release: apis dev to main

corpus/dac.yaml

@@ -161,11 +161,9 @@
 
 - id: dq-dac-016
   query: GNI of Denmark
-  why: DAC1 carries GNI only as the denominator of the ODA ratio; a plain GNI question elects the World Bank series.
+  why: The DFO series carries the World Bank indicator NY.GNP.MKTP.CD, and DAC1 serves it for Denmark.
   tags: {theme: macro, entity: country, direction: subject, unit: default, time: none, operation: none, breakdown: none, door: raw_text, phrasing: casual, overlap: [dac1], probe: [source_election]}
-  # Known failure: "GNI of Denmark" returns candidates, not a definite answer.
-  known_failure: {issue: 1249, checks: [disposition]}
-  expect: {definite: {measure: {family: base.gni}, entities: [{dcid: country/DNK, role: observationAbout}]}}
+  expect: {definite: {measure: {family: dac1.gni}, entities: [{dcid: country/DNK, role: observationAbout}]}}
 
 - id: dq-dac-017
   query: oda gni ratio swedan

corpus/families/dac.yaml

@@ -59,8 +59,10 @@ dac1.private_mobilised:
   shape: {measured_property: amountMobilised}
 dac1.gni:
   origin: dfo
-  dataset: {gl: DAC1 provenance}
-  shape: {gl: GNI series about the provider}
+  dataset: provenance/OECDDAC1
+  shape: {population_type: EconomicUnit, measured_property: primaryIncome, stat_type: measuredValue, measurement_qualifier: Nominal, measurement_denominator: null}
+  fixed: {fixedCapitalConsumptionBasis: GrossOfFixedCapitalConsumption}
+  stat_vars: [PrimaryIncome_EconomicUnit_GrossOfFixedCapitalConsumption_Nominal]
 dac2a.oda_net_received:
   origin: dfo
   dataset: provenance/OECDDAC2A

docs/datasets-facet-policy.md

@@ -362,7 +362,14 @@ by facet id and each facet's series count across the members
 (`materialise.group_facets.group_breadth`), and then the one the earliest
 member holds. Every member dispatches only its facets at that period. A
 member holding observations at another period alone dispatches nothing,
-and the result carries a `coverage_below_scope` warning naming it. A group
+and the result carries a `coverage_below_scope` warning naming it by ref
+in a structured field. The warning counts such members apart from those whose
+observations all fall outside a range window. The first report the
+requested years at another frequency, so the frame leaves their figures
+out. The second report nothing in those years, and raise no warning when
+the served dataset reads an absent member as zero
+(`code_index.members.absent_member_reads_zero`), since they add nothing to
+a sum over the requested years. A group
 chunk fetches the union of its members' dispatch sets, and each member
 serves only rows from its own (`materialise.columnar.dispatched_rows`).
 

docs/datasets-sync-ceiling.md

@@ -62,11 +62,14 @@ set is never returned to a submission that resolved another.
 `0024_operation_attempt_fencing.sql`) stores the request identity, rendering options, the
 admission coverage stamp, catalogue build and estimate, the lifecycle state (`pending`, `running`,
 `done`, `failed`), and the dispatch and fencing columns described below. A submit that finds an
-existing row returns it. A `failed` row marked `retryable`, a `running` row whose lease has ended,
-or a `pending` row past the operation deadline (see [Abandoned runs](#abandoned-runs)) is reset to
-`pending` as a new submission, with `submission_count` incremented. A new row or a reset first
+existing row returns it. A `done` row, a `failed` row marked `retryable`, a `running` row whose
+lease has ended, or a `pending` row past the operation deadline (see
+[Abandoned runs](#abandoned-runs)) is reset to `pending` as a new submission, with
+`submission_count` incremented. A `done` row resets because a submission reaches the store only
+when the subject's head could not answer it, and the rerun serves what the head gives at run
+time. A new row or a reset first
 counts the other live rows (every `pending` or `running` row not abandoned) under an advisory
-lock and refuses `service_unavailable` (503, retryable) at `DATASETS_OPERATION_BACKLOG_MAX` (20).
+lock and refuses `service_unavailable` (503, retryable) at `DATASETS_OPERATION_BACKLOG_MAX` (20). A backlog refusal re-reads the subject's head, and when the head serves the request it answers 200 from the head.
 Every mint and definition door call that reaches this point raises `OperationSubmitted`, caught by
 the door itself. The response is `202 Accepted` with a `Location` header pointing at
 `/api/datasets/operations/{operation_id}` and a body carrying `operation_id`, `status`, `poll`, and

packages/dfo-data-tools/pyproject.toml

@@ -88,10 +88,10 @@ graph-release = ["google-cloud-spanner>=3.57", "dfo-data-tools[coverage]"]
 [tool.uv.sources]
 # See the root pyproject.toml's dfo-spec-kit source comment for the
 # pin-consistency invariant this rev must satisfy.
-dfo-spec-kit-frame-grammar = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "ba976a2a74173e1c31b6c03d802cbcf71a022273", subdirectory = "python/frame-grammar" }
-dfo-spec-kit-components = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "ba976a2a74173e1c31b6c03d802cbcf71a022273", subdirectory = "python/dfo-spec-kit-components" }
-dfo-spec-kit-provenance-docs = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "ba976a2a74173e1c31b6c03d802cbcf71a022273", subdirectory = "python/provenance-docs" }
-dfo-spec-kit-descriptions = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "ba976a2a74173e1c31b6c03d802cbcf71a022273", subdirectory = "python/descriptions" }
+dfo-spec-kit-frame-grammar = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "0d19172e3695c311ed48ff43a885a9995869e6a5", subdirectory = "python/frame-grammar" }
+dfo-spec-kit-components = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "0d19172e3695c311ed48ff43a885a9995869e6a5", subdirectory = "python/dfo-spec-kit-components" }
+dfo-spec-kit-provenance-docs = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "0d19172e3695c311ed48ff43a885a9995869e6a5", subdirectory = "python/provenance-docs" }
+dfo-spec-kit-descriptions = { git = "https://github.com/DevelopmentFinanceObservatory/spec-kit.git", rev = "0d19172e3695c311ed48ff43a885a9995869e6a5", subdirectory = "python/descriptions" }
 
 [build-system]
 requires = ["uv_build>=0.11,<0.12"]

packages/dfo-data-tools/src/assembler/mint/__init__.py

@@ -10,7 +10,7 @@
 from assembler.mint.builders import complete_window, default_window
 from assembler.mint.card_description import DescribedRevision, with_card_description
 from assembler.mint.dispatch import MintedComponent, mint_component, stamp_envelope
-from assembler.mint.provenance import DESCRIPTION_PROVENANCE_MEMBERS
+from assembler.mint.provenance import DESCRIPTION_PROVENANCE_MEMBERS, warned_refs
 from assembler.mint.selection import selection_facts
 
 __all__ = [
@@ -22,5 +22,6 @@
     "mint_component",
     "selection_facts",
     "stamp_envelope",
+    "warned_refs",
     "with_card_description",
 ]

packages/dfo-data-tools/src/assembler/mint/provenance.py

@@ -14,6 +14,8 @@
 from dataclasses import dataclass
 from typing import cast
 
+from dfo_frame_grammar.grammar import STAT_VAR_REF_PREFIX
+
 from assembler.bind import BoundSeries
 from assembler.limits import LABEL_MAX
 from assembler.ports import NamedSeries, ResolvedFrame
@@ -178,10 +180,70 @@ def _build_citation(
     return citation_line(retrieved=retrieved, sources=names, processed=processed)[:_CITATION_MAX]
 
 
-def _translate_warnings(warnings: Sequence[Mapping[str, object]]) -> list[dict[str, object]]:
+@dataclass(frozen=True)
+class _RefField:
+    """A counted field of a manifest warning: its key, the words that lead its names on a
+    card, and the noun, one and many, that counts them when no name fits. A field `with_years`
+    is `{count, entities}`, each entity a `{ref, years}` (`datasets_api.manifest_models.
+    CountedEntityYearsModel`); any other is `{count, refs}` (`CountedRefsModel`)."""
+
+    key: str
+    lead: str
+    one: str
+    many: str
+    with_years: bool = False
+
+
+_REF_FIELDS = (
+    _RefField("unserved_entities", "Not served", "entity", "entities"),
+    _RefField("off_cadence_members", "At a different frequency", "series", "series"),
+    _RefField(
+        "out_of_window_members", "No observations in the requested years", "series", "series"
+    ),
+    _RefField("omitted_entities", "Left out", "entity", "entities"),
+    _RefField("omitted_entity_years", "Left out", "entity", "entities", with_years=True),
+)
+
+
+def warned_refs(*, manifest: Mapping[str, object]) -> list[str]:
+    """The `dcid:` refs the counted fields of `manifest`'s warnings carry, in warning and
+    field order: the entities and group members a card names by label
+    (`_translate_warnings`)."""
+    warnings = cast("Sequence[Mapping[str, object]]", manifest.get("warnings") or ())
+    return [
+        ref
+        for warning in warnings
+        for field in _REF_FIELDS
+        for ref, _years in _named_items(_counted(warning, field), field)
+    ]
+
+
+def _counted(warning: Mapping[str, object], field: _RefField) -> Mapping[str, object]:
+    return cast("Mapping[str, object]", warning.get(field.key) or {})
+
+
+def _named_items(counted: Mapping[str, object], field: _RefField) -> list[tuple[str, str]]:
+    """Each ref `counted` carries, with the text that follows its name on a card: its years
+    in brackets for a field `with_years`, `" (1990-2001, 2004)"`, else nothing."""
+    if not field.with_years:
+        return [(ref, "") for ref in cast("Sequence[str]", counted.get("refs", ()))]
+    entities = cast("Sequence[Mapping[str, object]]", counted.get("entities", ()))
+    return [
+        (cast(str, entity["ref"]), f" ({', '.join(cast('Sequence[str]', entity['years']))})")
+        for entity in entities
+    ]
+
+
+def _translate_warnings(
+    warnings: Sequence[Mapping[str, object]], *, labels: Mapping[str, str]
+) -> list[dict[str, object]]:
     """Maps manifest warnings to the envelope's closed `{kind, detail}` vocabulary.
 
-    Drops `severity` and raises `AssertionError` for an unmapped code."""
+    A warning's `detail` is its message, then, for each counted field it carries, the field's
+    lead words and its refs' `labels`, the bare dcid where `labels` holds none, each followed
+    by its years where the field holds them (`_named_refs`). The fields share the room the message leaves within
+    `_WARNING_DETAIL_MAX` equally, and a field's unused share passes to the next. Drops
+    `severity` and raises `AssertionError` for an unmapped code."""
     translated: list[dict[str, object]] = []
     for warning in warnings:
         code = cast(str, warning["code"])
@@ -190,11 +252,36 @@ def _translate_warnings(warnings: Sequence[Mapping[str, object]]) -> list[dict[s
                 f"manifest warning code {code!r} has no entry in mint's own "
                 "envelope-kind translation table (_WARNING_KIND_BY_CODE)"
             )
-        detail = str(warning["message"])[:_WARNING_DETAIL_MAX]
-        translated.append({"kind": _WARNING_KIND_BY_CODE[code], "detail": detail})
+        detail = str(warning["message"])
+        present = [
+            (field, counted) for field in _REF_FIELDS if (counted := _counted(warning, field))
+        ]
+        for index, (field, counted) in enumerate(present):
+            names = [
+                labels.get(ref, ref.removeprefix(STAT_VAR_REF_PREFIX))[:LABEL_MAX] + years
+                for ref, years in _named_items(counted, field)
+            ]
+            room = (_WARNING_DETAIL_MAX - len(detail)) // (len(present) - index)
+            detail += _named_refs(field, names, count=cast(int, counted["count"]), budget=room)
+        translated.append(
+            {"kind": _WARNING_KIND_BY_CODE[code], "detail": detail[:_WARNING_DETAIL_MAX]}
+        )
     return translated
 
 
+def _named_refs(field: _RefField, names: Sequence[str], *, count: int, budget: int) -> str:
+    """`" {lead}: "`, then as many whole `names` as fit within `budget` characters, with
+    `" and N more"` for the `count` not named, closed by a full stop. When no name fits, the
+    count stands for them all, whatever the budget, so the field keeps its count."""
+    lead = field.lead
+    for shown in range(len(names), 0, -1):
+        hidden = count - shown
+        text = f" {lead}: {', '.join(names[:shown])}{f' and {hidden} more' if hidden > 0 else ''}."
+        if len(text) <= budget:
+            return text
+    return f" {lead}: {count} {field.one if count == 1 else field.many}."
+
+
 @dataclass(frozen=True)
 class LeftOut:
     """The series a rolling component leaves out, one per display label, and the period they
@@ -289,7 +376,7 @@ def _build_provenance(
             "next_expected": None,
             "partial_period": False,
         },
-        "warnings": _translate_warnings(warnings),
+        "warnings": _translate_warnings(warnings, labels=resolved_frame.labels),
         "generated_at": clock().astimezone(datetime.UTC).isoformat().replace("+00:00", "Z"),
         "citation": _build_citation(
             sources=sources,

packages/dfo-data-tools/src/code_index/export.py

@@ -5,7 +5,7 @@
 document: provenances and parties from the coverage it joins, and the stamp.
 `graph_release.dfo` builds a release's code index part with it. `index_bytes` is the one
 serialisation of an index document, so the same document gives the same bytes. Nothing here
-imports `dc_graph`: a reader supplies the graph reads, and the two names the export needs from
+imports `dc_graph`: a reader supplies the graph reads, and the names the export needs from
 `dc_graph` are imported where they are used.
 """
 
@@ -377,7 +377,7 @@ def export_parties(
             "dcid": dcid,
             "kind": kind,
             "party_class": _party_class(kind, private=dcid in private),
-            "label": _first_or_none(party_arcs.get("name", ())),
+            "label": _graph_label(dcid, party_arcs),
             "totals": None if total is None else total.group,
             "residuals": [] if total is None else sorted(total.residuals),
             "sums_members": True if total is None else total.sums_members,
@@ -538,6 +538,20 @@ def node_arcs_batch(self, dcids: list[str]) -> Mapping[str, dict | None]:
         ...
 
 
+def _graph_label(dcid: str, arcs: Mapping[str, tuple[str, ...]]) -> str | None:
+    """The node's label as `GraphClient.node_label` gives it (`dc_graph.label_or_self_name`)."""
+    from dc_graph import label_or_self_name
+
+    return label_or_self_name(arcs.get("name", ()), dcid)
+
+
+def _required_name(dcid: str, arcs: Mapping[str, tuple[str, ...]]) -> str:
+    name = _graph_label(dcid, arcs)
+    if name is None:
+        raise ExportRefused(f"{dcid} carries no name arc -- cannot export a row missing it")
+    return name
+
+
 def _first_or_none(values: tuple[str, ...]) -> str | None:
     return values[0] if values else None
 
@@ -554,7 +568,7 @@ def _fetch_arcs_many(
 
 def _build_code_row(dcid: str, code_system: str, arcs: dict[str, tuple[str, ...]]) -> dict:
     code = _single_value(dcid, "classificationCode", arcs.get("classificationCode", ()))
-    name = _single_value(dcid, "name", arcs.get("name", ()))
+    name = _required_name(dcid, arcs)
     label = strip_system_suffix(name)
     synonyms = sorted(
         {v for v in arcs.get("alternateName", ()) if v and v != label}
@@ -1746,7 +1760,7 @@ def export_measure_values(
                     f"{dcid} is a {axis.axis!r} measure value an exported StatVar row carries "
                     "or the export declares, but the graph holds no node for it"
                 )
-            name = _single_value(dcid, "name", arcs.get("name", ()))
+            name = _required_name(dcid, arcs)
             overrides = MEASURE_VALUE_TEXT.get(dcid, MeasureValueText())
             label = overrides.label if overrides.label is not None else strip_system_suffix(name)
             synonyms = sorted(

packages/dfo-data-tools/src/dc_graph/__init__.py

@@ -18,8 +18,10 @@
     SubjectPage,
     SubjectsCoverageFlag,
     declared_role_refresh_deadline,
+    display_name,
     earliest_date,
     is_front_door_refusal,
+    label_or_self_name,
     latest_date,
     parse_arcs,
     reject_unsafe_filter_token,
@@ -62,8 +64,10 @@
     "SubjectsCoverageFlag",
     "declared_role_refresh_deadline",
     "dfo_visible",
+    "display_name",
     "earliest_date",
     "is_front_door_refusal",
+    "label_or_self_name",
     "latest_date",
     "mixer_visible",
     "parse_arcs",