Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions corpus/standard.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2010,8 +2010,6 @@
query: German aid to Kenya
why: The demonym "German" names Germany as the donor, which extraction reads as the place the resolver holds.
tags: {theme: aid, entity: two_roles, direction: both, time: none, operation: none, door: raw_text}
# Known failure: the binder's reply leaves dac2aAidType out, and a slot left out is not read as open, so the total ODA default does not bind.
known_failure: {issue: 99, checks: [measure]}
expect:
definite:
measure: {family: dac2a.oda_gross_received}
Expand Down
19 changes: 19 additions & 0 deletions src/search/engine/bind.py
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,25 @@ def _names_its_values(self) -> SlotBindingDraft:
return self


def sole_option_properties(bindings: Iterable[SlotBindingDraft]) -> frozenset[str]:
"""The properties the bind reply bound to a value only because the slot offered no
other (`SlotBindingDraft.evidence` ``sole_option``, `drafted_by` the reply): the
request's words name none of them, and each claims that every member asserts its
value, which a member conflicting on it refutes. A ``sole_option`` value the engine
wrote makes no such claim: a partition value (`standard.bind_partition_values`) sits
on a slot the members differ on, and a series code's shared values
(`bound_values.bind_member_values`) are shared by that code's StatVars alone. Those
give way as values the request named."""
return frozenset(
b.property_dcid
for b in bindings
if b.kind == "value"
and b.evidence == "sole_option"
and b.drafted_by == "reply"
and b.property_dcid is not None
)


class ShapeRating(BaseModel):
"""A candidate shape the bind reply says measures the variable phrase, with its rating."""

Expand Down
8 changes: 7 additions & 1 deletion src/search/engine/region/grounding.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
from dfo_platform.executor import submit_with_context
from materialise import GraphReader
from materialise.roles import axis_noun
from search.engine.bind import SlotBindingDraft
from search.engine.bind import SlotBindingDraft, sole_option_properties
from search.engine.clients.graph import EngineGraphClient
from search.engine.clients.sdmx import EngineSdmxClient
from search.engine.config import settings
Expand Down Expand Up @@ -1444,6 +1444,9 @@ def _suggest_nearest_real(
warns it. Each probe uses `deadline`, and any error yields no suggestion.
Option-coverage failures add warnings. Callers fetch observations separately, from each
suggestion's plan and places.
A value bound as its slot's sole option (`bind.sole_option_properties`) is not relaxed:
the request did not name it, and every member of the family asserts it, so the
relaxed query reads the same members as the answer that had no data.
`text_recognized_properties`, `measure_elected_properties`, and `defaulted_properties`
preserve the original query's slot provenance after relaxation, and `named_groups` the
groups the request named, which a suggestion states for the roles that still bind
Expand Down Expand Up @@ -1484,9 +1487,12 @@ def _suggest_nearest_real(
relaxations: list[tuple[list[SlotBindingDraft], Relaxation]] = (
[] if level is None else [(list(bindings), level[1])]
)
unnamed = sole_option_properties(bindings)
for i, binding in enumerate(bindings if level is None else ()):
if binding.kind not in ("value", "set") or binding.axis in ("where", "when", "source"):
continue
if binding.property_dcid in unnamed:
continue
relaxed = list(bindings)
relaxed[i] = SlotBindingDraft(
axis=binding.axis,
Expand Down
108 changes: 91 additions & 17 deletions src/search/engine/region/phase_election.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@
bind_shared_values,
canonical_pin_bindings,
elect_measure_family,
five_tuple_member_facts,
identity_member_facts,
named_scope,
part_source,
read_held_families,
Expand Down Expand Up @@ -286,7 +286,9 @@ def _settle_bindings(
reading the shape keeps of each two-reading span
(`reading_for_shape`), which admission read.

The source drafts leave and invalid ``requested_all`` claims are demoted
Each constraint slot of the shape the reply drafts nothing for gets an ``unbound``
draft (`_with_every_slot_drafted`), so every later pass reads one draft per slot. The
source drafts leave and invalid ``requested_all`` claims are demoted
(`bind.validate_requested_all_bindings`). The binder applies it to its pick's drafts,
and this pass keeps a claim only on a slot of the elected shape. Each slot
whose taxonomy holds one value and that the reply left unbound is claimed as that
Expand All @@ -306,7 +308,9 @@ def _settle_bindings(
shape admits, with the codes the model chose from a phrase's candidate menu, are
forced into the bindings (`_force_qualifier_bindings`).
"""
bindings = without_dataset_drafts(state.bind_result.bindings)
bindings = _with_every_slot_drafted(
without_dataset_drafts(state.bind_result.bindings), shape_draft
)
bindings, state.applied_dropped_qualifier_phrases = validate_requested_all_bindings(
bindings,
state.dropped_qualifier_phrases,
Expand Down Expand Up @@ -407,6 +411,27 @@ def _settle_bindings(
return unresolved, readings


def _with_every_slot_drafted(
bindings: list[SlotBindingDraft], shape_draft: ShapeDraft
) -> list[SlotBindingDraft]:
"""`bindings` with an ``unbound`` engine draft for each constraint slot of
`shape_draft` they hold no draft for, in slot order after the reply's own.

The bind prompt asks the reply for one binding per constraint slot, and a reply can
leave one out. Every pass after it reads the slot through its draft: a single-member
slot is claimed as its sole option (`_claim_single_member_slots`), and a declared
default fires on an open slot's draft (`bind_repair.apply_declared_defaults`). A slot
left out is open, so it reads as one drafted ``unbound``: "German aid to Kenya" binds
DAC2a's ODA aid type by its default whether or not the reply lists the slot."""
drafted = {draft.property_dcid for draft in bindings}
missing = [
SlotBindingDraft(axis=slot.axis, property_dcid=slot.property_dcid, kind="unbound")
for slot in shape_draft.slot_keys
if slot.property_dcid is not None and slot.property_dcid not in drafted
]
return [*bindings, *missing] if missing else bindings


def _settle_sole_options(
bindings: list[SlotBindingDraft], shape_draft: ShapeDraft
) -> list[SlotBindingDraft]:
Expand Down Expand Up @@ -513,10 +538,12 @@ def _validate_sole_option_bindings(
"""Validate model ``sole_option`` claims against the shape's population.

A ``sole_option`` claim says the slot forces one value and the phrase says nothing
about it. The population is the members the shape's slot taxonomy describes
(`five_tuple_member_facts`): the rows of the family the index carries for the shape,
else the members of the family the catalogue holds for it beside its recalled ones,
else its recalled members alone. The phrase did not name the value and a slot that
about it. The population is the family the shape's identity denotes
(`identity_member_facts`): the rows the index carries for the dataset the part names,
or for every dataset of the family when it names none, else the members of the family
the catalogue holds for it beside its recalled ones, else its recalled members alone.
A part excluding a dataset names none, so it mints the identity of the whole family,
and a row of the excluded dataset holding another value keeps the slot open. The phrase did not name the value and a slot that
the population leaves a reading without it does not force it, so no binding rule
produced the draft and the slot is left unbound.

Expand Down Expand Up @@ -545,7 +572,7 @@ def _validate_sole_option_bindings(
(`without_sampled_claims`), settles it (`phase_place_materialise._judge_sampled_claims`).
A shape with no members passes every claim through.
"""
population, index_sourced = five_tuple_member_facts(shape_draft)
population, index_sourced = identity_member_facts(shape_draft)
if not population:
return bindings
complete = index_sourced or shape_draft.held_family is not None
Expand Down Expand Up @@ -584,8 +611,8 @@ def _unbind_implied_values(
binding is released where the family has totals under the other bindings
(`family_totals`) and none of them asserts the property, so the total answers. A
family with no total keeps the value, as does one whose totals assert the property or
have unknown constraints: there the value may separate readings the family holds. The population is the one
`_validate_sole_option_bindings` reads (`five_tuple_member_facts`).
have unknown constraints: there the value may separate readings the family holds. The
population is the one `_validate_sole_option_bindings` reads (`identity_member_facts`).

The draft's evidence does not matter: the recorded bind reply to "life expectancy"
drafts ``age=Years0`` as the slot's ``sole_option``, and a reply naming the age as
Expand All @@ -594,7 +621,7 @@ def _unbind_implied_values(
implied = get_index().implied_values_of(shape_draft.pop_type_dcid, shape_draft.meas_prop_dcid)
if not implied:
return bindings
population, _ = five_tuple_member_facts(shape_draft)
population, _ = identity_member_facts(shape_draft)
settled = list(bindings)
for i, binding in enumerate(settled):
prop = binding.property_dcid
Expand Down Expand Up @@ -644,7 +671,7 @@ def _keep_the_noun_pinned_property(
and len(b.value_dcids) == 1
):
properties_by_value.setdefault(b.value_dcids[0], []).append(b.property_dcid)
population, _ = five_tuple_member_facts(shape_draft)
population, _ = identity_member_facts(shape_draft)
stated = _collapse_bindings(bindings)
released: set[str] = set()
for value, drafted in properties_by_value.items():
Expand Down Expand Up @@ -705,21 +732,68 @@ def _sample_forces(
stated_values: Mapping[str, list[str]],
) -> bool:
"""Whether a recalled sample leaves `prop` no reading but `value`: no member asserts
another value on it (`constraints` are the members' `read_constraints`), every total
of the sample under the part's other bindings (`family_totals`), the member a slot
left open serves, asserts `value`, and the sample holds no broader reading of the part
without it (`holds_broader_reading`). A sample that splits on another open slot holds
no total, and the broader reading is the member an open `prop` serves there."""
another value on it (`constraints` are the members' `read_constraints`), no member of
the part asserting none is a reading without it, and every total of the sample under
the part's other bindings (`family_totals`), the member a slot left open serves,
asserts `value`.

A member of the part asserting nothing on `prop` is one of three readings beside the
members asserting `value`. Its constraints can be a subset of an asserting member's,
so it is the broader reading an open `prop` serves (`holds_broader_reading`): in a
sample that splits on another open slot, which holds no total, it is the broadest
member of its side. It can differ from an asserting member on a property both state,
so the two are siblings across a split and it is the reading of the part without
`value` (`_splits_without_value`): gross domestic product beside the gross value added
of agriculture. Or it can narrow on properties every asserting member leaves unstated,
which forces nothing against the claim. The first two refute it; the third does not.

The rule governs only answers the walk of the family cannot judge: where the walk
reads the slot open, it confirms or refutes the claim on the whole family
(`phase_place_materialise._judge_sampled_claims`)."""
if any(member.get(prop) not in (None, value) for member in constraints):
return False
if holds_broader_reading(population, stated_values, prop, value):
return False
if _splits_without_value(constraints, prop, value, stated_values):
return False
return all(
population[sv].read_constraints().get(prop) == value
for sv in family_totals(population, stated_values)
)


def _splits_without_value(
constraints: Sequence[Mapping[str, str]],
prop: str,
value: str,
stated_values: Mapping[str, list[str]],
) -> bool:
"""Whether a member of the part (the members stating one of the `stated_values` on
each bound property) asserting nothing on `prop` asserts, on another property, a value
other than the one a member asserting `value` there asserts. The two are the sides of
a slot the part leaves open, and the side without `value` is a reading of the part
without it: "GDP" recalls gross domestic product, which states no economic sector,
beside the gross value added of agriculture, so the sector is not forced.
`holds_broader_reading` misses such a pair, since neither member's constraints are a
subset of the other's. A member asserting none that narrows on a property the members
asserting `value` leave unstated splits nowhere with them: the age group
classification members beside the age of zero of "infant mortality rate"."""
part = [
member
for member in constraints
if all(member.get(bound) in values for bound, values in stated_values.items())
]
asserting = [member for member in part if member.get(prop) == value]
return any(
other.get(other_prop) not in (None, other_value)
for other in part
if prop not in other
for member in asserting
for other_prop, other_value in member.items()
if other_prop != prop
)


def _name_served_dataset(
state: ResolveState,
shape: ShapeDraft,
Expand Down
41 changes: 26 additions & 15 deletions src/search/engine/region/phase_place_materialise.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
from dfo_platform.trace import get_trace_id
from materialise.place_scope import DefaultPresence
from materialise.roles import RoleAxis, axis_for_role
from search.engine.bind import SlotBindingDraft
from search.engine.bind import SlotBindingDraft, sole_option_properties
from search.engine.clients.graph import EngineGraphClient, force_full_facets
from search.engine.clients.sdmx import EngineSdmxClient
from search.engine.config import SEARCH_MAX_UNIT_ALTERNATIVES, settings
Expand Down Expand Up @@ -1759,11 +1759,29 @@ def _members_holding_bound_values(
}


def _stand_in_members(shape: ShapeDraft, bindings: Sequence[SlotBindingDraft]) -> dict[str, dict]:
"""The members of the stand-in `shape` that keep the elected `bindings`, with their arcs:
those asserting every property a binding the request named binds at a bound value
(`_members_holding_bound_values`), and no value other than the one a value the request
did not name binds (`sole_option_properties`). Such a value is the elected family's
own description of itself, so a member stating nothing on its property reads the same
quantity another way: the IDS gross national income states its income as
``economicActivityType`` beside the Atlas method's ``activitySource=GrossNationalIncome``."""
unnamed = sole_option_properties(bindings)
asked = bound_values_from_bindings([b for b in bindings if b.property_dcid not in unnamed])
sole = bound_values_from_bindings([b for b in bindings if b.property_dcid in unnamed])
return {
sv: arcs
for sv, arcs in _members_holding_bound_values(shape, asked).items()
if all(read_constraints(arcs).get(prop) in (None, *values) for prop, values in sole.items())
}


def _stand_in_indices(state: ResolveState) -> list[int]:
"""The capped-shape indices of the readings status rule 1 tries in the elected shape's
place when it has no observations at its places, in the order the rule takes them: its
equivalent readings, then its measurement-qualifier siblings that hold the elected
bindings' values (`_members_holding_bound_values`). Each passes the alternate guard
equivalent readings, then its measurement-qualifier siblings that keep the elected
bindings (`_stand_in_members`). Each passes the alternate guard
(`AlternateJudge.inadmissible`) judged beside an elected shape without data, so no
default price basis is imposed, within the candidate budget. A part with a registry twin
of the elected shape takes none: the equivalent-series diversion answers with the twin
Expand All @@ -1772,14 +1790,9 @@ def _stand_in_indices(state: ResolveState) -> list[int]:
plan = state.status_plan
if plan.twins:
return []
bound_values = bound_values_from_bindings(state.bindings)
indices = [
*plan.equivalents,
*(
i
for i in plan.stand_ins
if _members_holding_bound_values(state.capped_shapes[i], bound_values)
),
*(i for i in plan.stand_ins if _stand_in_members(state.capped_shapes[i], state.bindings)),
]
if indices:
inadmissible = _guard_judge(state, elected_has_data=False).inadmissible(
Expand All @@ -1791,14 +1804,12 @@ def _stand_in_indices(state: ResolveState) -> list[int]:

def _sibling_shapes_of(state: ResolveState) -> dict[int, ShapeDraft]:
"""Each re-election sibling (`state.reelection_siblings`) by capped-shape index, holding
the members it is probed and answered through: those that hold the elected bindings'
values among every member it stands for (`_members_holding_bound_values`), and those
alone (`narrowed_to`)."""
elected_bound_values = bound_values_from_bindings(state.bindings)
the members it is probed and answered through: those that keep the elected bindings
among every member it stands for (`_stand_in_members`), and those alone
(`narrowed_to`)."""
return {
i: narrowed_to(
state.capped_shapes[i],
_members_holding_bound_values(state.capped_shapes[i], elected_bound_values),
state.capped_shapes[i], _stand_in_members(state.capped_shapes[i], state.bindings)
)
for i in state.reelection_siblings
}
Expand Down
11 changes: 10 additions & 1 deletion src/search/engine/region/phase_rank_bind.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
_slot_name,
_UniversalBindOutput,
rank_elected,
sole_option_properties,
split_stray_bindings,
)
from search.engine.bind_repair import (
Expand Down Expand Up @@ -1186,12 +1187,20 @@ def _out_of_scope_records(
def _bound_code_records(bindings: Sequence[SlotBindingDraft]) -> tuple[ResolvedQualifier, ...]:
"""Each code `bindings` bind on a what or how slot, as an unbound record under its
label: what a part cannot bind when no table it may answer from holds a row with all
of them."""
of them. A value bound as its slot's sole option (`sole_option_properties`) is one the
request did not name, so it is no code the part cannot bind, whether a recall sample
or the whole family backs it. A sampled claim can sit on an indexed five-tuple whose
part datasets publish none of its rows; such a part, unserved, would answer once the
claim is relaxed, as materialise relaxes it, and this rule only keeps the value out of
the refusal."""
index = get_index()
unnamed = sole_option_properties(bindings)
records: list[ResolvedQualifier] = []
for binding in bindings:
if binding.axis not in ("what", "how") or binding.kind not in ("value", "set"):
continue
if binding.property_dcid in unnamed:
continue
for dcid in binding.value_dcids:
row = index.code_by_dcid(dcid)
label = row.label if row is not None else dcid
Expand Down
Loading
Loading