Skip to content

Commit f23548f

Browse files
igerberclaude
andcommitted
Wire allow_zero_se through survey bootstrap callers, add TROP deviation note
- Fix allow_zero_se to return p=NaN (not p=0.0) for census FPC - Pass allow_zero_se=True from all 6 survey bootstrap callers (17 sites) - Document TROP Rao-Wu frozen-tau as deviation (mathematically equivalent to refit since tau_{it} is deterministic given Y, D, lambda) - Add TODO.md entry for survey bootstrap test coverage gaps Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent f9f4ac3 commit f23548f

9 files changed

Lines changed: 34 additions & 7 deletions

TODO.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@ Deferred items from PR reviews that were not addressed before merge.
7878
| CS R helpers hard-code `xformla = ~ 1`; no covariate-adjusted R benchmark for IRLS path | `tests/test_methodology_callaway.py` | #202 | Low |
7979
| ~376 `duplicate object description` Sphinx warnings — caused by autodoc `:members:` on dataclass attributes within manual API pages (not from autosummary stubs); fix requires restructuring `docs/api/*.rst` pages to avoid documenting the same attribute via both `:members:` and inline `autosummary` tables | `docs/api/*.rst` || Low |
8080
| Plotly renderers silently ignore styling kwargs (marker, markersize, linewidth, capsize, ci_linewidth) that the matplotlib backend honors; thread them through or reject when `backend="plotly"` | `visualization/_event_study.py`, `_diagnostic.py`, `_power.py` | #222 | Medium |
81+
| Survey bootstrap test coverage: add FPC census zero-variance, single-PSU NaN, full-design bootstrap for CS/ContinuousDiD/EfficientDiD, and TROP Rao-Wu vs block bootstrap equivalence tests | `tests/test_survey_phase*.py` | #237 | Medium |
8182

8283
---
8384

diff_diff/bootstrap_utils.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -286,7 +286,7 @@ def compute_effect_bootstrap_stats(
286286
if not np.isfinite(se) or se <= 0:
287287
# Census FPC: all bootstrap estimates identical → SE=0 is legitimate
288288
if allow_zero_se and se == 0.0:
289-
return 0.0, (original_effect, original_effect), 0.0
289+
return 0.0, (original_effect, original_effect), np.nan
290290
warnings.warn(
291291
f"Bootstrap SE is non-finite or zero (n_valid={n_valid}) in {context}. "
292292
"Returning NaN for SE/CI/p-value.",
@@ -406,7 +406,7 @@ def compute_effect_bootstrap_stats_batch(
406406
ses[zero_idx] = 0.0
407407
ci_lowers[zero_idx] = original_effects[zero_idx]
408408
ci_uppers[zero_idx] = original_effects[zero_idx]
409-
p_values[zero_idx] = 0.0
409+
p_values[zero_idx] = np.nan # p undefined when SE=0
410410
n_bad_se = int(np.sum(~se_valid & ~se_zero)) if allow_zero_se else int(np.sum(~se_valid))
411411
if n_bad_se > 0:
412412
warnings.warn(

diff_diff/continuous_did.py

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1498,6 +1498,7 @@ def _bootstrap_gt_cell(gt, info):
14981498
boot_att_d[:, idx],
14991499
alpha=self.alpha,
15001500
context=f"ATT(d) at grid point {idx}",
1501+
allow_zero_se=_use_survey_bootstrap,
15011502
)
15021503
att_d_se[idx] = se
15031504
att_d_ci_lower[idx] = ci[0]
@@ -1509,6 +1510,7 @@ def _bootstrap_gt_cell(gt, info):
15091510
boot_acrt_d[:, idx],
15101511
alpha=self.alpha,
15111512
context=f"ACRT(d) at grid point {idx}",
1513+
allow_zero_se=_use_survey_bootstrap,
15121514
)
15131515
acrt_d_se[idx] = se
15141516
acrt_d_ci_lower[idx] = ci[0]
@@ -1530,6 +1532,7 @@ def _bootstrap_gt_cell(gt, info):
15301532
boot_att_glob,
15311533
alpha=self.alpha,
15321534
context="overall ATT_glob",
1535+
allow_zero_se=_use_survey_bootstrap,
15331536
)
15341537
result["overall_att_se"] = se
15351538
result["overall_att_ci"] = ci
@@ -1540,6 +1543,7 @@ def _bootstrap_gt_cell(gt, info):
15401543
boot_acrt_glob,
15411544
alpha=self.alpha,
15421545
context="overall ACRT_glob",
1546+
allow_zero_se=_use_survey_bootstrap,
15431547
)
15441548
result["overall_acrt_se"] = se
15451549
result["overall_acrt_ci"] = ci
@@ -1556,6 +1560,7 @@ def _bootstrap_gt_cell(gt, info):
15561560
boot_es[e],
15571561
alpha=self.alpha,
15581562
context=f"event study e={e}",
1563+
allow_zero_se=_use_survey_bootstrap,
15591564
)
15601565
es_se[e] = se_e
15611566
es_ci[e] = ci_e

diff_diff/efficient_did_bootstrap.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -207,6 +207,7 @@ def _run_multiplier_bootstrap(
207207
bootstrap_atts[:, j],
208208
alpha=self.alpha,
209209
context=f"ATT(g={gt[0]}, t={gt[1]})",
210+
allow_zero_se=_use_survey_bootstrap,
210211
)
211212
gt_ses[gt] = se
212213
gt_cis[gt] = ci
@@ -220,6 +221,7 @@ def _run_multiplier_bootstrap(
220221
bootstrap_overall,
221222
alpha=self.alpha,
222223
context="overall ATT",
224+
allow_zero_se=_use_survey_bootstrap,
223225
)
224226

225227
es_ses = es_cis = es_pvs = None
@@ -231,6 +233,7 @@ def _run_multiplier_bootstrap(
231233
bootstrap_event_study[e],
232234
alpha=self.alpha,
233235
context=f"event study (e={e})",
236+
allow_zero_se=_use_survey_bootstrap,
234237
)
235238
es_ses[e] = se
236239
es_cis[e] = ci
@@ -245,6 +248,7 @@ def _run_multiplier_bootstrap(
245248
bootstrap_group[g],
246249
alpha=self.alpha,
247250
context=f"group effect (g={g})",
251+
allow_zero_se=_use_survey_bootstrap,
248252
)
249253
g_ses[g] = se
250254
g_cis[g] = ci

diff_diff/imputation_bootstrap.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -349,6 +349,7 @@ def _run_bootstrap(
349349
boot_overall_shifted,
350350
alpha=self.alpha,
351351
context="ImputationDiD overall ATT",
352+
allow_zero_se=_use_survey_bootstrap,
352353
)
353354

354355
event_study_ses = None
@@ -366,6 +367,7 @@ def _run_bootstrap(
366367
shifted_h,
367368
alpha=self.alpha,
368369
context=f"ImputationDiD event study (h={h})",
370+
allow_zero_se=_use_survey_bootstrap,
369371
)
370372
event_study_ses[h] = se_h
371373
event_study_cis[h] = ci_h
@@ -386,6 +388,7 @@ def _run_bootstrap(
386388
shifted_g,
387389
alpha=self.alpha,
388390
context=f"ImputationDiD group effect (g={g})",
391+
allow_zero_se=_use_survey_bootstrap,
389392
)
390393
group_ses[g] = se_g
391394
group_cis[g] = ci_g

diff_diff/staggered_bootstrap.py

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -426,7 +426,8 @@ def _run_multiplier_bootstrap(
426426

427427
# Batch compute bootstrap statistics for ATT(g,t)
428428
batch_ses, batch_ci_lo, batch_ci_hi, batch_pv = _compute_effect_bootstrap_stats_batch_func(
429-
original_atts, bootstrap_atts_gt, alpha=self.alpha
429+
original_atts, bootstrap_atts_gt, alpha=self.alpha,
430+
allow_zero_se=_use_survey_bootstrap,
430431
)
431432
gt_ses = {}
432433
gt_cis = {}
@@ -442,8 +443,9 @@ def _run_multiplier_bootstrap(
442443
overall_ci = (np.nan, np.nan)
443444
overall_p_value = np.nan
444445
else:
445-
overall_se, overall_ci, overall_p_value = self._compute_effect_bootstrap_stats(
446-
original_overall, bootstrap_overall, context="overall ATT"
446+
overall_se, overall_ci, overall_p_value = _compute_effect_bootstrap_stats_func(
447+
original_overall, bootstrap_overall, alpha=self.alpha, context="overall ATT",
448+
allow_zero_se=_use_survey_bootstrap,
447449
)
448450

449451
# Batch compute bootstrap statistics for event study effects
@@ -455,7 +457,8 @@ def _run_multiplier_bootstrap(
455457
es_effects = np.array([event_study_info[e]["effect"] for e in rel_periods])
456458
es_boot_matrix = np.column_stack([bootstrap_event_study[e] for e in rel_periods])
457459
es_ses, es_ci_lo, es_ci_hi, es_pv = _compute_effect_bootstrap_stats_batch_func(
458-
es_effects, es_boot_matrix, alpha=self.alpha
460+
es_effects, es_boot_matrix, alpha=self.alpha,
461+
allow_zero_se=_use_survey_bootstrap,
459462
)
460463
event_study_ses = {e: float(es_ses[i]) for i, e in enumerate(rel_periods)}
461464
event_study_cis = {
@@ -472,7 +475,8 @@ def _run_multiplier_bootstrap(
472475
grp_effects = np.array([group_agg_info[g]["effect"] for g in group_list])
473476
grp_boot_matrix = np.column_stack([bootstrap_group[g] for g in group_list])
474477
grp_ses, grp_ci_lo, grp_ci_hi, grp_pv = _compute_effect_bootstrap_stats_batch_func(
475-
grp_effects, grp_boot_matrix, alpha=self.alpha
478+
grp_effects, grp_boot_matrix, alpha=self.alpha,
479+
allow_zero_se=_use_survey_bootstrap,
476480
)
477481
group_effect_ses = {g: float(grp_ses[i]) for i, g in enumerate(group_list)}
478482
group_effect_cis = {

diff_diff/sun_abraham.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1476,6 +1476,7 @@ def _run_rao_wu_bootstrap(
14761476
boot_dist,
14771477
alpha=self.alpha,
14781478
context=f"event study e={e}",
1479+
allow_zero_se=True,
14791480
)
14801481
event_study_ses[e] = se
14811482
event_study_cis[e] = ci
@@ -1487,6 +1488,7 @@ def _run_rao_wu_bootstrap(
14871488
bootstrap_overall,
14881489
alpha=self.alpha,
14891490
context="overall ATT",
1491+
allow_zero_se=True,
14901492
)
14911493

14921494
return SABootstrapResults(

diff_diff/two_stage_bootstrap.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -311,6 +311,7 @@ def _run_bootstrap(
311311
boot_overall_shifted,
312312
alpha=self.alpha,
313313
context="TwoStageDiD overall ATT",
314+
allow_zero_se=_use_survey_bootstrap,
314315
)
315316

316317
# --- Event study bootstrap ---
@@ -411,6 +412,7 @@ def _run_bootstrap(
411412
shifted_h,
412413
alpha=self.alpha,
413414
context=f"TwoStageDiD event study (h={h})",
415+
allow_zero_se=_use_survey_bootstrap,
414416
)
415417
event_study_ses[h] = se_h
416418
event_study_cis[h] = ci_h
@@ -474,6 +476,7 @@ def _run_bootstrap(
474476
shifted_g,
475477
alpha=self.alpha,
476478
context=f"TwoStageDiD group effect (g={g})",
479+
allow_zero_se=_use_survey_bootstrap,
477480
)
478481
group_ses[g] = se_g
479482
group_cis[g] = ci_g

docs/methodology/REGISTRY.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1986,6 +1986,11 @@ ContinuousDiD, EfficientDiD):
19861986
TROP uses treatment group (treated vs control) as pseudo-strata for Rao-Wu resampling
19871987
to preserve treatment ratio. FPC is applied within these pseudo-strata. This matches
19881988
TROP's existing treatment-stratified resampling pattern.
1989+
- **Note (deviation from block bootstrap):** In Rao-Wu survey bootstrap, per-observation
1990+
treatment effects tau_{it} are deterministic given (Y, D, lambda) because survey weights
1991+
do not enter the kernel-weighted matrix completion. The Rao-Wu path therefore precomputes
1992+
tau values once and only varies the ATT aggregation weights across draws. This is
1993+
mathematically equivalent to refitting per draw and avoids redundant computation.
19891994

19901995
---
19911996

0 commit comments

Comments
 (0)