From 130f69498623aec77ec22c2e77e1603e6bb89f9e Mon Sep 17 00:00:00 2001 From: Lukas Wallrich Date: Sun, 28 Jun 2026 23:57:02 +0200 Subject: [PATCH 1/2] Add many-analyst study findings to Ch2 (#15) Add a focused paragraph after the Landy et al. discussion covering many-analyst studies (Silberzahn et al. 2018, Botvinik-Nezer et al. 2020, Breznau et al. 2022), where teams analyse the same data and reach divergent conclusions, making the analytic-flexibility point explicit for interpreting reproductions/replications. References: BotvinikNezerEtAl2020 already existed (added verified DOI 10.1038/s41586-020-2314-9). New entries SilberzahnEtAl2018 (doi 10.1177/2515245917747646) and BreznauEtAl2022 (doi 10.1073/pnas.2203150119), both inserted alphabetically. All three DOIs verified via Crossref. --- references.bib | 25 ++++++++++++++++++++++++- understanding.qmd | 16 ++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/references.bib b/references.bib index 22d468a..ff1a93f 100644 --- a/references.bib +++ b/references.bib @@ -198,7 +198,8 @@ @article{BotvinikNezerEtAl2020 volume = {582}, number = {7810}, pages = {84-88}, - year = {2020} + year = {2020}, + doi = {10.1038/s41586-020-2314-9} } @article{BoyceEtAl2024, @@ -219,6 +220,17 @@ @article{BrandtEtAl2014 doi = {10.1016/j.jesp.2013.10.005} } +@article{BreznauEtAl2022, + author = {Breznau, N. and Rinke, E. M. and Wuttke, A. and Nguyen, H. H. V. and Adem, M. and Adriaans, J. and Alvarez-Benjumea, A.}, + title = {Observing many researchers using the same data and hypothesis reveals a hidden universe of uncertainty}, + journal = {Proceedings of the National Academy of Sciences}, + volume = {119}, + number = {44}, + pages = {e2203150119}, + year = {2022}, + doi = {10.1073/pnas.2203150119} +} + @article{BrodeurEtAl2024a, author = {Brodeur, A. and Cook, N. M. and Hartley, J. S. and Heyes, A.}, title = {Do Preregistration and Preanalysis Plans Reduce p-Hacking and Publication Bias? Evidence from 15,992 Test Statistics and Suggestions for Improvement}, @@ -1409,6 +1421,17 @@ @article{SchultzeEtAl2018 doi = {10.1002/bdm.2065} } +@article{SilberzahnEtAl2018, + author = {Silberzahn, R. and Uhlmann, E. L. and Martin, D. P. and Anselmi, P. and Aust, F. and Awtrey, E. and Bahník, Š.}, + title = {Many analysts, one data set: Making transparent how variations in analytic choices affect results}, + journal = {Advances in Methods and Practices in Psychological Science}, + volume = {1}, + number = {3}, + pages = {337-356}, + year = {2018}, + doi = {10.1177/2515245917747646} +} + @article{SimmonsEtAl2011, author = {Simmons, J. P. and Nelson, L. D. and Simonsohn, U.}, title = {False-positive psychology: Undisclosed flexibility in data collection and analysis allows presenting anything as significant}, diff --git a/understanding.qmd b/understanding.qmd index dad2563..86ebfde 100644 --- a/understanding.qmd +++ b/understanding.qmd @@ -125,6 +125,22 @@ close replications more directly test the credibility of original results, while conceptual replications that vary features of the design are concerned with generalizability. +A related line of work isolates the analysis stage by holding the data +fixed. In so-called many-analyst studies, multiple independent teams +analyse the same dataset to test the same hypothesis, yet reach divergent +conclusions. @SilberzahnEtAl2018 had numerous teams estimate whether +football referees are more likely to give red cards to dark-skinned +players, and the teams reported a wide spread of effect estimates despite +working from identical data. @BotvinikNezerEtAl2020 observed comparable +variability when many teams tested hypotheses on a single neuroimaging +dataset, and @BreznauEtAl2022 described a “hidden universe of +uncertainty” across the many teams that analysed the same survey data. +Because the data are held constant, this variability stems from analytic +flexibility alone, which matters for interpreting reproductions and +replications: even when the same data are reanalysed, defensible +differences in modelling decisions can move a result from supporting to +contradicting the original claim. + Note that @NosekErrington2020 define replication as a study “for which any outcome would be considered diagnostic evidence about a claim from prior research”. This can lead to issues when the original claim is not From 9ad114e657d7023ea332c7cba532f6be77177a18 Mon Sep 17 00:00:00 2001 From: Lukas Wallrich Date: Mon, 29 Jun 2026 17:43:29 +0200 Subject: [PATCH 2/2] Address codex review: qualify many-analyst claims (#15) --- understanding.qmd | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/understanding.qmd b/understanding.qmd index 86ebfde..1757cfd 100644 --- a/understanding.qmd +++ b/understanding.qmd @@ -131,15 +131,17 @@ analyse the same dataset to test the same hypothesis, yet reach divergent conclusions. @SilberzahnEtAl2018 had numerous teams estimate whether football referees are more likely to give red cards to dark-skinned players, and the teams reported a wide spread of effect estimates despite -working from identical data. @BotvinikNezerEtAl2020 observed comparable +working from identical data. @BotvinikNezerEtAl2020 observed substantial variability when many teams tested hypotheses on a single neuroimaging dataset, and @BreznauEtAl2022 described a “hidden universe of uncertainty” across the many teams that analysed the same survey data. -Because the data are held constant, this variability stems from analytic -flexibility alone, which matters for interpreting reproductions and +Because the data are held constant, this variability cannot be attributed +to differences in data collection and instead reflects choices made in the +analysis pipeline, which matters for interpreting reproductions and replications: even when the same data are reanalysed, defensible -differences in modelling decisions can move a result from supporting to -contradicting the original claim. +differences in modelling decisions can change effect estimates, their +uncertainty, statistical significance, and sometimes the direction of the +substantive conclusion. Note that @NosekErrington2020 define replication as a study “for which any outcome would be considered diagnostic evidence about a claim from