Add many-analyst findings alongside Landy et al. (Ch2, #15)

references.bib

@@ -198,7 +198,8 @@ @article{BotvinikNezerEtAl2020
   volume = {582},
   number = {7810},
   pages = {84-88},
-  year = {2020}
+  year = {2020},
+  doi = {10.1038/s41586-020-2314-9}
 }
 
 @article{BoyceEtAl2024,
@@ -219,6 +220,17 @@ @article{BrandtEtAl2014
   doi = {10.1016/j.jesp.2013.10.005}
 }
 
+@article{BreznauEtAl2022,
+  author = {Breznau, N. and Rinke, E. M. and Wuttke, A. and Nguyen, H. H. V. and Adem, M. and Adriaans, J. and Alvarez-Benjumea, A.},
+  title = {Observing many researchers using the same data and hypothesis reveals a hidden universe of uncertainty},
+  journal = {Proceedings of the National Academy of Sciences},
+  volume = {119},
+  number = {44},
+  pages = {e2203150119},
+  year = {2022},
+  doi = {10.1073/pnas.2203150119}
+}
+
 @article{BrodeurEtAl2024a,
   author = {Brodeur, A. and Cook, N. M. and Hartley, J. S. and Heyes, A.},
   title = {Do Preregistration and Preanalysis Plans Reduce p-Hacking and Publication Bias? Evidence from 15,992 Test Statistics and Suggestions for Improvement},
@@ -1409,6 +1421,17 @@ @article{SchultzeEtAl2018
   doi = {10.1002/bdm.2065}
 }
 
+@article{SilberzahnEtAl2018,
+  author = {Silberzahn, R. and Uhlmann, E. L. and Martin, D. P. and Anselmi, P. and Aust, F. and Awtrey, E. and Bahník, Š.},
+  title = {Many analysts, one data set: Making transparent how variations in analytic choices affect results},
+  journal = {Advances in Methods and Practices in Psychological Science},
+  volume = {1},
+  number = {3},
+  pages = {337-356},
+  year = {2018},
+  doi = {10.1177/2515245917747646}
+}
+
 @article{SimmonsEtAl2011,
   author = {Simmons, J. P. and Nelson, L. D. and Simonsohn, U.},
   title = {False-positive psychology: Undisclosed flexibility in data collection and analysis allows presenting anything as significant},

understanding.qmd

@@ -125,6 +125,24 @@ close replications more directly test the credibility of original
 results, while conceptual replications that vary features of the design
 are concerned with generalizability.
 
+A related line of work isolates the analysis stage by holding the data
+fixed. In so-called many-analyst studies, multiple independent teams
+analyse the same dataset to test the same hypothesis, yet reach divergent
+conclusions. @SilberzahnEtAl2018 had numerous teams estimate whether
+football referees are more likely to give red cards to dark-skinned
+players, and the teams reported a wide spread of effect estimates despite
+working from identical data. @BotvinikNezerEtAl2020 observed substantial
+variability when many teams tested hypotheses on a single neuroimaging
+dataset, and @BreznauEtAl2022 described a “hidden universe of
+uncertainty” across the many teams that analysed the same survey data.
+Because the data are held constant, this variability cannot be attributed
+to differences in data collection and instead reflects choices made in the
+analysis pipeline, which matters for interpreting reproductions and
+replications: even when the same data are reanalysed, defensible
+differences in modelling decisions can change effect estimates, their
+uncertainty, statistical significance, and sometimes the direction of the
+substantive conclusion.
+
 Note that @NosekErrington2020 define replication as a study “for which
 any outcome would be considered diagnostic evidence about a claim from
 prior research”. This can lead to issues when the original claim is not