diff --git a/references.bib b/references.bib index 22d468a..ff1a93f 100644 --- a/references.bib +++ b/references.bib @@ -198,7 +198,8 @@ @article{BotvinikNezerEtAl2020 volume = {582}, number = {7810}, pages = {84-88}, - year = {2020} + year = {2020}, + doi = {10.1038/s41586-020-2314-9} } @article{BoyceEtAl2024, @@ -219,6 +220,17 @@ @article{BrandtEtAl2014 doi = {10.1016/j.jesp.2013.10.005} } +@article{BreznauEtAl2022, + author = {Breznau, N. and Rinke, E. M. and Wuttke, A. and Nguyen, H. H. V. and Adem, M. and Adriaans, J. and Alvarez-Benjumea, A.}, + title = {Observing many researchers using the same data and hypothesis reveals a hidden universe of uncertainty}, + journal = {Proceedings of the National Academy of Sciences}, + volume = {119}, + number = {44}, + pages = {e2203150119}, + year = {2022}, + doi = {10.1073/pnas.2203150119} +} + @article{BrodeurEtAl2024a, author = {Brodeur, A. and Cook, N. M. and Hartley, J. S. and Heyes, A.}, title = {Do Preregistration and Preanalysis Plans Reduce p-Hacking and Publication Bias? Evidence from 15,992 Test Statistics and Suggestions for Improvement}, @@ -1409,6 +1421,17 @@ @article{SchultzeEtAl2018 doi = {10.1002/bdm.2065} } +@article{SilberzahnEtAl2018, + author = {Silberzahn, R. and Uhlmann, E. L. and Martin, D. P. and Anselmi, P. and Aust, F. and Awtrey, E. and Bahník, Š.}, + title = {Many analysts, one data set: Making transparent how variations in analytic choices affect results}, + journal = {Advances in Methods and Practices in Psychological Science}, + volume = {1}, + number = {3}, + pages = {337-356}, + year = {2018}, + doi = {10.1177/2515245917747646} +} + @article{SimmonsEtAl2011, author = {Simmons, J. P. and Nelson, L. D. and Simonsohn, U.}, title = {False-positive psychology: Undisclosed flexibility in data collection and analysis allows presenting anything as significant}, diff --git a/understanding.qmd b/understanding.qmd index dad2563..1757cfd 100644 --- a/understanding.qmd +++ b/understanding.qmd @@ -125,6 +125,24 @@ close replications more directly test the credibility of original results, while conceptual replications that vary features of the design are concerned with generalizability. +A related line of work isolates the analysis stage by holding the data +fixed. In so-called many-analyst studies, multiple independent teams +analyse the same dataset to test the same hypothesis, yet reach divergent +conclusions. @SilberzahnEtAl2018 had numerous teams estimate whether +football referees are more likely to give red cards to dark-skinned +players, and the teams reported a wide spread of effect estimates despite +working from identical data. @BotvinikNezerEtAl2020 observed substantial +variability when many teams tested hypotheses on a single neuroimaging +dataset, and @BreznauEtAl2022 described a “hidden universe of +uncertainty” across the many teams that analysed the same survey data. +Because the data are held constant, this variability cannot be attributed +to differences in data collection and instead reflects choices made in the +analysis pipeline, which matters for interpreting reproductions and +replications: even when the same data are reanalysed, defensible +differences in modelling decisions can change effect estimates, their +uncertainty, statistical significance, and sometimes the direction of the +substantive conclusion. + Note that @NosekErrington2020 define replication as a study “for which any outcome would be considered diagnostic evidence about a claim from prior research”. This can lead to issues when the original claim is not