[{"k":"22WZMMFB","t":"The Equivalence of Weighted Kappa and the Intraclass Correlation Coefficient as Measures of Reliability","au":"Fleiss, Joseph L.","y":1973,"vn":"Educational and Psychological Measurement","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/001316447303300309","cx":"Analogue","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"closed","cby":3357,"self":false,"s":"A statistical paper proving that weighted kappa, a chance-corrected agreement statistic for categorical ratings, is mathematically equivalent to the intraclass correlation coefficient under general conditions.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"23VVRPE8","t":"An Empirical Comparative Assessment of Inter-Rater Agreement of Binary Outcomes and Multiple Raters","au":"Konstantinidis, Menelaos","y":2022,"vn":"Symmetry","ty":"journalArticle","tyl":"Journal article","doi":"10.3390/sym14020262","cx":"Analogue","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"gold","cby":26,"self":false,"s":"Simulation study comparing four inter-rater agreement statistics (Fleiss', Light's, Conger's Kappa, Gwet's AC1) for binary ratings across varying rater and sample counts, finding Gwet's AC1 more stable when outcome prevalence is uneven.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"24E7BBHB","t":"The reliability of ethical reviews: Is it desirable?","au":"Doob, Anthony N","y":1983,"vn":"Canadian Psychology / Psychologie canadienne","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/h0080918","cx":"General","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":7,"self":false,"s":"A commentary responding to an earlier paper on ethics-review reliability, arguing that low agreement among ethics reviewers need not signal a problem and may reflect appropriately diverse viewpoints.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"24YIFL73","t":"‘Conditional Acceptance’ (additional experiments required): A scoping review of recent evidence on key aspects of Open Peer Review","au":"Ross-Hellauer, Tony","y":2022,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31222/osf.io/r6t8p","cx":"Journal","co":["Interventions"],"sg":["whole-process"],"x":0,"st":"systematic-review-meta-analysis","oast":"gold","cby":4,"self":false,"s":"A scoping review synthesising 52 studies (2017 onward) on open identities and open reports in journal peer review, finding growing but still uncommon adoption, mixed attitudes, and no evidence that openness slows reviews or changes acceptance rates.","pg":"preregistered","pgr":"OSF","pgid":"https://osf.io/dvhxt/","da":"open","dau":"https://doi.org/10.5281/zenodo.7454789","ca":"not-applicable","cau":""},{"k":"28GK7NTQ","t":"Assessing quality of selection procedures: Lower bound of false positive rate as a function of inter‐rater reliability","au":"Bartoš, František","y":2024,"vn":"British Journal of Mathematical and Statistical Psychology","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/bmsp.12343","cx":"Grant","co":["Reliability-noise"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"simulation-methodological","oast":"hybrid","cby":2,"self":false,"s":"Proposes a method linking inter-rater reliability coefficients to false positive and false negative rates in binary selection decisions, and provides an R package for computing these error bounds in grant review.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://osf.io/674fk/","ca":"open","cau":"https://osf.io/674fk/"},{"k":"29UTGYZQ","t":"Peer review of interdisciplinary research proposals","au":"Porter, Alan L.","y":1985,"vn":"Science, Technology, & Human Values","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/016224398501000304","cx":"Grant","co":["Reliability-noise","Bias"],"sg":["individual-scoring","whole-process"],"x":0,"st":"observational-quantitative","oast":"closed","cby":107,"self":false,"s":"Discusses why judging future research proposals is harder than judging finished work, since projects can change direction, staffing, or scope after funding, undermining rating consistency.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"2BMCWPPH","t":"Breaking the gatekeepers: how AI will revolutionize scientific funding","au":"Mangalam, Madhur","y":2025,"vn":"Frontiers in Artificial Intelligence","ty":"journalArticle","tyl":"Journal article","doi":"10.3389/frai.2025.1667752","cx":"Grant","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":0,"self":false,"s":"An analysis piece arguing that AI could disrupt entrenched, seniority-favoring grant peer review, potentially reducing bias and broadening opportunities for early-career and unconventional researchers.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"2BQGZI7A","t":"Persistent nepotism in peer-review","au":"Sandström, Ulf","y":2007,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-008-0211-3","cx":"Grant","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"bronze","cby":183,"self":false,"s":"A replication of Wennerås and Wold's Swedish study finds that principal investigators with a reviewer affiliation score notably higher on grant applications, pointing to persistent favouritism in medical research council funding decisions.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"www.forskningspolitik.se/data.asp","ca":"no-statement","cau":""},{"k":"2F4I83RM","t":"Peer Review and Citation Data in Predicting University Rankings, a Large-Scale Analysis","au":"Pride, David","y":2018,"vn":"Lecture notes in computer science","ty":"book","tyl":"Book","doi":"10.1007/978-3-030-00066-0_17","cx":"General","co":["Validity"],"sg":["whole-process"],"x":0,"st":"book-monograph","oast":"closed","cby":15,"self":false,"s":"An analysis of nearly 200,000 papers and millions of citations finding that citation-based metrics correlate strongly with peer review judgments when aggregated at the institutional level, suggesting they could reduce reliance on peer review in national research evaluations.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://figshare.com/s/69199811238dcb4ca987","ca":"no-statement","cau":""},{"k":"2F6KAXNL","t":"The Reliability of Peer Reviews of Papers on Information Systems","au":"Wood, Michael","y":2004,"vn":"Journal of Information Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/0165551504041673","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":30,"self":false,"s":"Examines reliability of double-blind peer review at two UK information-systems conferences, finding agreement close to or somewhat above chance levels and discussing what this implies for how reviewing is organized.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"2G4INPDL","t":"A retrospective analysis of the peer review of more than 75,000 Marie Curie proposals between 2007 and 2018.","au":"Pina, David G","y":2021,"vn":"eLife","ty":"journalArticle","tyl":"Journal article","doi":"10.7554/eLife.59338","cx":"Grant","co":["Reliability-noise"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"gold","cby":26,"self":false,"s":"Retrospective study of over 75,000 Marie Curie grant applications (2007-2018) finding that fewer evaluation criteria and virtual panel meetings had little effect on funding outcomes compared with grant type or field.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"","ca":"no-statement","cau":""},{"k":"2G777HQX","t":"A scoping review of simulation models of peer review.","au":"Feliciani, Thomas","y":2019,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-019-03205-w","cx":"General","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"systematic-review-meta-analysis","oast":"hybrid","cby":36,"self":false,"s":"A scoping review of 46 articles modelling peer review through simulation, proposing a taxonomy of model types and summarising findings on decision-making, reviewer matching, editorial strategy, and bias across the fragmented literature.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"2GEXWPD3","t":"Race, ethnicity, and NIH research awards.","au":"Ginther, Donna K","y":2011,"vn":"Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1126/science.1196783","cx":"Grant","co":["Bias"],"sg":["funding-decision"],"x":0,"st":"observational-quantitative","oast":"green","cby":1087,"self":false,"s":"Analysis of NIH R01 award data finds Black and Asian applicants were less likely than White applicants to receive funding, a gap that narrowed but persisted after controlling for background and track record.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"http://report.nih.gov/investigators_and_trainees/index.aspx","ca":"no-statement","cau":""},{"k":"2GKBJDYD","t":"Sample size and precision in NIH peer review.","au":"Kaplan, David","y":2008,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0002761","cx":"Grant","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"gold","cby":56,"self":false,"s":"A statistical analysis argues that NIH's minimum of four reviewers per grant application is too few to achieve the level of precision the review process is meant to provide.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"2H7BLLRW","t":"A structured, journal-led peer-review mentoring program enhances peer review training","au":"Lyons‐Warren, Ariel M.","y":2024,"vn":"Research Integrity and Peer Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s41073-024-00143-x","cx":"Journal","co":["Interventions"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":13,"self":false,"s":"A pre-post study of 55 mentees across five cohorts found that a structured, mentor-guided peer-review training program raised review-quality scores and boosted participants' confidence and comfort with reviewing.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"no-statement","cau":""},{"k":"2I7XLXLK","t":"Gender bias goes away when grant reviewers focus on the science.","au":"Guglielmi, Giorgia","y":2018,"vn":"Nature","ty":"journalArticle","tyl":"Journal article","doi":"10.1038/d41586-018-01212-0","cx":"Grant","co":["Bias","Interventions"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"bronze","cby":23,"self":false,"s":"A brief piece (with abstract text unavailable/mismatched) reportedly relating to grant reviewers and gender bias, suggesting that focusing evaluation on scientific content narrows gender-based score differences.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"2KETM58Y","t":"A new approach to grant review assessments: score, then rank","au":"Gallo, Stephen A.","y":2023,"vn":"Research Integrity and Peer Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s41073-023-00131-7","cx":"Grant","co":["Reliability-noise","Interventions"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"simulation-methodological","oast":"gold","cby":9,"self":false,"s":"Describes a statistical method combining reviewer rankings with numeric ratings to resolve ties and inconsistencies in grant scoring, and tests it on both simulated examples and real panel review data.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"10.6084/m9.figshare.20505747","ca":"no-statement","cau":"https://cran.r-project.org/package=rankrate"},{"k":"2KPUFM47","t":"Using prediction markets to forecast research evaluations","au":"Munafò, Marcus R.","y":2015,"vn":"Royal Society Open Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1098/rsos.150287","cx":"General","co":["Validity"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"gold","cby":9,"self":false,"s":"Tests whether a prediction market among researchers could approximate the outcomes of the UK's 2014 Research Excellence Framework for chemistry departments, as a cheaper alternative to full peer evaluation.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"doi:10.5523/bris.1s8bmxopxry716vkmluw1aii5","ca":"no-statement","cau":""},{"k":"2LKYJJFR","t":"Addressing animal methods bias through government-led initiatives","au":"Kopew, Jessica S","y":2026,"vn":"Frontiers in Toxicology","ty":"journalArticle","tyl":"Journal article","doi":"10.3389/ftox.2026.1761759","cx":"General","co":["Bias","Interventions"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":0,"self":false,"s":"Discusses \"animal methods bias,\" where reviewers favour animal-based research or lack expertise in non-animal methods, and describes 2025 US and UK government initiatives aiming to counter this bias in funding decisions.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"2LXPUFDJ","t":"Does the inclusion of non-academic reviewers make any difference for grant impact panels?","au":"Luo, Junwen","y":2021,"vn":"Science and Public Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/scipol/scab046","cx":"Grant","co":["Interventions"],"sg":["panel-consensus","reviewer-selection"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":8,"self":false,"s":"An analysis of 164 Science Foundation Ireland review reports comparing panels with and without non-academic members, finding mixed panels produced longer, broader discussions of societal impact and attributed outcomes differently than academic-only panels.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://doi.org/10.6084/m9.figshare.13651058.v1","ca":"no-statement","cau":""},{"k":"2MRIZ4HG","t":"A systematic approach to assessing the accuracy of a peer review process in an academic journal","au":"García, J. A.","y":2025,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-025-05452-6","cx":"Journal","co":["Validity"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"simulation-methodological","oast":"green","cby":0,"self":false,"s":"Applies Bayesian reasoning to estimate how often acceptance and rejection signals from journal reviewers correctly identify manuscript quality, finding weaker predictive value at lower-impact, higher-cost journals.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"2NTZHLW6","t":"Reliability in ethics reviews: Some initial empirical findings.","au":"Eaton, Warren O","y":1983,"vn":"Canadian Psychology / Psychologie canadienne","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/h0080684","cx":"General","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":11,"self":false,"s":"An early study of a university ethics review committee found reviewers agreed on about two-thirds of decisions, but this was barely above what chance alone would produce.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"2QWWWN8B","t":"Commentary: New Guidelines for NIH Peer Review: Improving the System or Undermining It?","au":"Spiegel, Allen M.","y":2010,"vn":"Academic Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.1097/acm.0b013e3181d7e130","cx":"Grant","co":["Validity","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":3,"self":false,"s":"A commentary responding to criticism of NIH peer-review reforms, defending policies favoring new investigators and innovation by placing them in the historical context of NIH funding priorities.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"2SCD2GXP","t":"Using democracy to award research funding: an observational study.","au":"Barnett, Adrian G","y":2017,"vn":"Research integrity and peer review","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s41073-017-0040-0","cx":"Grant","co":["Bias","Interventions"],"sg":["funding-decision"],"x":0,"st":"observational-quantitative","oast":"gold","cby":11,"self":false,"s":"A pilot study asked Australian researchers to nominate scientists most deserving of funding, comparing the results to conventional peer-reviewed fellowship winners; voting was faster than standard review but showed bias toward voters' own institutions.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://figshare.com/articles/Democratic_funding/5197387","ca":"no-statement","cau":""},{"k":"2SJ6PABX","t":"Inter-reviewer reliability of human literature reviewing and implications for the introduction of machine-assisted systematic reviews: a mixed-methods review","au":"Hanegraaf, P.","y":2024,"vn":"BMJ Open","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmjopen-2023-076912","cx":"Analogue","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"systematic-review-meta-analysis","oast":"gold","cby":31,"self":false,"s":"A mixed-methods review of published systematic reviews finding reported inter-reviewer kappa scores around 0.8 for screening and data extraction, with no link to team size or review quality, and a survey showing researchers expect similarly moderate-to-high agreement from machine-assisted review.","pg":"preregistered","pgr":"PROSPERO","pgid":"CRD42023386706","da":"restricted","dau":"","ca":"no-statement","cau":""},{"k":"2TIYPCIB","t":"Scientific research funding criteria: an empirical study of peer review and scientometrics","au":"Devyatkin, D.","y":2018,"vn":"Practical issues of intelligent innovations","ty":"bookSection","tyl":"Book chapter","doi":"10.1007/978-3-319-78437-3_12","cx":"Grant","co":["Validity"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"book-monograph","oast":"closed","cby":3,"self":false,"s":"Data-mining analysis of Russian Foundation for Basic Research review forms and funding decisions, finding that bibliometric indicators added little explanatory power beyond reviewers' own assessments.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"no-statement","cau":""},{"k":"2TWYBDP6","t":"Overlooked, Underlying: Understanding tacit criteria of proposal reviewing during a mock panel review","au":"Sims, Randi","y":2024,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.18260/1-2--43835","cx":"Grant","co":["Interventions"],"sg":["individual-scoring","panel-consensus"],"x":0,"st":"qualitative","oast":"gold","cby":1,"self":false,"s":"Case study of a mentoring program pairing novice NSF grant reviewers with former program officers, examining what unstated criteria new reviewers rely on and how a mock panel discussion changes their evaluations.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"2UD55EPP","t":"Guardians of science or stewards of society? The role of grant peer reviewer identity in governing socially responsible innovation","au":"Wu, Y","y":2026,"vn":"Journal of Responsible Innovation","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/23299460.2026.2674322","cx":"Grant","co":["general-systemic"],"sg":["individual-scoring"],"x":0,"st":"qualitative","oast":"gold","cby":0,"self":false,"s":"Interviews and vignettes with 28 Chinese grant reviewers examine how they reconcile scientific gatekeeping with responsible-innovation mandates, identifying three coping strategies (separating, merging, or ranking scientific versus societal criteria) that can sideline social considerations.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"2VFUUIMD","t":"Open peer review: A randomised controlled trial","au":"Walsh, Elizabeth","y":2000,"vn":"The British Journal of Psychiatry","ty":"journalArticle","tyl":"Journal article","doi":"10.1192/bjp.176.1.47","cx":"Journal","co":["Interventions"],"sg":["individual-scoring"],"x":1,"st":"experimental-rct","oast":"bronze","cby":268,"self":false,"s":"Randomised trial at a psychiatry journal found reviewers who agreed to sign their reviews wrote higher-quality, more courteous reviews, took longer, and recommended publication more often than unsigned reviewers.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"2XED69NE","t":"Canada's First Experience With a Patient-Adjudicated Cancer Research Grant Competition: A Case Study","au":"Ackerman, Ruth","y":2022,"vn":"Journal of Patient Experience","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/23743735221123424","cx":"Grant","co":["Reliability-noise","Interventions"],"sg":["reviewer-selection"],"x":0,"st":"observational-quantitative","oast":"diamond","cby":0,"self":false,"s":"A case study describes a Canadian cancer-research funding competition in which patient partners, rather than scientific experts, held sole authority to decide which proposals were funded.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"2ZR5VGF7","t":"Reply to “Funding by Lottery: Political Problems and Research Opportunities”","au":"Casadevall, Arturo","y":2016,"vn":"mBio","ty":"journalArticle","tyl":"Journal article","doi":"10.1128/mbio.01401-16","cx":"Grant","co":["Reliability-noise","Bias","Validity"],"sg":["funding-decision"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":0,"self":false,"s":"A brief reply in a scholarly exchange responding to comments on lottery-based grant funding, addressing a perceived obstacle to adopting a modified lottery model.","pg":"not-codeable","pgr":"","pgid":"","da":"","dau":"","ca":"","cau":""},{"k":"2ZZ5VVPC","t":"Blinding reduces institutional prestige bias during initial review of applications for a young investigator award.","au":"Hultgren, Anne E","y":2024,"vn":"eLife","ty":"journalArticle","tyl":"Journal article","doi":"10.7554/eLife.92339","cx":"Grant","co":["Bias","Interventions"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":9,"self":false,"s":"After a foundation blinded applicant identity and institution in the first stage of a young-investigator award, the pool advancing to full review shifted toward less prestigious institutions, with no corresponding shift by gender.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"","ca":"open","cau":""},{"k":"32NEFZ9T","t":"A Difficult Balance: Editorial Peer Review in Medicine","au":"Carney, Martha J.","y":1986,"vn":"JAMA","ty":"journalArticle","tyl":"Journal article","doi":"10.1001/jama.1986.03380020117036","cx":"Journal","co":["Validity","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":192,"self":false,"s":"A review of a book by a former British Medical Journal editor examining strengths and flaws of editorial peer review in medicine from multiple stakeholder perspectives.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"33C9QVWD","t":"Editorial behaviors in peer review","au":"Wang, Wei","y":2016,"vn":"SpringerPlus","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s40064-016-2601-y","cx":"Journal","co":["Bias"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"simulation-methodological","oast":"diamond","cby":26,"self":false,"s":"Agent-based simulation of author-editor-reviewer interactions showing that editorial bias degrades the quality of accepted papers more severely than reviewer bias does.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"3455VSI4","t":"External Tests of Peer Review Validity Via Impact Measures","au":"Gallo, Stephen A","y":2018,"vn":"Frontiers in Research Metrics and Analytics","ty":"journalArticle","tyl":"Journal article","doi":"10.3389/frma.2018.00022","cx":"Grant","co":["Validity"],"sg":["post-award-validity"],"x":0,"st":"systematic-review-meta-analysis","oast":"gold","cby":11,"self":false,"s":"Literature review of 48 studies testing whether peer review scores for funded research predict later real-world impact, finding modest predictive validity overall but weak ability to distinguish among top-tier proposals.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"35UMJRRJ","t":"Gender disparity in funding rates in double-blind grant peer review: The case of the Villum Experiment","au":"Madsen, Emil Bargmann","y":2025,"vn":"Quantitative Science Studies","ty":"journalArticle","tyl":"Journal article","doi":"10.1162/qss.a.5","cx":"Grant","co":["Bias"],"sg":["funding-decision"],"x":0,"st":"observational-quantitative","oast":"gold","cby":1,"self":false,"s":"A study of over 2,000 Villum Experiment applications under double-blind review finds a small gender gap in funding success concentrated in life sciences, attributable to structural workforce composition rather than direct reviewer bias.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://osf.io/n5f4u/","ca":"open","cau":"https://osf.io/n5f4u/"},{"k":"36AR47M3","t":"GKAPPA: Generalized Kappa Coefficient","au":"Uebersax, John S.","y":1981,"vn":"Applied Psychological Measurement","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/014662168100500104","cx":"Analogue","co":["Reliability-noise"],"sg":["whole-process"],"x":0,"st":"simulation-methodological","oast":"closed","cby":40,"self":false,"s":"Describes a FORTRAN program, GKAPPA, that computes a generalized kappa statistic and its significance via simulation for irregular multi-rater rating designs.","pg":"not-codeable","pgr":"","pgid":"","da":"","dau":"","ca":"","cau":""},{"k":"37SMEXTX","t":"Randomisation as a tool for organisational decision-making: a debatable or debilitating proposition?","au":"Dushnitsky, Gary","y":2023,"vn":"Industry and Innovation","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/13662716.2023.2281983","cx":"General","co":["Bias","Validity","Interventions"],"sg":["funding-decision"],"x":0,"st":"conceptual-commentary","oast":"bronze","cby":4,"self":false,"s":"An essay weighs arguments for and against replacing expert judgment with randomisation in funding and management decisions, and considers how AI might reshape this debate.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"3956JT8Q","t":"Development and application of a comprehensive glossary for the identification of statistical and methodological concepts in peer review reports","au":"Buljan, Ivan","y":2024,"vn":"Journal of Informetrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.joi.2024.101555","cx":"Journal","co":["general-systemic"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"green","cby":2,"self":false,"s":"Researchers built a glossary of over a thousand terms to code methodological and statistical issues raised in nearly 500,000 peer review reports, finding disciplinary and gender differences in which issues reviewers flagged.","pg":"preregistered","pgr":"OSF","pgid":"https://osf.io/tzb6n","da":"restricted","dau":"https://osf.io/d34b9/","ca":"no-statement","cau":""},{"k":"3B9JRHFK","t":"Panel peer review of grant applications: what do we know from research in social psychology on judgment and decision-making in groups?","au":"Olbrecht, Meike","y":2010,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.3152/095820210X12809191250762","cx":"Grant","co":["Bias","Interventions"],"sg":["panel-consensus"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":65,"self":false,"s":"A conceptual paper applies social-psychological research on group judgment, such as groupthink and polarization, to grant review panels, arguing these dynamics remain unexamined in peer-review scholarship and proposing countermeasures.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"3DCVN5UP","t":"Bias and effort in peer review","au":"García, J. A.","y":2015,"vn":"Journal of the Association for Information Science and Technology","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/asi.23307","cx":"Journal","co":["Bias"],"sg":["reviewer-selection","individual-scoring"],"x":0,"st":"simulation-methodological","oast":"closed","cby":25,"self":false,"s":"A theoretical economics paper modeling why journal editors might tolerate biased reviewers because they exert more effort, and how oversight by associate editors can offset the resulting bias.","pg":"no-statement","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"3FFMLXGD","t":"Blinding Models for Scientific Peer-Review of Biomedical Research Proposals: A Systematic Review","au":"Qussini, Seba","y":2023,"vn":"Journal of Empirical Research on Human Research Ethics","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/15562646231191424","cx":"Grant","co":["Bias","Interventions"],"sg":["individual-scoring"],"x":0,"st":"systematic-review-meta-analysis","oast":"closed","cby":6,"self":false,"s":"Systematic review of studies comparing blinding approaches in peer review of biomedical grant proposals, concluding evidence is sparse and calling for more randomized trials on this question.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"3FUV2PR2","t":"How do referees integrate evaluation criteria into their overall judgment? Evidence from grant peer review","au":"Hug, Sven E.","y":2024,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-023-04915-y","cx":"Grant","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":13,"self":false,"s":"Analysis of scoring data from a career funding scheme testing whether reviewers apply evaluation criteria uniformly or case-by-case, concluding that reviewers use many criteria combined through complex, though largely consistent, rules.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"no-statement","cau":""},{"k":"3HN82VIN","t":"No Agreement Without Loss: Learning and Social Choice in Peer Review","au":"Barceló, Pablo","y":2023,"vn":"Frontiers in artificial intelligence and applications","ty":"book","tyl":"Book","doi":"10.3233/faia230270","cx":"Journal","co":["Reliability-noise","Bias"],"sg":["individual-scoring"],"x":0,"st":"book-monograph","oast":"hybrid","cby":0,"self":false,"s":"A theoretical critique of a proposed method for aggregating multi-criteria reviewer scores into overall recommendations, arguing its assumptions and axioms have unrealistic trade-offs, including a discontinuity problem once an unrealistic condition is relaxed.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"3KXX595H","t":"Menage a quoi? Optimal number of peer reviewers.","au":"Snell, Richard R","y":2015,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0120838","cx":"Grant","co":["Reliability-noise"],"sg":["reviewer-selection","individual-scoring"],"x":0,"st":"simulation-methodological","oast":"gold","cby":39,"self":false,"s":"Bootstrap analysis of fellowship competition data argues that five reviewers per application is a practical optimum, balancing reliability gains against the diminishing returns and cost of adding more reviewers.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"","ca":"open","cau":""},{"k":"3LCPQ3XU","t":"Developing a Criteria Framework for Peer Review: A Critical Interpretive Synthesis","au":"Li, Yifei","y":2025,"vn":"Learned Publishing","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/leap.2016","cx":"Journal","co":["general-systemic"],"sg":["individual-scoring"],"x":0,"st":"systematic-review-meta-analysis","oast":"hybrid","cby":1,"self":false,"s":"A synthesis of 157 sources on journal manuscript review builds a four-level framework of criteria editors and reviewers use, organised into categories such as rigor, clarity, and value to the journal.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"no-statement","cau":""},{"k":"3MLFBQHS","t":"Exploring the Impact of Generative AI on Peer Review: Insights from Journal Reviewers","au":"Ebadi, Saman","y":2025,"vn":"Journal of Academic Ethics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s10805-025-09604-4","cx":"Journal","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"qualitative","oast":"hybrid","cby":30,"self":false,"s":"Interviews with twelve journal reviewers found they see generative AI as useful for routine tasks like screening and language checks, but raised concerns about bias, transparency, and the need to keep human judgment central.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"not-applicable","cau":""},{"k":"3MVDPR4T","t":"Peer reviewer topic choice and its impact on interrater reliability: A mixed-method study","au":"Feliciani, Thomas","y":2022,"vn":"Quantitative Science Studies","ty":"journalArticle","tyl":"Journal article","doi":"10.1162/qss_a_00207","cx":"Grant","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"gold","cby":8,"self":false,"s":"Analysing reviewer forms and a survey at an Irish funding agency, the study finds reviewers vary in which proposal topics they emphasize and how they map them to scoring criteria, but simulations suggest this variation barely affects overall reviewer disagreement.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"https://doi.org/10.6084/m9.figshare.13651058.v1","ca":"open","cau":"https://github.com/thomasfeliciani/TC-mapping"},{"k":"3MWL4DVH","t":"Beyond declarations: Metrics, rankings and responsible assessment","au":"Morgan-Thomas, Anna","y":2024,"vn":"Research Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.respol.2024.105093","cx":"General","co":["Bias","Interventions"],"sg":["funding-decision"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":24,"self":false,"s":"Using UK national research assessment data in business and management, this study finds that the correlation between journal prestige rankings and expert quality judgments has not weakened despite institutions formally endorsing responsible-assessment principles like DORA.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"open","cau":""},{"k":"3QAW4CJ4","t":"NIH Peer Review Reform—Change We Need, or Lipstick on a Pig?","au":"Fang, Ferric C.","y":2009,"vn":"Infection and Immunity","ty":"journalArticle","tyl":"Journal article","doi":"10.1128/iai.01567-08","cx":"Grant","co":["Reliability-noise","Bias","Validity","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"green","cby":41,"self":false,"s":"Opinion piece questioning whether a proposed NIH peer-review reform genuinely addresses underlying problems or merely adjusts the process superficially.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"3RPSC5JB","t":"The Feeling Rules of Peer Review: Defining, Displaying, and Managing Emotions in Evaluation for Research Funding","au":"Brunet, Lucas","y":2023,"vn":"Minerva","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11024-023-09518-8","cx":"Grant","co":["general-systemic"],"sg":["individual-scoring","panel-consensus"],"x":0,"st":"qualitative","oast":"hybrid","cby":10,"self":false,"s":"Interview-based study of European Research Council and Marie Curie grant panels describing unwritten norms about which emotions reviewers are expected to show or suppress during proposal evaluation.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"3S8IX9X5","t":"A Note on Estimating the Reliability of Categorical Data","au":"Finn, Rachael","y":1970,"vn":"Educational and Psychological Measurement","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/001316447003000106","cx":"Analogue","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"closed","cby":205,"self":false,"s":"An older methodological paper proposing an alternative statistic for estimating rater reliability on categorical data, addressing limitations of the intraclass correlation when item variance is low.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"3TDTASF9","t":"Grant funding: Playing the odds","au":"Fang, Ferric C.","y":2016,"vn":"Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1126/science.352.6282.158-a","cx":"Grant","co":["Bias","Validity","Interventions"],"sg":["funding-decision"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":13,"self":false,"s":"Commentary questioning whether grant peer review reliably identifies the strongest science, citing evidence of imprecise and inconsistent funding decisions.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"3TRWDK3E","t":"Decentralized knowledge assessment","au":"Sun, Kai","y":2025,"vn":"The Innovation","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.xinn.2025.100945","cx":"Journal","co":["Interventions"],"sg":["reviewer-selection","whole-process"],"x":0,"st":"simulation-methodological","oast":"gold","cby":0,"self":false,"s":"Proposes replacing centralized editor-assigned peer review with a decentralized model in which a scholarly community collectively assigns and rewards reviewing, tested through conference simulations.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"open","cau":""},{"k":"43PNVEVR","t":"'Are you siding with a personality or the grant proposal?': observations on how peer review panels function.","au":"Coveney, John","y":2017,"vn":"Research Integrity and Peer Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s41073-017-0043-x","cx":"Grant","co":["Bias","Interventions"],"sg":["panel-consensus"],"x":0,"st":"qualitative","oast":"gold","cby":32,"self":false,"s":"Interviews with Australian grant review panelists examined how panel procedures, composition, and dynamics shape funding decisions, comparing a simplified review process to a more conventional one.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"not-applicable","cau":""},{"k":"43RMRTWZ","t":"Augmented Peer Review: A Framework for AI-Supervised Academic Publishing","au":"Martin, Matt","y":2026,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31222/osf.io/cxarz_v1","cx":"Journal","co":["Reliability-noise","Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":0,"self":false,"s":"Proposes a governance framework in which an AI system, termed an \"AI Board Chair,\" screens manuscripts against methodological and transparency standards to support, not replace, human editors, aiming to shorten review timelines and reduce bias.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"44PDV7ET","t":"Studying grant decision-making: a linguistic analysis of review reports.","au":"van den Besselaar, Peter","y":2018,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-018-2848-x","cx":"Grant","co":["Reliability-noise","Bias","Validity"],"sg":["individual-scoring","panel-consensus"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":61,"self":false,"s":"A linguistic analysis of grant review reports, combined with interviews and bibliometric data, finding that panels apply the same criteria to past track record and proposed ideas alike and tend to focus on finding flaws rather than identifying high-potential proposals.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"no-statement","cau":""},{"k":"4BW48ANW","t":"Participation and influence in publication review proceedings: A reply.","au":"Lindsey, Duncan","y":1977,"vn":"American Psychologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1037//0003-066x.32.7.579","cx":"Journal","co":["general-systemic"],"sg":["reviewer-selection"],"x":0,"st":"observational-quantitative","oast":"closed","cby":18,"self":false,"s":"A reply/commentary piece discussing status and influence dynamics among journal editors and reviewers, including whether editorial board members are actually the most eminent figures in their fields.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"4C6DKMM7","t":"Evidence on peer review—scientific quality control or smokescreen?","au":"Goldbeck-Wood, Sandy","y":1999,"vn":"BMJ","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmj.318.7175.44","cx":"Journal","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"green","cby":96,"self":false,"s":"Editorial questioning the evidence base for peer review, noting that despite its central role in academic publishing, few studies test whether it reliably identifies good science.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"4CDE9Y6H","t":"Peer review and gender bias: A study on 145 scholarly journals","au":"Squazzoni, Flaminio","y":2021,"vn":"Science Advances","ty":"journalArticle","tyl":"Journal article","doi":"10.1126/sciadv.abd0299","cx":"Journal","co":["Bias"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"gold","cby":200,"self":false,"s":"A large-scale analysis of 145 journals, 1.7 million authors and 740,000 referees examines whether editorial and referee decisions disadvantage women, finding manuscripts by women were treated at least as favourably as those by men.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://doi.org/10.7910/DVN/3IKRGI","ca":"no-statement","cau":""},{"k":"4EBV6BAJ","t":"Academic Journals, Incentives, and the Quality of Peer Review: A Model","au":"Zollman, Kevin","y":2023,"vn":"Philosophy of Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/psa.2023.81","cx":"Journal","co":["Reliability-noise"],"sg":["funding-decision"],"x":0,"st":"simulation-methodological","oast":"hybrid","cby":8,"self":false,"s":"Theoretical model of journal incentives shows that pursuing a high rejection rate, rather than publishing top-quality work, can push editors toward less accurate review to attract more mediocre submissions.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://doi.org/10.26180/20499582.v1","ca":"no-statement","cau":""},{"k":"4EEVRSNC","t":"A Computer Program for Assessing the Reliability and Systematic Bias of Individual Measurements1","au":"Cicchetti, Domenic V.","y":1976,"vn":"Educational and Psychological Measurement","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/001316447603600329","cx":"Analogue","co":["Reliability-noise","Bias"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"closed","cby":79,"self":false,"s":"Describes a computer program built to calculate both agreement between raters and any systematic bias one rater shows relative to others.","pg":"no-statement","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"on-request","cau":""},{"k":"4F4Y5QU5","t":"Should we fund research randomly? An epistemological criticism of the lottery model as an alternative to peer review for the funding of science","au":"Bedessem, Baptiste","y":2019,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvz034","cx":"Grant","co":["Bias","Validity"],"sg":["funding-decision"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":20,"self":false,"s":"A philosophical critique of proposals to fund science by lottery, arguing the model overlooks how research projects gain significance through their place in interconnected systems of scientific practice.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"4F79EKHL","t":"Evaluation, culture, and the world of the American professor.","au":"McLaughlin, Neil","y":2012,"vn":"Sociological Forum","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/j.1573-7861.2012.01331.x","cx":"Grant","co":["Reliability-noise","Bias"],"sg":["individual-scoring","panel-consensus"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":0,"self":false,"s":"A review of Michèle Lamont's book on how academic evaluation panels judge grant and fellowship proposals across disciplines, highlighting her account of disciplinary clashes, especially involving philosophy, in interdisciplinary funding panels.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"4FDFYDIR","t":"An output evaluation of a health research foundation’s enhanced grant review process for new investigators","au":"Hammond, Gregory","y":2017,"vn":"Health Research Policy and Systems","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s12961-017-0220-x","cx":"Grant","co":["Validity"],"sg":["post-award-validity"],"x":0,"st":"observational-quantitative","oast":"gold","cby":1,"self":false,"s":"A retrospective comparison of researchers funded versus not funded by a Canadian medical foundation's early-career grant programme, finding funded applicants went on to publish more, achieve higher citation impact, and secure substantially more follow-on national funding.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"no-statement","cau":""},{"k":"4H22B8FV","t":"The effectiveness of implementing statcheck in the peer review process to avoid statistical reporting errors","au":"Nuijten, Michele B.","y":2023,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31234/osf.io/bxau9","cx":"Journal","co":["Interventions"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":9,"self":false,"s":"Preregistered comparison of statistical-error rates in journals that adopted the statcheck screening tool during review versus matched journals that did not, finding a steeper decline in reporting inconsistencies at the adopting journals.","pg":"preregistered","pgr":"OSF","pgid":"https://osf.io/umwea","da":"open","dau":"https://osf.io/q84jn/","ca":"open","cau":"https://osf.io/q84jn/"},{"k":"4H7URGN2","t":"Partial lottery can make grant allocation more fair, more efficient, and more diverse","au":"Horbach, Serge P. J. M.","y":2022,"vn":"Science and Public Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/scipol/scac009","cx":"Grant","co":["Reliability-noise","Bias","Validity","Interventions"],"sg":["funding-decision"],"x":0,"st":"conceptual-commentary","oast":"bronze","cby":11,"self":false,"s":"Position paper proposing that funders test partial lottery-based allocation of grants as a way to improve fairness, efficiency, and diversity compared with pure peer-review ranking.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"4IAVPQUX","t":"Early Career Researchers on all Aspects of Peer Review: A Deep Dive Into the Data","au":"Nicholas, David","y":2025,"vn":"Learned Publishing","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/leap.2002","cx":"Journal","co":["general-systemic"],"sg":["whole-process"],"x":0,"st":"qualitative","oast":"hybrid","cby":5,"self":false,"s":"Interviews with 91 international early-career researchers examine their peer review experience and attitudes across disciplines, reporting more trust than distrust in the system and widespread expectation that AI will meaningfully transform reviewing.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"no-statement","cau":""},{"k":"4IR9JMGM","t":"How random is the review outcome? A systematic study of the impact of external factors on eLife peer review","au":"Liang, Weixin","y":2023,"vn":"bioRxiv (Cold Spring Harbor Laboratory)","ty":"journalArticle","tyl":"Journal article","doi":"10.1101/2023.01.04.522708","cx":"Journal","co":["Reliability-noise","Bias"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"green","cby":1,"self":false,"s":"Analysis of over 23,000 eLife submissions from 2016-2018 finds that random factors such as timing or editor-reviewer matching had little measurable influence on review outcomes at each decision stage.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"open","cau":"https://github.com/Weixin-Liang/eLife-peer-review-random-external-factors-study"},{"k":"4JQYGGWI","t":"Alternative models of funding curiosity-driven research","au":"Gigerenzer, Gerd","y":2025,"vn":"Proceedings of the National Academy of Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1073/pnas.2401237121","cx":"Grant","co":["Reliability-noise","Bias","Validity","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"hybrid","cby":5,"self":false,"s":"Surveys problems with current science-funding systems, including low agreement between reviewers, and discusses possible reforms ranging from modest panel changes to a global funding body.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"4KJWXB4W","t":"Towards a more informed and balanced use of scientific performance metrics","au":"Denissen, Jaap J A","y":2026,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvag023","cx":"General","co":["Bias","Validity"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"hybrid","cby":0,"self":false,"s":"Drawing on psychometric theory, this paper proposes five criteria for sound academic assessment and argues that abandoning quantitative performance metrics in favor of purely qualitative judgment is not the best path forward.","pg":"not-applicable","pgr":"","pgid":"","da":"open","dau":"","ca":"not-applicable","cau":""},{"k":"4RYFCKV2","t":"Does the perceived quality of interdisciplinary research vary between fields?","au":"Thelwall, Mike","y":2023,"vn":"Journal of Documentation","ty":"journalArticle","tyl":"Journal article","doi":"10.1108/jd-01-2023-0012","cx":"General","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"green","cby":8,"self":false,"s":"Analysing UK REF2021 data, the authors find that the same interdisciplinary journal articles receive notably different quality scores when assessed by different subject panels than when reassessed within one panel, indicating cross-field evaluation inconsistency.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"4SWWEB8Z","t":"Personal experience with AI-generated peer reviews: a case study","au":"Vecchio, Nicholas Lo","y":2025,"vn":"Research Integrity and Peer Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s41073-025-00161-3","cx":"Journal","co":["Interventions"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":9,"self":false,"s":"A first-person case report describing an author's suspicion that two anonymous peer reviews of their manuscript were AI-generated, the resulting dispute with the journal, and recommendations for handling suspected undisclosed AI use in review.","pg":"not-applicable","pgr":"","pgid":"","da":"open","dau":"https://nicospage.eu/publications/ai-peer-review","ca":"not-applicable","cau":""},{"k":"4T8H83NM","t":"The effect of masking manuscripts for the peer-review process of an ophthalmic journal","au":"Isenberg, S J","y":2009,"vn":"British Journal of Ophthalmology","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bjo.2008.151886","cx":"Journal","co":["Bias"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"closed","cby":33,"self":false,"s":"Retrospective study of manuscripts at a pediatric ophthalmology journal finding reviewers rated papers more favorably when they believed they knew the authors' identity, supporting masking as a way to reduce bias.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"4WZHZP49","t":"To fund or not to fund?","au":"Shailes, Sarah","y":2017,"vn":"eLife","ty":"journalArticle","tyl":"Journal article","doi":"10.7554/eLife.32015","cx":"Grant","co":["Bias"],"sg":["funding-decision"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":2,"self":false,"s":"Brief piece noting that funding agencies apply varied criteria and review procedures when deciding which grant proposals to support.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"4XB7F74M","t":"Are numerical scores important for grant assessment? A cross-sectional study","au":"Buljan, Ivan","y":2024,"vn":"F1000Research","ty":"journalArticle","tyl":"Journal article","doi":"10.12688/f1000research.139743.2","cx":"Grant","co":["Validity","Interventions"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":0,"self":false,"s":"A study of Marie Skłodowska-Curie Actions evaluation reports compares reviewer comment language before and after numerical scores were removed from individual assessments, finding no meaningful change in linguistic style despite the scoring change.","pg":"preregistered","pgr":"OSF","pgid":"https://osf.io/t84ba","da":"open","dau":"https://osf.io/6bpvu/?view_only=","ca":"open","cau":"https://osf.io/6bpvu/?view_only="},{"k":"4YLCI9SK","t":"Kappa Reliabilities for Continuous Behaviors and Events","au":"Conger, Anthony J.","y":1985,"vn":"Educational and Psychological Measurement","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/0013164485454016","cx":"Analogue","co":["Reliability-noise"],"sg":["whole-process"],"x":0,"st":"simulation-methodological","oast":"closed","cby":59,"self":false,"s":"Extends Cohen's kappa statistic from discrete category counts to continuous-time behavioral observation, addressing limitations of dividing observation periods into artificial discrete intervals.","pg":"not-codeable","pgr":"","pgid":"","da":"","dau":"","ca":"","cau":""},{"k":"4YUT5W5I","t":"The effect of number of rating scale categories on levels of interrater reliability : A monte carlo investigation","au":"Cicchetti, Domenic V.","y":1985,"vn":"Applied psychological measurement","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/014662168500900103","cx":"Analogue","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"green","cby":291,"self":false,"s":"A simulation study found that interrater reliability of rating scales improves as the number of scale points increases up to about seven, with little further gain beyond that.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"4ZG9JFXU","t":"Does incentive provision increase the quality of peer review? An experimental study","au":"Squazzoni, Flaminio","y":2012,"vn":"Research Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.respol.2012.04.014","cx":"Journal","co":["Interventions"],"sg":["individual-scoring"],"x":1,"st":"experimental-rct","oast":"closed","cby":103,"self":false,"s":"An experimental study found that paying reviewers material rewards for peer review reduced, rather than improved, the quality and efficiency of their reviews.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"no-statement","cau":""},{"k":"52D3XAIF","t":"Differences in Review Quality and Recommendations for Publication Between Peer Reviewers Suggested by Authors or by Editors","au":"Schroter, Sara","y":2006,"vn":"JAMA","ty":"journalArticle","tyl":"Journal article","doi":"10.1001/jama.295.3.314","cx":"Journal","co":["Bias"],"sg":["reviewer-selection","individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":147,"self":false,"s":"Comparing reviewers suggested by authors versus chosen by editors across ten biomedical journals, the study finds similar review quality between the two groups, but author-suggested reviewers were more likely to recommend accepting the manuscript.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"54CMD3XJ","t":"The many shapes of peer review: how to make journal review and preprint review activities more complementary?","au":"Puebla, Iratxe","y":2021,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31219/osf.io/t65a2","cx":"Journal","co":["Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":0,"self":false,"s":"Summarizes discussions from a 2021 workshop exploring how preprint review platforms could complement traditional journal peer review and help diversify who participates in evaluating research.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"54HK26IV","t":"Effect of written feedback by editors on quality of reviews: two randomized trials.","au":"Callaham, Michael L","y":2002,"vn":"The Journal of the American Medical Association","ty":"journalArticle","tyl":"Journal article","doi":"10.1001/jama.287.21.2781","cx":"Journal","co":["Interventions"],"sg":["individual-scoring"],"x":1,"st":"experimental-rct","oast":"closed","cby":75,"self":false,"s":"Two randomized trials testing whether giving reviewers written feedback on their review quality improved later reviews, finding no measurable improvement and even a slight negative trend for weaker reviewers.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"55TCQDVA","t":"What is expected of people who lead meetings where the goal is to reach consensus? A scoping review with implications for improving the quality of health research grant peer review and clinical guideline development","au":"Ghannad, Mona","y":2025,"vn":"medRxiv","ty":"journalArticle","tyl":"Journal article","doi":"10.1101/2025.02.03.25320968","cx":"General","co":["Interventions"],"sg":["panel-consensus"],"x":0,"st":"systematic-review-meta-analysis","oast":"green","cby":0,"self":false,"s":"Scoping review synthesizing literature on leadership roles in consensus-based meetings such as grant panels and guideline committees, identifying four recurring leadership functions: organizing, facilitating, adjudicating, and administering.","pg":"preregistered","pgr":"OSF","pgid":"https://doi.org/10.17605/OSF.IO/AXMJZ","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"57U6GUR2","t":"On various intraclass correlation reliability coefficients.","au":"Bartko, John J.","y":1976,"vn":"Psychological Bulletin","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/0033-2909.83.5.762","cx":"Analogue","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"closed","cby":816,"self":false,"s":"A methodological review of intraclass correlation reliability coefficients recommends the one-way ANOVA approach while cautioning against several commonly used but flawed alternatives, including one shown to yield perfect reliability even when raters share a constant bias.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"58QYQUYU","t":"“Excellence R Us”: university research and the fetishisation of excellence","au":"Moore, Samuel","y":2017,"vn":"Palgrave Communications","ty":"journalArticle","tyl":"Journal article","doi":"10.1057/palcomms.2016.105","cx":"General","co":["Reliability-noise","Bias","Validity"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":275,"self":false,"s":"Argues that \"excellence\" functions as an empty rhetorical device in academia rather than a meaningful standard, and that combined with competitive scarcity it contributes to reproducibility problems, fraud, and homogeneity in research.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"5BYI68MN","t":"Peer review of grant applications in biology and medicine. Reliability, fairness, and validity","au":"Reinhart, Martin","y":2009,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-008-2220-7","cx":"Grant","co":["Reliability-noise"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"qualitative","oast":"closed","cby":78,"self":false,"s":"An analysis of nearly 500 Swiss biology and medicine grant applications found moderate reviewer agreement, no evidence that demographic factors biased funding decisions, and later publication success aligned with the decisions made.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"5D8PZXA5","t":"Effective operation of competitive research funding systems","au":"OECD","y":2018,"vn":"OECD science, technology and industry policy papers","ty":"journalArticle","tyl":"Journal article","doi":"10.1787/2ae8c0dc-en","cx":"Grant","co":["Reliability-noise","Validity","Interventions"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"bronze","cby":23,"self":false,"s":"A policy report evaluating how competitive research funding systems operate in practice, identifying contextual factors that affect their effectiveness and offering recommendations for funders and policymakers.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"5HK3SW77","t":"Can generative AI effectively perform quality evaluation within social sciences? A case study in library and information science","au":"Zhu, Yu","y":2026,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-026-05570-9","cx":"General","co":["Reliability-noise","Bias","Validity"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":3,"self":false,"s":"Testing GPT-4o and DeepSeek-V3 on selecting the highest-quality papers from a pool of already high-quality library and information science articles, the authors find only weak correlation with citation/download counts and a bias favouring technical, jargon-heavy writing over theoretical work.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"no-statement","cau":""},{"k":"5IMAHJFT","t":"“The Grievance Studies Affair” Project: Reconstructing and Assessing the Experimental Design","au":"Lagerspetz, Mikko","y":2020,"vn":"Science Technology & Human Values","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/0162243920923087","cx":"Journal","co":["Reliability-noise","Bias","Validity"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":37,"self":false,"s":"A critical assessment of a well-publicized hoax-paper experiment targeting gender studies journals, arguing its design was methodologically and ethically flawed and did not demonstrate the low quality standards it claimed to expose.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"5KLQT2J8","t":"Editor and reviewer gender influence the peer review process but not peer review outcomes at an ecology journal","au":"Fox, Charles W.","y":2015,"vn":"Functional Ecology","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/1365-2435.12529","cx":"Journal","co":["Bias"],"sg":["reviewer-selection","individual-scoring"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":129,"self":false,"s":"Ten-year dataset from an ecology journal shows editor and reviewer gender shaped who was invited to review and response rates, but review scores and final accept/reject outcomes did not differ by gender.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"doi:10.5061/dryad.5090r","ca":"no-statement","cau":""},{"k":"5MPYUIC3","t":"An Investigation of Social-Behavioral Phenomena in the Peer-Review Processes of Scientific Foundations","au":"Kleiner, George","y":2019,"vn":"Modeling and Simulation of Social-Behavioral Phenomena in Creative Societies: First International EURO Mini Conference, MSBC 2019, Vilnius, Lithuania, September 18–20, 2019, Proceedings","ty":"bookSection","tyl":"Book chapter","doi":"10.1007/978-3-030-29862-3_6","cx":"Grant","co":["Bias"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"book-monograph","oast":"closed","cby":2,"self":false,"s":"Presents an agent-based simulation model of grant peer review exploring how reviewers' psychological traits, reputational concerns, and academic affiliations shape funding decisions.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"5NYG9LP8","t":"Inequity in peer review in communication sciences and disorders","au":"Girolamo, Teresa","y":2021,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31234/osf.io/ygprn","cx":"Journal","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":0,"self":false,"s":"A viewpoint article on how peer review practices in communication sciences and disorders may disadvantage Black, Indigenous, and other scholars of colour, proposing practical strategies for more equitable review and editorial structures.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"5P62SBYZ","t":"Effect of Blinded Peer Review on Abstract Acceptance","au":"Ross, Joseph S.","y":2006,"vn":"JAMA","ty":"journalArticle","tyl":"Journal article","doi":"10.1001/jama.295.14.1675","cx":"Journal","co":["Bias","Interventions"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":249,"self":false,"s":"Comparison of abstract acceptance at a cardiology meeting before and after blinding reviewers to author identity found blinding narrowed, but did not eliminate, an advantage for US and prestigious-institution submissions.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"5PTU7R5C","t":"Author review of reviewers.","au":"Hall, Judith A.","y":1979,"vn":"American Psychologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/0003-066x.34.9.798.a","cx":"Journal","co":["Interventions"],"sg":["reviewer-selection"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":96,"self":false,"s":"Proposes a system where journal authors rate the fairness and quality of the reviews they receive, giving editors feedback to identify and address poorly performing reviewers.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"5RWLPM53","t":"Science policies: How should science funding be allocated? An evolutionary biologists’ perspective","au":"Meirmans, Stephanie","y":2019,"vn":"Journal of Evolutionary Biology","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/jeb.13497","cx":"Grant","co":["general-systemic"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"hybrid","cby":31,"self":false,"s":"Nine evolutionary biologists compare grant funding practices across eleven national funding agencies, examining differences in application burden, funding rates, and review procedures, and suggest ways to improve fairness.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"5T5MS6AZ","t":"The Matthew effect and early-career setbacks in research funding—a replication study","au":"Traag, Vincent","y":2025,"vn":"eLife","ty":"journalArticle","tyl":"Journal article","doi":"10.7554/elife.109042.1","cx":"Grant","co":["Bias","Validity"],"sg":["post-award-validity"],"x":0,"st":"observational-quantitative","oast":"gold","cby":1,"self":false,"s":"A large multi-funder replication confirms that early-career researchers who win initial grants are more likely to secure later funding (the Matthew effect), while a proposed compensating \"setback\" advantage for initially unfunded high-impact applicants proves less consistent across funders.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"5U7HVU43","t":"Equitable Research Funding: Strategies, Challenges and the Role of Funding Agencies.","au":"Ruediger, Sarah","y":2025,"vn":"The European journal of neuroscience","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/ejn.70160","cx":"Grant","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"bronze","cby":7,"self":false,"s":"Drawing on discussions with several major funders, this paper examines how equity, diversity and inclusion principles are incorporated into research funding decisions and what systemic barriers, including biases in peer review, still limit progress.","pg":"not-applicable","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"5YACCVHF","t":"Jury Theorems for Peer Review","au":"Arvan, Marcus","y":2022,"vn":"The British Journal for the Philosophy of Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1086/719117","cx":"Journal","co":["Reliability-noise","Validity"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"hybrid","cby":3,"self":false,"s":"Argues, using jury-theorem reasoning, that open crowd-sourced review (as on arXiv) should outperform traditional journal review because more independent evaluators tend to produce better collective judgments.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"5YREXS39","t":"Conclave in the Tower of Babel: how peers review interdisciplinary research proposals","au":"Laudel, Grit","y":2006,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.3152/147154406781776048","cx":"Grant","co":["Bias","Interventions"],"sg":["reviewer-selection","panel-consensus"],"x":0,"st":"qualitative","oast":"closed","cby":69,"self":false,"s":"Case study of an institutional peer-review format designed for interdisciplinary grant proposals, describing procedures such as applicant empowerment and mandated cross-disciplinary learning among panel members.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"5ZF6XD7M","t":"Can ChatGPT evaluate research quality?","au":"Thelwall, Mike","y":2024,"vn":"Journal of Data and Information Science","ty":"journalArticle","tyl":"Journal article","doi":"10.2478/jdis-2024-0013","cx":"General","co":["Reliability-noise","Bias","Validity"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"diamond","cby":58,"self":false,"s":"A single-author case study tests whether ChatGPT-4 can grade the quality of 51 of the author's own articles against UK Research Excellence Framework criteria, finding weak correlation with self-assessment and limited reliability for fine distinctions.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"5ZRC7NDD","t":"Impact of Alumni Connections on Peer Review Ratings and Selection Success Rate in National Research","au":"Jang, Duckhee","y":2016,"vn":"Science Technology & Human Values","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/0162243916665466","cx":"Grant","co":["Bias"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"closed","cby":20,"self":false,"s":"Analysis of over 8,000 Korean national R&D proposals found evaluators gave higher scores and more favorable outcomes to applicants sharing their alma mater.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"5ZWLBYYK","t":"American Idol and NIH Grant Review","au":"Pagano, Michele","y":2006,"vn":"Cell","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.cell.2006.08.004","cx":"Grant","co":["Interventions"],"sg":["individual-scoring","panel-consensus"],"x":0,"st":"conceptual-commentary","oast":"bronze","cby":17,"self":false,"s":"A personal account from an NIH study-section member describes how panels triage most grant applications quickly and compares reviewer dynamics to judges on a television talent competition.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"625FMAPA","t":"Improving Participation and Interrater Agreement in Scoring Ambulatory Pediatric Association Abstracts","au":"Kemper, Kathi J.","y":1996,"vn":"Archives of Pediatrics and Adolescent Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.1001/archpedi.1996.02170290046007","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":21,"self":false,"s":"Comparing 1990 and 1995 conference abstract reviews, this study found that increasing the number and expertise of reviewers did not improve interrater agreement, which stayed low in both years.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"64U3MRH9","t":"Uncertainty in peer review","au":"Barnett, Adrian","y":2024,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31219/osf.io/fm6st","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":1,"self":false,"s":"An observational study examining how much uncertainty characterizes outcomes in the peer-review process.","pg":"preregistered","pgr":"OSF","pgid":"https://osf.io/ucmxw/","da":"open","dau":"https://github.com/agbarnett/uncertainty","ca":"open","cau":"https://github.com/agbarnett/uncertainty"},{"k":"6754BVQM","t":"Quantitative Impact of Including Consumers in the Scientific Review of Breast Cancer Research Proposals","au":"Andejeski, Yvonne","y":2002,"vn":"Journal of Women s Health & Gender-Based Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.1089/152460902317586010","cx":"Grant","co":["Reliability-noise","Interventions"],"sg":["panel-consensus","individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":53,"self":false,"s":"A study of a US Department of Defense breast cancer research programme finding that including patient-advocate consumers as voting panel members changed relatively few proposal scores and was viewed positively by both consumers and scientists.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"67M7Y9GH","t":"Fund ideas, not pedigree, to find fresh insight","au":"Sinkjær, Thomas","y":2018,"vn":"Nature","ty":"journalArticle","tyl":"Journal article","doi":"10.1038/d41586-018-02743-2","cx":"Grant","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"bronze","cby":14,"self":false,"s":"Commentary describing a Danish foundation's blinded, background-free grant scheme designed to fund innovative ideas rather than applicant pedigree, situating it alongside a similar German initiative.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"68C8KWK7","t":"La revisión por pares en las áreas de filosofía y filosofía moral en España. Creencias y debates sobre su consolidación, su idoneidad y su tipo ideal","au":"Feenstra, Ramón A.","y":2024,"vn":"Revista Española de Sociología","ty":"journalArticle","tyl":"Journal article","doi":"10.22325/fes/res.2024.209","cx":"Journal","co":["Reliability-noise","Bias","Validity"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":1,"self":false,"s":"Spanish-language survey and interview study of philosophy researchers in Spain finding broad acceptance of peer review's legitimacy and a strong preference for double-blind review, alongside lingering doubts about biases.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"6BPUB4IR","t":"Challenges, experiments, and computational solutions in peer review","au":"Shah, Nihar B.","y":2022,"vn":"Communications of the ACM","ty":"journalArticle","tyl":"Journal article","doi":"10.1145/3528086","cx":"Journal","co":["Reliability-noise","Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"bronze","cby":62,"self":false,"s":"A short piece arguing that applying scientific and experimental methods to studying and reforming peer review is a promising direction for addressing its known problems.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"6E7729LN","t":"Peer review of grant applications: a simple method to identify proposals with discordant reviews.","au":"Giraudeau, Bruno","y":2011,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0027557","cx":"Grant","co":["Reliability-noise"],"sg":["individual-scoring","panel-consensus"],"x":0,"st":"simulation-methodological","oast":"gold","cby":8,"self":false,"s":"Proposes using the intraclass correlation coefficient as a practical criterion for flagging grant applications with discordant reviewer scores that warrant committee discussion.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"open","cau":""},{"k":"6EA2KUS7","t":"Innovating peer review, reconfiguring scholarly communication: an analytical overview of ongoing peer review innovation activities","au":"Kaltenbrunner, Wolfgang","y":2022,"vn":"Journal of Documentation","ty":"journalArticle","tyl":"Journal article","doi":"10.1108/JD-01-2022-0022","cx":"Journal","co":["Interventions"],"sg":["whole-process"],"x":0,"st":"qualitative","oast":"hybrid","cby":31,"self":false,"s":"A survey of 95 self-reported peer-review innovations finds that many initiatives pull in conflicting directions, such as efficiency versus rigor, or transparency versus anonymity-based objectivity.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"doi: 10.6084/m9.figshare.17161835.v1","ca":"no-statement","cau":""},{"k":"6EE23NG2","t":"Forming inferences about some intraclass correlation coefficients.","au":"McGraw, Kenneth O.","y":1996,"vn":"Psychological Methods","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/1082-989X.1.1.30","cx":"Analogue","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"closed","cby":6705,"self":false,"s":"A methodological article explaining statistical procedures for constructing confidence intervals and hypothesis tests for intraclass correlation coefficients derived from different analysis-of-variance rater-agreement models.","pg":"no-statement","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"6ELG25E6","t":"Decision-making approaches used by UK and international health funding organisations for allocating research funds: A survey of current practice.","au":"Meadmore, Katie","y":2020,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0239757","cx":"Grant","co":["Reliability-noise","Bias"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"gold","cby":12,"self":false,"s":"A survey of UK and international health funders finds that external peer review, triage and committee meetings dominate funding decisions, with transparency valued highly but bias, reviewer burden and resource limits cited as recurring problems.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"","ca":"no-statement","cau":""},{"k":"6EQT27QI","t":"Do Peer Reviews Predict Impact? Evidence from the American Sociological Review , 1978 to 1982","au":"Teplitskiy, Misha","y":2016,"vn":"Socius : sociological research for a dynamic world","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/2378023116640278","cx":"Journal","co":["Validity"],"sg":["post-award-validity"],"x":0,"st":"observational-quantitative","oast":"gold","cby":4,"self":false,"s":"An analysis of peer review outcomes for articles in a major sociology journal in the late 1970s/early 1980s finding no relationship between reviewers' assessments and how often the articles were later cited, even decades on.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"6H732CUX","t":"Interreferee agreement and acceptance rates in physics","au":"Lazarus, David","y":1982,"vn":"Behavioral and Brain Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/s0140525x00011468","cx":"Journal","co":["Reliability-noise","Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":50,"self":false,"s":"Brief item examining agreement between referees and acceptance rates for physics manuscripts (abstract not available).","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"6HCFA5SL","t":"Effect on peer review of telling reviewers that their signed reviews might be posted on the web: randomised controlled trial.","au":"van Rooyen, Susan","y":2010,"vn":"BMJ (Clinical Research Ed.)","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmj.c5729","cx":"Journal","co":["Interventions"],"sg":["individual-scoring"],"x":1,"st":"experimental-rct","oast":"hybrid","cby":169,"self":false,"s":"Randomized trial at a medical journal testing whether telling reviewers their signed reports might be posted online affected review quality, finding no significant difference between the informed and control groups.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"on-request","cau":""},{"k":"6I4774BQ","t":"A scoping review on the roles and tasks of peer reviewers in the manuscript review process in biomedical journals.","au":"Glonti, Ketevan","y":2019,"vn":"BMC Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s12916-019-1347-0","cx":"Journal","co":["general-systemic"],"sg":["individual-scoring"],"x":0,"st":"systematic-review-meta-analysis","oast":"gold","cby":84,"self":false,"s":"A scoping review of the literature up to 2017 on how peer reviewer roles and tasks are described in biomedical journal publishing, synthesising over a thousand statements into thematic categories such as expertise, ethics, and gatekeeping.","pg":"preregistered","pgr":"other","pgid":"BMJ Open. 2017;7(10):e017468","da":"open","dau":"","ca":"not-applicable","cau":""},{"k":"6ISHLVGN","t":"Examining uncertainty in journal peer reviewers’ recommendations: a cross-sectional study","au":"Barnett, Adrian","y":2024,"vn":"Royal Society Open Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1098/rsos.240612","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":6,"self":false,"s":"Survey of reviewers at three journals found about a quarter reported no uncertainty in their recommendation, while most expressed some hesitation between adjacent categories such as accept and minor revision.","pg":"preregistered","pgr":"OSF","pgid":"https://osf.io/ucmxw/","da":"open","dau":"doi:10.5281/zenodo.10892583","ca":"open","cau":"https://github.com/agbarnett/uncertainty"},{"k":"6JA7SUSX","t":"An evolutionary explanation of assassins and zealots in peer review","au":"Chamorro-Padial, Jorge","y":2019,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-019-03171-3","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"green","cby":7,"self":false,"s":"Uses an evolutionary modeling approach to explain why some reviewers become consistently harsh or consistently lenient over time, framing reviewer types as strategies that persist through reviewer training.","pg":"no-statement","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"no-statement","cau":""},{"k":"6PGVZK3J","t":"Improving Peer Review in Economics: Stocktaking and Proposals","au":"Charness, Gary","y":2025,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31222/osf.io/k9ybz_v1","cx":"Journal","co":["Interventions"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"gold","cby":3,"self":false,"s":"A survey of more than 1,400 economists documents current peer-review practices in the field, common frustrations, and proposed reforms to improve the process.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://osf.io/eczkv/","ca":"open","cau":"https://osf.io/eczkv/"},{"k":"6VBQCX25","t":"Mitigating animal methods bias to reduce animal use and improve biomedical translation","au":"Kavanagh, Owen","y":2024,"vn":"Science Progress","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/00368504241253693","cx":"Journal","co":["Bias","Interventions"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"diamond","cby":2,"self":false,"s":"Describes an international coalition's effort to document and counter \"animal methods bias,\" the tendency of reviewers to favour animal-based research designs over adequate non-animal alternatives, including a new author guide for addressing this bias in publishing.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"6W6MH3MT","t":"NIH peer review percentile scores are poorly predictive of grant productivity.","au":"Fang, Ferric C","y":2016,"vn":"eLife","ty":"journalArticle","tyl":"Journal article","doi":"10.7554/eLife.13323","cx":"Grant","co":["Validity"],"sg":["post-award-validity"],"x":0,"st":"observational-quantitative","oast":"gold","cby":113,"self":false,"s":"A re-analysis of over 100,000 NIH-funded grants concluding that percentile scores from peer review panels only weakly predict later research productivity, despite an earlier study suggesting a stronger link.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"no-statement","cau":""},{"k":"6W76X9PV","t":"Defenses against LLM prompt injections in academic peer review","au":"Zika, Ondrej","y":2025,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31234/osf.io/zxuwf_v2","cx":"Journal","co":["Bias","Interventions"],"sg":["individual-scoring"],"x":1,"st":"experimental-rct","oast":"gold","cby":0,"self":false,"s":"Experiment testing whether hidden instructions embedded in manuscripts can manipulate LLM-generated peer reviews, finding such prompt injections raised acceptance recommendations, though explicitly instructing models to check for them mostly reversed the effect.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"6XTFN68E","t":"Evaluation of research proposals by peer review panels: broader panels for broader assessments?","au":"Abma-Schouten, Rebecca","y":2023,"vn":"Science and Public Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/scipol/scad009","cx":"Grant","co":["general-systemic"],"sg":["panel-consensus"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":13,"self":false,"s":"Observational study of ten grant review panel meetings at two Dutch health funders finding that more diverse panels raised more arguments about societal relevance, while less diverse panels focused mainly on scientific merit.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"no-statement","cau":""},{"k":"6YBF2W5W","t":"Registered report protocol: Factors associated with inter-rater agreement in grant peer review.","au":"Hesselberg, Jan-Ole","y":2025,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0322696","cx":"Grant","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":0,"self":true,"s":"A registered protocol for a study analysing about 135,000 grant reviews from four Norwegian funders to identify which reviewer, applicant, and application characteristics predict agreement between reviewers, with open data and code planned.","pg":"registered-report","pgr":"other","pgid":"","da":"open","dau":"kaggle.com/datasets/janolehesselberg","ca":"open","cau":""},{"k":"6Z32RSBF","t":"The state of the art in peer review","au":"Tennant, Jonathan","y":2018,"vn":"FEMS Microbiology Letters","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/femsle/fny204","cx":"Journal","co":["general-systemic"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"hybrid","cby":165,"self":false,"s":"Overview article surveys recent innovation in peer review platforms and practices, reflecting on why adoption of newer models has lagged behind traditional methods, and speculates on future directions.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"72ZM4PVC","t":"Designing grant-review panels for better funding decisions: Lessons from an empirically calibrated simulation model","au":"Feliciani, Thomas","y":2022,"vn":"Research Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.respol.2021.104467","cx":"Grant","co":["Reliability-noise","Validity","Interventions"],"sg":["individual-scoring","panel-consensus","funding-decision"],"x":0,"st":"simulation-methodological","oast":"hybrid","cby":14,"self":false,"s":"A simulation calibrated to a funding agency's procedures finds that panels using finer grading scales and stronger collective discrimination make choices closer to an expert benchmark, while differing grading standards among reviewers matter less than expected.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"https://github.com/thomasfeliciani/wisdom-of-expert-crowds/","ca":"open","cau":"https://github.com/thomasfeliciani/wisdom-of-expert-crowds/"},{"k":"73TNGMX8","t":"Hidden connections: Network effects on editorial decisions in four computer science journals","au":"Bravo, Giangiacomo","y":2018,"vn":"Journal of Informetrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.joi.2017.12.002","cx":"Journal","co":["Bias"],"sg":["funding-decision"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":54,"self":false,"s":"Analyzing eight years of editorial decisions across four computer science journals, the authors found that authors with higher network-based reputation were less likely to be rejected even after receiving negative reviews.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"74M956VR","t":"The effect of the “very important paper” (VIP) designation in Angewandte Chemie International Edition on citation impact: A propensity score matching analysis","au":"Mutz, Rüdiger","y":2016,"vn":"Journal of the Association for Information Science and Technology","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/asi.23701","cx":"Journal","co":["Reliability-noise","Validity"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":18,"self":false,"s":"Using matched comparisons, the study finds that chemistry papers designated \"very important\" by a leading journal accumulate substantially more citations, and reach their citation peak faster, than similar non-designated papers.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"7596286J","t":"Malice's wonderland: Research funding and peer review","au":"Osmond, D. H.","y":1983,"vn":"Journal of Neurobiology","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/neu.480140202","cx":"Grant","co":["Bias","Validity"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":46,"self":false,"s":"An essay using the Alice in Wonderland motif of arbitrary, capricious judgment as a metaphor for the unpredictability researchers experience in grant funding and peer review.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"76KIJ2D8","t":"Efficacy of Double-Blind Peer Review in an Imaging Subspecialty Journal","au":"O’Connor, Erin E.","y":2016,"vn":"American Journal of Neuroradiology","ty":"journalArticle","tyl":"Journal article","doi":"10.3174/ajnr.a5017","cx":"Journal","co":["Bias"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"bronze","cby":37,"self":false,"s":"Survey-based study at a neuroradiology journal finding reviewers who felt they recognized the authors or institution behind a submission were more likely to recommend acceptance, despite double-blind procedures.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"77VRC7BI","t":"Can \"results blind manuscript evaluation\" assuage \"publication bias\"?","au":"Hyman, Michael R","y":2017,"vn":"Basic and Applied Social Psychology","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/01973533.2017.1350581","cx":"Journal","co":["Bias","Interventions"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":15,"self":false,"s":"Short commentary critiquing a proposal for \"results-blind\" manuscript review as a remedy for publication bias.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"78GNLVBD","t":"Peer review in a changing world: An international study measuring the attitudes of researchers","au":"Mulligan, Adrian","y":2013,"vn":"Journal of the American Society for Information Science and Technology","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/asi.22798","cx":"Journal","co":["Bias"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"closed","cby":400,"self":false,"s":"A 2009 international survey of over 4,000 researchers finds broad confidence in peer review's value, a preference for double-blind review, and support for reviewer training, alongside scepticism about review's ability to catch fraud.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"78VLE8B6","t":"Peer review assessment of originality in tourism journals: critical perspective of key gatekeepers","au":"Rodríguez, Isabel","y":2019,"vn":"Annals of Tourism Research","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.annals.2019.04.003","cx":"Journal","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"qualitative","oast":"green","cby":29,"self":false,"s":"Interviews with 26 tourism journal editors reveal disagreement over how to define and weigh originality against other review criteria, and identify challenges in recognizing it consistently.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"7EKMP546","t":"Heterogeneity of inter-rater reliabilities of grant peer reviews and its determinants: a general estimating equations approach.","au":"Mutz, Rüdiger","y":2012,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0048509","cx":"Grant","co":["Reliability-noise","Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":81,"self":false,"s":"Analysis of over 8,000 Austrian Science Fund proposals found reviewer agreement varied by research field, with humanities showing higher consistency than most other disciplines and biosciences showing comparatively low agreement.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"7GYSZ988","t":"Does the disconnect between the peer-reviewed label and reality explain the peer review crisis, and can open peer review or preprints resolve it? A narrative review","au":"Silva, Jaime A. Teixeira da","y":2025,"vn":"Naunyn-Schmiedeberg s Archives of Pharmacology","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s00210-025-04486-0","cx":"Journal","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":4,"self":false,"s":"A narrative review argues that traditional peer review's flaws, and similar problems in newer alternatives like open review and preprints, complicate simple labeling of journals as reliably \"peer reviewed,\" proposing a new classification scheme.","pg":"not-applicable","pgr":"","pgid":"","da":"on-request","dau":"","ca":"not-applicable","cau":""},{"k":"7HL64R5H","t":"Examining Personality Psychology to Unpack the Peer Review System: Towards a More Diverse, Inclusive, and Equitable Psychological Science","au":"Atherton, Olivia E.","y":2024,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31234/osf.io/mxcj3","cx":"Journal","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"qualitative","oast":"gold","cby":1,"self":false,"s":"Drawing on personal accounts, a survey of over 100 personality psychologists, and prior literature, the authors trace historical roots of bias and exclusion in psychology's peer-review system and propose ways to make it more equitable.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://osf.io/tcwxd/","ca":"open","cau":"https://osf.io/tcwxd/"},{"k":"7JG4HLD9","t":"Substantial Agreement of Referee Recommendations at a General Medical Journal – A Peer Review Evaluation at Deutsches Ärzteblatt International","au":"Baethge, Christopher","y":2013,"vn":"PLoS ONE","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0061401","cx":"Journal","co":["Reliability-noise","Validity"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":34,"self":false,"s":"A study of reviewer recommendations at a German medical journal reporting that standard kappa statistics showed poor agreement, but overall raw agreement was high and an alternative statistic (Gwet's AC1) indicated substantial agreement, with no link found between reviewer recommendations and later citations.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"no-statement","cau":""},{"k":"7KXBVRH5","t":"Transparency of peer review: a semi-structured interview study with editors in chief from Social Sciences and Humanities","au":"Karhulahti, Veli-Matti","y":2021,"vn":"Research Integrity and Peer Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s41073-021-00116-4","cx":"Journal","co":["general-systemic"],"sg":["whole-process"],"x":0,"st":"qualitative","oast":"gold","cby":23,"self":false,"s":"Interview study with social-science and humanities journal editors-in-chief exploring their views and practices on making peer review more transparent.","pg":"preregistered","pgr":"OSF","pgid":"https://osf.io/wjztp","da":"restricted","dau":"","ca":"no-statement","cau":""},{"k":"7KZ9486Z","t":"Author-suggested reviewers rate manuscripts much more favorably: A cross-sectional analysis of the neuroscience section of PLOS ONE","au":"Acuña, Daniel E.","y":2022,"vn":"PLoS ONE","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0273994","cx":"Journal","co":["Bias"],"sg":["reviewer-selection","individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":2,"self":false,"s":"Analysis of nearly 8,000 PLOS ONE neuroscience manuscripts finding that reviewers suggested by authors gave substantially more favorable evaluations than editor-selected reviewers, raising the paper's acceptance chances.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"no-statement","cau":"https://github.com/daniel-acuna/scopus_helper"},{"k":"7LDKWQXY","t":"Gender differences in peer review of innovation","au":"Belz, Andrea","y":2022,"vn":"Strategic Entrepreneurship Journal","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/sej.1429","cx":"Grant","co":["Bias"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"closed","cby":24,"self":false,"s":"A study of NASA's Small Business Innovation Research programme finds evidence of gender bias disadvantaging newer female applicants in initial scoring, with a subsequent corrective mechanism shown to offset the disparity in final funding outcomes.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"7QJBZY4V","t":"Beyond Gatekeeping: Reassessing Peer Review and Establishing Preprints as the Future of Scientific Communication","au":"Baker, Maher Asaad","y":2025,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31235/osf.io/69uhe_v1","cx":"Journal","co":["Reliability-noise","Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":0,"self":false,"s":"Narrative literature review argues traditional peer review is slow and inconsistent, and makes the case for preprints and open hybrid review models becoming the primary channel for sharing research.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"7S4GVWVN","t":"Validating Grant-Making Processes: Construct Validity of the 2013 Senior Corps RSVP Grant Review","au":"Tan, Erwin","y":2016,"vn":"VOLUNTAS: International Journal of Voluntary and Nonprofit Organizations","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11266-015-9602-2","cx":"Grant","co":["Reliability-noise","Validity"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":3,"self":false,"s":"Develops a four-step framework for checking that a grant-selection process is valid and fair, applied to a 2013 US community-service grant program, finding high reviewer agreement and reliability.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"7TV4V8FE","t":"Making Reviewers Visible","au":"Godlee, Fiona","y":2002,"vn":"JAMA","ty":"journalArticle","tyl":"Journal article","doi":"10.1001/jama.287.21.2762","cx":"Journal","co":["Interventions"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":168,"self":false,"s":"An opinion piece arguing for open (non-anonymous) peer review in biomedical journals on ethical, practical, and accountability grounds, contending it should eventually replace anonymous prepublication review.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"7U835JX8","t":"What does better peer review look like? Underlying principles and recommendations for better practice","au":"Allen, Heidi","y":2019,"vn":"Learned Publishing","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/leap.1222","cx":"Journal","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"guideline-policy","oast":"hybrid","cby":59,"self":false,"s":"A literature review distils five principles, covering content integrity, ethics, fairness, usefulness and timeliness, intended to guide journals toward better peer-review practice, accompanied by a practical checklist.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"7UYA6YG5","t":"Transparency in peer review: Exploring the content and tone of reviewers’ confidential comments to editors","au":"OʼBrien, Bridget C.","y":2021,"vn":"PLoS ONE","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0260558","cx":"Journal","co":["general-systemic"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":16,"self":false,"s":"Reports on a workshop's exploration of how confidential comments reviewers send only to editors compare in content and tone to what they tell authors directly, based on manuscripts submitted to a health-education journal.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://doi.org/10.5281/zenodo.5128723","ca":"no-statement","cau":""},{"k":"7VRBLWI2","t":"Do peers share the same criteria for assessing grant applications?","au":"Hug, Sven E.","y":2021,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvab034","cx":"Grant","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"green","cby":17,"self":false,"s":"Survey of Swiss humanities scholars using latent class analysis to test whether reviewers share common grant-evaluation criteria, identifying several distinct criteria clusters rather than one shared consensus.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"open","cau":"https://github.com/MattisvdBergh/LCT"},{"k":"7VUDTF33","t":"Changing the Culture of Peer Review for a More Inclusive and Equitable Psychological Science","au":"Aly, Mariam","y":2022,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31234/osf.io/435xz","cx":"General","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"gold","cby":7,"self":false,"s":"Reviews problems in psychology and neuroscience peer review, including negativity and lack of reviewer training, and calls for changes that make feedback more constructive and sensitive to diverse researchers and topics.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://osf.io/jqy4k","ca":"no-statement","cau":""},{"k":"7W8CFBU4","t":"NIH peer review: Criterion scores completely account for racial disparities in overall impact scores.","au":"Erosheva, Elena A","y":2020,"vn":"Science Advances","ty":"journalArticle","tyl":"Journal article","doi":"10.1126/sciadv.aaz4868","cx":"Grant","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":128,"self":false,"s":"Analysis of NIH R01 grant reviews from 2014-2016 finds that racial disparities in overall impact scores are largely explained by disparities already present in the underlying criterion scores, rather than by how those scores are combined.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://osf.io/4d6rx/","ca":"open","cau":"https://osf.io/4d6rx/"},{"k":"7WB2GCPB","t":"“Let the community decide”? The vision and reality of soundness-only peer review in open-access mega-journals","au":"Spezi, Valérie","y":2017,"vn":"Journal of Documentation","ty":"journalArticle","tyl":"Journal article","doi":"10.1108/jd-06-2017-0092","cx":"Journal","co":["general-systemic"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"qualitative","oast":"hybrid","cby":61,"self":false,"s":"Interviews with 31 publishers and editors find that open-access mega-journals marketed as reviewing only technical soundness often still apply broader traditional criteria in practice, complicating their stated model.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"7WU5LEHX","t":"Who chooses open peer review and is it an indicator of article quality? An observational study of PLOS journals.","au":"Barnett, Adrian","y":2026,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31222/osf.io/2b7z5_v1","cx":"Journal","co":["Validity"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"gold","cby":0,"self":false,"s":"An analysis of over 115,000 PLOS articles found that authors who opted into open peer review varied by country and email-domain type, and that their articles were cited more and retracted less often than average.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://github.com/agbarnett/open_peer_review","ca":"open","cau":"https://github.com/agbarnett/open_peer_review"},{"k":"7Z69DLU7","t":"Emerging trends in peer review—a survey","au":"Walker, Richard","y":2015,"vn":"Frontiers in Neuroscience","ty":"journalArticle","tyl":"Journal article","doi":"10.3389/fnins.2015.00169","cx":"Journal","co":["Interventions"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"gold","cby":188,"self":false,"s":"A survey of 82 scientific communication channels identifies major trends in peer review reform, including the growth of preprint servers and non-selective review, alongside persistent disciplinary differences in practice.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"85LYQU4R","t":"Peering at peer review revealed high degree of chance associated with funding of grant applications.","au":"Mayo, Nancy E","y":2006,"vn":"Journal of Clinical Epidemiology","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.jclinepi.2005.12.007","cx":"Grant","co":["Reliability-noise"],"sg":["individual-scoring","panel-consensus","funding-decision"],"x":0,"st":"observational-quantitative","oast":"bronze","cby":70,"self":false,"s":"Comparing two grant-ranking methods on the same pilot-project applications, the study finds only modest agreement between them, with top-ranked projects sometimes failing to be funded depending on which reviewer pair was assigned, underscoring a large role for chance.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"85QBBTET","t":"Low agreement among reviewers evaluating the same NIH grant applications.","au":"Pier, Elizabeth L","y":2018,"vn":"Proceedings of the National Academy of Sciences of the United States of America","ty":"journalArticle","tyl":"Journal article","doi":"10.1073/pnas.1714379115","cx":"Grant","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":209,"self":false,"s":"An experiment replicating the NIH grant review process with independent reviewers rating the same 25 applications, finding essentially no agreement between reviewers on quality, either in scores or in how they translated critiques into numbers.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"open","cau":""},{"k":"86MJ77BB","t":"Unprofessional peer reviews disproportionately harm underrepresented groups in STEM","au":"Silbiger, Nyssa J.","y":2019,"vn":"PeerJ","ty":"journalArticle","tyl":"Journal article","doi":"10.7717/peerj.8247","cx":"Journal","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":190,"self":false,"s":"International STEM survey finding unprofessional peer-review comments were received at similar rates across demographic groups, but underrepresented respondents were more likely to report lasting harm to confidence and career progress.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"http://dx.doi.org/10.7717/peerj.8247#supplemental-information","ca":"open","cau":"https://github.com/njsilbiger/UnproReviewsInSTEM"},{"k":"86W5FZUM","t":"Strategies to prevent or reduce gender bias in peer review of research grants: A rapid scoping review.","au":"Tricco, Andrea C","y":2017,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0169718","cx":"Grant","co":["Bias","Interventions"],"sg":["individual-scoring"],"x":0,"st":"systematic-review-meta-analysis","oast":"gold","cby":68,"self":false,"s":"Scoping review of interventions to reduce gender bias in grant peer review, finding only one relevant study (on blinding applicant gender), which showed no change in funding success for women.","pg":"preregistered","pgr":"OSF","pgid":"https://osf.io/qdzpt/","da":"open","dau":"","ca":"not-applicable","cau":""},{"k":"87BCIXU7","t":"When AI turns grant evaluation into a lottery","au":"Bulla, Martin","y":2026,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31222/osf.io/d8gcu_v1","cx":"Grant","co":["Reliability-noise","Validity"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":1,"self":false,"s":"An analysis of recent Marie Skłodowska-Curie postdoctoral fellowship evaluations argues that widespread AI-assisted proposal writing has sharply compressed quality scores, undermining fine-grained ranking and pushing funders toward alternative allocation methods.","pg":"not-applicable","pgr":"","pgid":"","da":"open","dau":"https://github.com/MartinBulla/correspondance_funding","ca":"no-statement","cau":"https://github.com/MartinBulla/correspondance_funding"},{"k":"88FVADPB","t":"The influence of first author sex on acceptance rates of submissions to Anaesthesia Cases.","au":"Laycock, H","y":2019,"vn":"Anaesthesia","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/anae.14797","cx":"Journal","co":["Bias"],"sg":["funding-decision"],"x":0,"st":"observational-quantitative","oast":"bronze","cby":12,"self":false,"s":"Study of case report submissions to one anaesthesia journal since 2013 found a slightly higher acceptance rate for female than male first authors, contrasting with gender gaps reported elsewhere in publishing.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"89999KMX","t":"Authors and editors assort on gender and geography in high-rank ecological publications","au":"Manlove, Kezia R.","y":2018,"vn":"PLoS ONE","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0192481","cx":"Journal","co":["Bias"],"sg":["funding-decision"],"x":0,"st":"observational-quantitative","oast":"gold","cby":16,"self":false,"s":"Examining ecology journals, the study found women were underrepresented among lead authors and editors relative to men, editors more often handled papers by same-gender authors, and both groups skewed toward researchers from wealthy countries and top institutions.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"","ca":"open","cau":""},{"k":"8AAUI9F9","t":"Interrater reliability and convergent validity of F1000Prime peer review","au":"Bornmann, Lutz","y":2015,"vn":"Journal of the Association for Information Science and Technology","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/asi.23334","cx":"General","co":["Reliability-noise","Validity"],"sg":["individual-scoring","post-award-validity"],"x":0,"st":"observational-quantitative","oast":"closed","cby":36,"self":false,"s":"Analyzing about 100,000 papers, this study found low interrater agreement among F1000Prime post-publication reviewers, but showed that papers rated more highly were more often highly cited, supporting the system's validity.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"8AIUI3PN","t":"High-scoring grant applications yield more highly cited papers","au":"Baker, Monya","y":2015,"vn":"Nature","ty":"journalArticle","tyl":"Journal article","doi":"10.1038/nature.2015.17380","cx":"Grant","co":["Validity"],"sg":["individual-scoring","post-award-validity"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":1,"self":false,"s":"A large-scale analysis of NIH grants finds that higher peer-review scores are associated with more subsequent patents and highly cited publications, challenging earlier claims that review scores poorly predict research impact.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"8DH86VMV","t":"Critical reflections on the journal peer review process","au":"Tight, Malcolm","y":2024,"vn":"Higher Education Research & Development","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/07294360.2024.2366035","cx":"Journal","co":["Reliability-noise"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"green","cby":8,"self":false,"s":"A commentary on inconsistencies and frustrations in academic journal peer review, arguing for clearer guidance to reviewers and authors and greater institutional recognition of reviewing work.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"8DHMK6TJ","t":"Peer review revisited.","au":"Meyen, Michael","y":2016,"vn":"Studies in Communication and Media","ty":"journalArticle","tyl":"Journal article","doi":"10.5771/2192-4007-2016-1-8","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"qualitative","oast":"diamond","cby":0,"self":false,"s":"Content analysis of reviews from a German communication-studies journal describing how reviewers act as both critical evaluators and supportive advocates, reinforcing the field's existing disciplinary norms.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"8FCS5PFQ","t":"Bibliometrically Disciplined Peer Review: on Using Indicators in Research Evaluation","au":"Besselaar, Peter van den","y":2020,"vn":"Scholarly Assessment Reports","ty":"journalArticle","tyl":"Journal article","doi":"10.29024/sar.16","cx":"General","co":["Interventions"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"diamond","cby":14,"self":false,"s":"Case study of a university evaluation combining peer review with bibliometric indicators, proposing this \"disciplined peer review\" approach as a way to reduce subjectivity in panel-based assessments.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"8FW39FHY","t":"Is There Gender Bias in JAMA's Peer Review Process?","au":"Gilbert, Julie R.","y":1994,"vn":"JAMA","ty":"journalArticle","tyl":"Journal article","doi":"10.1001/jama.1994.03520020065018","cx":"Journal","co":["Bias"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"closed","cby":130,"self":false,"s":"A 1994 retrospective study of JAMA's 1991 manuscripts examines gender patterns among editors, reviewers, and authors, finding some gender-linked differences in process (e.g., reviewer assignment, turnaround) but no effect on final acceptance rates.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"8H8IKI6S","t":"And Who Will Review the Review(er)s?","au":"Potvin, Sarah","y":2017,"vn":"College & Research Libraries","ty":"journalArticle","tyl":"Journal article","doi":"10.5860/crl.78.6.734","cx":"General","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"diamond","cby":3,"self":false,"s":"Commentary reflecting on peer review's entrenched authority in academic life despite persistent doubts about its reliability and fairness.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"8HWJHBN9","t":"Consensus Pervades Scientific Processes","au":"Jorm, Anthony F.","y":2024,"vn":"","ty":"book","tyl":"Book","doi":"10.1007/978-981-97-9222-1_2","cx":"General","co":["general-systemic"],"sg":["whole-process"],"x":0,"st":"book-monograph","oast":"gold","cby":0,"self":false,"s":"Argues that consensus-building is embedded throughout scientific practice, not just in review, identifying roughly ten distinct points in a typical research project where group agreement plays a role.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"8JXS4WJA","t":"Teleconference versus face-to-face scientific peer review of grant application: effects on review outcomes.","au":"Gallo, Stephen A","y":2013,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0071693","cx":"Grant","co":["Reliability-noise","Interventions"],"sg":["individual-scoring","panel-consensus"],"x":0,"st":"observational-quantitative","oast":"gold","cby":38,"self":false,"s":"A retrospective analysis of roughly 1,600 grant applications found that switching from face-to-face to teleconference panel review had little effect on scores, score distributions, or interrater reliability, apart from discussion time.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"no-statement","cau":""},{"k":"8LVJQ749","t":"Predicting article quality scores with machine learning: The U.K. Research Excellence Framework","au":"Thelwall, Mike","y":2023,"vn":"Quantitative Science Studies","ty":"journalArticle","tyl":"Journal article","doi":"10.1162/qss_a_00258","cx":"General","co":["Validity","Interventions"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":26,"self":false,"s":"Machine-learning models trained on bibliometric and metadata features predicted UK REF2021 peer-review quality scores with reasonable accuracy in medical, physical science and economics fields, but performed poorly in social sciences, humanities and mathematics.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"open","cau":"https://doi.org/10.6084/m9.figshare.21723227"},{"k":"8P8K5JWV","t":"Gender and other potential biases in peer review: cross-sectional analysis of 38 250 external peer review reports.","au":"Severin, Anna","y":2020,"vn":"BMJ Open","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmjopen-2019-035058","cx":"Grant","co":["Bias"],"sg":["reviewer-selection","individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":50,"self":false,"s":"A large cross-sectional analysis of grant reviews at the Swiss National Science Foundation finding that male applicants and male reviewers received or gave somewhat higher scores, along with effects tied to who nominated the reviewer and their country of affiliation.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"open","cau":"www.git.io/fhaJx"},{"k":"8PTHSIIY","t":"Mavericks and lotteries","au":"Avin, Shahar","y":2018,"vn":"Studies in History and Philosophy of Science Part A","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.shpsa.2018.11.006","cx":"Grant","co":["Reliability-noise","Bias","Validity"],"sg":["funding-decision"],"x":0,"st":"conceptual-commentary","oast":"hybrid","cby":43,"self":false,"s":"Reviews the emergence of lottery-based grant allocation across several funders and disciplines, tracing the arguments for randomizing funding decisions and how they have (or have not) shaped actual policy.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"8PYAHQPH","t":"A Guide to Peer Review in Developmental Psychology","au":"Singh, Leher","y":2025,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31234/osf.io/uzxp2_v1","cx":"Journal","co":["Bias","Interventions"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":0,"self":false,"s":"Editorial offering guidance for constructive peer review in developmental psychology, covering qualities like clarity, consistency, and bias awareness, aimed particularly at early-career and first-time reviewers.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"8RFSBJI6","t":"Statistical analysis of the National Institutes of Health peer review system","au":"Johnson, Valen E.","y":2008,"vn":"Proceedings of the National Academy of Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1073/pnas.0804538105","cx":"Grant","co":["Reliability-noise","Bias"],"sg":["individual-scoring","panel-consensus"],"x":0,"st":"simulation-methodological","oast":"closed","cby":56,"self":false,"s":"A statistical model of NIH R01 grant scoring accounts for reviewer-specific rating tendencies and how preliminary scores and panel discussion shape final ratings, estimating that adjusting for these patterns would meaningfully change which proposals get funded.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"no-statement","cau":""},{"k":"8RWURVCA","t":"Feyerabend’s well-ordered science: how an anarchist distributes funds","au":"Shaw, Jamie","y":2018,"vn":"Synthese","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11229-018-02026-3","cx":"Grant","co":["Validity"],"sg":["funding-decision"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":24,"self":false,"s":"Philosophical argument draws on Feyerabend's pluralism and Peircean theory-pursuit economics to propose funding science partly through lottery, contrasting this with \"well-ordered science\" models that assume predictable research trajectories.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"8STS2ZAR","t":"Double-blind peer review is detrimental to scientific integrity","au":"Mebane, Christopher","y":2024,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31219/osf.io/shp6a","cx":"Journal","co":["Bias","Interventions"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":0,"self":false,"s":"An opinion piece argues that concealing author and reviewer identities in double-blind review can actually make research misconduct harder to detect and proposes more transparent alternatives.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"8T5LNS52","t":"Justice, efficiency and epistemology in the peer review of scientific manuscripts","au":"Mahoney, Michael J.","y":1991,"vn":"Behavioral and Brain Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/s0140525x00065900","cx":"General","co":["Reliability-noise"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"closed","cby":3,"self":false,"s":"A brief, abstract-unavailable piece discussing fairness, efficiency, and epistemological issues in the peer review of scientific manuscripts.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"8TYZS6JZ","t":"Does mentoring new peer reviewers improve review quality? A randomized trial.","au":"Houry, Debra","y":2012,"vn":"BMC Medical Education","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/1472-6920-12-83","cx":"Journal","co":["Interventions"],"sg":["individual-scoring"],"x":1,"st":"experimental-rct","oast":"gold","cby":90,"self":false,"s":"A randomized trial paired new medical-journal reviewers with experienced mentors for their first several reviews, but found no improvement in subsequent review quality compared with reviewers receiving only standard written guidance.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"8V294IF3","t":"Opening the Black Box of Grant Peer Review: Exploratory Analysis of Linguistic and Sentiment Signals in Funding Outcomes","au":"Bin, Adriana","y":2026,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31235/osf.io/ykca7_v1","cx":"Grant","co":["Bias"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"closed","cby":0,"self":false,"s":"An exploratory study of linguistic and sentiment patterns in review reports from a Brazilian funding agency, finding that the language reviewers use to justify approval or rejection differs by funding scheme, field, and reviewer/applicant gender.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"8VT8CAJR","t":"Assassins and zealots: variations in peer review. Special report.","au":"Siegelman, SS","y":1991,"vn":"Radiology","ty":"journalArticle","tyl":"Journal article","doi":"10.1148/radiology.178.3.1994394","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":93,"self":false,"s":"Analysis of radiology journal reviewer ratings identifying persistently more lenient or more critical reviewers (\"zealots\" and \"assassins\"), with the extreme raters spread across editorial board members and specialty areas.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"8YL5NW33","t":"Experience with NIH Peer Review: Researchers' Cynicism and Desire for Change","au":"Gillespie, Gilbert W.","y":1985,"vn":"Science Technology & Human Values","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/016224398501000306","cx":"Grant","co":["Bias"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"closed","cby":47,"self":false,"s":"Discusses researcher skepticism toward NIH-style peer review, noting persistent debate over how to define \"peers\" and concerns that reviewers cannot stay disinterested when funding is scarce.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"8ZGHI9RN","t":"Facts and Myths about Refereeing","au":"Hamermesh, Daniel S.","y":1994,"vn":"The Journal of Economic Perspectives","ty":"journalArticle","tyl":"Journal article","doi":"10.1257/jep.8.1.153","cx":"Journal","co":["Bias","Interventions"],"sg":["reviewer-selection","individual-scoring"],"x":0,"st":"observational-quantitative","oast":"bronze","cby":153,"self":false,"s":"An analysis of referee behaviour at seven economics journals finds most invited referees agree to review and complete reports reasonably quickly, and that paying referees speeds up the slowest respondents.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"no-statement","cau":""},{"k":"93G8L2PU","t":"Structured peer review: pilot results from 23 Elsevier journals","au":"Malički, Mario","y":2024,"vn":"PeerJ","ty":"journalArticle","tyl":"Journal article","doi":"10.7717/peerj.17514","cx":"Journal","co":["Reliability-noise","Interventions"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":18,"self":false,"s":"Pilot of a nine-question structured review form across 23 Elsevier journals found reviewers engaged with most prompts, with agreement between reviewers highest on manuscript structure and lowest on statistical and interpretive rigor.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://purl.stanford.edu/fn500hr0903","ca":"no-statement","cau":""},{"k":"948KR92W","t":"Editorial peer review for improving the quality of reports of biomedical studies.","au":"Jefferson, T","y":2007,"vn":"Cochrane Database of Systematic Reviews","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/14651858.MR000016.pub3","cx":"Journal","co":["Bias","Validity","Interventions"],"sg":["whole-process"],"x":0,"st":"systematic-review-meta-analysis","oast":"green","cby":316,"self":false,"s":"A Cochrane systematic review of biomedical journal peer review found little solid evidence that blinding, checklists, or reviewer training improve manuscript quality, based on 28 included studies.","pg":"registered-report","pgr":"Cochrane","pgid":"10.1002/14651858.MR000016 (protocol)","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"952G7EYT","t":"Research funding randomly allocated? A survey of scientists’ views on peer review and lottery","au":"Philipps, Axel","y":2021,"vn":"Science and Public Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/scipol/scab084","cx":"Grant","co":["Validity"],"sg":["funding-decision"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":22,"self":false,"s":"A survey of PhD holders found most opposed pure random allocation of grant funding but were more open to combining lottery elements with conventional peer review.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"95WGD5RH","t":"Peer Review in Law Journals","au":"Stojanovski, Jadranka","y":2021,"vn":"Frontiers in Research Metrics and Analytics","ty":"journalArticle","tyl":"Journal article","doi":"10.3389/frma.2021.787768","cx":"Journal","co":["general-systemic"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"gold","cby":2,"self":false,"s":"Content analysis of peer-review policy documents from 119 Croatian, Italian and Spanish law journals finds that most rely on traditional anonymized review by editorial boards, with country differences in how transparency and misconduct issues are addressed.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://www.frontiersin.org/articles/10.3389/frma.2021.787768/full#supplementary-material","ca":"no-statement","cau":""},{"k":"96JCP7V7","t":"How Values and Uncertainty Shape Scientific Advance in Peer Review","au":"Smith, Daniel Scott","y":2025,"vn":"American Sociological Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/00031224251362254","cx":"Journal","co":["Reliability-noise","Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":1,"self":false,"s":"A large-scale study combining close reading with language models to analyze tens of thousands of journal reviews, finding that reviewers weigh evaluative criteria differently from one another and that editors tend to side with the most critical reviewer.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"99FF6I9A","t":"Gender Effects in the Peer Reviews of Grant Proposals: A Comprehensive Meta-Analysis Comparing Traditional and Multilevel Approaches","au":"Marsh, Herbert W.","y":2009,"vn":"Review of Educational Research","ty":"journalArticle","tyl":"Journal article","doi":"10.3102/0034654309334143","cx":"Grant","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"systematic-review-meta-analysis","oast":"closed","cby":179,"self":false,"s":"Meta-analysis re-examining gender differences in grant-proposal peer review scores using multilevel modeling, finding no reliable gender gap once methodological improvements over earlier meta-analyses are applied.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"9C6LPVHL","t":"Peer Review 2027: Scenarios for Academic Publishing in the Age of AI","au":"Munger, Kevin","y":2026,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31235/osf.io/594zj_v1","cx":"Journal","co":["general-systemic"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":1,"self":false,"s":"Report from a meeting of social-science journal editors outlining four possible future scenarios for how large language models could reshape peer review and academic publishing.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"9CQYZK6Q","t":"Peer Review, Innovation, and Predicting the Future of Science: The Scope of Lotteries in Science Funding Policy","au":"Shaw, Jamie","y":2023,"vn":"Philosophy of Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/psa.2023.35","cx":"Grant","co":["Reliability-noise","Validity"],"sg":["funding-decision"],"x":0,"st":"conceptual-commentary","oast":"hybrid","cby":6,"self":false,"s":"A conceptual paper argues that advocates of introducing lottery elements into grant funding overstate how well traditional peer review actually performs, and proposes a different framing for where lotteries could fit into funding policy.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"9E2KEZVY","t":"All that glitters is not gold: The shaping of contemporary journal peer review at scientific and medical journals","au":"Gaudet, Joanne","y":2017,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31219/osf.io/4954h","cx":"Journal","co":["general-systemic"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":3,"self":false,"s":"Sociological analysis argues that economic pressures, more than a need for objectivity, have shaped traditional pre-publication peer review as the dominant model over more open alternatives.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"9E2S4NZV","t":"Identifying research quality in the social sciences","au":"Ochsner, Michael","y":2022,"vn":"Edward Elgar Publishing eBooks","ty":"book","tyl":"Book","doi":"10.4337/9781800372559.00010","cx":"General","co":["Validity"],"sg":["individual-scoring"],"x":0,"st":"book-monograph","oast":"green","cby":5,"self":false,"s":"Argues that research quality is not an objective, fixed property but a concept that depends on disciplinary and institutional context, and proposes combining peer judgment with quantitative indicators to define it.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"9E4R28VE","t":"Trends in Peer Review","au":"Reinhart, Martin","y":2021,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31235/osf.io/nzsp5","cx":"Grant","co":["general-systemic"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":5,"self":false,"s":"A synthesis piece arguing peer review persists despite known flaws because it provides scientifically grounded and externally legitimate decisions, proposing a framework of eight elemental peer-review practices.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"9EBX8PWV","t":"The influence of peer reviewer expertise on the evaluation of research funding applications.","au":"Gallo, Stephen A","y":2016,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0165147","cx":"Grant","co":["Reliability-noise","Bias"],"sg":["reviewer-selection","individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":74,"self":false,"s":"A large observational study of biomedical grant reviews finds that reviewers with greater self-assessed subject expertise tend to score applications more harshly, replicating earlier experimental findings and pointing to a possible role for social networks in scoring.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"","ca":"no-statement","cau":""},{"k":"9EY3H6LA","t":"AI-assisted peer review","au":"Checco, Alessandro","y":2021,"vn":"Humanities and Social Sciences Communications","ty":"journalArticle","tyl":"Journal article","doi":"10.1057/s41599-020-00703-8","cx":"Journal","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"gold","cby":264,"self":false,"s":"Discusses how uncertainty and differing personal values, rather than shared consensus, shape reviewers' and editors' judgments, based on a large-scale text analysis of manuscript reviews across engineering and life-science journals.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"no-statement","cau":""},{"k":"9FKXJGFE","t":"Investigating journal peer review as scientific object of study: unabridged version – Part I","au":"Gaudet, Joanne","y":2017,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31219/osf.io/kmcen","cx":"Journal","co":["general-systemic"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":3,"self":false,"s":"Historical analysis arguing that journal peer review should be studied as a social practice with roots in older censorship and licensing systems, rather than taken for granted as an inherently rational process.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"9G3YRFSM","t":"The author–editor game","au":"García, J. A.","y":2015,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-015-1566-x","cx":"Journal","co":["Bias"],"sg":["funding-decision"],"x":0,"st":"simulation-methodological","oast":"closed","cby":30,"self":false,"s":"Game-theoretic model of author-editor interactions during manuscript revision, showing how an editor's stringency shapes which revisions authors make and how author satisfaction sorts submissions across journals.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"9HNM58YP","t":"Choosing Reviewers: Predictors of Undergraduate Manuscript Evaluations","au":"Walker, Christina","y":2022,"vn":"PS Political Science & Politics","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/s1049096521001888","cx":"Journal","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":0,"self":false,"s":"A study of seven years of undergraduate peer reviews for a political science student journal finds that a reviewer's prior editorial experience and race are the strongest predictors of how manuscripts are evaluated.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://doi.org/10.7910/DVN/HLWNGC","ca":"no-statement","cau":""},{"k":"9IU8ALQ2","t":"Ethical Theories in Research Evaluation: An Exploratory Approach","au":"Biagetti, Maria Teresa","y":2020,"vn":"Scholarly Assessment Reports","ty":"journalArticle","tyl":"Journal article","doi":"10.29024/sar.19","cx":"General","co":["general-systemic"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"diamond","cby":12,"self":false,"s":"Discusses deontological, consequentialist, and virtue-based ethical frameworks for research evaluation, proposing a combined approach centered on the common good to address known problems in peer review and metrics use.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"9LLPJNVM","t":"No evidence of any systematic bias against manuscripts by women in the peer review process of 145 scholarly journals","au":"Squazzoni, Flaminio","y":2020,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31235/osf.io/gh4rv","cx":"Journal","co":["Bias"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"gold","cby":9,"self":false,"s":"A large-scale analysis of 145 journals and roughly 1.7 million authors found no consistent gender penalty against women's manuscripts, though editors tended to match reviewers to authors by gender.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"","ca":"no-statement","cau":""},{"k":"9NBTMKDY","t":"Neglected considerations in the analysis of agreement among journal referees","au":"Hargens, Lowell L.","y":1990,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/bf02130467","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"closed","cby":82,"self":false,"s":"Reanalyzing prior studies of journal referee agreement, this paper argues associations between reviewers' recommendations are moderately strong given the inherent variability of scholarly evaluation, and that composite reviewer scores usefully aid editorial decisions.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"9P5NC8TZ","t":"A System-Level Analysis of Conference Peer Review","au":"Zhang, Yichi","y":2022,"vn":"Proceedings of the 23rd ACM Conference on Economics and Computation","ty":"journalArticle","tyl":"Journal article","doi":"10.1145/3490486.3538235","cx":"Journal","co":["Reliability-noise","Interventions"],"sg":["whole-process"],"x":0,"st":"simulation-methodological","oast":"green","cby":5,"self":false,"s":"A theoretical and simulation-based analysis models conference peer review as a system balancing authors, reviewers and organizers with conflicting goals, examining how policies such as acceptance thresholds and review counts perform given inherently noisy evaluation.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"9P8K8DT7","t":"Factores de rechazo en el arbitraje de artículos científicos de la revista Tecnociencia Chihuahua","au":"Picazo, David","y":2025,"vn":"Investigación Bibliotecológica Archivonomía Bibliotecología e Información","ty":"journalArticle","tyl":"Journal article","doi":"10.22201/iibi.24488321xe.2025.102.58956","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"diamond","cby":0,"self":false,"s":"A Spanish-language study of a decade of manuscript rejections at a multidisciplinary Mexican journal, finding methodological weakness as the leading rejection reason and notable variation in rejection rates across disciplines.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"9QMZUVL4","t":"Citations versus expert opinions: Citation analysis of Featured Reviews of the American Mathematical Society","au":"Smolinsky, Lawrence","y":2020,"vn":"arXiv (Cornell University)","ty":"journalArticle","tyl":"Journal article","doi":"10.48550/arxiv.2005.05389","cx":"General","co":["Validity"],"sg":["post-award-validity"],"x":0,"st":"observational-quantitative","oast":"green","cby":1,"self":false,"s":"Compares mathematics articles singled out by post-publication expert review with the most-cited articles from the same period, finding the two ways of identifying important work overlap only weakly.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"9QWVUD3K","t":"Evaluating large language models for abstract evaluation tasks: an empirical study","au":"Liu, Yinuo","y":2026,"vn":"Frontiers in Research Metrics and Analytics","ty":"journalArticle","tyl":"Journal article","doi":"10.3389/frma.2026.1807672","cx":"Journal","co":["Reliability-noise","Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":0,"self":false,"s":"Compares scoring of 160 conference abstracts by three large language models against 14 human reviewers, finding the models agreed strongly with each other but only moderately with humans, especially on subjective criteria.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"on-request","cau":""},{"k":"9RLJISM4","t":"Innovation Suppression and Clique Evolution in Peer-Review-Based, Competitive Research Funding Systems: An Agent-Based Model","au":"Sobkowicz, Paweł","y":2015,"vn":"Journal of Artificial Societies and Social Simulation","ty":"journalArticle","tyl":"Journal article","doi":"10.18564/jasss.2750","cx":"Grant","co":["Bias","Validity","Interventions"],"sg":["whole-process"],"x":0,"st":"simulation-methodological","oast":"gold","cby":14,"self":false,"s":"An agent-based model of competitive grant-funded science simulates how imperfect peer review can promote mediocre research and self-serving reviewer cliques, testing measures intended to counteract these dynamics.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"9TFXN7CG","t":"Do Expert Panelists Herd? Evidence from FDA Committees","au":"Newham, Melissa","y":2019,"vn":"SSRN Electronic Journal","ty":"journalArticle","tyl":"Journal article","doi":"10.2139/ssrn.3476536","cx":"Analogue","co":["Bias","Interventions"],"sg":["panel-consensus"],"x":0,"st":"observational-quantitative","oast":"green","cby":2,"self":false,"s":"Structural model of FDA advisory committee votes finds evidence that panelists sometimes vote in line with earlier votes rather than their own assessment, and that simultaneous voting reduces this herding.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"9VB4W7T9","t":"Grant Review","au":"Yan, Zheng","y":2025,"vn":"Cambridge University Press eBooks","ty":"book","tyl":"Book","doi":"10.1017/9781009381338.009","cx":"Grant","co":["Reliability-noise","Bias"],"sg":["whole-process"],"x":0,"st":"book-monograph","oast":"closed","cby":0,"self":false,"s":"A book chapter surveys the current state of knowledge on grant review, covering reviewers, criteria, processes, and outcomes as background for applicants and funders.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"A42AQLJA","t":"Optimizing ChatGPT-Assisted Evaluation of Vietnamese English Majors’ Research Proposals: A Tree-of-Thought and Rubric-Based Approach","au":"Tran, The Phi","y":2025,"vn":"Studies in computational intelligence","ty":"book","tyl":"Book","doi":"10.1007/978-3-032-01348-4_3","cx":"General","co":["Reliability-noise","Interventions"],"sg":["proposal-quality","individual-scoring"],"x":0,"st":"book-monograph","oast":"closed","cby":0,"self":false,"s":"A study of ChatGPT-assisted grading of 40 Vietnamese undergraduate research proposals found that a Tree-of-Thought prompting approach improved reliability on complex evaluation criteria but reduced it on simpler ones.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"A42HFS8G","t":"How many reviewers are required to obtain reliable evaluations of NIH R01 grant proposals?","au":"Forscher, Patrick S.","y":2019,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31234/osf.io/483zj","cx":"Grant","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":24,"self":false,"s":"Analysing an NIH experiment with hundreds of reviewers scoring dozens of grant proposals, the authors find low agreement on scientific merit and estimate that roughly a dozen reviewers would be needed to achieve moderately reliable overall-impact ratings.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://osf.io/uy7vq/","ca":"open","cau":"https://osf.io/c5csm/"},{"k":"A49B9ZX7","t":"Gender bias in female physician assessments: Women considered better suited for qualitative research.","au":"Johansson, Eva E","y":2002,"vn":"Scandinavian Journal of Primary Health Care","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/02813430215553","cx":"Journal","co":["Bias"],"sg":["individual-scoring"],"x":1,"st":"experimental-rct","oast":"bronze","cby":14,"self":false,"s":"An experimental survey of Swedish physicians found that a fictitious research abstract with a qualitative design was judged more favorably when attributed to a female rather than male author, while judgments of a quantitative-design abstract showed no such gender effect.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"A58BN9MR","t":"Rethinking the funding line at the swiss national science foundation: bayesian ranking and lottery","au":"Heyard, Rachel","y":2022,"vn":"Statistics and Public Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/2330443X.2022.2086190","cx":"Grant","co":["Reliability-noise","Bias","Interventions"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"simulation-methodological","oast":"gold","cby":28,"self":false,"s":"Describes a Bayesian ranking method the Swiss National Science Foundation uses to identify proposals near the funding threshold for entry into a lottery, aiming to account for both estimated quality and uncertainty.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://doi.org/10.5281/zenodo.4531160","ca":"open","cau":"snsf-data.github.io/ERpaper-online-supplement/"},{"k":"A7ERVCRX","t":"Reproducibility of peer review in clinical neuroscience: Is agreement between reviewers any greater than would be expected by chance alone?","au":"Rothwell, Peter M.","y":2000,"vn":"Brain","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/brain/123.9.1964","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"bronze","cby":251,"self":false,"s":"Analysis of manuscript and abstract review data from clinical neuroscience journals and conferences finding that agreement between independent reviewers on accept/reject decisions was little better than chance.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"A8F6GAJN","t":"Decisions, decisions.","au":"Rodgers, Peter","y":2017,"vn":"eLife","ty":"journalArticle","tyl":"Journal article","doi":"10.7554/eLife.32011","cx":"Journal","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":16,"self":false,"s":"A brief overview describes how journals and funders are experimenting with new peer review approaches to address bias and transparency concerns and better meet author expectations.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"A8QVM43M","t":"Heuristics as conceptual lens for understanding and studying the usage of bibliometrics in research evaluation","au":"Bornmann, Lutz","y":2019,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-019-03018-x","cx":"General","co":["general-systemic"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"hybrid","cby":46,"self":false,"s":"Proposes applying the fast-and-frugal heuristics framework from cognitive science to explain how evaluators use bibliometric indicators as simplified decision shortcuts under uncertainty in research assessment.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"ABQTK2RB","t":"Conservatism and risk-taking in peer review: Emerging ERC practices","au":"Luukkonen, Terttu","y":2012,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvs001","cx":"Grant","co":["Bias"],"sg":["panel-consensus","individual-scoring"],"x":0,"st":"qualitative","oast":"closed","cby":187,"self":false,"s":"Examines how the European Research Council's peer reviewers handle proposals for genuinely novel research, arguing that existing knowledge boundaries and informal interpretive norms limit reviewers' willingness to take risks.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"ACMIRSDD","t":"Integration and generalization of kappas for multiple raters.","au":"Conger, Anthony J.","y":1980,"vn":"Psychological Bulletin","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/0033-2909.88.2.322","cx":"Analogue","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"closed","cby":527,"self":false,"s":"A methodological paper extending Cohen's kappa statistic for two raters into generalized agreement indices applicable to multiple raters using a nominal scale.","pg":"no-statement","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"AEDP93ZE","t":"Interrater reliability estimators tested against true interrater reliabilities","au":"Zhao, Xinshu","y":2022,"vn":"BMC Medical Research Methodology","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s12874-022-01707-5","cx":"Analogue","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"gold","cby":57,"self":false,"s":"A controlled experiment compares several well-known statistical formulas for interrater reliability against directly observed true agreement rates, aiming to clarify which chance-correction methods perform accurately under realistic rating behaviour.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"no-statement","cau":""},{"k":"AEP27EJR","t":"Open up: a survey on open and non-anonymized peer reviewing.","au":"Besançon, Lonni","y":2020,"vn":"Research Integrity and Peer Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s41073-020-00094-z","cx":"Journal","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"green","cby":0,"self":false,"s":"A survey of authors and reviewers involved in an open, non-anonymous peer review track at a major computing conference, finding general support for openness within that track but reluctance to extend it elsewhere.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://osf.io/vuw7h/","ca":"open","cau":"https://osf.io/vuw7h/"},{"k":"AGV9EQ59","t":"Epistemic Diversity and Editor Decisions: A Statistical Matthew Effect","au":"Heesen, Remco","y":2019,"vn":"UWA Profiles and Research Repository (University of Western Australia)","ty":"journalArticle","tyl":"Journal article","doi":"","cx":"Journal","co":["Bias"],"sg":["funding-decision"],"x":0,"st":"simulation-methodological","oast":"green","cby":39,"self":false,"s":"Formal modeling argument that even impartial editorial procedures can structurally disadvantage less-established research programs, amplifying existing reputation gaps over time.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"AHI3KN85","t":"Non-Financial Conflicts of Interest in Academic Grant Evaluation: A Qualitative Study of Multiple Stakeholders in France","au":"Abdoul, Hendy","y":2012,"vn":"PLoS ONE","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0035247","cx":"Grant","co":["Bias"],"sg":["reviewer-selection"],"x":0,"st":"qualitative","oast":"gold","cby":50,"self":false,"s":"Qualitative interview study of French grant reviewers and applicants finding that non-financial conflicts of interest, such as rivalry and cronyism, were seen as more common than financial ones, without undermining trust in peer review overall.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"AIII4UT9","t":"Inappropriate and appropriate selection of \"peers\" in grant review.","au":"Glantz, Stanton A","y":1994,"vn":"JAMA: Journal of the American Medical Association","ty":"journalArticle","tyl":"Journal article","doi":"10.1001/jama.272.2.114","cx":"Grant","co":["Bias"],"sg":["reviewer-selection"],"x":0,"st":"observational-quantitative","oast":"closed","cby":23,"self":false,"s":"A comparison of two US tobacco-research grant review panels finds many assigned reviewers lacked relevant publication expertise on the specific topic, raising concerns about the appropriateness of reviewer selection in those programmes.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"AIQ3U7NC","t":"An overview of innovations in the external peer review of journal manuscripts.","au":"Woods, Helen Buckley","y":2023,"vn":"Wellcome Open Research","ty":"journalArticle","tyl":"Journal article","doi":"10.12688/wellcomeopenres.17715.2","cx":"Journal","co":["Interventions"],"sg":["whole-process"],"x":0,"st":"systematic-review-meta-analysis","oast":"green","cby":0,"self":false,"s":"Review of six prior review articles catalogues innovations in external peer review of journal manuscripts, organizing them into categories covering process approaches, reviewer-focused initiatives, and supporting technology.","pg":"no-statement","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"AKUVGJ36","t":"Community review: a robust and scalable selection system for resource allocation within open science and innovation communities","au":"Graham, Chris L. B.","y":2022,"vn":"F1000Research","ty":"journalArticle","tyl":"Journal article","doi":"10.12688/f1000research.125886.1","cx":"Grant","co":["Reliability-noise","Bias"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"gold","cby":1,"self":false,"s":"Describes and evaluates a fast, applicant-involved \"community review\" system used to distribute small emergency grants across 147 pandemic-related projects, finding it quick, scalable, and resistant to bias from reviewer removal.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://doi.org/10.17605/OSF.IO/CAZ4N","ca":"open","cau":"https://doi.org/10.17605/OSF.IO/W5Q9B"},{"k":"AMFWGDX4","t":"Towards theorizing peer review","au":"Hug, Sven E.","y":2022,"vn":"Quantitative Science Studies","ty":"journalArticle","tyl":"Journal article","doi":"10.1162/qss_a_00195","cx":"General","co":["general-systemic"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":30,"self":false,"s":"An argument that peer review research is undertheorized, focusing too much on documenting phenomena like bias rather than building explanatory theory, with suggestions for advancing theoretical work in the field.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"ANYJYNQA","t":"The Delphi process: a solution for reviewing novel grant applications","au":"Robotin, Monica","y":2010,"vn":"International Journal of General Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.2147/ijgm.s11117","cx":"Grant","co":["Interventions"],"sg":["panel-consensus"],"x":0,"st":"observational-quantitative","oast":"gold","cby":19,"self":false,"s":"A funding body used a three-round Delphi voting process among external reviewers to rank novel pancreatic-cancer proposals when local expertise was scarce, converging on four recommended grants with reviewer agreement.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"AQBH53NF","t":"Peer reviewers altered their recommendation based on whether they were cited or wanted to be cited. A matched study of open peer review at four journals","au":"Barnett, Adrian","y":2025,"vn":"eLife","ty":"journalArticle","tyl":"Journal article","doi":"10.7554/elife.108748.1","cx":"Journal","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":0,"self":false,"s":"A study of over 37,000 open peer reviews across four journals finding that reviewers who were cited in a manuscript were more likely to recommend acceptance, while those who requested a self-citation were markedly less likely to approve the paper.","pg":"preregistered","pgr":"AsPredicted","pgid":"https://aspredicted.org/rn8vg.pdf","da":"open","dau":"https://doi.org/10.5281/zenodo.16551814","ca":"open","cau":"https://doi.org/10.5281/zenodo.16551814"},{"k":"AR9S4GQ4","t":"CRITICAL EVALUATION OF THE GRANT FUNDING ALLOCATION MECHANISM IN SOCIAL AND HUMANITARIAN SCIENCES","au":"Молдашев, Кайрат","y":2025,"vn":"Central Asian Economic Review","ty":"journalArticle","tyl":"Journal article","doi":"10.52821/2789-4401-2024-5-48-58","cx":"Grant","co":["Reliability-noise","Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":0,"self":false,"s":"Analyzes correlation between expert review scores and national council scores in Kazakhstan's social-science grant funding, finding only weak-to-moderate agreement between the two evaluation stages.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"","ca":"no-statement","cau":""},{"k":"ASUP2B28","t":"A Dataset of Peer Reviews (PeerRead): Collection, Insights and NLP Applications","au":"Kang, Dongyeop","y":2018,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.18653/v1/n18-1149","cx":"Journal","co":["general-systemic"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":165,"self":false,"s":"Describes the construction of PeerRead, a public dataset of peer reviews compiled for natural language processing research on review text.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://github.com/allenai/PeerRead","ca":"open","cau":"https://github.com/allenai/PeerRead"},{"k":"AUAWRZC8","t":"A framework for assessing the trustworthiness of scientific research findings","au":"Nosek, Brian A.","y":2026,"vn":"Proceedings of the National Academy of Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1073/pnas.2536736123","cx":"General","co":["general-systemic"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"hybrid","cby":3,"self":false,"s":"A conceptual paper proposes a systems framework for assessing research trustworthiness spanning accountability, methodological rigor, bias control, and evidentiary support, intended to generate measurable indicators applicable across research fields and organizations.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"AV5X84YW","t":"What Authors Want From Journal Reviewers and Editors.","au":"Nickerson, Raymond S.","y":2005,"vn":"American Psychologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/0003-066x.60.6.661","cx":"Journal","co":["general-systemic"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"closed","cby":26,"self":false,"s":"Commentary reflecting on divergent views among clinicians and researchers over interpreting subjective, informal judgment in psychological assessment debates.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"AWK223K2","t":"Societal impact evaluation: Exploring evaluator perceptions of the characterization of impact under the REF2014: Table 1.","au":"Samuel, Gabrielle","y":2015,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvv007","cx":"General","co":["general-systemic"],"sg":["individual-scoring"],"x":0,"st":"qualitative","oast":"hybrid","cby":95,"self":false,"s":"Interviews with UK Research Excellence Framework evaluators reveal differing personal understandings of what \"societal impact\" means before they began scoring submissions under that criterion.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"AWNC85V8","t":"Identification and comparison of key criteria of feedback of funding decisions: mixed-methods analysis of funder and applicant perspectives","au":"Fackrell, Kathryn","y":2021,"vn":"BMJ Open","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmjopen-2021-048979","cx":"Grant","co":["general-systemic"],"sg":["funding-decision"],"x":0,"st":"observational-quantitative","oast":"gold","cby":6,"self":false,"s":"A mixed-methods study of UK NIHR funding feedback examined what makes reviewer feedback to unsuccessful and successful grant applicants valuable, developing a framework of key feedback components from documents and a survey.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"no-statement","cau":""},{"k":"AX5647XH","t":"Expertise versus Bias in Evaluation: Evidence from the NIH","au":"Li, Danielle","y":2017,"vn":"American Economic Journal Applied Economics","ty":"journalArticle","tyl":"Journal article","doi":"10.1257/app.20150421","cx":"Grant","co":["Bias","Validity"],"sg":["reviewer-selection","individual-scoring"],"x":0,"st":"observational-quantitative","oast":"bronze","cby":171,"self":false,"s":"Analysing NIH review data, the study finds reviewers score more favourably and more informatively in their own area of expertise, concluding that the benefits of subject-matter expertise generally outweigh the biases it introduces.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://doi.org/10.1257/app.20150421","ca":"no-statement","cau":""},{"k":"AXTNPFSW","t":"Peer review in academic publishing: Challenges in achieving the gold standard","au":"Gonzalez, Prue","y":2022,"vn":"Journal of University Teaching and Learning Practice","ty":"journalArticle","tyl":"Journal article","doi":"10.53761/1.19.5.1","cx":"Journal","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"diamond","cby":29,"self":false,"s":"An editorial surveying current challenges facing academic peer review, including bias, reviewer fatigue, and fabricated reviews, and tracing the system's historical development toward proposals for reform.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"AZU4V3L3","t":"Crowd-sourced peer review: wisdom or tyranny of the crowd?","au":"Muller, Seán","y":2025,"vn":"South African Journal of Philosophy","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/02580136.2025.2462433","cx":"Journal","co":["Bias","Validity"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"bronze","cby":3,"self":false,"s":"Critiques the argument that crowd-sourced peer review outperforms traditional review, presenting a counterexample where larger open reviewer pools can actually suppress unconventional or revolutionary work.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"B32YEVN2","t":"Evaluation in research funding agencies: Are structurally diverse teams biased against?","au":"Banal‐Estañol, Albert","y":2019,"vn":"Research Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.respol.2019.04.008","cx":"Grant","co":["Bias","Validity"],"sg":["funding-decision","post-award-validity"],"x":0,"st":"observational-quantitative","oast":"closed","cby":43,"self":false,"s":"Analysis of UK EPSRC funding decisions finding that structurally diverse research teams were less likely to receive funding despite being more likely to succeed after being funded, indicating bias against diversity.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"B68EMUVK","t":"Does gender matter in grant peer review?: an empirical investigation using the example of the austrian science fund.","au":"Mutz, Rüdiger","y":2012,"vn":"Zeitschrift für Psychologie","ty":"journalArticle","tyl":"Journal article","doi":"10.1027/2151-2604/a000103","cx":"Grant","co":["Bias"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"green","cby":75,"self":false,"s":"An analysis of nearly 8,500 Austrian Science Fund grant applications (1999-2009) finds no gender effect on funding decisions overall, but reports lower approval odds when reviewer panels have gender parity or a female majority.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"B6DG7TSN","t":"Analyzing sentiments in peer review reports: Evidence from two science funding agencies","au":"Luo, Junwen","y":2021,"vn":"Quantitative Science Studies","ty":"journalArticle","tyl":"Journal article","doi":"10.1162/qss_a_00156","cx":"Grant","co":["Reliability-noise","Validity"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"gold","cby":27,"self":false,"s":"Compares manual and automated sentiment analysis of grant review text across two funding agencies, finding sentiment correlates with review scores but manually coded sentiment predicts funding outcomes more accurately than algorithmic scoring.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"open","cau":"https://github.com/thomasfeliciani/SFI-and-SNSF-review-sentiment"},{"k":"B6VDY29W","t":"No Peeking: Peer Review and Presumptive Blinding","au":"Ballantyne, Nathan","y":2024,"vn":"Canadian Journal of Philosophy","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/can.2024.25","cx":"Journal","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"hybrid","cby":1,"self":false,"s":"A philosophical essay defends blind review as the appropriate default practice in science, arguing it strengthens the fairness and credibility of the review process.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"B7BZH6ZQ","t":"Why Proposal Review Should Be More Like Meteorology","au":"Buck, Stuart","y":2023,"vn":"Seeds of Science","ty":"journalArticle","tyl":"Journal article","doi":"10.53975/cwwp-mrgc","cx":"Grant","co":["Interventions"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"bronze","cby":0,"self":false,"s":"A proposal, drafted with ChatGPT assistance, suggesting that grant reviewers assign quantitative probability estimates to proposal outcomes rather than subjective quality judgments, to improve precision and enable calibration tracking.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"B8MCFZB5","t":"What feedback do reviewers give when reviewing qualitative manuscripts? A focused mapping review and synthesis","au":"Herber, Oliver Rudolf","y":2020,"vn":"BMC Medical Research Methodology","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s12874-020-01005-y","cx":"Journal","co":["general-systemic"],"sg":["individual-scoring"],"x":0,"st":"systematic-review-meta-analysis","oast":"gold","cby":32,"self":false,"s":"A qualitative synthesis of open peer-review comments on qualitative-research manuscripts submitted to BioMed Central journals finds that reviewers focus most often on writing style and methods, sometimes applying evaluation standards suited to quantitative work.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"no-statement","cau":""},{"k":"B9A9JT4V","t":"Reliability, bias and randomisation in peer review: a simulation","au":"Hulkes, Alexander","y":2025,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31235/osf.io/4gqce_v2","cx":"Grant","co":["Reliability-noise","Bias","Interventions"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"simulation-methodological","oast":"gold","cby":0,"self":false,"s":"Uses computer simulations of a simplified grant peer review process to show that introducing randomization into funding decisions can reduce bias but may also reduce the overall reliability of outcomes.","pg":"no-statement","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"open","cau":"https://github.com/CillianUKRI/DPR_MetascienceAIFellowships"},{"k":"B9VSF35G","t":"Modeling Preferences: A Bayesian Mixture of Finite Mixtures for Rankings and Ratings","au":"Pearce, Michael","y":2024,"vn":"Journal of the American Statistical Association","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/01621459.2024.2444700","cx":"General","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"hybrid","cby":0,"self":false,"s":"Proposes a joint Bayesian statistical model combining ranking and rating data to capture both reviewer effects and differing preference groups, illustrated with peer-review datasets.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"BB5XVIJ5","t":"Streamlined research funding using short proposals and accelerated peer review: an observational study.","au":"Barnett, Adrian G","y":2015,"vn":"BMC Health Services Research","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s12913-015-0721-7","cx":"Grant","co":["Interventions"],"sg":["proposal-quality","individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":3,"self":false,"s":"Observational study of an Australian health-services funding scheme using short proposals and fast-tracked review, reporting reduced applicant preparation time and quick turnaround from submission to funding decision.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"BBVN6S7M","t":"Enhanced peer-review for optimising publication of biomedical papers submitted from low- and middle-income countries: feasibility study for a randomised controlled trial.","au":"Pitman, Alexandra","y":2019,"vn":"BJPsych open","ty":"journalArticle","tyl":"Journal article","doi":"10.1192/bjo.2018.89","cx":"Journal","co":["Interventions"],"sg":["individual-scoring"],"x":1,"st":"experimental-rct","oast":"gold","cby":7,"self":false,"s":"A feasibility study tests a peer-review support intervention for authors from low- and middle-income countries whose papers were rejected by a psychiatry journal, finding low uptake and mixed acceptability that argue against a full-scale trial as designed.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"no-statement","cau":""},{"k":"BCUNWS6G","t":"The limited value of journal peer review in public health: a case series of tobacco harm reduction articles.","au":"Phillips, Carl V.","y":2019,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31219/osf.io/q7jdv","cx":"Journal","co":["Validity"],"sg":["individual-scoring"],"x":0,"st":"qualitative","oast":"gold","cby":0,"self":false,"s":"Independent re-review of a dozen published tobacco-related articles in one public health journal found substantial unaddressed flaws that the original journal review process had not corrected.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://docs.google.com/spreadsheets/d/1Mq-bucQhgeTAK2cWlWZw7Fiy1Xt0xNbm1FISLeuAo_E/pubhtml","ca":"not-applicable","cau":""},{"k":"BENYRE6T","t":"Double anonymous peer review in JVSCIT—lessons from our pilot trial and next steps","au":"Smeds, Matthew R.","y":2025,"vn":"Journal of Vascular Surgery Cases and Innovative Techniques","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.jvscit.2025.102028","cx":"Journal","co":["Bias","Interventions"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":0,"self":false,"s":"Reports on a vascular-surgery journal's year-long pilot of double-anonymous review, comparing it with the single-blind and open models and describing lessons learned for further rollout.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"BFIPR3E7","t":"Social Biases and Solutions for Procedural Objectivity","au":"Lee, Carole J.","y":2011,"vn":"Hypatia","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/j.1527-2001.2011.01178.x","cx":"Journal","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"bronze","cby":68,"self":false,"s":"Drawing on Helen Longino's philosophy of science, this paper argues that shared evaluation standards can still unintentionally reproduce social bias, illustrating the point with survey and editorial data from philosophy journals.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"BFVZSMKT","t":"Peer review bias: A critical review.","au":"Haffar, Samir","y":2019,"vn":"Mayo Clinic Proceedings","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.mayocp.2018.09.004","cx":"Journal","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"bronze","cby":163,"self":false,"s":"A narrative literature review outlines mechanisms by which the peer-review process can introduce bias into published research and evaluates evidence for blinding and open-review strategies intended to reduce it.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"BFW9NA83","t":"Bibliometric indicators: opportunities and limits.","au":"Belter, Christopher W","y":2015,"vn":"Journal of the Medical Library Association","ty":"journalArticle","tyl":"Journal article","doi":"10.3163/1536-5050.103.4.014","cx":"General","co":["Reliability-noise","Bias","Validity"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"diamond","cby":302,"self":false,"s":"Discusses the rising use of bibliometric indicators alongside or instead of peer review to cope with growing publication volume and reviewer scarcity in evaluating scientific output.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"BG57BA84","t":"The effect of the scale of grant scoring on ranking accuracy","au":"Visscher, Peter M.","y":2023,"vn":"F1000Research","ty":"journalArticle","tyl":"Journal article","doi":"10.12688/f1000research.125400.2","cx":"Grant","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"gold","cby":2,"self":false,"s":"Statistical modeling study finding that scoring grants on a small number of discrete categories loses little information compared with a continuous scale, so adding more scoring categories offers limited benefit.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://github.com/loic-yengo/GrantSCoring_Figures","ca":"open","cau":"https://github.com/loic-yengo/GrantSCoring_Figures"},{"k":"BHZQYW9A","t":"Analysis of reporting quality for oral presentations of observational studies at 19th National Surgical Congress: Proposal for a national evaluation system.","au":"Hasbahçeci, Mustafa","y":2016,"vn":"Ulusal cerrahi dergisi","ty":"journalArticle","tyl":"Journal article","doi":"10.5152/UCD.2016.3195","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"green","cby":0,"self":false,"s":"Compares three scoring systems for evaluating conference abstracts, finding a proposed national evaluation system produced consistent inter-reviewer agreement comparable to established international tools.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"BI6REK75","t":"(Fehlende) Anwesenheit bei Begutachtungsprozessen: Zum Wandel der sozialen Interaktionen im virtuellen Format","au":"Paruschke, Laura","y":2023,"vn":"Zeitschrift für Soziologie","ty":"journalArticle","tyl":"Journal article","doi":"10.1515/zfsoz-2023-2027","cx":"Grant","co":["general-systemic"],"sg":["panel-consensus"],"x":0,"st":"qualitative","oast":"hybrid","cby":3,"self":false,"s":"A German qualitative interview study compares in-person versus virtual site-visit grant review formats within two DFG programmes, finding virtual formats constrain social interaction in ways participants view as detrimental to thorough evaluation.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"BJ4METPH","t":"Improving peer review: What reviewers can do","au":"DiDomenico, Robert J.","y":2017,"vn":"American Journal of Health-System Pharmacy","ty":"journalArticle","tyl":"Journal article","doi":"10.2146/ajhp170190","cx":"Journal","co":["Interventions"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"bronze","cby":30,"self":false,"s":"Practical guidance article offering suggestions for reviewers on conducting more effective and constructive manuscript reviews.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"BJHV5PDY","t":"Gender differences in authorships are not associated with publication bias in an evolutionary journal","au":"Edwards, Hannah A","y":2018,"vn":"PLoS ONE","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0201725","cx":"Journal","co":["Bias"],"sg":["funding-decision"],"x":0,"st":"observational-quantitative","oast":"gold","cby":54,"self":false,"s":"A study of an evolutionary biology journal found gender differences existed in who served as corresponding author, but overall editorial acceptance decisions were not biased by author gender, though manuscripts from Asia were rejected more often.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://doi.org/10.6084/m9.figshare.5821110.v1","ca":"no-statement","cau":""},{"k":"BJLC5S5R","t":"Fragments of peer review: A quantitative analysis of the literature (1969-2015)","au":"Grimaldo, Francisco","y":2018,"vn":"PLoS ONE","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0193148","cx":"General","co":["general-systemic"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"gold","cby":56,"self":false,"s":"A bibliometric analysis of peer review research from 1969 to 2015 found publications on the topic doubled after 2005 but that the research community remains fragmented, with little large-scale collaboration.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"","ca":"open","cau":""},{"k":"BLZITU4C","t":"A reader's, writer's, and reviewer's guide to assessing research reports in clinical psychology.","au":"Maher, Brendan A.","y":1978,"vn":"Journal of Consulting and Clinical Psychology","ty":"journalArticle","tyl":"Journal article","doi":"10.1037//0022-006x.46.4.835","cx":"Journal","co":["Interventions"],"sg":["proposal-quality","individual-scoring"],"x":0,"st":"guideline-policy","oast":"closed","cby":80,"self":false,"s":"Drawing on experience reviewing thousands of manuscripts, the author sets out general guidelines for evaluating research reports in clinical and consulting psychology, covering structure, citations, hypotheses and methodology.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"BML9YX8S","t":"Improving the Evaluation of Submitted Manuscripts.","au":"Newman, Sidney H.","y":1966,"vn":"American Psychologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/h0021046","cx":"Journal","co":["Reliability-noise","Bias","Interventions"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":48,"self":false,"s":"An early opinion piece calling for more systematic, empirically grounded procedures to reduce subjective error when evaluating manuscripts submitted to psychology journals.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"BQHSYZPG","t":"Editorial decisions with informed and uninformed reviewers","au":"Rodríguez‐Sánchez, Rosa","y":2018,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-018-2875-7","cx":"Journal","co":["general-systemic"],"sg":["panel-consensus","funding-decision"],"x":0,"st":"simulation-methodological","oast":"green","cby":3,"self":false,"s":"Game-theoretic model comparing reviewer groups who can see each other's comments with those who cannot, examining how this affects the probabilities of editors' accept/reject decisions.","pg":"no-statement","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"no-statement","cau":""},{"k":"BQP92QB2","t":"Peer review: from “sacred ideals” to “profane realities”","au":"Johnson, David R.","y":2017,"vn":"Higher education: handbook of theory and research","ty":"bookSection","tyl":"Book chapter","doi":"10.1007/978-3-319-48983-4_10","cx":"Journal","co":["Reliability-noise","Bias"],"sg":["whole-process"],"x":0,"st":"book-monograph","oast":"closed","cby":46,"self":false,"s":"Book chapter examining how academic peer review functions as a socially structured system of judgment among colleagues, contrasting its idealized purpose with everyday practice.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"BRRTSI2P","t":"Transforming the Academy of Community Reviewers (ACR) course into an E-Learning course in the Post COVID-19 Pandemic Era","au":"Neal J.","y":2024,"vn":"Journal of Clinical and Translational Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/cts.2024.215","cx":"Grant","co":["Interventions"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":0,"self":false,"s":"A description of moving a community grant-reviewer training programme online during the pandemic, restructuring content into modules covering equity and bias alongside practical grant-review skills for lay reviewers.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"BS9CHRDT","t":"Three years of quality assurance data assessing the performance of over 4000 grant peer review contributions to the Canadian Institutes of Health Research Project Grant Competition","au":"Ardern, Clare L.","y":2023,"vn":"FACETS","ty":"journalArticle","tyl":"Journal article","doi":"10.1139/facets-2022-0175","cx":"Grant","co":["general-systemic"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":4,"self":false,"s":"Summarizes three years of a Canadian funding agency's internal quality-assurance program rating thousands of grant reviewer contributions, finding most reviewers performed adequately while a small share were rated as underperforming.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://doi.org/10.5683/SP3/KZL1M1","ca":"no-statement","cau":""},{"k":"BSI9PZ5P","t":"Not all peer reviews are significant: a dataset of exhaustive vs. trivial scientific peer reviews leveraging chain-of-thought reasoning","au":"Bharti, Prabhat Kumar","y":2026,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-025-05435-7","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":1,"self":false,"s":"Uses large language models to automatically classify peer reviews as thorough or superficial, testing whether AI-based sorting of review quality aligns with human judgments.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://github.com/PrabhatkrBharti/InsightfulPeer-CoT.git","ca":"open","cau":"https://github.com/PrabhatkrBharti/InsightfulPeer-CoT.git"},{"k":"BTE5MMT9","t":"The changing forms and expectations of peer review","au":"Horbach, Serge P. J. M.","y":2018,"vn":"Research Integrity and Peer Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s41073-018-0051-5","cx":"Journal","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":176,"self":false,"s":"A literature review traces how expectations for peer review, particularly its role in safeguarding research integrity, have evolved, and systematizes the range of peer review models that have emerged in response to publishing crises.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"BTNL5XNW","t":"The effect of public funding on research output: the New Zealand Marsden Fund","au":"Gush, Jason","y":2017,"vn":"New Zealand Economic Papers","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/00779954.2017.1325921","cx":"Grant","co":["Validity"],"sg":["post-award-validity"],"x":0,"st":"observational-quantitative","oast":"green","cby":38,"self":false,"s":"Comparing researchers funded by New Zealand's Marsden Fund with unfunded applicants while adjusting for panel rankings, the study finds funding is linked to modest increases in publication output and citation-weighted output.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"BU8EHDHM","t":"Expert decision support system as a means of improving the quality of reviewing articles in a scientific journal","au":"Spasennikov, Valeriy","y":2022,"vn":"Ergodesign","ty":"journalArticle","tyl":"Journal article","doi":"10.30987/2658-4026-2022-2-128-136","cx":"Journal","co":["Interventions"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"closed","cby":4,"self":false,"s":"Describes the design of a computerized expert decision-support tool intended to standardize manuscript quality assessment and reduce reviewer subjectivity at a specific journal.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"BVGT7SQ9","t":"Post-Publication Peer Review for Real","au":"Ikeda, Koki","y":2020,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31234/osf.io/sp3j5","cx":"General","co":["Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":6,"self":false,"s":"Argues that post-publication peer review has underdelivered due to weak incentives, proposing that journals publish review commentaries as citable works to motivate participation, modeled on a commentary format used by one journal.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"BX3VF92R","t":"Large sample variance of kappa in the case of different sets of raters.","au":"Fleiss, Joseph L.","y":1979,"vn":"Psychological Bulletin","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/0033-2909.86.5.974","cx":"Analogue","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"closed","cby":391,"self":false,"s":"Methodological paper deriving corrected statistical formulas for the variance of the kappa agreement statistic when different raters are used for different subjects.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"BZ2KEKP2","t":"Blinding in peer review: the preferences of reviewers for nursing journals","au":"Baggs, Judith Gedney","y":2008,"vn":"Journal of Advanced Nursing","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/j.1365-2648.2008.04816.x","cx":"Journal","co":["Bias","Interventions"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":89,"self":false,"s":"A 2007 survey of over 1,600 nursing journal reviewers finds double-blind review is the dominant and strongly preferred method, with most reviewers unable to identify blinded authors and expressing satisfaction with the practice.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"BZ4JJDAM","t":"Peer Review: A System under Stress","au":"Gropp, Robert E.","y":2017,"vn":"BioScience","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/biosci/bix034","cx":"General","co":["Reliability-noise","Bias","Validity","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":53,"self":false,"s":"Report on a multi-stakeholder meeting on peer review identifies mounting strain from rising submission volumes, funding pressure, reviewer shortages, and calls for better training and more research into the process.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"BZDWP8J8","t":"How much would each researcher receive if competitive government research funding were distributed equally among researchers?","au":"Vaesen, Krist","y":2017,"vn":"PLoS ONE","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0183967","cx":"Grant","co":["Validity"],"sg":["funding-decision"],"x":0,"st":"observational-quantitative","oast":"gold","cby":64,"self":false,"s":"Estimates how much individual researchers would receive if national research funding were distributed equally rather than through competitive peer review, finding that most researchers could sustain current staffing plus a modest discretionary budget.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"","ca":"no-statement","cau":""},{"k":"BZFYKEBB","t":"The Novelty Paradox & Bias for Normal Science: Evidence from Randomized Medical Grant Proposal Evaluations","au":"Boudreau, Kevin","y":2012,"vn":"Harvard Business School working paper series # 13-053","ty":"journalArticle","tyl":"Journal article","doi":"10.2139/ssrn.2184791","cx":"Grant","co":["Bias"],"sg":["proposal-quality","individual-scoring"],"x":1,"st":"experimental-rct","oast":"green","cby":17,"self":false,"s":"An experiment with 142 faculty evaluating 150 medical grant proposals found that more novel proposals received systematically lower rankings, a penalty attributed mainly to reviewers' uncertainty rather than strategic behavior.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"C2RR4L79","t":"Bias and truth in science evaluation: a simulation model of grant review panel discussions.","au":"Bethencourt, Adrián Martín","y":2021,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"","cx":"Grant","co":["Reliability-noise","Bias"],"sg":["panel-consensus"],"x":0,"st":"simulation-methodological","oast":"","cby":0,"self":false,"s":"An agent-based simulation calibrated to real Science Foundation Ireland panel data explores how social influence and bias shape grant-panel discussions, suggesting consensus-seeking dynamics and a stronger role for negative than positive bias.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"C3BI4GWS","t":"How professors think: inside the curious world of academic judgment","au":"Álvaro, SANTANA ACUÑA","y":2009,"vn":"Choice Reviews Online","ty":"journalArticle","tyl":"Journal article","doi":"10.5860/choice.47-1492","cx":"Grant","co":["Reliability-noise","Bias"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"green","cby":612,"self":false,"s":"A book examining how academic evaluation committees actually form judgments of quality, portraying the process as shaped by emotion and social dynamics rather than purely objective analysis.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"C3F8YBY5","t":"Heuristics, not plumage: A response to Osterloh and Frey's discussion paper on ‘Borrowed plumes’","au":"Wooding, Steven","y":2019,"vn":"Research Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.respol.2019.103871","cx":"General","co":["Validity"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"green","cby":5,"self":false,"s":"Commentary responding to a critique of journal impact factor use, arguing that heuristics are unavoidable in research evaluation and the real challenge is choosing which ones are appropriate.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"C3QMZY4M","t":"Peer Reviewed Evaluation of Registered End-Points of Randomised Trials (the PRE-REPORT study): a stepped wedge, cluster-randomised trial","au":"Jones, Christopher W.","y":2022,"vn":"BMJ Open","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmjopen-2022-066624","cx":"Journal","co":["Interventions"],"sg":["individual-scoring"],"x":1,"st":"experimental-rct","oast":"gold","cby":11,"self":false,"s":"Stepped-wedge cluster trial across thirteen medical journals testing whether giving peer reviewers trial-registry information reduced discrepancies between registered and published outcomes, finding no significant effect.","pg":"preregistered","pgr":"ISRCTN","pgid":"ISRCTN41225307","da":"restricted","dau":"","ca":"no-statement","cau":""},{"k":"C3XF2ZR4","t":"Reminding Peer Reviewers of Reporting Guideline Items to Improve Completeness in Published Articles","au":"Speich, Benjamin","y":2023,"vn":"JAMA Network Open","ty":"journalArticle","tyl":"Journal article","doi":"10.1001/jamanetworkopen.2023.17651","cx":"Journal","co":["Interventions"],"sg":["individual-scoring"],"x":1,"st":"experimental-rct","oast":"gold","cby":44,"self":false,"s":"Two randomized trials test whether prompting peer reviewers to check specific reporting-guideline items (CONSORT, SPIRIT) during review improves completeness of subsequent published trial reports and protocols compared to standard review practice.","pg":"preregistered","pgr":"OSF","pgid":"https://osf.io/c4hn8; https://osf.io/z2hm9","da":"open","dau":"","ca":"open","cau":""},{"k":"C3XQ6W24","t":"Gender differences in grant peer review: A meta-analysis","au":"Bornmann, L","y":2007,"vn":"Journal of Informetrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.joi.2007.03.001","cx":"Grant","co":["Bias"],"sg":["funding-decision"],"x":0,"st":"systematic-review-meta-analysis","oast":"closed","cby":347,"self":false,"s":"Meta-analysis of 21 studies finds men have roughly seven percent higher odds than women of receiving research grants, contradicting earlier narrative reviews that found little evidence of gender bias.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"C5N7Q8TK","t":"Symbolic scarcity and epistemic waste: a systems theory of academic publishing failure.","au":"Hansen, Trym","y":2026,"vn":"Research integrity and peer review","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s41073-026-00193-3","cx":"Journal","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":0,"self":false,"s":"A theoretical paper argues that publishing incentives favoring novel and attention-grabbing findings create structural waste in reviewer effort and systematically crowd out replication and confirmatory research.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"C5SPC6BS","t":"Funding lotteries for research grant allocation: An extended taxonomy and evaluation of their fairness","au":"Feliciani, Thomas","y":2024,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvae025","cx":"Grant","co":["Bias","Validity"],"sg":["funding-decision"],"x":0,"st":"simulation-methodological","oast":"hybrid","cby":4,"self":false,"s":"Using an extended taxonomy and Monte Carlo simulations, this paper compares how different funding-lottery designs balance selecting the best proposals against avoiding bias and concentration of resources, versus traditional peer review.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://github.com/thomasfeliciani/mixed-lotteries-for-science-funding/","ca":"open","cau":"https://github.com/thomasfeliciani/mixed-lotteries-for-science-funding/"},{"k":"C7YCC4A7","t":"Variability and negligence: grant peer review panels evaluating impact ex ante","au":"Oxley, Kristin","y":2025,"vn":"Science and Public Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/scipol/scae081","cx":"Grant","co":["Reliability-noise","Interventions"],"sg":["panel-consensus"],"x":0,"st":"qualitative","oast":"hybrid","cby":5,"self":false,"s":"Based on observations and interviews across five funding organizations, the study finds that grant panels give little structured weight to societal-impact criteria unless explicitly built into scoring, and proposes distinguishing \"productive\" from \"unproductive\" reviewer variability.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"no-statement","cau":""},{"k":"C8THDZNN","t":"A randomized controlled study of reviewer bias against an unconventional therapy","au":"Resch, K I","y":2000,"vn":"Journal of the Royal Society of Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/014107680009300402","cx":"Journal","co":["Bias"],"sg":["individual-scoring"],"x":1,"st":"experimental-rct","oast":"bronze","cby":94,"self":false,"s":"A randomized trial in which nearly 400 reviewers evaluated one of two otherwise-identical manuscripts describing either a conventional or an unconventional obesity treatment, finding a significant bias favoring the conventional-therapy version.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"C9INWUTB","t":"Can journal reviewers dependably assess rigour, significance, and originality in theoretical papers? Evidence from physics","au":"Thelwall, Mike","y":2023,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvad018","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"green","cby":5,"self":false,"s":"Compares reviewer scores for rigour, significance, and originality in theoretical physics papers, finding only moderate agreement between reviewers even in this well-defined subject area.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"CAM2G6QZ","t":"The Peer Review Process and Publishing Academic Work","au":"Reyes, Andres De Los","y":2024,"vn":"","ty":"book","tyl":"Book","doi":"10.1007/978-3-031-54284-8_7","cx":"Journal","co":["Reliability-noise","Bias"],"sg":["whole-process"],"x":0,"st":"book-monograph","oast":"closed","cby":0,"self":false,"s":"Book chapter framing how early-career researchers can understand and respond strategically to peer review feedback within their broader academic network.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"CBSAJ6NJ","t":"PP021 peer review innovations for grant applications: efficient and effective?","au":"Shepherd, Jonathan","y":2017,"vn":"International Journal of Technology Assessment in Health Care","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/S0266462317002124","cx":"Grant","co":["Interventions"],"sg":["whole-process"],"x":0,"st":"systematic-review-meta-analysis","oast":"bronze","cby":0,"self":false,"s":"A systematic mapping exercise catalogues 81 empirical studies of health-research grant peer review innovations aimed at efficiency and effectiveness, noting most are observational, US-based, and rarely test genuine procedural alternatives.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"CDSKP5Q8","t":"Funding grant proposals for scientific research: retrospective analysis of scores by members of grant review panel.","au":"Graves, Nicholas","y":2011,"vn":"BMJ (Clinical Research Ed.)","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmj.d4797","cx":"Grant","co":["Reliability-noise"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":138,"self":false,"s":"Retrospective analysis of Australian national health grant panel scores estimates that most funding decisions could have gone differently given normal scorer variability, and calculates the added cost of funding more reliably.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"CM2NSUHH","t":"And the winner is … : inter-rater reliability among scholarship assessors","au":"Johnston, Lucy","y":2015,"vn":"Studies in Higher Education","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/03075079.2015.1124849","cx":"Grant","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":4,"self":false,"s":"A multi-year study of two New Zealand postgraduate scholarship competitions found good to excellent agreement among assessors, with agreement strongest for academic-merit ratings and weaker on other criteria.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"CM3DJF35","t":"Is Living Easier With Eyes Closed?","au":"Wien, Charlotte","y":2024,"vn":"European Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/s1062798723000558","cx":"Journal","co":["Bias","Interventions"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"hybrid","cby":1,"self":false,"s":"An opinion piece questioning why blind peer review remains standard despite tensions with openness values increasingly embraced elsewhere in scholarly communication.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"CNGBYJUU","t":"Threats to grant peer review: a qualitative study","au":"Sims‐Gould, Joanie","y":2025,"vn":"BMJ Open","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmjopen-2024-091666","cx":"Grant","co":["Reliability-noise","Bias","Interventions"],"sg":["reviewer-selection","individual-scoring","panel-consensus"],"x":0,"st":"qualitative","oast":"gold","cby":4,"self":false,"s":"Interviews with 18 grant-panel chairs, reviewers and staff at a national funder identify three recurring threats to grant peer review: inadequate reviewer training, difficulty distinguishing similarly strong applications, and reviewers weighing reputation over merit.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"no-statement","cau":""},{"k":"CPIAV23Z","t":"Do bibliometrics introduce gender, institutional or interdisciplinary biases into research evaluations?","au":"Thelwall, Mike","y":2023,"vn":"Research Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.respol.2023.104829","cx":"General","co":["Bias"],"sg":["funding-decision"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":29,"self":false,"s":"Comparing peer review scores with citation-based metrics across the UK's national research assessment, this study finds bibliometrics can disadvantage otherwise high-quality departments, while showing small favorable effects for gender and interdisciplinary research in some fields.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"no-statement","cau":""},{"k":"CQUBJBF5","t":"Reading Peer Review","au":"Eve, Martin Paul","y":2020,"vn":"Cambridge University Press eBooks","ty":"book","tyl":"Book","doi":"10.1017/9781108783521","cx":"Journal","co":["general-systemic"],"sg":["individual-scoring"],"x":0,"st":"book-monograph","oast":"hybrid","cby":13,"self":false,"s":"Describes a unique dataset of peer review reports from PLOS ONE, presenting the history, structure, and typical features of its reviews and situating the journal's model within a shifting scientific-publishing landscape.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"no-statement","cau":""},{"k":"CSUMH6XZ","t":"Comparison of the Null Distributions of Weighted Kappa and the C Ordinal Statistic","au":"Cicchetti, Dominic","y":1977,"vn":"Applied Psychological Measurement","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/014662167700100206","cx":"Analogue","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"green","cby":206,"self":false,"s":"Simulation study comparing weighted kappa with an alternative ordinal agreement statistic, concluding weighted kappa is generally preferable and specifying minimum sample sizes needed for its valid use.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"CSVNG9TD","t":"Double-blind review in software engineering venues","au":"Beller, Moritz","y":2016,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.7287/peerj.preprints.1757v2","cx":"Journal","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"gold","cby":1,"self":false,"s":"A mixed-methods study surveys and interviews the software engineering research community about adopting double-blind review, finding perceived logistical costs generally outweigh benefits, though most respondents still would not oppose a switch for major venues.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"http://sback.it/dblind","ca":"open","cau":"http://sback.it/dblind"},{"k":"CUA96JA7","t":"Ex ante evaluation of interdisciplinary research projects: A literature review","au":"Oviedo‐García, M. Ángeles","y":2016,"vn":"Social Science Information","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/0539018416658147","cx":"Grant","co":["Reliability-noise","Bias","Validity","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":5,"self":false,"s":"Literature review on evaluating interdisciplinary research proposals ahead of funding, describing two alternative approaches to standard peer review designed for this context.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"CUHSPUCH","t":"Textual analysis of artificial intelligence manuscripts reveals features associated with peer review outcome","au":"Vincent‐Lamarre, Philippe","y":2021,"vn":"Quantitative Science Studies","ty":"journalArticle","tyl":"Journal article","doi":"10.1162/qss_a_00125","cx":"Journal","co":["Bias"],"sg":["funding-decision"],"x":0,"st":"observational-quantitative","oast":"gold","cby":3,"self":false,"s":"Text analysis of AI-conference manuscripts found accepted papers used more technical jargon, less common vocabulary, and cited more overlapping references than rejected papers, and word choice alone predicted review outcomes.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://github.com/lamvin/PeerReviewAI.git","ca":"open","cau":"https://github.com/lamvin/PeerReviewAI.git"},{"k":"CUKLB2A9","t":"Anosmic peer review: A rose by another name is evidently not a rose","au":"Scarr, Sandra","y":1982,"vn":"Behavioral and Brain Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/s0140525x0001164x","cx":"Journal","co":["Reliability-noise","Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":50,"self":false,"s":"A brief, abstract-unavailable commentary using an analogy to discuss shortcomings in how peer review is conducted or labeled.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"CWDVSG5K","t":"The limitations to our understanding of peer review.","au":"Tennant, Jonathan P","y":2020,"vn":"Research Integrity and Peer Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s41073-020-00092-1","cx":"Journal","co":["Bias","Validity"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":355,"self":false,"s":"A conceptual review maps major gaps in current knowledge about peer review, including reviewer bias and editorial accountability, and calls for shared standards and data infrastructure to support a dedicated field of peer-review research.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"CWZ994PR","t":"How to sponsor ground-breaking research: a comparison of funding schemes","au":"Heinze, Thomas","y":2008,"vn":"Science and Public Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.3152/030234208X317151","cx":"Grant","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"qualitative","oast":"green","cby":125,"self":false,"s":"Compares funding schemes designed to support unconventional, high-risk research across institutional features like selection criteria and budget structure, and argues such programmes should account for peer review's inherent bias against novel work.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"CY45T4US","t":"The AI Review Lottery: Widespread AI-Assisted Peer Reviews Boost Paper Scores and Acceptance Rates","au":"Russo, Giuseppe","y":2025,"vn":"Proceedings of the ACM on Human-Computer Interaction","ty":"journalArticle","tyl":"Journal article","doi":"10.1145/3757667","cx":"Journal","co":["Bias"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"green","cby":5,"self":false,"s":"Analysis of a major machine-learning conference finding a substantial share of reviews showed signs of AI assistance, and that AI-assisted reviews tended to score papers higher and modestly raise acceptance likelihood for borderline submissions.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"CYVRJWAL","t":"Peer review","au":"Hyland, Ken","y":2020,"vn":"Journal of English for Research Publication Purposes","ty":"journalArticle","tyl":"Journal article","doi":"10.1075/jerpp.19010.hyl","cx":"Journal","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"green","cby":11,"self":false,"s":"Essay-style piece arguing that despite common academic criticisms of peer review as slow or biased, it still serves an important quality-control function worth defending and teaching.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"D2ITMW42","t":"The Cultivation of Social Work Knowledge: Towards a More Robust System of Peer Review","au":"Dunleavy, Daniel J.","y":2020,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31235/osf.io/rvqze","cx":"Journal","co":["Reliability-noise","Validity","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":4,"self":false,"s":"A commentary argues social work scholarship should move beyond strict double-blind peer review toward more open, dynamic reviewing practices, situating this shift within the broader open science movement.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"D2RXKQ8L","t":"A Coefficient of Agreement for Nominal Scales","au":"Cohen, Jacob","y":1960,"vn":"Educational and Psychological Measurement","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/001316446002000104","cx":"Analogue","co":["Reliability-noise"],"sg":["whole-process"],"x":0,"st":"simulation-methodological","oast":"closed","cby":41373,"self":false,"s":"Introduces Cohen's kappa, a chance-corrected statistic for measuring agreement between two independent raters classifying items into unordered categories.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"D69EUCWP","t":"The Review Process at PSPB: Correlates of Interreviewer Agreement and Manuscript Acceptance","au":"Petty, Richard E.","y":1999,"vn":"Personality and Social Psychology Bulletin","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/0146167299025002005","cx":"Journal","co":["Reliability-noise","Bias"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"closed","cby":62,"self":false,"s":"A multi-year analysis of one psychology journal found reviewer judgments were shaped by multiple factors, reviewer agreement was higher among prestigious reviewer pairs, and articles by female authors received less favorable editorial decisions.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"D8EVAPCQ","t":"Multiple evaluations of grant proposals by independent assessors: confirmatory factor analysis evaluations of reliability, validity, and structure.","au":"Marsh, H W","y":1999,"vn":"Multivariate behavioral research","ty":"journalArticle","tyl":"Journal article","doi":"10.1207/s15327906mbr3401_1","cx":"Grant","co":["Reliability-noise","Validity"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":16,"self":false,"s":"Using confirmatory factor analysis on Australian Research Council grant ratings, this study found modest reliability across independent assessors and evidence that ratings of proposals and researchers largely reflect a single underlying factor rather than two distinct ones.","pg":"not-codeable","pgr":"","pgid":"","da":"","dau":"","ca":"","cau":""},{"k":"DBHUKEVE","t":"Evaluating the Pros and Cons of Different Peer Review Policies via Simulation","au":"Zhu, Jia","y":2015,"vn":"Science and Engineering Ethics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11948-015-9683-8","cx":"Journal","co":["Bias","Validity","Interventions"],"sg":["whole-process"],"x":0,"st":"simulation-methodological","oast":"closed","cby":6,"self":false,"s":"Computer simulations of a conference peer-review process compare single-blind and double-blind review with author-feedback mechanisms, finding that programme-chair decisions and reviewer discussion matter more than blinding policy alone, and highlighting a strong role for randomness.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"DBUHQVVR","t":"Does the pressure to fill journal quotas bias evaluation?: Evidence from publication delays and rejection rates","au":"Park, Brian","y":2020,"vn":"PLoS ONE","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0236927","cx":"Journal","co":["Bias"],"sg":["funding-decision"],"x":0,"st":"observational-quantitative","oast":"gold","cby":7,"self":false,"s":"Analyzes how publication backlog pressure at academic journals correlates with subsequent manuscript rejection rates, suggesting time-driven quota pressures may introduce an evaluation bias into editorial decisions.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://www.apa.org/pubs/journals/statistics","ca":"no-statement","cau":""},{"k":"DBXX2Q3H","t":"Gender Gaps in the Evaluation of Research: Evidence from Submissions to Economics Conferences*","au":"Hospido, Laura","y":2020,"vn":"Oxford Bulletin of Economics and Statistics","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/obes.12409","cx":"Journal","co":["Bias"],"sg":["funding-decision"],"x":0,"st":"observational-quantitative","oast":"bronze","cby":57,"self":false,"s":"Analysis of thousands of economics conference submissions finding papers by all-female author teams were accepted at a lower rate than those by all-male teams, with the gap attributed mainly to male reviewers' evaluations.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"DM28BPQZ","t":"Grant review feedback: appropriateness and usefulness.","au":"Gallo, Stephen A","y":2021,"vn":"Science and engineering ethics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11948-021-00295-9","cx":"Grant","co":["Reliability-noise","Bias"],"sg":["proposal-quality","individual-scoring"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":20,"self":false,"s":"Survey study of grant applicants' views on review feedback finding that most rated feedback as fair, but usefulness varied by funding outcome, applicant gender, and race, with poor inter-panel reliability cited as a limiting factor.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://doi.org/10.6084/m9.figshare.8132453.v1","ca":"no-statement","cau":""},{"k":"DNH4PGL6","t":"Can transparency undermine peer review? A simulation model of scientist behavior under open peer review","au":"Bianchi, Federico","y":2022,"vn":"Science and Public Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/scipol/scac027","cx":"Journal","co":["Bias","Interventions"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"hybrid","cby":21,"self":false,"s":"An agent-based simulation of scientist behaviour under open peer review suggests that when reviewers are sensitive to reputation and competition, increased transparency can paradoxically reduce review quality rather than improve it.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"","ca":"open","cau":"https://www.comses.net/codebases/3d99eb9f-ae4f-42d0-8c58-9d28757161c0/releases/1.0.0/"},{"k":"DRPY6FB5","t":"Contributions of reviewer judgements to editorial decision-making for the Canadian Journal of Behavioural Science: 1985–1986.","au":"Linden, Wolfgang","y":1992,"vn":"Canadian Journal of Behavioural Science/Revue canadienne des sciences du comportement","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/h0078757","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"closed","cby":2,"self":false,"s":"Analysis of 120 manuscripts at a behavioural science journal found low agreement between reviewers on specific quality criteria, though reviewer recommendations still predicted a large share of the editor's final decision.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"DS5G6JKI","t":"International peer review and decision-making practices, bias, and burden: The funders' perspective","au":"Meadmore K.","y":2019,"vn":"BMJ Evidence-Based Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmjebm-2019-EBMLive.19","cx":"Grant","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"bronze","cby":0,"self":false,"s":"A survey of health-research funders internationally examined which decision-making practices they use for grant allocation and how much attention has been paid to evaluating bias and reviewer burden in those approaches.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"DSQ6UWH2","t":"Who stands to win from double-blind peer review?","au":"Garvalov, Boyan K.","y":2015,"vn":"Advances in Regenerative Biology","ty":"journalArticle","tyl":"Journal article","doi":"10.3402/arb.v2.26879","cx":"Journal","co":["Bias","Interventions"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"hybrid","cby":7,"self":false,"s":"A commentary on the launch of a new biology journal, discussing the introduction of double-blind peer review and considering who benefits from it.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"DTFX59UM","t":"What Factors are Important to the Success of Resubmitted Grant Applications in Health Research? A Retrospective Study of Over 20,000 Applications to the Canadian Institutes of Health Research","au":"Wrightson, James G.","y":2024,"vn":"medRxiv","ty":"journalArticle","tyl":"Journal article","doi":"10.1101/2024.05.29.24308137","cx":"Grant","co":["Bias","Validity"],"sg":["proposal-quality","funding-decision"],"x":0,"st":"observational-quantitative","oast":"green","cby":4,"self":false,"s":"Analysing over 20,000 Canadian health-research grant applications, the study finds resubmissions succeed more often than new applications, with prior ranking, prior score and the applicant's history of other funded grants as the strongest predictors of success.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"open","cau":"https://osf.io/pw45z/"},{"k":"DU3KIZEC","t":"Journal Peer Review and Editorial Evaluation: Cautious Innovator or Sleepy Giant?","au":"Horbach, Serge P. J. M.","y":2019,"vn":"Minerva","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11024-019-09388-z","cx":"Journal","co":["Interventions"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":48,"self":false,"s":"Examines how widely journal editors have actually adopted proposed innovations in peer review procedures, finding adoption has been slow and largely confined to narrow niches, aside from plagiarism-detection tools.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"DUZT4KCB","t":"Feedback practices in journal peer-review: a systematic literature review","au":"Chong, Sin Wang","y":2023,"vn":"Assessment & Evaluation in Higher Education","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/02602938.2022.2164757","cx":"Journal","co":["Bias","Interventions"],"sg":["individual-scoring"],"x":0,"st":"systematic-review-meta-analysis","oast":"hybrid","cby":21,"self":false,"s":"Systematic review synthesizing what published peer-review feedback studies identify as effective versus poor feedback practices, and the personal and institutional factors shaping how reviewers give feedback.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"DVDHBSTH","t":"An experimental study of simulated grant peer review: Gender differences and psychometric characteristics of proposal scores","au":"Schmaling, Karen B.","y":2024,"vn":"PLOS ONE","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0315567","cx":"Grant","co":["Reliability-noise","Bias"],"sg":["individual-scoring"],"x":1,"st":"experimental-rct","oast":"gold","cby":0,"self":false,"s":"Experiment with 605 experienced peer reviewers scoring simulated NIH-style grant summaries, finding generally consistent scoring but noting gender-associated differences in how proposals and reviewers rated them.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://doi.org/10.17605/OSF.IO/92QEZ","ca":"no-statement","cau":""},{"k":"DVXN4LHZ","t":"Measuring bias, burden and conservatism in research funding processes","au":"Guthrie, Susan","y":2019,"vn":"F1000Research","ty":"journalArticle","tyl":"Journal article","doi":"10.12688/f1000research.19156.1","cx":"Grant","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"systematic-review-meta-analysis","oast":"gold","cby":36,"self":false,"s":"A review of international funder practices, combining a rapid evidence assessment and interviews, finds systematic measurement of bias, burden, and conservatism in grant funding processes remains limited, and proposes a broader framework for where these issues arise.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://doi.org/10.6084/m9.figshare.8184113.v1","ca":"no-statement","cau":""},{"k":"DX8A4GJ8","t":"Commensuration Bias in Peer Review","au":"Lee, Carole J.","y":2015,"vn":"Philosophy of Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1086/683652","cx":"General","co":["Bias","Interventions"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":88,"self":false,"s":"Introduces \"commensuration bias\" as a concept describing how converting varied review criteria into one overall quality score can systematically favor certain criteria, shaping funding and publication patterns.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"DYZ8TVFC","t":"A maturity model for the scientific review of clinical trial designs and their informativeness","au":"Dolley, Shawn","y":2024,"vn":"Trials","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s13063-024-08099-5","cx":"Grant","co":["Interventions"],"sg":["proposal-quality"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":6,"self":false,"s":"Proposes a maturity model with eleven process areas to help funders systematically improve scientific review of clinical trial design quality, addressing the common problem of trials that end without clear answers.","pg":"not-applicable","pgr":"","pgid":"","da":"open","dau":"https://clinicaltrials.gov/","ca":"no-statement","cau":""},{"k":"DZ9QX9UK","t":"European Research Council: excellence and leadership over time from a gender perspective","au":"Bautista‐Puig, Núria","y":2019,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvz023","cx":"Grant","co":["Bias"],"sg":["reviewer-selection","funding-decision"],"x":0,"st":"observational-quantitative","oast":"bronze","cby":22,"self":false,"s":"An analysis of European Research Council grants found women remain underrepresented as applicants, awardees, and panel members, with projections showing no clear improvement in gender balance over time.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"DZKVKM62","t":"Gender bias in funding evaluation: A randomized experiment","au":"Castro, Laura Cruz","y":2023,"vn":"Quantitative Science Studies","ty":"journalArticle","tyl":"Journal article","doi":"10.1162/qss_a_00263","cx":"Grant","co":["Bias"],"sg":["individual-scoring"],"x":1,"st":"experimental-rct","oast":"gold","cby":13,"self":false,"s":"An experiment embedded in a live funding call randomly varied whether a hypothetical proposal described a male or female principal investigator, finding no significant difference in the scores reviewers assigned by gender.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"no-statement","cau":""},{"k":"E28UBWHZ","t":"Double‐blind peer review—An experiment","au":"Fox, Charles W.","y":2019,"vn":"Functional Ecology","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/1365-2435.13269","cx":"Journal","co":["Bias","Interventions"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"bronze","cby":10,"self":false,"s":"An editorial from an ecology journal announcing a planned experiment testing alternative peer review models to address perceived systemic bias, alongside a summary of the journal's recent submission and publication statistics.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"E3U9A6YC","t":"A protocol of a cross-sectional study evaluating an online tool for early career peer reviewers assessing reports of randomised controlled trials.","au":"Chauvin, Anthony","y":2017,"vn":"BMJ Open","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmjopen-2017-017462","cx":"Journal","co":["Interventions"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":10,"self":false,"s":"Study protocol for a planned comparison of an online training tool against standard practice in helping early-career reviewers detect incomplete or altered outcome reporting in randomized trial manuscripts.","pg":"preregistered","pgr":"ClinicalTrials.gov","pgid":"NCT03119376","da":"open","dau":"","ca":"no-statement","cau":""},{"k":"E4VMBDGP","t":"The jury is out: a new approach to awarding science prizes","au":"Hill, Michael","y":2021,"vn":"F1000Research","ty":"journalArticle","tyl":"Journal article","doi":"10.12688/f1000research.75098.1","cx":"General","co":["Interventions"],"sg":["panel-consensus","funding-decision"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":3,"self":false,"s":"Describes a new evaluation procedure developed by the Swiss National Science Foundation for an academic prize, modeled on jury deliberation with anonymized documents, structured panel discussion, and ranking rather than rating.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"E4ZVSVLA","t":"A study of criteria for journal manuscripts.","au":"Wolff, Wirt M.","y":1970,"vn":"American Psychologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/h0029770","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":114,"self":false,"s":"An older paper studying criteria used to judge journal manuscripts calls for further research into the reliability of such judgments and comparisons between editors' and authors' views of manuscript quality.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"E5YQG6G8","t":"Development of the review quality instrument (RQI) for assessing peer reviews of manuscripts.","au":"van Rooyen, S","y":1999,"vn":"Journal of Clinical Epidemiology","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/s0895-4356(99)00047-5","cx":"Journal","co":["Reliability-noise","Validity"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":177,"self":false,"s":"Describes development and validation of the Review Quality Instrument, a seven-item scale for rating peer reviews, reporting strong internal consistency and good test-retest and inter-rater reliability.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"E69UQ2W5","t":"Are reviewer scores consistent with citations?","au":"Xie, Weixi","y":2024,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-024-05103-2","cx":"Journal","co":["Reliability-noise","Validity"],"sg":["individual-scoring","post-award-validity"],"x":0,"st":"observational-quantitative","oast":"closed","cby":2,"self":false,"s":"Analysis of machine-learning conference reviews found peer-review scores correlated positively but only moderately with later citation counts, and papers with more disagreement among reviewers tended to be cited more.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"E6NDGFIC","t":"Personal attributes of authors and reviewers, social bias and the outcomes of peer review: a case study.","au":"Walker, Richard","y":2015,"vn":"F1000Research","ty":"journalArticle","tyl":"Journal article","doi":"10.12688/f1000research.6012.2","cx":"Journal","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":55,"self":false,"s":"Analyzing three peer-review datasets, this study found little evidence of gender or institutional-prestige bias by reviewers but some evidence of regional bias, with author characteristics showing stronger effects than reviewer characteristics.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"no-statement","cau":""},{"k":"E6QJ72I8","t":"Prospective analyses of sex/gender-related publication decisions in general medical journals: editorial rejection of population-based women's reproductive physiology.","au":"Kalidasan, Dharani","y":2022,"vn":"BMJ open","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmjopen-2021-057854","cx":"Journal","co":["Bias"],"sg":["funding-decision"],"x":0,"st":"observational-quantitative","oast":"gold","cby":11,"self":false,"s":"Tracking editorial rejections at major medical journals, the study finds men substantially outnumbered women as authors of published clinical research, and found no non-disease-focused articles on women's reproductive health despite topic relevance to both sexes.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"no-statement","cau":""},{"k":"E7836ZR3","t":"Using ChatGPT to Evaluate the Methodological Components of Research Proposals: An Experimental Study on Undergraduate English Majors in Vietnam","au":"Tran, The Phi","y":2025,"vn":"Computer-Assisted Language Learning Electronic Journal","ty":"journalArticle","tyl":"Journal article","doi":"10.54855/callej.252637","cx":"Analogue","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":1,"self":false,"s":"Tests ChatGPT-4o's consistency in scoring undergraduate research proposals against human lecturer ratings, finding moderate agreement on straightforward criteria but weaker performance on more abstract, judgment-heavy evaluation areas.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"E7IAN24K","t":"A year in metascience (2025): the past, present and future of UK metascience","au":"UK Metascience Unit","y":2025,"vn":"","ty":"report","tyl":"Report","doi":"","cx":"Grant","co":["Reliability-noise","Interventions"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"","cby":0,"self":false,"s":"Commentary discussing UK funders' difficulties recruiting enough peer reviewers and criticisms of multi-stage application processes intended to reduce reviewer burden.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"E7VGIDPE","t":"Automated scholarly paper review: Concepts, technologies, and challenges","au":"Lin, Jialiang","y":2023,"vn":"Information Fusion","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.inffus.2023.101830","cx":"Journal","co":["Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":31,"self":false,"s":"Reviews the concept, technologies, and current limitations of fully automated scholarly paper review using artificial intelligence, identifying gaps in data, reasoning, and human-computer interaction.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"E88CKB29","t":"Pork Barrel or Barrel of Gold? Examining the performance implications of earmarking in public R&D grants","au":"Kok, Holmer","y":2022,"vn":"Research Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.respol.2022.104514","cx":"Grant","co":["Bias","Validity"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":3,"self":false,"s":"A study of 321 US Department of Energy Hydrogen Program R&D projects compares congressionally earmarked funding against competitive peer review, finding earmarked projects scored lower in review despite not underperforming on research outcomes, suggesting reviewer bias against earmarks.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"","ca":"no-statement","cau":""},{"k":"E8UCA4GJ","t":"Author‐suggested reviewers: gender differences and influences on the peer review process at an ecology journal","au":"Fox, Charles W.","y":2016,"vn":"Functional Ecology","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/1365-2435.12665","cx":"Journal","co":["Bias"],"sg":["reviewer-selection"],"x":0,"st":"observational-quantitative","oast":"bronze","cby":43,"self":false,"s":"Study of an ecology journal's author-suggested reviewer system found most authors nominate preferred reviewers, mostly male, though the share of women nominated grew over eleven years and female nominees were more often selected.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"http://dx.doi.org/10.5061/dryad.37312","ca":"no-statement","cau":""},{"k":"E93QFXS4","t":"Trust, transparency, and disciplinary alignment in research funding evaluations: an illustrative case from the Norwegian AI Center call (2024–2025)","au":"Krumsvik, Rune Johan","y":2026,"vn":"Frontiers in Research Metrics and Analytics","ty":"journalArticle","tyl":"Journal article","doi":"10.3389/frma.2026.1874986","cx":"Grant","co":["Bias"],"sg":["panel-consensus","funding-decision"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":0,"self":false,"s":"A perspective piece uses one rejected proposal from a Norwegian national AI-funding call as a case study to discuss transparency, disciplinary fit between reviewers and applicants, and fairness in evaluation panels.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"E96MVBF3","t":"Negotiating science funding: The interplay of merit, bias, and administrative discretion in grant allocation in Kazakhstan","au":"Medeuov, Darkhan","y":2025,"vn":"PLoS ONE","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0318875","cx":"Grant","co":["Bias"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"gold","cby":1,"self":false,"s":"Analyzing thousands of Kazakhstani grant applications, this study found review scores explained little variation in funding success, with institutional proximity to decision-makers and, in one research domain, gender playing larger roles.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://doi.org/10.7910/DVN/8UZ7EY","ca":"open","cau":"https://doi.org/10.7910/DVN/8UZ7EY"},{"k":"E9LBD95C","t":"Why the referees’ reports I receive as an editor are so much better than the reports I receive as an author?","au":"García, J. A.","y":2016,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-015-1827-8","cx":"Journal","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"green","cby":7,"self":false,"s":"A Bayesian model and an author survey explain why researchers, acting as authors, tend to attribute rejections to reviewer bias while crediting acceptances to their own ability, a self-serving asymmetry that also affects how much they learn from criticism.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"EBBEPJCN","t":"Single- versus double-blind reviewing","au":"Snodgrass, Richard T.","y":2006,"vn":"ACM SIGMOD Record","ty":"journalArticle","tyl":"Journal article","doi":"10.1145/1168092.1168094","cx":"Journal","co":["Bias","Validity","Interventions"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"bronze","cby":123,"self":false,"s":"A short piece introducing the debate between single-blind and double-blind manuscript review practices and their implications for fairness in scholarly publishing.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"EBE9WAEU","t":"Strategyproof peer selection using randomization, partitioning, and apportionment","au":"Aziz, Haris","y":2019,"vn":"Artificial Intelligence","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.artint.2019.06.004","cx":"General","co":["Interventions"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"simulation-methodological","oast":"green","cby":10,"self":false,"s":"Proposes and tests mechanisms for peer selection (e.g., awarding grants or prizes based on member votes) designed so that participants cannot benefit by misrepresenting their true evaluations.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"open","cau":"https://github.com/nmattei/peerselection"},{"k":"EBJ79HYD","t":"Talent Identification at the limits of Peer Review: an analysis of the EMBO Postdoctoral Fellowships Selection Process","au":"Klaus, Bernd","y":2018,"vn":"bioRxiv (Cold Spring Harbor Laboratory)","ty":"journalArticle","tyl":"Journal article","doi":"10.1101/481655","cx":"Grant","co":["Bias","Validity"],"sg":["funding-decision","post-award-validity"],"x":0,"st":"observational-quantitative","oast":"green","cby":12,"self":false,"s":"Analysis of EMBO postdoctoral fellowship selection from 2007, comparing award decisions to applicants' career progress a decade later and concluding peer review performed little better than random selection among high-quality applicants.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://git.embl.de/klaus/embo_ltf_analysis","ca":"open","cau":"https://git.embl.de/klaus/embo_ltf_analysis"},{"k":"EC8W23YY","t":"Gender, race, and grant reviews: Translating and responding to research feedback.","au":"Biernat, Monica","y":2020,"vn":"Personality and Social Psychology Bulletin","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/0146167219845921","cx":"Grant","co":["Bias"],"sg":["individual-scoring"],"x":1,"st":"experimental-rct","oast":"bronze","cby":23,"self":false,"s":"An experiment with NIH career-award recipients finds women interpret both encouraging and discouraging hypothetical grant feedback more negatively than men do, with this interpretation pattern linked to reduced motivation to resubmit after critical reviews.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"ED6VSDD5","t":"Exploring Bias in Scientific Peer Review: An ASCO Initiative","au":"McKenzie, Nathalie D.","y":2022,"vn":"JCO Oncology Practice","ty":"journalArticle","tyl":"Journal article","doi":"10.1200/op.22.00275","cx":"General","co":["Bias","Interventions"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"qualitative","oast":"closed","cby":7,"self":false,"s":"ASCO working group combined literature review, internal data, interviews, and a member survey to examine implicit bias in its peer review, finding disproportionate representation of men and prestigious institutions among reviewers and awardees.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"EFBDCCW7","t":"Estimating Interrater Agreement with the Average Deviation Index: A User’s Guide","au":"Burke, Michael J.","y":2002,"vn":"Organizational Research Methods","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/1094428102005002002","cx":"Analogue","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"closed","cby":443,"self":false,"s":"Proposes practical guidelines and numerical cutoffs for interpreting the average deviation index, a common statistic for judging whether raters agree closely enough on rating-scale items.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"EFWSGJML","t":"A comparison of Cohen's Kappa and Gwet's AC1 when calculating inter-rater reliability coefficients: a study conducted with personality disorder samples.","au":"Wongpakaran, Nahathai","y":2013,"vn":"BMC Medical Research Methodology","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/1471-2288-13-61","cx":"Analogue","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":983,"self":false,"s":"Comparing two statistical measures of rater agreement using personality-disorder diagnoses from nine clinicians, this study found Gwet's AC1 produced more stable reliability estimates than Cohen's Kappa, which varied more with diagnosis prevalence.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"EGVIHGTT","t":"Peer review reduces spin in PCORI research reports","au":"Mayo-Wilson, Evan","y":2021,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31222/osf.io/xknhp","cx":"Grant","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":0,"self":false,"s":"Reviewing PCORI-funded research reports, the study finds that peer review identified misleading \"spin,\" such as biased reporting or overreaching conclusions, in the large majority of submitted reports, though similar spin often persisted in related published journal articles.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://osf.io/fh7cz/","ca":"no-statement","cau":""},{"k":"EIPTF2DN","t":"Conservatism Gets Funded? A Field Experiment on the Role of Negative Information in Novel Project Evaluation","au":"Lane, Jacqueline N.","y":2021,"vn":"Management Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1287/mnsc.2021.4107","cx":"Grant","co":["Bias"],"sg":["panel-consensus","individual-scoring"],"x":1,"st":"experimental-rct","oast":"green","cby":55,"self":false,"s":"A field experiment with hundreds of grant evaluators finds a negativity bias: seeing lower scores from other reviewers pushed evaluators to cut their own scores more than seeing higher scores pushed them to raise them.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://doi.org/10.1287/mnsc.2021.4107","ca":"no-statement","cau":""},{"k":"EJFLDVJU","t":"Retrospective analysis of the quality of reports by author-suggested and non-author-suggested reviewers in journals operating on open or single-blind peer review models","au":"Kowalczuk, Maria","y":2015,"vn":"BMJ Open","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmjopen-2015-008707","cx":"Journal","co":["Reliability-noise","Bias"],"sg":["reviewer-selection","individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":61,"self":false,"s":"Retrospective comparison finding author-suggested reviewers produced reports of similar quality to other reviewers but were substantially more likely to recommend acceptance, regardless of the journal's peer-review model.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"no-statement","cau":""},{"k":"EJTCGZTW","t":"A Generalized Kappa Coefficient","au":"Uebersax, John S.","y":1982,"vn":"Educational and Psychological Measurement","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/0013164482421018","cx":"Analogue","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"closed","cby":76,"self":false,"s":"Presents a generalized computational method for calculating the kappa agreement statistic across more varied and flexible multi-rater study designs than previous formulas allowed.","pg":"not-codeable","pgr":"","pgid":"","da":"","dau":"","ca":"","cau":""},{"k":"EK4WFSX9","t":"Is the quality of reviews reflected in editors' and authors' satisfaction with peer review? A cross‐sectional study in 12 journals across four research fields","au":"Pranić, Shelly Melissa","y":2020,"vn":"Learned Publishing","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/leap.1344","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":18,"self":false,"s":"A cross-sectional study of 12 Elsevier journals examines whether measured review quality aligns with authors' and editors' satisfaction, finding authors were most satisfied with reviews recommending acceptance, while revision-recommending reviews scored highest on a quality instrument.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"EQNEHVU3","t":"Does the Gender Composition of Scientific Committees Matter?","au":"Bagues, Manuel","y":2017,"vn":"American Economic Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1257/aer.20151211","cx":"General","co":["Bias","Validity"],"sg":["panel-consensus"],"x":0,"st":"observational-quantitative","oast":"bronze","cby":274,"self":false,"s":"Study of 100,000 professorship applications in Italy and Spain finds adding women to evaluation committees did not increase female candidates' success, though male evaluators became less favorable to women once a female evaluator joined.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"ES4ZGEIL","t":"Peer-review for the peer-review system","au":"Ruhli F.J.","y":2009,"vn":"Human Ontogenetics","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/huon.200900004","cx":"Journal","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"hybrid","cby":8,"self":false,"s":"An opinion piece argues that anonymous peer review can let reviewers suppress work that challenges their own views, and advocates moving toward fully open, transparent review.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"EZ496VHZ","t":"Peer review across borders: benefits and challenges of international review panels in research funding organizations","au":"Peterson, Helen","y":2024,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvaf030","cx":"Grant","co":["Reliability-noise","Bias"],"sg":["reviewer-selection","individual-scoring"],"x":0,"st":"qualitative","oast":"hybrid","cby":1,"self":false,"s":"Interviews with staff and panel members at a national funding organization explore the tradeoffs of replacing national review panels with international ones, finding benefits for impartiality and diversity but risks to reviewer agreement and local context.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"EZA7L96Y","t":"Nobel and novice: Author prominence affects peer review.","au":"Huber, Jürgen","y":2022,"vn":"Proceedings of the National Academy of Sciences of the United States of America","ty":"journalArticle","tyl":"Journal article","doi":"10.1073/pnas.2205779119","cx":"Journal","co":["Bias"],"sg":["individual-scoring"],"x":1,"st":"experimental-rct","oast":"hybrid","cby":127,"self":false,"s":"A large preregistered field experiment shows reviewers were far more likely to recommend rejecting a finance paper when only the early-career author's name was visible than when a Nobel laureate co-author's name was shown, demonstrating strong status bias.","pg":"preregistered","pgr":"OSF","pgid":"https://osf.io/mjh8s/","da":"open","dau":"https://osf.io/mjh8s/","ca":"open","cau":"https://osf.io/mjh8s/"},{"k":"EZNFKUCC","t":"Is There Gender Bias in the Peer Review Process at Journal of Neurophysiology?","au":"Lane, John A.S.","y":2009,"vn":"Journal of Neurophysiology","ty":"journalArticle","tyl":"Journal article","doi":"10.1152/jn.00196.2009","cx":"Journal","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":16,"self":false,"s":"An editorial examining whether gender bias exists in the peer review process at a neurophysiology journal, prompted by broader concerns about the underrepresentation of women in senior scientific positions.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"","ca":"no-statement","cau":""},{"k":"F29KWSCM","t":"Gender-equal funding rates conceal unequal evaluations","au":"Bol, Thijs","y":2021,"vn":"Research Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.respol.2021.104399","cx":"Grant","co":["Bias"],"sg":["individual-scoring","panel-consensus","funding-decision"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":68,"self":false,"s":"Analysis of a Dutch funding competition's scoring data finding men received higher initial evaluation scores than women, while panel-level adjustments then redistributed funding to restore equal outcomes overall.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"no-statement","cau":""},{"k":"F2AW6R6R","t":"The validation of peer review through research impact measures and the implications for funding strategies.","au":"Gallo, Stephen A","y":2014,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0106474","cx":"Grant","co":["Validity"],"sg":["funding-decision","post-award-validity"],"x":0,"st":"observational-quantitative","oast":"gold","cby":49,"self":false,"s":"Retrospective analysis of over 2,000 grant applications and their bibliometric output over eight years, finding peer review scores moderately correlated with later citation impact but with considerable variability.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"","ca":"no-statement","cau":""},{"k":"F2TL4U3X","t":"Peer review and publication. Presidential address before the 70th annual meeting of the American Society for Clinical Investigation, San Francisco, California, 30 April 1978.","au":"Wilson, Jess","y":1978,"vn":"Journal of Clinical Investigation","ty":"journalArticle","tyl":"Journal article","doi":"10.1172/jci109091","cx":"Journal","co":["Reliability-noise","Bias","Validity"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"bronze","cby":105,"self":false,"s":"A 1978 presidential address reflects on the history and fairness of journal manuscript peer review, questioning both its institutional legitimacy and what the process actually achieves.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"F46K5IYY","t":"Gatekeeping Without Peer Review","au":"Tarr, Michael","y":2024,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31234/osf.io/g75c4","cx":"Journal","co":["Bias","Interventions"],"sg":["funding-decision"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":0,"self":false,"s":"Commentary calling for journals to improve pre-review editorial screening (desk rejection) alongside efforts to make full peer review fairer.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"F4C6TEFK","t":"Meta-research: justifying career disruption in funding applications, a survey of Australian researchers","au":"Barnett, Adrian","y":2022,"vn":"eLife","ty":"journalArticle","tyl":"Journal article","doi":"10.7554/elife.76123","cx":"Grant","co":["Reliability-noise","Bias"],"sg":["individual-scoring","proposal-quality"],"x":0,"st":"observational-quantitative","oast":"gold","cby":8,"self":false,"s":"A survey of Australian researchers found many hesitate to disclose career disruptions such as illness or childbirth in grant applications for fear it will hurt their chances, and reviewers applied inconsistent adjustments when disruptions were disclosed.","pg":"preregistered","pgr":"other","pgid":"","da":"open","dau":"https://github.com/agbarnett/career_disruption","ca":"open","cau":"https://github.com/agbarnett/career_disruption"},{"k":"F68GYWKS","t":"Improving the peer-review process for grant applications: reliability, validity, bias, and generalizability.","au":"Marsh, Herbert W","y":2008,"vn":"The American Psychologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/0003-066X.63.3.160","cx":"Grant","co":["Reliability-noise","Bias","Validity","Interventions"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":300,"self":false,"s":"Summarizing a research program with the Australian Research Council, this paper reports that grant peer reviews showed poor reliability and little systematic bias except for inflated scores from reviewers nominated by applicants, proposing an alternative \"reader system.\"","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"F6AMXHF7","t":"Gender differences in peer reviews of grant applications: A substantive-methodological synergy in support of the null hypothesis model","au":"Marsh, Herbert W.","y":2011,"vn":"Journal of Informetrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.joi.2010.10.004","cx":"Grant","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":50,"self":false,"s":"Analysing over 10,000 reviews of grant proposals across disciplines using multilevel statistical models, the study finds no meaningful effect of principal investigator gender on review scores, a result that held across reviewer gender, discipline and country.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"F6KINVIB","t":"Suggesting or Excluding Reviewers Can Help Get Your Paper Published","au":"Grimm, David","y":2005,"vn":"Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1126/science.309.5743.1974","cx":"Journal","co":["Bias"],"sg":["reviewer-selection"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":40,"self":false,"s":"A news piece on how journals' practice of letting authors suggest or exclude specific reviewers can influence a manuscript's chances of publication.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"F83SPSMN","t":"Peer review and innovation","au":"Spier, R.E.","y":2002,"vn":"Science and Engineering Ethics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11948-002-0035-0","cx":"General","co":["Bias","Validity"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"bronze","cby":48,"self":false,"s":"Discusses tension between peer review and truly innovative work in both journal publishing and grant funding, arguing that some portion of research funds should be allocated by alternative mechanisms.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"F8KRUZLI","t":"Reviewing peer review at the NIH.","au":"Lauer, Michael S","y":2015,"vn":"The New England Journal of Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.1056/NEJMp1507427","cx":"Grant","co":["Validity"],"sg":["post-award-validity"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":21,"self":false,"s":"Short piece discussing reports that NIH peer review scores are weak predictors of a funded project's eventual scientific impact, while cautioning that this does not necessarily mean the review system has failed.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"F9GMEZ7U","t":"AI-assisted prescreening of biomedical research proposals: ethical considerations and the pilot case of “la Caixa” Foundation","au":"Cortes, Carla M.","y":2024,"vn":"Data & Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/dap.2024.41","cx":"Grant","co":["Bias","Interventions"],"sg":["proposal-quality"],"x":0,"st":"observational-quantitative","oast":"gold","cby":12,"self":false,"s":"A case study examines a Spanish foundation's pilot use of an AI algorithm to flag immature biomedical grant proposals before human review, assessing the ethical conditions for responsible AI use in funding pipelines.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"no-statement","cau":""},{"k":"F9HSQVFX","t":"Is there a gender gap in chemical sciences scholarly communication?","au":"Day, A E","y":2020,"vn":"Chemical science","ty":"journalArticle","tyl":"Journal article","doi":"10.1039/c9sc04090k","cx":"Journal","co":["Bias"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"diamond","cby":56,"self":false,"s":"Analysis of Royal Society of Chemistry journal data from 2014-2018 finds the proportion of women decreases at each stage from submitting authors through to cited authors, with women's success rates lower at every stage.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://bitbucket.org/rscapplications/genderdiversity/src/master/","ca":"open","cau":"https://bitbucket.org/rscapplications/genderdiversity/src/master/"},{"k":"F9U369U4","t":"Potential sources of bias in research fellowship assessments: effects of university prestige and field of study","au":"Bornmann, Lutz","y":2006,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.3152/147154406781775850","cx":"Grant","co":["Bias"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"closed","cby":41,"self":false,"s":"Analysis of a biomedical research foundation's fellowship decisions found evidence that certain scientific fields were favored over others, but no evidence that applicants from prestigious universities received an advantage.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"F9YGPA3S","t":"‘Science by consensus’ impedes scientific creativity and progress: An alternative to funding biomedical research","au":"Düzgüneş, Nejat","y":2022,"vn":"F1000Research","ty":"journalArticle","tyl":"Journal article","doi":"10.12688/f1000research.124082.1","cx":"Grant","co":["Reliability-noise","Bias","Validity","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":3,"self":false,"s":"An opinion piece criticizing NIH and NSF grant peer review as an unproven \"consensus\" system, proposing an alternative that funds many more investigators using simpler procedures without traditional review.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"FAWCL6KR","t":"Development of a rating form to evaluate grant applications to the Hogg Foundation for Mental Health.","au":"Whaley, Arthur L","y":2006,"vn":"Evaluation Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/0193841X05275586","cx":"Grant","co":["Reliability-noise","Validity"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":5,"self":false,"s":"Describes how a mental-health foundation developed and tested a structured rating form for grant applications, finding good internal consistency and interrater reliability among independent raters compared with the foundation's prior subjective review process.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"FBSYGXK4","t":"Development and assessment of criteria to select projects for funding in the EU health programme.","au":"Van den Broucke, Stephan","y":2012,"vn":"European journal of public health","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/eurpub/ckr066","cx":"Grant","co":["Validity","Interventions"],"sg":["individual-scoring","panel-consensus"],"x":0,"st":"observational-quantitative","oast":"bronze","cby":7,"self":false,"s":"Describes the development and validation of a scoring checklist used to evaluate EU public health grant applications across relevance, technical quality, and management dimensions, finding all three contributed to funding decisions.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"FDNSXEWP","t":"Selection of research fellowship recipients by committee peer review. Reliability, fairness and predictive validity of Board of Trustees' decisions","au":"Bornmann, Lutz","y":2005,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-005-0214-2","cx":"Grant","co":["Reliability-noise","Bias","Validity"],"sg":["individual-scoring","funding-decision","post-award-validity"],"x":0,"st":"observational-quantitative","oast":"green","cby":128,"self":false,"s":"Large-scale study of a biomedical fellowship funder's selection procedure assessing reviewer agreement, fairness across applicant characteristics, and whether review scores predicted later research success.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"FE53H48F","t":"Questionable research practices in competitive grant funding: A survey.","au":"Conix, Stijn","y":2023,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0293310","cx":"Grant","co":["general-systemic"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"gold","cby":2,"self":false,"s":"Pre-registered survey of grant applicants, reviewers, and panel members at a Belgian funding agency finding that questionable research practices in the funding context were widespread and that many respondents doubted the reliability of grant peer review.","pg":"preregistered","pgr":"OSF","pgid":"https://osf.io/jk6wd/","da":"open","dau":"https://zenodo.org/record/7673890","ca":"open","cau":"https://osf.io/jk6wd/"},{"k":"FEDSMXXE","t":"Ladies, Gentlemen, and Scientific Publication at the Royal Society, 1945–1990","au":"Røstvik, Camilla Mørk","y":2018,"vn":"Open Library of Humanities","ty":"journalArticle","tyl":"Journal article","doi":"10.16995/olh.265","cx":"Journal","co":["Bias"],"sg":["reviewer-selection"],"x":0,"st":"observational-quantitative","oast":"diamond","cby":11,"self":false,"s":"An archival study of the Royal Society (1945-1990) traces women's participation as authors, referees, and editors, finding representation in both visible and anonymous roles was higher in the 1950s than in later decades.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"FEGJPLXX","t":"Developing a rubric to distinguish translational science from translational research in CTSA pilot projects","au":"Dillon P.","y":2023,"vn":"Journal of Clinical and Translational Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/cts.2023.437","cx":"Grant","co":["Reliability-noise","Interventions"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":1,"self":false,"s":"Describes development of a 31-item rubric for distinguishing translational science from translational research proposals within a CTSA pilot grant review consortium, to be assessed for inter-rater reliability.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"FFGAWNZF","t":"Tres siglos después. . . ¿Es vigente el arbitraje por pares en las publicaciones científicas?","au":"Sepúlveda-Vildósola, Ana Carolina","y":2015,"vn":"Investigación en Educación Médica","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.riem.2015.09.001","cx":"Journal","co":["Reliability-noise","Bias","Validity","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"diamond","cby":3,"self":false,"s":"A narrative review traces roughly three centuries of peer review history in scientific publishing, discussing its persistent criticisms and emerging alternatives.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"FGY2CT6A","t":"Does double-blind peer review effectively correct for gender disparities in research funding?","au":"Madsen, Emil Bargmann","y":2024,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31235/osf.io/uw3xe","cx":"Grant","co":["Bias","Interventions"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"gold","cby":1,"self":false,"s":"Analyzing over 2,000 Villum Foundation grant applications under a double-blind review scheme, this study found small gender differences in funding persisted, attributable to disciplinary composition and underrepresentation of experienced women applicants.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"","ca":"open","cau":""},{"k":"FJLLQTJJ","t":"Measuring nominal scale agreement among many raters.","au":"Fleiss, Joseph L.","y":1971,"vn":"Psychological Bulletin","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/h0031619","cx":"Analogue","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"closed","cby":8552,"self":false,"s":"Extends Cohen's kappa statistic to settings with more than two raters per subject, where different subjects may be rated by different, non-overlapping sets of raters, deriving standard errors and illustrating the method with an example.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"FLGSGDMR","t":"A randomized trial of an editorial intervention to reduce spin in the abstract's conclusion of manuscripts showed no significant effect.","au":"Ghannad, Mona","y":2021,"vn":"Journal of Clinical Epidemiology","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.jclinepi.2020.10.014","cx":"Journal","co":["Bias","Interventions"],"sg":["proposal-quality"],"x":1,"st":"experimental-rct","oast":"hybrid","cby":26,"self":false,"s":"A randomized controlled trial testing whether giving authors short instructions during peer review reduces exaggerated (\"spin\") language in revised manuscript abstracts, finding no statistically significant reduction.","pg":"preregistered","pgr":"OSF","pgid":"osf.io/xnuyt","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"FM5RIB4P","t":"An Overview of Post-Publication Peer Review","au":"O’Sullivan, Lydia","y":2021,"vn":"Scholarly Assessment Reports","ty":"journalArticle","tyl":"Journal article","doi":"10.29024/sar.26","cx":"General","co":["general-systemic"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"diamond","cby":26,"self":false,"s":"Narrative review describing forms of post-publication peer review (letters, blogs, platforms) and weighing their speed and openness advantages against risks of disseminating unvetted findings.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"FPITCKY4","t":"Evaluation of stakeholder views on peer review of NIHR applications for funding: a qualitative study.","au":"Turner, Sheila","y":2018,"vn":"BMJ Open","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmjopen-2018-022548","cx":"Grant","co":["Interventions"],"sg":["whole-process"],"x":0,"st":"qualitative","oast":"gold","cby":16,"self":false,"s":"Qualitative interview study with UK National Institute for Health Research stakeholders identifying suggested improvements to grant peer review, including more proportionate processes, clearer guidance, and structured feedback for reviewers.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"not-applicable","cau":""},{"k":"FPWBZM9M","t":"Influence of peer review on the reporting of primary outcome(s) and statistical analyses of randomised trials.","au":"Hopewell, Sally","y":2018,"vn":"Trials","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s13063-017-2395-4","cx":"Journal","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":10,"self":false,"s":"A survey-based study of authors of randomized trials in high-impact journals examines how often peer review prompted changes to reporting of primary outcomes and statistical analyses, finding requests for statistical clarification more common than outcome changes, with high author satisfaction overall.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"no-statement","cau":""},{"k":"FQMUE9X2","t":"Targeted, actionable and fair: Reviewer reports as feedback and its effect on ECR career choices.","au":"Derrick, Gemma Elizabeth","y":2023,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvad034","cx":"Grant","co":["Interventions"],"sg":["individual-scoring"],"x":0,"st":"qualitative","oast":"hybrid","cby":7,"self":false,"s":"Interviews with unsuccessful early-career Wellcome Trust applicants and coded reviewer comments identify actionable, targeted, and fair feedback as the characteristics that motivate applicants to reapply rather than leave academia.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"FS9SHXLZ","t":"The consequences of competition: simulating the effects of research grant allocation strategies","au":"Höylä, Tuomas","y":2016,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-016-1940-3","cx":"Grant","co":["Reliability-noise","Validity"],"sg":["funding-decision"],"x":0,"st":"simulation-methodological","oast":"closed","cby":16,"self":false,"s":"A simulation model of proposal review and funding compares different resource-allocation strategies, showing that policy choice strongly affects researcher career outcomes and examining how review errors and writing skill interact with these effects.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"open","cau":"www.bartneck.de/publications/2016/PEGS/index.html"},{"k":"FSW6MJXT","t":"Quantitative evaluation of peer review of program project and center applications in allergy and immunology.","au":"Das, N K","y":1985,"vn":"Journal of clinical immunology","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/BF00929456","cx":"Grant","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":4,"self":false,"s":"Comparing review outcomes for large multi-investigator grants versus individual research grants in immunology, this study found high consistency in approval rates, scores, and reviewer consensus between the two review formats.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"FT37VU8M","t":"Evaluation for the allocation of university research project funding: Can rules improve the peer review?","au":"Reale, Emanuela","y":2017,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvx019","cx":"Grant","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"qualitative","oast":"closed","cby":29,"self":false,"s":"A case study of an Italian national research-funding scheme shows that repeated rule changes intended to make ex-ante grant evaluation more accountable had little effect on how peer-review panels actually behaved in practice.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"FUM3UBJI","t":"Problems with Peer Review Shine a Light on Gaps in Scientific Training","au":"Proctor, Diana M.","y":2023,"vn":"mBio","ty":"journalArticle","tyl":"Journal article","doi":"10.1128/mbio.03183-22","cx":"Journal","co":["Reliability-noise","Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":21,"self":false,"s":"An opinion piece arguing that persistent problems in peer review, such as failure to catch flawed statistics or bias, reflect broader gaps in how scientists are trained to evaluate research.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"FV9M3PCW","t":"Gatekeeping should be conserved in the open science era","au":"Desmond, Hugh","y":2024,"vn":"Synthese","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11229-024-04559-2","cx":"Journal","co":["Bias"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"hybrid","cby":10,"self":false,"s":"Argues that removing publication gatekeepers would not eliminate prestige hierarchies but merely shift them onto citation counts or search-ranking algorithms, which carry their own biases.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"FXQ5RBYN","t":"Ideology Critique, Representational Pluralism, and the Abolition of Peer Review. A Marxist Addendum to McKeown’s Criticism of Analytic Philosophy","au":"Jeuk, Alexander Albert","y":2022,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31219/osf.io/x4b2e","cx":"Journal","co":["Bias"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":1,"self":false,"s":"Philosophical essay applying Marxist ideology critique to analytic philosophy's academic dominance and arguing for abolishing peer review and journal editing as opaque, corruptible mechanisms of institutional power.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"G2B4D2J6","t":"Peer reviewers’ dilemmas: a qualitative exploration of decisional conflict in the evaluation of grant applications in the medical humanities and social sciences","au":"Vallée-Tourangeau, Gaëlle","y":2022,"vn":"Humanities and Social Sciences Communications","ty":"journalArticle","tyl":"Journal article","doi":"10.1057/s41599-022-01050-6","cx":"Grant","co":["Reliability-noise","Bias"],"sg":["individual-scoring"],"x":0,"st":"qualitative","oast":"gold","cby":12,"self":false,"s":"Think-aloud interviews with 16 grant reviewers in medical humanities and social sciences identify five recurring ethical dilemmas reviewers face, showing idiosyncratic personal resolutions that may contribute to variation in review outcomes.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"http://data-archive.ac.uk","ca":"no-statement","cau":""},{"k":"G526NSHB","t":"The role of metrics in peer assessments","au":"Langfeldt, Liv","y":2020,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvaa032","cx":"General","co":["general-systemic"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":46,"self":false,"s":"Cross-country survey of researchers in three fields finds most consider publication and citation metrics important when reviewing grant proposals and job candidates, even though few view metrics as defining the \"best\" research.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"","ca":"no-statement","cau":""},{"k":"G6BW4RFI","t":"Using peer review to evaluate the societal relevance of humanities research","au":"Conix, Stijn","y":2025,"vn":"Quantitative Science Studies","ty":"journalArticle","tyl":"Journal article","doi":"10.1162/qss.a.19","cx":"General","co":["Reliability-noise","Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":0,"self":false,"s":"A study asked early-career humanities researchers to rate the societal relevance of hundreds of published articles, finding reviewer identity mattered more than article content in shaping ratings, and suggesting much published humanities work may not register as societally relevant.","pg":"preregistered","pgr":"OSF","pgid":"https://doi.org/10.17605/OSF.IO/TNSFZ","da":"open","dau":"https://doi.org/10.5281/zenodo.7768577","ca":"open","cau":"https://doi.org/10.17605/OSF.IO/TNSFZ"},{"k":"G6K4RHJ6","t":"Grant Peer Review: Improving Inter-Rater Reliability with Training.","au":"Sattler, David N","y":2015,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0130450","cx":"Grant","co":["Reliability-noise","Interventions"],"sg":["individual-scoring"],"x":1,"st":"experimental-rct","oast":"gold","cby":86,"self":false,"s":"A training study with 75 public health professors found that a short training video improved scoring accuracy and interrater reliability for grant review, for both novice and experienced reviewers.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"http://figshare.com/authors/Patrick_Mcknight/695619","ca":"no-statement","cau":""},{"k":"G7V7A6A4","t":"Bias in peer review: a case study","au":"Walker, Richard","y":2015,"vn":"F1000Research","ty":"journalArticle","tyl":"Journal article","doi":"10.12688/f1000research.6012.1","cx":"Journal","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":16,"self":false,"s":"Statistical modelling of three peer-review datasets, from an open-review publisher and two computer-science conferences, finds little evidence of gender or institutional-prestige bias by reviewers, though weak regional bias appears across all three.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"no-statement","cau":""},{"k":"G87QSGJQ","t":"Comparing human and AI expertise in the academic peer review process: towards a hybrid approach","au":"Farber, Shai","y":2025,"vn":"Higher Education Research & Development","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/07294360.2024.2445575","cx":"Journal","co":["Reliability-noise","Bias","Interventions"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":22,"self":false,"s":"A comparative study contrasting human and AI (Claude-3) reviews of ten unpublished humanities and social science papers, finding AI stronger on efficiency and consistency while humans excelled at contextual and ethical judgment, and proposing a hybrid review approach.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"G8RWYA8C","t":"What professors do in peer review: Interrogating assessment practices in the recruitment of professors in Sweden.","au":"Mahlck, Paula","y":2020,"vn":"Gender, Work and Organization","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/gwao.12500","cx":"General","co":["Reliability-noise","Bias"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"qualitative","oast":"hybrid","cby":9,"self":false,"s":"Multi-method study of professor recruitment at a Swedish university finding wide variation in how evaluation criteria were applied, frequent reviewer disagreement, and bias against applicants from non-Western institutions.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"G9XZ5UV7","t":"A Proposal for More Informative Abstracts of Clinical Articles","au":"LITERATURE, AD HOC WORKING GROUP FOR CRITICAL APPRAISAL OF THE MEDICAL","y":1987,"vn":"Annals of Internal Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.7326/0003-4819-106-4-598","cx":"Journal","co":["Interventions"],"sg":["proposal-quality"],"x":0,"st":"guideline-policy","oast":"closed","cby":236,"self":false,"s":"Proposes a structured format for clinical article abstracts to help readers and reviewers quickly assess a paper's purpose, methods, and results before publication.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"GB3BX4WQ","t":"Standardising Peer Review in Paleontology journals","au":"Tennant, Jonathan","y":2020,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31233/osf.io/qzycs","cx":"Journal","co":["Interventions"],"sg":["individual-scoring"],"x":0,"st":"guideline-policy","oast":"gold","cby":1,"self":false,"s":"A proposal for a standardized operational checklist intended to give paleontology journals a consistent quality threshold for peer review, aiming to formalize integrity expectations across authors, editors, reviewers, and publishers.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"GDWBNVPZ","t":"Assessment of gender and geographical bias in the editorial decision-making process of biomedical journals: A Case-Control study","au":"Gayet-Ageron A.","y":2024,"vn":"medRxiv","ty":"journalArticle","tyl":"Journal article","doi":"10.1101/2024.03.15.24304220","cx":"Journal","co":["Bias"],"sg":["funding-decision"],"x":0,"st":"observational-quantitative","oast":"green","cby":1,"self":false,"s":"Case-control study of manuscripts across 22 biomedical journals found a raw association between female first authorship and acceptance that became non-significant after adjustment, while authors from lower-income regions faced lower acceptance odds.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"on-request","cau":""},{"k":"GE2AQA7U","t":"An observational study of the Patient-Centered Outcomes Research Institute’s switch from in-person to virtual research application review panels during the COVID-19 pandemic","au":"Bloodworth, Robin","y":2026,"vn":"Research Integrity and Peer Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s41073-026-00198-y","cx":"Grant","co":["Interventions"],"sg":["panel-consensus"],"x":0,"st":"observational-quantitative","oast":"gold","cby":0,"self":false,"s":"Comparing review panels before and after a patient-research funder shifted to virtual meetings during COVID-19, the study found broadly similar scoring patterns, with panelists giving somewhat higher post-discussion scores online, especially patient and stakeholder reviewers.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"no-statement","cau":""},{"k":"GEWF5FVV","t":"Do Readers and Peer Reviewers Agree on Manuscript Quality?","au":"Justice, Amy C.","y":1994,"vn":"JAMA","ty":"journalArticle","tyl":"Journal article","doi":"10.1001/jama.1994.03520020043011","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":43,"self":false,"s":"A study of manuscripts at a medical journal found readers, peer reviewers, and methods experts generally agreed on overall favorable ratings but showed poor agreement beyond chance, and experts were more critical than readers or reviewers.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"GFRCZP3P","t":"Journal editors’ perspectives on the roles and tasks of peer reviewers in biomedical journals: a qualitative study","au":"Glonti, Ketevan","y":2019,"vn":"BMJ Open","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmjopen-2019-033421","cx":"Journal","co":["general-systemic"],"sg":["individual-scoring"],"x":0,"st":"qualitative","oast":"gold","cby":74,"self":false,"s":"Interviews with 56 biomedical journal editors reveal that expectations for peer reviewers' roles and depth of scrutiny vary widely by journal context, and that editors generally value prior authorship experience over formal reviewer training.","pg":"no-statement","pgr":"","pgid":"BMJ Open 2018;8:e020568 (published protocol)","da":"on-request","dau":"","ca":"not-applicable","cau":""},{"k":"GGQQTLWV","t":"Language and socioeconomics predict geographic variation in peer review outcomes at an ecology journal","au":"Burns, C. Sean","y":2017,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-017-2517-5","cx":"Journal","co":["Bias"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"closed","cby":20,"self":false,"s":"Analyzing thousands of submissions to an ecology journal, this study finds that language proficiency and national socioeconomic status largely explain why manuscripts from certain world regions fare worse in peer review than others.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"http://dx.doi.org/10.5061/dryad.37312","ca":"open","cau":"https://github.com/cseanburns/peer_review_geography"},{"k":"GHKXSBKL","t":"Influence of reviewer interaction network on long-term citations: a case study of the scientific peer-review system of the journal of high energy physics","au":"Sikdar, Sandipan","y":2017,"vn":"arXiv (Cornell University)","ty":"journalArticle","tyl":"Journal article","doi":"10.5555/3200334.3200354","cx":"Journal","co":["Validity"],"sg":["reviewer-selection","post-award-validity"],"x":0,"st":"observational-quantitative","oast":"green","cby":5,"self":false,"s":"Analysis of a large physics journal's review records showing that structural features of the network connecting reviewers assigned by the same editor strongly predict a paper's eventual long-term citation impact.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"GHRIZ2A3","t":"UK Doubles Its “World-Leading” Research in Life Sciences and Medicine in Six Years: Testing the Claim?","au":"Wooding, Steven","y":2015,"vn":"PLoS ONE","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0132990","cx":"General","co":["Validity"],"sg":["post-award-validity"],"x":0,"st":"observational-quantitative","oast":"gold","cby":16,"self":false,"s":"Compares UK Research Excellence Framework peer-review quality ratings against bibliometric citation data, finding no evidence to support the reported doubling of \"world-leading\" outputs in life sciences and medicine between 2008 and 2014.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"no-statement","cau":""},{"k":"GHRV2IBK","t":"A Social Epistemological Inquiry into Biases in Journal Peer Review","au":"Jukola, Saana","y":2016,"vn":"Perspectives on Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1162/posc_a_00237","cx":"Journal","co":["Bias"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":16,"self":false,"s":"A philosophical analysis argues that journal peer review is shaped not only by individual reviewer biases but also by the broader institutional and career incentives surrounding academic publishing.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"GIY7LUIX","t":"Development and Validation of a Scoring Rubric for Editorial Evaluation of Peer-review Quality: A Pilot Study.","au":"Love, Jeffrey N.","y":2024,"vn":"PubMed","ty":"journalArticle","tyl":"Journal article","doi":"10.5811/westjem.18432","cx":"Journal","co":["Reliability-noise","Validity","Interventions"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"green","cby":1,"self":false,"s":"Pilot study develops and validates a holistic rubric for editors to score peer review quality, finding it produced a better score distribution and predicted editorial decisions better than a prior rating scale.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"GKF84MXT","t":"Sceptics and champions: participant insights on the use of partial randomization to allocate research culture funding","au":"Davies, Catherine","y":2024,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvaf006","cx":"Grant","co":["Bias","Validity","Interventions"],"sg":["funding-decision"],"x":0,"st":"qualitative","oast":"hybrid","cby":0,"self":false,"s":"Describes a University of Leeds trial that randomly allocated some research-culture funding among similarly scored mid-tier applications, finding modest reductions in perceived bias alongside concerns from participants about fairness and rigor.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"","ca":"no-statement","cau":""},{"k":"GL6NPLR6","t":"Open science and modified funding lotteries can impede the natural selection of bad science","au":"Smaldino, Paul E.","y":2019,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31219/osf.io/zvkwq","cx":"General","co":["Validity","Interventions"],"sg":["funding-decision"],"x":0,"st":"simulation-methodological","oast":"gold","cby":18,"self":false,"s":"Using computational modeling, this study found that lottery-based grant allocation restricted to methodologically rigorous proposals, especially combined with open-science practices, can reduce the rate of false scientific discoveries.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"https://github.com/mt-digital/badscience-solutions","ca":"open","cau":"https://github.com/mt-digital/badscience-solutions"},{"k":"GLBUJJ2K","t":"The Reliability of Peer Assessments","au":"Goldman, Ronald L.","y":1994,"vn":"Evaluation & the Health Professions","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/016327879401700101","cx":"General","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"systematic-review-meta-analysis","oast":"closed","cby":62,"self":false,"s":"A meta-analysis of studies from 1966 to 1991 estimates the average chance-corrected agreement among peer assessors of care quality at a weighted kappa of about 0.31, indicating limited reliability that the authors say warrants process improvement.","pg":"not-codeable","pgr":"","pgid":"","da":"","dau":"","ca":"","cau":""},{"k":"GNW32DYJ","t":"But the reviewers are making different criticisms of my paper! Diversity and uniqueness in reviewer comments.","au":"Fiske, Donald W.","y":1990,"vn":"American Psychologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/0003-066x.45.5.591","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":144,"self":false,"s":"A content analysis of hundreds of reviews submitted to psychology journal editors finding that different reviewers of the same paper rarely raised the same criticisms, instead each highlighting distinct, non-overlapping concerns.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"GPQAWJJ9","t":"An analysis of rater effects in reviews of scientific manuscripts","au":"Turner, D. P.","y":2017,"vn":"undefined","ty":"journalArticle","tyl":"Journal article","doi":"","cx":"Journal","co":["Reliability-noise","Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"","cby":0,"self":false,"s":"Applies Generalizability Theory and Many-Facet Rasch measurement to journal reviews, finding reviewer severity accounts for a substantial share of score variance and that adding more reviewers would not meaningfully fix this.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"GPUHMSPG","t":"Double anonymity in peer review within the chemistry periodicals community","au":"Brown, Richard J. C.","y":2007,"vn":"Learned Publishing","ty":"journalArticle","tyl":"Journal article","doi":"10.1087/174148507x185108","cx":"Journal","co":["Bias","Interventions"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":11,"self":false,"s":"Survey of over 1,400 chemistry journal editors and board members on attitudes toward double-anonymous peer review, the first such survey conducted in that field.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"GRDE4DDS","t":"Mentored peer review of standardized manuscripts as a teaching tool for residents: a pilot randomized controlled multi-center study.","au":"Wong, Victoria S S","y":2017,"vn":"Research Integrity and Peer Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s41073-017-0032-0","cx":"Journal","co":["Interventions"],"sg":["individual-scoring"],"x":1,"st":"experimental-rct","oast":"gold","cby":21,"self":false,"s":"A multi-center pilot randomized trial of 78 neurology residents tests whether mentored peer review of standardized manuscripts improves research-methods knowledge more than unmentored review, finding no significant difference in knowledge gains between groups.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"no-statement","cau":""},{"k":"GS27TS6D","t":"Responses by four Local Research Ethics Committees to submitted proposals.","au":"Kent, G","y":1999,"vn":"Journal of medical ethics","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/jme.25.3.274","cx":"General","co":["Reliability-noise"],"sg":["panel-consensus"],"x":0,"st":"observational-quantitative","oast":"bronze","cby":23,"self":false,"s":"Content analysis of letters from four UK research ethics committees finds substantial variation in initial approval rates and identifies four recurring types of requested changes, most often to participant information sheets.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"GSEWAUVH","t":"Peer review for funding decisions","au":"Liu, Yuxian","y":2025,"vn":"Journal of Data and Information Science","ty":"journalArticle","tyl":"Journal article","doi":"10.2478/jdis-2025-0050","cx":"Grant","co":["Reliability-noise","Bias","Validity","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"diamond","cby":1,"self":false,"s":"A review of funding-focused peer review identifies persistent reviewer disagreement as a core structural problem and surveys emerging alternatives such as partial lotteries and distributed review.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"GSWIY3RL","t":"Reimagining Peer Review Needs Publishers and Institutions to Collaborate More","au":"King, Stuart","y":2024,"vn":"Exchanges The Interdisciplinary Research Journal","ty":"journalArticle","tyl":"Journal article","doi":"10.31273/eirj.v11i3.1519","cx":"Journal","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"diamond","cby":1,"self":false,"s":"A commentary arguing that meaningful peer-review reform requires closer collaboration between academic institutions and publishers, building on momentum from preprints and related initiatives.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"GUAUHBD5","t":"Peer review procedures as practice, decision, and governance – the road to theories of peer review","au":"Reinhart, Martin","y":2021,"vn":"Science and Public Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/scipol/scad089","cx":"General","co":["general-systemic"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"hybrid","cby":12,"self":false,"s":"A theoretical paper argues peer review should be understood simultaneously as an evaluative practice, a resource-allocation mechanism, and an instrument of scientific self-governance, proposing groundwork for a more integrated theory of peer review.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"GUJQFPNY","t":"What Are Reviewers Looking For? A Qualitative Analysis of Open-Ended Responses from a Questionnaire Sent to Faculty in Agricultural Communications","au":"Settle, Quisto","y":2020,"vn":"Journal of Applied Communications","ty":"journalArticle","tyl":"Journal article","doi":"10.4148/1051-0834.2318","cx":"Journal","co":["general-systemic"],"sg":["individual-scoring"],"x":0,"st":"qualitative","oast":"diamond","cby":1,"self":false,"s":"A survey of agricultural communications faculty reviewers finds wide variation in how they evaluate manuscripts, though most look for clear structure, overall quality, writing clarity, and originality of contribution.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"GUX5QL5F","t":"Criticism and Judgment: A Critical Look at Scientific Peer Review","au":"Hope, Aluko A.","y":2019,"vn":"American Journal of Critical Care","ty":"journalArticle","tyl":"Journal article","doi":"10.4037/ajcc2019152","cx":"Journal","co":["Bias","Interventions"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"bronze","cby":14,"self":false,"s":"Essay reflecting on the philosophical basis of scientific judgment in peer review, drawing on Francis Bacon's account of how careful observation and reasoning inform expert assessment.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"GVVSKS4V","t":"Peer review reliability: The hierarchy of the sciences","au":"Crothers, Charles","y":1993,"vn":"Behavioral and Brain Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/s0140525x00030764","cx":"General","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":25,"self":false,"s":"Brief work discussing how the reliability of peer review varies across scientific disciplines, framed within a broader hierarchy-of-sciences perspective.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"GZBYTT5I","t":"The time has come for big changes to improve research funding","au":"Kolarz, Peter","y":2026,"vn":"PLoS Biology","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pbio.3003754","cx":"Grant","co":["Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":0,"self":false,"s":"A commentary argues the competitive research funding system faces fundamental limits that incremental fixes cannot resolve, calling for large-scale structural transformation.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"H3V2822W","t":"Does the use of open, non-anonymous peer review in scholarly publishing introduce bias? Evidence from the F1000 post-publication open peer review publishing model","au":"Thelwall, Mike","y":2019,"vn":"arXiv (Cornell University)","ty":"journalArticle","tyl":"Journal article","doi":"10.48550/arxiv.1911.03379","cx":"Journal","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"green","cby":0,"self":false,"s":"Analysis of F1000Research's open post-publication review system finds a tendency for reviewers to rate same-country authors more favorably, but no evidence that reviewers conform to previously published reviewers' judgments.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"H4CUS6M8","t":"An examination of the federal psychiatric training grant peer review process.","au":"Wilkinson, Charles B","y":1977,"vn":"The American Journal of Psychiatry","ty":"journalArticle","tyl":"Journal article","doi":"10.1176/ajp.134.6.637","cx":"Grant","co":["Reliability-noise","Bias"],"sg":["reviewer-selection"],"x":0,"st":"observational-quantitative","oast":"closed","cby":2,"self":false,"s":"A 1970s analysis of a federal psychiatric-training grant program examined reviewer demographics and procedural safeguards, concluding the review process was reasonably representative and fair despite common criticisms of federal peer review.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"H73S9JI4","t":"Peer Advocacy: Expressions of Loyalty in Peer Review","au":"Geschwind, Lars","y":2022,"vn":"","ty":"book","tyl":"Book","doi":"10.1007/978-3-030-75263-7_9","cx":"General","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"book-monograph","oast":"gold","cby":4,"self":false,"s":"Drawing on Swedish higher-education case studies, this book chapter identifies \"peer advocacy,\" where reviewers act as promoters rather than neutral evaluators, and offers recommendations to preserve review integrity.","pg":"not-applicable","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"H8TDR5AF","t":"Are female applicants disadvantaged in National Institutes of Health peer review? Combining algorithmic text mining and qualitative methods to detect evaluative differences in R01 reviewers' critiques.","au":"Magua, Wairimu","y":2017,"vn":"Journal of Women's Health","ty":"journalArticle","tyl":"Journal article","doi":"10.1089/jwh.2016.6021","cx":"Grant","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":127,"self":false,"s":"Text-mining analysis of NIH grant renewal critiques finds reviewers described male applicants using more assertive, leadership-oriented language than female applicants with comparable productivity, and male-led applications received better scores overall.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"H9B5WQZN","t":"Funding decision-making systems: An empirical comparison of continuous and dichotomous approaches based on psychometric theory","au":"Mutz, Rüdiger","y":2016,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvw002","cx":"Grant","co":["Reliability-noise","Bias"],"sg":["funding-decision"],"x":0,"st":"simulation-methodological","oast":"bronze","cby":5,"self":false,"s":"Using data from over 8,000 proposals to the Austrian Science Fund, this study compares simulated dichotomous (fund/reject) and continuous funding-allocation systems, showing that a continuous approach based on peer review scores would approve substantially more proposals than a strict cutoff system.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"HAKSM7H8","t":"Comments on Single-Blind Reviewing from the Editorial Staff","au":"Atkeson, Lonna Rae","y":2018,"vn":"Political Analysis","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/pan.2018.34","cx":"Journal","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"bronze","cby":0,"self":false,"s":"Editorial explaining a journal's move to single-blind reviewing and responding to reader feedback and debate about the decision.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"HBP5E7SN","t":"Do funding applications where peer reviewers disagree have higher citations? A cross-sectional study.","au":"Barnett, Adrian G","y":2018,"vn":"F1000Research","ty":"journalArticle","tyl":"Journal article","doi":"10.12688/f1000research.15479.2","cx":"Grant","co":["Reliability-noise","Validity"],"sg":["individual-scoring","post-award-validity"],"x":0,"st":"observational-quantitative","oast":"gold","cby":3,"self":false,"s":"Analysis of scores for 227 funded grant applications finding that disagreement among reviewers did not predict whether an application later achieved higher-than-average citation impact.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"http://doi.org/10.5281/zenodo.1299384","ca":"open","cau":"https://github.com/agbarnett/funding.disagree"},{"k":"HBXRKG4U","t":"‘Fund people, not projects’: From narrative CVs to lotteries in science funding policy","au":"Shaw, Jamie","y":2024,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvae035","cx":"Grant","co":["Bias","Validity","Interventions"],"sg":["funding-decision"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":1,"self":false,"s":"A conceptual paper analyzes the \"fund people, not projects\" movement in science funding, distinguishing its variants and proposing narrative CVs as an evaluation mechanism, while arguing debiasing efforts should target funding policy broadly rather than peer review alone.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"HC9VZTTZ","t":"Patterns of evaluation in science: Institutionalisation, structure and functions of the referee system","au":"Zuckerman, Harriet","y":1971,"vn":"Minerva","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/bf01553188","cx":"Journal","co":["Reliability-noise"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"closed","cby":800,"self":false,"s":"Sociological account describes journal editors and referees as institutional \"status-judges\" whose confidential evaluations shape scientific reputations and resource allocation, comparing this role to analogous evaluators in other social systems.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"HE2HZJ8P","t":"Quality assessment of scientific manuscripts in peer review and education","au":"Augusteijn, Hilde Elisabeth Maria","y":2023,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31219/osf.io/7dc6a","cx":"Journal","co":["Bias"],"sg":["individual-scoring"],"x":1,"st":"experimental-rct","oast":"gold","cby":5,"self":false,"s":"A vignette-based survey found academic quality ratings were influenced by sample size but not by whether results were statistically significant, and that students and academics rated manuscript-quality criteria similarly.","pg":"preregistered","pgr":"OSF","pgid":"https://osf.io/cd3uw","da":"no-statement","dau":"https://osf.io/cd3uw","ca":"open","cau":"https://osf.io/fn5tu/"},{"k":"HEYTAC8Z","t":"Evaluation of Large Language Model Performance in Assessing Health Economic Study Quality","au":"Dun, Chen","y":2025,"vn":"Journal of health economics and outcomes research","ty":"journalArticle","tyl":"Journal article","doi":"10.36469/jheor.2025.145214","cx":"General","co":["Reliability-noise","Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":0,"self":false,"s":"Comparing large language model evaluations of 110 health-economic studies against two human reviewers using a standard reporting checklist, this study found substantial human-human disagreement and moderate-to-high agreement between the AI and human consensus scores.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"HF9NCMFH","t":"An exploration of referees’ comments published in open peer review journals: The characteristics of review language and the association between review scrutiny and citations","au":"Wolfram, Dietmar","y":2021,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvab005","cx":"Journal","co":["Validity"],"sg":["individual-scoring","post-award-validity"],"x":0,"st":"observational-quantitative","oast":"closed","cby":26,"self":false,"s":"Examining open peer-review reports from 17 journals, the study finds reviewer self-identification made little difference to hedging language, and greater review scrutiny (more reviewers or review rounds) did not correlate with a paper's eventual citation count.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"","ca":"no-statement","cau":""},{"k":"HFSYDHZ3","t":"Topic choice contributes to the lower rate of NIH awards to African-American/black scientists","au":"Hoppe, Travis","y":2019,"vn":"Science Advances","ty":"journalArticle","tyl":"Journal article","doi":"10.1126/sciadv.aaw7238","cx":"Grant","co":["Bias","Validity"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"gold","cby":629,"self":false,"s":"An analysis of NIH grant application stages from 2011-2015 finds that Black scientists' lower funding rate is partly explained by their tendency to propose research on topics that historically receive lower award rates.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"no-statement","cau":""},{"k":"HH53MPTF","t":"Evaluating psychological research reports: Dimensions, reliability, and correlates of quality judgments.","au":"Gottfredson, Stephen D.","y":1978,"vn":"American Psychologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/0003-066x.33.10.920","cx":"Journal","co":["Reliability-noise","Validity"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":256,"self":false,"s":"Survey of psychology journal editors identifying shared dimensions of manuscript quality and demonstrating that structuring reviews around these dimensions can raise reviewer reliability, though quality ratings correlated only weakly with later citations.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"HI2UVM24","t":"Effect of an editorial intervention to improve the completeness of reporting of randomised trials: a randomised controlled trial","au":"Blanco, David","y":2020,"vn":"BMJ Open","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmjopen-2020-036799","cx":"Journal","co":["Interventions"],"sg":["individual-scoring"],"x":1,"st":"experimental-rct","oast":"gold","cby":69,"self":false,"s":"Randomized trial at BMJ Open testing an editorial intervention providing reviewers with a CONSORT-based completeness report alongside standard peer review, finding it improved reporting quality of published randomized trials.","pg":"preregistered","pgr":"ClinicalTrials.gov","pgid":"NCT03751878","da":"no-statement","dau":"","ca":"open","cau":""},{"k":"HIEHTPKP","t":"What factors are important to the success of resubmitted grant applications in biomedical and health research? A retrospective study of over 20 000 applications to the Canadian Institutes of Health Research","au":"Wrightson, James G.","y":2025,"vn":"FACETS","ty":"journalArticle","tyl":"Journal article","doi":"10.1139/facets-2024-0116","cx":"Grant","co":["Bias","Validity"],"sg":["proposal-quality","funding-decision"],"x":0,"st":"observational-quantitative","oast":"gold","cby":0,"self":false,"s":"A retrospective study of over 20,000 Canadian Institutes of Health Research applications finds resubmissions succeed roughly twice as often as new applications, with prior ranking and the applicant's funding history mattering most.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"open","cau":"https://osf.io/pw45z/"},{"k":"HMW89QW7","t":"Male applicants are more likely to be awarded fellowships than female applicants: A case study of a Japanese national funding agency.","au":"Kyogoku, Daisuke","y":2023,"vn":"PloS one","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0291372","cx":"Grant","co":["Bias"],"sg":["funding-decision"],"x":0,"st":"observational-quantitative","oast":"gold","cby":3,"self":false,"s":"Analysis of Japan Society for the Promotion of Science fellowship data finds consistently higher success rates for male than female applicants across categories, with gaps varying by field.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"","ca":"no-statement","cau":""},{"k":"HNZ2E7JB","t":"Does double‐blind peer review reduce bias? Evidence from a top computer science conference","au":"Sun, Mengyi","y":2021,"vn":"Journal of the Association for Information Science and Technology","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/asi.24582","cx":"Journal","co":["Reliability-noise","Bias","Validity","Interventions"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":3,"self":false,"s":"Analyzing a computer science conference's shift from single- to double-blind review, the study found scores for prestigious authors dropped under double-blind review and reviewer disagreement increased, though overall acceptance rates changed little.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"no-statement","cau":""},{"k":"HPC6SYSG","t":"An Examination of Sources of Peer-Review Bias","au":"Blackburn, Jessica L.","y":2006,"vn":"Psychological Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/j.1467-9280.2006.01715.x","cx":"Journal","co":["Reliability-noise","Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":57,"self":false,"s":"Analyzing nearly 2,000 conference poster reviews, this study found reviewers who were also authors rated posters more critically, while posters with reviewer-authors received more favorable ratings, consistent with self-serving bias effects.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"HPEHPE8N","t":"Academic judgments under uncertainty: A study of collective anchoring effects in Swedish Research Council panel groups","au":"Roumbanis, Lambros","y":2016,"vn":"Social Studies of Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/0306312716659789","cx":"Grant","co":["Bias"],"sg":["panel-consensus"],"x":0,"st":"qualitative","oast":"closed","cby":60,"self":false,"s":"Based on direct observation of ten Swedish Research Council panels, the study attributes anchoring effects in proposal scoring to the interaction of numeric grading conventions with panellists' efforts to reach consensus under disagreement and uncertainty.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"HPGFXDLZ","t":"Relationship between journal impact factor and the thoroughness and helpfulness of peer reviews","au":"Severin, Anna","y":2023,"vn":"PLoS Biology","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pbio.3002238","cx":"Journal","co":["general-systemic"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":38,"self":false,"s":"A machine-learning-assisted text analysis of 10,000 peer review reports across journals with varying impact factors, finding that reviews for higher-impact journals tend to be longer and more focused on methodology, with somewhat less attention to formatting or suggested fixes.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://doi.org/10.5281/zenodo.8006829","ca":"open","cau":"https://doi.org/10.5281/zenodo.8006829"},{"k":"HQHC5EYY","t":"Centralized Funding and Epistemic Exploration","au":"Avin, Shahar","y":2017,"vn":"The British Journal for the Philosophy of Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/bjps/axx059","cx":"Grant","co":["Bias","Validity"],"sg":["funding-decision"],"x":0,"st":"simulation-methodological","oast":"green","cby":34,"self":false,"s":"Agent-based simulation modeling how centralized funding allocation affects a research community's balance between exploring new ideas and exploiting known ones, finding random allocation can outperform peer review when the space of possible discoveries is large.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"open","cau":"https://github.com/shaharavin/FundingSimulator"},{"k":"HQXJUIA5","t":"Journal and disciplinary variations in academic open peer review anonymity, outcomes, and length","au":"Thelwall, Mike","y":2022,"vn":"Journal of Librarianship and Information Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/09610006221079345","cx":"Journal","co":["general-systemic"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"green","cby":21,"self":false,"s":"Large-scale analysis of over 45,000 open peer reviews across MDPI journals finding substantial differences by discipline and journal in reviewer anonymity, review length, and outcomes, but no evidence that signed reviews compromised validity.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://doi.org/10.6084/m9.figshare.16677091","ca":"open","cau":"https://doi.org/10.6084/m9.figshare.16677091"},{"k":"HT7RTIKQ","t":"Prior publication productivity, grant percentile ranking, and topic-normalized citation impact of NHLBI cardiovascular R01 grants.","au":"Kaltman, Jonathan R","y":2014,"vn":"Circulation Research","ty":"journalArticle","tyl":"Journal article","doi":"10.1161/CIRCRESAHA.115.304766","cx":"Grant","co":["Validity"],"sg":["post-award-validity"],"x":0,"st":"observational-quantitative","oast":"closed","cby":52,"self":false,"s":"A study of nearly 1,500 NHLBI cardiovascular R01 grants finds that after normalizing citation impact, peer-review percentile ranking still shows no association with later scientific impact, while investigators' prior publication productivity does predict it.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"HUD5RNL8","t":"Peer Review Quality and Transparency of the Peer-Review Process in Open Access and Subscription Journals.","au":"Wicherts, Jelte M","y":2016,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0147913","cx":"Journal","co":["Reliability-noise","Validity"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":167,"self":false,"s":"Develops and validates a 14-item tool for rating the transparency of a journal's peer review process, finding transparency ratings correlated with independently assessed review quality and predicted resistance to a hoax paper.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://osf.io/5hn36/","ca":"no-statement","cau":""},{"k":"HUMIF4QI","t":"How do NIHR peer review panels use bibliometric information to support their decisions?","au":"Gunashekar, Salil","y":2017,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-017-2417-8","cx":"Grant","co":["Interventions"],"sg":["panel-consensus"],"x":0,"st":"qualitative","oast":"hybrid","cby":34,"self":false,"s":"Interviews with UK health-research funding panelists found members generally support using citation-based bibliometrics in decisions, though they raised concerns about comparing such metrics fairly across different fields.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"HUZ86E9X","t":"Is the Lack of Inter-referee Reliability a Threat to the Effectiveness of Peer Review in Science?","au":"Wray, K. Brad","y":2025,"vn":"Perspectives on Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1162/posc_a_00639","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":0,"self":false,"s":"A philosophical argument contending that low agreement among journal referees does not undermine peer review's value, because the system is nonetheless effective at catching errors in submitted manuscripts.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"HVR47UFQ","t":"Metascience on Peer Review: Testing the Effects of a Study’s Originality and Statistical Significance in a Field Experiment","au":"Elson, Malte","y":2020,"vn":"Advances in Methods and Practices in Psychological Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/2515245919895419","cx":"Journal","co":["Bias"],"sg":["individual-scoring"],"x":1,"st":"experimental-rct","oast":"bronze","cby":22,"self":false,"s":"A field experiment embedded in a conference's review process tests how a submission's originality and the statistical significance of its results affect reviewer evaluations, aiming to demonstrate the feasibility of experimental peer-review research.","pg":"preregistered","pgr":"OSF","pgid":"https://osf.io/t6hs9/","da":"open","dau":"https://osf.io/zajx3/","ca":"open","cau":"https://osf.io/cfnvu/"},{"k":"HY8N87UZ","t":"Clinical trials proposed for the VA Cooperative Studies Program: Success rates and factors impacting approval","au":"Burnaska, David R.","y":2021,"vn":"Contemporary Clinical Trials Communications","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.conctc.2021.100811","cx":"Grant","co":["general-systemic"],"sg":["funding-decision"],"x":0,"st":"guideline-policy","oast":"gold","cby":2,"self":false,"s":"Reviews outcomes of proposals submitted to a US Veterans Affairs clinical trials programme over several years, identifying investigator experience, preliminary data, and study design as key factors distinguishing funded from rejected applications.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"HYTZEAF9","t":"MERIT (Manuscript Evaluation Reflecting Intrinsic Tenets): A Framework for Efficient and Equitable Academic Peer Review","au":"Arboleda‐Velásquez, Joseph F.","y":2024,"vn":"ScienceBank","ty":"journalArticle","tyl":"Journal article","doi":"10.61340/rwng4653","cx":"Journal","co":["Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"hybrid","cby":0,"self":false,"s":"Proposes a set of new tools and concepts, including a structured \"justification sheet\" for reviewers, intended to make journal peer review faster and more transparent given the scarcity of formal reviewer training.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"HZ4YRKUY","t":"The modified lottery: Formalizing the intrinsic randomness of research funding.","au":"De Peuter, Steven","y":2022,"vn":"Accountability in research","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/08989621.2021.1927727","cx":"Grant","co":["Reliability-noise","Bias","Validity","Interventions"],"sg":["funding-decision"],"x":0,"st":"conceptual-commentary","oast":"green","cby":15,"self":false,"s":"Argues for testing a \"modified lottery\" system that formally incorporates randomness into grant funding decisions as a way to address shortcomings of traditional peer review.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"I358ZRGB","t":"Peer Review Under Review — a Statistical Study on Proposal Ranking at ESO. Part I: the Pre-meeting Phase","au":"Patat, Ferdinando","y":2018,"vn":"Publications of the Astronomical Society of the Pacific","ty":"journalArticle","tyl":"Journal article","doi":"10.1088/1538-3873/aac463","cx":"Grant","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"bronze","cby":2,"self":false,"s":"A statistical analysis of about 15,000 European Southern Observatory telescope-time proposals over eight years finds that reviewer rankings agree only marginally above chance in the middle quartiles, quantifying limited reproducibility in pre-meeting review.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"I42AJ26U","t":"What makes an effective grants peer reviewer? An exploratory study of the necessary skills.","au":"Steiner Davis, Miriam L E","y":2020,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0232327","cx":"Grant","co":["Interventions"],"sg":["individual-scoring"],"x":0,"st":"qualitative","oast":"gold","cby":14,"self":false,"s":"Mixed-methods study of grant review panelists and program officers identifies the skills considered necessary for effective reviewing and examines how experience and virtual versus in-person formats affect skill development.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"","ca":"no-statement","cau":""},{"k":"I4LKXWXM","t":"Communicating and compromising on disciplinary expertise in the peer review of research proposals","au":"Huutoniemi, Katri Ilona","y":2012,"vn":"Social Studies of Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/0306312712458478","cx":"Grant","co":["Reliability-noise","Bias"],"sg":["panel-consensus"],"x":0,"st":"qualitative","oast":"closed","cby":72,"self":false,"s":"Observing four interdisciplinary grant panels, the study describes how reviewers from different fields negotiate and sometimes reach consensus despite differing disciplinary standards, while also identifying limits set by disciplinary boundaries.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"I4PYTQC4","t":"Peer-review in a world with rational scientists: Toward selection of the average","au":"Thurner, S.","y":2011,"vn":"The European Physical Journal B","ty":"journalArticle","tyl":"Journal article","doi":"10.1140/epjb/e2011-20545-7","cx":"Journal","co":["Bias","Validity"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"green","cby":57,"self":false,"s":"Using an agent-based model, this study shows that even a small proportion of self-interested or incompetent referees can substantially degrade the average quality of work selected through peer review.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"I4ZU3XFU","t":"The reliability of reviews for the American Psychologist.","au":"Scarr, Sandra","y":1978,"vn":"American Psychologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/h0078544","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":151,"self":false,"s":"Analysing 87 manuscript decisions at a psychology journal, the authors report exact reviewer agreement in about two-thirds of paired ratings, a more favourable reliability picture than an earlier study of journal reviews had suggested.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"I5Y9JHPK","t":"Predicting productivity returns on investment: thirty years of peer review, grant funding, and publication of highly cited papers at the national heart, lung, and blood institute.","au":"Lauer, Michael S","y":2015,"vn":"Circulation Research","ty":"journalArticle","tyl":"Journal article","doi":"10.1161/CIRCRESAHA.115.306830","cx":"Grant","co":["Validity"],"sg":["post-award-validity"],"x":0,"st":"observational-quantitative","oast":"closed","cby":55,"self":false,"s":"A 30-year bibliometric analysis of NIH cardiovascular grants finding only a modest, poorly discriminating association between peer review percentile rankings and the likelihood of producing highly cited publications.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"I7KMMCXK","t":"What works for peer review and decision-making in research funding: a realist synthesis.","au":"Recio-Saucedo, Alejandra","y":2022,"vn":"Research Integrity and Peer Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s41073-022-00120-2","cx":"Grant","co":["Reliability-noise","Interventions"],"sg":["whole-process"],"x":0,"st":"systematic-review-meta-analysis","oast":"gold","cby":51,"self":false,"s":"Realist synthesis of around 50 documented interventions to grant peer review (shorter applications, reviewer training, virtual panels, etc.), summarizing which changes reduced reviewer variability, sped up the process, or lowered costs.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"no-statement","cau":""},{"k":"I8D6WFWV","t":"On the effectiveness of the scientific peer-review system: a case study of the Journal of High Energy Physics","au":"Sikdar, Sandipan","y":2018,"vn":"International Journal on Digital Libraries","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s00799-018-0247-9","cx":"Journal","co":["Validity"],"sg":["whole-process","post-award-validity"],"x":0,"st":"observational-quantitative","oast":"closed","cby":4,"self":false,"s":"Large-scale measurement study of nearly 29,000 physics papers and 70,000 review reports, building a predictive model of long-term paper impact from features related to the paper, authors, and reviewers.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"I8SJR4X6","t":"UnWEIRDing Peer Review in Human-Computer Interaction","au":"Nigatu, Hellina Hailu","y":2026,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.1145/3772318.3791553","cx":"Journal","co":["Bias","Interventions"],"sg":["individual-scoring"],"x":0,"st":"qualitative","oast":"gold","cby":1,"self":false,"s":"Focus groups with 16 HCI researchers studying the Global South describe experiences of reviews that dismiss theoretical contributions, tokenize them as cultural experts, and impose an implicit epistemic burden, framed through a lens of epistemic oppression.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"IBJ7WKTZ","t":"Is novel research worth doing? Evidence from peer review at 49 journals","au":"Teplitskiy, Misha","y":2022,"vn":"Proceedings of the National Academy of Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1073/pnas.2118046119","cx":"Journal","co":["Reliability-noise","Bias"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":38,"self":false,"s":"Analysis of over 27,000 submissions to major journals finds manuscripts combining novel and conventional ideas were more likely to be accepted and later more highly cited, challenging the view that review disfavors novelty.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"IBR4QCFH","t":"Journal policies and editors’ opinions on peer review","au":"Hamilton, Daniel G","y":2020,"vn":"eLife","ty":"journalArticle","tyl":"Journal article","doi":"10.7554/eLife.62529","cx":"Journal","co":["general-systemic"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"gold","cby":68,"self":false,"s":"A survey of 322 journal editors across five disciplines found wide variation in peer-review policies, including plagiarism checks and openness practices, along with broad editorial support for co-reviewing and replication studies.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://doi.org/10.17605/OSF.IO/CY2RE","ca":"no-statement","cau":""},{"k":"IBUMSYH9","t":"Determinants of quality of research environment: An assessment of the environment submissions in the UK’s Research Excellence Framework in 2014","au":"Pinar, Mehmet","y":2020,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvaa003","cx":"General","co":["Reliability-noise","Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":16,"self":false,"s":"An analysis of the UK's 2014 Research Excellence Framework found that research income was linked to higher environment-quality scores, and that panels weighted similar quantitative factors inconsistently across disciplines, alongside evidence of an institutional halo effect.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"","ca":"no-statement","cau":""},{"k":"ICFNMK39","t":"Attitudes of referees in a multidisciplinary journal: An empirical analysis","au":"Casnici, Niccolò","y":2016,"vn":"Journal of the Association for Information Science and Technology","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/asi.23665","cx":"Journal","co":["Reliability-noise","Bias","Validity"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"bronze","cby":59,"self":false,"s":"Ten years of review data from a multidisciplinary social-simulation journal show that referees' disciplinary background and seniority shape recommendation style, with humanities reviewers more lenient and senior reviewers harsher, while multidisciplinary review was linked to higher later citations.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"ID6UT6PK","t":"AI in the gatekeeper’s chair: elite researchers’ perceptions of AI-assisted feedback in journal peer review","au":"Li, Heng","y":2026,"vn":"Assessment & Evaluation in Higher Education","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/02602938.2026.2691499","cx":"Journal","co":["Bias","Interventions"],"sg":["individual-scoring"],"x":1,"st":"experimental-rct","oast":"closed","cby":0,"self":false,"s":"A mixed-methods study of elite Nature and Science authors finds they view AI-assisted peer review feedback as less fair and useful than human review, driven partly by distrust of researchers who delegate review tasks to AI.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"IDTRS5C7","t":"Illusory reliability in journal reviewing.","au":"Gilmore, J. Barnard","y":1979,"vn":"Canadian Psychological Review/Psychologie canadienne","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/h0081518","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":36,"self":false,"s":"Argues that apparent high agreement rates reported between journal reviewers can be misleading once compared against the agreement expected purely by chance.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"IFPUHR5K","t":"Biases in grant proposal success rates, funding rates and award sizes affect the geographical distribution of funding for biomedical research.","au":"Wahls, Wayne P","y":2016,"vn":"PeerJ","ty":"journalArticle","tyl":"Journal article","doi":"10.7717/peerj.1917","cx":"Grant","co":["Bias","Validity"],"sg":["funding-decision"],"x":0,"st":"observational-quantitative","oast":"gold","cby":26,"self":false,"s":"Analysis of a decade of NIH grant data finding large state-by-state disparities in funding success unrelated to scientific productivity, pointing to structural biases in how funding is distributed geographically.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"http://dx.doi.org/10.7717/peerj.1917#supplemental-information","ca":"no-statement","cau":""},{"k":"IGGDUDQL","t":"Does citation context information enhance the validity of citation analysis for measuring research quality? An empirical comparison of peer assessments and enriched citations","au":"Donner, Paul","y":2025,"vn":"Quantitative Science Studies","ty":"journalArticle","tyl":"Journal article","doi":"10.1162/qss.a.15","cx":"General","co":["Validity"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":1,"self":false,"s":"A study comparing citation-context data (e.g., where citations appear in a paper) against expert quality ratings finds that richer citation-context measures correlate with human judgments better than simple citation counts alone.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"no-statement","cau":""},{"k":"IH5URG3N","t":"The Game Between a Biased Reviewer and His Editor","au":"García, J. A.","y":2017,"vn":"Science and Engineering Ethics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11948-017-9998-8","cx":"Journal","co":["Bias"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"simulation-methodological","oast":"green","cby":13,"self":false,"s":"Game-theoretic model examines when a journal editor benefits from delegating review to a biased referee, depending on the cost and informativeness of the review process.","pg":"no-statement","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"II3R27M2","t":"The Bias of Sighted Reviewers in Research Proposal Evaluation: A Comparative Analysis of Blind and Open Review in Korea","au":"Lee, Mushin","y":2000,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1023/a:1005636503358","cx":"Grant","co":["Bias","Interventions"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":14,"self":false,"s":"Compares blind and open review of Korean research proposals to examine whether reviewers who can see applicant identity show measurable bias.","pg":"not-codeable","pgr":"","pgid":"","da":"","dau":"","ca":"","cau":""},{"k":"IJWHZBDG","t":"Grant reviewer perceptions of the quality, effectiveness, and influence of panel discussion.","au":"Gallo, Stephen A","y":2020,"vn":"Research Integrity and Peer Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s41073-020-00093-0","cx":"Grant","co":["Reliability-noise","Bias"],"sg":["panel-consensus"],"x":0,"st":"observational-quantitative","oast":"gold","cby":18,"self":false,"s":"A survey of grant reviewers found generally favorable views of panel discussions' fairness and influence on outcomes, but noted concerns about uneven participation, time constraints, and inconsistent chair facilitation.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://doi.org/10.6084/m9.figshare.8132453.v1","ca":"no-statement","cau":""},{"k":"IKBM99JX","t":"Effective Peer Review: Who, Where, or What?","au":"Hall, Russell P.","y":2022,"vn":"JID Innovations","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.xjidi.2022.100162","cx":"Journal","co":["Bias","Interventions"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":4,"self":false,"s":"A commentary on persistent problems in scientific peer review notes that assessing the quality of reviews themselves remains an unresolved and under-addressed challenge for the field.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"ILFZLJIP","t":"Associations of topic-specific peer review outcomes and institute and center award rates with funding disparities at the National Institutes of Health.","au":"Lauer, Michael S","y":2021,"vn":"eLife","ty":"journalArticle","tyl":"Journal article","doi":"10.7554/eLife.67173","cx":"Grant","co":["Bias"],"sg":["funding-decision","individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":39,"self":false,"s":"A re-analysis of NIH funding data finds that lower award rates for grant topics favored by Black applicants stem mainly from those topics being assigned to institutes with generally lower funding rates, not from biased peer review scoring itself.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"","ca":"open","cau":""},{"k":"IPM7YXWJ","t":"Anonymous reviewing and the peer-review process.","au":"Surwillo, Walter W.","y":1986,"vn":"American Psychologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/0003-066x.41.2.218.a","cx":"Journal","co":["Bias","Interventions"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":50,"self":false,"s":"A reply/commentary on memory-frequency judgment research, tangential to peer review methodology itself (full context not available from the excerpt).","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"IQ5FTA4C","t":"Double-blind reviewing and gender biases at EvoLang conferences: An update","au":"Cuskley, Christine","y":2019,"vn":"Journal of Language Evolution","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/jole/lzz007","cx":"Journal","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"green","cby":8,"self":false,"s":"Updated analysis of gender effects under double-blind review at Evolution of Language conferences, finding a bias effect seen at an earlier conference largely disappeared at the most recent iteration.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://github.com/seannyD/EvoLang12DoubleBlindData","ca":"open","cau":"https://github.com/seannyD/EvoLang12DoubleBlindData"},{"k":"IQURWTM7","t":"A primer on different models of peer review","au":"Alfred, Jane","y":2023,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31219/osf.io/jnehu","cx":"Journal","co":["Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":0,"self":false,"s":"An overview article for researchers explains how traditional and newer models of peer review differ and outlines what each requires of reviewers, aimed particularly at those new to reviewing.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"IRL44KLX","t":"FENS‐Kavli Network of Excellence: Postponed, non‐competitive peer review for research funding","au":"Dresler, Martin","y":2022,"vn":"European Journal of Neuroscience","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/ejn.15818","cx":"Grant","co":["Reliability-noise","Validity","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"bronze","cby":6,"self":false,"s":"Proposes replacing competitive grant review with a \"funding flat rate\" distributed more evenly across researchers, combined with quality control through delayed, non-competitive review under open science practices.","pg":"not-applicable","pgr":"","pgid":"","da":"open","dau":"https://osf.io/xpaqs","ca":"no-statement","cau":""},{"k":"ITEBWPJP","t":"Unravelling the panel contribution upon peer review evaluation of numerous, unstructured and highly interdisciplinary research proposals","au":"Baimpos, Theodoros","y":2020,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvz013","cx":"Grant","co":["Reliability-noise"],"sg":["panel-consensus","individual-scoring"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":2,"self":false,"s":"Analyzes a two-phase European Commission funding scheme in which remote reviewers first assess highly interdisciplinary proposals and a separate on-site panel later reinforces or revises scores, examining how much the panel stage adds to reviewer agreement and proposal selection.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"","ca":"no-statement","cau":""},{"k":"ITKLS9MM","t":"From funding agencies to scientific agency","au":"Bollen, Johan","y":2014,"vn":"EMBO Reports","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/embr.201338068","cx":"Grant","co":["Validity","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"bronze","cby":61,"self":false,"s":"A commentary examining criticisms of traditional grant peer review, including its cost and low success rates, in the context of major funders like NSF, NIH, and the European Research Council.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"ITXJM6F2","t":"Bias in awarding research grants.","au":"Martin, B","y":1986,"vn":"British medical journal (Clinical research ed.)","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmj.293.6546.550","cx":"Grant","co":["Bias"],"sg":["funding-decision"],"x":0,"st":"conceptual-commentary","oast":"bronze","cby":6,"self":false,"s":"Content unrelated to the stated title on grant bias; the passage discusses trends in UK hospital consultant staffing and specialisation, not peer review, so only a minimal placeholder description is possible from context provided.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"IV59GNII","t":"'Your comments are meaner than your score': score calibration talk influences intra- and inter-panel variability during scientific grant peer review.","au":"Pier, Elizabeth L","y":2017,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvw025","cx":"Grant","co":["Reliability-noise"],"sg":["panel-consensus"],"x":0,"st":"observational-quantitative","oast":"bronze","cby":49,"self":false,"s":"An experimental study with reconstructed NIH-style review panels finds that discussion increases agreement within a panel but decreases agreement between different panels reviewing the same applications, with reviewers' comparative commentary during discussion playing a key role.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"","ca":"no-statement","cau":""},{"k":"IVG78IE3","t":"Statistical reviewers improve reporting in biomedical articles: a randomized trial.","au":"Cobo, Erik","y":2007,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0000332","cx":"Journal","co":["Interventions"],"sg":["individual-scoring"],"x":1,"st":"experimental-rct","oast":"gold","cby":116,"self":false,"s":"Randomized factorial trial testing whether adding a statistical reviewer and/or suggesting reporting checklists to clinical reviewers improved manuscript quality after peer review at a medical journal.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"IWCKCW6T","t":"The forced battle between peer-review and scientometric research assessment: Why the CoARA initiative is unsound","au":"Abramo, Giovanni","y":2024,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvae021","cx":"General","co":["Reliability-noise","Bias","Validity"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"hybrid","cby":30,"self":false,"s":"Critiques the CoARA research-assessment reform initiative, arguing on theoretical and empirical grounds against its preference for qualitative peer judgment over quantitative scientometric methods.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"IWQZXBVF","t":"Randomised controlled trial of the READER method of critical appraisal in general practice","au":"MacAuley, Domhnall","y":1998,"vn":"BMJ","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmj.316.7138.1134","cx":"Analogue","co":["Reliability-noise","Validity","Interventions"],"sg":["individual-scoring"],"x":1,"st":"experimental-rct","oast":"bronze","cby":30,"self":false,"s":"A randomized trial with 243 UK general practitioners compares a structured critical-appraisal method (READER) against free-form appraisal of research papers, finding the structured method produced more consistent, lower quality scores viewed as more appropriately critical.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"IX2FS8HU","t":"Blinded versus unblinded review: A field study comparing the equity of peer review","au":"Pleskac, Timothy Joseph","y":2025,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31234/osf.io/q2tkw_v2","cx":"Journal","co":["Reliability-noise","Bias","Validity"],"sg":["individual-scoring","post-award-validity"],"x":1,"st":"experimental-rct","oast":"gold","cby":0,"self":false,"s":"Field experiment comparing single- and double-blind review of 530 conference submissions found moderate reliability in both, some demographic effects differing by blinding condition, and no clear reliability or validity advantage for double-blind review.","pg":"preregistered","pgr":"OSF","pgid":"","da":"open","dau":"","ca":"open","cau":""},{"k":"IXVXHAXS","t":"An Objective Rating Form to Evaluate Grant Proposals to the Hogg Foundation for Mental Health","au":"Whaley, Arthur L.","y":2006,"vn":"Evaluation Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/0193841x06288737","cx":"Grant","co":["Reliability-noise","Validity","Interventions"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":3,"self":false,"s":"A pilot study tested a structured rating form designed to make a mental-health foundation's grant review more objective, but found the tool's psychometric performance was weaker than in the original developer's study.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"IZAQ9Y4J","t":"role of peer review in the evaluation of research in Italy. Some remarks on the evaluation of PRINs","au":"Vivarelli, Maurizio","y":2022,"vn":"JLIS it","ty":"journalArticle","tyl":"Journal article","doi":"10.36253/jlis.it-500","cx":"Grant","co":["Bias"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"diamond","cby":0,"self":false,"s":"A case study examining peer review and funding decisions for a specific Italian national research grant scheme, focusing on projects funded in one discipline area between 1996 and 2020.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"J2S8SMJC","t":"Alternatives to Peer Review: Novel Approaches for Research Evaluation","au":"Birukou, Aliaksandr","y":2011,"vn":"Frontiers in Computational Neuroscience","ty":"journalArticle","tyl":"Journal article","doi":"10.3389/fncom.2011.00056","cx":"General","co":["Reliability-noise","Bias","Validity","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":67,"self":false,"s":"A review of alternative and complementary approaches to conventional peer review, including reputation-based systems developed by the LiquidPub project, weighing their advantages and disadvantages for research evaluation.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"J2WJBPIT","t":"Scientific publishing without gatekeeping: an empirical investigation of eLife’s new peer review process","au":"Mutz, Rüdiger","y":2025,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-025-05422-y","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":3,"self":false,"s":"Analyzes reviewer ratings under eLife's newer \"publish after review\" model versus its earlier gatekeeping model, finding similarly low interrater agreement and reliability on manuscript significance and rigor across both approaches.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"no-statement","cau":""},{"k":"J4LTG2X6","t":"Putting Journal-based Peer Review in Its Place: Transparency, Post-publication Review, and Systems for Self-correction","au":"Higgins, Wendy C","y":2025,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31234/osf.io/g6fut_v1","cx":"Journal","co":["Validity","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":0,"self":false,"s":"Argues that journal peer review rarely verifies accuracy and cannot alone provide rigorous vetting, proposing a transparent system combining initial screening with open post-publication review by a wider expert pool.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"J583ZHPU","t":"Weighted kappa: Nominal scale agreement provision for scaled disagreement or partial credit.","au":"Cohen, Jacob","y":1968,"vn":"Psychological Bulletin","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/h0026256","cx":"Analogue","co":["Reliability-noise","Validity"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"closed","cby":8557,"self":false,"s":"Original methodological paper introducing weighted kappa, extending the standard kappa agreement statistic to account for varying degrees of disagreement rather than treating all disagreements as equal.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"J5BUPAHX","t":"Administrative discretion in scientific funding: Evidence from a prestigious postdoctoral training program✰","au":"Ginther, Donna K.","y":2020,"vn":"Research Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.respol.2020.103953","cx":"Grant","co":["Validity"],"sg":["funding-decision"],"x":0,"st":"observational-quantitative","oast":"green","cby":21,"self":false,"s":"A study of a prestigious NIH postdoctoral training award compares outcomes predicted by peer-review recommendations versus staff discretion, finding peer review more accurately predicts later research independence and productivity than administrative override.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"J5PCTC3D","t":"Quantification of Agreement in Psychiatric Diagnosis Revisited","au":"Shrout, Patrick E.","y":1987,"vn":"Archives of General Psychiatry","ty":"journalArticle","tyl":"Journal article","doi":"10.1001/archpsyc.1987.01800140084013","cx":"Analogue","co":["Reliability-noise"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":351,"self":false,"s":"Revisits an earlier argument for using Cohen's kappa as the standard measure of diagnostic agreement in psychiatry, reviewing its advantages over older agreement measures.","pg":"no-statement","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"J6L3SI4Q","t":"Fair or flawed? Rethinking grant review with generative AI","au":"Sikimić, Vlasta","y":2025,"vn":"Synthese","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11229-025-05366-z","cx":"Grant","co":["Bias","Validity","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"green","cby":0,"self":false,"s":"A perspective paper outlines methodological and ethical risks of using AI in grant review, arguing that full automation is undesirable and that AI should instead supplement, not replace, human reviewer judgment.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"J92QMRFW","t":"A manifesto for fair and equitable research funding in ecology","au":"Winder, Linton","y":2017,"vn":"Rethinking Ecology","ty":"journalArticle","tyl":"Journal article","doi":"10.3897/rethinkingecology.2.21798","cx":"Grant","co":["Reliability-noise","Bias","Validity","Interventions"],"sg":["funding-decision"],"x":0,"st":"simulation-methodological","oast":"gold","cby":1,"self":false,"s":"A position paper on funding bias in ecology, proposing a combination of short proposals, blinded review, and lottery elements to reduce bias and support innovative research.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"JAKKEBE4","t":"ASSESSMENT OF SUPPORT OF COMPETITIVE SELECTION PROCESSES FOR FUNDING OF SCHOLARLY RESEARCH (R&D) PROJECTS PROVIDED BY THE NATIONAL ELECTRONIC SCIENTIFIC INFORMATION SYSTEM","au":"Жарінова, Алла","y":2026,"vn":"Science and innovation","ty":"journalArticle","tyl":"Journal article","doi":"10.15407/scine22.03.023","cx":"Grant","co":["Interventions"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"diamond","cby":0,"self":false,"s":"Evaluates a national electronic information system supporting competitive grant-selection procedures, concluding that it has reduced organisational and methodological risks and improved transparency, and offering recommendations for further refinement.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"JBIBRH23","t":"Features of successful bids for funding of applied health research: a cohort study","au":"Turner, Sheila","y":2014,"vn":"Health Research Policy and Systems","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/1478-4505-12-54","cx":"Grant","co":["Bias"],"sg":["proposal-quality","funding-decision"],"x":0,"st":"observational-quantitative","oast":"gold","cby":7,"self":false,"s":"A cohort study of health research funding applications finds that having a multidisciplinary team, including a statistician, and completing a feasibility study were the strongest predictors of a proposal's success, with no effect from investigator gender or institution.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"JCNY992P","t":"Panel discussion does not improve reliability of peer review for medical research grant proposals.","au":"Fogelholm, Mikael","y":2012,"vn":"Journal of Clinical Epidemiology","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.jclinepi.2011.05.001","cx":"Grant","co":["Reliability-noise"],"sg":["individual-scoring","panel-consensus"],"x":1,"st":"experimental-rct","oast":"closed","cby":87,"self":false,"s":"Compares scores from two independent expert panels reviewing the same medical research grant proposals, finding panel discussion did not improve agreement compared with simply averaging individual reviewers' scores.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"JD3VGE3T","t":"Bias and Groupthink in Science’s Peer-Review System","au":"Resnik, David B.","y":2020,"vn":"Groupthink in science: greed, pathological altruism, ideology, competition, and culture","ty":"bookSection","tyl":"Book chapter","doi":"10.1007/978-3-030-36822-7_9","cx":"General","co":["Bias","Interventions"],"sg":["individual-scoring"],"x":0,"st":"book-monograph","oast":"closed","cby":25,"self":false,"s":"Book chapter reviewing evidence on biases and groupthink dynamics in scientific peer review and discussing ways to reduce their influence on funding and publication decisions.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"JEMVX4GK","t":"Dear Reviewer 2: Go F’ Yourself","au":"Peterson, David A.","y":2020,"vn":"Social Science Quarterly","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/ssqu.12824","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":50,"self":false,"s":"An empirical test of the folk belief that \"Reviewer 2\" is unusually harsh, using a political science journal's reviewer database, finds no evidence supporting that reputation but identifies a different reviewer slot as disproportionately negative.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"no-statement","cau":""},{"k":"JEXNP37Z","t":"The need for a research agenda.","au":"Bailar, J C","y":1985,"vn":"The New England Journal of Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.1056/NEJM198503073121023","cx":"Journal","co":["general-systemic"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":163,"self":false,"s":"Essay proposes four competing metaphors, screening gatekeeper, arbitrary determinant of venue, manuscript-improving process, and unpredictable lottery, for characterizing what journal peer review actually does.","pg":"not-codeable","pgr":"","pgid":"","da":"","dau":"","ca":"","cau":""},{"k":"JFCN34TN","t":"NIH Peer Review of Grant Applications for Clinical Research","au":"Kotchen, Theodore A.","y":2004,"vn":"JAMA","ty":"journalArticle","tyl":"Journal article","doi":"10.1001/jama.291.7.836","cx":"Grant","co":["Bias"],"sg":["funding-decision"],"x":0,"st":"observational-quantitative","oast":"closed","cby":104,"self":false,"s":"An analysis of NIH grant outcomes between 1997 and 2002 found clinical-research applications received somewhat less favorable priority scores and funding rates than non-clinical research, despite similar volumes submitted by physician and non-physician investigators.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"JGLEEC55","t":"An experimental test of the effects of redacting grant applicant identifiers on peer review outcomes","au":"Nakamura, Richard","y":2021,"vn":"eLife","ty":"journalArticle","tyl":"Journal article","doi":"10.7554/eLife.71368","cx":"Grant","co":["Bias","Interventions"],"sg":["individual-scoring"],"x":1,"st":"experimental-rct","oast":"gold","cby":38,"self":false,"s":"An experiment redacting identifying information from 1,200 NIH grant applications found that redaction reduced but did not eliminate reviewers' ability to guess applicant identity, and roughly halved a scoring gap favoring White over Black investigators.","pg":"preregistered","pgr":"OSF","pgid":"https://osf.io/3vmfz","da":"open","dau":"","ca":"no-statement","cau":""},{"k":"JGPW43LL","t":"Erosion of peer review?","au":"van der Linden, Wim","y":2014,"vn":"Educational Measurement: Issues and Practice","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/emip.12044","cx":"Journal","co":["Validity","Interventions"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":0,"self":false,"s":"A commentary disputes a journal editor's view that rising submission and acceptance numbers at a conference necessarily lowered average presentation quality, addressing a restructured proposal-review process introduced to expand accepted slots.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"JHIX7RTB","t":"Does single blind peer review hinder newcomers?","au":"Seeber, Marco","y":2017,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-017-2264-7","cx":"Journal","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":64,"self":false,"s":"Comparing single-blind and double-blind review across thousands of computer science conference papers, this study finds single-blind review is associated with a lower share of contributions from newcomer authors relative to established researchers.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"JITSB8WY","t":"Evolution of Peer Review in Scientific Communication","au":"Kochetkov, Dmitry","y":2024,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31235/osf.io/b2ra3_v1","cx":"Journal","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":0,"self":false,"s":"Traces the historical development of journal peer review and identifies three current models (traditional pre-publication review, registered reports, post-publication review), arguing the last best serves science communication's core aims.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"JIUWDBMD","t":"An end to ‘God-like’ scientific knowledge? How non-anonymous referees and open review alter meanings for scientific knowledge","au":"Gaudet, Joanne","y":2017,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31219/osf.io/6ypuw","cx":"Journal","co":["general-systemic"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":0,"self":false,"s":"Theoretical essay examining how the shift from anonymous to non-anonymous, open journal peer review has changed the meaning and authority attributed to scientific knowledge.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"JJYTBKG4","t":"Assessing health research grant applications: A retrospective comparative review of a one-stage versus a two-stage application assessment process.","au":"Morgan, Ben","y":2020,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0230118","cx":"Grant","co":["Interventions"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"gold","cby":5,"self":false,"s":"A retrospective comparison of one-stage versus two-stage grant application review at a UK health research funder finds the two-stage process took longer overall but informed unsuccessful applicants sooner and required fewer total reviews.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"","ca":"no-statement","cau":""},{"k":"JMK4CLUJ","t":"Conditions that do or do not disadvantage interdisciplinary research proposals in project evaluation","au":"Seeber, Marco","y":2022,"vn":"Journal of the Association for Information Science and Technology","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/asi.24617","cx":"Grant","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":26,"self":false,"s":"Analysis of nearly 2,000 European COST funding proposals evaluated under a panel-free system found no meaningful funding disadvantage for more interdisciplinary proposals, regardless of scientific field.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"JPKCD6M8","t":"Statistical notes for clinical researchers: Evaluation of measurement error 2: Dahlberg's error, Bland-Altman method, and Kappa coefficient","au":"Kim, Hae‐Yeong","y":2013,"vn":"Restorative Dentistry & Endodontics","ty":"journalArticle","tyl":"Journal article","doi":"10.5395/rde.2013.38.3.182","cx":"Analogue","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"diamond","cby":87,"self":false,"s":"An educational note explains three statistical methods, Dahlberg's error, the Bland-Altman method, and the kappa coefficient, used by clinical researchers to quantify measurement error and agreement.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"JRSMWBU4","t":"Peer review and panel decisions in the assessment of Australian Research Council project grant applicants: what counts in a highly competitive context?","au":"Bazeley, Pat","y":1998,"vn":"Higher Education","ty":"journalArticle","tyl":"Journal article","doi":"10.1023/a:1003118502318","cx":"Grant","co":["Bias"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"closed","cby":38,"self":false,"s":"Analyzing Australian Research Council grant applications, this study found academic status and publication record, but not age or gender, significantly influenced assessor ratings, consistent with a cumulative-advantage (\"Matthew effect\") pattern.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"JTR3R42M","t":"Bayesian Nonparametric Models for Multiple Raters: A General Statistical Framework","au":"Mignemi, Giuseppe","y":2025,"vn":"Psychometrika","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/psy.2025.10035","cx":"Analogue","co":["Reliability-noise","Bias"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"hybrid","cby":2,"self":false,"s":"Proposes a flexible Bayesian nonparametric statistical framework for analysing ratings from multiple raters that relaxes the strict assumptions of standard multilevel models, allowing more accurate estimation of intraclass correlation despite rater and subject heterogeneity.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"https://osf.io/3yx4j/?view_only=98c600198a6b4807878989765118f97e","ca":"open","cau":"https://osf.io/3yx4j/?view_only=98c600198a6b4807878989765118f97e"},{"k":"JU4PBD6P","t":"Policy Considerations for Random Allocation of Research Funds","au":"Avin, Shahar","y":2017,"vn":"PhilSci-Archive (University of Pittsburgh)","ty":"journalArticle","tyl":"Journal article","doi":"10.13130/2282-5398/8626","cx":"Grant","co":["Reliability-noise","Validity","Interventions"],"sg":["funding-decision"],"x":0,"st":"conceptual-commentary","oast":"green","cby":27,"self":false,"s":"Explores practical policy considerations, including cost and fairness, for implementing proposals to allocate a portion of research funding through random selection rather than peer review alone.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"JX8TSFKY","t":"Chance and interrater agreement on manuscripts.","au":"Watkins, Marley W.","y":1979,"vn":"American Psychologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/0003-066x.34.9.796","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":115,"self":false,"s":"Discusses chance-corrected measures of reviewer agreement on manuscripts within a broader piece on behavioral assessment methodology (fuller context on peer review specifically not available from the excerpt).","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"JXSMF6RN","t":"Gender differences in peer reviewed grant applications, awards, and amounts: a systematic review and meta-analysis","au":"Schmaling, Karen B.","y":2023,"vn":"Research Integrity and Peer Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s41073-023-00127-3","cx":"Grant","co":["Bias"],"sg":["funding-decision"],"x":0,"st":"systematic-review-meta-analysis","oast":"gold","cby":70,"self":false,"s":"Systematic review and meta-analysis of 55 sources on gender differences in grant funding, finding award acceptance rates were only marginally higher for men and differences varied widely across studies.","pg":"preregistered","pgr":"PROSPERO","pgid":"CRD42021232153","da":"open","dau":"https://osf.io/e5t4v/","ca":"open","cau":"https://osf.io/e5t4v/"},{"k":"JXTFVYUU","t":"Funding Science: The Real Defects of Peer Review and An Alternative To It","au":"Roy, Rustum","y":1985,"vn":"Science Technology & Human Values","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/016224398501000309","cx":"Grant","co":["Validity","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":141,"self":false,"s":"An older critical essay argues that grant peer review has entrenched flaws rooted in disciplinary culture and career incentives, and that viable alternative funding-allocation systems already exist and are used elsewhere.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"JZW58HZM","t":"Sunlight not shadows: Double-anonymized peer review is not the progressive answer to status bias","au":"Horbach, Serge P. J. M.","y":2022,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31219/osf.io/fqb5c","cx":"Journal","co":["Bias","Interventions"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":4,"self":false,"s":"Response article argues that double-anonymized review is not an adequate progressive fix for status-related bias in journal peer review, advocating instead for more open and collaborative publishing models.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"K3DZVZXI","t":"A New Way to Distribute Research Seed Funding: Peer Review Without Formal Review","au":"Burns, Mark A.","y":2021,"vn":"Change The Magazine of Higher Learning","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/00091383.2021.1883976","cx":"Grant","co":["Reliability-noise","Interventions"],"sg":["funding-decision"],"x":0,"st":"observational-quantitative","oast":"closed","cby":0,"self":false,"s":"Describes a University of Michigan seed-funding program, Mcubed, designed to allocate small research grants with minimal formal review, motivated by evidence that traditional review is slow and only marginally better than random selection.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"K4EVXX88","t":"Retractions and Rewards in Science: An Open Question for Reviewers and Funders","au":"Ribeiro, Mariana Dias","y":2023,"vn":"Science and Engineering Ethics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11948-023-00446-0","cx":"Grant","co":["general-systemic"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":6,"self":false,"s":"Interviews with funding-agency representatives and a survey of 224 US grant reviewers found that retractions and research self-correction were seen as important for scientific integrity but were not perceived as factors actually influencing grant funding decisions.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"K5N2TIMX","t":"ChatGPT identifies gender disparities in scientific peer review","au":"Verharen, Jeroen P. H.","y":2023,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.7554/elife.90230.2","cx":"Journal","co":["Reliability-noise","Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":1,"self":false,"s":"Using ChatGPT to analyse language in peer-review reports for published neuroscience papers, the study finds most reviews were favourable and polite, but reviews of papers with female first authors were less polite, and papers with female senior authors received more favourable reviews.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"","ca":"no-statement","cau":""},{"k":"K64P7MNV","t":"Potential bias in peer review of grant applications at the Swiss National Science Foundation","au":"Severin, Anna","y":2019,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.7287/peerj.preprints.27587v1","cx":"Grant","co":["Bias"],"sg":["individual-scoring","reviewer-selection"],"x":0,"st":"observational-quantitative","oast":"gold","cby":3,"self":false,"s":"A large-scale analysis of Swiss National Science Foundation reviews finds that reviewer-nominating source, reviewer location, and gender were associated with score differences, though gender effects were mostly explained once other factors were controlled for.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"open","cau":"www.git.io/fhaJx"},{"k":"KARAZVW9","t":"Assessment of potential bias in research grant peer review in Canada.","au":"Tamblyn, Robyn","y":2018,"vn":"Canadian Medical Association Journal","ty":"journalArticle","tyl":"Journal article","doi":"10.1503/cmaj.170901","cx":"Grant","co":["Reliability-noise","Bias"],"sg":["reviewer-selection","individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"gold","cby":156,"self":false,"s":"Analysis of over 11,000 Canadian Institutes of Health Research grant applications finding lower scores associated with female applicants, applied researchers, and reviewers outside the applicant's field, alongside effects from productivity and resubmission history.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"KC5HMJS8","t":"A scoping review on biomedical journal peer review guides for reviewers","au":"Song, Eunhye","y":2021,"vn":"PLoS ONE","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0251440","cx":"Journal","co":["Interventions"],"sg":["individual-scoring"],"x":0,"st":"systematic-review-meta-analysis","oast":"gold","cby":40,"self":false,"s":"Scoping review of published peer-review guidelines and checklists for biomedical journal reviewers, finding wide variation across publishers and calling for more standardized guidance.","pg":"preregistered","pgr":"other","pgid":"reviewregistry881","da":"open","dau":"","ca":"no-statement","cau":""},{"k":"KFAYWFPV","t":"Does the committee peer review select the best applicants for funding? An investigation of the selection process for two European molecular biology organization programmes.","au":"Bornmann, Lutz","y":2008,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0003480","cx":"Grant","co":["Validity"],"sg":["post-award-validity","funding-decision"],"x":0,"st":"observational-quantitative","oast":"gold","cby":116,"self":false,"s":"A study of two European Molecular Biology Organization fellowship programmes compares funding decisions against later publication and citation performance, finding a statistically significant link between selection and future achievement, alongside a substantial rate of prediction errors.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"KIXZ6B2J","t":"Grant Research Support in Russia: What Can We Learn from the Russian Science Foundation’s First Grant Competition?","au":"Simachev, Yuri","y":2017,"vn":"Foresight-Russia","ty":"journalArticle","tyl":"Journal article","doi":"10.17323/2500-2597.2017.4.74.83","cx":"Grant","co":["Bias"],"sg":["funding-decision"],"x":0,"st":"observational-quantitative","oast":"green","cby":5,"self":false,"s":"Analysis of the Russian Science Foundation's first 2014 grant competition finds winners were more concentrated in life sciences, affiliated with the national academy or Moscow institutions, though external expert scores remained the strongest predictor of success.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"KMEYIQ9Y","t":"Does Masking Author Identity Improve Peer Review Quality?","au":"Justice, Amy C.","y":1998,"vn":"The Journal of the American Medical Association","ty":"journalArticle","tyl":"Journal article","doi":"10.1001/jama.280.3.240","cx":"Journal","co":["Bias","Interventions"],"sg":["individual-scoring"],"x":1,"st":"experimental-rct","oast":"closed","cby":280,"self":false,"s":"A randomized trial across five biomedical journals found masking author identity from reviewers made no significant difference in review quality as judged by authors and editors, and masking itself was often incomplete.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"KNJE6VS9","t":"A Bayesian approach to ranking and rater evaluation: An application to grant reviews.","au":"Cao, Jing","y":2010,"vn":"Journal of Educational and Behavioral Statistics","ty":"journalArticle","tyl":"Journal article","doi":"10.3102/1076998609353116","cx":"Grant","co":["Reliability-noise","Bias"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"closed","cby":16,"self":false,"s":"This paper develops a Bayesian hierarchical model for ranking items rated by multiple raters that separately estimates each rater's bias, discrimination, and error, applying the approach to grant review data.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"KPLP8QZE","t":"Gender Bias Impacts Top-Merited Candidates.","au":"Andersson, Emma Rachel","y":2021,"vn":"Frontiers in research metrics and analytics","ty":"journalArticle","tyl":"Journal article","doi":"10.3389/frma.2021.594424","cx":"Grant","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":20,"self":false,"s":"Analysing recruitment-grant reviews at a Swedish medical university, the study finds that although research fields were not gender-segregated, women's productivity translated into lower reviewer merit scores than equally productive men, with the gap widening at higher productivity levels.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"no-statement","cau":""},{"k":"KPSCZ7V2","t":"Intraclass correlations: Uses in assessing rater reliability.","au":"Shrout, Patrick E.","y":1979,"vn":"Psychological Bulletin","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/0033-2909.86.2.420","cx":"Analogue","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"closed","cby":23060,"self":false,"s":"A methodological guide explaining how to select the appropriate form of the intraclass correlation coefficient depending on study design and intended use in assessing rater reliability.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"KRNLC6TD","t":"A multi-disciplinary perspective on emergent and future innovations in peer review","au":"Tennant, Jonathan","y":2017,"vn":"F1000Research","ty":"journalArticle","tyl":"Journal article","doi":"10.12688/f1000research.12037.2","cx":"Journal","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":40,"self":false,"s":"Reviews emerging web-based peer-review models across disciplines against core functions of quality control and reputation-building, and proposes a hybrid platform aimed at reducing existing biases.","pg":"not-applicable","pgr":"","pgid":"","da":"open","dau":"https://doi.org/10.6084/m9.figshare.5117260","ca":"open","cau":"https://doi.org/10.6084/m9.figshare.5117260"},{"k":"KRP5YXFM","t":"Where do field-specific notions of research quality come from?","au":"Laudel, Grit","y":2024,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvae027","cx":"Journal","co":["general-systemic"],"sg":["individual-scoring"],"x":0,"st":"qualitative","oast":"hybrid","cby":7,"self":false,"s":"Qualitative study comparing how researchers in plant biology and medieval history define research quality, finding field-specific criteria shaped by each discipline's distinct research practices rather than generic notions like originality or validity.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"KVALUL6S","t":"The Necessity of Commensuration Bias in Grant Peer Review","au":"Heesen, Remco","y":2022,"vn":"Ergo an Open Access Journal of Philosophy","ty":"journalArticle","tyl":"Journal article","doi":"10.3998/ergo.2240","cx":"Grant","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"diamond","cby":4,"self":false,"s":"A philosophical paper applies social choice theory to argue that when reviewers aggregate separate criterion scores into an overall grant score, some degree of so-called commensuration bias is often unavoidable.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"KWATA3RR","t":"A new reader trial approach to peer review in funding research grants: An Australian experiment","au":"Jayasinghe, Upali W.","y":2006,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-006-0171-4","cx":"Grant","co":["Reliability-noise","Interventions"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":38,"self":false,"s":"Australian trial compared traditional grant peer review with a \"reader system\" where senior academics read all proposals in a subdiscipline, finding the reader system produced substantially higher reliability.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"KXATMKHW","t":"Impact of interventions to improve the quality of peer review of biomedical journals: a systematic review and meta-analysis.","au":"Bruce, Rachel","y":2016,"vn":"BMC Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s12916-016-0631-5","cx":"Journal","co":["Interventions"],"sg":["whole-process"],"x":0,"st":"systematic-review-meta-analysis","oast":"gold","cby":230,"self":false,"s":"A systematic review and meta-analysis of 22 trials found that reviewer training and manuscript checklists did not improve review or manuscript quality, while adding a statistical reviewer and open review showed some measurable benefit.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"no-statement","cau":""},{"k":"KXCEL6DG","t":"Does Peer Review Identify the Best Papers? A Simulation Study of Editors, Reviewers, and the Scientific Publication Process","au":"Esarey, Justin","y":2017,"vn":"PS Political Science & Politics","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/s1049096517001081","cx":"Journal","co":["Reliability-noise","Validity"],"sg":["whole-process"],"x":0,"st":"simulation-methodological","oast":"bronze","cby":23,"self":false,"s":"A simulation study of peer review systems found that, regardless of design, many accepted papers would not be judged as meeting journal standards by an average reader, though an active editor using desk rejection could reduce this problem.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"KXEIIMC8","t":"A survey of how biology researchers assess credibility when serving on grant and hiring committees","au":"Hrynaszkiewicz I.","y":2026,"vn":"PeerJ","ty":"journalArticle","tyl":"Journal article","doi":"10.7717/peerj.20502","cx":"General","co":["general-systemic"],"sg":["reviewer-selection","individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":0,"self":false,"s":"A survey of 485 biology researchers serving on grant and hiring committees finds that most consider assessing research credibility highly important, but fewer than half feel satisfied with their own ability to judge it, especially for signs of research misconduct.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://doi.org/10.6084/m9.figshare.22522372.v1","ca":"no-statement","cau":""},{"k":"KY6KDPFD","t":"The peer review process for awarding funds to international science research consortia: a qualitative developmental evaluation","au":"Gregorius, Stefanie","y":2018,"vn":"F1000Research","ty":"journalArticle","tyl":"Journal article","doi":"10.12688/f1000research.12496.3","cx":"Grant","co":["Reliability-noise","Interventions"],"sg":["panel-consensus"],"x":0,"st":"qualitative","oast":"gold","cby":9,"self":false,"s":"A qualitative study observing panel meetings and interviewing panelists for an international science funding programme, finding the review process was generally rigorous but inconsistently applied due to unclear criteria weighting.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"10.5256/f1000research.12496.d178727","ca":"no-statement","cau":""},{"k":"L32ZULYV","t":"Enhancing Consistency in Peer Review: A Statistical Analysis of Discrepancies and Proposals for Improvement","au":"Alarfaj, Maher Mohammed","y":2025,"vn":"Learned Publishing","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/leap.2033","cx":"General","co":["Reliability-noise","Interventions"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":1,"self":false,"s":"Statistical analysis of scoring discrepancies among reviewer groups judging manuscripts for an educational award, proposing calibration methods to improve consistency across reviewers.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"no-statement","cau":""},{"k":"L3X5I6CT","t":"Evaluating high risk research: an assessment of the Wellcome Trust's Sir Henry Wellcome Commemorative Awards for Innovative Research","au":"Grant, Julius","y":1999,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.3152/147154499781777513","cx":"Grant","co":["Validity"],"sg":["individual-scoring"],"x":1,"st":"experimental-rct","oast":"closed","cby":19,"self":false,"s":"Evaluates the Wellcome Trust's Showcase awards program for high-risk research, using an experimental panel assessment to show the scheme successfully identified more innovative proposals than standard grants.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"L6CC3BZQ","t":"How pre-publication journal peer review (re)produces ignorance at scientific and medical journals: a case study","au":"Gaudet, Joanne","y":2017,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31219/osf.io/3dq5j","cx":"Journal","co":["general-systemic"],"sg":["whole-process"],"x":0,"st":"qualitative","oast":"gold","cby":3,"self":false,"s":"A qualitative case study of journal peer review examines how its structural features, including secrecy and accountability norms, can reproduce rather than reduce scientific ignorance by shaping which manuscripts are deemed legitimate.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"L6N4AXR4","t":"“It takes a village to write a really good paper”: A normative framework for peer reviewing in philosophy","au":"Copeland, Samantha","y":2024,"vn":"Metaphilosophy","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/meta.12670","cx":"Journal","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"hybrid","cby":3,"self":false,"s":"Proposes a normative framework for reforming peer review specifically within philosophy, arguing that transparency and democratic participation should guide reform to make the discipline fairer and more inclusive.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"L6S33TPR","t":"Imperfect referees: Reducing the impact of multiple biases in peer review","au":"Yun, Zhao","y":2015,"vn":"Journal of the Association for Information Science and Technology","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/asi.23317","cx":"Journal","co":["Bias","Interventions"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"closed","cby":8,"self":false,"s":"Analyzing real conference review data, the study found reviewers' fixed characteristics, such as seniority and stated confidence, influenced bias more than the order in which they reviewed papers over time.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"L7CAZX6J","t":"The practicalities of a partial lottery to allocate research funding","au":"Meer, Martijn van der","y":2024,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvae023","cx":"Grant","co":["Validity","Interventions"],"sg":["funding-decision"],"x":0,"st":"conceptual-commentary","oast":"hybrid","cby":2,"self":false,"s":"Drawing on a lottery-based seed-funding scheme at a Dutch university, this paper describes the practical implementation of partial lottery allocation and shares lessons for designing similar procedures elsewhere.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"L7XCUZU9","t":"Effect of structured workshop training on subsequent performance of journal peer reviewers.","au":"Callaham, Michael L","y":2002,"vn":"Annals of Emergency Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.1067/mem.2002.127121","cx":"Journal","co":["Interventions"],"sg":["individual-scoring"],"x":1,"st":"experimental-rct","oast":"bronze","cby":52,"self":false,"s":"A controlled study of a formal, interactive peer-reviewer training workshop found that attendees' subsequent review quality ratings did not improve more than those of matched reviewers who had not attended.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"LAHV4J24","t":"Strengthening Community Involvement in Grant Review: Insights from the Community–University Research Partnership (CURES) Pilot Review Process","au":"Paberzs, Adam","y":2014,"vn":"Clinical and Translational Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/cts.12141","cx":"Grant","co":["Reliability-noise","Interventions"],"sg":["panel-consensus","reviewer-selection"],"x":0,"st":"observational-quantitative","oast":"green","cby":16,"self":false,"s":"Describes a pilot programme at a US academic medical center that involved community members in reviewing funding proposals for community-university research partnerships, outlining benefits and challenges observed over five years.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"LAPY8Q5J","t":"Improving editorial procedures.","au":"Crandall, Rick","y":1990,"vn":"American Psychologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/0003-066x.45.5.665","cx":"Journal","co":["Reliability-noise","Bias","Interventions"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":38,"self":false,"s":"Commentary responding to a proposal about journal editorial procedures, arguing that separating manuscript writing from methodological review stages would undermine narrative coherence and unfairly shift what publication decisions are based on.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"LB5K9AI7","t":"Networking and knowledge exchange to promote the formation of transdisciplinary coalitions and levels of agreement among transdisciplinary peer reviewers.","au":"Lobb, Rebecca","y":2013,"vn":"Journal of public health management and practice : JPHMP","ty":"journalArticle","tyl":"Journal article","doi":"10.1097/PHH.0b013e31823991c2","cx":"Grant","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":13,"self":false,"s":"Canadian study of a transdisciplinary funding initiative finding that pre-application networking activities were linked to funding success, while agreement among research, practice, and policy peer reviewers was low.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"LCGWFM9G","t":"Peer review: the attitudes and behaviours of Covid-19 pandemic-era early career researchers","au":"Nicholas, David","y":2023,"vn":"El Profesional de la Informacion","ty":"journalArticle","tyl":"Journal article","doi":"10.3145/epi.2023.may.06","cx":"Journal","co":["Interventions"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"diamond","cby":9,"self":false,"s":"A mixed-methods study interviews and surveys early-career researchers internationally about pandemic-era peer review experiences, finding continued reservations about trustworthiness, persistent slow and inadequate reviewing, and mostly temporary pandemic-related disruptions.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"LDQ9B77W","t":"The relationship of previous training and experience of journal peer reviewers to subsequent review quality.","au":"Callaham, Michael L","y":2007,"vn":"PLoS Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pmed.0040040","cx":"Journal","co":["Interventions"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":140,"self":false,"s":"Survey and four-year tracking of over 300 journal reviewers found most measured training and experience variables, including academic rank, did not predict review quality, with only workplace setting and relative junior status weakly associated.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"LEGPWSWB","t":"The validity of peer review in a general medicine journal.","au":"Jackson, Jeffrey L","y":2011,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0022475","cx":"Journal","co":["Reliability-noise","Validity"],"sg":["individual-scoring","funding-decision","post-award-validity"],"x":0,"st":"observational-quantitative","oast":"gold","cby":74,"self":false,"s":"A year-long study at a general medicine journal found reviewer quality ratings predicted later citation impact only weakly and inconsistently, though editorial accept/reject decisions distinguished high- and low-impact articles fairly well, and agreement between individual reviewers was low.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"LFCNU42R","t":"Peer review for improving the quality of grant applications.","au":"Demicheli, V","y":2007,"vn":"Cochrane Database of Systematic Reviews","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/14651858.MR000003.pub2","cx":"Grant","co":["Reliability-noise","Interventions"],"sg":["whole-process"],"x":0,"st":"systematic-review-meta-analysis","oast":"green","cby":100,"self":false,"s":"This Cochrane review of ten studies on grant peer review interventions found little strong evidence that different screening, blinding, or decision-making procedures affected the quality of funded research, though reviewer agreement was generally high.","pg":"registered-report","pgr":"Cochrane","pgid":"10.1002/14651858.MR000003 (protocol)","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"LFVVACBY","t":"Reviewer ratings and agreement on manuscripts reviewed for the Journal of Counseling Psychology..","au":"Munley, Patrick H.","y":1988,"vn":"Journal of Counseling Psychology","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/0022-0167.35.2.198","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":17,"self":false,"s":"Studying 263 manuscripts and 207 paired reviews at a counseling psychology journal, the authors find that ratings of a paper's overall importance and methodological quality were most linked to reviewer and editorial decisions, with reviewer agreement described as modest.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"LHCCRG7C","t":"Are gender gaps due to evaluations of the applicant or the science? A natural experiment at a national funding agency.","au":"Witteman, Holly O","y":2019,"vn":"The Lancet","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/S0140-6736(18)32611-4","cx":"Grant","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":561,"self":false,"s":"Analyzing a Canadian funding agency's natural experiment that separated evaluation of the investigator from evaluation of the proposed science, this study finds the gender funding gap is driven by less favorable assessments of women as investigators rather than of their research.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"no-statement","cau":""},{"k":"LHXMSP4I","t":"Perspective: Is NIH Funding the “Best Science by the Best Scientists”? A Critique of the NIH R01 Research Grant Review Policies","au":"Costello, Leslie C.","y":2010,"vn":"Academic Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.1097/acm.0b013e3181d74256","cx":"Grant","co":["general-systemic"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"bronze","cby":33,"self":false,"s":"Critiques recent changes to NIH R01 grant review policy, arguing certain new requirements introduce forms of discrimination that work against the stated goal of funding the highest-quality science.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"LIB9SBY5","t":"Is Peer Review a Good Idea?","au":"Heesen, Remco","y":2019,"vn":"The British Journal for the Philosophy of Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/bjps/axz029","cx":"Journal","co":["Validity","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"hybrid","cby":129,"self":false,"s":"Argument paper concluding that, based on currently available evidence, abolishing prepublication peer review would likely be neutral or beneficial for science rather than harmful.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"LJCKBTRE","t":"What is submitted and what gets accepted in Indian Pediatrics: Analysis of submissions, review process, decision making, and criteria for rejection","au":"Gupta, Piyush","y":2013,"vn":"Indian Pediatrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s13312-013-0001-8","cx":"Journal","co":["Reliability-noise","Bias"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"closed","cby":34,"self":false,"s":"A retrospective analysis of manuscripts submitted to an Indian pediatrics journal in 2002 examines acceptance patterns and reports low inter-reviewer agreement (kappa around 0.2-0.35) among assigned reviewer pairs.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"LKWH7CPF","t":"Corrigendum: Prestigious Science Journals Struggle to Reach Even Average Reliability (Frontiers in Human Neuroscience, (2018), 12, 10.3389/fnhum.2018.00037)","au":"Brembs B.","y":2018,"vn":"Frontiers in Human Neuroscience","ty":"journalArticle","tyl":"Journal article","doi":"10.3389/fnhum.2018.00376","cx":"Journal","co":["Validity"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":0,"self":false,"s":"Correction notice to an earlier article discussing declining acceptance rates and reliability concerns at prestigious science journals amid growing submission competition.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"LLMTWE7V","t":"Partial and particularistic: grant peer review panels evaluating interdisciplinarity","au":"Oxley, Kristin","y":2024,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvaf037","cx":"Grant","co":["Bias"],"sg":["panel-consensus"],"x":0,"st":"qualitative","oast":"hybrid","cby":1,"self":false,"s":"Based on observations of grant panels and interviews with panelists, the study finds that interdisciplinary proposals face disadvantages because individual reviewers tend to favor work resembling their own field, and panel dynamics can reinforce rather than correct this tendency.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"LLTEQ7U5","t":"Peer Review, Program Officers and Science Funding","au":"Roebber, Paul J.","y":2011,"vn":"PLoS ONE","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0018680","cx":"Grant","co":["Interventions"],"sg":["whole-process"],"x":0,"st":"simulation-methodological","oast":"gold","cby":35,"self":false,"s":"An agent-based model of the grant funding cycle examines how program officer and reviewer behavior affect proposal-submission strategies, finding that under scarce funding, submitting many proposals becomes scientists' most effective approach despite reducing overall quality.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"LNLZUDIW","t":"Gender differences in Dutch research funding over time: A statistical investigation of the innovation scheme 2012-2021.","au":"Albers, Casper","y":2024,"vn":"PloS one","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0297311","cx":"Grant","co":["Bias","Interventions"],"sg":["funding-decision"],"x":0,"st":"observational-quantitative","oast":"gold","cby":7,"self":false,"s":"Examining a decade of applications to a Dutch research council's tiered career-grant scheme, the study finds that after 2015 anti-bias measures were introduced, female applicants became more likely than male applicants to win the earliest-career grant tier.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://osf.io/8bfaz/","ca":"open","cau":"https://osf.io/8bfaz/"},{"k":"LP9VFUWK","t":"Frequency and Type of Conflicts of Interest in the Peer Review of Basic Biomedical Research Funding Applications: Self-Reporting Versus Manual Detection","au":"Gallo, Stephen A.","y":2015,"vn":"Science and Engineering Ethics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11948-015-9631-7","cx":"Grant","co":["Bias"],"sg":["reviewer-selection"],"x":0,"st":"observational-quantitative","oast":"green","cby":7,"self":false,"s":"A retrospective analysis of conflict-of-interest reporting in biomedical grant review panels finds most identified conflicts were institutional or collaborative, and that more conflicts were caught through manual staff checks than through reviewer self-reporting.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"LPDZSVPC","t":"First Comprehensive Survey of NSF Applicants Focuses on Their Concerns About Proposal Review","au":"McCullough, Jim","y":1989,"vn":"Science Technology & Human Values","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/016224398901400107","cx":"Grant","co":["Reliability-noise","Bias"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"closed","cby":38,"self":false,"s":"Describes the design and early findings of the first broad NSF survey of grant applicants about their concerns with the proposal review and decision-making process.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"LR2JK33N","t":"Back to the roots: reimagining scientific evaluation of research without peer review","au":"Sallam, Malik","y":2026,"vn":"Insights the UKSG journal","ty":"journalArticle","tyl":"Journal article","doi":"10.1629/uksg.714","cx":"Journal","co":["Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"diamond","cby":0,"self":false,"s":"Opinion piece calling for scholarly publishing to replace external peer review with an editor-driven model in which named academic editors take sole, publicly accountable responsibility for publication decisions.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"LVKA3ZWD","t":"Individual versus general structured feedback to improve agreement in grant peer review: a randomized controlled trial","au":"Hesselberg, Jan-Ole","y":2021,"vn":"Research Integrity and Peer Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s41073-021-00115-5","cx":"Grant","co":["Reliability-noise","Interventions"],"sg":["individual-scoring"],"x":1,"st":"experimental-rct","oast":"gold","cby":8,"self":true,"s":"A randomized controlled trial with 42 reviewers at a Norwegian funder compares individual versus general feedback reports on subsequent reviewer agreement, finding the general-feedback group showed higher eligibility agreement, while overall score agreement stayed low in both groups.","pg":"preregistered","pgr":"OSF","pgid":"osf.io/n4fq3","da":"open","dau":"OSF.io/6rdvc","ca":"no-statement","cau":""},{"k":"LW6BBXR7","t":"Effect of Blinding and Unmasking on the Quality of Peer Review","au":"Rooyen, Susan van","y":1998,"vn":"JAMA","ty":"journalArticle","tyl":"Journal article","doi":"10.1001/jama.280.3.234","cx":"Journal","co":["Interventions"],"sg":["individual-scoring"],"x":1,"st":"experimental-rct","oast":"bronze","cby":288,"self":false,"s":"Randomised trial of 527 BMJ submissions found that neither blinding reviewers to author identity nor revealing reviewer identity to a co-reviewer significantly changed review quality, recommendations, or review time.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"LYM4S2ML","t":"Positive bias for European men in peer reviewed applications for faculty position at Karolinska Institutet","au":"Hagg S.","y":2018,"vn":"F1000Research","ty":"journalArticle","tyl":"Journal article","doi":"10.12688/f1000research.13030.2","cx":"General","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":5,"self":false,"s":"A follow-up to a landmark 1997 study examined a 2014 Swedish faculty hiring competition and found merit scores for men, but not women, were linked to productivity and prior grants, and applicants from Middle Eastern backgrounds were disproportionately rated lowest.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"no-statement","cau":""},{"k":"LZBRQ7CQ","t":"PREPRINT: Open science and modified funding lotteries can impede the natural selection of bad science","au":"Smaldino","y":2019,"vn":"Open Science Framework","ty":"journalArticle","tyl":"Journal article","doi":"10.1098/rsos.190194","cx":"General","co":["Validity","Interventions"],"sg":["funding-decision"],"x":0,"st":"simulation-methodological","oast":"gold","cby":55,"self":false,"s":"A preprint version of a computational modeling study showing that lottery-based funding restricted by a methodological rigor threshold, combined with open-science practices, can reduce false discoveries in published research.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://doi.org/10.6084/m9.figshare.c.4554284","ca":"open","cau":"https://github.com/mt-digital/badscience-solutions"},{"k":"LZDVVTHX","t":"Using Peer Review to Improve Research and Promote Collaboration","au":"Kupfer, David J.","y":2014,"vn":"Academic Psychiatry","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s40596-013-0027-1","cx":"Grant","co":["Interventions"],"sg":["panel-consensus"],"x":0,"st":"observational-quantitative","oast":"green","cby":9,"self":false,"s":"Describes changes to an internal grant peer-review committee within a university psychiatry department, associating the revised process with higher success in obtaining external funding and increased interdisciplinary collaboration.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"LZLZ77Q3","t":"Changing peer review practices: transforming roles and future challenges","au":"Horbach, Serge P. J. M.","y":2025,"vn":"Acta Orthopaedica","ty":"journalArticle","tyl":"Journal article","doi":"10.2340/17453674.2025.44353","cx":"Journal","co":["general-systemic"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":2,"self":false,"s":"Discusses evolving roles and future challenges in peer review practices as the scholarly publishing landscape changes.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"M4FB9M3J","t":"Use of check lists in assessing the statistical content of medical studies.","au":"Gardner, M J","y":1986,"vn":"BMJ","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmj.292.6523.810","cx":"Journal","co":["Interventions"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"bronze","cby":141,"self":false,"s":"Describes standardized checklists used by a medical journal's statistical reviewers, finding their use revealed widespread, often easily correctable statistical shortcomings in submitted manuscripts.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"M4FE5YNE","t":"Involving clinical experts in prioritising topics for health technology assessment: a randomised controlled trial","au":"Cook, Andrew","y":2017,"vn":"BMJ Open","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmjopen-2017-016104","cx":"Grant","co":["Interventions"],"sg":["individual-scoring"],"x":1,"st":"experimental-rct","oast":"gold","cby":1,"self":false,"s":"Randomized trial testing whether shorter review documents and different response formats affected the quality of clinical experts' input into prioritizing health technology assessment topics, finding structured response forms improved quality more than document length did.","pg":"preregistered","pgr":"other","pgid":"ANZCTR12614000167662","da":"on-request","dau":"","ca":"no-statement","cau":""},{"k":"M4I4S2CC","t":"Reimagining peer review: the emergence of peer community in registered reports system.","au":"Zoccali, Carmine","y":2023,"vn":"Journal of nephrology","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s40620-023-01709-6","cx":"Journal","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":10,"self":false,"s":"A commentary describes the Peer Community In Research Reports model as an emerging alternative to traditional pre-publication review, evaluating preprints through a transparent, community-driven process intended to improve trust and speed.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"M653B5D9","t":"Cognitive bias in peer review: A new approach","au":"Ström","y":2009,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"","cx":"Grant","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":10,"self":false,"s":"Proposes a bibliometric method for detecting \"cognitive similarity\" between reviewers and applicants as a way to identify a specific form of bias in peer review.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"M72NE68B","t":"Evaluating science: A comparison of human and AI reviewers","au":"Shcherbiak, Anna","y":2024,"vn":"Judgment and Decision Making","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/jdm.2024.24","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"diamond","cby":11,"self":false,"s":"A field experiment at an academic conference compared human and AI (GPT-4) reviewers, finding human reviewers were somewhat better at spotting AI-written abstracts, and that AI and human quality ratings agreed only loosely overall but converged more on identifying the top-rated abstracts.","pg":"preregistered","pgr":"OSF","pgid":"https://osf.io/zaj6p","da":"open","dau":"https://osf.io/nje23","ca":"open","cau":"https://osf.io/nje23"},{"k":"M8NNE35R","t":"The Costs of Competition in Distributing Scarce Research Funds","au":"Schweiger, Gerald","y":2024,"vn":"PNAS","ty":"journalArticle","tyl":"Journal article","doi":"10.1073/pnas.2407644121","cx":"Grant","co":["Reliability-noise","Bias","Validity"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"hybrid","cby":18,"self":false,"s":"A review paper examines the reliability and economic costs of competitive research funding, its effects on risk-taking, and related ethical issues, proposing recommendations for improving funding-decision research.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"MAPYT9ZR","t":"The effectiveness of the peer review process: inter-referee agreement and predictive validity of manuscript refereeing at Angewandte Chemie.","au":"Bornmann, Lutz","y":2008,"vn":"Angewandte Chemie International Edition","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/anie.200800513","cx":"Journal","co":["Reliability-noise","Validity"],"sg":["individual-scoring","post-award-validity"],"x":0,"st":"observational-quantitative","oast":"closed","cby":69,"self":false,"s":"Examining referee agreement and the link between publication decisions and later citation rates at a chemistry journal, the study concludes its peer-review process maintained consistent quality despite a sharp rise in submissions and falling acceptance rates.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"MBW2C84N","t":"An assessment of selection processes among candidates for public research grants: the case of the Ramón y Cajal Programme in Spain","au":"Cañibano, Carolina","y":2009,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.3152/095820209x444968","cx":"Grant","co":["Reliability-noise","Bias"],"sg":["funding-decision"],"x":0,"st":"observational-quantitative","oast":"closed","cby":15,"self":false,"s":"Uses applicant CVs to build statistical models replicating the outcomes of a Spanish research grant programme, finding the selection process aligns closely with applicants' prior research productivity rather than demographic factors.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"no-statement","cau":""},{"k":"MC7H39UQ","t":"The Decision-Making Constraints and Processes of Grant Peer Review, and Their Effects on the Review Outcome","au":"Langfeldt, Liv","y":2001,"vn":"Social studies of science","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/030631201031006002","cx":"Grant","co":["Reliability-noise"],"sg":["individual-scoring","panel-consensus","funding-decision"],"x":0,"st":"qualitative","oast":"green","cby":206,"self":false,"s":"Case study of Research Council of Norway grant panels finding that rating scales and budget constraints shaped review criteria and outcomes more than formal guidelines, with different ranking methods favoring safer versus more novel proposals.","pg":"not-codeable","pgr":"","pgid":"","da":"","dau":"","ca":"","cau":""},{"k":"MFK36ARS","t":"Selecting manuscripts for a high‐impact journal through peer review: A citation analysis of communications that were accepted by Angewandte Chemie International Edition, or rejected but published elsewhere","au":"Bornmann, Lutz","y":2008,"vn":"Journal of the American Society for Information Science and Technology","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/asi.20901","cx":"Journal","co":["Validity"],"sg":["funding-decision","post-award-validity"],"x":0,"st":"observational-quantitative","oast":"green","cby":69,"self":false,"s":"Citation analysis of manuscripts submitted to a major chemistry journal comparing accepted papers with those rejected but published elsewhere, finding accepted papers received substantially more citations on average.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"MGSKAUXW","t":"No evidence that gender contributes to personal research funding success in The Netherlands: A reaction to van der Lee and Ellemers","au":"Völker, Beate","y":2015,"vn":"Proceedings of the National Academy of Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1073/pnas.1519046112","cx":"Grant","co":["Bias"],"sg":["funding-decision"],"x":0,"st":"observational-quantitative","oast":"green","cby":36,"self":false,"s":"A response paper reacting to earlier findings on gender and Dutch research funding, reporting no evidence that gender affects personal funding success in that context.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"no-statement","cau":""},{"k":"MHK2CREE","t":"Comparing ChatGPT and human expertise: exploring new avenues in peer review of single-case experimental research","au":"Rakap, Salih","y":2025,"vn":"European Journal of Special Needs Education","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/08856257.2025.2533555","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"closed","cby":0,"self":false,"s":"Comparison of ChatGPT and human expert reviews of 18 single-case experimental design manuscripts found substantial agreement on quality assessment but weaker agreement on final publication recommendations.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"MIAXJ264","t":"Scientific peer review","au":"Bornmann, Lutz","y":2011,"vn":"Annual Review of Information Science and Technology","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/aris.2011.1440450112","cx":"General","co":["Reliability-noise","Bias","Validity"],"sg":["whole-process"],"x":0,"st":"systematic-review-meta-analysis","oast":"closed","cby":566,"self":false,"s":"A literature review traces the history and evidence base of peer review across journals, fellowships, and grants, focusing on three long-studied criteria: reliability, fairness, and predictive validity.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"MJUP8VVE","t":"Peer Review at the Crossroads","au":"Kochetkov, Dmitry","y":2025,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31235/osf.io/b2ra3_v3","cx":"Journal","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":0,"self":false,"s":"A mixed-methods literature review and process-modeling study traces the history of peer review and identifies four current models, including registered reports and Publish-Review-Curate, concluding no single model is universally best.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"MLXXE7WL","t":"Improving peer review of systematic reviews and related review types by involving librarians and information specialists as methodological peer reviewers: a randomized controlled trial","au":"Rethlefsen, Melissa L.","y":2024,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31222/osf.io/7hu3v","cx":"Journal","co":["Interventions"],"sg":["reviewer-selection","individual-scoring"],"x":1,"st":"experimental-rct","oast":"closed","cby":1,"self":false,"s":"A randomized controlled trial tests whether adding librarians as methodological peer reviewers to systematic-review manuscripts at BMJ journals improves the completeness of search reporting and reduces risk of bias, comparing outcomes against usual editorial review.","pg":"preregistered","pgr":"OSF","pgid":"https://doi.org/10.17605/OSF.IO/W4CK2","da":"open","dau":"https://dx.doi.org/10.17605/OSF.IO/ZY547","ca":"no-statement","cau":""},{"k":"MM2N6SHA","t":"Opening Pandora’s box: Developing reviewer rhetorical sensitivity through retracted articles","au":"Khuder, Baraa","y":2025,"vn":"Accountability in Research","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/08989621.2025.2607681","cx":"Journal","co":["Interventions"],"sg":["individual-scoring"],"x":0,"st":"qualitative","oast":"hybrid","cby":0,"self":false,"s":"Describes a pedagogical exercise in which doctoral STEM students analyzed retracted articles to identify rhetorical warning signs of unreliable research, aiming to build critical reviewing skills.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"MM5CTRWM","t":"Editorial peer reviewers' recommendations at a general medical journal: are they reliable and do editors care?","au":"Kravitz, Richard L","y":2010,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0010072","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"gold","cby":176,"self":false,"s":"Analysis of thousands of reviews at a general medical journal finding reviewers' recommendations to accept or reject agreed only slightly above chance, yet editors' decisions still tracked reviewer consensus closely.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"MMDJHRAP","t":"Distrust in grant peer review—reasons and remedies","au":"Langfeldt, Liv","y":2023,"vn":"Science and Public Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/scipol/scad051","cx":"Grant","co":["general-systemic"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":14,"self":false,"s":"Survey-based study exploring why researchers report less trust in grant peer reviewers than in journal reviewers or colleagues, linking trust to reviewers' field expertise and process transparency.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"no-statement","cau":""},{"k":"MQ4IPI6N","t":"Surveys of current status in biomedical science grant review: funding organisations' and grant reviewers' perspectives.","au":"Schroter, Sara","y":2010,"vn":"BMC Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/1741-7015-8-62","cx":"Grant","co":["general-systemic"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"gold","cby":48,"self":false,"s":"Surveys of grant funders and reviewers across dozens of countries document common problems such as declined invitations, late reports, and administrative burden in biomedical grant review, alongside broad support for more standardized review requirements.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"MRNKJKY2","t":"Peer review in an Era of Evaluation","au":"Forsberg, Eva","y":2022,"vn":"","ty":"book","tyl":"Book","doi":"10.1007/978-3-030-75263-7","cx":"Grant","co":["general-systemic"],"sg":["whole-process"],"x":0,"st":"book-monograph","oast":"gold","cby":26,"self":false,"s":"Historical overview traces three developmental stages of peer review research literature since before 1983, describing shifting dominant topics and journal venues across each period.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"MTAZBL3E","t":"Large-scale language analysis of peer review reports","au":"Buljan, Ivan","y":2020,"vn":"eLife","ty":"journalArticle","tyl":"Journal article","doi":"10.7554/elife.53249","cx":"Journal","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":35,"self":false,"s":"A text-analysis study of nearly half a million peer review reports found that a reviewer's overall recommendation shaped the language of the report far more than the research field, review type, or reviewer's gender did.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"no-statement","cau":""},{"k":"MTIHLHV3","t":"Is the expertise of evaluation panels congruent with the research interests of the research groups: A quantitative approach based on barycenters","au":"Rahman, A.I.M. Jakaria","y":2015,"vn":"Journal of Informetrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.joi.2015.07.009","cx":"General","co":["general-systemic"],"sg":["reviewer-selection"],"x":0,"st":"simulation-methodological","oast":"closed","cby":11,"self":false,"s":"Using bibliometric mapping of two university department evaluations, this study introduces a method to measure how well an expert review panel's research expertise overlaps with that of the units being assessed, finding some mismatch.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"MTLGNRD3","t":"Flipping the grant application review process","au":"Dinov, Ivo D.","y":2019,"vn":"Studies in Higher Education","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/03075079.2019.1628201","cx":"Grant","co":["Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"green","cby":3,"self":false,"s":"An opinion piece proposes restructuring grant review from an upfront, front-loaded evaluation into a back-loaded assessment of research's actual return on investment, arguing this could counter unreliability and excessive focus on grant-writing skill.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"MVK2CRH7","t":"Inference About Weighted Kappa in the Non-Null Case","au":"Fleiss, Joseph L.","y":1978,"vn":"Applied Psychological Measurement","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/014662167800200111","cx":"Analogue","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"green","cby":142,"self":false,"s":"A simulation study examining how large a sample needs to be to reliably test hypotheses about weighted kappa values and to construct confidence intervals around them.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"MVKIF9TD","t":"Gender bias in scholarly peer review.","au":"Helmer, Markus","y":2017,"vn":"eLife","ty":"journalArticle","tyl":"Journal article","doi":"10.7554/eLife.21718","cx":"Journal","co":["Bias"],"sg":["reviewer-selection","individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":380,"self":false,"s":"Large-scale analysis of Frontiers journal editors and reviewers finding women underrepresented in review roles and both male and female editors showing a same-gender preference when selecting reviewers.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"","ca":"no-statement","cau":""},{"k":"MY2NAXXH","t":"Why can't we make research grant allocation systems more consistent? A personal opinion","au":"Cousens, Roger","y":2019,"vn":"Ecology and Evolution","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/ece3.4855","cx":"Grant","co":["Reliability-noise","Validity"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":2,"self":false,"s":"Opinion piece arguing that grant allocation involves inherent statistical uncertainty at multiple stages, cautioning against replacing peer review with lotteries while calling for more clarity and innovation incentives.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"N2LFYALW","t":"Emerging new methods of peer review in scholarly journals","au":"Björk, Bo‐Christer","y":2015,"vn":"Learned Publishing","ty":"journalArticle","tyl":"Journal article","doi":"10.1087/20150202","cx":"Journal","co":["Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"bronze","cby":24,"self":false,"s":"An overview article proposes a taxonomy of emerging peer review innovations in scholarly journals, discussing new reviewer-assignment approaches and models like megajournals that prioritize scientific validity over perceived novelty.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"N4W7NGXC","t":"Selecting competent referees to assess research projects proposals: A study of referees' registers","au":"Abramo, Giovanni","y":2012,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvs031","cx":"Grant","co":["general-systemic"],"sg":["reviewer-selection"],"x":0,"st":"observational-quantitative","oast":"closed","cby":6,"self":false,"s":"Proposes and applies a methodology for assessing how well a register of expert referees covers the range of expertise needed to evaluate proposals in a national competitive funding program.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"N4Z5ARMM","t":"For what it’s worth – the open peer review landscape","au":"Tattersall, Andy","y":2015,"vn":"Online Information Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1108/oir-06-2015-0182","cx":"Journal","co":["Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":37,"self":false,"s":"Surveys the landscape of post-publication open peer review, discussing key platforms, barriers to academic adoption, and whether open review could become a mainstream model.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"N5TQTHKV","t":"A radical change in peer review","au":"Mervis, Jeffrey","y":2014,"vn":"Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1126/science.345.6194.248","cx":"Grant","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":10,"self":false,"s":"A short report on an NSF pilot program requiring grant applicants to review several competing proposals, intended to reduce strain on the traditional peer-review system.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"N8M2AUCU","t":"Peer Review Interrater Reliability of Scientific Abstracts: A Study of an Anesthesia Subspecialty Society","au":"Cohen, Ira Todd","y":2005,"vn":"Journal of Education in Perioperative Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.46374/volvii-issue2-cohen2","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"bronze","cby":2,"self":false,"s":"Assessing interrater reliability among 11 reviewers rating abstracts for an anesthesiology subspecialty meeting, the study finds only fair agreement (kappa around 0.2 to 0.4), consistent with similarly low reliability reported for other medical society abstract reviews.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"N8W8PGRJ","t":"Percentile Ranking and Citation Impact of a Large Cohort of National Heart, Lung, and Blood Institute–Funded Cardiovascular R01 Grants","au":"Danthi, Narasimhan","y":2014,"vn":"Circulation Research","ty":"journalArticle","tyl":"Journal article","doi":"10.1161/circresaha.114.302656","cx":"Grant","co":["Validity"],"sg":["post-award-validity"],"x":0,"st":"observational-quantitative","oast":"bronze","cby":84,"self":false,"s":"Tracking a large set of NHLBI cardiovascular grants and their resulting publications, this study finds no consistent association between peer review percentile ranking and later citation impact, though some benefit emerged in exploratory machine-learning analysis.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"no-statement","cau":""},{"k":"N99QJWN2","t":"Peer review: Risk and risk tolerance","au":"Gallo, Stephen A.","y":2022,"vn":"PLOS ONE","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0273813","cx":"Grant","co":["Reliability-noise"],"sg":["individual-scoring"],"x":1,"st":"experimental-rct","oast":"gold","cby":6,"self":false,"s":"Experiment testing how manipulated descriptions of risk affected reviewers' scoring of mock grant reviews, finding perceived proposal risk influenced scores more than proposal strengths did.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"10.17605/OSF.IO/FU83D","ca":"no-statement","cau":""},{"k":"NAZXFWMK","t":"A statistical approach to calibrating the scores of biased reviewers of scientific papers","au":"Kuhlisch, Wiltrud","y":2015,"vn":"Metrika","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s00184-015-0542-z","cx":"Journal","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"closed","cby":3,"self":false,"s":"Proposes and compares several statistical methods, including mixed and generalized linear models, for aggregating and calibrating biased or inconsistent peer reviewer scores in evaluating papers and grants.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"NB4IK4KH","t":"Design and Analysis of the NIPS 2016 Review Process","au":"Shah, Nihar B.","y":2017,"vn":"arXiv (Cornell University)","ty":"journalArticle","tyl":"Journal article","doi":"10.48550/arxiv.1708.09794","cx":"Journal","co":["Reliability-noise","Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"green","cby":48,"self":false,"s":"An analysis of the 2016 NeurIPS conference's large-scale review process examines submission and reviewer growth alongside an experiment on collecting ordinal reviewer rankings, aiming to assess review quality and inform future conference design.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"NCCTU8QK","t":"How to best evaluate applications for junior fellowships? Remote evaluation and face-to-face panel meetings compared","au":"Bieri, Marco","y":2020,"vn":"bioRxiv (Cold Spring Harbor Laboratory)","ty":"journalArticle","tyl":"Journal article","doi":"10.1101/2020.11.26.400028","cx":"Grant","co":["Reliability-noise","Validity","Interventions"],"sg":["individual-scoring","panel-consensus","funding-decision"],"x":0,"st":"observational-quantitative","oast":"green","cby":0,"self":false,"s":"Comparison of a simplified two- or three-reviewer scoring procedure against official Swiss National Science Foundation fellowship panel decisions found roughly 80 to 87 percent agreement, with lower agreement for borderline, panel-discussed applications.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"NCMPJGAC","t":"Review for \"That's a Lot to Process! Pitfalls of Popular Path Models\"","au":"Montoya, Amanda Kay","y":2021,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/25152459221095827/v1/review3","cx":"Journal","co":["Interventions"],"sg":["individual-scoring"],"x":0,"st":"guideline-policy","oast":"gold","cby":0,"self":false,"s":"Offers a worked example of a peer review aimed at improving methodological rigor, illustrating concrete practices reviewers can use to promote research-transparency reforms in psychology.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"NG8FPPXE","t":"Does the use of open, nonanonymous peer review in scholarly publishing introduce bias? Evidence from the F1000 postpublication open peer review publishing model","au":"Thelwall, Mike","y":2019,"vn":"Journal of Information Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/0165551520938678","cx":"Journal","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"green","cby":41,"self":false,"s":"Analyzing over four years of open, named peer review data from F1000Research, this study found weak evidence of same-country bias between reviewers and authors but no clear evidence that seeing a prior review influenced subsequent reviewers.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"NGZJ7MXU","t":"Assessing inter-rater reliability for rating scales: resolving some basic issues.","au":"Cicchetti, D V","y":1976,"vn":"The British Journal of Psychiatry","ty":"journalArticle","tyl":"Journal article","doi":"10.1192/bjp.129.5.452","cx":"Analogue","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"closed","cby":322,"self":false,"s":"A statistical paper extends earlier work on sample-size and weighting requirements for reliably assessing agreement on rating scales used in psychiatric and clinical settings, accounting for scale structure and number of categories.","pg":"no-statement","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"on-request","cau":""},{"k":"NJCR7T7F","t":"Towards improving peer review: Crowd-sourced insights from Twitter","au":"Allen, Kelly‐Ann","y":2022,"vn":"Journal of University Teaching and Learning Practice","ty":"journalArticle","tyl":"Journal article","doi":"10.53761/1.19.3.2","cx":"Journal","co":["Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"diamond","cby":7,"self":false,"s":"A commentary combining a literature review with crowdsourced suggestions gathered from academics on Twitter about how to address common problems in peer review, such as publication delays and reviewer overload.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"NL2R3R8Q","t":"Reflections on a mentored group peer review process","au":"Auby, Harpreet","y":2024,"vn":"Journal of Engineering Education","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/jee.20616","cx":"Journal","co":["Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"bronze","cby":2,"self":false,"s":"Editorial reflecting on a mentored group peer-review program, situating it against broader concerns about reviewer burden, bias, and proposed alternatives such as crowdsourced or AI-assisted review.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"NLNWACRS","t":"Foucault, the “Facts,” and the Fiction of Neutrality: Neutrality in Librarianship and Peer Review","au":"Johnson, Heidi R.","y":2016,"vn":"Canadian Journal of Academic Librarianship","ty":"journalArticle","tyl":"Journal article","doi":"10.33137/cjal-rcbu.v1.24304","cx":"Journal","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"diamond","cby":11,"self":false,"s":"Applies Foucauldian and science-studies theory to critique the ideal of reviewer neutrality in peer review, drawing parallels with debates about neutrality in librarianship.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"NN8254C6","t":"Large sample standard errors of kappa and weighted kappa.","au":"Fleiss, Joseph L.","y":1969,"vn":"Psychological Bulletin","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/h0028106","cx":"Analogue","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"closed","cby":1525,"self":false,"s":"A statistical methods paper identifies errors in the originally published formulas for the standard errors of kappa and weighted kappa agreement statistics and derives corrected expressions.","pg":"no-statement","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"no-statement","cau":""},{"k":"NND4U3YX","t":"A physics editor comments on Peters and Ceci's peer-review study","au":"Adair, Robert K.","y":1982,"vn":"Behavioral and Brain Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/s0140525x00011195","cx":"Journal","co":["Reliability-noise","Bias"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"closed","cby":57,"self":false,"s":"Commentary from a physics journal editor responding to and reflecting on the well-known Peters and Ceci peer-review resubmission study.","pg":"no-statement","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"NNIEFSDW","t":"Unique review criteria and patient and stakeholder reviewers: analysis of pcori's approach to research funding.","au":"Forsythe, Laura P","y":2018,"vn":"Value in Health","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.jval.2018.03.017","cx":"Grant","co":["Reliability-noise"],"sg":["individual-scoring","panel-consensus","funding-decision"],"x":0,"st":"observational-quantitative","oast":"bronze","cby":11,"self":false,"s":"Analysis of over 1,300 applications to a patient-centered research funder found that scientists' technical-merit ratings were the strongest predictor of final funding scores, and that scoring agreement between scientist, patient, and stakeholder reviewers improved after panel discussion.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"NPECNPPD","t":"Arbitrariness in the peer review process","au":"Brezis, Elise S.","y":2020,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-020-03348-1","cx":"General","co":["Reliability-noise","Bias"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"hybrid","cby":81,"self":false,"s":"A modeling study replicating a well-known NIPS peer-review experiment finds that reviewer heterogeneity in taste and time investment produces substantial arbitrariness, and shows innovative papers tend to be ranked lower.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"","ca":"no-statement","cau":""},{"k":"NPXHHPC4","t":"Extraneous factors in judicial decisions","au":"Danziger, Shai","y":2011,"vn":"Proceedings of the National Academy of Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1073/pnas.1018033108","cx":"Analogue","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":1411,"self":false,"s":"Studying sequential parole rulings by experienced judges, the study finds the proportion of favourable decisions declines steadily within each session between food breaks and resets sharply afterward, suggesting extraneous factors can influence high-stakes judicial decisions.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"NRNXMEJZ","t":"The Role of Peer Review in the Scientific Process","au":"Garza, Henriette De La","y":2022,"vn":"","ty":"book","tyl":"Book","doi":"10.1007/978-3-030-99680-2_41","cx":"Journal","co":["Bias"],"sg":["whole-process"],"x":0,"st":"book-monograph","oast":"closed","cby":4,"self":false,"s":"A general overview of peer review's role in maintaining research quality and scientific trust, alongside its known weaknesses and susceptibility to bias.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"NSYAMRTC","t":"Perceptions of grant peer reviewers: a mixed methods study","au":"Schmaling, Karen B.","y":2024,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvae050","cx":"Grant","co":["Reliability-noise","Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":3,"self":false,"s":"Mixed-methods study coding grant reviewers' unstructured comments after scoring mock proposals, finding reviewers who expressed more negative sentiment tended to give lower scores.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"osf.io/g9ew4","ca":"no-statement","cau":""},{"k":"NUWKCHMA","t":"Is Peer Review a Game of Chance?","au":"Neff, Bryan D.","y":2006,"vn":"BioScience","ty":"journalArticle","tyl":"Journal article","doi":"10.1641/0006-3568(2006)56[333:iprago]2.0.co;2","cx":"Journal","co":["Reliability-noise","Validity"],"sg":["whole-process"],"x":0,"st":"simulation-methodological","oast":"bronze","cby":82,"self":false,"s":"Uses probability and Bayesian modeling with citation data to estimate how much chance influences journal peer-review outcomes, concluding editorial prescreening and multiple-referee decision rules reduce, but do not eliminate, a \"lottery\" element.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"NYR9UWFI","t":"Fast, Furious and Dubious? MDPI and the Depth of Peer Review Reports","au":"Maddi, Abdelghani","y":2023,"vn":"Research Square","ty":"journalArticle","tyl":"Journal article","doi":"10.21203/rs.3.rs-3027724/v1","cx":"Journal","co":["general-systemic"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"green","cby":1,"self":false,"s":"A bibliometric study of over 61,000 publications and 86,000 reviewers analyzes whether the publisher MDPI's fast turnaround times correspond to shorter, less thorough reviewer reports, finding comparable report length to other publishers despite tighter deadlines.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"P2DZ89LR","t":"Research funding: the case for a modified lottery.","au":"Fang, Ferric C","y":2016,"vn":"mBio","ty":"journalArticle","tyl":"Journal article","doi":"10.1128/mBio.00422-16","cx":"Grant","co":["Reliability-noise","Bias","Validity","Interventions"],"sg":["funding-decision"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":163,"self":false,"s":"Essay argues traditional peer-review ranking can no longer reliably distinguish the most meritorious grant proposals, and proposes a modified lottery that screens for basic merit before randomly selecting among qualified applicants.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"P5PUM6V5","t":"Peer review’s irremediable flaws: Scientists’ perspectives on grant evaluation in Germany","au":"Barlösius, Eva","y":2023,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvad032","cx":"Grant","co":["Bias","Validity"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":7,"self":false,"s":"A survey of German scientists found that certain flaws in grant peer review, such as networking effects and difficulty distinguishing similarly strong proposals, were viewed as more fundamentally unfixable than other commonly criticized problems.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://doi.org/10.21249/DZHW:vorausquanti:1.0.0","ca":"no-statement","cau":""},{"k":"P5TFHNG5","t":"Zoom fatigue saps grant reviewers’ attention","au":"Chawla, Dalmeet Singh","y":2021,"vn":"Nature","ty":"journalArticle","tyl":"Journal article","doi":"10.1038/d41586-021-00161-5","cx":"Grant","co":["general-systemic"],"sg":["panel-consensus"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":17,"self":false,"s":"A news report on a survey of NIH grant reviewers found many felt less attentive and engaged during Zoom-based review meetings than in-person ones, with mixed preferences between video and face-to-face formats.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"P64HCKBT","t":"Reimagining the peer‐review system for translational health science journals","au":"Smith, Elise","y":2021,"vn":"Clinical and Translational Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/cts.13050","cx":"Journal","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":24,"self":false,"s":"Prompted by COVID-19-era retractions in major medical journals, this paper argues that scapegoating individuals overlooks structural limits of traditional peer review for complex interdisciplinary research and proposes a more open, collaborative review model.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"P6D79UMP","t":"Medical Journal Peer Review: Process and Bias","au":"Manchikanti, Laxmaiah","y":2015,"vn":"Pain Physician","ty":"journalArticle","tyl":"Journal article","doi":"10.36076/ppj/2015.18.e1","cx":"Journal","co":["Bias"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"hybrid","cby":129,"self":false,"s":"A historical and topical overview of medical journal peer review, covering its origins, evolving challenges such as bias and plagiarism detection, and its continued central role despite criticism.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"P7XBS7BU","t":"Peer‐reviewing geen valide basis voor erkennen en waarderen van onderzoeksprestaties","au":"Breetvelt, Iris","y":2021,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31234/osf.io/9n5mq","cx":"General","co":["Reliability-noise","Bias","Validity"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":0,"self":false,"s":"Dutch-language commentary arguing peer review lacks sufficient reliability and validity to serve as a basis for recognizing and rewarding researchers' performance, and calling for its reconsideration.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"PAEJWKA6","t":"Gender differences in grant and personnel award funding rates at the Canadian Institutes of Health Research based on research content area: A retrospective analysis.","au":"Burns, Karen E A","y":2019,"vn":"PLoS Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pmed.1002935","cx":"Grant","co":["Bias"],"sg":["funding-decision"],"x":0,"st":"observational-quantitative","oast":"gold","cby":110,"self":false,"s":"Retrospective analysis of nearly 60,000 Canadian Institutes of Health Research applications over 15 years finding women had modestly lower grant funding success than men overall, with variation by research institute.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"no-statement","cau":""},{"k":"PB3TYFEA","t":"Improving manuscript evaluation procedures.","au":"Bowen, Donald D.","y":1972,"vn":"American Psychologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/h0033849","cx":"Journal","co":["Reliability-noise","Bias","Interventions"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":160,"self":false,"s":"An older commentary discusses longstanding dissatisfaction among journal editors and authors with manuscript evaluation procedures, describing gatekeeping challenges and reviewer disagreement from an editor's perspective.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"PBECA3WZ","t":"Grant application review: The case of transparency.","au":"Gurwitz, David","y":2014,"vn":"PLoS Biology","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pbio.1002010","cx":"Grant","co":["Interventions"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"gold","cby":30,"self":false,"s":"Commentary describes peer review's centrality to legitimizing competitive research funding decisions to both the scientific community and the public.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"PBV9GKQU","t":"Rational Disagreement in Peer Review","au":"Harnad, Stevan","y":1985,"vn":"Science Technology & Human Values","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/016224398501000307","cx":"Grant","co":["Reliability-noise","Validity"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":74,"self":false,"s":"A short essay argues that some level of disagreement among reviewers is a healthy and informative feature of scientific evaluation rather than simply a flaw to be eliminated.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"PEMI66JZ","t":"An intervention to improve the reliability of manuscript reviews for the Journal of the American Academy of Child and Adolescent Psychiatry.","au":"Strayhorn, J","y":1993,"vn":"The American Journal of Psychiatry","ty":"journalArticle","tyl":"Journal article","doi":"10.1176/ajp.150.6.947","cx":"Journal","co":["Reliability-noise","Interventions"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":53,"self":false,"s":"This intervention study found that introducing a multi-item rating scale with training manuals for a child psychiatry journal's manuscript reviewers modestly improved the interrater reliability of quality ratings compared to prior methods.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"PFRFLMJE","t":"Open Science, Closed Peer Review?","au":"Evans, Daniel","y":2025,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31222/osf.io/dmgex_v2","cx":"Journal","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"gold","cby":0,"self":false,"s":"A survey of over 800 experimental and behavioural economists finds general support for keeping referee identities anonymous, mixed views on author and editor anonymity, and openness to publishing referee reports and allowing appeals, with active reviewers less favourable to transparency reforms.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://osf.io/eczkv/","ca":"no-statement","cau":""},{"k":"PH5T9KLF","t":"Peer-review for selection of oral presentations for conferences: Are we reliable?","au":"Deveugele, Myriam","y":2017,"vn":"Patient Education and Counseling","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.pec.2017.06.007","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":21,"self":false,"s":"Reports results of a calibration exercise at an international healthcare communication conference, finding low reliability among reviewers scoring the same conference abstracts, with the same abstracts often receiving the full range of possible scores.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"PKDGKHJ5","t":"Rethinking (un)blinding in biomedical proposal peer review: A multi-stakeholder qualitative study","au":"Qussini, Seba","y":2025,"vn":"Accountability in Research","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/08989621.2025.2593625","cx":"Grant","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"qualitative","oast":"hybrid","cby":0,"self":false,"s":"Qualitative interview study with funding-agency stakeholders in Belgium and Qatar on blinding and openness in biomedical grant review, finding continued preference for double-blind review alongside growing calls for more transparency.","pg":"preregistered","pgr":"OSF","pgid":"https://osf.io/dp65f","da":"on-request","dau":"","ca":"no-statement","cau":""},{"k":"PP7EU66X","t":"Lack of peer reviewer diversity advantages scientists from wealthier countries","au":"Zumel Dumlao, James M.","y":2025,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31235/osf.io/754e3_v3","cx":"Journal","co":["Bias","Interventions"],"sg":["reviewer-selection","individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":1,"self":false,"s":"Analysis of over 200,000 physics-related journal submissions finding reviewers were more likely to rate manuscripts favorably when they shared the author's home country, disadvantaging authors from underrepresented countries even after blinding.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"","ca":"open","cau":""},{"k":"PQZETV2H","t":"Building a Data-Driven Model of Peer Review: The Case of Science Foundation Ireland","au":"Feliciani, Thomas","y":2021,"vn":"Springer proceedings in complexity","ty":"book","tyl":"Book","doi":"10.1007/978-3-030-61503-1_21","cx":"Grant","co":["Validity"],"sg":["whole-process"],"x":0,"st":"book-monograph","oast":"closed","cby":0,"self":false,"s":"A brief report outlines efforts to build a data-driven understanding of peer review at Science Foundation Ireland, referencing prior findings on inconsistent reviewer interpretation of criteria and various bias patterns in review outcomes.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"PR6SUMXI","t":"Researchers’ Goals When Assessing Credibility and Impact in Committees and in Their Own Work","au":"Harney, James","y":2021,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31219/osf.io/ryds4","cx":"General","co":["general-systemic"],"sg":["panel-consensus"],"x":0,"st":"qualitative","oast":"gold","cby":6,"self":false,"s":"Interviews with 52 cell biology researchers find substantial overlap in the goals researchers apply when judging credibility and impact, whether evaluating their own work or serving on grant and hiring committees.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://osf.io/sphfj","ca":"not-applicable","cau":""},{"k":"PSBEMHHH","t":"An efficient system to fund science: from proposal review to peer-to-peer distributions.","au":"Bollen, Johan","y":2017,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-016-2110-3","cx":"Grant","co":["Interventions"],"sg":["funding-decision"],"x":0,"st":"simulation-methodological","oast":"green","cby":49,"self":false,"s":"Proposes a science-funding model where each researcher receives equal base funding but must redistribute part of it to peers, aiming to reach a crowd-informed funding distribution without formal proposals or review.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"PUG2N7TW","t":"NSF's ‘Big Pitch’ Tests Anonymized Grant Reviews","au":"Bhattacharjee, Yudhijit","y":2012,"vn":"Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1126/science.336.6084.969","cx":"Grant","co":["Reliability-noise","Bias","Interventions"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":24,"self":false,"s":"A news report on an NSF experiment testing anonymized grant review, suggesting the standard process may disadvantage some deserving proposals due to potential reviewer bias.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"PWQWRMFY","t":"Peer review of health research funding proposals: A systematic map and systematic review of innovations for effectiveness and efficiency.","au":"Shepherd, Jonathan","y":2018,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0196914","cx":"Grant","co":["Interventions"],"sg":["whole-process"],"x":0,"st":"systematic-review-meta-analysis","oast":"gold","cby":41,"self":false,"s":"A systematic map and review of 83 studies on innovations in health-research grant peer review finds that streamlining processes, such as shorter forms or smaller panels, can speed review but that effects on review quality are rarely assessed.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"","ca":"no-statement","cau":""},{"k":"PYVKKJGJ","t":"A harm reduction approach to improving peer review by acknowledging its imperfections","au":"Cooke, Steven J.","y":2024,"vn":"FACETS","ty":"journalArticle","tyl":"Journal article","doi":"10.1139/facets-2024-0102","cx":"Journal","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":8,"self":false,"s":"A multidisciplinary perspective piece proposing a \"harm reduction\" framework, focused on training editors, reviewers and authors, to address acknowledged flaws and eroding trust in scholarly peer review.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"PZBSQXF9","t":"Engineering Features From Advanced Medical Technology Initiative Submissions to Enable Predictive Modeling for Proposal Success.","au":"Pavliscsak, Holly","y":2024,"vn":"Military medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/milmed/usae063","cx":"Grant","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":0,"self":false,"s":"Describes early-stage work extracting text-based features from military medical-technology grant proposals to explore whether machine learning could help predict proposal evaluation outcomes alongside expert review.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"no-statement","cau":""},{"k":"Q3AG2M2K","t":"Envisioning the Future of Peer Review: Investigating LLM-Assisted Reviewing Using ChatGPT as a Case Study","au":"Chen, Shiping","y":2025,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.1145/3729176.3729196","cx":"Journal","co":["Interventions"],"sg":["individual-scoring"],"x":1,"st":"experimental-rct","oast":"gold","cby":2,"self":false,"s":"Experimental study with 24 human-computer interaction reviewers testing ChatGPT-assisted peer review, finding it reduced perceived workload but did not meaningfully speed up reviews or improve their quality.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"Q4YMV96Z","t":"Reviewer training for improving grant and journal peer review.","au":"Hesselberg, Jan-Ole","y":2023,"vn":"Cochrane Database of Systematic Reviews","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/14651858.MR000056.pub2","cx":"General","co":["Interventions"],"sg":["individual-scoring"],"x":0,"st":"systematic-review-meta-analysis","oast":"green","cby":14,"self":true,"s":"A Cochrane systematic review of 10 randomized trials evaluates whether reviewer training improves grant and journal peer review quality, finding inconsistent and generally weak evidence of benefit across outcomes like error detection and inter-reviewer agreement.","pg":"registered-report","pgr":"Cochrane","pgid":"10.1002/14651858.MR000056 (protocol)","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"Q5HY3MN9","t":"Enhancing peer review skills in higher education: A mixed-methods study on challenges and training needs","au":"Stupacher, Jan","y":2025,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31219/osf.io/89xju_v2","cx":"Journal","co":["Interventions"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":0,"self":false,"s":"Mixed-methods study of peer review training in higher education finds formal instruction is rare, and identifies common challenges reviewers face around objectivity, tone, and assessing research quality.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://researchbox.org/3626","ca":"no-statement","cau":""},{"k":"Q6KQ5BNR","t":"Publication problems in psychology and an explicit evaluation schema for manuscripts.","au":"Wolff, Wirt M.","y":1973,"vn":"American Psychologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/h0034678","cx":"Journal","co":["Bias","Interventions"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":133,"self":false,"s":"An older commentary on publication problems in psychology proposes an explicit scoring scheme for evaluating journal manuscripts to address concerns about inconsistency and bias in editorial decisions.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"Q9DUXILF","t":"How reliable is peer review of scientific abstracts?","au":"Rubin, Haya R.","y":1993,"vn":"Journal of General Internal Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/bf02600092","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":63,"self":false,"s":"Analyzing reviews of medical conference abstracts, this study found substantial reviewer disagreement, with most score variance attributable to idiosyncratic reviewer judgment rather than the abstract itself, and recommends structured review criteria.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"QA4UL4N5","t":"Pride and prejudice – What can we learn from peer review?","au":"Sueur, Helen Le","y":2020,"vn":"Medical Teacher","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/0142159x.2020.1774527","cx":"Journal","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":28,"self":false,"s":"Analysing hundreds of peer reviews, the study finds reviews written during northern-hemisphere winter were harsher and reviews at open-review journals less harsh than anonymous ones, and identifies three latent reviewer types differing in tone and constructiveness.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"no-statement","cau":""},{"k":"QCWPTK8N","t":"Can small and reasoning large language models score journal articles for research quality and do averaging and few-shot help?","au":"Thelwall, Mike","y":2026,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-026-05585-2","cx":"General","co":["Validity","Interventions"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":3,"self":false,"s":"Tests whether several medium-sized and smaller large language models can rate the research quality of journal articles as well as bigger commercial models, finding comparable performance from models with at least a few billion parameters, especially when scores are averaged across repeated queries.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://doi.org/10.6084/m9.figshare.30382651","ca":"no-statement","cau":""},{"k":"QFEN9AY7","t":"An evaluation of the process of peer review","au":"Riding, James B.","y":2022,"vn":"Palynology","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/01916122.2022.2151052","cx":"Journal","co":["Bias"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":5,"self":false,"s":"General overview describing the standard mechanics of pre-publication peer review across manuscripts, grants, and related academic documents.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"QFLU6VYM","t":"Confirmatory bias in peer review","au":"García, J. A.","y":2020,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-020-03357-0","cx":"Journal","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"closed","cby":28,"self":false,"s":"Presents a probabilistic model of reviewer decision-making under information overload, showing how reviewers' prior beliefs and evaluation costs can produce confirmatory bias favoring information that matches their expectations.","pg":"no-statement","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"no-statement","cau":""},{"k":"QFPVXIEC","t":"Unpacking the black box - a questionnaire study of how medical journals identify and manage conflicts of interest of editors and peer reviewers","au":"Korfitsen, Christoffer Bruun","y":2026,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31222/osf.io/g4v2n_v1","cx":"Journal","co":["Bias"],"sg":["reviewer-selection"],"x":0,"st":"observational-quantitative","oast":"gold","cby":0,"self":false,"s":"A survey of 250 randomly sampled medical journals, with 80 responding, examines how conflicts of interest among editors and reviewers are identified and managed, finding substantial gaps between stated policies and actual enforcement practices.","pg":"preregistered","pgr":"OSF","pgid":"https://doi.org/10.17605/OSF.IO/3GFN9","da":"open","dau":"https://osf.io/3gfn9","ca":"no-statement","cau":""},{"k":"QI9FZXDI","t":"Insiders, outsiders, and efficiency in a National Science Foundation panel.","au":"Klahr, David","y":1985,"vn":"American Psychologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/0003-066x.40.2.148","cx":"Grant","co":["Reliability-noise","Interventions"],"sg":["panel-consensus","individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":25,"self":false,"s":"Analysis of nearly 1,400 reviews across NSF panel meetings finds ad hoc external reviewers had little independent influence on final proposal ratings compared with panel members, though outcomes for the highest- and lowest-rated proposals were predictable in advance.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"QIPW3GDB","t":"Reviewer training to assess knowledge translation in funding applications is long overdue","au":"Scarrow, Gayle","y":2017,"vn":"Research Integrity and Peer Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s41073-017-0037-8","cx":"Grant","co":["Interventions"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":8,"self":false,"s":"Argues that grant reviewers often lack training to properly judge the knowledge-translation or dissemination plans in health-research applications, and proposes reviewer training and guidelines to strengthen assessment of this criterion.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"QJ4FU8AL","t":"Impact of a short version of the CONSORT checklist for peer reviewers to improve the reporting of randomised controlled trials published in biomedical journals: study protocol for a randomised controlled trial","au":"Speich, Benjamin","y":2020,"vn":"BMJ Open","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmjopen-2019-035114","cx":"Journal","co":["Interventions"],"sg":["proposal-quality","individual-scoring"],"x":1,"st":"experimental-rct","oast":"gold","cby":33,"self":false,"s":"This study protocol describes a planned randomized trial testing whether giving peer reviewers a short checklist of key CONSORT reporting items improves adherence to reporting guidelines in published randomized trials.","pg":"preregistered","pgr":"OSF","pgid":"https://osf.io/c4hn8","da":"open","dau":"","ca":"no-statement","cau":""},{"k":"QJGGX98U","t":"Peer review: A flawed process at the heart of science and journals","au":"Smith, Richard","y":2006,"vn":"Journal of the Royal Society of Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/014107680609900414","cx":"Journal","co":["Reliability-noise","Bias","Validity","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"bronze","cby":848,"self":false,"s":"An extract from a book on peer review argues that the practice, central to science publishing and funding, resists clear operational definition despite its perceived authority, illustrated by an anecdote about a controversial BMJ paper.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"QM7APGKP","t":"Reliability, fairness, objectivity and other inappropriate goals in peer review","au":"Bailar, John C.","y":1991,"vn":"Behavioral and Brain Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/S0140525X00065705","cx":"General","co":["Reliability-noise"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":54,"self":false,"s":"Argues that goals commonly set for peer review, such as reliability, fairness, and objectivity, may be inappropriate or unrealistic standards for the process.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"QN8S7TQ4","t":"Communicating the Significance of Research Questions: Insights from Peer Review at a Flagship Journal","au":"Cai, Jinfa","y":2020,"vn":"International Journal of Science and Mathematics Education","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s10763-020-10073-x","cx":"Journal","co":["general-systemic"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":5,"self":false,"s":"Analyzes peer reviews at an education-research journal to identify what reviewers look for when judging a study's significance and how authors are advised to communicate it.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"QPFJ9ABU","t":"Risking Groupthink in Impact Assessment","au":"Derrick, Gemma","y":2018,"vn":"","ty":"book","tyl":"Book","doi":"10.1007/978-3-319-63627-6_6","cx":"General","co":["Reliability-noise","Bias"],"sg":["panel-consensus"],"x":0,"st":"book-monograph","oast":"closed","cby":1,"self":false,"s":"Book chapter examining how academic evaluation panels socially construct and reach consensus on the concept of research \"impact,\" highlighting risks of groupthink in this group decision-making process.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"QPIKNW58","t":"Blinding applicants in a first-stage peer-review process of biomedical research grants: An observational study","au":"Solans‐Domènech, Maite","y":2017,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvx021","cx":"Grant","co":["Reliability-noise","Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"bronze","cby":16,"self":false,"s":"An observational study of over 5,000 biomedical grant evaluations compares blinded versus unblinded assessments by the same reviewers, finding scores changed in about a fifth of cases after unblinding, often correlating with applicant reputation.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"QQRFQCNU","t":"A critique of Whitehurst's \"Interrater agreement for journal manuscript reviews\": De omnibus, disputandem est.","au":"Cicchetti, Domenic V.","y":1985,"vn":"American Psychologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/0003-066x.40.5.563","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"closed","cby":43,"self":false,"s":"Commentary disputes an earlier author's critique of a study on inter-rater agreement in journal manuscript review, addressing disagreement over the framing of politics and science in diagnostic classification debates.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"QR68P67D","t":"Improving the efficiency of research proposals evaluation: A two-stage procedure","au":"Seeber, Marco","y":2024,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvae020","cx":"Grant","co":["Reliability-noise","Interventions"],"sg":["proposal-quality","individual-scoring"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":1,"self":false,"s":"Studying Stiftelsen Dam's 2020 switch to a two-stage review process, short proposal first, then a longer one for finalists, the study found it cut applicant and reviewer time substantially while producing more reliable scores than the earlier one-stage system.","pg":"preregistered","pgr":"OSF","pgid":"https://osf.io/y65j8","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"QWNRA5FG","t":"Comparison of the effects of using the Crowe Critical Appraisal Tool versus informal appraisal in assessing health research: a randomised trial.","au":"Crowe, Michael","y":2011,"vn":"International journal of evidence-based healthcare","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/j.1744-1609.2011.00237.x","cx":"General","co":["Reliability-noise","Interventions"],"sg":["individual-scoring"],"x":1,"st":"experimental-rct","oast":"closed","cby":157,"self":false,"s":"A small randomized trial comparing a structured critical appraisal tool to informal appraisal of health research papers found the structured tool produced higher interrater agreement and reduced variance attributable to individual raters.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"QXKD6QZG","t":"A&A community survey on the future of scientific publishing","au":"Alves, J.","y":2026,"vn":"Astronomy and Astrophysics","ty":"journalArticle","tyl":"Journal article","doi":"10.1051/0004-6361/202661365","cx":"Journal","co":["Reliability-noise","Bias"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"diamond","cby":0,"self":false,"s":"A large survey of nearly 3,000 astronomy journal authors and reviewers finds that journal reputation and cost most influence where researchers publish, that reviewer expertise and fairness are bigger concerns than review speed, and that views on AI in publishing are sharply divided.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"R2SIKQTM","t":"Healing the NIH-Funded Biomedical Research Enterprise","au":"Germain, Ronald N.","y":2015,"vn":"Cell","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.cell.2015.05.052","cx":"Grant","co":["Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"bronze","cby":26,"self":false,"s":"An opinion piece proposing that NIH shift from project-based to person-based grant funding to better support young investigators and reduce misaligned incentives in the current R01 system.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"R3CGAELC","t":"Grant writing and grant peer review as questionable research practices","au":"Conix, Stijn","y":2021,"vn":"F1000Research","ty":"journalArticle","tyl":"Journal article","doi":"10.12688/f1000research.73893.1","cx":"Grant","co":["general-systemic"],"sg":["proposal-quality","whole-process"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":15,"self":false,"s":"Argues that grant peer review systems create pressure on researchers to compromise several core research-integrity values, adding an ethical dimension to existing critiques of proposal-based funding.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"R4APKMSR","t":"Citizen Scholar Chapter 5: Peer Review, Unleashed","au":"Cohen, Philip N.","y":2023,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31235/osf.io/nax2p","cx":"General","co":["general-systemic"],"sg":["whole-process"],"x":0,"st":"book-monograph","oast":"gold","cby":0,"self":false,"s":"Book chapter arguing that public discussion of research on social media extends and reinforces, rather than undermines, the traditional principles of scientific peer review.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"R4QME8PM","t":"Comparing 2 approaches to residency application file review","au":"Younan J.","y":2019,"vn":"CMAJ. Canadian Medical Association Journal","ty":"journalArticle","tyl":"Journal article","doi":"10.1503/cjs.011719","cx":"Analogue","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":1,"self":false,"s":"A comparison of two residency-application file review approaches, trait-based versus element-based scoring, with ten reviewers assessing seven files each, finds the trait-based method achieved substantially higher inter-rater reliability despite requiring more review time.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"R733ES7Q","t":"Rethink funding by putting the lottery first","au":"Luebber, Finn","y":2023,"vn":"Nature Human Behaviour","ty":"journalArticle","tyl":"Journal article","doi":"10.1038/s41562-023-01649-y","cx":"Grant","co":["Bias","Validity","Interventions"],"sg":["funding-decision"],"x":0,"st":"simulation-methodological","oast":"closed","cby":9,"self":false,"s":"Presents a simulation tool modeling how introducing a lottery early in grant decision-making would affect funding costs, diversity, and quality, arguing this timing better promotes inclusion than a lottery used only as a late tiebreaker.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"open","cau":""},{"k":"R8HWP4XI","t":"‘Science by consensus’ impedes scientific creativity and progress: A simple alternative to funding biomedical research","au":"Düzgüneş, Nejat","y":2023,"vn":"F1000Research","ty":"journalArticle","tyl":"Journal article","doi":"10.12688/f1000research.124082.2","cx":"Grant","co":["Reliability-noise","Bias","Validity","Interventions"],"sg":["funding-decision"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":0,"self":false,"s":"A pointed critique lists fifteen problems with NIH and NSF grant peer review and proposes replacing it with a system funding many more investigators directly, forgoing the traditional review stage entirely, and testing both approaches against each other.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"R8Q4R9V6","t":"Manuscript review in psychology: An alternative model.","au":"Bornstein, Robert F.","y":1990,"vn":"American Psychologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/0003-066x.45.5.672","cx":"Journal","co":["Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":41,"self":false,"s":"An opinion piece critiquing the reliability and fairness of psychology journal manuscript review and proposing an alternative review model.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"RA5UQF8G","t":"When journal editors play favorites","au":"Heesen, Remco","y":2017,"vn":"Philosophical Studies","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11098-017-0895-4","cx":"Journal","co":["Bias","Validity"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"simulation-methodological","oast":"hybrid","cby":14,"self":false,"s":"A philosophical analysis considers whether journal editors should adopt triple-anonymous review, concluding that non-anonymous review can create injustice toward some authors, while its effect on published quality depends on the type of bias involved and varies by field.","pg":"no-statement","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"RATFKYTW","t":"Exploring virtual funding committee practices in the allocation of National Institute for Health and Care Research funding: A netnographic study","au":"Blatch-Jones, Amanda","y":2024,"vn":"F1000Research","ty":"journalArticle","tyl":"Journal article","doi":"10.12688/f1000research.145582.1","cx":"Grant","co":["general-systemic"],"sg":["panel-consensus"],"x":0,"st":"qualitative","oast":"gold","cby":1,"self":false,"s":"A mixed-methods study observing virtual funding committee meetings for a UK health research funder, examining meeting efficiency, member wellbeing, and social dynamics including awareness of unconscious bias.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"https://doi.org/10.17605/OSF.IO/RZ6VT","ca":"not-applicable","cau":""},{"k":"RB5785EC","t":"Peer review process: Assessments by applicant-nominated referees are biased, inflated, unreliable and invalid","au":"Marsh, Herbert W.","y":2007,"vn":"Australian psychologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/00050060600823275","cx":"Grant","co":["Reliability-noise","Bias","Validity"],"sg":["reviewer-selection","individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"closed","cby":37,"self":false,"s":"Analysis of Australian Research Council reviews finding assessors nominated by applicants themselves gave higher, less consistent, and less predictive ratings than assessors chosen independently, leading the funder to stop using applicant-nominated reviewers.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"RB6NP5W7","t":"Assessing precision in the manuscript review process: A little better than a dice roll","au":"Lindsey, Duncan","y":1988,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/bf02020243","cx":"Journal","co":["Reliability-noise"],"sg":["funding-decision"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":98,"self":false,"s":"Reviews and reanalyzes prior studies on the precision of manuscript review decisions, concluding that highly selective editorial decisions made with imprecise reviewers perform only slightly better than chance.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"RBI5CT7L","t":"Peer Review in Academia","au":"Forsberg, Eva","y":2022,"vn":"","ty":"book","tyl":"Book","doi":"10.1007/978-3-030-75263-7_1","cx":"General","co":["general-systemic"],"sg":["whole-process"],"x":0,"st":"book-monograph","oast":"gold","cby":8,"self":false,"s":"An introductory book chapter frames peer review in relation to academic autonomy and the broader science-society contract, introducing key themes explored elsewhere in the volume.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"RCUHNQYG","t":"Double‐blind peer review affects reviewer ratings and editor decisions at an ecology journal","au":"Fox, Charles W.","y":2023,"vn":"Functional Ecology","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/1365-2435.14259","cx":"Journal","co":["Bias","Interventions"],"sg":["individual-scoring","funding-decision"],"x":1,"st":"experimental-rct","oast":"bronze","cby":105,"self":false,"s":"Randomised trial at an ecology journal found double-blind review lowered average manuscript ratings and revision invitations, with the effect concentrated among authors from lower-development, lower English-proficiency countries.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://doi.org/10.5061/dryad.m63xsj466","ca":"no-statement","cau":""},{"k":"RFR8CR44","t":"How Masked Is the “Masked Peer Review” of Abstracts Submitted to International Medical Conferences?","au":"Falagas, Matthew E.","y":2006,"vn":"Mayo Clinic Proceedings","ty":"journalArticle","tyl":"Journal article","doi":"10.4065/81.5.705","cx":"Journal","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"bronze","cby":6,"self":false,"s":"Analyzing conference abstract books, the study found reviewers could often still identify authors' institutions or countries despite blinding, casting doubt on how effectively masked review actually conceals author identity.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"RGKZZYMF","t":"Fairness of Peer Review Systems in Hospitality and Tourism Journals","au":"Cheong, Fan","y":2024,"vn":"Journal of China Tourism Research","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/19388160.2024.2441362","cx":"Journal","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"qualitative","oast":"closed","cby":0,"self":false,"s":"Using grounded theory, this study develops a five-theme, eleven-subtheme framework for fair peer review in hospitality and tourism journals and offers recommendations for editors, reviewers, and authors.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"RJEGBDFX","t":"A game theoretic approach to peer review of grant proposals","au":"Bayindir, Esra Eren","y":2019,"vn":"Journal of Informetrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.joi.2019.100981","cx":"Grant","co":["Reliability-noise","Interventions"],"sg":["panel-consensus"],"x":0,"st":"simulation-methodological","oast":"closed","cby":3,"self":false,"s":"Models the grant peer-review process used by Turkish regional development agencies as a Bayesian game among three reviewers balancing false-acceptance and false-rejection risks, comparing its outcomes to two simpler review procedures.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"on-request","cau":""},{"k":"RJLFSWWX","t":"Standardizing an approach to the evaluation of implementation science proposals.","au":"Crable, Erika L","y":2018,"vn":"Implementation Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s13012-018-0770-5","cx":"Grant","co":["Reliability-noise","Interventions"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":38,"self":false,"s":"Describes development of a scoring tool tailored to implementation-science grant proposals, applied to 30 pilot proposals, finding most scored poorly on criteria like theoretical grounding and readiness for adoption.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"no-statement","cau":""},{"k":"RJR9QVGF","t":"Wie zuverlässig ist das Peer-Review-Verfahren? Eine Untersuchung der Interrater-Reliabilität von Gutachter*innen auf DGPuK-Tagungen","au":"Koch, Thomas","y":2019,"vn":"Studies in Communication and Media","ty":"journalArticle","tyl":"Journal article","doi":"10.5771/2192-4007-2019-2-203","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"diamond","cby":2,"self":false,"s":"German-language study of reviewer agreement across five years of communication-studies conference submissions, finding disagreement on both overall judgments and individual criteria, though averaging across criteria improved consistency.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"RMNMTAKV","t":"Gender differences in peer review outcomes and manuscript impact at six journals of ecology and evolution.","au":"Fox, Charles W","y":2019,"vn":"Ecology and Evolution","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/ece3.4993","cx":"Journal","co":["Bias"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"gold","cby":190,"self":false,"s":"Analysis of over 23,000 manuscripts across six ecology and evolution journals finding female first authors received slightly worse peer-review scores and citations, partly linked to differences in who served as corresponding author.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://doi.org/10.5061/dryad.7p048mk; https://doi.org/10.5061/dryad.6nh4fc2","ca":"no-statement","cau":""},{"k":"RN4Z3CSB","t":"Defining the role of cognitive distance in the peer review process with an explorative study of a grant scheme in infection biology","au":"Wang, Qi","y":2015,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvv009","cx":"Grant","co":["Bias"],"sg":["reviewer-selection"],"x":0,"st":"observational-quantitative","oast":"green","cby":31,"self":false,"s":"An exploratory study of a Swedish infection-biology grant scheme develops a method for measuring cognitive distance between applicants and reviewers based on shared references and topics, finding this distance may affect review outcomes in unexpected ways.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"RP9WIA7I","t":"Differences in Editorial Board Reviewer Behavior Based on Gender","au":"Wing, Deborah A.","y":2010,"vn":"Journal of Women s Health","ty":"journalArticle","tyl":"Journal article","doi":"10.1089/jwh.2009.1904","cx":"Journal","co":["Bias"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"closed","cby":76,"self":false,"s":"Analysis of over 6,000 manuscript reviews at one journal from 2002-2008 found female editorial board members were less likely to recommend acceptance and had longer turnaround times than male members, though overall agreement with editors' final decisions did not differ.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"RQS39XWL","t":"Impact from the Evaluators’ Eye","au":"Derrick, Gemma","y":2018,"vn":"","ty":"book","tyl":"Book","doi":"10.1007/978-3-319-63627-6_1","cx":"General","co":["general-systemic"],"sg":["panel-consensus"],"x":0,"st":"book-monograph","oast":"closed","cby":1,"self":false,"s":"A book examines how evaluators in the UK's 2014 Research Excellence Framework collectively interpreted and scored the newly introduced \"impact\" criterion, focusing on group dynamics rather than the fairness of the outcomes themselves.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"RR7TA29K","t":"Revealing Reviewers’ Identities as Part of Open Peer Review and Analysis of the Review Reports","au":"Bolek, Cezary","y":2020,"vn":"LIBER Quarterly The Journal of the Association of European Research Libraries","ty":"journalArticle","tyl":"Journal article","doi":"10.18352/lq.10347","cx":"Journal","co":["Bias","Interventions"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"diamond","cby":6,"self":false,"s":"Analyzing 343 review reports from one journal's optional open-review system, this study found female reviewers more often chose to stay anonymous, and reviewers recommending rejection were less willing to reveal their identity than those recommending acceptance.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://cbolek.page.link/pr2020","ca":"open","cau":"https://cbolek.page.link/pr2020"},{"k":"RR8Z3GMV","t":"Is Lottery a Better Way of Resource Distribution Than Baseline Funding?","au":"Sakamoto, Takahiro","y":2023,"vn":"Philosophy of Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/psa.2023.49","cx":"Grant","co":["Validity"],"sg":["funding-decision"],"x":0,"st":"simulation-methodological","oast":"bronze","cby":2,"self":false,"s":"A theoretical model incorporating baseline block-grant funding argues that combining conventional peer review with baseline funding outperforms combining peer review with a lottery system in many research-funding scenarios, contrary to some prior simulation studies.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"open","cau":"https://github.com/TSakamoto-evo/funding"},{"k":"RRJ5XCFM","t":"Reliability of reviews for the American Psychologist: A biostatistical assessment of the data.","au":"Cicchetti, Domenic V.","y":1980,"vn":"American Psychologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/0003-066x.35.3.300","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":160,"self":false,"s":"A statistical assessment of reviewer evaluations submitted to a major psychology journal, situated within a broader literature critical of reviewer competence, consistency, and fairness in the review process.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"RRPR8N5D","t":"Development and evaluation of a quality score for abstracts","au":"Timmer, Antje","y":2003,"vn":"BMC Medical Research Methodology","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/1471-2288-3-2","cx":"Journal","co":["Reliability-noise","Validity"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":121,"self":false,"s":"Describes development of a standardized scoring instrument for conference abstracts, finding it achieved good reviewer agreement and that higher scores predicted acceptance.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"RRRS56JK","t":"Role, function, and expectations of research funding committees: Perspectives from committee members.","au":"Blatch-Jones, Amanda","y":2024,"vn":"F1000Research","ty":"journalArticle","tyl":"Journal article","doi":"10.12688/f1000research.154665.3","cx":"Grant","co":["general-systemic"],"sg":["panel-consensus","funding-decision"],"x":0,"st":"qualitative","oast":"gold","cby":2,"self":false,"s":"Survey of members of UK research funding committees describing their perceived purpose and role, with most emphasizing fair, unbiased assessment and prioritizing which research to fund.","pg":"preregistered","pgr":"other","pgid":"https://ror-hub.org/study/805/","da":"restricted","dau":"https://doi.org/10.17605/OSF.IO/RZ6VT","ca":"not-applicable","cau":""},{"k":"RUSREZRN","t":"Exploring the potential role of community engagement in evaluating clinical and translational science grant proposals.","au":"Treem, Jeffrey W","y":2018,"vn":"Journal of clinical and translational science","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/cts.2018.311","cx":"Grant","co":["Reliability-noise"],"sg":["individual-scoring","panel-consensus"],"x":0,"st":"observational-quantitative","oast":"gold","cby":4,"self":false,"s":"A study integrating 21 community members into a two-stage biomedical grant review process finds their ratings closely matched scientific reviewers' ratings, with community reviewers rarely commenting on technical aspects, suggesting engagement without compromising rigor.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"RV5VHIXB","t":"Uncertainty and Individual Discretion in Allocating Research Funds","au":"Goldstein, Anna","y":2017,"vn":"SSRN Electronic Journal","ty":"journalArticle","tyl":"Journal article","doi":"10.2139/ssrn.3012169","cx":"Grant","co":["Reliability-noise","Validity"],"sg":["funding-decision"],"x":0,"st":"observational-quantitative","oast":"green","cby":13,"self":false,"s":"Analysis of ARPA-E's internal project selection data finds program directors using individual discretion tend to favor proposals with more disagreement among expert reviewers, resulting in funding of more uncertain, creative projects.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"RVJKZWBR","t":"The Selection of Scientific Talent in the Allocation of Research Grants","au":"Arensbergen, P. van","y":2012,"vn":"Higher Education Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1057/hep.2012.15","cx":"Grant","co":["Reliability-noise"],"sg":["individual-scoring","panel-consensus","funding-decision"],"x":0,"st":"observational-quantitative","oast":"closed","cby":44,"self":false,"s":"Analysis of nearly 900 grant applications for career-development awards found that interviews and committee discussion, more than the initial written peer reviews, shaped decisions about which applicants were judged to show research talent.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"RYZBUT3Q","t":"Cost of the NSERC Science Grant Peer Review System Exceeds the Cost of Giving Every Qualified Researcher a Baseline Grant","au":"Gordon, Richard","y":2009,"vn":"Accountability in Research","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/08989620802689821","cx":"Grant","co":["Reliability-noise","Validity","Interventions"],"sg":["funding-decision"],"x":0,"st":"observational-quantitative","oast":"closed","cby":56,"self":false,"s":"Using Canadian NSERC statistics, this analysis argues the cost of preparing and reviewing grant applications exceeds what it would cost to give every qualified researcher a smaller baseline grant directly, proposing direct funding as more efficient.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"RZQ4LY8Z","t":"Dutch research funding, gender bias, and Simpson's paradox.","au":"Albers, Casper J","y":2015,"vn":"Proceedings of the National Academy of Sciences of the United States of America","ty":"journalArticle","tyl":"Journal article","doi":"10.1073/pnas.1518936112","cx":"Grant","co":["Bias"],"sg":["funding-decision"],"x":0,"st":"conceptual-commentary","oast":"bronze","cby":58,"self":false,"s":"A short piece (bibliographic metadata only, no substantive abstract available) apparently concerning gender bias in Dutch research funding and the statistical phenomenon of Simpson's paradox.","pg":"not-applicable","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"S2F2F3ZG","t":"Funding by lottery: political problems and research opportunities.","au":"Barnett, Adrian G","y":2016,"vn":"mBio","ty":"journalArticle","tyl":"Journal article","doi":"10.1128/mBio.01369-16","cx":"Grant","co":["Validity"],"sg":["funding-decision"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":24,"self":false,"s":"A commentary responding to a lottery-based grant funding proposal, arguing existing peer-review-based funding systems show significant biases and inefficiencies that a lottery approach could help address.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"S2HNC4AW","t":"Blinded vs. unblinded peer review of manuscripts submitted to a dermatology journal: a randomized multi-rater study","au":"Alam, M Fasihul","y":2011,"vn":"British Journal of Dermatology","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/j.1365-2133.2011.10432.x","cx":"Journal","co":["Bias","Interventions"],"sg":["individual-scoring"],"x":1,"st":"experimental-rct","oast":"closed","cby":57,"self":false,"s":"Randomized study at a dermatology journal comparing blinded and unblinded reviewers' scores on the same manuscripts, finding no significant difference in outcomes between the two conditions.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"S36DXTBV","t":"Anti-Asian biases in federal grant reviews: Commentary on Yip et al. (2021).","au":"Yip, Tiffany","y":2024,"vn":"American Psychologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/amp0001337","cx":"Grant","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"qualitative","oast":"green","cby":9,"self":false,"s":"Qualitative commentary and small survey study identifying themes of anti-Asian bias experienced during NIH grant review, proposing policy recommendations to address underrepresentation of this population in funded research.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"S38N4UBK","t":"How qualitative criteria can improve the assessment process of interdisciplinary research proposals","au":"Schölvinck, Anne-Floor M","y":2024,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvae049","cx":"Grant","co":["Validity","Interventions"],"sg":["individual-scoring"],"x":0,"st":"qualitative","oast":"hybrid","cby":5,"self":false,"s":"A study develops and tests qualitative criteria and bibliometric indicators intended to help review panels assess interdisciplinary research proposals, finding the qualitative criteria improved evaluative discussions more than the bibliometric measures did.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"","ca":"no-statement","cau":""},{"k":"S49RVGAY","t":"Toward a New Model of Scientific Publishing: Discussion and a Proposal","au":"Kravitz, Dwight J.","y":2011,"vn":"Frontiers in Computational Neuroscience","ty":"journalArticle","tyl":"Journal article","doi":"10.3389/fncom.2011.00055","cx":"Journal","co":["Validity","Interventions"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"gold","cby":62,"self":false,"s":"Proposes restructuring scientific publishing around post-publication evaluation of merit rather than journal-based accept or reject decisions, arguing current peer review is redundant and poorly incentivized.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"S4CDNSIP","t":"Research on research funding: an imperative for science and society","au":"Severin, Anna","y":2020,"vn":"British Journal of Sports Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bjsports-2020-103340","cx":"Grant","co":["Bias","Validity"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"green","cby":11,"self":false,"s":"An editorial argues that grant peer review in health sciences shows documented biases, tends toward conservatism, and only weakly predicts future research performance, calling for more systematic research on funding-decision processes.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"S4D7BM4N","t":"Scientific Grant Funding","au":"Azoulay, Pierre","y":2020,"vn":"National Bureau of Economic Research","ty":"journalArticle","tyl":"Journal article","doi":"10.3386/w26889","cx":"Grant","co":["general-systemic"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"green","cby":46,"self":false,"s":"A book chapter reviewing grant funding as a policy tool, covering its history, program design choices including peer review, incentives for risk-taking, and the case for more rigorous evaluation of funding programs.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"S59DARDU","t":"A Kuhnian Critique of Psychometric Research on Peer Review","au":"Lee, Carole J.","y":2012,"vn":"Philosophy of Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1086/667841","cx":"General","co":["Reliability-noise","Validity"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":43,"self":false,"s":"A philosophical critique argues that low interrater reliability among peer reviewers need not indicate a flawed evaluation system, since some disagreement may reflect legitimate differences in how reviewers interpret and apply disciplinary standards.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"S5ZNPKAX","t":"How reliable is peer review? An examination of operating grant proposals simultaneously submitted to two similar peer review systems.","au":"Hodgson, C","y":1997,"vn":"Journal of Clinical Epidemiology","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/S0895-4356(97)00167-4","cx":"Grant","co":["Reliability-noise"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"closed","cby":48,"self":false,"s":"Comparing scores given to the same grant proposals when submitted simultaneously to two similar review agencies, this study finds a moderate positive correlation between the two systems' scores and only moderate agreement on which proposals were fundable.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"S7AYT2Z6","t":"Is peer review useful in assessing research proposals in Indigenous health? A case study","au":"Street, Jackie","y":2009,"vn":"Health Research Policy and Systems","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/1478-4505-7-2","cx":"Grant","co":["Interventions"],"sg":["whole-process"],"x":0,"st":"qualitative","oast":"gold","cby":16,"self":false,"s":"Interview study asking Australian Indigenous health researchers and stakeholders about traditional academic peer review, finding broad agreement that it is poorly suited to evaluating Indigenous research proposals.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"S9EHKIC4","t":"Frame Search and Re-search: How Quantitative Sociological Articles Change During Peer Review","au":"Teplitskiy, Misha","y":2015,"vn":"The American Sociologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s12108-015-9288-3","cx":"Journal","co":["general-systemic"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":29,"self":false,"s":"Analyzes how sociology manuscripts change between conference presentation and peer-reviewed publication, finding that peer review prompts bigger changes to theoretical framing than to data analysis.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"SA7AMBC5","t":"Reliability of editors' subjective quality ratings of peer reviews of manuscripts.","au":"Callaham, M L","y":1998,"vn":"The Journal of the American Medical Association","ty":"journalArticle","tyl":"Journal article","doi":"10.1001/jama.280.3.229","cx":"Journal","co":["Reliability-noise","Validity"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":123,"self":false,"s":"A 3.5-year observational study of a peer-reviewed journal finds editors' subjective quality ratings of individual reviewers are moderately reliable and correlate with reviewers' ability to catch errors in a test manuscript, more so than with their acceptance recommendation rates.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"SA84R68P","t":"Inter-Rater Reliability in Assessing the Methodological Quality of Research Papers in Psychology","au":"Etzel, Franka Tabitha","y":2025,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31234/osf.io/4w7rb_v2","cx":"General","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":0,"self":false,"s":"Two studies asking groups of raters to assess methodological rigor of psychology papers using structured criteria found good to excellent inter-rater reliability, even among minimally trained raters, though a recent paper sample showed generally low methodological rigor.","pg":"preregistered","pgr":"OSF","pgid":"https://osf.io/4dw9g","da":"no-statement","dau":"","ca":"open","cau":"https://osf.io/4dw9g"},{"k":"SAJ2V3PH","t":"An Analysis of Preliminary and Post-Discussion Priority Scores for Grant Applications Peer Reviewed by the Center for Scientific Review at the NIH","au":"Martin, Michael R.","y":2010,"vn":"PLoS ONE","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0013526","cx":"Grant","co":["Reliability-noise"],"sg":["individual-scoring","panel-consensus"],"x":0,"st":"observational-quantitative","oast":"gold","cby":31,"self":false,"s":"Analysis of NIH grant scoring found panel discussion produced a meaningful shift in outcome for more than one in ten applications, contrary to the assumption that preliminary reviewer scores alone determine the final result.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"SAUTPX5E","t":"Peer review in funding-by-lottery: A systematic overview and expansion","au":"Shaw, Jamie","y":2023,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvac022","cx":"Grant","co":["Reliability-noise","Bias","Validity"],"sg":["funding-decision"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":29,"self":false,"s":"A conceptual paper develops a taxonomy of funding-by-lottery variants, clarifying that lotteries differ substantially in design and often incorporate peer review rather than replacing it, and identifies research gaps for future study.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"SBVJDN7K","t":"A randomized trial of fellowships for early career researchers finds a high reliability in funding decisions.","au":"Clarke, Philip","y":2016,"vn":"Journal of Clinical Epidemiology","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.jclinepi.2015.04.010","cx":"Grant","co":["Reliability-noise"],"sg":["panel-consensus","funding-decision"],"x":1,"st":"experimental-rct","oast":"closed","cby":21,"self":false,"s":"An experiment duplicating applications across two independent Australian early-career fellowship review panels finds relatively high agreement on funding outcomes (about 83 percent, with substantial chance-adjusted agreement), higher than reported for some other grant schemes.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"SC63XAI2","t":"Journal ratings as predictors of articles quality in Arts, Humanities and Social Sciences: an analysis based on the Italian Research Evaluation Exercise","au":"Bonaccorsi, Andrea","y":2015,"vn":"F1000Research","ty":"journalArticle","tyl":"Journal article","doi":"10.12688/f1000research.6478.1","cx":"General","co":["Validity"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":15,"self":false,"s":"Using a large Italian dataset of published articles and their independent peer review scores, this study finds a positive relationship between journal prestige rankings and article-level quality assessments in the humanities and social sciences.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"no-statement","cau":""},{"k":"SCBRUUIQ","t":"The sociology of scientific validity: How professional networks shape judgement in peer review","au":"Teplitskiy, Misha","y":2018,"vn":"Research Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.respol.2018.06.014","cx":"Journal","co":["Bias"],"sg":["reviewer-selection","individual-scoring"],"x":0,"st":"observational-quantitative","oast":"green","cby":112,"self":false,"s":"Analysis of thousands of neuroscience manuscript reviews at PLOS ONE finding reviewers gave more favorable scores to authors closer to them in the co-authorship network, even though instructed to judge only scientific validity.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"SDFDFFNQ","t":"Contest models highlight inherent inefficiencies of scientific funding competitions","au":"Gross, Kevin","y":2019,"vn":"PLoS Biology","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pbio.3000065","cx":"Grant","co":["Validity"],"sg":["proposal-quality","funding-decision"],"x":0,"st":"simulation-methodological","oast":"gold","cby":122,"self":false,"s":"Applies economic contest theory to grant competitions, estimating that researchers' time spent writing unsuccessful proposals can rival the scientific value of funded work, and suggesting partial lotteries or track-record-based funding as more efficient alternatives.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"","ca":"no-statement","cau":""},{"k":"SEPHFSGV","t":"Using simplified peer review processes to fund research: a prospective study","au":"Herbert, Danielle L.","y":2015,"vn":"BMJ Open","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmjopen-2015-008380","cx":"Grant","co":["Reliability-noise","Interventions"],"sg":["funding-decision","panel-consensus"],"x":0,"st":"observational-quantitative","oast":"gold","cby":32,"self":false,"s":"A prospective study compares two simplified grant review processes, shorter panels and paired independent review, against Australia's official longer process, finding agreement just below an acceptable threshold while offering substantial potential cost savings.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"on-request","cau":""},{"k":"SHG9FRIZ","t":"Funding Risky Research","au":"Franzoni, Chiara","y":2022,"vn":"Entrepreneurship and Innovation Policy and the Economy","ty":"journalArticle","tyl":"Journal article","doi":"10.1086/719252","cx":"Grant","co":["Bias","Interventions"],"sg":["funding-decision"],"x":0,"st":"conceptual-commentary","oast":"green","cby":0,"self":false,"s":"Drawing on the pace of mRNA vaccine development, this paper reviews evidence that funding systems are biased against risky research and discusses possible interventions to encourage more transformative research funding.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"SHHLHEPD","t":"Reliability of reviewers' ratings when using public peer review: a case study","au":"Bornmann, Lutz","y":2010,"vn":"Learned Publishing","ty":"journalArticle","tyl":"Journal article","doi":"10.1087/20100207","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":50,"self":false,"s":"A study of one open-access journal's public peer review process found reviewer agreement was still low to only moderate, similar to findings previously reported for traditional closed peer review.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"SI6TYDVR","t":"Testing Heterogeneity in Inter-Rater Reliability","au":"Bartos̆, Frantis̆ek","y":2020,"vn":"Springer proceedings in mathematics & statistics","ty":"book","tyl":"Book","doi":"10.1007/978-3-030-43469-4_26","cx":"Analogue","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"book-monograph","oast":"closed","cby":5,"self":false,"s":"A simulation study compares statistical methods for detecting differences in interrater reliability across groups, finding that two approaches perform similarly well while a third can yield inaccurate results.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://osf.io/9sajx/","ca":"open","cau":"https://osf.io/9sajx/"},{"k":"SIN7UYQK","t":"Scientists from Minority-Serving Institutions and Their Participation in Grant Peer Review","au":"Gallo, Stephen A.","y":2021,"vn":"BioScience","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/biosci/biab130","cx":"Grant","co":["Bias"],"sg":["reviewer-selection"],"x":0,"st":"observational-quantitative","oast":"bronze","cby":4,"self":false,"s":"A survey of scientists at minority-serving institutions finds substantially lower rates of participation in grant peer review than previously reported for scientists at predominantly white institutions, mainly due to lack of invitations and time constraints, despite strong interest in reviewing.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://doi.org/10.6084/m9.figshare.16540206","ca":"no-statement","cau":""},{"k":"SL7FVHLY","t":"Improving Scientific Judgments in Law and Government: A Field Experiment of Patent Peer Review","au":"Ho, Daniel E.","y":2020,"vn":"Journal of Empirical Legal Studies","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/jels.12249","cx":"General","co":["Interventions"],"sg":["funding-decision"],"x":1,"st":"experimental-rct","oast":"hybrid","cby":10,"self":false,"s":"A three-year randomized field experiment recruiting scientific experts to comment on US patent applications finds the intervention increased examiners' engagement with scientific literature and lowered initial grant rates, though effects were weaker than expected.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"SLY82ACP","t":"Should Authors be Requested to Suggest Peer Reviewers?","au":"Silva, Jaime A. Teixeira da","y":2017,"vn":"Science and Engineering Ethics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11948-016-9842-6","cx":"Journal","co":["Bias"],"sg":["reviewer-selection"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":23,"self":false,"s":"Argues against the common practice of asking authors to suggest their own peer reviewers, contending it undermines the fairness and trustworthiness of the review process.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"SNXCY9C9","t":"The iSchool Community: A Case Study of iConference Reviews","au":"Bogers, Toine","y":2016,"vn":"iConference 2016 Proceedings","ty":"journalArticle","tyl":"Journal article","doi":"10.9776/16247","cx":"Journal","co":["Bias"],"sg":["reviewer-selection","individual-scoring"],"x":0,"st":"observational-quantitative","oast":"green","cby":8,"self":false,"s":"Analyzes reviewing practices and reviewer demographics at an information-science conference, recommending a more diverse reviewer pool and clearer review instructions.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"SQH3JZM2","t":"A Computer Program for Assessing Specific Category Rater Agreement for Qualitative Data","au":"Cicchetti, Domenic V.","y":1978,"vn":"Educational and Psychological Measurement","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/001316447803800322","cx":"Analogue","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"closed","cby":93,"self":false,"s":"A methods paper describes a computer program for calculating category-specific rater agreement on nominal or ordinal qualitative data, including an option to collapse scale categories to improve reliability estimates.","pg":"no-statement","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"on-request","cau":""},{"k":"SQSVEPGJ","t":"Peer review--the newcomers' perspective.","au":"Mainguy, Gaell","y":2005,"vn":"PLoS biology","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pbio.0030326","cx":"Journal","co":["Bias","Interventions"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":27,"self":false,"s":"Commentary from early-career researchers argues that double-blind review would improve perceived fairness in global scientific funding and publishing.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"SR9TSCSZ","t":"2  Does end-user-engaged peer review improve the relevance, replicability, and accessibility of research evidence?","au":"Li, Edward W","y":2022,"vn":"BMJ Publishing Group Ltd","ty":"conferencePaper","tyl":"Conference paper","doi":"10.1136/ebm-2022-EBMLive.24","cx":"General","co":["Interventions"],"sg":["reviewer-selection","individual-scoring"],"x":0,"st":"systematic-review-meta-analysis","oast":"closed","cby":0,"self":false,"s":"A systematic review protocol outlines plans to examine whether involving patients and other end-users as reviewers improves the relevance, reproducibility, and accessibility of published and funded research.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"SS4GAFL8","t":"Extending ArXiv.org to Achieve Open Peer Review and Publishing","au":"Boldt, Axel","y":2010,"vn":"Journal of Scholarly Publishing","ty":"journalArticle","tyl":"Journal article","doi":"10.3138/jsp.42.2.238","cx":"Journal","co":["general-systemic"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":23,"self":false,"s":"A proposal describing a simple extension of the arXiv preprint server to enable open, signed peer review and a lightweight publishing workflow.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"SSVEGHMV","t":"Innovations in peer review in scholarly publishing: a meta-summary","au":"Woods, Helen Buckley","y":2022,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31235/osf.io/qaksd","cx":"Journal","co":["Interventions"],"sg":["whole-process"],"x":0,"st":"systematic-review-meta-analysis","oast":"gold","cby":6,"self":false,"s":"A literature review summarising six prior review articles categorises innovations in scholarly peer review into approaches to review, reviewer-focused initiatives, and supporting technologies, drawing overall conclusions about current practice and future research needs.","pg":"no-statement","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"STUQWYY7","t":"Agreeing to disagree: A study of the reliability of manuscript reviews","au":"Kirk, S. A.","y":1997,"vn":"Social Work Research","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/swr/21.2.121","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":37,"self":false,"s":"An editorial note describing an unusual case where a journal's own editorial board became both subjects and evaluators of a study on manuscript review reliability, complicating the normal blind review process.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"SUR92HZC","t":"A Quantitative Linguistic Analysis of National Institutes of Health R01 Application Critiques From Investigators at One Institution","au":"Kaatz, Anna","y":2014,"vn":"Academic Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.1097/acm.0000000000000442","cx":"Grant","co":["Bias"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"green","cby":83,"self":false,"s":"Text analysis of NIH R01 grant critiques from one university finding funded applications received more positive language, and identifying differences in critique wording associated with applicant gender.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"SUY5YMZQ","t":"Analysis of National Institutes of Health R01 Application Critiques, Impact, and Criteria Scores: Does the Sex of the Principal Investigator Make a Difference?","au":"Kaatz, Anna","y":2016,"vn":"Academic medicine : journal of the Association of American Medical Colleges","ty":"journalArticle","tyl":"Journal article","doi":"10.1097/ACM.0000000000001272","cx":"Grant","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"bronze","cby":157,"self":false,"s":"Text analysis of NIH R01 grant critiques finding that under the post-2009 review format, female principal investigators received worse priority scores on renewal applications despite similarly positive language in critiques.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"SVBDILLU","t":"A unified reviewing format for grant applications and evaluations.","au":"Janke, Carsten","y":2018,"vn":"EMBO reports","ty":"journalArticle","tyl":"Journal article","doi":"10.15252/embr.201745611","cx":"Grant","co":["Interventions"],"sg":["proposal-quality","individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"bronze","cby":2,"self":false,"s":"A commentary proposes standardizing the format used for grant applications and reviews as a way to ease reviewer recruitment burden and improve the consistency and quality of peer review.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"SWHA4YCC","t":"Peer Review or Lottery? A Critical Analysis of Two Different Forms of Decision-making Mechanisms for Allocation of Research Grants","au":"Roumbanis, Lambros","y":2019,"vn":"Science, Technology, & Human Values","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/0162243918822744","cx":"Grant","co":["Reliability-noise","Bias","Validity"],"sg":["funding-decision"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":83,"self":false,"s":"Critical analysis compares peer review against lottery-based allocation of research grants, arguing a lottery among minimally qualified applicants could reduce bias and disagreement while increasing diversity of funded research.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"SXIK8HML","t":"Reviewer recommendations and editorial outcomes in peer review: a longitudinal analysis of agreement and disagreement across review rounds","au":"Cuevas, Carlos De las","y":2026,"vn":"Research Integrity and Peer Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s41073-026-00200-7","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"gold","cby":0,"self":false,"s":"Analysis of over 15,000 manuscripts at one journal found reviewers frequently disagreed in the first review round, but disagreement declined markedly by the final round for manuscripts that were eventually published.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"no-statement","cau":""},{"k":"SYCLI3BC","t":"Towards a More Structured Peer Review Process with Empirical Standards","au":"Arshad, Arham","y":2021,"vn":"Evaluation and Assessment in Software Engineering","ty":"journalArticle","tyl":"Journal article","doi":"10.1145/3463274.3463359","cx":"Journal","co":["Reliability-noise","Interventions"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":6,"self":false,"s":"This paper presents a tool that generates structured, checklist-based peer review forms from \"empirical standards\" documenting community expectations for software engineering research methodologies, aiming to improve review consistency.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"SYDFSMXV","t":"Assessing interrater agreement via the average deviation index given a variety of theoretical and methodological problems","au":"Smith-Crowe, Kristin","y":2013,"vn":"Organizational Research Methods","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/1094428112465898","cx":"Analogue","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"closed","cby":29,"self":false,"s":"A methodological paper provides reference values for interpreting an average-deviation index of interrater agreement across various measurement and theoretical scenarios in organizational psychology research.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"SYUM6PRB","t":"Examining the predictive validity of NIH peer review scores.","au":"Lindner, Mark D","y":2015,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0126938","cx":"Grant","co":["Validity"],"sg":["post-award-validity"],"x":0,"st":"observational-quantitative","oast":"gold","cby":27,"self":false,"s":"Argues that comparing NIH peer review percentile scores to bibliometric outcomes is not a valid test of predictive validity because funded applications are a restricted, non-random sample shaped by negotiation and revision after review.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"http://public.csr.nih.gov/aboutcsr/NewsAndPublications/News/Documents/DEIDENTIFIEDDATASETUSINGZSCORES.XLSX","ca":"no-statement","cau":""},{"k":"SZ554632","t":"Assessing Inter-rater Reliability With Heterogeneous Variance Components Models: Flexible Approach Accounting for Contextual Variables","au":"Martinková, Patrícia","y":2023,"vn":"Journal of Educational and Behavioral Statistics","ty":"journalArticle","tyl":"Journal article","doi":"10.3102/10769986221150517","cx":"Grant","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"closed","cby":7,"self":false,"s":"Proposes a Bayesian statistical method for estimating inter-rater reliability while accounting for contextual factors like rater gender or experience, illustrated with grant peer review data.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://osf.io/bk8a7/","ca":"open","cau":"https://osf.io/bk8a7/"},{"k":"T275N5RX","t":"Can Oncologists Predict the Efficacy of Treatments in Randomized Trials?","au":"Benjamin, D. M.","y":2020,"vn":"The Oncologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1634/theoncologist.2020-0054","cx":"Analogue","co":["Validity"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"bronze","cby":15,"self":false,"s":"Study testing whether oncologists could forecast trial outcomes, finding expert predictions performed only modestly better than chance and worse than simple statistical base-rate models, questioning reliance on expert judgment for funding or policy decisions.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"T3A36PFH","t":"Interrater agreement for journal manuscript reviews.","au":"Whitehurst, Grover J.","y":1984,"vn":"American Psychologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/0003-066x.39.1.22","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":190,"self":false,"s":"An older methodological paper argues that standard intraclass correlation measures understate reviewer agreement on journal manuscripts when manuscript quality variance is low, proposing an alternative statistic (Finn's r) and reanalyzing past agreement data.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"T4CIJ7Y9","t":"Hidden Inequities in Peer Review: Integrating Geographic, Ethnic, and Linguistic Bias","au":"Machado, Maria","y":2026,"vn":"Trends in Scholarly Publishing","ty":"journalArticle","tyl":"Journal article","doi":"10.21124/tsp.2026.37.42","cx":"Journal","co":["Bias","Interventions"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"diamond","cby":0,"self":false,"s":"Review article synthesizes evidence on geographic, ethnic, and linguistic biases in peer review, highlighting how non-native English writing and reviewer composition can disadvantage authors independent of research quality.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"T584RU39","t":"Gender bias in the allocation of student grants","au":"Wijnen, Marjolijn N.","y":2021,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-021-03985-0","cx":"Grant","co":["Bias"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":22,"self":false,"s":"A 24-year analysis of a Dutch university study-abroad fund found female student applicants had lower success rates than male applicants despite equal application rates and equally rated proposal quality once gender cues were removed.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"","ca":"open","cau":""},{"k":"T5FYTZ8E","t":"Structured peer review","au":"Malički, Mario","y":2023,"vn":"PUBMET","ty":"journalArticle","tyl":"Journal article","doi":"10.15291/pubmet.4268","cx":"Journal","co":["Reliability-noise","Interventions"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"diamond","cby":0,"self":false,"s":"A pilot study of structured peer review across 220 Elsevier journals analyzed how reviewers answered nine standardized questions on 107 manuscripts, finding patterns of agreement that the authors used to refine the question set.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"T67A4S8E","t":"Review panel consensus and post-decision commercial performance: a study of early stage technologies","au":"Galbraith, Craig S.","y":2010,"vn":"The Journal of technology transfer","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s10961-009-9122-6","cx":"Grant","co":["Reliability-noise","Validity"],"sg":["panel-consensus","post-award-validity"],"x":0,"st":"observational-quantitative","oast":"closed","cby":12,"self":false,"s":"Studying grant review panels evaluating early-stage technologies, the authors find panellists tend to converge toward consensus after discussion, with technical experts most resistant to changing their views, though shared information did not clearly improve predictions of commercial success.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"T6YQ33G6","t":"Inter-rater agreement in the scoring of abstracts submitted to a primary care research conference","au":"Montgomery, Alan","y":2002,"vn":"BMC Health Services Research","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/1472-6963-2-8","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":147,"self":false,"s":"A study of two-reviewer scoring of primary care conference abstracts finds moderate agreement on objective study-design criteria but much weaker agreement on more subjective elements like perceived importance.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"T6Z9QVYH","t":"How Gendered Is the Peer-Review Process? A Mixed-Design Analysis of Reviewer Feedback","au":"König, Thomas","y":2021,"vn":"PS Political Science & Politics","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/s1049096521000937","cx":"Journal","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":7,"self":false,"s":"Analysis of a political science journal's reviews over 14 years finding reviewers tended to favor authors of their own gender, an effect reduced when both a male and female reviewer evaluated the same manuscript.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://doi.org/10.7910/DVN/LMDUEQ","ca":"no-statement","cau":"https://doi.org/10.7910/DVN/LMDUEQ"},{"k":"TA7PAIQN","t":"Lottery before peer review is associated with increased female representation and reduced estimated economic cost in a German funding line.","au":"Luebber, Finn","y":2025,"vn":"Nature communications","ty":"journalArticle","tyl":"Journal article","doi":"10.1038/s41467-025-65660-9","cx":"Grant","co":["Bias","Validity","Interventions"],"sg":["funding-decision"],"x":0,"st":"observational-quantitative","oast":"gold","cby":2,"self":false,"s":"Analysis of a German funding scheme using a lottery-first, then peer-review approach, finding increased submissions and awards to women along with substantially lower administrative costs compared with conventional review.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://osf.io/4ufrb/","ca":"open","cau":"https://doi.org/10.17605/OSF.IO/4UFRB"},{"k":"TCT2KQAT","t":"Similar‐to‐me effects in the grant application process: Applicants, panellists, and the likelihood of obtaining funds","au":"Banal‐Estañol, Albert","y":2023,"vn":"R and D Management","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/radm.12601","cx":"Grant","co":["Bias"],"sg":["panel-consensus","funding-decision"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":6,"self":false,"s":"A study of UK Engineering and Physical Sciences Research Council funding decisions tests whether panel members favor applicants who share their characteristics, finding evidence of such similarity effects across disciplinary orientation and gender composition of panels.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"no-statement","cau":""},{"k":"TE7UT3JG","t":"Looking Across and Looking Beyond the Knowledge Frontier: Intellectual Distance, Novelty, and Resource Allocation in Science.","au":"Boudreau, Kevin J","y":2016,"vn":"Management science","ty":"journalArticle","tyl":"Journal article","doi":"10.1287/mnsc.2015.2207","cx":"Grant","co":["Reliability-noise","Validity"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"green","cby":1,"self":false,"s":"Randomized field experiment assigning over 2,000 evaluator-proposal pairs at a research university finds evaluators give lower scores to proposals closer to their own expertise and to highly novel proposals, consistent with bounded rationality rather than random noise.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"THI5QKB2","t":"How to improve scientific peer review: Four schools of thought","au":"Waltman, Ludo","y":2023,"vn":"Learned Publishing","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/leap.1544","cx":"Journal","co":["Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"hybrid","cby":65,"self":false,"s":"Proposes organizing the many current peer-review reform initiatives into four distinct schools of thought, each prioritizing different goals such as reproducibility, transparency, equity, or efficiency.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"TIAHXM4B","t":"Interjudgmental reliability of reviews for the Journal of Educational Psychology..","au":"Marsh, Herbert W.","y":1981,"vn":"Journal of Educational Psychology","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/0022-0663.73.6.872","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":89,"self":false,"s":"A study of manuscript reviews for an educational psychology journal found only moderate reliability in reviewers' overall recommendations, with agreement on specific rating subscales even lower than for overall judgments.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"TLKPSVAY","t":"Review of Peer Review Final report","au":"Kolarz, Peter","y":null,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"","cx":"Grant","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"systematic-review-meta-analysis","oast":"","cby":0,"self":false,"s":"A commissioned final report reviewing interventions in research-funding peer review, covering measures applied before, during and after grant calls, intended to inform funders on choices such as demand management and application-length reduction.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"TLPKXX2Q","t":"Cognitive Bias in the Peer Review Process","au":"Street, Chris","y":2019,"vn":"ACM SIGMIS Database the DATABASE for Advances in Information Systems","ty":"journalArticle","tyl":"Journal article","doi":"10.1145/3371041.3371046","cx":"Journal","co":["Bias"],"sg":["individual-scoring"],"x":1,"st":"experimental-rct","oast":"closed","cby":11,"self":false,"s":"A survey-based study finds that people's judgments about which research quality practices matter shift depending on whether they are answering as a researcher seeking to publish or as a reviewer evaluating others' work, suggesting a motivational cognitive bias.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"TLSP5JN5","t":"Do Author-Suggested Reviewers Rate Submissions More Favorably than Editor-Suggested Reviewers? A Study on Atmospheric Chemistry and Physics","au":"Bornmann, Lutz","y":2010,"vn":"PLoS ONE","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0013345","cx":"Journal","co":["Bias"],"sg":["reviewer-selection","individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":43,"self":false,"s":"Study of an atmospheric science journal finding reviewers suggested by authors rated manuscripts substantially more favorably than editor-selected reviewers did, even after accounting for manuscript quality.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"TTLBYQFY","t":"Citation Impact of NHLBI R01 Grants Funded Through the American Recovery and Reinvestment Act as Compared to R01 Grants Funded Through a Standard Payline","au":"Danthi, Narasimhan","y":2015,"vn":"Circulation Research","ty":"journalArticle","tyl":"Journal article","doi":"10.1161/circresaha.116.305894","cx":"Grant","co":["Validity"],"sg":["funding-decision","post-award-validity"],"x":0,"st":"observational-quantitative","oast":"bronze","cby":21,"self":false,"s":"Compares citation impact of NIH grants funded through emergency stimulus funding, which had scored below the normal cutoff, against regularly funded grants, finding comparable impact per dollar spent despite lower initial peer-review scores.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"TTVVB5VB","t":"Factors influencing the scientific performance of Momentum grant holders: an evaluation of the first 117 research groups","au":"Győrffy, Balázs","y":2018,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-018-2852-1","cx":"Grant","co":["Validity"],"sg":["post-award-validity"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":24,"self":false,"s":"An evaluation of the first 117 research groups funded by Hungary's Momentum programme finds scientific output was largely independent of gender, degree, and prior grant-review scores, but strongly linked to prior citation record and impact factor.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"TVAKSECW","t":"Supporting robust, rigorous, and reliable reviewing as the cornerstone of our profession: Introducing a competency framework for peer review","au":"Köhler, Tine","y":2020,"vn":"Industrial and organizational psychology","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/iop.2019.121","cx":"General","co":["Reliability-noise","Bias","Interventions"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"green","cby":58,"self":false,"s":"Proposes a competency framework defining the skills and behaviors associated with high-quality peer review, intended to guide reviewer training and standard-setting in industrial-organizational psychology.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"TZXYKKCC","t":"A retrospective analysis of the effect of discussion in teleconference and face-to-face scientific peer-review panels.","au":"Carpenter, Afton S","y":2015,"vn":"BMJ Open","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmjopen-2015-009138","cx":"Grant","co":["Reliability-noise","Interventions"],"sg":["panel-consensus"],"x":0,"st":"observational-quantitative","oast":"gold","cby":32,"self":false,"s":"A retrospective analysis of NIH grant panels found that discussion produced only small average score changes but shifted more than one in ten applications across the funding threshold, regardless of whether the panel met by teleconference or in person.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"http://dx.doi.org/10.6084/m9.figshare.1495503","ca":"no-statement","cau":""},{"k":"U2VS5L4D","t":"Does the NIH Fund Edge Science?","au":"Packalén, Mikko","y":2018,"vn":"National Bureau of Economic Research","ty":"journalArticle","tyl":"Journal article","doi":"10.3386/w24860","cx":"Grant","co":["Validity"],"sg":["funding-decision"],"x":0,"st":"observational-quantitative","oast":"green","cby":12,"self":false,"s":"A short piece examining the extent to which NIH-funded research supports high-risk, boundary-pushing (\"edge\") science as part of the agency's broader funding mandate.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"U3FISC8H","t":"Peer review reports of randomized controlled trials in oncology can be short and superficial","au":"Logullo, Patrícia","y":2025,"vn":"Journal of Clinical Epidemiology","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.jclinepi.2025.111893","cx":"Journal","co":["general-systemic"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":2,"self":false,"s":"Evaluating open peer-review reports for oncology randomized trials published in BioMed Central journals, the study finds reviews were typically brief and rarely constructive, with limited attention paid to result interpretation or trial reporting completeness.","pg":"preregistered","pgr":"OSF","pgid":"https://osf.io/nyx8v/","da":"open","dau":"https://osf.io/2j43q","ca":"no-statement","cau":""},{"k":"U43AIBHF","t":"Working double-blind","au":"Nature","y":2008,"vn":"Nature","ty":"journalArticle","tyl":"Journal article","doi":"10.1038/451605b","cx":"Journal","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"bronze","cby":26,"self":false,"s":"An editorial commenting on debates around bringing science into US political campaigns, briefly referencing broader themes of double-blind scrutiny applied to public claims.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"U4JKK6XI","t":"How do authors’ perceptions of their papers compare with co-authors’ perceptions and peer-review decisions?","au":"Rastogi, Charvi","y":2024,"vn":"PLoS ONE","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0300710","cx":"Journal","co":["Reliability-noise","Bias"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"gold","cby":4,"self":false,"s":"Survey of authors at a major computer science conference finding they substantially overestimated their papers' acceptance chances and often disagreed with peer-review outcomes and even with their own co-authors' rankings.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"no-statement","cau":""},{"k":"U4LVMCYZ","t":"Peer review of grant applications: what do we know?","au":"Wessely, S","y":1998,"vn":"The Lancet","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/S0140-6736(97)11129-1","cx":"Grant","co":["Reliability-noise","Bias","Validity","Interventions"],"sg":["reviewer-selection","individual-scoring"],"x":0,"st":"systematic-review-meta-analysis","oast":"closed","cby":143,"self":false,"s":"Review discussing what is empirically known about grant peer review, covering topics such as reviewer expertise, institutional bias, and inconsistent evidence on cronyism across funding agencies.","pg":"no-statement","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"U5BTC2NZ","t":"Ejes de discusión en la evaluación de la ciencia: revisión por pares, bibliometría y pertinencia","au":"Sarthou, Nerina Fernanda","y":2016,"vn":"Revista de Estudios Sociales","ty":"journalArticle","tyl":"Journal article","doi":"10.7440/res58.2016.06","cx":"General","co":["Reliability-noise","Bias","Validity","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"diamond","cby":14,"self":false,"s":"A Spanish-language review article surveys debates in science evaluation around peer review, bibliometrics, and relevance criteria, and notes the limited attention this topic has received in Latin American scholarship.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"U6HNRM36","t":"Scientists’ perception of pilot study quality was influenced by statistical significance and study design","au":"Klinggraeff, Lauren von","y":2023,"vn":"Journal of Clinical Epidemiology","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.jclinepi.2023.05.011","cx":"Journal","co":["Bias"],"sg":["individual-scoring"],"x":1,"st":"experimental-rct","oast":"hybrid","cby":5,"self":false,"s":"Survey experiment varying obesity-prevention pilot study abstracts found statistically significant results and randomized designs were rated as higher quality regardless of actual study strength, while pilot study status itself had no effect.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"no-statement","cau":""},{"k":"UAGF8GCJ","t":"Grant reviewer perceptions of the quality and effectiveness of panel discussion","au":"Gallo, Stephen A.","y":2019,"vn":"bioRxiv (Cold Spring Harbor Laboratory)","ty":"journalArticle","tyl":"Journal article","doi":"10.1101/586685","cx":"Grant","co":["Reliability-noise","Bias"],"sg":["panel-consensus"],"x":0,"st":"observational-quantitative","oast":"green","cby":2,"self":false,"s":"A survey of grant reviewers found panel discussions were generally viewed positively for clarifying disagreements, though many reviewers noted uneven participation and inconsistent chair facilitation as recurring weaknesses.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://doi.org/10.6084/m9.figshare.8132453.v1","ca":"no-statement","cau":""},{"k":"UEURBNET","t":"The troubles with peer review for allocating research funding: Funders need to experiment with versions of peer review and decision-making.","au":"Bendiscioli, Sandra","y":2019,"vn":"EMBO Reports","ty":"journalArticle","tyl":"Journal article","doi":"10.15252/embr.201949472","cx":"Grant","co":["Bias","Validity","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"hybrid","cby":51,"self":false,"s":"A brief commentary noting recent criticism of grant peer review and highlighting that various alternative funding and decision-making approaches are being tested by funders.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"UF4JGDEA","t":"Peer review practices for evaluating biomedical research grants: A scientific statement from the american heart association.","au":"Liaw, Lucy","y":2017,"vn":"Circulation Research","ty":"journalArticle","tyl":"Journal article","doi":"10.1161/RES.0000000000000158","cx":"Grant","co":["Reliability-noise","Bias","Validity","Interventions"],"sg":["whole-process"],"x":0,"st":"guideline-policy","oast":"closed","cby":15,"self":false,"s":"A scientific statement from the American Heart Association reviews published evidence on biomedical grant peer-review practices, compares its own process with other funders, and recommends best practices for improving review at the association.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"UFFKJYI4","t":"Reliability of ratings of research papers.","au":"McReynolds, Paul","y":1971,"vn":"American Psychologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/h0037937","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":109,"self":false,"s":"An early study measuring how consistently members of an American Psychological Association conference programme committee rated the same submitted papers.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"UJX27BA2","t":"Peer-Review and Academic Archaeology: Quality, Epistemology and Science Policies","au":"Salazar, Diego","y":2019,"vn":"Archaeologies","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11759-019-09367-6","cx":"Journal","co":["Validity","Interventions"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"closed","cby":4,"self":false,"s":"Explores how epistemological criteria feature in archaeology journal peer review and argues for strengthening this dimension to better justify the discipline's claims about the past.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"UKMK3ZNF","t":"Evaluation of scientific research projects on the basis of evidential reasoning approach under the perspective of expert reliability","au":"Zhu, Weidong","y":2021,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-021-04201-9","cx":"Grant","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"closed","cby":11,"self":false,"s":"Proposes a data-driven \"evidential reasoning\" method for aggregating multiple experts' evaluations of research proposals that accounts for differences in individual reviewer reliability.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"UKWAWEDT","t":"Interreferee agreement on some characteristics of manuscripts submitted to the Journal of Personality and Social Psychology.","au":"Scott, William A.","y":1974,"vn":"American Psychologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/h0037631","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":167,"self":false,"s":"An older paper on manuscript submissions to a psychology journal discusses proposals for standardized rating forms to help discriminate among the many high-quality manuscripts competing for limited publication space.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"ULDI5EEF","t":"How to Fix Peer Review: Separating Its Two Functions--Improving Manuscripts and Judging Their Scientific Merit--Would Help.","au":"Kaplan, David","y":2005,"vn":"Journal of Child and Family Studies","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s10826-005-6845-3","cx":"Journal","co":["Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":19,"self":false,"s":"Proposes separating peer review's two functions, improving a manuscript and judging its scientific significance, into independent processes as a way to reduce bias and inefficiency.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"UPSXNUDQ","t":"The present and future of peer review: Ideas, interventions, and evidence","au":"Aczél, Balázs","y":2025,"vn":"Proceedings of the National Academy of Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1073/pnas.2401232121","cx":"General","co":["Reliability-noise","Bias","Validity","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"hybrid","cby":72,"self":false,"s":"A perspective piece surveys proposed reforms to the peer-review system and concludes that better, more comprehensive data on how review actually functions is needed before major changes can be evaluated.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"UQ4VV8EV","t":"Reviewer bias in single- versus double-blind peer review","au":"Tomkins, Andrew","y":2017,"vn":"Proceedings of the National Academy of Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1073/pnas.1707323114","cx":"Journal","co":["Bias"],"sg":["individual-scoring"],"x":1,"st":"experimental-rct","oast":"bronze","cby":608,"self":false,"s":"A controlled experiment at a major computer science conference found single-blind reviewers were significantly more likely than double-blind reviewers to recommend acceptance for papers from famous authors and prestigious institutions.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"no-statement","cau":""},{"k":"UQL4XDCH","t":"Interdisciplinarity can aid the spread of better methods between scientific communities","au":"Smaldino, Paul E.","y":2022,"vn":"Collective Intelligence","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/26339137221131816","cx":"General","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"diamond","cby":23,"self":false,"s":"A formal modelling paper examines why inferior research methods persist within some scientific disciplines and argues that input from other disciplines can help identify and displace such methods by counteracting reviewers' bias toward familiar approaches.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"open","cau":"https://www.comses.net/codebase-release/74086cfd-3398-44b3-bf9c-ada0d70e1d86/"},{"k":"USAZDF6Y","t":"Authors and reviewers who suffer from confirmatory bias","au":"García, J. A.","y":2016,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-016-2079-y","cx":"Journal","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"green","cby":16,"self":false,"s":"Presents a Bayesian model of how confirmatory bias can lead authors to overestimate the role of reviewer bias in their manuscript's rejection, plus an accompanying online tool that lets authors examine their own beliefs about why a paper was rejected.","pg":"no-statement","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"no-statement","cau":""},{"k":"USLLK4XA","t":"Double entropy inter-rater agreement indices.","au":"Olenko, Andriy","y":2016,"vn":"Applied Psychological Measurement","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/0146621615592718","cx":"Analogue","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"closed","cby":12,"self":false,"s":"Proposes a new entropy-based statistical index for measuring rater agreement when only a single target (such as one grant proposal) is being judged, comparing it against established agreement measures.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://sites.google.com/site/olenkoandriy/","ca":"open","cau":"https://sites.google.com/site/olenkoandriy/"},{"k":"UTBUWH8P","t":"Context-Aware Reviewer Assignment for Trust Enhanced Peer Review","au":"Li, Lei","y":2015,"vn":"PLoS ONE","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0130493","cx":"Journal","co":["Bias","Interventions"],"sg":["reviewer-selection"],"x":0,"st":"simulation-methodological","oast":"gold","cby":27,"self":false,"s":"Proposes a reviewer-assignment algorithm that accounts for expertise, conflicts of interest, and reviewer strictness, aiming to avoid grouping overly harsh or lenient reviewers on the same submission.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"","ca":"no-statement","cau":""},{"k":"UTCS2K7S","t":"Double-blind peer review of manuscripts: opportunities, challenges, and way forward","au":"Arumugam, Ashokan","y":2020,"vn":"Physical Therapy Reviews","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/10833196.2019.1698161","cx":"Journal","co":["Bias","Interventions"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"bronze","cby":13,"self":false,"s":"A commentary traces the history of double-blind peer review and discusses its potential advantages and practical challenges as a review model for scholarly journals.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"UUUT86RV","t":"Female-dominated disciplines have lower evaluated research quality and funding success rates, for men and women.","au":"James, Alex","y":2024,"vn":"eLife","ty":"journalArticle","tyl":"Journal article","doi":"10.7554/eLife.97613","cx":"General","co":["Bias"],"sg":["funding-decision"],"x":0,"st":"observational-quantitative","oast":"gold","cby":6,"self":false,"s":"Cross-country analysis finds research quality ratings and funding success are lower in disciplines with higher proportions of women, affecting both male and female researchers in those fields.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"no-statement","cau":""},{"k":"UX8S6N6I","t":"The future of societal impact assessment using peer review: pre-evaluation training, consensus building and inter-reviewer reliability","au":"Derrick, Gemma","y":2017,"vn":"Palgrave Communications","ty":"journalArticle","tyl":"Journal article","doi":"10.1057/palcomms.2017.40","cx":"General","co":["Reliability-noise","Interventions"],"sg":["individual-scoring","panel-consensus"],"x":0,"st":"qualitative","oast":"gold","cby":30,"self":false,"s":"Interviews with UK Research Excellence Framework evaluators, conducted before and after their assessments, examine whether structured pre-evaluation training helped panels reach more consistent judgments on the novel societal-impact criterion.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"no-statement","cau":""},{"k":"UXZF2E9W","t":"Bias, Lotteries, and Affirmative Action in Science Funding Policy","au":"Shaw, Jamie","y":2024,"vn":"The British Journal for the Philosophy of Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1086/730218","cx":"Grant","co":["Bias"],"sg":["funding-decision"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":3,"self":false,"s":"A philosophical paper argues that criticisms of lottery-based funding as biased mainly apply to one type of partial lottery, and that other lottery variants could better address bias while also supporting complementary affirmative-action funding schemes.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"UYD3UVJU","t":"Engaging doctoral students in peer review: a pre-post study evaluating the effectiveness of the “Peerspectives” course on review quality, knowledge and skills","au":"Rohmann, Jessica L.","y":2025,"vn":"medRxiv","ty":"journalArticle","tyl":"Journal article","doi":"10.1101/2025.02.11.25322060","cx":"Journal","co":["Interventions"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"green","cby":2,"self":false,"s":"A single-arm pre-post study of a semester-long peer-review training course for doctoral students finds participants' review quality, as judged by journal editors, and self-assessed reviewing skills both improved after completing the course.","pg":"preregistered","pgr":"OSF","pgid":"https://osf.io/vndcx","da":"open","dau":"https://osf.io/wyegc/","ca":"open","cau":"https://osf.io/wyegc/"},{"k":"UYGWPDNF","t":"The Reliability and Influence of Peer Reviews","au":"Chibnik, Michael","y":2016,"vn":"American Anthropologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/aman.12523","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"closed","cby":0,"self":false,"s":"Analyzing manuscripts submitted to a major anthropology journal, this study finds reviewer agreement is only modestly better than chance despite an initial impression of good agreement, alongside a fairly strong link between reviewer recommendations and editors' final decisions.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"V2ANRITC","t":"Open peer review: promoting transparency in open science","au":"Wolfram, Dietmar","y":2020,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-020-03488-4","cx":"Journal","co":["general-systemic"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":165,"self":false,"s":"Bibliometric study identifying over 600 journals that had adopted open peer review by 2019, describing growth patterns, disciplinary spread, and the varying ways publishers implement open identities and open reports.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://doi.org/10.5281/zenodo.3737197","ca":"no-statement","cau":""},{"k":"V5I4EX6D","t":"A Robust Community-Based Credit System to Enhance Peer Review in Scientific Research","au":"Tan, Wanpeng","y":2023,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31222/osf.io/y9qh6","cx":"General","co":["Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":1,"self":false,"s":"Proposes a community-based credit and reputation system as an alternative structure for organizing peer review, academic recognition, and progression within the scientific community.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"V5LUUSHS","t":"When zero may not be zero: A cautionary note on the use of inter‐rater reliability in evaluating grant peer review","au":"Erosheva, Elena A.","y":2021,"vn":"Journal of the Royal Statistical Society: Series A (Statistics in Society)","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/rssa.12681","cx":"Grant","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":30,"self":false,"s":"A methodological paper uses NIH and biological-sciences grant review data to show that inter-rater reliability estimates can misleadingly equal zero when calculated from a narrow range of top-quality proposals, even though full-range agreement is good.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://doi.org/10.6084/m9.figshare.12728087.v1","ca":"open","cau":"https://osf.io/knph8/"},{"k":"V6JGCBSK","t":"What do we know about grant peer review in the health sciences?","au":"Guthrie, Susan","y":2017,"vn":"F1000Research","ty":"journalArticle","tyl":"Journal article","doi":"10.12688/f1000research.11917.1","cx":"Grant","co":["Reliability-noise","Bias","Validity"],"sg":["whole-process"],"x":0,"st":"systematic-review-meta-analysis","oast":"gold","cby":68,"self":false,"s":"Review of 105 papers on grant peer review in health sciences finds evidence of bias against innovative research, weak prediction of future performance, considerable inter-reviewer variation, and disproportionate burden falling on applicants.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"","ca":"no-statement","cau":""},{"k":"V6S3FITK","t":"Criteria for assessing grant applications: a systematic review","au":"Hug, Sven E.","y":2020,"vn":"Palgrave Communications","ty":"journalArticle","tyl":"Journal article","doi":"10.1057/s41599-020-0412-9","cx":"Grant","co":["general-systemic"],"sg":["individual-scoring"],"x":0,"st":"systematic-review-meta-analysis","oast":"gold","cby":47,"self":false,"s":"A systematic review of twelve empirical studies identifies fifteen distinct criteria and thirty evaluated aspects that reviewers use to judge grant applications, organized into a framework covering project aims, methods, feasibility, and applicant traits.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"","ca":"no-statement","cau":""},{"k":"V73EMDT7","t":"Peer Review and Bibliometric: Potentials and Problems","au":"Bornmann, Lutz","y":2011,"vn":"","ty":"book","tyl":"Book","doi":"10.1007/978-94-007-1116-7_8","cx":"General","co":["Reliability-noise","Bias","Validity"],"sg":["whole-process"],"x":0,"st":"book-monograph","oast":"closed","cby":16,"self":false,"s":"A book chapter reviewing how peer review and bibliometric indicators are used together in research evaluation and university rankings, discussing the strengths and limitations of each approach.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"V74GUKBP","t":"Science Evaluation: Peer Review, Bibliometrics, and Research Impact Assessment","au":"Redman, Barbara K.","y":2023,"vn":"","ty":"book","tyl":"Book","doi":"10.1007/978-3-031-27111-3_8","cx":"General","co":["Reliability-noise","Bias","Validity","Interventions"],"sg":["whole-process"],"x":0,"st":"book-monograph","oast":"closed","cby":2,"self":false,"s":"A book chapter surveys peer review, bibliometrics and research-impact assessment methods, discussing how machine-learning models of paper content may predict real-world impact better than citation counts and highlighting problems such as continued citation of retracted work.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"V75ZGITL","t":"Computing Inter-Rater Reliability for Observational Data: An Overview and Tutorial.","au":"Hallgren, Kevin A","y":2012,"vn":"Tutorials in quantitative methods for psychology","ty":"journalArticle","tyl":"Journal article","doi":"10.20982/tqmp.08.1.p023","cx":"Analogue","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"diamond","cby":3872,"self":false,"s":"A methodological tutorial covering study design, statistic selection, and computation (including software syntax) for correctly assessing inter-rater reliability of observational data.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"V7CCKM45","t":"Ranking versus rating in peer review of research grant applications","au":"Tamblyn, Robyn","y":2023,"vn":"PLOS ONE","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0292306","cx":"Grant","co":["Reliability-noise","Bias"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"gold","cby":5,"self":false,"s":"Compares rating versus ranking approaches to scoring thousands of Canadian health-research grant applications, finding ranking more reliable and less affected by reviewer characteristics, though both approaches disadvantaged early-career applicants.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"no-statement","cau":""},{"k":"V8UFU356","t":"Meetings that matter: the dual benefits of panel peer review","au":"Oxley, Kristin","y":2024,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvaf047","cx":"Grant","co":["Interventions"],"sg":["panel-consensus"],"x":0,"st":"qualitative","oast":"hybrid","cby":1,"self":false,"s":"Analysis exploring how grant review panels add value beyond simply averaging individual reviewers' scores, suggesting panel discussion promotes more systematic evaluation but that the benefit depends on the review task's characteristics.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"","ca":"not-applicable","cau":""},{"k":"VAJCBBM4","t":"Distortions from Unbiased Noise in Sequential Review: Why the Research Excellence Framework Needs a Noise Audit","au":"Ward, Robert","y":2026,"vn":"Quarterly Journal of Experimental Psychology","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/17470218261446939","cx":"General","co":["Reliability-noise","Bias"],"sg":["individual-scoring","panel-consensus"],"x":0,"st":"simulation-methodological","oast":"bronze","cby":0,"self":false,"s":"Simulations of the UK Research Excellence Framework's multi-stage review process show that ordinary judgment noise, especially at the panel stage, combined with a bounded rating scale, systematically causes top-performing departments to be underrated.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://osf.io/fwbqr/","ca":"open","cau":"https://osf.io/fwbqr/"},{"k":"VDGEQNPW","t":"Scientific autonomy in the structural bubble: from institutional bias to AI-mediated consensus","au":"Rubenstein, Elias","y":2026,"vn":"Frontiers in Research Metrics and Analytics","ty":"journalArticle","tyl":"Journal article","doi":"10.3389/frma.2026.1766504","cx":"General","co":["Bias"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":0,"self":false,"s":"Theoretical paper models how institutional funding pressures, publication and peer review systems, consensus platforms, and AI-mediated information filtering interact to constrain which scientific questions and conclusions gain visibility.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"VDYRFLE8","t":"Ten considerations for open peer review","au":"Schmidt, Birgit","y":2018,"vn":"F1000Research","ty":"journalArticle","tyl":"Journal article","doi":"10.12688/f1000research.15334.1","cx":"Journal","co":["general-systemic"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":80,"self":false,"s":"Drawing on discussions with authors, reviewers, editors, and publishers, the paper proposes ten practical considerations for implementing open peer review effectively.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"VFV5RJU5","t":"Are peer reviewers influenced by their work being cited?","au":"Barnett, Adrian","y":2025,"vn":"eLife","ty":"journalArticle","tyl":"Journal article","doi":"10.7554/elife.108748.2","cx":"Journal","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":0,"self":false,"s":"Analyzing over 37,000 open peer reviews across four journals, this study found reviewers were more likely to recommend approval of articles that cited their own work, and less likely when they explicitly requested such citations.","pg":"preregistered","pgr":"AsPredicted","pgid":"https://aspredicted.org/rn8vg","da":"open","dau":"https://github.com/agbarnett/cited_reviewers","ca":"open","cau":"https://github.com/agbarnett/cited_reviewers"},{"k":"GETDEYXE","t":"Scholars as government-appointed research evaluators: Do they create congruence between their professional quality standards and political demands?","au":"Woiwode, Hendrik","y":2020,"vn":"PLoS ONE","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0239336","cx":"General","co":["general-systemic"],"sg":["individual-scoring"],"x":0,"st":"qualitative","oast":"gold","cby":0,"self":false,"s":"A small-sample study compares the research-quality standards scholars use when evaluating their own work with those they apply as government-appointed evaluators, finding a notable inconsistency between the two sets of criteria.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"","ca":"no-statement","cau":""},{"k":"45K94TWP","t":"Applying psychometric theory and research to developing a continuously distributed approach to making research funding decisions.","au":"De Los Reyes, Andres","y":2012,"vn":"Review of General Psychology","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/a0027250","cx":"Grant","co":["Reliability-noise","Validity","Interventions"],"sg":["funding-decision"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":0,"self":false,"s":"Applies psychometric theory to propose a continuously distributed funding allocation model as an alternative to dichotomous fund/no-fund decisions, illustrating with a hypothetical example that it could extend support to substantially more proposals using the same budget.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"49ZXXL9X","t":"Latent Markov modeling applied to grant peer review","au":"Bornmann, Lutz","y":2008,"vn":"Journal of Informetrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.joi.2008.05.003","cx":"Grant","co":["Reliability-noise"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"closed","cby":18,"self":false,"s":"Latent Markov model analysis of a three-stage fellowship review process finding that a positive first-stage external review was essentially a precondition for eventual funding, supporting the value of early-stage screening.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"5CTCSBI3","t":"Gatekeepers of science—Effects of external reviewers’ attributes on the assessments of fellowship applications","au":"Bornmann, Lutz","y":2006,"vn":"Journal of Informetrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.joi.2006.09.005","cx":"Grant","co":["Bias","Validity"],"sg":["reviewer-selection","individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":44,"self":false,"s":"Study of a European fellowship program's peer review finding that reviewers' own experience, country, and gender had no significant effect on their ratings, which instead tracked applicants' scientific achievement.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"6LMXE72N","t":"Measuring Agreement for Multinomial Data","au":"Davies, Mark","y":1982,"vn":"Biometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.2307/2529886","cx":"Analogue","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"closed","cby":353,"self":false,"s":"A statistical methods paper proposes a kappa-like coefficient for measuring agreement among multiple categorical raters arranged in a two-way table, deriving it from both chance-corrected agreement and analysis-of-variance approaches.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"on-request","cau":""},{"k":"836GCTFJ","t":"Interrater agreement for reviews for Developmental Review","au":"Whitehurst, Grover J.","y":1983,"vn":"Developmental Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/0273-2297(83)90009-6","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":47,"self":false,"s":"Analysis of reviewer agreement for one developmental psychology journal reports higher reliability than earlier studies of other journals, and argues a particular statistical method better captures cross-journal agreement differences than the intraclass correlation.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"9GSQFNSJ","t":"Extension of the Kappa Coefficient","au":"Kraemer, Helena C.","y":1980,"vn":"Biometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.2307/2529972","cx":"Analogue","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"closed","cby":237,"self":false,"s":"Proposes a statistical extension of the kappa coefficient that allows it to handle multiple observations per subject and more than two response categories, illustrated using previously published data.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"ABE5N6J3","t":"Agreement among reviewers of review articles","au":"Oxman, Andrew D","y":1991,"vn":"Journal of Clinical Epidemiology","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/0895-4356(91)90205-n","cx":"General","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":286,"self":false,"s":"A study of nine judges rating the quality of 36 review articles found generally consistent agreement across judges of varying research expertise on most quality criteria, based on intraclass correlation coefficients.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"no-statement","cau":""},{"k":"ADI4QBWF","t":"A proposed index for measuring agreement in test-retest studies","au":"Rogot, Eugene","y":1966,"vn":"Journal of Chronic Diseases","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/0021-9681(66)90032-4","cx":"Analogue","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"closed","cby":113,"self":false,"s":"Proposes a statistical index for quantifying agreement between two classification methods in test-retest reliability studies where neither method serves as an established standard, illustrated with examples from health survey research.","pg":"no-statement","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"CLMQHE39","t":"Editorial Peer Review: Its Strengths and Weaknesses","au":"Hepfer, Cindy","y":2002,"vn":"Serials Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/s0098-7913(02)00181-8","cx":"Journal","co":["Reliability-noise","Bias","Validity"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":82,"self":false,"s":"A brief review discussing a book examining the strengths and weaknesses of editorial peer review, part of a set of short library-science book reviews.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"EAXQ23NK","t":"A proposal for an 'equal peer-review' statement.","au":"Moustafa, Khaled","y":2015,"vn":"Trends in pharmacological sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.tips.2015.06.001","cx":"Journal","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"green","cby":8,"self":false,"s":"Short proposal for an \"equal peer-review\" statement that would preserve author anonymity throughout review, intended to remove bias linked to author or institutional prestige.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"G4TTQ7ZR","t":"High agreement but low Kappa: I. the problems of two paradoxes","au":"Feinstein, Alvan R.","y":1990,"vn":"Journal of Clinical Epidemiology","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/0895-4356(90)90158-l","cx":"Analogue","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"closed","cby":2903,"self":false,"s":"Methodological paper describing two statistical paradoxes in which high observed agreement between raters can coincide with a low kappa statistic due to imbalanced marginal totals.","pg":"no-statement","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"JN8MZ48Y","t":"Peer Review Interrater Concordance of Scientific Abstracts: A Study of Anesthesiology Subspecialty and Component Societies","au":"Cohen, Ira Todd","y":2006,"vn":"Anesthesia & Analgesia","ty":"journalArticle","tyl":"Journal article","doi":"10.1213/01.ane.0000200314.73035.4d","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":6,"self":false,"s":"A study of abstract selection at several anesthesiology subspecialty meetings finds reviewer agreement ranging from poor to moderate, only slightly better than chance, and suggests clearer criteria and interval scoring scales could improve consistency.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"JZNXVKWD","t":"Reviewer bias against the unconventional? A randomized double-blind study of peer review.","au":"Ernst, E","y":1999,"vn":"Complementary therapies in medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/S0965-2299(99)80054-5","cx":"Journal","co":["Reliability-noise","Bias"],"sg":["individual-scoring"],"x":1,"st":"experimental-rct","oast":"closed","cby":17,"self":false,"s":"Randomized double-blind trial testing whether reviewers are biased against an unconventional treatment found no such bias, but did find low inter-rater agreement, raising separate concerns about review validity.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"K52L9EZ2","t":"Research funding: past performance is a stronger predictor of future scientific output than reviewer scores","au":"Győrffy, Balázs","y":2020,"vn":"Journal of Informetrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.joi.2020.101050","cx":"Grant","co":["Validity"],"sg":["individual-scoring","post-award-validity"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":70,"self":false,"s":"Large-scale analysis of Hungarian science-fund applications found a researcher's past publication record predicted future scientific output far more strongly than the scores their proposal received from reviewers.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"KCX9AZ3T","t":"Implicit criteria weighting and evaluators' backgrounds in grant peer review","au":"Franzoni, Chiara","y":2026,"vn":"Research Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.respol.2026.105535","cx":"Grant","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":0,"self":false,"s":"Analyzing over 2,000 review reports for Swedish research grant proposals, this study found reviewers implicitly weighted research quality roughly twice as heavily as societal relevance, with applicant qualifications playing a comparatively minor role.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"no-statement","cau":""},{"k":"KULA287G","t":"Editorial Comment: Agreement between reviewers of Journal of Economic Psychology submissions","au":"Kemp, Simon","y":2005,"vn":"Journal of Economic Psychology","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.joep.2005.05.004","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":6,"self":false,"s":"Reports moderate correlations (roughly 0.5 to 0.6) between reviewers rating manuscript quality and publishability at a psychology journal over several years, comparable to reliability seen in other journals but not high enough to trust individual paper judgments confidently.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"KZDA7KMW","t":"Measuring Agreement between Two Judges on the Presence or Absence of a Trait","au":"Fleiss, Joseph L.","y":1975,"vn":"Biometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.2307/2529549","cx":"Analogue","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"closed","cby":506,"self":false,"s":"A foundational statistical paper presenting and critically comparing several proposed indexes for measuring two-judge agreement on a binary categorical trait, emphasizing the need to correct for chance agreement.","pg":"no-statement","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"LJ3DDLY7","t":"A new approach to referees' assessments of manuscripts","au":"Hargens, Lowell L.","y":1990,"vn":"Social Science Research","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/0049-089x(90)90012-8","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":66,"self":false,"s":"Statistical analysis of referee recommendations across five journals testing whether a single underlying \"publishability\" dimension explains reviewer judgments, finding rejection recommendations more reliable than more favorable ones.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"MMQAABEQ","t":"Artificial intelligence in scholarly peer review: a scoping review of applications, risks, and governance challenges","au":"Nabavi, Ali","y":2026,"vn":"International Journal of Medical Informatics","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.ijmedinf.2026.106418","cx":"Journal","co":["Reliability-noise","Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"systematic-review-meta-analysis","oast":"closed","cby":2,"self":false,"s":"Scoping review of 189 studies on artificial intelligence in scholarly peer review, mapping its current uses in triage and review assistance alongside risks such as confidentiality breaches and bias toward elite institutions.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"no-statement","cau":""},{"k":"P2AKBBFQ","t":"Peer review: and the winner is …","au":"Greenberg, Daniel S.","y":1999,"vn":"The Lancet","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/s0140-6736(05)76844-6","cx":"Grant","co":["Validity"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":5,"self":false,"s":"An interview-style profile piece touches on calls for evidence-based evaluation of competitive grant peer review, framed around a researcher's personal reflections and career anecdotes.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"QUNM7567","t":"Peer review in biomedical publication","au":"Ingelfinger, Franz J.","y":1974,"vn":"The American Journal of Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/0002-9343(74)90635-4","cx":"Journal","co":["Reliability-noise","Bias","Validity","Interventions"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"closed","cby":243,"self":false,"s":"Discusses persistent problems in biomedical journal peer review, including low reviewer agreement approaching chance levels in one large journal, and proposes several possible reforms to reviewer training and incentives.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"TPJDUPCA","t":"Evaluating Grant Peer Review in the Health Sciences: A review of the literature","au":"Ismail, Sharif","y":2009,"vn":"RAND Corporation eBooks","ty":"report","tyl":"Report","doi":"10.7249/tr742","cx":"Grant","co":["Reliability-noise","Bias","Validity","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":19,"self":false,"s":"A literature review commissioned by a health department evaluates common criticisms of grant peer review in the health sciences and discusses modest procedural changes that might address some of the concerns raised.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"TT68QNR5","t":"John Snow’s Grant Application","au":"Rothman, Kenneth J.","y":2016,"vn":"Epidemiology","ty":"journalArticle","tyl":"Journal article","doi":"10.1097/ede.0000000000000453","cx":"Grant","co":["Bias","Validity"],"sg":["proposal-quality"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":6,"self":false,"s":"A satirical piece imagining John Snow's historic 1853 cholera investigation proposal being evaluated through a modern grant peer-review summary statement format.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"TUSX26VN","t":"High agreement but low kappa: II. Resolving the paradoxes","au":"Cicchetti, Domenic V.","y":1990,"vn":"Journal of Clinical Epidemiology","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/0895-4356(90)90159-m","cx":"Analogue","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"closed","cby":1840,"self":false,"s":"A statistical paper addresses known paradoxes in the kappa coefficient for rater agreement, proposing that separate positive- and negative-agreement indices, alongside kappa, give a fuller picture of observer variability.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"5PN6GE4C","t":"Relationship between peer review quality and scientific impact: Insights from LLMs-assessed reviews","au":"Sun, Zhuanlan","y":2026,"vn":"Journal of Informetrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.joi.2026.101801","cx":"Journal","co":["Reliability-noise","Validity"],"sg":["individual-scoring","post-award-validity"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":0,"self":false,"s":"Using Google's Gemini model to automatically score peer review quality, this study finds a positive association between higher review scores and a paper's subsequent citation count, with strong agreement between human and AI-assessed review scores.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"on-request","cau":""},{"k":"667Z2XRJ","t":"The predictive validity of peer review: A selective review of the judgmental forecasting qualities of peers, and implications for innovation in science","au":"Benda, Wim G.G.","y":2010,"vn":"International Journal of Forecasting","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.ijforecast.2010.03.003","cx":"General","co":["Reliability-noise","Bias","Validity"],"sg":["individual-scoring","post-award-validity"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":58,"self":false,"s":"Reviews evidence on how accurately peer review predicts a paper's or proposal's eventual impact, concluding it performs reasonably well on average but shows tension with recognizing genuinely innovative work.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"8DLAPIKQ","t":"Threats to objectivity in peer review: the case of gender","au":"Kaatz, Anna","y":2014,"vn":"Trends in Pharmacological Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.tips.2014.06.005","cx":"General","co":["Bias","Interventions"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":139,"self":false,"s":"Discusses how implicit cognitive biases related to gender can affect scientific peer review of grants and manuscripts, drawing on broader psychological research on gender stereotypes in science.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"A2LG7MLV","t":"How to detect indications of potential sources of bias in peer review: A generalized latent variable modeling approach exemplified by a gender study","au":"Bornmann, Lutz","y":2008,"vn":"Journal of Informetrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.joi.2008.09.003","cx":"Grant","co":["Bias"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"simulation-methodological","oast":"closed","cby":25,"self":false,"s":"A methodological paper presents a latent variable modeling approach that funding organizations can use to detect potential bias, such as gender bias, in approval and rejection patterns across applicant groups in peer review.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"open","cau":"http://www.gllamm.org/books/gum.html"},{"k":"BIYZNW7P","t":"Unpacking the predictive power of peer review under scientific disruption: Evidence from AI research","au":"Zhang, Yujie","y":2026,"vn":"Journal of Informetrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.joi.2026.101811","cx":"Journal","co":["Validity"],"sg":["individual-scoring","post-award-validity"],"x":0,"st":"observational-quantitative","oast":"closed","cby":0,"self":false,"s":"Analysis of a large longitudinal dataset of AI research papers and reviews from 2017-2024 finds peer review still predicts a paper's later influence and disruption, but both predictive power and reviewer confidence have declined over time.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"CN7CIUKK","t":"The big consequences of small biases: A simulation of peer review","au":"Day, Theodore E.","y":2015,"vn":"Research Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.respol.2015.01.006","cx":"Grant","co":["Bias"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"simulation-methodological","oast":"closed","cby":40,"self":false,"s":"A simulation study found that even small, consistent scoring biases against a disadvantaged group of investigators become statistically detectable in funding rates once a large pool of grant applications is reviewed.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"CQTBQ2KA","t":"Selection committee membership: Service or self-service","au":"Besselaar, Peter van den","y":2012,"vn":"Journal of Informetrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.joi.2012.05.003","cx":"Grant","co":["Bias"],"sg":["reviewer-selection"],"x":0,"st":"observational-quantitative","oast":"closed","cby":17,"self":false,"s":"This study found that members of grant selection committees were, on average, more successful in later winning their own grant funding than comparable non-member researchers, beyond what their research performance would predict.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"DLJNLM4I","t":"The fading of status bias during the open peer review process","au":"Sun, Zhuanlan","y":2024,"vn":"Journal of Informetrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.joi.2024.101528","cx":"Journal","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":9,"self":false,"s":"Analysing thousands of open peer-review reports at Nature Communications, the study finds no evidence that reviewers treated papers differently based on exposure to prestigious versus lesser-known authors' identities during open review.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://osf.io/5xc9v/","ca":"no-statement","cau":""},{"k":"EY4N66YR","t":"Large language models for peer review in biotechnology.","au":"Li, Wenyu","y":2026,"vn":"New biotechnology","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.nbt.2026.03.007","cx":"General","co":["Reliability-noise","Bias","Validity"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":1,"self":false,"s":"Tests several large language models as peer reviewers on preprints and grant proposals in biotechnology, finding AI reviewers produced substantive but more lenient feedback than humans and that AI-generated review text was difficult to reliably detect.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"open","cau":"https://github.com/wenyuli23/LLM_peer_review"},{"k":"FNNGDXAV","t":"Generative artificial intelligence for project evaluation: comparing large language models and human experts","au":"Santos, José M.R.C.A.","y":2026,"vn":"Procedia Computer Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.procs.2026.03.136","cx":"Grant","co":["Reliability-noise","Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"diamond","cby":0,"self":false,"s":"Compares scores given by generative AI models and human experts reviewing research proposals, finding AI-generated scores and review quality were broadly comparable to human averages, though human oversight remained necessary.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"KNAEUISZ","t":"Procedural problems with peer review, impact factor and citation index: A response to Borja and Elliott (2024)","au":"Bianchi, Carlo Nike","y":2025,"vn":"Estuarine Coastal and Shelf Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.ecss.2025.109184","cx":"Journal","co":["Bias"],"sg":["reviewer-selection","whole-process"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":1,"self":false,"s":"Commentary responding to a prior paper on peer review challenges, adding further discussion of problems with reviewer availability, journal impact factors, and citation indices in evaluating scientific quality.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"PSNKTKAU","t":"The “invisible hand” of peer review: The implications of author-referee networks on peer review in a scholarly journal","au":"Dondio, Pierpaolo","y":2019,"vn":"Journal of Informetrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.joi.2019.03.018","cx":"Journal","co":["Bias"],"sg":["individual-scoring","reviewer-selection"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":49,"self":false,"s":"A network analysis of 2,232 author-referee pairs at an interdisciplinary journal finds referees gave more favorable recommendations to authors closer to them in prior collaboration networks, and that peer review itself appeared to accelerate subsequent co-authorship ties.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"U5A9M4EQ","t":"Development and Implementation of a Peer Review Training Program","au":"Thigpen, Jonathan","y":2025,"vn":"American Journal of Pharmaceutical Education","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.ajpe.2025.101884","cx":"Journal","co":["Interventions"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":2,"self":false,"s":"Describes design and early results of a mentorship-based peer review training program at one pharmacy education journal, reporting strong and sustained applicant interest across its first cohorts.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"U8R56WTD","t":"The effectiveness of peer review in identifying issues leading to retractions","au":"Zheng, Xiang","y":2023,"vn":"Journal of Informetrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.joi.2023.101423","cx":"Journal","co":["Validity"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":16,"self":false,"s":"Analyzing peer-review comments on later-retracted publications, the study found reviewers rarely recommended rejection outright, and were better at catching problems with data and methods than with plagiarism or citation issues.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"29KVHR3C","t":"A model of the editorial process in academic journals","au":"Bayar, Onur","y":2021,"vn":"Research Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.respol.2021.104339","cx":"Journal","co":["Bias"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"simulation-methodological","oast":"closed","cby":14,"self":false,"s":"A theoretical model of the academic journal editorial process examines how editors' decisions to desk-reject, desk-accept, or send papers to referees affect the balance between accepting high-quality and rejecting low-quality work.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"JU6II4Y3","t":"Journal review and gender equality: a critical comment on Budden et al.","au":"Whittaker, Robert J.","y":2008,"vn":"Trends in Ecology & Evolution","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.tree.2008.06.003","cx":"Journal","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":36,"self":false,"s":"Content mismatch: the retrieved text discusses insect-plant coevolution genetics rather than the stated topic of journal review and gender equality, so only the title indicates the paper is a critical commentary on prior gender-bias research in review.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"KFD5UHRM","t":"Comparative outcomes of face-to-face and virtual review meetings","au":"Vo, Nghia M.","y":2016,"vn":"International Journal of Surgery Open","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.ijso.2016.07.002","cx":"Grant","co":["Reliability-noise"],"sg":["panel-consensus"],"x":0,"st":"observational-quantitative","oast":"gold","cby":5,"self":false,"s":"Compares scoring consistency between face-to-face and virtual grant review panel meetings, finding minimal differences in how much discussion changed preliminary scores between the two formats.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"DK7Z8PE6","t":"Who Benefits from Peer Review? An Analysis of the Outcome of 100 Requests for Review by Plastic and Reconstructive Surgery","au":"Loonen, Martijn P. J.","y":2005,"vn":"Plastic & Reconstructive Surgery","ty":"journalArticle","tyl":"Journal article","doi":"10.1097/01.prs.0000178796.82273.7c","cx":"Journal","co":["Reliability-noise","Bias"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"closed","cby":27,"self":false,"s":"Bibliometric analysis of 100 review requests at a plastic surgery journal examining benefits to editors, authors, and readers, and testing for publication delay or bias against non-Anglo-American submissions.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"F2Z5HBK3","t":"Empowering peer reviewers with a checklist to improve transparency","au":"Parker, Timothy","y":2018,"vn":"Nature Ecology & Evolution","ty":"journalArticle","tyl":"Journal article","doi":"10.1038/s41559-018-0545-z","cx":"Journal","co":["Bias","Interventions"],"sg":["individual-scoring"],"x":0,"st":"guideline-policy","oast":"closed","cby":49,"self":false,"s":"Proposes a checklist tool to help peer reviewers more systematically assess study design soundness and methods reporting when evaluating manuscripts.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"II5SPEMP","t":"Chance and consensus in peer review.","au":"Cole, S","y":1981,"vn":"Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1126/science.7302566","cx":"Grant","co":["Reliability-noise","Bias"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"closed","cby":624,"self":false,"s":"A classic experiment reviewing 150 US National Science Foundation proposals with a fresh set of reviewers finds substantial disagreement among eligible reviewers, indicating funding outcomes depend considerably on which reviewers happen to be assigned, without evidence of systematic selection bias.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"L47VGY36","t":"Advancing nursing education science: an analysis of the NLN's Grants Program 2008-2010.","au":"Duffy, Joanne R","y":2011,"vn":"Nursing education perspectives","ty":"journalArticle","tyl":"Journal article","doi":"10.5480/1536-5026-32.1.10","cx":"Grant","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":4,"self":false,"s":"Descriptive analysis of a US nursing education grant program's proposals from 2008-2010 reports high inter-rater reliability among reviewers and improving scientific rigor over the period studied.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"9ARNWGFR","t":"Halo Effect in Peer Review: Exploring the Possibility of Bias Associated with the Feeling of Belonging to a Group","au":"Sordi, José Osvaldo De","y":2019,"vn":"Perspectivas em Ciência da Informação","ty":"journalArticle","tyl":"Journal article","doi":"10.1590/1981-5344/3860","cx":"Journal","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"qualitative","oast":"diamond","cby":2,"self":false,"s":"A case study proposes that reviewers sharing a professional or disciplinary in-group with authors may rate manuscripts more favorably due to a halo effect, based on content analysis of one journal's review context.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"38BW3IRN","t":"What does better peer review look like? Definitions, essential areas, and recommendations for better practice","au":"Allen, Heidi","y":2018,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31219/osf.io/4mfk2","cx":"Journal","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"guideline-policy","oast":"gold","cby":4,"self":false,"s":"Drawing on 40 case studies from Wiley journals, this paper defines standards for good peer review across integrity, ethics, fairness, usefulness, and timeliness, synthesizing them into a practical checklist for journal teams.","pg":"not-applicable","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"4EXX72HE","t":"A Multilevel Modelling Approach to Investigating the Predictive Validity of Editorial Decisions: Do the Editors of a High Profile Journal Select Manuscripts that are Highly Cited After Publication?","au":"Bornmann, Lutz","y":2011,"vn":"Journal of the Royal Statistical Society Series A (Statistics in Society)","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/j.1467-985x.2011.00689.x","cx":"Journal","co":["Validity"],"sg":["funding-decision","post-award-validity"],"x":0,"st":"observational-quantitative","oast":"bronze","cby":89,"self":false,"s":"Using multilevel logistic regression on manuscripts submitted to a chemistry journal, the study proposes a new method for testing whether editors' accept/reject decisions validly predict which papers go on to be highly cited.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"open","cau":""},{"k":"5QKAGHIT","t":"Experimenting with Grant Peer Review: A Mixed Methods Case Study of the Effects on Time Use and the Quality of Reviewing","au":"Besselaar, Peter van den","y":2026,"vn":"Publications","ty":"journalArticle","tyl":"Journal article","doi":"10.3390/publications14020033","cx":"Grant","co":["Validity","Interventions"],"sg":["panel-consensus","individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":0,"self":false,"s":"A mixed-methods case study testing a new \"Peer Circle\" team-based grant review format against conventional review, finding it saved reviewer time and expanded the reviewer pool while maintaining perceived review quality.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"no-statement","cau":""},{"k":"648JUZU3","t":"Moving to the Double-Blind Review System","au":"Boeck, Paul De","y":2015,"vn":"Journal of Intelligence","ty":"journalArticle","tyl":"Journal article","doi":"10.3390/jintelligence3040158","cx":"Journal","co":["Bias"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":0,"self":false,"s":"Editorial discussing a journal's consideration of switching from single-blind to double-blind peer review and weighing the tradeoffs between the two systems.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"6NFKSA2S","t":"Peer review in megajournals compared with traditional scholarly journals: Does it make a difference?","au":"Björk, Bo‐Christer","y":2016,"vn":"Learned Publishing","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/leap.1007","cx":"Journal","co":["Validity"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":60,"self":false,"s":"Pilot study comparing citation distributions of megajournal articles with traditional peer-reviewed journals, finding traditional journals had fewer uncited papers but a larger share of minimally cited papers than expected.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"85KL54GC","t":"Judging research papers for research excellence","au":"Tymms, Peter","y":2017,"vn":"Studies in Higher Education","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/03075079.2016.1266609","cx":"General","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"green","cby":9,"self":false,"s":"A Rasch model analysis of 710 internal ratings from 23 senior staff assessing 42 academics' outputs, mimicking the UK's Research Excellence Framework, finds most raters reliable but with meaningful variation in severity or leniency across raters.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"8JJIF6SZ","t":"Where next for partial randomisation of research funding? The feasibility of RCTs and alternatives","au":"Stafford, Tom","y":2024,"vn":"Wellcome Open Research","ty":"journalArticle","tyl":"Journal article","doi":"10.12688/wellcomeopenres.19565.2","cx":"Grant","co":["Validity","Interventions"],"sg":["funding-decision"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":0,"self":false,"s":"Discusses methodological considerations for studying partial randomisation in research funding, arguing that sample size and statistical power often make randomized controlled trials less practical than they initially appear, and reviews alternative study designs.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"9CDDJRXJ","t":"The association of gender, experience, and academic rank in peer-reviewed manuscript evaluation","au":"García, J. A.","y":2024,"vn":"Accountability in Research","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/08989621.2024.2440098","cx":"Journal","co":["Reliability-noise","Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"green","cby":5,"self":false,"s":"A survey of 573 reviewers tested whether gender, experience, or academic rank explained inconsistent manuscript evaluations, finding little difference except that lower-ranked reviewers relied more heavily on norms carried over from past reviewing experience.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://github.com/rosadecsai/survey2023.git","ca":"no-statement","cau":"https://github.com/rosadecsai/survey2023.git"},{"k":"A8TZ4TZK","t":"A Psychometric Analysis and Primer for Decision Making in Public Third-Party Funding Allocation","au":"Rahal, Rima-Maria","y":2025,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31234/osf.io/mp8y2_v3","cx":"Grant","co":["Bias","Validity"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":0,"self":false,"s":"This primer applies psychometric concepts to treat research-funding selection as a diagnostic procedure, highlighting gaps in how well allocation processes are shown to identify societally beneficial research and calling for greater transparency.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"open","cau":"https://osf.io/gdqyu/overview?view_only=f75952982ab2473bbd2e2975ebbbf656"},{"k":"BGW9Q5PA","t":"Applicants as reviewers: A mixed methods evaluation of the risks, benefits, and potential of distributed peer review for grant funding allocations","au":"Butters, Anna","y":2026,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31222/osf.io/t2p56_v2","cx":"Grant","co":["Reliability-noise","Interventions"],"sg":["reviewer-selection","panel-consensus","funding-decision"],"x":0,"st":"observational-quantitative","oast":"gold","cby":1,"self":false,"s":"A mixed-methods evaluation of a foundation's trial of \"distributed peer review,\" where grant applicants reviewed each other's proposals, finds moderate overlap with a parallel panel's funding choices and substantial variability between individual reviewers.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://osf.io/cws8b/","ca":"open","cau":"https://osf.io/cws8b/"},{"k":"C5VVQXIS","t":"Where are the carrots? A proposal to start crediting the peer reviewers for their contribution to science","au":"Laxdal, Aron Gauti","y":2023,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31219/osf.io/7c5xr","cx":"Journal","co":["Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":4,"self":false,"s":"A proposal to formally credit peer reviewers' contributions, such as counting reviews toward tenure criteria, to improve incentives for participating in scholarly review.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"CSH8NIEK","t":"Little race or gender bias in an experiment of initial review of NIH R01 grant proposals.","au":"Forscher, Patrick S","y":2019,"vn":"Nature Human Behaviour","ty":"journalArticle","tyl":"Journal article","doi":"10.1038/s41562-018-0517-y","cx":"Grant","co":["Bias"],"sg":["individual-scoring"],"x":1,"st":"experimental-rct","oast":"green","cby":44,"self":false,"s":"Experimental study modifying principal investigator names on real NIH R01 proposals to test for race and gender bias in initial review, finding little to no detectable bias across a large sample of reviewer evaluations.","pg":"preregistered","pgr":"OSF","pgid":"https://osf.io/vhwnd/","da":"open","dau":"https://osf.io/uy7vq/","ca":"no-statement","cau":""},{"k":"F8ULRBVB","t":"Open Peer Review: Fast Forward for a New Science","au":"Hachani, Samir","y":2015,"vn":"","ty":"book","tyl":"Book","doi":"10.1108/s0065-283020150000039012","cx":"Journal","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"book-monograph","oast":"green","cby":8,"self":false,"s":"Book chapter examining whether open peer review, exemplified by three specific platforms, has made scientific evaluation more transparent and objective compared with traditional closed review.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"GPSLVGVP","t":"Mitigating Negativity in Science Funding: The Role of Three-Step Procedures and Group Decision-Making","au":"Hottenrott, Hanna","y":2025,"vn":"SSRN Electronic Journal","ty":"journalArticle","tyl":"Journal article","doi":"10.2139/ssrn.5379818","cx":"Grant","co":["Bias"],"sg":["individual-scoring","panel-consensus"],"x":0,"st":"observational-quantitative","oast":"green","cby":0,"self":false,"s":"A study of 11 years of grant applications examines \"negative potency,\" the tendency to overweight negative information, finding it strongly biases individual expert assessments but that subsequent panel deliberation largely neutralizes this effect on final funding outcomes.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"GWDJVJX5","t":"When do experts listen to other experts? the role of negative information in expert evaluations for novel projects","au":"Lane, Jacqueline","y":2020,"vn":"SSRN Electronic Journal","ty":"journalArticle","tyl":"Journal article","doi":"10.2139/ssrn.3656495","cx":"Grant","co":["Bias"],"sg":["panel-consensus"],"x":1,"st":"experimental-rct","oast":"green","cby":4,"self":false,"s":"Field experiments with hundreds of evaluators assessing university grant proposals find evaluators lower their scores more after seeing harsher peer scores than they raise them after seeing more favorable ones, indicating a negativity bias that favors caution.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"H9DZ6W5V","t":"Research Conducted in Women Was Deemed More Impactful but Less Publishable than the Same Research Conducted in Men","au":"Murrar, Sohad","y":2021,"vn":"Journal of Women s Health","ty":"journalArticle","tyl":"Journal article","doi":"10.1089/jwh.2020.8666","cx":"Journal","co":["Bias"],"sg":["individual-scoring"],"x":1,"st":"experimental-rct","oast":"green","cby":37,"self":false,"s":"A randomized experiment found reviewers rated an identical clinical-study abstract as more scientifically valuable when it described research on women, yet were far more likely to recommend publishing the same study when it described research on men.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"IMRJ7IJ3","t":"Organization of research project competitions in the Republic of Moldova: lottery is better than incompetence and partiality","au":"Moldova, Agentia Nationala de Asigurare a Calitatii in Educatie si Cercetare, Republica","y":2021,"vn":"Akademos","ty":"journalArticle","tyl":"Journal article","doi":"10.52673/18570461.21.2-61.01","cx":"Grant","co":["Reliability-noise","Bias","Validity","Interventions"],"sg":["funding-decision"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":0,"self":false,"s":"Analyzing a Moldovan national research grant competition, this paper compares peer review and bibliometric evaluation methods and proposes combining bibliometrics with a lottery to address shortcomings identified in the process.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"J4QQRCKM","t":"Variables Affecting Evaluation and Publication of Oncology Case Reports: A Systematic Analysis","au":"Florita, Antonio","y":2016,"vn":"The International Journal of Biological Markers","ty":"journalArticle","tyl":"Journal article","doi":"10.5301/jbm.5000237","cx":"Journal","co":["Bias"],"sg":["funding-decision"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":1,"self":false,"s":"Analysing oncology case reports submitted to two journals, the study finds acceptance rates differed significantly by the type of case described and were markedly higher for authors from developed countries than developing countries.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"MRCX6QN8","t":"Publications as a measure of scientific advancement and of scientists' productivity","au":"Daniel, Hans‐Dieter","y":2005,"vn":"Learned Publishing","ty":"journalArticle","tyl":"Journal article","doi":"10.1087/0953151053584939","cx":"Journal","co":["Reliability-noise","Bias","Validity"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"bronze","cby":53,"self":false,"s":"A literature review of journal peer review research concluding that peer review generally functions as a valid quality filter, while noting inconsistent findings warrant supplementing it with post-publication evaluation.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"Q5KUVRG8","t":"Why Not Open the Black Box of Journal Editing in Philosophy? Make Peer Reviews of Published Papers Available","au":"Muckadell, Caroline Schaffalitzky de","y":2017,"vn":"Metaphilosophy","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/meta.12240","cx":"Journal","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"green","cby":6,"self":false,"s":"Proposes that philosophy journals publish peer reviews alongside accepted papers to increase transparency about editorial decisions while still protecting reviewer anonymity.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"QMJ45GTT","t":"Peer review training: A catalyst for quality Physiotherapy research","au":"Kasi, Madhuri","y":2025,"vn":"Physiotherapy - The Journal of Indian Association of Physiotherapists","ty":"journalArticle","tyl":"Journal article","doi":"10.4103/pjiap.pjiap_48_25","cx":"Journal","co":["Interventions"],"sg":["reviewer-selection","individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"hybrid","cby":0,"self":false,"s":"Discusses the role of peer review training in improving research quality in physiotherapy, motivated by concerns about declining quality of published systematic reviews and meta-analyses in the field.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"U5QL8TKC","t":"Rubrics, Not Vibes: Structured AI Integration as Quality Control for Peer Review","au":"Turner, Stephen D.","y":2026,"vn":"SSRN Electronic Journal","ty":"journalArticle","tyl":"Journal article","doi":"10.2139/ssrn.6314421","cx":"General","co":["Reliability-noise","Bias","Interventions"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"green","cby":0,"self":false,"s":"A position paper argues peer review's known unreliability could be improved by using AI as a structured rubric-enforcement and quality-control tool rather than a replacement for human judgment, proposing which review tasks suit AI versus human expertise.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"UBXPPIM6","t":"Ensuring Sustainable Evaluation: How to Improve Quality of Evaluating Grant Proposals?","au":"Wieczorkowska, Grażyna","y":2021,"vn":"Sustainability","ty":"journalArticle","tyl":"Journal article","doi":"10.3390/su13052842","cx":"Grant","co":["Reliability-noise","Bias","Interventions"],"sg":["individual-scoring"],"x":1,"st":"experimental-rct","oast":"gold","cby":4,"self":false,"s":"Analysis of two national grant funding agencies identifies reviewer calibration problems, rating-style differences, and order effects as limitations in proposal evaluation, and suggests evaluating reviewer performance and using consistent panels as remedies.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"no-statement","cau":""},{"k":"UUS7MZZX","t":"Reviewers’ Perceptions of the Peer Review Process in Journalism and Mass Communication","au":"Curtin, Patricia A.","y":2017,"vn":"Journalism & Mass Communication Quarterly","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/1077699017736031","cx":"Journal","co":["Bias","Interventions"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"bronze","cby":16,"self":false,"s":"A survey of journalism and mass-communication scholars found reviewers rated their own reviewing more highly than the reviews they received, and despite general ambivalence about peer review, most preferred keeping the double-blind model.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"CTHBPG4W","t":"Gender and Geographic Origin as Determinants of Manuscript Publication Outcomes: JBMR® Bibliometric Analysis from 2017 to 2019","au":"Rivadeneira, Fernando","y":2020,"vn":"Journal of Bone and Mineral Research","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/jbmr.4696","cx":"Journal","co":["Bias"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"green","cby":3,"self":false,"s":"A bibliometric analysis of a bone-research journal's 2017 to 2019 submissions found manuscript acceptance varied by author gender pairing and geographic region, with editors tending to select reviewers from their own region.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"no-statement","cau":""},{"k":"F7BJWUEL","t":"Peer Review Agreement or Peer Review Disagreement: Which Is Better?","au":"Hemlin, Sven","y":2009,"vn":"Journal of Psychology of Science and Technology","ty":"journalArticle","tyl":"Journal article","doi":"10.1891/1939-7054.2.1.5","cx":"General","co":["Reliability-noise","Bias"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":11,"self":false,"s":"A conceptual analysis examines why reviewers often disagree about manuscripts and grant applications, arguing that both agreement and disagreement among reviewers can have positive or negative consequences depending on their underlying causes.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"LLZCSWHH","t":"A New Paradigm for Scientific Publishing, Peer Review, and Impact Assessment","au":"Argibay, Nicolas","y":2026,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31222/osf.io/gcdqf_v1","cx":"General","co":["Bias","Validity","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":0,"self":false,"s":"A position paper proposing a new, AI-era framework for scientific publishing and peer review built around trust-weighted impact scores, transparency, and community governance, informed by prior open-science initiatives like DORA.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"thescientificreview.org/impact-factors","ca":"open","cau":"thescientificreview.org/impact-factors"},{"k":"4BPNLHNQ","t":"Validation of the CoRE Questionnaire for a Medical Journal Peer Review","au":"Doi, Suhail A.R.","y":2015,"vn":"Accountability in Research","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/08989621.2014.1002835","cx":"Journal","co":["Reliability-noise","Interventions"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":2,"self":false,"s":"Validation study of a structured review questionnaire (CoRE) for a medical journal, finding its concrete, expertise-sensitive items produced strong agreement between reviewer recommendations and their scored answers.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"6S73VTHR","t":"Promoting equity in the peer review process of journal publication","au":"Bancroft, Senetta F.","y":2022,"vn":"Science Education","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/sce.21733","cx":"Journal","co":["Bias","Interventions"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"closed","cby":19,"self":false,"s":"Analysis of editorial board composition and review policies at six science education journals finding boards were gender-balanced but predominantly North American and White, with limited explicit attention to bias in review policies.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"no-statement","cau":""},{"k":"7GZFFKLQ","t":"Triple-blind review as a solution to gender bias in academic publishing, a theoretical approach","au":"Conklin, Michael","y":2022,"vn":"Studies in Higher Education","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/03075079.2022.2081681","cx":"Journal","co":["Bias","Interventions"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":17,"self":false,"s":"A theoretical study examines whether extending double-blind review to triple-blind review could further reduce gender bias in academic publishing beyond what double-blind review already achieves.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"8IUUWWR8","t":"Ein Blick in die Black Box des Review-Verfahrens","au":"Otte, Gunnar","y":2019,"vn":"Zeitschrift für Soziologie","ty":"journalArticle","tyl":"Journal article","doi":"10.1515/zfsoz-2019-0001","cx":"Journal","co":["Reliability-noise","Bias"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"closed","cby":5,"self":false,"s":"German-language article examines the double-blind review process of a sociology journal, questioning whether its editorial and review procedures fairly represent the discipline's full range of subfields and theoretical approaches.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"96KKLHBV","t":"Battling bias at NIH.","au":"Sherley, James L","y":2012,"vn":"Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1126/science.337.6094.523-b","cx":"Grant","co":["Bias","Interventions"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":0,"self":false,"s":"A commentary on evidence of bias against Black scientists in NIH grant review argues that beyond preserving anonymity, NIH should actively adjust its review procedures to reduce racial bias and its downstream career effects.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"97SX9MRQ","t":"Peer review at National Institutes of Health: Small steps forward.","au":"Johnston, S. Claiborne","y":2008,"vn":"Annals of Neurology","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/ana.21260","cx":"Grant","co":["Reliability-noise","Bias","Validity","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":2,"self":false,"s":"A commentary on incremental NIH peer-review reforms argues the changes are reasonable but not evidence-based, noting that few scientific studies have rigorously examined variability and other properties of grant review.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"BDU9ZWUC","t":"The bibliometric indicators as predictors of the final decision of the peer review","au":"Vieira, Elizabeth S.","y":2015,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvv037","cx":"Grant","co":["Validity"],"sg":["funding-decision"],"x":0,"st":"observational-quantitative","oast":"closed","cby":7,"self":false,"s":"Analysing a specific researcher-selection process, the study finds two composite bibliometric indicators strongly correlate with peer-review outcomes, estimating about a 75 percent chance that a selected applicant's bibliometric score exceeds a rejected applicant's.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"BZH4FMTG","t":"Does “Decision Fatigue” Impact Manuscript Acceptance? An Analysis of Editorial Decisions by the American Journal of Gastroenterology","au":"Kwan, James","y":2016,"vn":"The American Journal of Gastroenterology","ty":"journalArticle","tyl":"Journal article","doi":"10.1038/ajg.2016.452","cx":"Journal","co":["Bias"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"closed","cby":6,"self":false,"s":"An analysis of editorial acceptance decisions at a gastroenterology journal examining whether \"decision fatigue\" affects how editors rule on manuscripts over time.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"DRNLYALD","t":"The Air We Breathe: A Critical Look at Practices and Alternatives in the Peer-Review Process","au":"Suls, Jerry","y":2009,"vn":"Perspectives on Psychological Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/j.1745-6924.2009.01105.x","cx":"Journal","co":["Reliability-noise","Bias","Validity","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":85,"self":false,"s":"Reviews criticisms of traditional anonymous peer review in psychology and surveys alternative practices from other disciplines, calling for broader discussion of reform within the field.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"DTEQFJLY","t":"Double-Blind Review in Software Engineering Venues: The Community's Perspective","au":"Bacchelli, Alberto","y":2017,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.1109/icse-c.2017.49","cx":"Journal","co":["Bias","Interventions"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"green","cby":13,"self":false,"s":"Survey and interview study of the software engineering research community on double-blind review, finding perceived logistical costs outweighed benefits, though many still supported adopting it for journals and conferences.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"http://sback.it/dblind","ca":"open","cau":"http://sback.it/dblind"},{"k":"HHTIHQYC","t":"Peer review in the development of academic articles: Experiences of Finnish authors in the educational sciences","au":"Atjonen, Päivi","y":2018,"vn":"Learned Publishing","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/leap.1204","cx":"Journal","co":["Reliability-noise","Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":7,"self":false,"s":"A survey of 121 Finnish education-sciences authors finds most viewed peer review feedback as constructive and helpful for improving manuscripts, though a substantial minority reported experiencing biased or paradigm-mismatched criticism.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"IZUPZGW2","t":"Rebound Peer Review: A Viable Recourse for Aggrieved Authors?","au":"Sen, Chandan K.","y":2011,"vn":"Antioxidants and Redox Signaling","ty":"journalArticle","tyl":"Journal article","doi":"10.1089/ars.2011.4424","cx":"Journal","co":["Bias","Interventions"],"sg":["funding-decision"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":10,"self":false,"s":"Editorial introduces a \"rebound peer review\" track allowing authors of rejected manuscripts a second, hybrid blinded and open review pathway intended to address possible unfairness in the original review outcome.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"LFSAIIW3","t":"The policy challenges of peer review: managing bias, conflict of interests and interdisciplinary assessments","au":"Langfeldt, Liv","y":2006,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.3152/147154406781776039","cx":"Grant","co":["Reliability-noise","Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":180,"self":false,"s":"Drawing on multiple empirical studies, the paper analyzes peer review through social, uncertainty, and organizational lenses, arguing that its typically cautious, risk-averse tendencies can disadvantage interdisciplinary or unconventional proposals.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"LIK4IE3M","t":"Refinement: Measuring informativeness of ratings in the absence of a gold standard","au":"Grant, Sheridan","y":2022,"vn":"British Journal of Mathematical and Statistical Psychology","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/bmsp.12268","cx":"Grant","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"closed","cby":2,"self":false,"s":"This paper proposes an information-theoretic metric, \"entropic refinement,\" to measure how informative a rater's ratings are without needing a gold standard, applying it to real grant-review data to compare overall versus criterion scores.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://doi.org/10.6084/m9.figshare.12728087","ca":"no-statement","cau":""},{"k":"N77BA9EY","t":"Editorial Judgments","au":"Hirschauer, Stefan","y":2009,"vn":"Social Studies of Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/0306312709335405","cx":"Journal","co":["general-systemic"],"sg":["funding-decision"],"x":0,"st":"qualitative","oast":"closed","cby":116,"self":false,"s":"Based on observing a sociology journal's editorial process, the study argues editorial judgments involve interpretive reading and rationalized group discussion rather than simple reliability, describing peer review as a form of mutual, ongoing monitoring among experts.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"P9WMK7AM","t":"National institutes of health: The lottery.","au":"Heilman, Kenneth M","y":2007,"vn":"Annals of Neurology","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/ana.21133","cx":"Grant","co":["Reliability-noise","Bias","Validity"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":1,"self":false,"s":"A commentary proposing that NIH peer review be tested for bias, reliability and validity using independent duplicate reviews and funding of borderline proposals to assess whether scores predict later research impact.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"PH3ATIG8","t":"Is three better than one? simulating the effect of reviewer selection and behavior on the quality and efficiency of peer review","au":"Bianchi, Federico","y":2015,"vn":"2015 Winter Simulation Conference (WSC)","ty":"journalArticle","tyl":"Journal article","doi":"10.1109/wsc.2015.7408561","cx":"Journal","co":["Reliability-noise","Bias"],"sg":["reviewer-selection","individual-scoring"],"x":0,"st":"simulation-methodological","oast":"green","cby":12,"self":false,"s":"Simulation study examining how the number and behavior of reviewers (fair, random, or strategic) affect peer-review quality and efficiency, finding added reviewers reduce bias but can strain reviewing resources if reviewers are highly diligent.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"PI3PZFJZ","t":"What We Still Don't Know About Peer Review","au":"Sabaj, Omar","y":2016,"vn":"Journal of Scholarly Publishing","ty":"journalArticle","tyl":"Journal article","doi":"10.3138/jsp.47.2.180","cx":"General","co":["general-systemic"],"sg":["whole-process"],"x":0,"st":"systematic-review-meta-analysis","oast":"closed","cby":40,"self":false,"s":"Reviews 150 empirical studies on peer review, finding most focus on system-level evaluation or reviewer demographics rather than on the actual texts and discourse exchanged during review.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"PL3YJB2Z","t":"Peer review research assessment: are the reviewers really experts?","au":"Abramo, Giovanni","y":2024,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvaf043","cx":"General","co":["general-systemic"],"sg":["reviewer-selection"],"x":0,"st":"observational-quantitative","oast":"closed","cby":3,"self":false,"s":"A study of Italy's national research assessment exercise examines whether reviewers assigned to evaluate submissions actually have matching disciplinary expertise and strong citation records, finding considerable variability and some reviewers with weak topical alignment or performance.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"QMMNS2MV","t":"Research Note—Making Peer Review Evidence-Based: It’s Time to Open the “Black Box”","au":"Dunleavy, Daniel J.","y":2024,"vn":"Journal of Social Work Education","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/10437797.2024.2378685","cx":"Journal","co":["Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":1,"self":false,"s":"Research note argues social work peer review remains closed and unaccountable despite being viewed as the best available system, and calls for an evidence-based research agenda on open peer review.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"QWVEZ7QZ","t":"On the Nature and Role of Peer Review in Mathematics","au":"Andersen, Line Edslev","y":2016,"vn":"Accountability in Research","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/08989621.2016.1274885","cx":"Journal","co":["Reliability-noise","Validity"],"sg":["individual-scoring","reviewer-selection"],"x":0,"st":"qualitative","oast":"closed","cby":29,"self":false,"s":"Based on interviews with seven mathematicians, the paper argues that mathematics journals' common practice of using a single referee is justified by the discipline's particular approach to verifying proof validity.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"2CHZ5LDK","t":"Guardians of Science: Fairness and Reliability of Peer Review","au":"Fletcher, Sara","y":1994,"vn":"BMJ","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmj.309.6952.488","cx":"Journal","co":["Reliability-noise","Bias","Validity"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":28,"self":false,"s":"A review of a book on peer review's fairness and reliability, framed around a historical anecdote about a surgeon who pioneered outcome tracking to improve medical standards.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"2IQ63KYT","t":"New Light on Old Boys: Cognitive and Institutional Particularism in the Peer Review System","au":"Travis, G.D.L.","y":1991,"vn":"Science, technology & human values","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/016224399101600303","cx":"Grant","co":["Bias"],"sg":["reviewer-selection","individual-scoring"],"x":0,"st":"qualitative","oast":"closed","cby":247,"self":false,"s":"Drawing on observations of a British research council's grant committees, the authors distinguish \"cognitive particularism,\" reviewers favouring familiar ways of framing problems, from institutional cronyism, arguing the former is the more consequential and overlooked bias in peer review.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"3MQGCJCA","t":"How blind is blind review?","au":"Yankauer, Alfred","y":1991,"vn":"American Journal of Public Health","ty":"journalArticle","tyl":"Journal article","doi":"10.2105/ajph.81.7.843","cx":"Journal","co":["Bias","Interventions"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"green","cby":65,"self":false,"s":"A survey of reviewers at a public health journal examining how often they could correctly identify blinded authors or institutions, finding blinding was imperfect but generally successful and favored by most respondents.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"3PY7SBW2","t":"Peer Review for Journals as it Stands Today—Part 1","au":"Campanario, Juan Miguel","y":1998,"vn":"Science Communication","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/1075547098019003002","cx":"Journal","co":["Reliability-noise","Bias"],"sg":["reviewer-selection","individual-scoring","whole-process"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":200,"self":false,"s":"First part of a two-part literature review summarizing research on who participates in journal peer review and on systemic problems of reliability, accuracy, and bias.","pg":"not-codeable","pgr":"","pgid":"","da":"","dau":"","ca":"","cau":""},{"k":"8TI3P7Z6","t":"Variation in journal peer review systems. Possible causes and consequences.","au":"Hargens, Lowell L.","y":1990,"vn":"PubMed","ty":"journalArticle","tyl":"Journal article","doi":"","cx":"Journal","co":["Reliability-noise"],"sg":["funding-decision"],"x":0,"st":"observational-quantitative","oast":"closed","cby":81,"self":false,"s":"Compares peer review practices across three disciplines, finding large differences in rejection rates, revision patterns, and review timelines between an astrophysics, a biology, and a sociology journal.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"B4FTVF6S","t":"Do Journal Rejection Rates Index Consensus?","au":"Cole, Stephen","y":1988,"vn":"American Sociological Review","ty":"journalArticle","tyl":"Journal article","doi":"10.2307/2095740","cx":"Journal","co":["Reliability-noise"],"sg":["funding-decision"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":66,"self":false,"s":"A commentary responds to prior work on journal rejection rates as an indicator of consensus across scientific disciplines, questioning whether rejection rates reliably reflect underlying differences in cognitive agreement between fields.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"BCKFAVIX","t":"Scholarly Consensus and Journal Rejection Rates","au":"Hargens, Lowell L.","y":1988,"vn":"American Sociological Review","ty":"journalArticle","tyl":"Journal article","doi":"10.2307/2095739","cx":"Journal","co":["Reliability-noise"],"sg":["funding-decision"],"x":0,"st":"observational-quantitative","oast":"closed","cby":239,"self":false,"s":"Analyzes long-term stability in journal rejection rates across disciplines, concluding that variation is better explained by differing levels of scholarly consensus than by space constraints on publishing.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"CG7WES8M","t":"The nonuse of psychological research at two federal agencies.","au":"Arkes, Hal R","y":2003,"vn":"Psychological science","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/1467-9280.01410","cx":"Grant","co":["Reliability-noise","Interventions"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"closed","cby":58,"self":false,"s":"Reflecting on a 1994 government report critical of federal grant review procedures, the paper illustrates alternative approaches the agencies could have adopted and discusses their reluctance to apply psychological decision-making research to reform review.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"E3GA6AEC","t":"What is the source of bias in peer review?","au":"Over, Ray","y":1982,"vn":"Behavioral and Brain Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/s0140525x00011559","cx":"Journal","co":["Reliability-noise","Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":47,"self":false,"s":"A brief, abstract-unavailable piece discussing possible sources of bias in the peer review process.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"EIL6RVZV","t":"Peer reviewing: Improve or be rejected","au":"Howe, Michael J. A.","y":1982,"vn":"Behavioral and Brain Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/s0140525x00011456","cx":"Journal","co":["Reliability-noise","Bias"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"closed","cby":41,"self":false,"s":"A short piece, whose text was unavailable beyond the title, apparently concerned with the need to improve peer reviewing or risk having submissions rejected.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"FPDF4D3R","t":"Peer review and editorial decision-making","au":"Howard, Louise M.","y":1998,"vn":"The British Journal of Psychiatry","ty":"journalArticle","tyl":"Journal article","doi":"10.1192/bjp.173.2.110","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"closed","cby":42,"self":false,"s":"An analysis of manuscripts submitted to a psychiatry journal finding poor interrater reliability among assessors' rankings, though assessors and editors agreed well on which papers to clearly reject.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"HP47KBFE","t":"On peer review: “We have met the enemy and he is us”","au":"Cicchetti, Domenic V.","y":1982,"vn":"Behavioral and Brain Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/s0140525x00011286","cx":"Journal","co":["Reliability-noise","Bias"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"closed","cby":47,"self":false,"s":"Brief commentary on peer review's shortcomings, framed around the idea that reviewers themselves are often the source of the process's problems (fuller abstract not available).","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"IBG58DEY","t":"Theoretical implications of failure to detect prepublished submissions","au":"Eckberg, Douglas Lee","y":1982,"vn":"Behavioral and Brain Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/s0140525x0001133x","cx":"Journal","co":["Reliability-noise","Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":38,"self":false,"s":"Discusses theoretical implications of cases where reviewers fail to detect that a submission has already been published elsewhere.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"KKY4QUVR","t":"The Manuscript Review and Decision-Making Process","au":"Bakanic, Von","y":1987,"vn":"American Sociological Review","ty":"journalArticle","tyl":"Journal article","doi":"10.2307/2095599","cx":"Journal","co":["Reliability-noise","Bias"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"closed","cby":203,"self":false,"s":"A study of manuscripts submitted to a major sociology journal (1977-1981) finds that referee recommendations and the number of revisions requested account for most of the variance in editors' final publication decisions, more than authors' characteristics.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"QTGZWS97","t":"Reflections on the peer review process","au":"Marsh, Herbert W.","y":1991,"vn":"Behavioral and Brain Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/s0140525x00065912","cx":"General","co":["Reliability-noise"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"closed","cby":19,"self":false,"s":"Commentary offering reflections on the peer review process, with no further detail available beyond the title.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"RXDUVK5E","t":"Manuscript evaluation by journal referees and editors: Randomness or bias?","au":"Colman, Andrew M.","y":1982,"vn":"Behavioral and Brain Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/s0140525x00011298","cx":"Journal","co":["Reliability-noise","Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":50,"self":false,"s":"A brief note on whether journal manuscript evaluation by referees and editors reflects genuine quality judgment or effectively amounts to randomness or bias; full text not available for this record.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"SDDU5JNV","t":"Confirmational Response Bias Among Social Work Journals","au":"Epstein, William","y":1990,"vn":"Science Technology & Human Values","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/016224399001500102","cx":"Journal","co":["Bias"],"sg":["individual-scoring"],"x":1,"st":"experimental-rct","oast":"closed","cby":143,"self":false,"s":"An experiment submitting matched papers with opposite findings to social work and other journals found limited quantitative evidence of confirmation bias but consistent qualitative evidence that many reviews were not rigorous or objective.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"U2FK4Y5H","t":"US and Non-US Submissions","au":"Link, Ann Marie","y":1998,"vn":"JAMA","ty":"journalArticle","tyl":"Journal article","doi":"10.1001/jama.280.3.246","cx":"Journal","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":323,"self":false,"s":"Analysing manuscripts submitted to a gastroenterology journal, the study finds reviewers from any country rated non-US and US papers similarly, but US-based reviewers rated papers from US authors distinctly more favourably than non-US reviewers did.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"8GRXGMM9","t":"The Effects of Blinding on the Quality of Peer Review","au":"Mcnutt, Robert","y":1990,"vn":"JAMA","ty":"journalArticle","tyl":"Journal article","doi":"10.1001/jama.1990.03440100079012","cx":"Journal","co":["Interventions"],"sg":["individual-scoring"],"x":1,"st":"experimental-rct","oast":"closed","cby":258,"self":false,"s":"A randomized trial testing blinded versus non-blinded peer review at a medical journal, finding blinded reviews were rated modestly higher in quality, though blinding was only partially successful.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"QIRA4ABW","t":"Reviewer Bias","au":"Ernst, E","y":1992,"vn":"Annals of Internal Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.7326/0003-4819-116-11-958_2","cx":"Journal","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":22,"self":false,"s":"Letter describing an experiment that sent a fabricated manuscript with a predetermined positive result to investigators known to hold opposing views on a treatment, to test for reviewer bias linked to prior beliefs.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"2PDM7FYG","t":"Peer-review research: Objections and obligations","au":"Peters, Douglas","y":1982,"vn":"Behavioral and Brain Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/s0140525x00011754","cx":"Journal","co":["Reliability-noise","Bias"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"closed","cby":27,"self":false,"s":"Discusses methodological objections to peer-review research and the obligations that arise when studying the review process itself.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"6BRBUR5I","t":"Peer Review and Manuscript Management in Scientific Journals","au":"Irene, Hames","y":2007,"vn":"","ty":"book","tyl":"Book","doi":"10.1002/9780470750803","cx":"Journal","co":["Interventions"],"sg":["whole-process"],"x":0,"st":"book-monograph","oast":"closed","cby":184,"self":false,"s":"A practitioner guidebook on peer review and manuscript management in scientific journals aimed at editors and publishers, covering practical guidelines for good editorial practice.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"848B4KFS","t":"Estimation of the Reliability of Ratings","au":"Ebel, Robert L.","y":1951,"vn":"Psychometrika","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/bf02288803","cx":"Analogue","co":["Reliability-noise"],"sg":["whole-process"],"x":0,"st":"simulation-methodological","oast":"closed","cby":986,"self":false,"s":"Introduces an analysis-of-variance-based procedure for estimating the reliability of ratings, applicable to both complete and incomplete rating designs, and provides confidence intervals for individual and averaged ratings.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"8RNI7EWX","t":"Taming complexity: narrative CVs in grant funding evaluations (RoRI Working Paper No. 14)","au":"Research on Research Institute (RoRI)","y":2025,"vn":"Figshare","ty":"report","tyl":"Report","doi":"10.6084/m9.figshare.27291537.v4","cx":"Grant","co":["Interventions"],"sg":["proposal-quality","panel-consensus"],"x":0,"st":"qualitative","oast":"green","cby":0,"self":false,"s":"A working paper examines how narrative CV formats, which let researchers describe achievements in prose rather than only lists, are changing evaluative practice in European grant panels, based on interviews from a Dutch funding program.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"92UR7R8Y","t":"Publish or Politic: Referee Bias in Manuscript Review1","au":"Abramowitz, Stephen I.","y":1975,"vn":"Journal of Applied Social Psychology","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/j.1559-1816.1975.tb00675.x","cx":"Journal","co":["Bias"],"sg":["individual-scoring"],"x":1,"st":"experimental-rct","oast":"closed","cby":137,"self":false,"s":"An experiment sending politically liberal and conservative reviewers different versions of a manuscript about student activists found publication ratings were sometimes biased by reviewers' political orientation, though specific quality criteria were less affected.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"9C2AQMRD","t":"Statistical Methods for Rates and Proportions","au":"Fleiss, Joseph L.","y":2003,"vn":"Wiley series in probability and statistics","ty":"book","tyl":"Book","doi":"10.1002/0471445428","cx":"Analogue","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"book-monograph","oast":"closed","cby":18058,"self":false,"s":"A statistics textbook covering methods for analysing rates and proportions, including a chapter on measuring interrater agreement alongside broader topics such as regression models and sampling for comparative studies.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"BMWJHC4K","t":"Peer-review practices of psychological journals: The fate of published articles, submitted again","au":"Peters, Douglas P.","y":1982,"vn":"Behavioral and Brain Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/S0140525X00011183","cx":"Journal","co":["Reliability-noise","Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":1083,"self":false,"s":"Twelve previously published psychology articles were resubmitted to their original journals under fictitious author names, and nearly all were rejected on review, with reviewers rarely detecting the resubmission.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"D32A3NXV","t":"The experimental research funder’s handbook (RoRI Working Paper No.6)","au":"Bendiscioli, Sandra","y":2021,"vn":"Open MIND","ty":"report","tyl":"Report","doi":"10.6084/m9.figshare.17102426.v1","cx":"Grant","co":["Interventions"],"sg":["whole-process"],"x":0,"st":"guideline-policy","oast":"green","cby":4,"self":false,"s":"Practical handbook compiling case studies and lessons from research funders who have run trials of peer-review and evaluation process changes, intended to guide other funders through designing similar experiments.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"FZ3ZTGAR","t":"Questioning peer review: Understanding peer review system-wide (RoRI Working Paper No. 21)","au":"Gregory, Kathleen","y":2026,"vn":"Figshare","ty":"report","tyl":"Report","doi":"10.6084/m9.figshare.32685546.v4","cx":"General","co":["general-systemic"],"sg":["whole-process"],"x":0,"st":"qualitative","oast":"green","cby":0,"self":false,"s":"Working paper drawing on expert interviews to examine peer review's aims, actors, and procedures across research funding, publishing, and career domains, identifying cross-cutting themes of quality, capacity, diversity, openness, and trust.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"H7D6ZSQU","t":"The Marks of Examiners.","au":"S., F.","y":1937,"vn":"Journal Of The Royal Statistical Society","ty":"journalArticle","tyl":"Journal article","doi":"10.2307/2980288","cx":"Analogue","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":98,"self":false,"s":"An archival statistical review discusses examiner scoring practices, part of an older body of work on evaluating scientific and educational assessments through peer or expert judgment.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"K6QJQFGS","t":"Editorial : Pourquoi l'évaluation par les pairs s’impose - Why peer review is needed","au":"This, Hervé","y":2023,"vn":"Notes Académiques de l Académie d agriculture de France / Academic Notes of the French Academy of Agriculture","ty":"journalArticle","tyl":"Journal article","doi":"10.58630/pubac.not.690522","cx":"Journal","co":["general-systemic"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"bronze","cby":0,"self":false,"s":"French-language editorial argues peer review consistently improves the quality of published scientific articles and responds to common criticisms of the practice with counter-arguments and empirical evidence.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"MBCQSMRJ","t":"Applicants as Reviewers: Evaluating the Risks, Benefits, and Potential of Distributed Peer Review for Grant Funding Allocations  (RoRI Working Paper No. 17)","au":"Research on Research Institute (RoRI)","y":2026,"vn":"Figshare","ty":"report","tyl":"Report","doi":"10.6084/m9.figshare.29994841.v4","cx":"Grant","co":["Reliability-noise","Interventions"],"sg":["reviewer-selection","funding-decision"],"x":0,"st":"observational-quantitative","oast":"green","cby":0,"self":false,"s":"A mixed-methods evaluation of a Volkswagen Foundation trial in which grant applicants also served as reviewers for other proposals, comparing this distributed model with conventional panel review on efficiency, fairness, and risk of gaming.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"https://github.com/A-Butters/VWS_DPR","ca":"open","cau":"https://github.com/A-Butters/VWS_DPR"},{"k":"N6R93HK3","t":"Manuscript Characteristics Which Influence Acceptance for Management and Social Science Journals.","au":"Kerr, S. J.","y":1977,"vn":"Academy of Management Journal","ty":"journalArticle","tyl":"Journal article","doi":"10.2307/255467","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"closed","cby":239,"self":false,"s":"A survey of editors and board members from 19 management and social science journals examined which manuscript characteristics they believe influence acceptance decisions.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"NZSRHRPT","t":"Peer Review: Quality Control of Applied Social Research","au":"Noble, John H.","y":1974,"vn":"Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1126/science.185.4155.916","cx":"Grant","co":["Reliability-noise","Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":55,"self":false,"s":"An opinion piece argues that peer review needs strengthening and broader application to better ensure the quality and relevance of policy-oriented applied social research amid shifts in federal research funding priorities.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"QR6S3V6J","t":"Rejection, rebuttal, revision: Some flexible features of peer review","au":"Rubin, Donald B.","y":1982,"vn":"Behavioral and Brain Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/s0140525x00011638","cx":"Journal","co":["Reliability-noise","Bias"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"closed","cby":37,"self":false,"s":"A discussion of adaptable aspects of the peer review process, including how rejection, author rebuttal, and revision interact.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"RIVJNS9P","t":"Where next for partial randomisation of research funding? The feasibility of RCTs and alternatives (RoRI Working Paper No.9)","au":"Stafford, Tom","y":2022,"vn":"Figshare","ty":"report","tyl":"Report","doi":"10.6084/m9.figshare.20101118.v1","cx":"Grant","co":["Validity","Interventions"],"sg":["funding-decision","whole-process"],"x":0,"st":"conceptual-commentary","oast":"green","cby":0,"self":false,"s":"Discusses methodological considerations for studying partial randomization of research funding, including when randomized trials are feasible given sample size and statistical power constraints, and outlines alternative study designs.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"S4NYFMCH","t":"Reviewer “bias”: Do Peters and Ceci protest too much?","au":"Perlman, Daniel","y":1982,"vn":"Behavioral and Brain Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/s0140525x00011572","cx":"Journal","co":["Reliability-noise","Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":17,"self":false,"s":"Commentary questioning claims made in an earlier influential study about reviewer bias in the peer review of manuscripts.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"SWUKRL6L","t":"Peer review in funding organizations: An analytical literature review (RoRI Working Paper No.11)","au":"Gregory, Kathleen","y":2024,"vn":"Figshare","ty":"report","tyl":"Report","doi":"10.6084/m9.figshare.26861680.v5","cx":"Grant","co":["general-systemic"],"sg":["whole-process"],"x":0,"st":"systematic-review-meta-analysis","oast":"green","cby":0,"self":false,"s":"A working paper reviews the literature on peer review across funding organizations, building on an earlier publishing-focused taxonomy to map shared problems like bias, burden, and duplicated reviewer effort, and to identify opportunities for cross-domain synergy.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"T2KC2YGE","t":"Why draw lots? Funder motivations for using partial randomisation to allocate research grants (RoRI Working Paper No.7)","au":"Woods, Helen Buckley","y":2021,"vn":"Figshare","ty":"report","tyl":"Report","doi":"10.6084/m9.figshare.17102495.v2","cx":"Grant","co":["Validity","Interventions"],"sg":["funding-decision"],"x":0,"st":"qualitative","oast":"green","cby":0,"self":false,"s":"Qualitative interview study across six funding agencies in six countries explores institutional motivations and constraints behind adopting partial randomisation to allocate research grants.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"not-applicable","cau":""},{"k":"TKIAN9Z9","t":"Why Are Research Grant Applications Disapproved?","au":"Allen, Ernest M.","y":1960,"vn":"Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1126/science.132.3439.1532","cx":"Grant","co":["general-systemic"],"sg":["proposal-quality","funding-decision"],"x":0,"st":"observational-quantitative","oast":"closed","cby":77,"self":false,"s":"A 1960 analysis of nearly 2,000 rejected NIH biomedical research grant applications from a single year describes the common shortcomings that led review boards to disapprove them.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"VAJTKYSK","t":"Perhaps it was right to reject the resubmitted manuscripts","au":"Thomas, Garth J.","y":1982,"vn":"Behavioral and Brain Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/s0140525x00011675","cx":"Journal","co":["Reliability-noise","Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":41,"self":false,"s":"A brief, abstract-unavailable commentary discussing a case where resubmitted manuscripts were rejected, questioning or defending that outcome.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"WR6XY2LQ","t":"Statistical significance criteria for the rWG and average deviation interrater agreement indices.","au":"Smith-Crowe, Kristin","y":2014,"vn":"The Journal of applied psychology","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/a0034556","cx":"Analogue","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"closed","cby":66,"self":false,"s":"Simulation study generating statistical significance guidelines for the rWG and average deviation interrater agreement indices, extending inference beyond the narrow conditions covered by existing guidance.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"W3TP344G","t":"Rating intervals: an experiment in peer review.","au":"Green, J G","y":1989,"vn":"The FASEB Journal","ty":"journalArticle","tyl":"Journal article","doi":"10.1096/fasebj.3.8.2721858","cx":"Grant","co":["Reliability-noise","Interventions"],"sg":["individual-scoring","panel-consensus"],"x":1,"st":"experimental-rct","oast":"closed","cby":13,"self":false,"s":"An experimental study within NIH grant peer review testing whether standard descriptors and different voting scale intervals (0.1 versus 0.5) across 24 study sections improve rating uniformity and score spread, with results adopted into standard NIH practice.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"WU4XSEWV","t":"Effect of using reporting guidelines during peer review on quality of final manuscripts submitted to a biomedical journal: masked randomised trial.","au":"Cobo, E","y":2011,"vn":"BMJ (Clinical Research Ed.)","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmj.d6783","cx":"Journal","co":["Interventions"],"sg":["proposal-quality","individual-scoring"],"x":1,"st":"experimental-rct","oast":"hybrid","cby":297,"self":false,"s":"Masked randomised trial at a","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"on-request","cau":""},{"k":"XUCCE9SV","t":"Calibration with confidence: a principled method for panel assessment.","au":"MacKay, R S","y":2017,"vn":"Royal Society Open Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1098/rsos.160760","cx":"General","co":["Reliability-noise","Bias","Interventions"],"sg":["individual-scoring","panel-consensus"],"x":0,"st":"simulation-methodological","oast":"gold","cby":16,"self":false,"s":"A methods paper developing a graph-based algorithm that calibrates panel assessors' scores for differing standards and self-declared confidence, tested on simulated and realistic data and proposed for grant proposal panels, national research assessments and similar evaluations.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://github.com/ralphkenna/CWC.git","ca":"open","cau":"https://github.com/ralphkenna/CWC.git"},{"k":"XIFAVINM","t":"Tools used to assess the quality of peer review reports: a methodological systematic review.","au":"Superchi, Cecilia","y":2019,"vn":"BMC Medical Research Methodology","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s12874-019-0688-x","cx":"Journal","co":["Validity"],"sg":["individual-scoring"],"x":0,"st":"systematic-review-meta-analysis","oast":"gold","cby":111,"self":false,"s":"A methodological systematic review identifying 24 tools for assessing biomedical journal peer-review report quality, finding weak development and validation processes and wide variation in the quality domains evaluated.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://zenodo.org/communities/miror/?page=1&size=20","ca":"no-statement","cau":""},{"k":"WXYAW7F6","t":"Examining the value added by committee discussion in the review of applications for research awards","au":"Obrecht, Michael","y":2007,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.3152/095820207X223785","cx":"Grant","co":["Reliability-noise","Bias"],"sg":["individual-scoring","panel-consensus"],"x":0,"st":"observational-quantitative","oast":"closed","cby":42,"self":false,"s":"An observational study of research fellowship review committees, comparing reviewers' pre-meeting scores with post-discussion scores and finding that committee discussion added no improvement in fairness or effectiveness beyond the initial independent evaluations.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"VQLA65TQ","t":"On average deviation indices for estimating interrater agreement","au":"Burke, Michael J.","y":1999,"vn":"Organizational Research Methods","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/109442819921004","cx":"Analogue","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":457,"self":false,"s":"A methodological paper proposing average deviation indices of interrater agreement, showing across two large retail employee surveys that they correlate highly with established agreement indices and yield consistent decisions across different Likert scale formats.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"Y26RHK69","t":"Effects of training on quality of peer review: randomised controlled trial.","au":"Schroter, Sara","y":2004,"vn":"BMJ (Clinical Research Ed.)","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmj.38023.700775.AE","cx":"Journal","co":["Interventions"],"sg":["individual-scoring"],"x":1,"st":"experimental-rct","oast":"bronze","cby":269,"self":false,"s":"A randomised controlled trial of training for reviewers at a general medical journal, finding that short workshop or self-taught packages produced only small, short-lived improvements in review quality and error detection.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"WV6LSXZU","t":"Disparities in ratings of internal and external applicants: A case for model-based inter-rater reliability.","au":"Martinková, Patrícia","y":2018,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0203002","cx":"Analogue","co":["Reliability-noise","Bias","Validity"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":17,"self":false,"s":"An empirical study of teacher hiring ratings in Spokane Public Schools, developing model-based inter-rater reliability estimates and finding that external applicants receive lower and less consistent ratings than internal ones, even after controlling for qualifications and quality.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"open","cau":""},{"k":"WSUJ7BB2","t":"How Criterion Scores Predict the Overall Impact Score and Funding Outcomes for National Institutes of Health Peer-Reviewed Applications.","au":"Eblen, Matthew K","y":2016,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0155060","cx":"Grant","co":["Bias","Validity"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"gold","cby":43,"self":false,"s":"An analysis of over 123,700 NIH grant applications finding that Approach and, to a lesser extent, Significance scores most strongly predict overall peer-review scores and funding outcomes.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"no-statement","cau":""},{"k":"VXQ8SIJC","t":"The leaky pipeline in research grant peer review and funding decisions: challenges and future directions.","au":"Sato, Sayaka","y":2021,"vn":"Higher Education","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s10734-020-00626-y","cx":"Grant","co":["Bias"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"conceptual-commentary","oast":"hybrid","cby":87,"self":false,"s":"A review and commentary examining why studies of gender bias in grant peer review and funding decisions reach conflicting conclusions, highlighting methodological and contextual weaknesses and proposing directions for more comprehensive future research.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"ZY2PTU76","t":"Answers to 20 Questions About Interrater Reliability and Interrater Agreement","au":"LeBreton, J. M.","y":2007,"vn":"Organizational Research Methods","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/1094428106296642","cx":"Analogue","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"closed","cby":3524,"self":false,"s":"A tutorial article using a question-and-answer format to explain when and how to use interrater reliability and interrater agreement indices in organizational multilevel research, with SPSS worked examples in the appendices.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"no-statement","cau":""},{"k":"YYCAX8PN","t":"Are peer-reviews of grant proposals reliable? An analysis of Economic and Social Research Council (ESRC) funding applications","au":"Jerrim, John","y":2020,"vn":"The Social science journal","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/03623319.2020.1728506","cx":"Grant","co":["Reliability-noise","Bias"],"sg":["reviewer-selection","individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"closed","cby":31,"self":false,"s":"An empirical analysis of 4,000 ESRC grant proposals and 15,000 reviews, finding low agreement between reviewers (correlation 0.2), overly generous applicant-nominated reviewers who nonetheless influence funding, and a single negative review halving a proposal's chance of success.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"WY96SPAI","t":"NIH peer review: scored review criteria and overall impact.","au":"Lindner, Mark D","y":2016,"vn":"The American journal of evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/1098214015582049","cx":"Grant","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":21,"self":false,"s":"An analysis of NIH R01 grant peer review finding that all five scored criteria, including innovation, influence overall impact scores and must receive good scores for an application to achieve a strong overall rating.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"YSPY8PEM","t":"Funding science by lottery","au":"Avin, Shahar","y":2015,"vn":"Recent developments in the philosophy of science: EPSA13 helsinki","ty":"bookSection","tyl":"Book chapter","doi":"10.1007/978-3-319-23015-3_9","cx":"Grant","co":["Reliability-noise","Validity"],"sg":["funding-decision"],"x":0,"st":"book-monograph","oast":"green","cby":27,"self":false,"s":"A philosophy paper arguing that scientific merit is causal and changes rapidly, making it hard for grant peer reviewers to estimate reliably, and using this to motivate allocating some research funding by lottery.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"open","cau":"https://www.repository.cam.ac.uk/handle/1810/247434"},{"k":"XZ8J64TH","t":"Peer review in the funding of research in higher education: the australian experience","au":"Jayasinghe, U. W.","y":2001,"vn":"Educational evaluation and policy analysis","ty":"journalArticle","tyl":"Journal article","doi":"10.3102/01623737023004343","cx":"Grant","co":["Reliability-noise","Bias"],"sg":["reviewer-selection","individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"closed","cby":57,"self":false,"s":"Empirical evaluation of grant peer review at the Australian Research Council (2,989 proposals, 6,233 reviewers), finding low inter-rater reliability (.53), higher and less reliable ratings from researcher-nominated reviewers, and recommending fewer, better-selected reviewers and more reviews per proposal.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"YXJVI9WJ","t":"Peer Review Evaluation Process of Marie Curie Actions under EU's Seventh Framework Programme for Research.","au":"Pina, David G","y":2015,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0130753","cx":"Grant","co":["Reliability-noise"],"sg":["individual-scoring","panel-consensus"],"x":0,"st":"observational-quantitative","oast":"gold","cby":28,"self":false,"s":"Analysis of nearly 25,000 Marie Curie grant proposals under the EU's Seventh Framework Programme, finding high agreement between remote individual assessments and consensus scores, with disagreement more common for social sciences and humanities panels and lower-scored proposals.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"no-statement","cau":""},{"k":"XTPUUZW7","t":"Peer review of grant applications: criteria used and qualitative study of reviewer practices.","au":"Abdoul, Hendy","y":2012,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0046054","cx":"Grant","co":["Reliability-noise"],"sg":["proposal-quality","individual-scoring"],"x":0,"st":"qualitative","oast":"gold","cby":76,"self":false,"s":"Qualitative study of French hospital research grant reviewers, using interviews and observation, finding that assessment criteria such as originality and methodology are applied subjectively and weighted differently across reviewers, which may undermine review reliability.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"WS9B8LJP","t":"Early career grants, performance, and careers: A study on predictive validity of grant decisions","au":"van den Besselaar, Peter","y":2015,"vn":"Journal of Informetrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.joi.2015.07.011","cx":"Grant","co":["Validity"],"sg":["funding-decision","post-award-validity"],"x":0,"st":"observational-quantitative","oast":"closed","cby":92,"self":false,"s":"An empirical study of 260 early career grant applications in three social science fields, finding grant panels have low to moderate ability to predict later research performance and none when grantees are compared with the best rejected applicants, though winning a grant boosts careers.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"WNJ55J6C","t":"Influence of external peer reviewer scores for funding applications on funding board decisions: a retrospective analysis of 1561 reviews.","au":"Sorrell, Lexy","y":2018,"vn":"BMJ Open","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmjopen-2018-022547","cx":"Grant","co":["Reliability-noise","Validity"],"sg":["reviewer-selection","individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"gold","cby":5,"self":false,"s":"Retrospective analysis of 1561 external peer reviewer scores for UK NIHR grant applications, finding that mean reviewer scores predict funding board decisions moderately well, with no detectable gain from using more than four reviewers and equal influence across reviewer expertise types.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"no-statement","cau":""},{"k":"VHL3ICQ9","t":"The acceptability of using a lottery to allocate research funding: a survey of applicants.","au":"Liu, Mengyao","y":2020,"vn":"Research Integrity and Peer Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s41073-019-0089-z","cx":"Grant","co":["Validity"],"sg":["funding-decision"],"x":0,"st":"observational-quantitative","oast":"gold","cby":70,"self":false,"s":"Survey of applicants to the Health Research Council of New Zealand's Explorer Grant scheme, finding most considered a lottery an acceptable way to allocate grant funding, particularly when clearly ineligible or outstanding applications were dealt with first.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"no-statement","cau":""},{"k":"WH32N544","t":"Publication prejudices: An experimental study of confirmatory bias in the peer review system.","au":"Mahoney, MJ","y":1977,"vn":"Cognitive therapy and research","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/BF01173636","cx":"Journal","co":["Reliability-noise","Bias"],"sg":["individual-scoring"],"x":1,"st":"experimental-rct","oast":"closed","cby":1035,"self":false,"s":"An experimental study sending 75 journal reviewers identical manuscripts differing only in reported results, finding poor inter-rater agreement and strong bias against papers whose findings contradicted reviewers' theoretical perspectives.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"WIVHR46Q","t":"Effect on the Quality of Peer Review of Blinding Reviewers and Asking Them to Sign Their Reports","au":"Godlee, Fiona","y":1998,"vn":"The Journal of the American Medical Association","ty":"journalArticle","tyl":"Journal article","doi":"10.1001/jama.280.3.237","cx":"Journal","co":["Bias","Interventions"],"sg":["individual-scoring"],"x":1,"st":"experimental-rct","oast":"closed","cby":338,"self":false,"s":"A randomised controlled trial at a general medical journal testing whether blinding reviewers to author identity or requiring signed reports improved error detection in a deliberately flawed manuscript, finding neither intervention affected the number of weaknesses spotted.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"ZCR8KFF5","t":"Extent of type I and type II errors in editorial decisions: A case study on Angewandte Chemie International Edition","au":"Bornmann, Lutz","y":2009,"vn":"Journal of Informetrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.joi.2009.05.002","cx":"Journal","co":["Validity"],"sg":["funding-decision","post-award-validity"],"x":0,"st":"observational-quantitative","oast":"closed","cby":32,"self":false,"s":"A citation-based case study of manuscript decisions at the chemistry journal Angewandte Chemie International Edition, finding that about 15% of accepted papers underperformed the average rejected paper (type I errors) and 15% of rejected papers matched or exceeded accepted ones (type II errors).","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"XHLJRX3Y","t":"The peer review process used to evaluate manuscripts submitted to academic journals","au":"Marsh, Herbert W.","y":1989,"vn":"The Journal of Experimental Education","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/00220973.1989.10806503","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":119,"self":false,"s":"An empirical study of 278 manuscripts submitted to the Journal of Educational Psychology, finding single-reviewer reliability of about .30 for overall recommendations and showing that specific rating dimensions, singly or combined, agreed no better because of idiosyncratic reviewer response biases.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"ZN6UACIU","t":"Risk evaluation in peer review of grant applications","au":"Gallo, Stephen","y":2018,"vn":"Environment Systems and Decisions","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s10669-018-9677-6","cx":"Grant","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":36,"self":false,"s":"A survey of grant applicants and reviewers finding marked differences in perceptions of how innovation and risk influenced peer-review scores, indicating a potential source of bias in funding decisions.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"YNHSVDSY","t":"Improving the Peer review process: Capturing more information and enabling high-risk/high-return research","au":"Linton, Jonathan D.","y":2016,"vn":"Research Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.respol.2016.07.004","cx":"Grant","co":["Reliability-noise","Interventions"],"sg":["panel-consensus","funding-decision"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":31,"self":false,"s":"A methodological note applying the Black-Scholes options-pricing model to grant peer review, arguing that using reviewer score variance alongside the mean makes high-risk/high-return projects more attractive and can justify funding proposals with high panel disagreement.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"VPIZJDYQ","t":"How do journals of different rank instruct peer reviewers? Reviewer guidelines in the field of management","au":"Seeber, Marco","y":2020,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-019-03343-1","cx":"Journal","co":["Interventions"],"sg":["proposal-quality","individual-scoring"],"x":0,"st":"observational-quantitative","oast":"green","cby":38,"self":false,"s":"A content analysis of reviewer guidelines from 46 management journals, finding that lower-ranked journals emphasise forms and gatekeeping criteria while the highest-ranked journals give far more developmental instructions and stress theoretical contribution and methodological rigour.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"WC3AX6UH","t":"Inter-rater reliability and validity of peer reviews in an interdisciplinary field","au":"Jirschitzka, Jens","y":2017,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-017-2516-6","cx":"Journal","co":["Reliability-noise","Validity"],"sg":["reviewer-selection","individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":43,"self":false,"s":"Empirical study of 443 reviews from an interdisciplinary conference, finding poor inter-rater reliability and low construct validity across rating dimensions, with same-discipline reviewers' relevance ratings positively, and novelty ratings negatively, predicting later citations.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"VP2W7UDL","t":"Bias in research grant evaluation has dire consequences for small universities.","au":"Murray, Dennis L","y":2016,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0155876","cx":"Grant","co":["Bias"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"gold","cby":89,"self":false,"s":"An empirical analysis of Canada's NSERC Discovery Grant programme (2011-2014) showing that applicants from small universities receive consistently lower success rates and funding, attributing this to systemic evaluation bias and arguing for remedies such as blind review rather than post-evaluation corrections.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"http://dx.doi.org/10.5061/dryad.m33hv/1","ca":"no-statement","cau":""},{"k":"XUEM9UZN","t":"Alternatives to Peer Review in Research Project Funding: 2013 Update","au":"Guthrie, Susan","y":2013,"vn":"","ty":"webpage","tyl":"Webpage","doi":"","cx":"Grant","co":["Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":6,"self":false,"s":"A RAND Europe report reviewing alternative approaches to peer review for allocating research project grant funding, updating earlier work with developments to 2013.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"Y5JCHZCD","t":"Engaging Patients and Stakeholders in Research Proposal Review: The Patient-Centered Outcomes Research Institute","au":"Fleurence, Rachael","y":2014,"vn":"Annals of Internal Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.7326/m13-2412","cx":"Grant","co":["Reliability-noise"],"sg":["reviewer-selection","individual-scoring","panel-consensus"],"x":0,"st":"observational-quantitative","oast":"closed","cby":82,"self":false,"s":"An empirical analysis of PCORI's first grant merit review round, finding that scientist, patient and stakeholder reviewers initially scored proposals quite differently but converged substantially after in-person panel discussion, which markedly changed the final funding rankings.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"YXXSYUJR","t":"Assessing the potential of a Bayesian ranking as an alternative to consensus meetings for decision making in research funding: A case study of Marie Skłodowska-Curie actions","au":"Heyard, Rachel","y":2025,"vn":"PLoS ONE","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0317772","cx":"Grant","co":["Reliability-noise","Interventions"],"sg":["individual-scoring","panel-consensus","funding-decision"],"x":0,"st":"observational-quantitative","oast":"gold","cby":0,"self":false,"s":"A case study testing whether a Bayesian ranking of individual reviewer scores could replace consensus meetings in Marie Skłodowska-Curie grant evaluation, finding large discrepancies with panel outcomes and recommending pre-scores be used only to triage weak proposals.","pg":"preregistered","pgr":"OSF","pgid":"osf.io/cnurf","da":"open","dau":"https://gitlab.com/heyardr/msca-br-comparison","ca":"open","cau":"https://gitlab.com/heyardr/msca-br-comparison"},{"k":"YSVZ7IT3","t":"Do Grant Proposal Texts Matter for Funding Decisions? A Field Experiment","au":"Şimşek, Müge","y":2023,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31219/osf.io/yhs2e","cx":"Grant","co":["Reliability-noise"],"sg":["proposal-quality","individual-scoring"],"x":1,"st":"experimental-rct","oast":"gold","cby":1,"self":false,"s":"A field experiment with the Dutch Research Council in an early-career grant competition, finding that panellists shown only a CV and one-paragraph summary ranked proposals no differently from those shown the full text, questioning the value of lengthy proposals.","pg":"preregistered","pgr":"AsPredicted","pgid":"https://aspredicted.org/md45e.pdf","da":"open","dau":"","ca":"no-statement","cau":""},{"k":"ZPZW8PR3","t":"The reliability of peer review in anthrozoology","au":"Herzog, Harold A.","y":2005,"vn":"Anthrozoös","ty":"journalArticle","tyl":"Journal article","doi":"10.2752/089279305785594180","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"closed","cby":4,"self":false,"s":"An analysis of conference peer review in anthrozoology finding poor to fair agreement among reviewers, but substantially improved reliability in acceptance decisions when ratings from three reviewers were averaged.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"YBHRWT7Z","t":"Gender contributes to personal research funding success in The Netherlands","au":"Lee, Romy van der","y":2015,"vn":"Proceedings of the National Academy of Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1073/pnas.1510159112","cx":"Grant","co":["Bias"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"bronze","cby":320,"self":false,"s":"Empirical analysis of 2,823 early-career grant applications to the Netherlands Organization for Scientific Research, finding gender bias favouring men in reviewers' \"quality of researcher\" evaluations, success rates and gendered language in evaluation materials.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"YJ6FIFYN","t":"Association of percentile ranking with citation impact and productivity in a large cohort of de novo NIMH-funded R01 grants","au":"Doyle, Jamie Mihoko","y":2015,"vn":"Molecular Psychiatry","ty":"journalArticle","tyl":"Journal article","doi":"10.1038/mp.2015.71","cx":"Grant","co":["Validity"],"sg":["post-award-validity"],"x":0,"st":"observational-quantitative","oast":"green","cby":41,"self":false,"s":"An analysis of 1,755 NIMH-funded R01 grants finding no association between peer-review percentile rankings and subsequent productivity or citation impact, while investigators’ previous funding and academic productivity moderately predicted citation impact.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"on-request","cau":""},{"k":"VPW3P5QI","t":"Face-to-face panel meetings versus remote evaluation of fellowship applications: simulation study at the Swiss National Science Foundation","au":"Bieri, Marco","y":2021,"vn":"BMJ Open","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmjopen-2020-047386","cx":"Grant","co":["Reliability-noise","Interventions"],"sg":["panel-consensus","funding-decision"],"x":0,"st":"simulation-methodological","oast":"gold","cby":10,"self":false,"s":"Simulation study at the Swiss National Science Foundation comparing remote score-based evaluation of postdoctoral fellowship applications with face-to-face panel review, finding the simplified procedure agreed with official funding decisions in over 80% of cases while halving costs.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"no-statement","cau":""},{"k":"W53JXMKE","t":"Making peer review evidence-based: It’s time to open the \"black box\"","au":"Dunleavy, Daniel J.","y":2022,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31222/osf.io/9ka8q","cx":"Journal","co":["Reliability-noise","Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":2,"self":false,"s":"A research note arguing that journal peer review in social work is opaque and of unproven rigour, and proposing a research agenda on open peer review to make the process more transparent, accountable and evidence-based.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"VKJXIFFN","t":"Addressing Racial Disparities in NIH Funding","au":"Comfort, Nicole","y":2021,"vn":"Journal of Science Policy & Governance","ty":"journalArticle","tyl":"Journal article","doi":"10.38126/jspg180408","cx":"Grant","co":["Bias","Interventions"],"sg":["funding-decision"],"x":0,"st":"guideline-policy","oast":"diamond","cby":3,"self":false,"s":"A policy report evaluating NIH proposals to reduce racial bias in grant peer review and recommending relaxed paylines, prioritisation of topics relevant to under-represented investigators, and a modified funding lottery.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"ZE8HW5W5","t":"Quality and peer review of research: an adjudicating role for editors.","au":"Newton, Douglas P","y":2010,"vn":"Accountability in research","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/08989621003791945","cx":"Journal","co":["Reliability-noise","Bias"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"qualitative","oast":"green","cby":78,"self":false,"s":"A commentary and analysis of flaws in journal peer review, organising them into situational, personal, social, and ethical factors, and arguing that critically engaged editors who adjudicate reviews and guard against bias offer the best remedy.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"VN36QJYZ","t":"The foundation and consequences of gender bias in grant peer review processes.","au":"Morgan, Rosemary","y":2018,"vn":"CMAJ : Canadian Medical Association journal = journal de l'Association medicale canadienne","ty":"journalArticle","tyl":"Journal article","doi":"10.1503/cmaj.180188","cx":"Grant","co":["Bias"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":24,"self":false,"s":"A commentary arguing that gender bias in grant peer review, evidenced by a Canadian health research funding study showing female applicants received lower scores despite equivalent track records, reflects wider systemic gender inequity in academia.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"YR6REJFY","t":"Gender inequities in medical research funding is driving an exodus of women from Australian STEMM academia.","au":"Borger, Jessica G","y":2022,"vn":"Immunology and cell biology","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/imcb.12568","cx":"Grant","co":["Bias"],"sg":["funding-decision"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":20,"self":false,"s":"A commentary arguing that gender bias in grant peer review, not lower application rates, drives women's poorer funding outcomes in Australian medical research, cumulatively pushing women out of STEMM academia and reducing workforce diversity.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://www.nhmrc.gov.au/funding/data-research/outcomes","ca":"no-statement","cau":""},{"k":"ZSH388T5","t":"Gender and geographical bias in the editorial decision-making process of biomedical journals: a case-control study.","au":"Gayet-Ageron, Angèle","y":2025,"vn":"BMJ evidence-based medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmjebm-2024-113083","cx":"Journal","co":["Bias"],"sg":["funding-decision"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":3,"self":false,"s":"A case-control study of 6,213 biomedical manuscripts found no gender bias in editorial decisions after peer review, but lower acceptance rates for authors from less wealthy regions and countries, while greater geographical and economic diversity among lead authors increased acceptance.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"on-request","cau":""},{"k":"WCVGJTSZ","t":"Geographical diversity of peer reviewers shapes author success.","au":"Zumel Dumlao, James M","y":2025,"vn":"Proceedings of the National Academy of Sciences of the United States of America","ty":"journalArticle","tyl":"Journal article","doi":"10.1073/pnas.2507394122","cx":"Journal","co":["Bias","Interventions"],"sg":["reviewer-selection","individual-scoring"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":10,"self":false,"s":"An observational study of over 200,000 submissions to 60 physics journals showing that reviewers favour authors from their own country, giving well-represented nations a structural advantage that anonymisation does not remove, and arguing for a more geographically diverse reviewer pool.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"","ca":"open","cau":""},{"k":"W24XKFMR","t":"Does a paper's country of origin affect the length of the review process?","au":"Valkonen, Laura","y":2012,"vn":"Cortex: A Journal Devoted to the Study of the Nervous System and Behavior","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.cortex.2012.05.015","cx":"Journal","co":["Bias"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"closed","cby":0,"self":false,"s":"An editorial analysing review times for 136 journal articles in Cortex, finding no evidence that authors' country of origin affects the length of peer review.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"YIN5T6G3","t":"Bibliometrics as a performance measurement tool for research evaluation: The case of research funded by the National Cancer Institute of Canada.","au":"Campbell, David","y":2010,"vn":"American Journal of Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/1098214009354774","cx":"Grant","co":["Validity"],"sg":["post-award-validity"],"x":0,"st":"observational-quantitative","oast":"closed","cby":158,"self":false,"s":"A case study using bibliometric indicators to evaluate research funded by the National Cancer Institute of Canada, showing that they can assess whether grant peer review selected outstanding scientists and whether funding improved researchers' scientific performance.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"ZYMGMAKP","t":"Criteria Used by a Peer Review Committee for Selection of Research Fellows: A Boolean Probit Analysis.","au":"Bornmann, Lutz","y":2005,"vn":"International Journal of Selection and Assessment","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/j.1468-2389.2005.00326.x","cx":"Grant","co":["general-systemic"],"sg":["panel-consensus","funding-decision"],"x":0,"st":"observational-quantitative","oast":"bronze","cby":17,"self":false,"s":"An empirical study using Boolean probit analysis of a biomedical research foundation's fellowship peer review, finding that applications are approved only when all three stated criteria (applicant achievements, project originality, and host laboratory standing) are rated positively.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"XJZJLZUX","t":"Evaluating scientific papers and research proposals.","au":"Disney, Henry","y":2001,"vn":"Cortex: A Journal Devoted to the Study of the Nervous System and Behavior","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/S0010-9452(08)70600-2","cx":"General","co":["Validity"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":1,"self":false,"s":"A commentary criticising the use of journal status to score papers and the demand by funding agencies for highly specific predicted results, arguing these practices undervalue non-Western journals, ignore long-term impact, and push researchers towards formulaic work or subterfuge.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"YSR8YJQI","t":"Reexamining the NSF GRFP review process","au":"Evande R.","y":2023,"vn":"Journal of Biological Chemistry","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.jbc.2023.103515","cx":"Grant","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"gold","cby":0,"self":false,"s":"Survey-based policy analysis of the NSF Graduate Research Fellowship Program, gathering applicant and reviewer experiences to inform three recommendations for making the fellowship's peer-review process fairer and more equitable.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"WC3QYWZ4","t":"Predominance of animal-based expertise may bias NIH neuroscience grant review: A pilot study with implications for non-animal methodologies","au":"Trunnell E.R.","y":2025,"vn":"bioRxiv","ty":"journalArticle","tyl":"Journal article","doi":"10.1101/2025.02.28.640877","cx":"Grant","co":["Bias"],"sg":["reviewer-selection","funding-decision"],"x":0,"st":"observational-quantitative","oast":"green","cby":3,"self":false,"s":"A pilot study of NIH neuroscience grant review finding that animal-based reviewer expertise predominates and correlates positively with funding animal research and negatively with funding non-animal methods.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"10.17605/OSF.IO/FKPZS","ca":"no-statement","cau":""},{"k":"YM46CVWT","t":"Research funders should be more transparent: a plea for open applications","au":"","y":2022,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31219/osf.io/37tsu","cx":"Grant","co":["Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":1,"self":false,"s":"A mini-review and opinion piece arguing that research funders should adopt open grant applications, including publishing all submitted proposals, review reports, decision rationales and reviewer identities, to improve fairness, collaboration and trust in funding allocation.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"ZWJJHGJC","t":"An ethnographic analysis virtual peer review panels","au":"","y":2023,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31235/osf.io/fbkr6","cx":"Grant","co":["Reliability-noise"],"sg":["panel-consensus"],"x":0,"st":"qualitative","oast":"closed","cby":0,"self":false,"s":"An ethnographic observation study of four virtual grant review panels at the Research Council of Norway during 2020, finding that online deliberation saved time and money but made reaching consensus harder, increased chairs' workload, and disadvantaged new panel members.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"Y34R4SM5","t":"The Case for Using Educational Scholarship to Improve Peer Review","au":"","y":2024,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31234/osf.io/zak4f","cx":"Journal","co":["Interventions"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":0,"self":false,"s":"A commentary arguing that journal peer review's inconsistent and unconstructive reviewer comments could be improved by applying evidence-based feedback interventions from educational scholarship.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"XWC5YURW","t":"Professionalising peer reviewers: maturation of an old idea","au":"","y":2024,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31219/osf.io/zcgvr","cx":"Journal","co":["Bias","Interventions"],"sg":["reviewer-selection"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":0,"self":false,"s":"An essay proposing that journal peer review be professionalised through paid, certified full-time reviewers employed by research institutions, discussing scenarios for combining professional and academic referees, staffing ratios, training, funding and effects on review quality and integrity.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"VR97RI38","t":"Integrating Artificial Intelligence into Scholarly Peer Review: A Framework for Enhancing Efficiency and Quality","au":"","y":2025,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31219/osf.io/s764u_v1","cx":"Journal","co":["Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":0,"self":false,"s":"A framework paper proposing how artificial intelligence can be integrated into journal peer review, combining AI and human expertise with the aim of improving the efficiency, consistency, comprehensiveness and quality of manuscript evaluation.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"WMHFBK5H","t":"Hidden Prompts in Manuscripts Exploit AI-Assisted Peer Review","au":"","y":2025,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31234/osf.io/nea5u_v1","cx":"Journal","co":["Bias","Interventions"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":8,"self":false,"s":"A commentary analysing hidden prompts embedded in arXiv manuscripts to manipulate AI-assisted journal peer review, classifying four types of prompt injection as a novel form of research misconduct and calling for technical screening and harmonised publisher policies on generative AI.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"Z24XRKEM","t":"Good practice in the use of machine learning & AI by research funding organisations: insights from a workshop series","au":"Holm, Jon","y":2022,"vn":"Figshare","ty":"report","tyl":"Report","doi":"10.6084/m9.figshare.21710015.v1","cx":"Grant","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"green","cby":0,"self":false,"s":"A working paper distilling insights from a workshop series on good practice for research funding organisations using machine learning and AI, including in grant review and decision-making processes.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"VYVGQ37Z","t":"The Future of Peer Review: A system-wide perspective","au":"Pinfield, Stephen","y":2026,"vn":"Figshare","ty":"report","tyl":"Report","doi":"10.6084/m9.figshare.32685351.v1","cx":"General","co":["Interventions"],"sg":["whole-process"],"x":0,"st":"qualitative","oast":"green","cby":0,"self":false,"s":"A report from the Research on Research Institute offering a system-wide perspective on the future of peer review, examining how the practices of journals, funders and other actors might evolve across the research ecosystem.","pg":"not-applicable","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"ZPUABMMS","t":"Policy proposals to promote inclusion of caregivers in the research funding system","au":"Torres, Isabel","y":2024,"vn":"Frontiers in Education","ty":"journalArticle","tyl":"Journal article","doi":"10.3389/feduc.2024.1472517","cx":"Grant","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"guideline-policy","oast":"gold","cby":3,"self":false,"s":"A policy review of gender bias in STEMM grant funding, arguing that the pandemic worsened inequities for caregivers, especially mothers, and proposing actionable policies to make funding systems more inclusive.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"Y7ME8VV7","t":"Double blind reviews in software engineering venues: Practicability, promises and perils","au":"Beller, Moritz","y":2016,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.7287/peerj.preprints.1757v1","cx":"Journal","co":["Bias","Interventions"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":0,"self":false,"s":"A mixed-methods study of software engineering conference peer review, using surveys, expert interviews and author-guessing estimates, finding that double-blind reviewing could be introduced at lower cost than generally believed and is favoured by most of the community.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"http://sback.it/dblind","ca":"open","cau":"http://sback.it/dblind"},{"k":"ZA5D24NV","t":"A new approach to peer review assessments: Score, then rank","au":"Gallo, Stephen A.","y":2022,"vn":"Research Square","ty":"journalArticle","tyl":"Journal article","doi":"10.21203/rs.3.rs-2198949/v1","cx":"Grant","co":["Reliability-noise","Interventions"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"simulation-methodological","oast":"green","cby":0,"self":false,"s":"A methods paper proposing that grant reviewers supply rankings of top proposals alongside scores, with a statistical model combining both into integrated scores, shown on real and simulated data to better distinguish similarly scored proposals for funding decisions.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"10.6084/m9.figshare.20505747","ca":"no-statement","cau":"https://cran.r-project.org/package=rankrate"},{"k":"VUDVZI2F","t":"Are numerical scores important for grant proposals' evaluation? A cross sectional study","au":"Buljan, Ivan","y":2023,"vn":"F1000Research","ty":"journalArticle","tyl":"Journal article","doi":"10.12688/f1000research.139743.1","cx":"Grant","co":["Interventions"],"sg":["individual-scoring","panel-consensus"],"x":0,"st":"observational-quantitative","oast":"gold","cby":0,"self":false,"s":"Observational study of Marie Skłodowska-Curie grant reviews, finding that removing numerical scores from individual evaluations in 2020 was not associated with changes in the linguistic characteristics of reviewers' written comments compared with 2019.","pg":"preregistered","pgr":"OSF","pgid":"https://osf.io/t84ba","da":"open","dau":"https://osf.io/6bpvu/?view_only=","ca":"open","cau":"https://osf.io/6bpvu/?view_only="},{"k":"YD9UFWAY","t":"NIH's peer review stands up to scrutiny","au":"Mervis, Jeffrey","y":2015,"vn":"Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1126/science.348.6233.384","cx":"Grant","co":["Validity"],"sg":["post-award-validity"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":5,"self":false,"s":"An analysis of NIH grant peer review finding that better-scored proposals subsequently produce more publications and citations.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"Z2G4LSCA","t":"Chapter 5: Reviewing the Reviewers: Training Neural Networks to Read Peer Review Reports","au":"Eve, Martin Paul","y":2022,"vn":"Bielefeld University Press eBooks","ty":"book","tyl":"Book","doi":"10.1515/9783839455845-006","cx":"Journal","co":["general-systemic"],"sg":["individual-scoring"],"x":0,"st":"book-monograph","oast":"closed","cby":0,"self":false,"s":"A book chapter describing a unique database of peer review reports at the journal PLOS ONE and an attempt to train a multi-label neural text classifier on them, which performed poorly but highlights future possibilities and risks of machine-reading review archives.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"XVXGZW69","t":"Rethinking resource allocation in science","au":"Bollen, Johan","y":2019,"vn":"Ecology and Society","ty":"journalArticle","tyl":"Journal article","doi":"10.5751/es-11005-240329","cx":"Grant","co":["Reliability-noise","Bias","Validity","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":0,"self":false,"s":"A perspective piece arguing that grant proposal peer review is struggling to cope with the growing gap between funding demand and supply, and calling for a rethink of how research resources are allocated.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"VU5PYHEV","t":"The Many Dilemmas of Grant Peer Review","au":"Langfeldt, Liv","y":2024,"vn":"SpringerBriefs in political science","ty":"book","tyl":"Book","doi":"10.1007/978-3-031-69580-3_5","cx":"Grant","co":["Reliability-noise","Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"book-monograph","oast":"hybrid","cby":1,"self":false,"s":"A book chapter reviewing the aims and inherent dilemmas of grant peer review, including distrust, reviewer fatigue and low success rates, and offering advice on how funders can handle these tensions.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"YKBFB3V5","t":"Science rules! A qualitative study of scientists’ approaches to grant lottery","au":"Philipps, Axel","y":2020,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvaa027","cx":"Grant","co":["Validity","Interventions"],"sg":["funding-decision"],"x":0,"st":"qualitative","oast":"hybrid","cby":17,"self":false,"s":"A qualitative interview study of natural scientists in Germany exploring attitudes to allocating grant funding by lottery, finding openness to trying lotteries but opposition to pure randomisation, with most favouring a combination of lottery and peer review.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"ZYWQI8FV","t":"Objective and Numerical Method of Finding Importance of Medical Research and Prioritizing Grants and Publications","au":"Dyczkowski, Jerzy","y":2020,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"","cx":"General","co":["Validity","Interventions"],"sg":["proposal-quality","funding-decision"],"x":0,"st":"simulation-methodological","oast":"","cby":0,"self":false,"s":"A methodological paper proposing an objective numerical measure of medical research value based on reduced disease burden to prioritise grants and journal publications, increase transparency and allow peer reviewers to focus on scientific integrity.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"X8NPAVC9","t":"Prediction of SMEs’ R&D performances by machine learning for project selection","au":"Yoo, Hyoung Sun","y":2023,"vn":"Scientific Reports","ty":"journalArticle","tyl":"Journal article","doi":"10.1038/s41598-023-34684-w","cx":"Grant","co":["Validity"],"sg":["funding-decision","post-award-validity"],"x":0,"st":"observational-quantitative","oast":"gold","cby":15,"self":false,"s":"An applied machine learning study using 1771 South Korean government-funded SME R&D projects to predict success, commercialisation and patenting, finding the models more precise than qualitative expert evaluation and proposing they supplement expert scores in grant selection.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"no-statement","cau":""},{"k":"XEDBKFQC","t":"Toward predicting research proposal success","au":"Boyack, Kevin W.","y":2017,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-017-2609-2","cx":"Grant","co":["Bias","Validity"],"sg":["proposal-quality","funding-decision"],"x":0,"st":"observational-quantitative","oast":"green","cby":39,"self":false,"s":"An empirical study of 369 NIH R01 grant proposals using citation and discourse analysis, finding that clearer writing and greater topical overlap between a proposal's references and the applicant's prior publications are associated with funding success.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"VWS5K2BN","t":"Social Valuation Across Multiple Audiences: The Interplay of Ability and Identity Judgments","au":"Fini, Riccardo","y":2017,"vn":"Academy of Management Journal","ty":"journalArticle","tyl":"Journal article","doi":"10.5465/amj.2016.0661","cx":"Grant","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"green","cby":51,"self":false,"s":"A theory-and-evidence study of grant peer review at a research university, showing that applicants' prior recognition from external audiences raises peer evaluations up to a point but lowers them beyond it, producing an inverted U-shaped effect across 9,502 scientists.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"W68NZPTA","t":"The peer review process: a primer forJNISreaders","au":"Hirsch, Joshua A","y":2015,"vn":"Journal of NeuroInterventional Surgery","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/neurintsurg-2015-011781","cx":"Journal","co":["Bias"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":11,"self":false,"s":"An educational primer for readers of the Journal of NeuroInterventional Surgery explaining how journal peer review works and summarising key points from the wider literature on the process.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"WN5CRXC2","t":"Modern applications of cross-classified random effects models in social and behavioral research: Illustration with R package PLmixed","au":"Huang, Sijia","y":2022,"vn":"Frontiers in Psychology","ty":"journalArticle","tyl":"Journal article","doi":"10.3389/fpsyg.2022.976964","cx":"Analogue","co":["Reliability-noise"],"sg":["whole-process"],"x":0,"st":"simulation-methodological","oast":"gold","cby":5,"self":false,"s":"A methodological tutorial presenting cross-classified random effects models as a flexible framework for social and behavioural data, showing how many existing models are special cases and demonstrating their estimation with the free R package PLmixed.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"","ca":"open","cau":""},{"k":"VPZMMF5B","t":"Cross-classified models in I/O psychology","au":"Claus, Anna M.","y":2020,"vn":"Journal of Vocational Behavior","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.jvb.2020.103447","cx":"Analogue","co":["Reliability-noise","Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":20,"self":false,"s":"A methodological tutorial showing how cross-classified models handle data clustered by overlapping factors, illustrated with assessment centre ratings where scores are nested in both participants and raters, allowing sources of rater variance to be studied.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://doi.org/10.1016/j.jvb.2020.103447","ca":"open","cau":"https://doi.org/10.1016/j.jvb.2020.103447"},{"k":"VZZHSWUF","t":"Gender inequality in applying research project and funding","au":"Liao, Chien Hsiang","y":2022,"vn":"Journal of Information Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/01655515221097861","cx":"Grant","co":["Bias"],"sg":["funding-decision"],"x":0,"st":"observational-quantitative","oast":"closed","cby":14,"self":false,"s":"Empirical analysis of 1465 grant-funded research projects (2015-2021) testing gender effects in funding, finding no gender gap in Matthew-effect advantages but showing that institutional reputation and past performance boost future funding for male applicants only.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"XHKV6QPM","t":"Grants Peer Review in Theory and Practice","au":"Chubin, Daryl E.","y":1994,"vn":"Evaluation Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/0193841x9401800103","cx":"Grant","co":["general-systemic"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":51,"self":false,"s":"A conceptual review examining the assumptions, criticisms and practical operation of grant peer review at NSF and NIH, concluding with suggestions for reforming how peer evaluations inform funding decisions.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"WWP3Q9HN","t":"Arbitrariness of peer review: A Bayesian analysis of the NIPS experiment","au":"François, Olivier","y":2015,"vn":"arXiv (Cornell University)","ty":"journalArticle","tyl":"Journal article","doi":"10.48550/arxiv.1507.06411","cx":"Journal","co":["Reliability-noise"],"sg":["panel-consensus","funding-decision"],"x":0,"st":"simulation-methodological","oast":"green","cby":12,"self":false,"s":"A Bayesian reanalysis of the 2014 NIPS conference experiment, in which duplicated reviews showed 60% disagreement on accepted papers, estimating that around 56% of submissions met basic quality criteria and suggesting a higher acceptance rate would reduce arbitrariness.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"XTSCEH5T","t":"Reviewing the Test Reviews: Quality Judgments and Reviewer Agreements in the Mental Measurements Yearbook","au":"Hogan, Thomas P.","y":2021,"vn":"Applied Measurement in Education","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/08957347.2021.1890742","cx":"General","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":1,"self":false,"s":"An empirical study of 100 reviews from the 19th Mental Measurements Yearbook, finding that quality judgments ranged uniformly from very good to very bad and that agreement between independent reviewers of the same test was positive but weak.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"YNVIY6T2","t":"Peer review in practice","au":"Dorothy, Butchard","y":2017,"vn":"UCL Press eBooks","ty":"book","tyl":"Book","doi":"10.14324/111.9781911307679.15","cx":"General","co":["Reliability-noise","Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"book-monograph","oast":"green","cby":6,"self":false,"s":"A report reviewing the history, current practices, and innovative alternative models of peer review for journal publishing and institutional assessment, with particular focus on the Arts and Humanities, itself offered for post-publication review.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"ZZLRBXG8","t":"Gender Differences in Grant Submissions across Science and Engineering Fields at the NSF","au":"Rissler, Leslie J.","y":2020,"vn":"BioScience","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/biosci/biaa072","cx":"Grant","co":["Bias"],"sg":["funding-decision"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":47,"self":false,"s":"An observational analysis of NSF grant submissions by gender across six research directorates, finding women are funded as often as men but submit proposals less often than expected in all fields except engineering, partly reflecting lower self-reported research focus.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"","ca":"no-statement","cau":""},{"k":"XKNN8HWX","t":"Authors anonymous?","au":"Benwell, R","y":1979,"vn":"Physics Bulletin","ty":"journalArticle","tyl":"Journal article","doi":"10.1088/0031-9112/30/7/006","cx":"Journal","co":["Bias","Interventions"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":95,"self":false,"s":"A short commentary proposing that journals withhold authors' identities from referees to reduce bias against particular authors or institutions in peer review.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"XXJERAKS","t":"A Computer Program for Determining the Reliability of Dimensionally Scaled Data when the Numbers and Specific Sets of Examiners may Vary at Each Assessment","au":"Ciccheti, Domenic V.","y":1988,"vn":"Educational and Psychological Measurement","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/0013164488483020","cx":"Analogue","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"closed","cby":51,"self":false,"s":"A software paper using a variant of the intraclass correlation coefficient to calculate reliability for scaled ratings when the number and identities of examiners vary between assessments.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"on-request","cau":""},{"k":"ZHGU8LLV","t":"Testing the Normal Approximation and Minimal Sample Size Requirements of Weighted Kappa When the Number of Categories is Large","au":"Cicchetti, Domenic V.","y":1981,"vn":"Applied Psychological Measurement","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/014662168100500114","cx":"Analogue","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"green","cby":187,"self":false,"s":"Simulation study showing that the normal approximation for weighted kappa's standard error holds when ratings use up to ten categories, and that the minimum sample size needed can be approximated by twice the square of the number of categories.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"WAA8LCI8","t":"The Reliability of Dichotomous Judgments: Unequal Numbers of Judges per Subject","au":"Fleiss, Joseph L.","y":1979,"vn":"Applied Psychological Measurement","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/014662167900300410","cx":"Analogue","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"green","cby":190,"self":false,"s":"A statistical paper proposing a kappa-like reliability measure for dichotomous judgements by unequal numbers of judges, showing its correspondence to an intraclass correlation coefficient and presenting a significance test.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"XTWK9EL7","t":"Interrater reliability and agreement of subjective judgments.","au":"Tinsley, Howard E. A.","y":1975,"vn":"Journal of Counseling Psychology","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/h0076640","cx":"Analogue","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":861,"self":false,"s":"A methodological review clarifying the distinction between interrater reliability and agreement, and advising counselling psychology researchers on choosing appropriate indexes, including the intraclass correlation, according to measurement level and type of replication.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"XVMP7GIU","t":"The Kappa Coefficient of Agreement for Multiple Observers When the Number of Subjects is Small","au":"Gross, Shulamith T.","y":1986,"vn":"Biometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.2307/2530702","cx":"Analogue","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"closed","cby":80,"self":false,"s":"A statistical paper proposing a modified kappa coefficient for multiple observers when subjects are few, providing significance tests and confidence intervals and allowing unequal numbers of raters per subject.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"Z7YJFU77","t":"Paradigm Development and Particularism: Journal Publication in Three Scientific Disciplines","au":"Pfeffer, Jeffrey","y":1977,"vn":"Social Forces","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/sf/55.4.938","cx":"Journal","co":["Bias"],"sg":["reviewer-selection","funding-decision"],"x":0,"st":"observational-quantitative","oast":"closed","cby":195,"self":false,"s":"A sociological study of journal publication in three scientific disciplines, examining whether particularistic factors such as authors' institutional prestige influence acceptance more in fields with less developed paradigms.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"ZG9A9VH7","t":"Guarding the Guardians: A Conference on Editorial Peer Review","au":"Rennie, Drummond","y":1986,"vn":"JAMA","ty":"journalArticle","tyl":"Journal article","doi":"10.1001/jama.1986.03380170107031","cx":"Journal","co":["general-systemic"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":143,"self":false,"s":"An editorial announcing a conference on editorial peer review at journals, arguing that although the biomedical publication system is well described, the peer-review process itself has received inadequate scientific scrutiny and deserves syst","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"ZH29M6TV","t":"Bias, Controversy, and Abuse in the Study of the Scientific Publication System","au":"Mahoney, M. J.","y":1990,"vn":"Science Technology & Human Values","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/016224399001500105","cx":"Journal","co":["Bias"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":47,"self":false,"s":"A commentary criticising Epstein's covert experiment on journal peer review, arguing that despite raising real concerns about bias in scientific publication, the study's methods may have harmed rather than helped efforts to understand and improve scholarly communication.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"WJIKJ2M5","t":"Problems Identified by Secondary Review of Accepted Manuscripts","au":"Garfunkel, Joseph M.","y":1990,"vn":"JAMA","ty":"journalArticle","tyl":"Journal article","doi":"10.1001/jama.1990.03440100077011","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"closed","cby":72,"self":false,"s":"A journal peer-review study re-reviewing accepted manuscripts with new referees, finding frequent but often dissimilar criticisms, while 80% were still recommended for publication and editors rarely disagreed on acceptance decisions.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"ZE7G284F","t":"Judgment of counseling process: Reliability, agreement, and error.","au":"Lawlis, G. Frank","y":1972,"vn":"Psychological Bulletin","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/h0032935","cx":"Analogue","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"closed","cby":174,"self":false,"s":"A methodological paper distinguishing rater agreement from reliability in judgements of counselling sessions, proposing a chi-square analysis that lets researchers set their own criterion and compare observed agreement frequencies with expected probabilities.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"WV99LFB4","t":"2 x 2 Kappa Coefficients: Measures of Agreement or Association","au":"Blöch, D.","y":1989,"vn":"Biometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.2307/2532052","cx":"Analogue","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"closed","cby":293,"self":false,"s":"A statistical paper distinguishing kappa coefficients for agreement from those for association, deriving separate models and asymptotic sampling results and evaluating the agreement model in small samples.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"W48D45BZ","t":"We need research on what constitutes good journal papers--and good editing--not guesswork on how to improve manuscriptsp.","au":"Crandall, Rick","y":1987,"vn":"American Psychologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1037//0003-066x.42.4.407.b","cx":"Journal","co":["Reliability-noise","Validity"],"sg":["proposal-quality","individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":36,"self":false,"s":"A commentary responding to advice on improving manuscript writing, arguing that journal editorial review has been shown to lack validity and reliability, and calling for empirical research on what makes good papers and editing rather than unsubstantiated guesswork.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"YGF3URKN","t":"Issues Surrounding Publishing","au":"Coniam, David","y":2022,"vn":"Springer briefs in education","ty":"book","tyl":"Book","doi":"10.1007/978-981-19-3065-2_2","cx":"Journal","co":["Reliability-noise"],"sg":["whole-process"],"x":0,"st":"book-monograph","oast":"closed","cby":0,"self":false,"s":"A book chapter surveying issues in academic publishing, including the Research Assessment Exercise, journal peer-review practices, tenure and pressure to publish, open access, pay-to-publish models, and the skill of academic writing.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"ZCSN6RNZ","t":"External Reviewers’ Influence on Citations of Articles Published in Pedagogical Business Journals: Holding Acceptance Rate Constant","au":"Alam, Arshad","y":2016,"vn":"SSRN Electronic Journal","ty":"journalArticle","tyl":"Journal article","doi":"","cx":"Journal","co":["Validity"],"sg":["reviewer-selection","post-award-validity"],"x":0,"st":"observational-quantitative","oast":"green","cby":0,"self":false,"s":"An empirical study of 560 articles in 28 pedagogical business journals showing, with acceptance rate held constant, that citation counts vary by business field and rise as the number of external peer reviewers increases from one to two.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"WZVF5TQJ","t":"Do grant proposal texts matter for funding decisions? A field experiment","au":"Şimşek, Müge","y":2024,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-024-04968-7","cx":"Grant","co":["Reliability-noise"],"sg":["proposal-quality","individual-scoring"],"x":1,"st":"experimental-rct","oast":"hybrid","cby":9,"self":false,"s":"A field experiment with the Dutch Research Council in an early-career grant competition, finding that panellists shown only a CV and one-paragraph summary ranked proposals no differently from those shown full proposal texts.","pg":"preregistered","pgr":"AsPredicted","pgid":"https://aspredicted.org/md45e.pdf","da":"open","dau":"","ca":"open","cau":"https://doi.org/10.17605/OSF.IO/TRMEB"},{"k":"XY9T7EXT","t":"How to avoid borrowed plumes in academia","au":"Osterloh, Margit","y":2019,"vn":"Research Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.respol.2019.103831","cx":"Journal","co":["Reliability-noise","Validity","Interventions"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"closed","cby":98,"self":false,"s":"A policy essay arguing that journal rankings persist as evaluation tools because skewed citation distributions benefit most authors, and proposing that","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"YWD3GJ8R","t":"An Evaluation of the Peer Review Process at Angewandte Chemie","au":"Daniel, Hans‐Dieter","y":1993,"vn":"Angewandte Chemie International Edition in English","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/anie.199302341","cx":"Journal","co":["Reliability-noise","Bias","Validity"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":21,"self":false,"s":"An empirical evaluation of the journal peer-review system at Angewandte Chemie, reporting selected findings from Daniel's anonymised study of referees' judgements on submitted manuscripts.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"ZX28CA2C","t":"The Measurement of Observer Agreement for Categorical Data","au":"Landis, J. Richard","y":1977,"vn":"Biometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.2307/2529310","cx":"Analogue","co":["Reliability-noise","Bias"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"green","cby":79617,"self":false,"s":"A statistical paper developing tests for observer bias and generalised kappa statistics to measure agreement in multivariate categorical data, illustrated with clinical diagnoses.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"ZGLDA6ZW","t":"Reliability of a Structured Method of Selecting Abstracts for a Plastic Surgical Scientific Meeting","au":"Steen, Lydia P. E. van der","y":2003,"vn":"Plastic & Reconstructive Surgery","ty":"journalArticle","tyl":"Journal article","doi":"10.1097/01.prs.0000061092.88629.82","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring","post-award-validity"],"x":0,"st":"observational-quantitative","oast":"closed","cby":28,"self":false,"s":"An evaluation of structured abstract selection for a European plastic surgery conference, finding that participants' ratings of presentations were reliable but failing to demonstrate the validity of abstract selection or reviewer scores as indicators of presentation quality.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"YQ73QQX4","t":"A review of ideas and strategies to improve scientific research and its dissemination in life and social sciences","au":"Almgren, Hannes","y":2020,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31222/osf.io/9hg3j","cx":"General","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":0,"self":false,"s":"A literature review examining reproducibility, questionable research practices and analytical flexibility in life and social sciences, and proposing reforms including preregistration, transparent reporting, broader statistical methods, continuous peer review and incentives favouring research quality over quantity.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"VYR3JTDF","t":"A Website System for Communicating Psychological Science","au":"Diener, Ed","y":2017,"vn":"Perspectives on Psychological Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/1745691616685864","cx":"Journal","co":["Bias","Validity","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"bronze","cby":0,"self":false,"s":"A proposal paper recommending an open website platform to complement journal peer review in psychology, where all behavioural scientists could publish and review articles, with merit judged by community ratings and citations.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"XV9APMTQ","t":"Publizieren in der Deutschdidaktik","au":"Funke, Reinold","y":2021,"vn":"Zeitschrift für Sprachlich-Literarisches Lernen und Deutschdidaktik","ty":"journalArticle","tyl":"Journal article","doi":"10.46586/slld.z.2021.8770","cx":"Journal","co":["Reliability-noise","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"diamond","cby":1,"self":false,"s":"A discussion piece examining publishing conditions in German-language education research (Deutschdidaktik), covering demand versus supply orientation, internationality and interdisciplinarity, journal peer review as quality assurance, and citation and plagiarism practices.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"Y3RTHFB6","t":"Improving what is published: Toward an evidence-based framework for manuscript review.","au":"Bornstein, Robert F.","y":2021,"vn":"Archives of Scientific Psychology","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/arc0000076","cx":"Journal","co":["Reliability-noise","Bias","Validity"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":4,"self":false,"s":"A conceptual review article proposing an evidence-based framework for journal manuscript review, reframing it as both a signal-detection task and a complex social interaction among authors, reviewers and editors, and outlining practical steps for improvement.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"XIB72TAG","t":"Is Peer Review Fit for Purpose?","au":"Tight, Malcolm","y":2022,"vn":"","ty":"book","tyl":"Book","doi":"10.1007/978-3-030-75263-7_10","cx":"General","co":["Reliability-noise","Validity"],"sg":["whole-process"],"x":0,"st":"book-monograph","oast":"gold","cby":6,"self":false,"s":"A book chapter critically examining whether peer review across higher education remains fit for purpose, questioning its effectiveness and possible alternatives in contexts including refereed journal articles, doctoral degree assessment and the UK Research Excellence Framework.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"WGB2HQAZ","t":"Difficulty of recruiting reviewers predicts review scores and editorial decisions at six journals of ecology and evolution","au":"Fox, Charles W.","y":2017,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-017-2489-5","cx":"Journal","co":["Reliability-noise","Bias","Validity"],"sg":["reviewer-selection","individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"closed","cby":28,"self":false,"s":"An analysis of peer review at six ecology and evolution journals, finding that manuscripts for which reviewers were harder to recruit received less positive review scores and were more likely to be rejected.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"W5SHTMNM","t":"ACADEMIC TALENT SELECTION IN GRANT REVIEW PANELS","au":"Arensbergen, P. van","y":2014,"vn":"Data Archiving and Networked Services (DANS)","ty":"book","tyl":"Book","doi":"","cx":"Grant","co":["Reliability-noise","Bias"],"sg":["individual-scoring","panel-consensus","funding-decision"],"x":0,"st":"book-monograph","oast":"closed","cby":7,"self":false,"s":"An empirical study of 897 career grant applications examining how review panels select talent, finding no clear boundaries of excellence, a dominant role for the interview phase, little influence of external peer reviews, and no gender bias in decisions.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"X8UKNNYN","t":"Behind Door #3: The Hard-to-Please Grant Reviewer.","au":"Miner, Jeremy T.","y":2011,"vn":"Research management review","ty":"journalArticle","tyl":"Journal article","doi":"","cx":"Grant","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":3,"self":false,"s":"A practical article for research administrators explaining why grant reviewers at NIH, NSF and the US Department of Education often give discrepant ratings, and offering strategies to help applicants interpret conflicting feedback and decide whether to revise, resubmit or change direction.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"YJW7DQHS","t":"Institutionalized biases in the award of research grants: a preliminary analysis revisiting the principle of accumulative advantage","au":"Viner, Neil","y":2003,"vn":"Research Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.respol.2003.09.005","cx":"Grant","co":["Bias"],"sg":["funding-decision"],"x":0,"st":"observational-quantitative","oast":"closed","cby":101,"self":false,"s":"An empirical analysis of grant submission data from a UK Research Council, finding that success is linked to advantages such as membership of the peer-review pool, departmental standing and track record, with apparent biases against women and non-white applicants.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"ZPYJF2B9","t":"Assessing the potential of a Bayesian ranking as an alternative to consensus meetings for decision making in research funding: A Case Study of Marie Skłodowska-Curie Actions","au":"Heyard, Rachel","y":2024,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31222/osf.io/c4ujg","cx":"Grant","co":["Reliability-noise","Interventions"],"sg":["individual-scoring","panel-consensus","funding-decision"],"x":0,"st":"observational-quantitative","oast":"gold","cby":0,"self":false,"s":"A case study testing whether a Bayesian ranking of individual reviewer scores could replace consensus meetings in Marie Skłodowska-Curie grant evaluation, finding large discrepancies with panel outcomes and recommending pre-meeting scores be used only to triage out weak proposals.","pg":"preregistered","pgr":"OSF","pgid":"osf.io/cnurf","da":"open","dau":"https://gitlab.com/heyardr/msca-br-comparison","ca":"open","cau":"https://gitlab.com/heyardr/msca-br-comparison"},{"k":"X6QKSCWD","t":"Ponderings on peer review: Part I. Basic principles","au":"Seals, Douglas R.","y":2023,"vn":"American Journal of Physiology-Regulatory, Integrative and Comparative Physiology","ty":"journalArticle","tyl":"Journal article","doi":"10.1152/ajpregu.00062.2023","cx":"General","co":["Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":8,"self":false,"s":"A tutorial-style commentary introducing early-career researchers to the basic principles of peer review for grants and manuscripts, covering its goals, history, structure, reviewer responsibilities, common criticisms and proposed reforms, and ways to gain reviewing experience.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"VQRAA6KJ","t":"Peer Review for Journals as it Stands Today—Part 2","au":"Campanario, Juan Miguel","y":1998,"vn":"Science Communication","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/1075547098019004002","cx":"Journal","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":134,"self":false,"s":"Second part of a two-part literature review on journal peer review, covering research on fraud, favouritism, self-interest and double-blind review, and concluding that despite its problems peer review remains an essential quality control mechanism for science.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"WVBAFXRY","t":"Editors’ Requests of Peer Reviewers: A Study and a Proposal","au":"Frank, Erica","y":1996,"vn":"Preventive Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.1006/pmed.1996.0035","cx":"Journal","co":["Interventions"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":21,"self":false,"s":"A survey of reviewer forms and instructions from 71 top-cited US-based journals, finding substantial variation in what editors ask peer reviewers to assess and proposing a more standardised approach to eliciting reviews.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"XIQ6JFMS","t":"Fairness as Appropriateness","au":"Mallard, Grégoire","y":2009,"vn":"Science Technology & Human Values","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/0162243908329381","cx":"Grant","co":["Reliability-noise"],"sg":["panel-consensus"],"x":0,"st":"qualitative","oast":"green","cby":123,"self":false,"s":"An interview-based sociological study of eighty-one panellists on five multidisciplinary fellowship competitions, identifying four epistemological styles reviewers use and showing that fairness is defined as judging each proposal by the standards appropriate to its own discipline.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"Y2YMGN82","t":"An Exploratory Study of Statistical Assessment of Papers Published in the British Medical Journal","au":"Gardner, Martin J.","y":1990,"vn":"JAMA","ty":"journalArticle","tyl":"Journal article","doi":"10.1001/jama.1990.03440100061010","cx":"Journal","co":["Interventions"],"sg":["proposal-quality","individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":130,"self":false,"s":"An exploratory study of statistical review at the British Medical Journal, comparing checklist assessments of 45 papers at submission and after publication, finding statistical acceptability rose from 11% to 84% and concluding that statistical assessment is beneficial though improvable.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"WVWHEAHL","t":"Nepotism and sexism in peer-review","au":"Wennerås, Christine","y":1997,"vn":"Nature","ty":"journalArticle","tyl":"Journal article","doi":"10.1038/387341a0","cx":"Grant","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":1359,"self":false,"s":"A commentary reporting sexism and nepotism in peer review of postdoctoral fellowship applications and calling for an end to secrecy in fellowship evaluation.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"VIZQNEHX","t":"Are three heads better than two? How the number of reviewers and editor behavior affect the rejection rate","au":"Schultz, David M.","y":2009,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-009-0084-0","cx":"Journal","co":["Reliability-noise"],"sg":["reviewer-selection","funding-decision"],"x":0,"st":"observational-quantitative","oast":"green","cby":40,"self":false,"s":"Empirical analysis of 500 manuscripts submitted to the journal Monthly Weather Review, finding that rejection rates did not differ overall between manuscripts with two versus three reviewers, though outcomes varied by editor.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"XPFIBE54","t":"Influence of interdisciplinarity on peer-review and bibliometric evaluations in physics research","au":"Rinia, Ed J.","y":2001,"vn":"Research Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/s0048-7333(00)00082-2","cx":"General","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":108,"self":false,"s":"An empirical study of a nationwide peer-review evaluation of Dutch academic physics groups, finding no evidence that interdisciplinary research receives lower peer judgements or lower advanced bibliometric scores, though simple citation indicators correlate negatively with interdisciplinarity.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"YBW8PCPY","t":"Effect of open peer review on quality of reviews and on reviewers'recommendations: a randomised trial","au":"Rooyen, Susan van","y":1999,"vn":"BMJ","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmj.318.7175.23","cx":"Journal","co":["Interventions"],"sg":["individual-scoring"],"x":1,"st":"experimental-rct","oast":"bronze","cby":422,"self":false,"s":"A randomised trial at a journal finding that asking reviewers to reveal their identity to authors had no effect on review quality, publication recommendations or turnaround time, but made reviewers more likely to decline to review.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"ZI6TF964","t":"Equal opportunities in Canada","au":"Friesen, Henry G.","y":1998,"vn":"Nature","ty":"journalArticle","tyl":"Journal article","doi":"10.1038/34768","cx":"Grant","co":["Bias"],"sg":["funding-decision"],"x":0,"st":"observational-quantitative","oast":"bronze","cby":11,"self":false,"s":"A correspondence piece from MRC Canada reporting that women's approval rates in five fellowship competitions were significantly lower than men's, while operating grant competitions showed gender parity, and describing efforts to remove gender bias from fellowship assessment criteria.","pg":"not-codeable","pgr":"","pgid":"","da":"","dau":"","ca":"","cau":""},{"k":"ZVGM62KZ","t":"Referees’ Attitudes toward Open Peer Review and Electronic Transmission of Papers","au":"Melero, R.","y":2001,"vn":"Food Science and Technology International","ty":"journalArticle","tyl":"Journal article","doi":"10.1106/0mxd-ym6f-3lm6-g9eb","cx":"Journal","co":["Interventions"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":21,"self":false,"s":"A survey of 103 referees for the journal Food Science and Technology International, finding most preferred masked rather than open peer review and favoured electronic transmission of manuscripts, with attitudes toward e-transmission varying by age.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"Y2UWHWNU","t":"Citation gamesmanship: testing for evidence of ego bias in peer review","au":"Sugimoto, Cassidy R.","y":2012,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-012-0845-z","cx":"Journal","co":["Bias"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"closed","cby":41,"self":false,"s":"An empirical study of 442 manuscripts and 927 referee reports at a journal (JASIST), testing whether authors, reviewers or editors exploit peer review for self-serving citations, and finding little evidence of such citation gaming.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"W78VMZZS","t":"Journal peer review in context: A qualitative study of the social and subjective dimensions of manuscript review in biomedical publishing","au":"Lipworth, Wendy","y":2011,"vn":"Social Science & Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.socscimed.2011.02.002","cx":"Journal","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"qualitative","oast":"green","cby":70,"self":false,"s":"A qualitative interview study of biomedical journal peer review, characterising the social and subjective dimensions of manuscript reviewing from reviewers' perspectives and arguing that efforts to improve the process must take these dimensions into account.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"Y325AUKI","t":"The Gender Gap in NIH Grant Applications","au":"Ley, Timothy J.","y":2008,"vn":"Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1126/science.1165878","cx":"Grant","co":["Bias"],"sg":["funding-decision"],"x":0,"st":"observational-quantitative","oast":"closed","cby":279,"self":false,"s":"A brief analysis of NIH grant application data showing that many qualified women scientists stop applying for funding in the late postdoctoral and early faculty years, creating a gender gap in applications.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"WNWHJLDM","t":"Estimating within-group interrater reliability with and without response bias.","au":"James, Lawrence R.","y":1984,"vn":"Journal of Applied Psychology","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/0021-9010.69.1.85","cx":"Analogue","co":["Reliability-noise","Bias"],"sg":["individual-scoring"],"x":0,"st":"simulation-methodological","oast":"closed","cby":4663,"self":false,"s":"A methodological paper presenting new procedures for estimating within-group interrater reliability when multiple judges rate a single target, such as journal referees judging a manuscript's publishability, including adjustments to control for response biases like leniency and social desirability.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"WEW94PCC","t":"Institutions of performance: The journal as a theater of power and legitimacy","au":"Khaled, Ahmed","y":2026,"vn":"Political Studies Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/14789299261454217","cx":"Journal","co":["general-systemic"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":0,"self":false,"s":"A theoretical essay drawing on Foucault, Butler and performance theory to conceptualise journal peer review as a theatrical performance, arguing that scholarly legitimacy is enacted through institutional rituals of power rather than through neutral evaluation.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"YHAILXFM","t":"Does output-driven research culture hinder social outcome-focused funding? Insights from the STI hub program of DST, India","au":"Stanley, Ansel Elias","y":2026,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvag029","cx":"Grant","co":["Bias"],"sg":["proposal-quality","individual-scoring"],"x":0,"st":"qualitative","oast":"closed","cby":0,"self":false,"s":"A qualitative case study of grant proposal review in India's DST STI Hub programme, finding proposals poorly aligned with societal-impact goals and peer review lacking structured criteria and transparency, with likely biases towards institutional and investigator reputation.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://osf.io/mr3ka/","ca":"no-statement","cau":""},{"k":"WZH5R7CJ","t":"Peer Reviewing as a Distributed Cognitive Validation System: Reconceptualizing Peer Review in Networked Knowledge Systems","au":"KWONGO, JOEL MANTSHIUMBA","y":2026,"vn":"SSRN Electronic Journal","ty":"journalArticle","tyl":"Journal article","doi":"10.2139/ssrn.6362173","cx":"Journal","co":["general-systemic"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"green","cby":0,"self":false,"s":"A conceptual preprint proposing that journal peer review be reconceived as a distributed cognitive validation process across networked knowledge systems, arguing this framing better addresses scalability, delay and reviewer workload than the traditional sequential gatekeeping model.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"W7GNC85E","t":"All that Glitters: A Perspective on Peer Reviewed Publications","au":"Jameison, Danielle","y":2020,"vn":"Harvard Medical Student Review","ty":"journalArticle","tyl":"Journal article","doi":"10.65539/4yj42w18","cx":"Journal","co":["general-systemic"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"hybrid","cby":0,"self":false,"s":"A perspective piece questioning the status of the peer-reviewed publication as the gold standard of academic productivity in medicine, tracing the origins of journal peer review and discussing alternative measures of scholarly output.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"XJVRNWAZ","t":"Is there gender gap in research grant success in economics? Case study of Slovakia","au":"Némethová, Valéria","y":2025,"vn":"Sociologija","ty":"journalArticle","tyl":"Journal article","doi":"10.2298/soc2502304n","cx":"Grant","co":["Bias"],"sg":["funding-decision","post-award-validity"],"x":0,"st":"observational-quantitative","oast":"diamond","cby":0,"self":false,"s":"An empirical case study of gender differences in economics grant applications to Slovakia's largest funding agency, finding women apply less often than expected, receive slightly lower evaluation scores after adjustment, yet lead teams with marginally higher scientific output.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"XJ9E8VL4","t":"A Conceptual Framework for AI- and Blockchain-Enabled Research Project Evaluation Systems","au":"Amanzholova, Saule","y":2026,"vn":"Information","ty":"journalArticle","tyl":"Journal article","doi":"10.3390/info17020151","cx":"Grant","co":["Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":0,"self":false,"s":"A conceptual design paper proposing a framework for national research grant proposal evaluation that combines artificial intelligence for expert matching and review analysis with blockchain for tamper-proof record-keeping, offering an architectural reference model rather than an implemented or empirically tested system.","pg":"not-applicable","pgr":"","pgid":"","da":"on-request","dau":"","ca":"not-applicable","cau":""},{"k":"XKIJUH93","t":"O processo editorial de revistas científicas: o longo caminho entre a submissão e a publicação","au":"Almeida, Débora Rezende de","y":2026,"vn":"Revista de Sociologia e Política","ty":"journalArticle","tyl":"Journal article","doi":"10.1590/1678-98732434e004","cx":"Journal","co":["Bias"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"diamond","cby":0,"self":false,"s":"A reflective qualitative account, drawing on the authors' experience as journal editors, describing each stage of the editorial process from submission to publication, identifying common reasons for rejection and arguing for greater transparency towards authors, especially early-career researchers.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"VTMMCRS3","t":"Academic Peer Review: Eliminating the Option of ‘Reject’ Without Author Response","au":"Arabi, Alya A.","y":2026,"vn":"Learned Publishing","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/leap.2038","cx":"Journal","co":["Reliability-noise","Bias","Interventions"],"sg":["funding-decision"],"x":0,"st":"conceptual-commentary","oast":"hybrid","cby":0,"self":false,"s":"A commentary proposing that journals abolish first-round manuscript rejections issued without giving authors a chance to respond, arguing that appeals are too slow and burdensome to serve as an adequate remedy.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"WETTECGT","t":"Published peer review reports have higher informative content than unpublished reports","au":"Álvarez-García, Elena","y":2025,"vn":"Journal of Informetrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.joi.2025.101760","cx":"Journal","co":["Reliability-noise","Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":0,"self":false,"s":"Large-scale text analysis of over 250,000 journal peer review reports from 233 medical journals, finding that published reports are longer, more informative and more readable than unpublished ones, with further differences by reviewer gender and institutional region.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://doi.org/10.7910/DVN/BBBQDA","ca":"open","cau":"https://doi.org/10.7910/DVN/BBBQDA"},{"k":"VRIHCI6Q","t":"Funding big science: managing diversity, social responsibility, and limited resources","au":"Shaw, Jamie","y":2025,"vn":"European Journal for Philosophy of Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s13194-025-00698-6","cx":"Grant","co":["Validity","Interventions"],"sg":["funding-decision"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":0,"self":false,"s":"A philosophy of science paper distinguishing normal, revolutionary, and socially-oriented big science and arguing that grant funders should evaluate each through different mechanisms, namely modified crowdfunding, lotteries, and citizen juries, to protect pluralism and social responsibility.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"YUHH5UFA","t":"An expert criticism on post-publication peer review platforms: the case of pubpeer","au":"Tsatsakis, Aristidis","y":2025,"vn":"DARU Journal of Pharmaceutical Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s40199-025-00580-z","cx":"General","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"green","cby":0,"self":false,"s":"A short commentary evaluating PubPeer as a post-publication journal peer-review platform, arguing that public scrutiny can identify errors and improve transparency but requires safeguards for fair, accountable","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"VW2V5MMQ","t":"How Transport Journals Weed Out Qualitative Research and Why It Matters","au":"Klein, Nicholas J.","y":2025,"vn":"Transportation","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11116-026-10777-z","cx":"Journal","co":["Bias"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"qualitative","oast":"hybrid","cby":0,"self":false,"s":"A mixed-methods study combining a systematic review of top transport journals with interviews of 46 researchers and editors, finding that qualitative research is rarely published because journal reviewers and editors often misunderstand and lack expertise in evaluating qualitative methods.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://osf.io/9y85k/","ca":"open","cau":"https://osf.io/9y85k/"},{"k":"VHDT8P7V","t":"When Cognitive Proximity Leads to Higher Evaluation Decision Quality: A Study of Public Funding Allocation","au":"Zhang, Chuqing","y":2021,"vn":"Frontiers in Psychology","ty":"journalArticle","tyl":"Journal article","doi":"10.3389/fpsyg.2021.697989","cx":"Grant","co":["Validity"],"sg":["reviewer-selection","individual-scoring"],"x":0,"st":"observational-quantitative","oast":"gold","cby":0,"self":false,"s":"An empirical study of 923 grant evaluations from the 2017 Beijing Innofund, finding an inverted U-shaped relationship between reviewers' cognitive proximity to proposals and decision quality, moderated by evaluation experience and effort.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"no-statement","cau":""},{"k":"ZW2W6EES","t":"What Can Journals Do to Increase the Reliability of Scientific Research?","au":"Lee, W. Robert","y":2020,"vn":"Practical Radiation Oncology","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.prro.2020.03.001","cx":"Journal","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":0,"self":false,"s":"An editorial in Practical Radiation Oncology outlining strategies scholarly journals can use to improve the reliability and reproducibility of published research, framed by the ongoing debate over a reproducibility crisis in science.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"ZCRB5IEL","t":"Does Unblinding Make One King in the Land of the Blind(Ed)?","au":"Hee, Ng Heok","y":2015,"vn":"Proceedings of Singapore Healthcare","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/201010581502400201","cx":"Journal","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"diamond","cby":0,"self":false,"s":"An editorial discussing whether single or double blinding in journal peer review reduces reviewer bias and improves review quality, noting that the evidence from published studies is mixed.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"WVZ6EEK5","t":"Greenlighting Innovative Projects: How Evaluation Structure Shapes the Perceived Feasibility of Early-Stage Ideas","au":"Lane, Jacqueline N.","y":2024,"vn":"SSRN Electronic Journal","ty":"journalArticle","tyl":"Journal article","doi":"10.2139/ssrn.4769282","cx":"Grant","co":["Bias","Interventions"],"sg":["individual-scoring","funding-decision"],"x":1,"st":"experimental-rct","oast":"green","cby":3,"self":false,"s":"A working paper examining how the structure of evaluation processes shapes reviewers' judgements of the feasibility of early-stage innovative project ideas.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"Y99Z2GUP","t":"Author-Reviewer Homophily in Peer Review","au":"Murray, Dakota","y":2018,"vn":"bioRxiv (Cold Spring Harbor Laboratory)","ty":"journalArticle","tyl":"Journal article","doi":"10.1101/400515","cx":"Journal","co":["Bias"],"sg":["reviewer-selection","funding-decision"],"x":0,"st":"observational-quantitative","oast":"green","cby":179,"self":false,"s":"An exploratory analysis of journal peer review at eLife finding higher manuscript acceptance rates when authors and gatekeepers shared gender or country, alongside advantages for male authors and those from North America and Europe.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://github.com/murrayds/elife-analysis","ca":"open","cau":"https://github.com/murrayds/elife-analysis"},{"k":"ZZYPVDYT","t":"Behavioral Optimization in Scientific Publishing","au":"Watve, Milind","y":2023,"vn":"Qeios","ty":"journalArticle","tyl":"Journal article","doi":"10.32388/8w10nd.3","cx":"Journal","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"diamond","cby":2,"self":false,"s":"A conceptual essay arguing that biases in journal peer review arise because each player optimises personal cost-benefit, and proposing that publishing systems be redesigned around human behavioural incentives so individual self-interest aligns with the collective goals of science.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"WDD7VXQ5","t":"Identifying emerging scholars: seeing through the crystal ball of scholarship selection committees","au":"Chandler, Vincent","y":2019,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-019-03120-0","cx":"Grant","co":["Bias","Validity"],"sg":["individual-scoring","post-award-validity"],"x":0,"st":"observational-quantitative","oast":"closed","cby":2,"self":false,"s":"An empirical study of 1900 doctoral scholarship winners showing that higher selection-committee scores modestly predict PhD completion and tenure-track appointments ten years later, and that male evaluators reward candidates who finish quickly more than female evaluators do.","pg":"no-statement","pgr":"","pgid":"","da":"restricted","dau":"","ca":"no-statement","cau":""},{"k":"WCX3TBVD","t":"Barriers to Publication in the Middle East: A Categorization of Rejection Reasons among Arab Researchers","au":"Elgamri, Alya","y":2025,"vn":"Journal of Academic Ethics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s10805-025-09680-6","cx":"Journal","co":["Bias"],"sg":["proposal-quality","individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":1,"self":false,"s":"Survey-based study categorising journal rejection letters received by Arab researchers in Middle Eastern low- and middle-income countries, finding lack of novelty, methodological flaws and scope misalignment the most common reviewer criticisms contributing to publication inequities.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://osf.io/z4xj6/?view_only=59858ed3d9df4ccba414f0d50d02a073","ca":"no-statement","cau":""},{"k":"WNMMH2FJ","t":"A duoethnographic exploration of what peer reviewing teaches us about peer review","au":"Holly, James","y":2025,"vn":"Journal of Engineering Education","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/jee.70013","cx":"Journal","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"qualitative","oast":"hybrid","cby":1,"self":false,"s":"A duoethnographic study in which two journal reviewers reflect on how their racialised positionalities shaped their manuscript evaluations for the Journal of Engineering Education, calling for greater reviewer introspection and structural change to make peer review more equitable.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"XNSNZATC","t":"Critical Contextual Empiricism for Busy People: Scientific Argumentation as Epistemic Exchange","au":"Novaes, Catarina Dutilh","y":2025,"vn":"Topoi","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11245-025-10198-0","cx":"Journal","co":["general-systemic"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"hybrid","cby":1,"self":false,"s":"A philosophy of science paper extending Longino's critical contextual empiricism with an account of argumentation as epistemic exchange that acknowledges scientists' limited time and resources, using peer review to illustrate the system's vulnerabilities and suggest improvements.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"YH4UKLA2","t":"Scientific Peer Review in the Modern Era: A Comprehensive Guide","au":"Sandhu, Nimrat Kaur","y":2022,"vn":"Indian Journal of Rheumatology","ty":"journalArticle","tyl":"Journal article","doi":"10.4103/0973-3698.364681","cx":"Journal","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"diamond","cby":2,"self":false,"s":"A review article outlining the history, purpose and shortcomings of journal peer review, describing current ethical guidelines and newer approaches such as registered reports and dynamic review intended to make the process faster, fairer and more rigorous.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"XFRJ7EYE","t":"A Case Study for the Interrater Reliability of Journal Referees","au":"Marson, Stephen M.","y":2021,"vn":"Research on Social Work Practice","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/10497315211052456","cx":"Journal","co":["Reliability-noise"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":4,"self":false,"s":"A case study analysing interrater reliability among referees at The International Journal of Social Work Values and Ethics, finding moderate agreement that exceeds published baselines and offering eight recommendations for improving anonymous journal review.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"ZHTYCRJS","t":"Peer Review May Not Be Such a Bad Idea: Response to Heesen and Bright","au":"Rowbottom, Darrell P.","y":2021,"vn":"The British Journal for the Philosophy of Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1086/714787","cx":"Journal","co":["Bias"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":9,"self":false,"s":"A philosophical reply defending prepublication journal peer review against Heesen and Bright's abolition proposal, identifying three overlooked factors: transition costs, bias-avoiding benefits of prepublication review, and epistemic drawbacks of the larger literature that postpublication review would create.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"VJGI4LJQ","t":"Review of interactive open-access publishing with community-based open peer review for improved scientific discourse and quality assurance","au":"Ervens, Barbara","y":2025,"vn":"Atmospheric chemistry and physics","ty":"journalArticle","tyl":"Journal article","doi":"10.5194/acp-25-13903-2025","cx":"Journal","co":["Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":1,"self":false,"s":"A review describing 25 years of interactive open-access journal publishing with multi-stage public peer review at the European Geosciences Union, arguing this community-based model delivers high scientific quality, low cost and financial sustainability, and has influenced other major publishers.","pg":"not-applicable","pgr":"","pgid":"","da":"open","dau":"https://doi.org/10.5281/zenodo.14713159","ca":"no-statement","cau":""},{"k":"VLAGCTV8","t":"On arXiv moderation system","au":"Силагадзе, З. К.","y":2023,"vn":"Journal of Informetrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.joi.2023.101433","cx":"General","co":["Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":3,"self":false,"s":"A short communication arguing that arXiv's preprint moderation system, while necessary for quality control, has eroded the platform's original democratising spirit, and proposing improvements modelled on MathOverflow's community moderation approach.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"YHI9F8UZ","t":"Reducing bias in the peer‐review process","au":"Frair, Jacqueline L.","y":2025,"vn":"Journal of Wildlife Management","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/jwmg.70019","cx":"Journal","co":["Bias","Interventions"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"conceptual-commentary","oast":"bronze","cby":2,"self":false,"s":"An editorial reviewing evidence on double-blind journal peer review and urging policy and procedural changes to reduce systemic bias, while noting that robust studies do not support claimed benefits for female first authors.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"WYCLR47W","t":"Impact of author characteristics on outcomes of single- versus double-blind peer review: a systematic review of comparative studies in scientific abstracts and publications","au":"Giannakakos, Vasiliki P.","y":2024,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-024-05213-x","cx":"General","co":["Bias"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"systematic-review-meta-analysis","oast":"hybrid","cby":2,"self":false,"s":"A systematic review of 29 studies comparing single- and double-blind journal peer review, finding that single-blind review favours male, White, US-based, well-known or prestigious-institution authors, while evidence that double-blinding reduces these biases is mixed.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"ZM57TQNS","t":"Cognitive bias undermining intellectual capital development in the scientific contexts: a study to understand origins, effects and solutions","au":"Palmucci, Dario Natale","y":2024,"vn":"Journal of Intellectual Capital","ty":"journalArticle","tyl":"Journal article","doi":"10.1108/jic-07-2024-0199","cx":"Journal","co":["Bias","Interventions"],"sg":["individual-scoring"],"x":0,"st":"qualitative","oast":"closed","cby":3,"self":false,"s":"A literature review and expert-interview study identifying cognitive biases affecting journal editors and reviewers, such as framing, halo and confirmation effects, arguing these undermine intellectual capital in universities and offering practical tips to counter them.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"YDF4F296","t":"The role of non-scientific factors vis-à-vis the quality of publications in determining their scholarly impact","au":"Abramo, Giovanni","y":2024,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-024-05106-z","cx":"General","co":["Validity"],"sg":["post-award-validity"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":3,"self":false,"s":"An empirical study using peer-review quality scores from Italy's first national research assessment exercise and Web of Science citations, showing that non-scientific attributes of publications, beyond quality judged by reviewers, help determine short- and long-term scholarly impact.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"ZN37E6CT","t":"Preprint review services: Disrupting the scholarly communication landscape?","au":"Henriques, Susana Oliveira","y":2023,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31235/osf.io/8c6xm","cx":"General","co":["Interventions"],"sg":["whole-process"],"x":0,"st":"qualitative","oast":"gold","cby":6,"self":false,"s":"A descriptive study mapping the main characteristics of preprint review services and how they manage peer review, finding they currently complement preprint servers and journals but could eventually disrupt scholarly communication more radically.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://doi.org/10.6084/m9.figshare.24307312.v3","ca":"no-statement","cau":""},{"k":"Y4Z4GDFY","t":"Let’s agree to agree: The situational academic quality of the UK REF as consensual public knowledge","au":"Milyaeva, Sveta","y":2023,"vn":"Social Studies of Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/03063127231152915","cx":"General","co":["Reliability-noise"],"sg":["proposal-quality","panel-consensus"],"x":0,"st":"qualitative","oast":"hybrid","cby":6,"self":false,"s":"An interview study of UK Research Excellence Framework panellists showing how benchmarking, scoring and calibration practices push reviewers from epistemically grounded assessment towards consensus on a single quality scale, making REF academic quality distinct from scientific quality.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"Y6MKSCBH","t":"Editors are biased too: An extension of Fox et al. (2023)'s analysis makes the case for triple‐blind review","au":"Srivastava, Diane S.","y":2024,"vn":"Functional Ecology","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/1365-2435.14483","cx":"Journal","co":["Bias","Interventions"],"sg":["funding-decision"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":7,"self":false,"s":"A reanalysis of a randomised journal peer-review trial finding editor bias favouring high-scoring manuscripts by male first authors from very high HDI countries, and advocating triple-blind review to reduce editor and reviewer bias.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://doi.org/10.5061/dryad.m63xsj466","ca":"open","cau":"https://doi.org/10.5281/zenodo.8075130"},{"k":"WIFBZF3Z","t":"Reforms in Academic Publishing: Should Behavioral Disorders and Special Education Journals Embrace Them?","au":"Cook, Bryan G.","y":2016,"vn":"Behavioral Disorders","ty":"journalArticle","tyl":"Journal article","doi":"10.17988/0198-7429-41.3.161","cx":"Journal","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":9,"self":false,"s":"A review essay examining sources of research bias and proposed journal publishing reforms, such as preregistration, open data, registered reports and open review, weighing their potential problems and considering whether special education journals like Behavioral Disorders should adopt them.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"YNWHEN66","t":"What influences the regional diversity of reviewers: A study of medical and agricultural/biological sciences journals","au":"Gaston, Thomas","y":2018,"vn":"Learned Publishing","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/leap.1155","cx":"Journal","co":["Bias"],"sg":["reviewer-selection","individual-scoring"],"x":0,"st":"observational-quantitative","oast":"bronze","cby":14,"self":false,"s":"An analysis of journal peer review in medical and agricultural and biological sciences, finding reviewer location, acceptance of invitations and recommendations were associated with the regions of editors and corresponding authors, but not journal discipline, rank, size or difficulty obtaining reviews.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"ZAW6VU37","t":"Evaluation, ranking and selection of R&D projects by multiple experts: an evidential reasoning rule based approach","au":"Liu, Fang","y":2017,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-017-2278-1","cx":"Grant","co":["Reliability-noise"],"sg":["panel-consensus","funding-decision"],"x":0,"st":"simulation-methodological","oast":"green","cby":21,"self":false,"s":"A methods paper proposing an evidential reasoning rule model for grant-style R&D project selection that combines multiple expert reviewers' ratings while accounting for their differing weights and reliabilities, demonstrated on National Science Foundation of China project data.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"XRW5XR99","t":"Nurturing ideas, cultivating reviewers: The C.E.D.A.R. tree of generative peer review","au":"Cristofaro, Matteo","y":2025,"vn":"European Management Journal","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.emj.2025.10.011","cx":"Journal","co":["Interventions"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"hybrid","cby":2,"self":false,"s":"An editorial proposing the C.E.D.A.R. Tree framework for generative journal peer review, arguing that reviewing at the European Management Journal should grow from shared values into practices such as curiosity and empathy that develop both authors and reviewers.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"YARRVZJ4","t":"Investigating the effect of publication text similarity between reviewers and authors on the rigor of peer review: An intellectual proximity perspective","au":"Kang, Yanlan","y":2025,"vn":"Journal of Informetrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.joi.2025.101709","cx":"Journal","co":["Bias"],"sg":["reviewer-selection","individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":2,"self":false,"s":"An empirical bibliometric study measuring publication text similarity between reviewers and authors in journal peer review, examining whether reviewers whose expertise is intellectually closer to the authors' research produce more rigorous manuscript evaluations.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"on-request","cau":""},{"k":"ZEDT76D3","t":"Social behavioural epistemology and the scientific community","au":"Watve, Milind","y":2017,"vn":"Journal of Genetics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s12041-017-0790-y","cx":"General","co":["Bias"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"closed","cby":7,"self":false,"s":"A conceptual paper arguing that behavioural evolution can explain how social dominance and connectedness shape scientific acceptance of innovations, and that studying these dynamics could reduce bias in scientific communication.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"Y3NCX4KP","t":"Technology assisted research assessment: algorithmic bias and transparency issues","au":"Thelwall, Mike","y":2023,"vn":"Aslib Journal of Information Management","ty":"journalArticle","tyl":"Journal article","doi":"10.1108/ajim-04-2023-0119","cx":"General","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"green","cby":15,"self":false,"s":"A review of transparency and algorithmic bias concerns in technology assisted research assessment, concluding that it is often unclear whether bibliometric or AI tools worsen the bias and opacity problems already inherent in peer review.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"WTZECG89","t":"The academic status of reviewers predicts their language use","au":"Sun, Zhuanlan","y":2023,"vn":"Journal of Informetrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.joi.2023.101449","cx":"Journal","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"observational-quantitative","oast":"closed","cby":13,"self":false,"s":"An observational study of 2,580 eLife journal peer-review histories finding that reviewers’ academic status predicts emotional and linguistic features of their","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"on-request","cau":""},{"k":"ZR8TTA2L","t":"PEERRec: An AI-based approach to automatically generate recommendations and predict decisions in peer review","au":"Bharti, Prabhat Kumar","y":2023,"vn":"International Journal on Digital Libraries","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s00799-023-00375-0","cx":"Journal","co":["Interventions"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"simulation-methodological","oast":"closed","cby":13,"self":false,"s":"A machine-learning paper introducing PEERRec, a system that uses the interaction between a manuscript's full text and its reviews to automatically generate reviewer recommendations and predict acceptance decisions, aiming to ease reviewer workload in conference and journal peer review.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"open","cau":"https://github.com/PrabhatkrBharti/PEERRec.git"},{"k":"Z89VP5AE","t":"All-pervading insider bias alters review time in Turkish university journals","au":"Tütüncü, Lokman","y":2023,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-023-04724-3","cx":"Journal","co":["Bias"],"sg":["whole-process"],"x":0,"st":"observational-quantitative","oast":"green","cby":13,"self":false,"s":"An empirical study of 56,920 papers in 258 Turkish university Social Sciences journals, finding that insider authors are common and receive substantially faster journal peer review, with the bias strongest in Law and Theology journals.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"not-applicable","cau":""},{"k":"Z475W66K","t":"Editorial: Editorial Practice at the International Journal of Primatology: the Roles of Gender and Country of Affiliation in Participation in Scientific Publication","au":"Setchell, Joanna M.","y":2018,"vn":"International Journal of Primatology","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s10764-018-0067-1","cx":"Journal","co":["Bias"],"sg":["reviewer-selection","funding-decision"],"x":0,"st":"observational-quantitative","oast":"bronze","cby":15,"self":false,"s":"An editorial examining how gender and country of affiliation shape authorship and peer-review participation at the International Journal of Primatology, and arguing that journal editors can address or perpetuate inequalities.","pg":"no-statement","pgr":"","pgid":"","da":"on-request","dau":"","ca":"no-statement","cau":""},{"k":"XML2CJ6J","t":"Preserving the Quality of Scientific Research: Peer Review of Research Articles","au":"Silva, Pali U. K. De","y":2017,"vn":"Fascinating life sciences","ty":"book","tyl":"Book","doi":"10.1007/978-3-319-50627-2_6","cx":"Journal","co":["Bias"],"sg":["whole-process"],"x":0,"st":"book-monograph","oast":"closed","cby":17,"self":false,"s":"A book chapter reviewing the role of peer review in journal manuscript evaluation, tracing its history from seventeenth-century learned societies and discussing its wider use in assessing grants, awards and academic tenure.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"XFLLARLX","t":"Critical essay: Blinding faith – Paradoxes and pathologies of opacity in peer review","au":"Willmott, Hugh","y":2021,"vn":"Human Relations","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/00187267211016752","cx":"Journal","co":["Bias"],"sg":["individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"hybrid","cby":21,"self":false,"s":"A critical essay arguing that blinding in journal peer review, examined in management and organisation studies, contradicts science's ethos of openness and reflects asymmetrical power relations, advocating a shift towards more open review.","pg":"not-applicable","pgr":"","pgid":"","da":"open","dau":"","ca":"not-applicable","cau":""},{"k":"WZTKU9QM","t":"Disagreement and Agonistic Chance in Peer Review","au":"Roumbanis, Lambros","y":2021,"vn":"Science Technology & Human Values","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/01622439211026016","cx":"Grant","co":["Reliability-noise"],"sg":["panel-consensus"],"x":0,"st":"qualitative","oast":"hybrid","cby":29,"self":false,"s":"An ethnographic study of ten grant review panels at the Swedish Research Council, theorising how chance enters expert deliberations when reviewers strongly disagree and becomes woven into the consensus decisions panels ultimately reach.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"not-applicable","cau":""},{"k":"VTZWHCCL","t":"How peer-review constrains cognition: on the frontline in the knowledge sector","au":"Cowley, Stephen J.","y":2015,"vn":"Frontiers in Psychology","ty":"journalArticle","tyl":"Journal article","doi":"10.3389/fpsyg.2015.01706","cx":"Journal","co":["Reliability-noise","Bias","Validity"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":29,"self":false,"s":"A theoretical essay reframing journal peer review as a cognitive process that shapes how authors, editors and reviewers construct knowledge, illustrated by tracing how adversarial review reshaped the paper's own argument structure and wording.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"YKTWXZ98","t":"Megajournal mismanagement: Manuscript decision bias and anomalous editor activity at PLOS ONE","au":"Petersen, Alexander M.","y":2019,"vn":"Journal of Informetrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.joi.2019.100974","cx":"Journal","co":["Bias"],"sg":["reviewer-selection","funding-decision"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":31,"self":false,"s":"An analysis of PLOS ONE editors' activity over the journal's first decade, finding that a small number of extremely active editors show stronger manuscript decision bias and elevated self-citation in the papers they handle, suggesting perverse incentives in megajournal editorial gatekeeping.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"W2VF7UBP","t":"Reviewer recommendations and editors’ decisions for a conservation journal: Is it just a crapshoot? And do Chinese authors get a fair shot?","au":"Campos‐Arceiz, Ahimsa","y":2015,"vn":"Biological Conservation","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.biocon.2015.02.025","cx":"Journal","co":["Reliability-noise","Bias"],"sg":["individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"closed","cby":34,"self":false,"s":"An analysis of journal peer review at Biological Conservation examining agreement between reviewer recommendations and editors’ decisions, and whether manuscripts by Chinese authors receive fair treatment.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"YJIXI5AV","t":"Managing the Growth of Peer Review at the Royal Society Journals, 1865-1965","au":"Fyfe, Aileen","y":2019,"vn":"Science Technology & Human Values","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/0162243919862868","cx":"Journal","co":["general-systemic"],"sg":["reviewer-selection"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":36,"self":false,"s":"A historical study using archival data from the Royal Society's journals (1865-1965) showing how the society managed growing submissions by involving more fellows as referees, though the editorial workload remained unevenly distributed.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"WVHDHIT8","t":"Gender differences in patterns of authorship do not affect peer review outcomes at an ecology journal","au":"Fox, Charles W.","y":2015,"vn":"Functional Ecology","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/1365-2435.12587","cx":"Journal","co":["Bias"],"sg":["reviewer-selection","individual-scoring","funding-decision"],"x":0,"st":"observational-quantitative","oast":"bronze","cby":64,"self":false,"s":"An observational study of journal peer review at Functional Ecology finding gender differences in authorship patterns but no effect of author gender on review scores, editorial decisions or acceptance.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"doi: 10.5061/dryad.0mv3q","ca":"no-statement","cau":""},{"k":"XTFW3RID","t":"Bias against scientific novelty: A prepublication perspective","au":"Liang, Zhentao","y":2022,"vn":"Journal of the Association for Information Science and Technology","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/asi.24725","cx":"Journal","co":["Bias"],"sg":["proposal-quality","funding-decision"],"x":0,"st":"observational-quantitative","oast":"closed","cby":32,"self":false,"s":"A large-scale journal study finding that more novel manuscripts have longer publication handling times, with prestige reducing delays less as novelty increases.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"ZSKAUZH4","t":"The crisis of peer review: Part of the evolution of science","au":"Horta, Hugo","y":2024,"vn":"Higher Education Quarterly","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/hequ.12511","cx":"Journal","co":["Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"hybrid","cby":63,"self":false,"s":"An essay tracing the historical development of journal peer review and arguing that its current crisis, driven by publish-or-perish massification, is an evolutionary opportunity to correct biases, broaden inclusivity, and gain recognition for reviewing in researchers' careers and training.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"XIBX7QW6","t":"The ability of different peer review procedures to flag problematic publications","au":"Horbach, Serge P. J. M.","y":2018,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-018-2969-2","cx":"Journal","co":["Validity","Interventions"],"sg":["post-award-validity"],"x":0,"st":"observational-quantitative","oast":"hybrid","cby":73,"self":false,"s":"An empirical study linking survey data on journal peer-review procedures to the Retraction Watch database, comparing retraction rates across 361 journals to identify which review procedures better flag problematic publications and offering recommendations for editors.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"ZHEM7BQQ","t":"Can editors save peer review from peer reviewers?","au":"D’Andrea, Rafael","y":2017,"vn":"PLoS ONE","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0186111","cx":"Journal","co":["Bias","Interventions"],"sg":["individual-scoring","reviewer-selection","funding-decision"],"x":0,"st":"simulation-methodological","oast":"gold","cby":79,"self":false,"s":"A modelling study of journal peer review finding that indifferent acceptance of low-quality manuscripts is more damaging than competitive sabotage, and that editorial interventions reduce but cannot eliminate the harm.","pg":"no-statement","pgr":"","pgid":"","da":"open","dau":"https://github.com/odwyer-lab/PeerReviewFunctions","ca":"open","cau":"https://github.com/odwyer-lab/PeerReviewFunctions"},{"k":"X8TU7KXM","t":"Is the future of peer review automated?","au":"Schulz, Robert","y":2022,"vn":"BMC Research Notes","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s13104-022-06080-6","cx":"Journal","co":["Interventions"],"sg":["proposal-quality","individual-scoring"],"x":0,"st":"conceptual-commentary","oast":"gold","cby":87,"self":false,"s":"A commentary arguing that automated screening can support journal peer review by checking reporting and policy compliance, but cannot replace human judgement of study design, conclusions, quality and significance.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"X3JD6ZBB","t":"What Do Editors Maximize? Evidence from Four Economics Journals","au":"Card, David","y":2019,"vn":"The Review of Economics and Statistics","ty":"journalArticle","tyl":"Journal article","doi":"10.1162/rest_a_00839","cx":"Journal","co":["Bias","Validity"],"sg":["individual-scoring","funding-decision","post-award-validity"],"x":0,"st":"observational-quantitative","oast":"closed","cby":124,"self":false,"s":"An empirical study of editorial decision-making at four leading economics journals, finding that referee recommendations strongly predict citations and editors largely follow them, but papers by highly published authors are under-cited by referees and editors overweight recommendations from highly published referees.","pg":"preregistered","pgr":"other","pgid":"AEARCTR-0001669","da":"no-statement","dau":"","ca":"no-statement","cau":""},{"k":"ZV4UN4ZA","t":"Scientific Autonomy, Public Accountability, and the Rise of “Peer Review” in the Cold War United States","au":"Baldwin, Melinda","y":2018,"vn":"Isis","ty":"journalArticle","tyl":"Journal article","doi":"10.1086/700070","cx":"General","co":["general-systemic"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":161,"self":false,"s":"A historical essay tracing how refereeing at journals and funding bodies, long an optional editorial practice, was recast in Cold War America as \"peer review\", the essential guarantor of scientific legitimacy amid 1970s pressures for public accountability.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"Z7KLFNBP","t":"Ensuring the Quality, Fairness, and Integrity of Journal Peer Review: A Possible Role of Editors","au":"Resnik, David B.","y":2015,"vn":"Science and Engineering Ethics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11948-015-9625-5","cx":"Journal","co":["Reliability-noise","Bias","Interventions"],"sg":["whole-process"],"x":0,"st":"conceptual-commentary","oast":"closed","cby":161,"self":false,"s":"A review and commentary examining evidence on problems in journal peer review, such as inconsistent reviewer reports, bias and ethical misconduct, and recommending steps editors should take to promote quality, fairness and integrity throughout the process.","pg":"not-applicable","pgr":"","pgid":"","da":"not-applicable","dau":"","ca":"not-applicable","cau":""},{"k":"W8TKHTI7","t":"Project selection in NIH: A natural experiment from ARRA","au":"Park, Hyunwoo","y":2015,"vn":"Research Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.respol.2015.03.004","cx":"Grant","co":["Bias","Validity"],"sg":["funding-decision","post-award-validity"],"x":0,"st":"observational-quantitative","oast":"closed","cby":40,"self":false,"s":"An economics paper using the 2009 ARRA funding windfall as a natural experiment to reveal the NIH's preferences in grant selection, comparing characteristics of projects funded only because of the extra resources with those funded through the normal process.","pg":"no-statement","pgr":"","pgid":"","da":"no-statement","dau":"","ca":"no-statement","cau":""}]