[{"id":"22WZMMFB","t":"The Equivalence of Weighted Kappa and the Intraclass Correlation Coefficient as Measures of Reliability","au":"Fleiss, Joseph L.","y":1973,"vn":"Educational and Psychological Measurement","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/001316447303300309","cby":3357,"nref":15,"oast":"closed","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"4","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"23VVRPE8","t":"An Empirical Comparative Assessment of Inter-Rater Agreement of Binary Outcomes and Multiple Raters","au":"Konstantinidis, Menelaos","y":2022,"vn":"Symmetry","ty":"journalArticle","tyl":"Journal article","doi":"10.3390/sym14020262","cby":26,"nref":53,"oast":"gold","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"4","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"24E7BBHB","t":"The reliability of ethical reviews: Is it desirable?","au":"Doob, Anthony N","y":1983,"vn":"Canadian Psychology / Psychologie canadienne","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/h0080918","cby":7,"nref":3,"oast":"closed","cx":"General","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"3","cc":5,"cp":0,"self":false,"ncoefs":0},{"id":"24YIFL73","t":"‘Conditional Acceptance’ (additional experiments required): A scoping review of recent evidence on key aspects of Open Peer Review","au":"Ross-Hellauer, Tony","y":2022,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31222/osf.io/r6t8p","cby":4,"nref":68,"oast":"gold","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"systematic-review-meta-analysis","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"28GK7NTQ","t":"Assessing quality of selection procedures: Lower bound of false positive rate as a function of inter‐rater reliability","au":"Bartoš, František","y":2024,"vn":"British Journal of Mathematical and Statistical Psychology","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/bmsp.12343","cby":2,"nref":66,"oast":"hybrid","cx":"Grant","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring","funding-decision"],"st":"simulation-methodological","tp":"8","cc":6,"cp":2,"self":false,"ncoefs":0},{"id":"29UTGYZQ","t":"Peer review of interdisciplinary research proposals","au":"Porter, Alan L.","y":1985,"vn":"Science, Technology, & Human Values","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/016224398501000304","cby":107,"nref":0,"oast":"closed","cx":"Grant","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring","whole-process"],"st":"observational-quantitative","tp":"7","cc":4,"cp":0,"self":false,"ncoefs":0},{"id":"2BMCWPPH","t":"Breaking the gatekeepers: how AI will revolutionize scientific funding","au":"Mangalam, Madhur","y":2025,"vn":"Frontiers in Artificial Intelligence","ty":"journalArticle","tyl":"Journal article","doi":"10.3389/frai.2025.1667752","cby":0,"nref":34,"oast":"gold","cx":"Grant","co":["Bias","Interventions"],"pco":"Bias","bs":["prestige-halo","ai-llm"],"pbs":"prestige-halo","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"1","cc":4,"cp":2,"self":false,"ncoefs":0},{"id":"2BQGZI7A","t":"Persistent nepotism in peer-review","au":"Sandström, Ulf","y":2007,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-008-0211-3","cby":183,"nref":23,"oast":"bronze","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"2","cc":4,"cp":4,"self":false,"ncoefs":0},{"id":"2F4I83RM","t":"Peer Review and Citation Data in Predicting University Rankings, a Large-Scale Analysis","au":"Pride, David","y":2018,"vn":"Lecture notes in computer science","ty":"book","tyl":"Book","doi":"10.1007/978-3-030-00066-0_17","cby":15,"nref":15,"oast":"closed","cx":"General","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"book-monograph","tp":"9","cc":4,"cp":1,"self":false,"ncoefs":0},{"id":"2F6KAXNL","t":"The Reliability of Peer Reviews of Papers on Information Systems","au":"Wood, Michael","y":2004,"vn":"Journal of Information Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/0165551504041673","cby":30,"nref":20,"oast":"closed","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":3,"cp":1,"self":false,"ncoefs":1},{"id":"2G4INPDL","t":"A retrospective analysis of the peer review of more than 75,000 Marie Curie proposals between 2007 and 2018.","au":"Pina, David G","y":2021,"vn":"eLife","ty":"journalArticle","tyl":"Journal article","doi":"10.7554/eLife.59338","cby":26,"nref":45,"oast":"gold","cx":"Grant","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"8","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"2G777HQX","t":"A scoping review of simulation models of peer review.","au":"Feliciani, Thomas","y":2019,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-019-03205-w","cby":36,"nref":73,"oast":"hybrid","cx":"General","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"systematic-review-meta-analysis","tp":"0","cc":3,"cp":2,"self":false,"ncoefs":0},{"id":"2GEXWPD3","t":"Race, ethnicity, and NIH research awards.","au":"Ginther, Donna K","y":2011,"vn":"Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1126/science.1196783","cby":1087,"nref":9,"oast":"green","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["funding-decision"],"st":"observational-quantitative","tp":"14","cc":4,"cp":0,"self":false,"ncoefs":0},{"id":"2GKBJDYD","t":"Sample size and precision in NIH peer review.","au":"Kaplan, David","y":2008,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0002761","cby":56,"nref":1,"oast":"gold","cx":"Grant","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["value"],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"7","cc":1,"cp":0,"self":false,"ncoefs":0},{"id":"2H7BLLRW","t":"A structured, journal-led peer-review mentoring program enhances peer review training","au":"Lyons‐Warren, Ariel M.","y":2024,"vn":"Research Integrity and Peer Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s41073-024-00143-x","cby":13,"nref":32,"oast":"gold","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":2,"cp":1,"self":false,"ncoefs":2},{"id":"2I7XLXLK","t":"Gender bias goes away when grant reviewers focus on the science.","au":"Guglielmi, Giorgia","y":2018,"vn":"Nature","ty":"journalArticle","tyl":"Journal article","doi":"10.1038/d41586-018-01212-0","cby":23,"nref":9,"oast":"bronze","cx":"Grant","co":["Bias","Interventions"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"2","cc":4,"cp":5,"self":false,"ncoefs":0},{"id":"2KETM58Y","t":"A new approach to grant review assessments: score, then rank","au":"Gallo, Stephen A.","y":2023,"vn":"Research Integrity and Peer Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s41073-023-00131-7","cby":9,"nref":49,"oast":"gold","cx":"Grant","co":["Reliability-noise","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring","funding-decision"],"st":"simulation-methodological","tp":"8","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"2KPUFM47","t":"Using prediction markets to forecast research evaluations","au":"Munafò, Marcus R.","y":2015,"vn":"Royal Society Open Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1098/rsos.150287","cby":9,"nref":13,"oast":"gold","cx":"General","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"9","cc":1,"cp":0,"self":false,"ncoefs":0},{"id":"2LKYJJFR","t":"Addressing animal methods bias through government-led initiatives","au":"Kopew, Jessica S","y":2026,"vn":"Frontiers in Toxicology","ty":"journalArticle","tyl":"Journal article","doi":"10.3389/ftox.2026.1761759","cby":0,"nref":11,"oast":"gold","cx":"General","co":["Bias","Interventions"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"-1","cc":4,"cp":7,"self":false,"ncoefs":0},{"id":"2LXPUFDJ","t":"Does the inclusion of non-academic reviewers make any difference for grant impact panels?","au":"Luo, Junwen","y":2021,"vn":"Science and Public Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/scipol/scab046","cby":8,"nref":44,"oast":"hybrid","cx":"Grant","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["panel-consensus","reviewer-selection"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"2MRIZ4HG","t":"A systematic approach to assessing the accuracy of a peer review process in an academic journal","au":"García, J. A.","y":2025,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-025-05452-6","cby":0,"nref":30,"oast":"green","cx":"Journal","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["individual-scoring","funding-decision"],"st":"simulation-methodological","tp":"-1","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"2NTZHLW6","t":"Reliability in ethics reviews: Some initial empirical findings.","au":"Eaton, Warren O","y":1983,"vn":"Canadian Psychology / Psychologie canadienne","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/h0080684","cby":11,"nref":7,"oast":"closed","cx":"General","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"3","cc":5,"cp":3,"self":false,"ncoefs":4},{"id":"2QWWWN8B","t":"Commentary: New Guidelines for NIH Peer Review: Improving the System or Undermining It?","au":"Spiegel, Allen M.","y":2010,"vn":"Academic Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.1097/acm.0b013e3181d7e130","cby":3,"nref":3,"oast":"closed","cx":"Grant","co":["Validity","Interventions"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"-1","cc":1,"cp":0,"self":false,"ncoefs":0},{"id":"2SCD2GXP","t":"Using democracy to award research funding: an observational study.","au":"Barnett, Adrian G","y":2017,"vn":"Research integrity and peer review","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s41073-017-0040-0","cby":11,"nref":31,"oast":"gold","cx":"Grant","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["funding-decision"],"st":"observational-quantitative","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"2SJ6PABX","t":"Inter-reviewer reliability of human literature reviewing and implications for the introduction of machine-assisted systematic reviews: a mixed-methods review","au":"Hanegraaf, P.","y":2024,"vn":"BMJ Open","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmjopen-2023-076912","cby":31,"nref":21,"oast":"gold","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level","value"],"sg":["individual-scoring"],"st":"systematic-review-meta-analysis","tp":"11","cc":8,"cp":0,"self":false,"ncoefs":3},{"id":"2TIYPCIB","t":"Scientific research funding criteria: an empirical study of peer review and scientometrics","au":"Devyatkin, D.","y":2018,"vn":"Practical issues of intelligent innovations","ty":"bookSection","tyl":"Book chapter","doi":"10.1007/978-3-319-78437-3_12","cby":3,"nref":32,"oast":"closed","cx":"Grant","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["individual-scoring","funding-decision"],"st":"book-monograph","tp":"-1","cc":4,"cp":2,"self":false,"ncoefs":0},{"id":"2TWYBDP6","t":"Overlooked, Underlying: Understanding tacit criteria of proposal reviewing during a mock panel review","au":"Sims, Randi","y":2024,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.18260/1-2--43835","cby":1,"nref":24,"oast":"gold","cx":"Grant","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["individual-scoring","panel-consensus"],"st":"qualitative","tp":"8","cc":4,"cp":2,"self":false,"ncoefs":0},{"id":"2UD55EPP","t":"Guardians of science or stewards of society? The role of grant peer reviewer identity in governing socially responsible innovation","au":"Wu, Y","y":2026,"vn":"Journal of Responsible Innovation","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/23299460.2026.2674322","cby":0,"nref":37,"oast":"gold","cx":"Grant","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"qualitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"2VFUUIMD","t":"Open peer review: A randomised controlled trial","au":"Walsh, Elizabeth","y":2000,"vn":"The British Journal of Psychiatry","ty":"journalArticle","tyl":"Journal article","doi":"10.1192/bjp.176.1.47","cby":268,"nref":9,"oast":"bronze","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"experimental-rct","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"2XED69NE","t":"Canada's First Experience With a Patient-Adjudicated Cancer Research Grant Competition: A Case Study","au":"Ackerman, Ruth","y":2022,"vn":"Journal of Patient Experience","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/23743735221123424","cby":0,"nref":8,"oast":"diamond","cx":"Grant","co":["Reliability-noise","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["reviewer-selection"],"st":"observational-quantitative","tp":"7","cc":4,"cp":2,"self":false,"ncoefs":0},{"id":"2ZR5VGF7","t":"Reply to “Funding by Lottery: Political Problems and Research Opportunities”","au":"Casadevall, Arturo","y":2016,"vn":"mBio","ty":"journalArticle","tyl":"Journal article","doi":"10.1128/mbio.01401-16","cby":0,"nref":4,"oast":"gold","cx":"Grant","co":["Reliability-noise","Bias","Validity"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["funding-decision"],"st":"conceptual-commentary","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"2ZZ5VVPC","t":"Blinding reduces institutional prestige bias during initial review of applications for a young investigator award.","au":"Hultgren, Anne E","y":2024,"vn":"eLife","ty":"journalArticle","tyl":"Journal article","doi":"10.7554/eLife.92339","cby":9,"nref":16,"oast":"gold","cx":"Grant","co":["Bias","Interventions"],"pco":"Bias","bs":["prestige-halo","demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"2","cc":4,"cp":1,"self":false,"ncoefs":0},{"id":"32NEFZ9T","t":"A Difficult Balance: Editorial Peer Review in Medicine","au":"Carney, Martha J.","y":1986,"vn":"JAMA","ty":"journalArticle","tyl":"Journal article","doi":"10.1001/jama.1986.03380020117036","cby":192,"nref":2,"oast":"closed","cx":"Journal","co":["Validity","Interventions"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":3,"cp":0,"self":false,"ncoefs":0},{"id":"33C9QVWD","t":"Editorial behaviors in peer review","au":"Wang, Wei","y":2016,"vn":"SpringerPlus","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s40064-016-2601-y","cby":26,"nref":32,"oast":"diamond","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["individual-scoring","funding-decision"],"st":"simulation-methodological","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"3455VSI4","t":"External Tests of Peer Review Validity Via Impact Measures","au":"Gallo, Stephen A","y":2018,"vn":"Frontiers in Research Metrics and Analytics","ty":"journalArticle","tyl":"Journal article","doi":"10.3389/frma.2018.00022","cby":11,"nref":137,"oast":"gold","cx":"Grant","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["post-award-validity"],"st":"systematic-review-meta-analysis","tp":"9","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"35UMJRRJ","t":"Gender disparity in funding rates in double-blind grant peer review: The case of the Villum Experiment","au":"Madsen, Emil Bargmann","y":2025,"vn":"Quantitative Science Studies","ty":"journalArticle","tyl":"Journal article","doi":"10.1162/qss.a.5","cby":1,"nref":98,"oast":"gold","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["funding-decision"],"st":"observational-quantitative","tp":"2","cc":4,"cp":5,"self":false,"ncoefs":0},{"id":"36AR47M3","t":"GKAPPA: Generalized Kappa Coefficient","au":"Uebersax, John S.","y":1981,"vn":"Applied Psychological Measurement","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/014662168100500104","cby":40,"nref":1,"oast":"closed","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["whole-process"],"st":"simulation-methodological","tp":"4","cc":5,"cp":0,"self":false,"ncoefs":0},{"id":"37SMEXTX","t":"Randomisation as a tool for organisational decision-making: a debatable or debilitating proposition?","au":"Dushnitsky, Gary","y":2023,"vn":"Industry and Innovation","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/13662716.2023.2281983","cby":4,"nref":67,"oast":"bronze","cx":"General","co":["Bias","Validity","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["funding-decision"],"st":"conceptual-commentary","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"3956JT8Q","t":"Development and application of a comprehensive glossary for the identification of statistical and methodological concepts in peer review reports","au":"Buljan, Ivan","y":2024,"vn":"Journal of Informetrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.joi.2024.101555","cby":2,"nref":45,"oast":"green","cx":"Journal","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"3B9JRHFK","t":"Panel peer review of grant applications: what do we know from research in social psychology on judgment and decision-making in groups?","au":"Olbrecht, Meike","y":2010,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.3152/095820210X12809191250762","cby":65,"nref":0,"oast":"closed","cx":"Grant","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["panel-consensus"],"st":"conceptual-commentary","tp":"13","cc":1,"cp":0,"self":false,"ncoefs":0},{"id":"3DCVN5UP","t":"Bias and effort in peer review","au":"García, J. A.","y":2015,"vn":"Journal of the Association for Information Science and Technology","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/asi.23307","cby":25,"nref":23,"oast":"closed","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["reviewer-selection","individual-scoring"],"st":"simulation-methodological","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"3FFMLXGD","t":"Blinding Models for Scientific Peer-Review of Biomedical Research Proposals: A Systematic Review","au":"Qussini, Seba","y":2023,"vn":"Journal of Empirical Research on Human Research Ethics","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/15562646231191424","cby":6,"nref":51,"oast":"closed","cx":"Grant","co":["Bias","Interventions"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["individual-scoring"],"st":"systematic-review-meta-analysis","tp":"10","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"3FUV2PR2","t":"How do referees integrate evaluation criteria into their overall judgment? Evidence from grant peer review","au":"Hug, Sven E.","y":2024,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-023-04915-y","cby":13,"nref":90,"oast":"hybrid","cx":"Grant","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":4,"cp":2,"self":false,"ncoefs":0},{"id":"3HN82VIN","t":"No Agreement Without Loss: Learning and Social Choice in Peer Review","au":"Barceló, Pablo","y":2023,"vn":"Frontiers in artificial intelligence and applications","ty":"book","tyl":"Book","doi":"10.3233/faia230270","cby":0,"nref":29,"oast":"hybrid","cx":"Journal","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":["level"],"sg":["individual-scoring"],"st":"book-monograph","tp":"0","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"3KXX595H","t":"Menage a quoi? Optimal number of peer reviewers.","au":"Snell, Richard R","y":2015,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0120838","cby":39,"nref":41,"oast":"gold","cx":"Grant","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level","value"],"sg":["reviewer-selection","individual-scoring"],"st":"simulation-methodological","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"3LCPQ3XU","t":"Developing a Criteria Framework for Peer Review: A Critical Interpretive Synthesis","au":"Li, Yifei","y":2025,"vn":"Learned Publishing","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/leap.2016","cby":1,"nref":68,"oast":"hybrid","cx":"Journal","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"systematic-review-meta-analysis","tp":"-1","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"3MLFBQHS","t":"Exploring the Impact of Generative AI on Peer Review: Insights from Journal Reviewers","au":"Ebadi, Saman","y":2025,"vn":"Journal of Academic Ethics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s10805-025-09604-4","cby":30,"nref":39,"oast":"hybrid","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["ai-llm"],"pbs":"ai-llm","rs":[],"sg":["individual-scoring"],"st":"qualitative","tp":"6","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"3MVDPR4T","t":"Peer reviewer topic choice and its impact on interrater reliability: A mixed-method study","au":"Feliciani, Thomas","y":2022,"vn":"Quantitative Science Studies","ty":"journalArticle","tyl":"Journal article","doi":"10.1162/qss_a_00207","cby":8,"nref":66,"oast":"gold","cx":"Grant","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level","factors"],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"3MWL4DVH","t":"Beyond declarations: Metrics, rankings and responsible assessment","au":"Morgan-Thomas, Anna","y":2024,"vn":"Research Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.respol.2024.105093","cby":24,"nref":53,"oast":"hybrid","cx":"General","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["funding-decision"],"st":"observational-quantitative","tp":"9","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"3QAW4CJ4","t":"NIH Peer Review Reform—Change We Need, or Lipstick on a Pig?","au":"Fang, Ferric C.","y":2009,"vn":"Infection and Immunity","ty":"journalArticle","tyl":"Journal article","doi":"10.1128/iai.01567-08","cby":41,"nref":28,"oast":"green","cx":"Grant","co":["Reliability-noise","Bias","Validity","Interventions"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"3RPSC5JB","t":"The Feeling Rules of Peer Review: Defining, Displaying, and Managing Emotions in Evaluation for Research Funding","au":"Brunet, Lucas","y":2023,"vn":"Minerva","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11024-023-09518-8","cby":10,"nref":35,"oast":"hybrid","cx":"Grant","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["individual-scoring","panel-consensus"],"st":"qualitative","tp":"8","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"3S8IX9X5","t":"A Note on Estimating the Reliability of Categorical Data","au":"Finn, Rachael","y":1970,"vn":"Educational and Psychological Measurement","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/001316447003000106","cby":205,"nref":2,"oast":"closed","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"4","cc":5,"cp":0,"self":false,"ncoefs":0},{"id":"3TDTASF9","t":"Grant funding: Playing the odds","au":"Fang, Ferric C.","y":2016,"vn":"Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1126/science.352.6282.158-a","cby":13,"nref":10,"oast":"closed","cx":"Grant","co":["Bias","Validity","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["funding-decision"],"st":"conceptual-commentary","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"3TRWDK3E","t":"Decentralized knowledge assessment","au":"Sun, Kai","y":2025,"vn":"The Innovation","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.xinn.2025.100945","cby":0,"nref":60,"oast":"gold","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["reviewer-selection","whole-process"],"st":"simulation-methodological","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"43PNVEVR","t":"'Are you siding with a personality or the grant proposal?': observations on how peer review panels function.","au":"Coveney, John","y":2017,"vn":"Research Integrity and Peer Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s41073-017-0043-x","cby":32,"nref":21,"oast":"gold","cx":"Grant","co":["Bias","Interventions"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["panel-consensus"],"st":"qualitative","tp":"8","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"43RMRTWZ","t":"Augmented Peer Review: A Framework for AI-Supervised Academic Publishing","au":"Martin, Matt","y":2026,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31222/osf.io/cxarz_v1","cby":0,"nref":0,"oast":"gold","cx":"Journal","co":["Reliability-noise","Bias","Interventions"],"pco":"Reliability-noise","bs":["ai-llm","demographic","prestige-halo"],"pbs":"demographic","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"6","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"44PDV7ET","t":"Studying grant decision-making: a linguistic analysis of review reports.","au":"van den Besselaar, Peter","y":2018,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-018-2848-x","cby":61,"nref":37,"oast":"hybrid","cx":"Grant","co":["Reliability-noise","Bias","Validity"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring","panel-consensus"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"4BW48ANW","t":"Participation and influence in publication review proceedings: A reply.","au":"Lindsey, Duncan","y":1977,"vn":"American Psychologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1037//0003-066x.32.7.579","cby":18,"nref":0,"oast":"closed","cx":"Journal","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["reviewer-selection"],"st":"observational-quantitative","tp":"0","cc":5,"cp":0,"self":false,"ncoefs":0},{"id":"4C6DKMM7","t":"Evidence on peer review—scientific quality control or smokescreen?","au":"Goldbeck-Wood, Sandy","y":1999,"vn":"BMJ","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmj.318.7175.44","cby":96,"nref":23,"oast":"green","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"4CDE9Y6H","t":"Peer review and gender bias: A study on 145 scholarly journals","au":"Squazzoni, Flaminio","y":2021,"vn":"Science Advances","ty":"journalArticle","tyl":"Journal article","doi":"10.1126/sciadv.abd0299","cby":200,"nref":51,"oast":"gold","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"5","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"4EBV6BAJ","t":"Academic Journals, Incentives, and the Quality of Peer Review: A Model","au":"Zollman, Kevin","y":2023,"vn":"Philosophy of Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/psa.2023.81","cby":8,"nref":51,"oast":"hybrid","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["funding-decision"],"st":"simulation-methodological","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"4EEVRSNC","t":"A Computer Program for Assessing the Reliability and Systematic Bias of Individual Measurements1","au":"Cicchetti, Domenic V.","y":1976,"vn":"Educational and Psychological Measurement","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/001316447603600329","cby":79,"nref":10,"oast":"closed","cx":"Analogue","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"4","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"4F4Y5QU5","t":"Should we fund research randomly? An epistemological criticism of the lottery model as an alternative to peer review for the funding of science","au":"Bedessem, Baptiste","y":2019,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvz034","cby":20,"nref":43,"oast":"closed","cx":"Grant","co":["Bias","Validity"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["funding-decision"],"st":"conceptual-commentary","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"4F79EKHL","t":"Evaluation, culture, and the world of the American professor.","au":"McLaughlin, Neil","y":2012,"vn":"Sociological Forum","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/j.1573-7861.2012.01331.x","cby":0,"nref":7,"oast":"closed","cx":"Grant","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring","panel-consensus"],"st":"conceptual-commentary","tp":"-1","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"4FDFYDIR","t":"An output evaluation of a health research foundation’s enhanced grant review process for new investigators","au":"Hammond, Gregory","y":2017,"vn":"Health Research Policy and Systems","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s12961-017-0220-x","cby":1,"nref":12,"oast":"gold","cx":"Grant","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["post-award-validity"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":0,"self":false,"ncoefs":0},{"id":"4H22B8FV","t":"The effectiveness of implementing statcheck in the peer review process to avoid statistical reporting errors","au":"Nuijten, Michele B.","y":2023,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31234/osf.io/bxau9","cby":9,"nref":7,"oast":"gold","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"11","cc":2,"cp":0,"self":false,"ncoefs":0},{"id":"4H7URGN2","t":"Partial lottery can make grant allocation more fair, more efficient, and more diverse","au":"Horbach, Serge P. J. M.","y":2022,"vn":"Science and Public Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/scipol/scac009","cby":11,"nref":22,"oast":"bronze","cx":"Grant","co":["Reliability-noise","Bias","Validity","Interventions"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["funding-decision"],"st":"conceptual-commentary","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"4IAVPQUX","t":"Early Career Researchers on all Aspects of Peer Review: A Deep Dive Into the Data","au":"Nicholas, David","y":2025,"vn":"Learned Publishing","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/leap.2002","cby":5,"nref":39,"oast":"hybrid","cx":"Journal","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"qualitative","tp":"-1","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"4IR9JMGM","t":"How random is the review outcome? A systematic study of the impact of external factors on eLife peer review","au":"Liang, Weixin","y":2023,"vn":"bioRxiv (Cold Spring Harbor Laboratory)","ty":"journalArticle","tyl":"Journal article","doi":"10.1101/2023.01.04.522708","cby":1,"nref":29,"oast":"green","cx":"Journal","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":["factors"],"sg":["whole-process"],"st":"observational-quantitative","tp":"0","cc":3,"cp":5,"self":false,"ncoefs":0},{"id":"4JQYGGWI","t":"Alternative models of funding curiosity-driven research","au":"Gigerenzer, Gerd","y":2025,"vn":"Proceedings of the National Academy of Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1073/pnas.2401237121","cby":5,"nref":58,"oast":"hybrid","cx":"Grant","co":["Reliability-noise","Bias","Validity","Interventions"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"4KJWXB4W","t":"Towards a more informed and balanced use of scientific performance metrics","au":"Denissen, Jaap J A","y":2026,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvag023","cby":0,"nref":61,"oast":"hybrid","cx":"General","co":["Bias","Validity"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"4RYFCKV2","t":"Does the perceived quality of interdisciplinary research vary between fields?","au":"Thelwall, Mike","y":2023,"vn":"Journal of Documentation","ty":"journalArticle","tyl":"Journal article","doi":"10.1108/jd-01-2023-0012","cby":8,"nref":63,"oast":"green","cx":"General","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"9","cc":3,"cp":2,"self":false,"ncoefs":0},{"id":"4SWWEB8Z","t":"Personal experience with AI-generated peer reviews: a case study","au":"Vecchio, Nicholas Lo","y":2025,"vn":"Research Integrity and Peer Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s41073-025-00161-3","cby":9,"nref":25,"oast":"gold","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"6","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"4T8H83NM","t":"The effect of masking manuscripts for the peer-review process of an ophthalmic journal","au":"Isenberg, S J","y":2009,"vn":"British Journal of Ophthalmology","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bjo.2008.151886","cby":33,"nref":13,"oast":"closed","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["prestige-halo"],"pbs":"prestige-halo","rs":[],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"10","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"4WZHZP49","t":"To fund or not to fund?","au":"Shailes, Sarah","y":2017,"vn":"eLife","ty":"journalArticle","tyl":"Journal article","doi":"10.7554/eLife.32015","cby":2,"nref":1,"oast":"gold","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["funding-decision"],"st":"conceptual-commentary","tp":"-1","cc":6,"cp":0,"self":false,"ncoefs":0},{"id":"4XB7F74M","t":"Are numerical scores important for grant assessment? A cross-sectional study","au":"Buljan, Ivan","y":2024,"vn":"F1000Research","ty":"journalArticle","tyl":"Journal article","doi":"10.12688/f1000research.139743.2","cby":0,"nref":20,"oast":"gold","cx":"Grant","co":["Validity","Interventions"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"4YLCI9SK","t":"Kappa Reliabilities for Continuous Behaviors and Events","au":"Conger, Anthony J.","y":1985,"vn":"Educational and Psychological Measurement","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/0013164485454016","cby":59,"nref":8,"oast":"closed","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["whole-process"],"st":"simulation-methodological","tp":"4","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"4YUT5W5I","t":"The effect of number of rating scale categories on levels of interrater reliability : A monte carlo investigation","au":"Cicchetti, Domenic V.","y":1985,"vn":"Applied psychological measurement","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/014662168500900103","cby":291,"nref":44,"oast":"green","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"4","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"4ZG9JFXU","t":"Does incentive provision increase the quality of peer review? An experimental study","au":"Squazzoni, Flaminio","y":2012,"vn":"Research Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.respol.2012.04.014","cby":103,"nref":73,"oast":"closed","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"experimental-rct","tp":"-1","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"52D3XAIF","t":"Differences in Review Quality and Recommendations for Publication Between Peer Reviewers Suggested by Authors or by Editors","au":"Schroter, Sara","y":2006,"vn":"JAMA","ty":"journalArticle","tyl":"Journal article","doi":"10.1001/jama.295.3.314","cby":147,"nref":10,"oast":"closed","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["reviewer-selection","individual-scoring"],"st":"observational-quantitative","tp":"3","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"54CMD3XJ","t":"The many shapes of peer review: how to make journal review and preprint review activities more complementary?","au":"Puebla, Iratxe","y":2021,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31219/osf.io/t65a2","cby":0,"nref":18,"oast":"gold","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"54HK26IV","t":"Effect of written feedback by editors on quality of reviews: two randomized trials.","au":"Callaham, Michael L","y":2002,"vn":"The Journal of the American Medical Association","ty":"journalArticle","tyl":"Journal article","doi":"10.1001/jama.287.21.2781","cby":75,"nref":6,"oast":"closed","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"experimental-rct","tp":"11","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"55TCQDVA","t":"What is expected of people who lead meetings where the goal is to reach consensus? A scoping review with implications for improving the quality of health research grant peer review and clinical guideline development","au":"Ghannad, Mona","y":2025,"vn":"medRxiv","ty":"journalArticle","tyl":"Journal article","doi":"10.1101/2025.02.03.25320968","cby":0,"nref":47,"oast":"green","cx":"General","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["panel-consensus"],"st":"systematic-review-meta-analysis","tp":"7","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"57U6GUR2","t":"On various intraclass correlation reliability coefficients.","au":"Bartko, John J.","y":1976,"vn":"Psychological Bulletin","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/0033-2909.83.5.762","cby":816,"nref":5,"oast":"closed","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"4","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"58QYQUYU","t":"“Excellence R Us”: university research and the fetishisation of excellence","au":"Moore, Samuel","y":2017,"vn":"Palgrave Communications","ty":"journalArticle","tyl":"Journal article","doi":"10.1057/palcomms.2016.105","cby":275,"nref":152,"oast":"gold","cx":"General","co":["Reliability-noise","Bias","Validity"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"5BYI68MN","t":"Peer review of grant applications in biology and medicine. Reliability, fairness, and validity","au":"Reinhart, Martin","y":2009,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-008-2220-7","cby":78,"nref":49,"oast":"closed","cx":"Grant","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level","factors"],"sg":["individual-scoring","funding-decision"],"st":"qualitative","tp":"-1","cc":1,"cp":4,"self":false,"ncoefs":0},{"id":"5D8PZXA5","t":"Effective operation of competitive research funding systems","au":"OECD","y":2018,"vn":"OECD science, technology and industry policy papers","ty":"journalArticle","tyl":"Journal article","doi":"10.1787/2ae8c0dc-en","cby":23,"nref":44,"oast":"bronze","cx":"Grant","co":["Reliability-noise","Validity","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"5HK3SW77","t":"Can generative AI effectively perform quality evaluation within social sciences? A case study in library and information science","au":"Zhu, Yu","y":2026,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-026-05570-9","cby":3,"nref":32,"oast":"hybrid","cx":"General","co":["Reliability-noise","Bias","Validity"],"pco":"Reliability-noise","bs":["ai-llm"],"pbs":"ai-llm","rs":["level"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"6","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"5IMAHJFT","t":"“The Grievance Studies Affair” Project: Reconstructing and Assessing the Experimental Design","au":"Lagerspetz, Mikko","y":2020,"vn":"Science Technology & Human Values","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/0162243920923087","cby":37,"nref":21,"oast":"hybrid","cx":"Journal","co":["Reliability-noise","Bias","Validity"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"5","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"5KLQT2J8","t":"Editor and reviewer gender influence the peer review process but not peer review outcomes at an ecology journal","au":"Fox, Charles W.","y":2015,"vn":"Functional Ecology","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/1365-2435.12529","cby":129,"nref":68,"oast":"hybrid","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["reviewer-selection","individual-scoring"],"st":"observational-quantitative","tp":"5","cc":3,"cp":5,"self":false,"ncoefs":0},{"id":"5MPYUIC3","t":"An Investigation of Social-Behavioral Phenomena in the Peer-Review Processes of Scientific Foundations","au":"Kleiner, George","y":2019,"vn":"Modeling and Simulation of Social-Behavioral Phenomena in Creative Societies: First International EURO Mini Conference, MSBC 2019, Vilnius, Lithuania, September 18–20, 2019, Proceedings","ty":"bookSection","tyl":"Book chapter","doi":"10.1007/978-3-030-29862-3_6","cby":2,"nref":27,"oast":"closed","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["prestige-halo","cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["individual-scoring","funding-decision"],"st":"book-monograph","tp":"-1","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"5NYG9LP8","t":"Inequity in peer review in communication sciences and disorders","au":"Girolamo, Teresa","y":2021,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31234/osf.io/ygprn","cby":0,"nref":55,"oast":"gold","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"-1","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"5P62SBYZ","t":"Effect of Blinded Peer Review on Abstract Acceptance","au":"Ross, Joseph S.","y":2006,"vn":"JAMA","ty":"journalArticle","tyl":"Journal article","doi":"10.1001/jama.295.14.1675","cby":249,"nref":13,"oast":"closed","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["demographic","prestige-halo"],"pbs":"demographic","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"10","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"5PTU7R5C","t":"Author review of reviewers.","au":"Hall, Judith A.","y":1979,"vn":"American Psychologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/0003-066x.34.9.798.a","cby":96,"nref":0,"oast":"closed","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["reviewer-selection"],"st":"conceptual-commentary","tp":"3","cc":5,"cp":0,"self":false,"ncoefs":0},{"id":"5RWLPM53","t":"Science policies: How should science funding be allocated? An evolutionary biologists’ perspective","au":"Meirmans, Stephanie","y":2019,"vn":"Journal of Evolutionary Biology","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/jeb.13497","cby":31,"nref":58,"oast":"hybrid","cx":"Grant","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"5T5MS6AZ","t":"The Matthew effect and early-career setbacks in research funding—a replication study","au":"Traag, Vincent","y":2025,"vn":"eLife","ty":"journalArticle","tyl":"Journal article","doi":"10.7554/elife.109042.1","cby":1,"nref":32,"oast":"gold","cx":"Grant","co":["Bias","Validity"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["post-award-validity"],"st":"observational-quantitative","tp":"2","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"5U7HVU43","t":"Equitable Research Funding: Strategies, Challenges and the Role of Funding Agencies.","au":"Ruediger, Sarah","y":2025,"vn":"The European journal of neuroscience","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/ejn.70160","cby":7,"nref":41,"oast":"bronze","cx":"Grant","co":["Bias","Interventions"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"14","cc":4,"cp":2,"self":false,"ncoefs":0},{"id":"5YACCVHF","t":"Jury Theorems for Peer Review","au":"Arvan, Marcus","y":2022,"vn":"The British Journal for the Philosophy of Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1086/719117","cby":3,"nref":47,"oast":"hybrid","cx":"Journal","co":["Reliability-noise","Validity"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"5YREXS39","t":"Conclave in the Tower of Babel: how peers review interdisciplinary research proposals","au":"Laudel, Grit","y":2006,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.3152/147154406781776048","cby":69,"nref":0,"oast":"closed","cx":"Grant","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["reviewer-selection","panel-consensus"],"st":"qualitative","tp":"9","cc":1,"cp":0,"self":false,"ncoefs":0},{"id":"5ZF6XD7M","t":"Can ChatGPT evaluate research quality?","au":"Thelwall, Mike","y":2024,"vn":"Journal of Data and Information Science","ty":"journalArticle","tyl":"Journal article","doi":"10.2478/jdis-2024-0013","cby":58,"nref":23,"oast":"diamond","cx":"General","co":["Reliability-noise","Bias","Validity"],"pco":"Reliability-noise","bs":["ai-llm"],"pbs":"ai-llm","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"6","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"5ZRC7NDD","t":"Impact of Alumni Connections on Peer Review Ratings and Selection Success Rate in National Research","au":"Jang, Duckhee","y":2016,"vn":"Science Technology & Human Values","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/0162243916665466","cby":20,"nref":59,"oast":"closed","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"-1","cc":4,"cp":1,"self":false,"ncoefs":0},{"id":"5ZWLBYYK","t":"American Idol and NIH Grant Review","au":"Pagano, Michele","y":2006,"vn":"Cell","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.cell.2006.08.004","cby":17,"nref":2,"oast":"bronze","cx":"Grant","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["individual-scoring","panel-consensus"],"st":"conceptual-commentary","tp":"-1","cc":1,"cp":0,"self":false,"ncoefs":0},{"id":"625FMAPA","t":"Improving Participation and Interrater Agreement in Scoring Ambulatory Pediatric Association Abstracts","au":"Kemper, Kathi J.","y":1996,"vn":"Archives of Pediatrics and Adolescent Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.1001/archpedi.1996.02170290046007","cby":21,"nref":28,"oast":"closed","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level","value"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"15","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"64U3MRH9","t":"Uncertainty in peer review","au":"Barnett, Adrian","y":2024,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31219/osf.io/fm6st","cby":1,"nref":59,"oast":"gold","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"6754BVQM","t":"Quantitative Impact of Including Consumers in the Scientific Review of Breast Cancer Research Proposals","au":"Andejeski, Yvonne","y":2002,"vn":"Journal of Women s Health & Gender-Based Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.1089/152460902317586010","cby":53,"nref":15,"oast":"closed","cx":"Grant","co":["Reliability-noise","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["panel-consensus","individual-scoring"],"st":"observational-quantitative","tp":"7","cc":1,"cp":0,"self":false,"ncoefs":0},{"id":"67M7Y9GH","t":"Fund ideas, not pedigree, to find fresh insight","au":"Sinkjær, Thomas","y":2018,"vn":"Nature","ty":"journalArticle","tyl":"Journal article","doi":"10.1038/d41586-018-02743-2","cby":14,"nref":0,"oast":"bronze","cx":"Grant","co":["Bias","Interventions"],"pco":"Bias","bs":["prestige-halo"],"pbs":"prestige-halo","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"1","cc":4,"cp":0,"self":false,"ncoefs":0},{"id":"68C8KWK7","t":"La revisión por pares en las áreas de filosofía y filosofía moral en España. Creencias y debates sobre su consolidación, su idoneidad y su tipo ideal","au":"Feenstra, Ramón A.","y":2024,"vn":"Revista Española de Sociología","ty":"journalArticle","tyl":"Journal article","doi":"10.22325/fes/res.2024.209","cby":1,"nref":36,"oast":"hybrid","cx":"Journal","co":["Reliability-noise","Bias","Validity"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"-1","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"6BPUB4IR","t":"Challenges, experiments, and computational solutions in peer review","au":"Shah, Nihar B.","y":2022,"vn":"Communications of the ACM","ty":"journalArticle","tyl":"Journal article","doi":"10.1145/3528086","cby":62,"nref":19,"oast":"bronze","cx":"Journal","co":["Reliability-noise","Bias","Interventions"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":3,"cp":0,"self":false,"ncoefs":0},{"id":"6E7729LN","t":"Peer review of grant applications: a simple method to identify proposals with discordant reviews.","au":"Giraudeau, Bruno","y":2011,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0027557","cby":8,"nref":12,"oast":"gold","cx":"Grant","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring","panel-consensus"],"st":"simulation-methodological","tp":"8","cc":1,"cp":4,"self":false,"ncoefs":1},{"id":"6EA2KUS7","t":"Innovating peer review, reconfiguring scholarly communication: an analytical overview of ongoing peer review innovation activities","au":"Kaltenbrunner, Wolfgang","y":2022,"vn":"Journal of Documentation","ty":"journalArticle","tyl":"Journal article","doi":"10.1108/JD-01-2022-0022","cby":31,"nref":28,"oast":"hybrid","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"qualitative","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"6EE23NG2","t":"Forming inferences about some intraclass correlation coefficients.","au":"McGraw, Kenneth O.","y":1996,"vn":"Psychological Methods","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/1082-989X.1.1.30","cby":6705,"nref":26,"oast":"closed","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"4","cc":6,"cp":3,"self":false,"ncoefs":0},{"id":"6ELG25E6","t":"Decision-making approaches used by UK and international health funding organisations for allocating research funds: A survey of current practice.","au":"Meadmore, Katie","y":2020,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0239757","cby":12,"nref":42,"oast":"gold","cx":"Grant","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"7","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"6EQT27QI","t":"Do Peer Reviews Predict Impact? Evidence from the American Sociological Review , 1978 to 1982","au":"Teplitskiy, Misha","y":2016,"vn":"Socius : sociological research for a dynamic world","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/2378023116640278","cby":4,"nref":54,"oast":"gold","cx":"Journal","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["post-award-validity"],"st":"observational-quantitative","tp":"-1","cc":3,"cp":2,"self":false,"ncoefs":0},{"id":"6H732CUX","t":"Interreferee agreement and acceptance rates in physics","au":"Lazarus, David","y":1982,"vn":"Behavioral and Brain Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/s0140525x00011468","cby":50,"nref":199,"oast":"closed","cx":"Journal","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":["level"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"12","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"6HCFA5SL","t":"Effect on peer review of telling reviewers that their signed reviews might be posted on the web: randomised controlled trial.","au":"van Rooyen, Susan","y":2010,"vn":"BMJ (Clinical Research Ed.)","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmj.c5729","cby":169,"nref":22,"oast":"hybrid","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"experimental-rct","tp":"11","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"6I4774BQ","t":"A scoping review on the roles and tasks of peer reviewers in the manuscript review process in biomedical journals.","au":"Glonti, Ketevan","y":2019,"vn":"BMC Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s12916-019-1347-0","cby":84,"nref":27,"oast":"gold","cx":"Journal","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"systematic-review-meta-analysis","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"6ISHLVGN","t":"Examining uncertainty in journal peer reviewers’ recommendations: a cross-sectional study","au":"Barnett, Adrian","y":2024,"vn":"Royal Society Open Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1098/rsos.240612","cby":6,"nref":72,"oast":"gold","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["factors"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"3","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"6JA7SUSX","t":"An evolutionary explanation of assassins and zealots in peer review","au":"Chamorro-Padial, Jorge","y":2019,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-019-03171-3","cby":7,"nref":23,"oast":"green","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"6PGVZK3J","t":"Improving Peer Review in Economics: Stocktaking and Proposals","au":"Charness, Gary","y":2025,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31222/osf.io/k9ybz_v1","cby":3,"nref":0,"oast":"gold","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"0","cc":3,"cp":0,"self":false,"ncoefs":0},{"id":"6VBQCX25","t":"Mitigating animal methods bias to reduce animal use and improve biomedical translation","au":"Kavanagh, Owen","y":2024,"vn":"Science Progress","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/00368504241253693","cby":2,"nref":22,"oast":"diamond","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"-1","cc":2,"cp":7,"self":false,"ncoefs":0},{"id":"6W6MH3MT","t":"NIH peer review percentile scores are poorly predictive of grant productivity.","au":"Fang, Ferric C","y":2016,"vn":"eLife","ty":"journalArticle","tyl":"Journal article","doi":"10.7554/eLife.13323","cby":113,"nref":18,"oast":"gold","cx":"Grant","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["post-award-validity"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"6W76X9PV","t":"Defenses against LLM prompt injections in academic peer review","au":"Zika, Ondrej","y":2025,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31234/osf.io/zxuwf_v2","cby":0,"nref":0,"oast":"gold","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["ai-llm"],"pbs":"ai-llm","rs":[],"sg":["individual-scoring"],"st":"experimental-rct","tp":"6","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"6XTFN68E","t":"Evaluation of research proposals by peer review panels: broader panels for broader assessments?","au":"Abma-Schouten, Rebecca","y":2023,"vn":"Science and Public Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/scipol/scad009","cby":13,"nref":55,"oast":"hybrid","cx":"Grant","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["panel-consensus"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"6YBF2W5W","t":"Registered report protocol: Factors associated with inter-rater agreement in grant peer review.","au":"Hesselberg, Jan-Ole","y":2025,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0322696","cby":0,"nref":26,"oast":"gold","cx":"Grant","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level","factors"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"8","cc":6,"cp":2,"self":true,"ncoefs":0},{"id":"6Z32RSBF","t":"The state of the art in peer review","au":"Tennant, Jonathan","y":2018,"vn":"FEMS Microbiology Letters","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/femsle/fny204","cby":165,"nref":103,"oast":"hybrid","cx":"Journal","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"72ZM4PVC","t":"Designing grant-review panels for better funding decisions: Lessons from an empirically calibrated simulation model","au":"Feliciani, Thomas","y":2022,"vn":"Research Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.respol.2021.104467","cby":14,"nref":46,"oast":"hybrid","cx":"Grant","co":["Reliability-noise","Validity","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level","factors"],"sg":["individual-scoring","panel-consensus","funding-decision"],"st":"simulation-methodological","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"73TNGMX8","t":"Hidden connections: Network effects on editorial decisions in four computer science journals","au":"Bravo, Giangiacomo","y":2018,"vn":"Journal of Informetrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.joi.2017.12.002","cby":54,"nref":38,"oast":"hybrid","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["prestige-halo"],"pbs":"prestige-halo","rs":[],"sg":["funding-decision"],"st":"observational-quantitative","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"74M956VR","t":"The effect of the “very important paper” (VIP) designation in Angewandte Chemie International Edition on citation impact: A propensity score matching analysis","au":"Mutz, Rüdiger","y":2016,"vn":"Journal of the Association for Information Science and Technology","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/asi.23701","cby":18,"nref":78,"oast":"hybrid","cx":"Journal","co":["Reliability-noise","Validity"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":3,"cp":4,"self":false,"ncoefs":1},{"id":"7596286J","t":"Malice's wonderland: Research funding and peer review","au":"Osmond, D. H.","y":1983,"vn":"Journal of Neurobiology","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/neu.480140202","cby":46,"nref":0,"oast":"closed","cx":"Grant","co":["Bias","Validity"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"-1","cc":1,"cp":0,"self":false,"ncoefs":0},{"id":"76KIJ2D8","t":"Efficacy of Double-Blind Peer Review in an Imaging Subspecialty Journal","au":"O’Connor, Erin E.","y":2016,"vn":"American Journal of Neuroradiology","ty":"journalArticle","tyl":"Journal article","doi":"10.3174/ajnr.a5017","cby":37,"nref":41,"oast":"bronze","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["prestige-halo"],"pbs":"prestige-halo","rs":[],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"10","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"77VRC7BI","t":"Can \"results blind manuscript evaluation\" assuage \"publication bias\"?","au":"Hyman, Michael R","y":2017,"vn":"Basic and Applied Social Psychology","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/01973533.2017.1350581","cby":15,"nref":16,"oast":"closed","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"10","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"78GNLVBD","t":"Peer review in a changing world: An international study measuring the attitudes of researchers","au":"Mulligan, Adrian","y":2013,"vn":"Journal of the American Society for Information Science and Technology","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/asi.22798","cby":400,"nref":34,"oast":"closed","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"78VLE8B6","t":"Peer review assessment of originality in tourism journals: critical perspective of key gatekeepers","au":"Rodríguez, Isabel","y":2019,"vn":"Annals of Tourism Research","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.annals.2019.04.003","cby":29,"nref":60,"oast":"green","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"qualitative","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"7EKMP546","t":"Heterogeneity of inter-rater reliabilities of grant peer reviews and its determinants: a general estimating equations approach.","au":"Mutz, Rüdiger","y":2012,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0048509","cby":81,"nref":34,"oast":"gold","cx":"Grant","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":["level","factors","value"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":6,"cp":4,"self":false,"ncoefs":2},{"id":"7GYSZ988","t":"Does the disconnect between the peer-reviewed label and reality explain the peer review crisis, and can open peer review or preprints resolve it? A narrative review","au":"Silva, Jaime A. Teixeira da","y":2025,"vn":"Naunyn-Schmiedeberg s Archives of Pharmacology","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s00210-025-04486-0","cby":4,"nref":198,"oast":"closed","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"-1","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"7HL64R5H","t":"Examining Personality Psychology to Unpack the Peer Review System: Towards a More Diverse, Inclusive, and Equitable Psychological Science","au":"Atherton, Olivia E.","y":2024,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31234/osf.io/mxcj3","cby":1,"nref":0,"oast":"gold","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["whole-process"],"st":"qualitative","tp":"-1","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"7JG4HLD9","t":"Substantial Agreement of Referee Recommendations at a General Medical Journal – A Peer Review Evaluation at Deutsches Ärzteblatt International","au":"Baethge, Christopher","y":2013,"vn":"PLoS ONE","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0061401","cby":34,"nref":18,"oast":"gold","cx":"Journal","co":["Reliability-noise","Validity"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":5,"cp":3,"self":false,"ncoefs":6},{"id":"7KXBVRH5","t":"Transparency of peer review: a semi-structured interview study with editors in chief from Social Sciences and Humanities","au":"Karhulahti, Veli-Matti","y":2021,"vn":"Research Integrity and Peer Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s41073-021-00116-4","cby":23,"nref":38,"oast":"gold","cx":"Journal","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"qualitative","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"7KZ9486Z","t":"Author-suggested reviewers rate manuscripts much more favorably: A cross-sectional analysis of the neuroscience section of PLOS ONE","au":"Acuña, Daniel E.","y":2022,"vn":"PLoS ONE","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0273994","cby":2,"nref":18,"oast":"gold","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["reviewer-selection","individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"7LDKWQXY","t":"Gender differences in peer review of innovation","au":"Belz, Andrea","y":2022,"vn":"Strategic Entrepreneurship Journal","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/sej.1429","cby":24,"nref":101,"oast":"closed","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"2","cc":4,"cp":5,"self":false,"ncoefs":0},{"id":"7QJBZY4V","t":"Beyond Gatekeeping: Reassessing Peer Review and Establishing Preprints as the Future of Scientific Communication","au":"Baker, Maher Asaad","y":2025,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31235/osf.io/69uhe_v1","cby":0,"nref":0,"oast":"gold","cx":"Journal","co":["Reliability-noise","Bias","Interventions"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":["factors"],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"7S4GVWVN","t":"Validating Grant-Making Processes: Construct Validity of the 2013 Senior Corps RSVP Grant Review","au":"Tan, Erwin","y":2016,"vn":"VOLUNTAS: International Journal of Voluntary and Nonprofit Organizations","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11266-015-9602-2","cby":3,"nref":37,"oast":"hybrid","cx":"Grant","co":["Reliability-noise","Validity"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["factors","value"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"8","cc":4,"cp":2,"self":false,"ncoefs":1},{"id":"7TV4V8FE","t":"Making Reviewers Visible","au":"Godlee, Fiona","y":2002,"vn":"JAMA","ty":"journalArticle","tyl":"Journal article","doi":"10.1001/jama.287.21.2762","cby":168,"nref":14,"oast":"closed","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"7U835JX8","t":"What does better peer review look like? Underlying principles and recommendations for better practice","au":"Allen, Heidi","y":2019,"vn":"Learned Publishing","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/leap.1222","cby":59,"nref":68,"oast":"hybrid","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"guideline-policy","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"7UYA6YG5","t":"Transparency in peer review: Exploring the content and tone of reviewers’ confidential comments to editors","au":"OʼBrien, Bridget C.","y":2021,"vn":"PLoS ONE","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0260558","cby":16,"nref":26,"oast":"gold","cx":"Journal","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"7VRBLWI2","t":"Do peers share the same criteria for assessing grant applications?","au":"Hug, Sven E.","y":2021,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvab034","cby":17,"nref":102,"oast":"green","cx":"Grant","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"8","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"7VUDTF33","t":"Changing the Culture of Peer Review for a More Inclusive and Equitable Psychological Science","au":"Aly, Mariam","y":2022,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31234/osf.io/435xz","cby":7,"nref":135,"oast":"gold","cx":"General","co":["Bias","Interventions"],"pco":"Bias","bs":["demographic","bias-general"],"pbs":"demographic","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"-1","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"7W8CFBU4","t":"NIH peer review: Criterion scores completely account for racial disparities in overall impact scores.","au":"Erosheva, Elena A","y":2020,"vn":"Science Advances","ty":"journalArticle","tyl":"Journal article","doi":"10.1126/sciadv.aaz4868","cby":128,"nref":41,"oast":"gold","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"14","cc":4,"cp":2,"self":false,"ncoefs":0},{"id":"7WB2GCPB","t":"“Let the community decide”? The vision and reality of soundness-only peer review in open-access mega-journals","au":"Spezi, Valérie","y":2017,"vn":"Journal of Documentation","ty":"journalArticle","tyl":"Journal article","doi":"10.1108/jd-06-2017-0092","cby":61,"nref":34,"oast":"hybrid","cx":"Journal","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["individual-scoring","funding-decision"],"st":"qualitative","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"7WU5LEHX","t":"Who chooses open peer review and is it an indicator of article quality? An observational study of PLOS journals.","au":"Barnett, Adrian","y":2026,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31222/osf.io/2b7z5_v1","cby":0,"nref":0,"oast":"gold","cx":"Journal","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"0","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"7Z69DLU7","t":"Emerging trends in peer review—a survey","au":"Walker, Richard","y":2015,"vn":"Frontiers in Neuroscience","ty":"journalArticle","tyl":"Journal article","doi":"10.3389/fnins.2015.00169","cby":188,"nref":88,"oast":"gold","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"85LYQU4R","t":"Peering at peer review revealed high degree of chance associated with funding of grant applications.","au":"Mayo, Nancy E","y":2006,"vn":"Journal of Clinical Epidemiology","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.jclinepi.2005.12.007","cby":70,"nref":47,"oast":"bronze","cx":"Grant","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level","factors"],"sg":["individual-scoring","panel-consensus","funding-decision"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":4,"self":false,"ncoefs":1},{"id":"85QBBTET","t":"Low agreement among reviewers evaluating the same NIH grant applications.","au":"Pier, Elizabeth L","y":2018,"vn":"Proceedings of the National Academy of Sciences of the United States of America","ty":"journalArticle","tyl":"Journal article","doi":"10.1073/pnas.1714379115","cby":209,"nref":35,"oast":"closed","cx":"Grant","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"7","cc":1,"cp":2,"self":false,"ncoefs":6},{"id":"86MJ77BB","t":"Unprofessional peer reviews disproportionately harm underrepresented groups in STEM","au":"Silbiger, Nyssa J.","y":2019,"vn":"PeerJ","ty":"journalArticle","tyl":"Journal article","doi":"10.7717/peerj.8247","cby":190,"nref":43,"oast":"gold","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":2,"cp":5,"self":false,"ncoefs":0},{"id":"86W5FZUM","t":"Strategies to prevent or reduce gender bias in peer review of research grants: A rapid scoping review.","au":"Tricco, Andrea C","y":2017,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0169718","cby":68,"nref":39,"oast":"gold","cx":"Grant","co":["Bias","Interventions"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring"],"st":"systematic-review-meta-analysis","tp":"2","cc":4,"cp":5,"self":false,"ncoefs":0},{"id":"87BCIXU7","t":"When AI turns grant evaluation into a lottery","au":"Bulla, Martin","y":2026,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31222/osf.io/d8gcu_v1","cby":1,"nref":0,"oast":"gold","cx":"Grant","co":["Reliability-noise","Validity"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["value"],"sg":["individual-scoring","funding-decision"],"st":"conceptual-commentary","tp":"-1","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"88FVADPB","t":"The influence of first author sex on acceptance rates of submissions to Anaesthesia Cases.","au":"Laycock, H","y":2019,"vn":"Anaesthesia","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/anae.14797","cby":12,"nref":24,"oast":"bronze","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["funding-decision"],"st":"observational-quantitative","tp":"5","cc":4,"cp":5,"self":false,"ncoefs":0},{"id":"89999KMX","t":"Authors and editors assort on gender and geography in high-rank ecological publications","au":"Manlove, Kezia R.","y":2018,"vn":"PLoS ONE","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0192481","cby":16,"nref":30,"oast":"gold","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["demographic","prestige-halo"],"pbs":"demographic","rs":[],"sg":["funding-decision"],"st":"observational-quantitative","tp":"5","cc":3,"cp":5,"self":false,"ncoefs":0},{"id":"8AAUI9F9","t":"Interrater reliability and convergent validity of F1000Prime peer review","au":"Bornmann, Lutz","y":2015,"vn":"Journal of the Association for Information Science and Technology","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/asi.23334","cby":36,"nref":72,"oast":"closed","cx":"General","co":["Reliability-noise","Validity"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level","factors"],"sg":["individual-scoring","post-award-validity"],"st":"observational-quantitative","tp":"-1","cc":3,"cp":4,"self":false,"ncoefs":4},{"id":"8AIUI3PN","t":"High-scoring grant applications yield more highly cited papers","au":"Baker, Monya","y":2015,"vn":"Nature","ty":"journalArticle","tyl":"Journal article","doi":"10.1038/nature.2015.17380","cby":1,"nref":4,"oast":"closed","cx":"Grant","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["individual-scoring","post-award-validity"],"st":"conceptual-commentary","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"8DH86VMV","t":"Critical reflections on the journal peer review process","au":"Tight, Malcolm","y":2024,"vn":"Higher Education Research & Development","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/07294360.2024.2366035","cby":8,"nref":13,"oast":"green","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"8DHMK6TJ","t":"Peer review revisited.","au":"Meyen, Michael","y":2016,"vn":"Studies in Communication and Media","ty":"journalArticle","tyl":"Journal article","doi":"10.5771/2192-4007-2016-1-8","cby":0,"nref":19,"oast":"diamond","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"qualitative","tp":"0","cc":2,"cp":4,"self":false,"ncoefs":0},{"id":"8FCS5PFQ","t":"Bibliometrically Disciplined Peer Review: on Using Indicators in Research Evaluation","au":"Besselaar, Peter van den","y":2020,"vn":"Scholarly Assessment Reports","ty":"journalArticle","tyl":"Journal article","doi":"10.29024/sar.16","cby":14,"nref":41,"oast":"diamond","cx":"General","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"9","cc":4,"cp":2,"self":false,"ncoefs":0},{"id":"8FW39FHY","t":"Is There Gender Bias in JAMA's Peer Review Process?","au":"Gilbert, Julie R.","y":1994,"vn":"JAMA","ty":"journalArticle","tyl":"Journal article","doi":"10.1001/jama.1994.03520020065018","cby":130,"nref":13,"oast":"closed","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"5","cc":3,"cp":3,"self":false,"ncoefs":0},{"id":"8H8IKI6S","t":"And Who Will Review the Review(er)s?","au":"Potvin, Sarah","y":2017,"vn":"College & Research Libraries","ty":"journalArticle","tyl":"Journal article","doi":"10.5860/crl.78.6.734","cby":3,"nref":15,"oast":"diamond","cx":"General","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"8HWJHBN9","t":"Consensus Pervades Scientific Processes","au":"Jorm, Anthony F.","y":2024,"vn":"","ty":"book","tyl":"Book","doi":"10.1007/978-981-97-9222-1_2","cby":0,"nref":24,"oast":"gold","cx":"General","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"book-monograph","tp":"-1","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"8JXS4WJA","t":"Teleconference versus face-to-face scientific peer review of grant application: effects on review outcomes.","au":"Gallo, Stephen A","y":2013,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0071693","cby":38,"nref":22,"oast":"gold","cx":"Grant","co":["Reliability-noise","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring","panel-consensus"],"st":"observational-quantitative","tp":"13","cc":1,"cp":2,"self":false,"ncoefs":2},{"id":"8LVJQ749","t":"Predicting article quality scores with machine learning: The U.K. Research Excellence Framework","au":"Thelwall, Mike","y":2023,"vn":"Quantitative Science Studies","ty":"journalArticle","tyl":"Journal article","doi":"10.1162/qss_a_00258","cby":26,"nref":70,"oast":"gold","cx":"General","co":["Validity","Interventions"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"9","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"8P8K5JWV","t":"Gender and other potential biases in peer review: cross-sectional analysis of 38 250 external peer review reports.","au":"Severin, Anna","y":2020,"vn":"BMJ Open","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmjopen-2019-035058","cby":50,"nref":23,"oast":"gold","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["demographic","prestige-halo"],"pbs":"demographic","rs":[],"sg":["reviewer-selection","individual-scoring"],"st":"observational-quantitative","tp":"2","cc":4,"cp":5,"self":false,"ncoefs":0},{"id":"8PTHSIIY","t":"Mavericks and lotteries","au":"Avin, Shahar","y":2018,"vn":"Studies in History and Philosophy of Science Part A","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.shpsa.2018.11.006","cby":43,"nref":51,"oast":"hybrid","cx":"Grant","co":["Reliability-noise","Bias","Validity"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["funding-decision"],"st":"conceptual-commentary","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"8PYAHQPH","t":"A Guide to Peer Review in Developmental Psychology","au":"Singh, Leher","y":2025,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31234/osf.io/uzxp2_v1","cby":0,"nref":0,"oast":"gold","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"0","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"8RFSBJI6","t":"Statistical analysis of the National Institutes of Health peer review system","au":"Johnson, Valen E.","y":2008,"vn":"Proceedings of the National Academy of Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1073/pnas.0804538105","cby":56,"nref":20,"oast":"closed","cx":"Grant","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":["factors"],"sg":["individual-scoring","panel-consensus"],"st":"simulation-methodological","tp":"-1","cc":1,"cp":8,"self":false,"ncoefs":0},{"id":"8RWURVCA","t":"Feyerabend’s well-ordered science: how an anarchist distributes funds","au":"Shaw, Jamie","y":2018,"vn":"Synthese","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11229-018-02026-3","cby":24,"nref":139,"oast":"closed","cx":"Grant","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["funding-decision"],"st":"conceptual-commentary","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"8STS2ZAR","t":"Double-blind peer review is detrimental to scientific integrity","au":"Mebane, Christopher","y":2024,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31219/osf.io/shp6a","cby":0,"nref":24,"oast":"gold","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["prestige-halo"],"pbs":"prestige-halo","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"10","cc":2,"cp":0,"self":false,"ncoefs":0},{"id":"8T5LNS52","t":"Justice, efficiency and epistemology in the peer review of scientific manuscripts","au":"Mahoney, Michael J.","y":1991,"vn":"Behavioral and Brain Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/s0140525x00065900","cby":3,"nref":363,"oast":"closed","cx":"General","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"0","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"8TYZS6JZ","t":"Does mentoring new peer reviewers improve review quality? A randomized trial.","au":"Houry, Debra","y":2012,"vn":"BMC Medical Education","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/1472-6920-12-83","cby":90,"nref":22,"oast":"gold","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"experimental-rct","tp":"-1","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"8V294IF3","t":"Opening the Black Box of Grant Peer Review: Exploratory Analysis of Linguistic and Sentiment Signals in Funding Outcomes","au":"Bin, Adriana","y":2026,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31235/osf.io/ykca7_v1","cby":0,"nref":0,"oast":"closed","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"-1","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"8VT8CAJR","t":"Assassins and zealots: variations in peer review. Special report.","au":"Siegelman, SS","y":1991,"vn":"Radiology","ty":"journalArticle","tyl":"Journal article","doi":"10.1148/radiology.178.3.1994394","cby":93,"nref":1,"oast":"closed","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"10","cc":3,"cp":0,"self":false,"ncoefs":0},{"id":"8YL5NW33","t":"Experience with NIH Peer Review: Researchers' Cynicism and Desire for Change","au":"Gillespie, Gilbert W.","y":1985,"vn":"Science Technology & Human Values","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/016224398501000306","cby":47,"nref":3,"oast":"closed","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"1","cc":3,"cp":0,"self":false,"ncoefs":0},{"id":"8ZGHI9RN","t":"Facts and Myths about Refereeing","au":"Hamermesh, Daniel S.","y":1994,"vn":"The Journal of Economic Perspectives","ty":"journalArticle","tyl":"Journal article","doi":"10.1257/jep.8.1.153","cby":153,"nref":11,"oast":"bronze","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["reviewer-selection","individual-scoring"],"st":"observational-quantitative","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"93G8L2PU","t":"Structured peer review: pilot results from 23 Elsevier journals","au":"Malički, Mario","y":2024,"vn":"PeerJ","ty":"journalArticle","tyl":"Journal article","doi":"10.7717/peerj.17514","cby":18,"nref":10,"oast":"gold","cx":"Journal","co":["Reliability-noise","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":2,"cp":1,"self":false,"ncoefs":2},{"id":"948KR92W","t":"Editorial peer review for improving the quality of reports of biomedical studies.","au":"Jefferson, T","y":2007,"vn":"Cochrane Database of Systematic Reviews","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/14651858.MR000016.pub3","cby":316,"nref":89,"oast":"green","cx":"Journal","co":["Bias","Validity","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"systematic-review-meta-analysis","tp":"-1","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"952G7EYT","t":"Research funding randomly allocated? A survey of scientists’ views on peer review and lottery","au":"Philipps, Axel","y":2021,"vn":"Science and Public Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/scipol/scab084","cby":22,"nref":53,"oast":"hybrid","cx":"Grant","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["funding-decision"],"st":"observational-quantitative","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"95WGD5RH","t":"Peer Review in Law Journals","au":"Stojanovski, Jadranka","y":2021,"vn":"Frontiers in Research Metrics and Analytics","ty":"journalArticle","tyl":"Journal article","doi":"10.3389/frma.2021.787768","cby":2,"nref":57,"oast":"gold","cx":"Journal","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"96JCP7V7","t":"How Values and Uncertainty Shape Scientific Advance in Peer Review","au":"Smith, Daniel Scott","y":2025,"vn":"American Sociological Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/00031224251362254","cby":1,"nref":82,"oast":"closed","cx":"Journal","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"99FF6I9A","t":"Gender Effects in the Peer Reviews of Grant Proposals: A Comprehensive Meta-Analysis Comparing Traditional and Multilevel Approaches","au":"Marsh, Herbert W.","y":2009,"vn":"Review of Educational Research","ty":"journalArticle","tyl":"Journal article","doi":"10.3102/0034654309334143","cby":179,"nref":90,"oast":"closed","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring"],"st":"systematic-review-meta-analysis","tp":"2","cc":4,"cp":4,"self":false,"ncoefs":0},{"id":"9C6LPVHL","t":"Peer Review 2027: Scenarios for Academic Publishing in the Age of AI","au":"Munger, Kevin","y":2026,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31235/osf.io/594zj_v1","cby":1,"nref":0,"oast":"closed","cx":"Journal","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"6","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"9CQYZK6Q","t":"Peer Review, Innovation, and Predicting the Future of Science: The Scope of Lotteries in Science Funding Policy","au":"Shaw, Jamie","y":2023,"vn":"Philosophy of Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/psa.2023.35","cby":6,"nref":35,"oast":"hybrid","cx":"Grant","co":["Reliability-noise","Validity"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["funding-decision"],"st":"conceptual-commentary","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":1},{"id":"9E2KEZVY","t":"All that glitters is not gold: The shaping of contemporary journal peer review at scientific and medical journals","au":"Gaudet, Joanne","y":2017,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31219/osf.io/4954h","cby":3,"nref":78,"oast":"gold","cx":"Journal","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"9E2S4NZV","t":"Identifying research quality in the social sciences","au":"Ochsner, Michael","y":2022,"vn":"Edward Elgar Publishing eBooks","ty":"book","tyl":"Book","doi":"10.4337/9781800372559.00010","cby":5,"nref":45,"oast":"green","cx":"General","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"book-monograph","tp":"9","cc":3,"cp":2,"self":false,"ncoefs":0},{"id":"9E4R28VE","t":"Trends in Peer Review","au":"Reinhart, Martin","y":2021,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31235/osf.io/nzsp5","cby":5,"nref":36,"oast":"gold","cx":"Grant","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"9EBX8PWV","t":"The influence of peer reviewer expertise on the evaluation of research funding applications.","au":"Gallo, Stephen A","y":2016,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0165147","cby":74,"nref":49,"oast":"gold","cx":"Grant","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["cognitive-COI","prestige-halo"],"pbs":"cognitive-COI","rs":[],"sg":["reviewer-selection","individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"9EY3H6LA","t":"AI-assisted peer review","au":"Checco, Alessandro","y":2021,"vn":"Humanities and Social Sciences Communications","ty":"journalArticle","tyl":"Journal article","doi":"10.1057/s41599-020-00703-8","cby":264,"nref":43,"oast":"gold","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["ai-llm"],"pbs":"ai-llm","rs":[],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"6","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"9FKXJGFE","t":"Investigating journal peer review as scientific object of study: unabridged version – Part I","au":"Gaudet, Joanne","y":2017,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31219/osf.io/kmcen","cby":3,"nref":112,"oast":"gold","cx":"Journal","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"9G3YRFSM","t":"The author–editor game","au":"García, J. A.","y":2015,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-015-1566-x","cby":30,"nref":26,"oast":"closed","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["funding-decision"],"st":"simulation-methodological","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"9HNM58YP","t":"Choosing Reviewers: Predictors of Undergraduate Manuscript Evaluations","au":"Walker, Christina","y":2022,"vn":"PS Political Science & Politics","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/s1049096521001888","cby":0,"nref":25,"oast":"hybrid","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":4,"cp":1,"self":false,"ncoefs":0},{"id":"9IU8ALQ2","t":"Ethical Theories in Research Evaluation: An Exploratory Approach","au":"Biagetti, Maria Teresa","y":2020,"vn":"Scholarly Assessment Reports","ty":"journalArticle","tyl":"Journal article","doi":"10.29024/sar.19","cby":12,"nref":16,"oast":"diamond","cx":"General","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"9","cc":4,"cp":2,"self":false,"ncoefs":0},{"id":"9LLPJNVM","t":"No evidence of any systematic bias against manuscripts by women in the peer review process of 145 scholarly journals","au":"Squazzoni, Flaminio","y":2020,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31235/osf.io/gh4rv","cby":9,"nref":46,"oast":"gold","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"5","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"9NBTMKDY","t":"Neglected considerations in the analysis of agreement among journal referees","au":"Hargens, Lowell L.","y":1990,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/bf02130467","cby":82,"nref":38,"oast":"closed","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"3","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"9P5NC8TZ","t":"A System-Level Analysis of Conference Peer Review","au":"Zhang, Yichi","y":2022,"vn":"Proceedings of the 23rd ACM Conference on Economics and Computation","ty":"journalArticle","tyl":"Journal article","doi":"10.1145/3490486.3538235","cby":5,"nref":38,"oast":"green","cx":"Journal","co":["Reliability-noise","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"simulation-methodological","tp":"0","cc":3,"cp":2,"self":false,"ncoefs":0},{"id":"9P8K8DT7","t":"Factores de rechazo en el arbitraje de artículos científicos de la revista Tecnociencia Chihuahua","au":"Picazo, David","y":2025,"vn":"Investigación Bibliotecológica Archivonomía Bibliotecología e Información","ty":"journalArticle","tyl":"Journal article","doi":"10.22201/iibi.24488321xe.2025.102.58956","cby":0,"nref":28,"oast":"diamond","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"9QMZUVL4","t":"Citations versus expert opinions: Citation analysis of Featured Reviews of the American Mathematical Society","au":"Smolinsky, Lawrence","y":2020,"vn":"arXiv (Cornell University)","ty":"journalArticle","tyl":"Journal article","doi":"10.48550/arxiv.2005.05389","cby":1,"nref":39,"oast":"green","cx":"General","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["post-award-validity"],"st":"observational-quantitative","tp":"9","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"9QWVUD3K","t":"Evaluating large language models for abstract evaluation tasks: an empirical study","au":"Liu, Yinuo","y":2026,"vn":"Frontiers in Research Metrics and Analytics","ty":"journalArticle","tyl":"Journal article","doi":"10.3389/frma.2026.1807672","cby":0,"nref":28,"oast":"gold","cx":"Journal","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["ai-llm"],"pbs":"ai-llm","rs":["level","factors"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"6","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"9RLJISM4","t":"Innovation Suppression and Clique Evolution in Peer-Review-Based, Competitive Research Funding Systems: An Agent-Based Model","au":"Sobkowicz, Paweł","y":2015,"vn":"Journal of Artificial Societies and Social Simulation","ty":"journalArticle","tyl":"Journal article","doi":"10.18564/jasss.2750","cby":14,"nref":90,"oast":"gold","cx":"Grant","co":["Bias","Validity","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"simulation-methodological","tp":"0","cc":4,"cp":2,"self":false,"ncoefs":0},{"id":"9TFXN7CG","t":"Do Expert Panelists Herd? Evidence from FDA Committees","au":"Newham, Melissa","y":2019,"vn":"SSRN Electronic Journal","ty":"journalArticle","tyl":"Journal article","doi":"10.2139/ssrn.3476536","cby":2,"nref":39,"oast":"green","cx":"Analogue","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["panel-consensus"],"st":"observational-quantitative","tp":"13","cc":4,"cp":2,"self":false,"ncoefs":0},{"id":"9VB4W7T9","t":"Grant Review","au":"Yan, Zheng","y":2025,"vn":"Cambridge University Press eBooks","ty":"book","tyl":"Book","doi":"10.1017/9781009381338.009","cby":0,"nref":65,"oast":"closed","cx":"Grant","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"book-monograph","tp":"8","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"A42AQLJA","t":"Optimizing ChatGPT-Assisted Evaluation of Vietnamese English Majors’ Research Proposals: A Tree-of-Thought and Rubric-Based Approach","au":"Tran, The Phi","y":2025,"vn":"Studies in computational intelligence","ty":"book","tyl":"Book","doi":"10.1007/978-3-032-01348-4_3","cby":0,"nref":27,"oast":"closed","cx":"General","co":["Reliability-noise","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["proposal-quality","individual-scoring"],"st":"book-monograph","tp":"6","cc":5,"cp":0,"self":false,"ncoefs":4},{"id":"A42HFS8G","t":"How many reviewers are required to obtain reliable evaluations of NIH R01 grant proposals?","au":"Forscher, Patrick S.","y":2019,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31234/osf.io/483zj","cby":24,"nref":11,"oast":"gold","cx":"Grant","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level","value"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"A49B9ZX7","t":"Gender bias in female physician assessments: Women considered better suited for qualitative research.","au":"Johansson, Eva E","y":2002,"vn":"Scandinavian Journal of Primary Health Care","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/02813430215553","cby":14,"nref":15,"oast":"bronze","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring"],"st":"experimental-rct","tp":"5","cc":4,"cp":5,"self":false,"ncoefs":0},{"id":"A58BN9MR","t":"Rethinking the funding line at the swiss national science foundation: bayesian ranking and lottery","au":"Heyard, Rachel","y":2022,"vn":"Statistics and Public Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/2330443X.2022.2086190","cby":28,"nref":47,"oast":"gold","cx":"Grant","co":["Reliability-noise","Bias","Interventions"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":["level"],"sg":["individual-scoring","funding-decision"],"st":"simulation-methodological","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":6},{"id":"A7ERVCRX","t":"Reproducibility of peer review in clinical neuroscience: Is agreement between reviewers any greater than would be expected by chance alone?","au":"Rothwell, Peter M.","y":2000,"vn":"Brain","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/brain/123.9.1964","cby":251,"nref":24,"oast":"bronze","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"3","cc":3,"cp":4,"self":false,"ncoefs":4},{"id":"A8F6GAJN","t":"Decisions, decisions.","au":"Rodgers, Peter","y":2017,"vn":"eLife","ty":"journalArticle","tyl":"Journal article","doi":"10.7554/eLife.32011","cby":16,"nref":11,"oast":"gold","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"A8QVM43M","t":"Heuristics as conceptual lens for understanding and studying the usage of bibliometrics in research evaluation","au":"Bornmann, Lutz","y":2019,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-019-03018-x","cby":46,"nref":217,"oast":"hybrid","cx":"General","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"9","cc":4,"cp":4,"self":false,"ncoefs":0},{"id":"ABQTK2RB","t":"Conservatism and risk-taking in peer review: Emerging ERC practices","au":"Luukkonen, Terttu","y":2012,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvs001","cby":187,"nref":15,"oast":"closed","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["panel-consensus","individual-scoring"],"st":"qualitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"ACMIRSDD","t":"Integration and generalization of kappas for multiple raters.","au":"Conger, Anthony J.","y":1980,"vn":"Psychological Bulletin","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/0033-2909.88.2.322","cby":527,"nref":3,"oast":"closed","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"4","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"AEDP93ZE","t":"Interrater reliability estimators tested against true interrater reliabilities","au":"Zhao, Xinshu","y":2022,"vn":"BMC Medical Research Methodology","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s12874-022-01707-5","cby":57,"nref":56,"oast":"gold","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level","factors"],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"4","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"AEP27EJR","t":"Open up: a survey on open and non-anonymized peer reviewing.","au":"Besançon, Lonni","y":2020,"vn":"Research Integrity and Peer Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s41073-020-00094-z","cby":0,"nref":2,"oast":"green","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"0","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"AGV9EQ59","t":"Epistemic Diversity and Editor Decisions: A Statistical Matthew Effect","au":"Heesen, Remco","y":2019,"vn":"UWA Profiles and Research Repository (University of Western Australia)","ty":"journalArticle","tyl":"Journal article","doi":"","cby":39,"nref":19,"oast":"green","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["prestige-halo"],"pbs":"prestige-halo","rs":[],"sg":["funding-decision"],"st":"simulation-methodological","tp":"-1","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"AHI3KN85","t":"Non-Financial Conflicts of Interest in Academic Grant Evaluation: A Qualitative Study of Multiple Stakeholders in France","au":"Abdoul, Hendy","y":2012,"vn":"PLoS ONE","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0035247","cby":50,"nref":59,"oast":"gold","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["cognitive-COI","demographic"],"pbs":"demographic","rs":[],"sg":["reviewer-selection"],"st":"qualitative","tp":"8","cc":1,"cp":1,"self":false,"ncoefs":0},{"id":"AIII4UT9","t":"Inappropriate and appropriate selection of \"peers\" in grant review.","au":"Glantz, Stanton A","y":1994,"vn":"JAMA: Journal of the American Medical Association","ty":"journalArticle","tyl":"Journal article","doi":"10.1001/jama.272.2.114","cby":23,"nref":0,"oast":"closed","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["reviewer-selection"],"st":"observational-quantitative","tp":"7","cc":1,"cp":0,"self":false,"ncoefs":0},{"id":"AIQ3U7NC","t":"An overview of innovations in the external peer review of journal manuscripts.","au":"Woods, Helen Buckley","y":2023,"vn":"Wellcome Open Research","ty":"journalArticle","tyl":"Journal article","doi":"10.12688/wellcomeopenres.17715.2","cby":0,"nref":0,"oast":"green","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"systematic-review-meta-analysis","tp":"0","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"AKUVGJ36","t":"Community review: a robust and scalable selection system for resource allocation within open science and innovation communities","au":"Graham, Chris L. B.","y":2022,"vn":"F1000Research","ty":"journalArticle","tyl":"Journal article","doi":"10.12688/f1000research.125886.1","cby":1,"nref":21,"oast":"gold","cx":"Grant","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"AMFWGDX4","t":"Towards theorizing peer review","au":"Hug, Sven E.","y":2022,"vn":"Quantitative Science Studies","ty":"journalArticle","tyl":"Journal article","doi":"10.1162/qss_a_00195","cby":30,"nref":125,"oast":"gold","cx":"General","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"ANYJYNQA","t":"The Delphi process: a solution for reviewing novel grant applications","au":"Robotin, Monica","y":2010,"vn":"International Journal of General Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.2147/ijgm.s11117","cby":19,"nref":23,"oast":"gold","cx":"Grant","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["panel-consensus"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":0,"self":false,"ncoefs":0},{"id":"AQBH53NF","t":"Peer reviewers altered their recommendation based on whether they were cited or wanted to be cited. A matched study of open peer review at four journals","au":"Barnett, Adrian","y":2025,"vn":"eLife","ty":"journalArticle","tyl":"Journal article","doi":"10.7554/elife.108748.1","cby":0,"nref":58,"oast":"gold","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"AR9S4GQ4","t":"CRITICAL EVALUATION OF THE GRANT FUNDING ALLOCATION MECHANISM IN SOCIAL AND HUMANITARIAN SCIENCES","au":"Молдашев, Кайрат","y":2025,"vn":"Central Asian Economic Review","ty":"journalArticle","tyl":"Journal article","doi":"10.52821/2789-4401-2024-5-48-58","cby":0,"nref":13,"oast":"hybrid","cx":"Grant","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":["factors"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"ASUP2B28","t":"A Dataset of Peer Reviews (PeerRead): Collection, Insights and NLP Applications","au":"Kang, Dongyeop","y":2018,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.18653/v1/n18-1149","cby":165,"nref":22,"oast":"gold","cx":"Journal","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"6","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"AUAWRZC8","t":"A framework for assessing the trustworthiness of scientific research findings","au":"Nosek, Brian A.","y":2026,"vn":"Proceedings of the National Academy of Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1073/pnas.2536736123","cby":3,"nref":80,"oast":"hybrid","cx":"General","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"-1","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"AV5X84YW","t":"What Authors Want From Journal Reviewers and Editors.","au":"Nickerson, Raymond S.","y":2005,"vn":"American Psychologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/0003-066x.60.6.661","cby":26,"nref":13,"oast":"closed","cx":"Journal","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"-1","cc":3,"cp":0,"self":false,"ncoefs":0},{"id":"AWK223K2","t":"Societal impact evaluation: Exploring evaluator perceptions of the characterization of impact under the REF2014: Table 1.","au":"Samuel, Gabrielle","y":2015,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvv007","cby":95,"nref":55,"oast":"hybrid","cx":"General","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"qualitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"AWNC85V8","t":"Identification and comparison of key criteria of feedback of funding decisions: mixed-methods analysis of funder and applicant perspectives","au":"Fackrell, Kathryn","y":2021,"vn":"BMJ Open","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmjopen-2021-048979","cby":6,"nref":21,"oast":"gold","cx":"Grant","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["funding-decision"],"st":"observational-quantitative","tp":"7","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"AX5647XH","t":"Expertise versus Bias in Evaluation: Evidence from the NIH","au":"Li, Danielle","y":2017,"vn":"American Economic Journal Applied Economics","ty":"journalArticle","tyl":"Journal article","doi":"10.1257/app.20150421","cby":171,"nref":40,"oast":"bronze","cx":"Grant","co":["Bias","Validity"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["reviewer-selection","individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":4,"cp":2,"self":false,"ncoefs":0},{"id":"AXTNPFSW","t":"Peer review in academic publishing: Challenges in achieving the gold standard","au":"Gonzalez, Prue","y":2022,"vn":"Journal of University Teaching and Learning Practice","ty":"journalArticle","tyl":"Journal article","doi":"10.53761/1.19.5.1","cby":29,"nref":41,"oast":"diamond","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"AZU4V3L3","t":"Crowd-sourced peer review: wisdom or tyranny of the crowd?","au":"Muller, Seán","y":2025,"vn":"South African Journal of Philosophy","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/02580136.2025.2462433","cby":3,"nref":27,"oast":"bronze","cx":"Journal","co":["Bias","Validity"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"-1","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"B32YEVN2","t":"Evaluation in research funding agencies: Are structurally diverse teams biased against?","au":"Banal‐Estañol, Albert","y":2019,"vn":"Research Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.respol.2019.04.008","cby":43,"nref":97,"oast":"closed","cx":"Grant","co":["Bias","Validity"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["funding-decision","post-award-validity"],"st":"observational-quantitative","tp":"-1","cc":4,"cp":2,"self":false,"ncoefs":0},{"id":"B68EMUVK","t":"Does gender matter in grant peer review?: an empirical investigation using the example of the austrian science fund.","au":"Mutz, Rüdiger","y":2012,"vn":"Zeitschrift für Psychologie","ty":"journalArticle","tyl":"Journal article","doi":"10.1027/2151-2604/a000103","cby":75,"nref":32,"oast":"green","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"2","cc":4,"cp":5,"self":false,"ncoefs":2},{"id":"B6DG7TSN","t":"Analyzing sentiments in peer review reports: Evidence from two science funding agencies","au":"Luo, Junwen","y":2021,"vn":"Quantitative Science Studies","ty":"journalArticle","tyl":"Journal article","doi":"10.1162/qss_a_00156","cby":27,"nref":46,"oast":"gold","cx":"Grant","co":["Reliability-noise","Validity"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["factors"],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"8","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"B6VDY29W","t":"No Peeking: Peer Review and Presumptive Blinding","au":"Ballantyne, Nathan","y":2024,"vn":"Canadian Journal of Philosophy","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/can.2024.25","cby":1,"nref":27,"oast":"hybrid","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["prestige-halo"],"pbs":"prestige-halo","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"10","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"B7BZH6ZQ","t":"Why Proposal Review Should Be More Like Meteorology","au":"Buck, Stuart","y":2023,"vn":"Seeds of Science","ty":"journalArticle","tyl":"Journal article","doi":"10.53975/cwwp-mrgc","cby":0,"nref":4,"oast":"bronze","cx":"Grant","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"-1","cc":1,"cp":0,"self":false,"ncoefs":0},{"id":"B8MCFZB5","t":"What feedback do reviewers give when reviewing qualitative manuscripts? A focused mapping review and synthesis","au":"Herber, Oliver Rudolf","y":2020,"vn":"BMC Medical Research Methodology","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s12874-020-01005-y","cby":32,"nref":44,"oast":"gold","cx":"Journal","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"systematic-review-meta-analysis","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"B9A9JT4V","t":"Reliability, bias and randomisation in peer review: a simulation","au":"Hulkes, Alexander","y":2025,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31235/osf.io/4gqce_v2","cby":0,"nref":0,"oast":"gold","cx":"Grant","co":["Reliability-noise","Bias","Interventions"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":["factors"],"sg":["individual-scoring","funding-decision"],"st":"simulation-methodological","tp":"1","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"B9VSF35G","t":"Modeling Preferences: A Bayesian Mixture of Finite Mixtures for Rankings and Ratings","au":"Pearce, Michael","y":2024,"vn":"Journal of the American Statistical Association","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/01621459.2024.2444700","cby":0,"nref":45,"oast":"hybrid","cx":"General","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"8","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"BB5XVIJ5","t":"Streamlined research funding using short proposals and accelerated peer review: an observational study.","au":"Barnett, Adrian G","y":2015,"vn":"BMC Health Services Research","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s12913-015-0721-7","cby":3,"nref":5,"oast":"gold","cx":"Grant","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["proposal-quality","individual-scoring"],"st":"observational-quantitative","tp":"7","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"BBVN6S7M","t":"Enhanced peer-review for optimising publication of biomedical papers submitted from low- and middle-income countries: feasibility study for a randomised controlled trial.","au":"Pitman, Alexandra","y":2019,"vn":"BJPsych open","ty":"journalArticle","tyl":"Journal article","doi":"10.1192/bjo.2018.89","cby":7,"nref":13,"oast":"gold","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"experimental-rct","tp":"11","cc":2,"cp":0,"self":false,"ncoefs":0},{"id":"BCUNWS6G","t":"The limited value of journal peer review in public health: a case series of tobacco harm reduction articles.","au":"Phillips, Carl V.","y":2019,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31219/osf.io/q7jdv","cby":0,"nref":15,"oast":"gold","cx":"Journal","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"qualitative","tp":"0","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"BENYRE6T","t":"Double anonymous peer review in JVSCIT—lessons from our pilot trial and next steps","au":"Smeds, Matthew R.","y":2025,"vn":"Journal of Vascular Surgery Cases and Innovative Techniques","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.jvscit.2025.102028","cby":0,"nref":9,"oast":"gold","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["prestige-halo"],"pbs":"prestige-halo","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"10","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"BFIPR3E7","t":"Social Biases and Solutions for Procedural Objectivity","au":"Lee, Carole J.","y":2011,"vn":"Hypatia","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/j.1527-2001.2011.01178.x","cby":68,"nref":60,"oast":"bronze","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["demographic","bias-general"],"pbs":"demographic","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"0","cc":3,"cp":4,"self":false,"ncoefs":0},{"id":"BFVZSMKT","t":"Peer review bias: A critical review.","au":"Haffar, Samir","y":2019,"vn":"Mayo Clinic Proceedings","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.mayocp.2018.09.004","cby":163,"nref":40,"oast":"bronze","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"BFW9NA83","t":"Bibliometric indicators: opportunities and limits.","au":"Belter, Christopher W","y":2015,"vn":"Journal of the Medical Library Association","ty":"journalArticle","tyl":"Journal article","doi":"10.3163/1536-5050.103.4.014","cby":302,"nref":17,"oast":"diamond","cx":"General","co":["Reliability-noise","Bias","Validity"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"9","cc":4,"cp":4,"self":false,"ncoefs":0},{"id":"BG57BA84","t":"The effect of the scale of grant scoring on ranking accuracy","au":"Visscher, Peter M.","y":2023,"vn":"F1000Research","ty":"journalArticle","tyl":"Journal article","doi":"10.12688/f1000research.125400.2","cby":2,"nref":8,"oast":"gold","cx":"Grant","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["value"],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"8","cc":4,"cp":2,"self":false,"ncoefs":0},{"id":"BHZQYW9A","t":"Analysis of reporting quality for oral presentations of observational studies at 19th National Surgical Congress: Proposal for a national evaluation system.","au":"Hasbahçeci, Mustafa","y":2016,"vn":"Ulusal cerrahi dergisi","ty":"journalArticle","tyl":"Journal article","doi":"10.5152/UCD.2016.3195","cby":0,"nref":14,"oast":"green","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"15","cc":5,"cp":6,"self":false,"ncoefs":3},{"id":"BI6REK75","t":"(Fehlende) Anwesenheit bei Begutachtungsprozessen: Zum Wandel der sozialen Interaktionen im virtuellen Format","au":"Paruschke, Laura","y":2023,"vn":"Zeitschrift für Soziologie","ty":"journalArticle","tyl":"Journal article","doi":"10.1515/zfsoz-2023-2027","cby":3,"nref":27,"oast":"hybrid","cx":"Grant","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["panel-consensus"],"st":"qualitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"BJ4METPH","t":"Improving peer review: What reviewers can do","au":"DiDomenico, Robert J.","y":2017,"vn":"American Journal of Health-System Pharmacy","ty":"journalArticle","tyl":"Journal article","doi":"10.2146/ajhp170190","cby":30,"nref":14,"oast":"bronze","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"BJHV5PDY","t":"Gender differences in authorships are not associated with publication bias in an evolutionary journal","au":"Edwards, Hannah A","y":2018,"vn":"PLoS ONE","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0201725","cby":54,"nref":54,"oast":"gold","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["funding-decision"],"st":"observational-quantitative","tp":"5","cc":3,"cp":5,"self":false,"ncoefs":0},{"id":"BJLC5S5R","t":"Fragments of peer review: A quantitative analysis of the literature (1969-2015)","au":"Grimaldo, Francisco","y":2018,"vn":"PLoS ONE","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0193148","cby":56,"nref":31,"oast":"gold","cx":"General","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"BLZITU4C","t":"A reader's, writer's, and reviewer's guide to assessing research reports in clinical psychology.","au":"Maher, Brendan A.","y":1978,"vn":"Journal of Consulting and Clinical Psychology","ty":"journalArticle","tyl":"Journal article","doi":"10.1037//0022-006x.46.4.835","cby":80,"nref":0,"oast":"closed","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["proposal-quality","individual-scoring"],"st":"guideline-policy","tp":"3","cc":5,"cp":0,"self":false,"ncoefs":0},{"id":"BML9YX8S","t":"Improving the Evaluation of Submitted Manuscripts.","au":"Newman, Sidney H.","y":1966,"vn":"American Psychologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/h0021046","cby":48,"nref":0,"oast":"closed","cx":"Journal","co":["Reliability-noise","Bias","Interventions"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"3","cc":5,"cp":0,"self":false,"ncoefs":0},{"id":"BQHSYZPG","t":"Editorial decisions with informed and uninformed reviewers","au":"Rodríguez‐Sánchez, Rosa","y":2018,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-018-2875-7","cby":3,"nref":16,"oast":"green","cx":"Journal","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["panel-consensus","funding-decision"],"st":"simulation-methodological","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"BQP92QB2","t":"Peer review: from “sacred ideals” to “profane realities”","au":"Johnson, David R.","y":2017,"vn":"Higher education: handbook of theory and research","ty":"bookSection","tyl":"Book chapter","doi":"10.1007/978-3-319-48983-4_10","cby":46,"nref":230,"oast":"closed","cx":"Journal","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"book-monograph","tp":"-1","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"BRRTSI2P","t":"Transforming the Academy of Community Reviewers (ACR) course into an E-Learning course in the Post COVID-19 Pandemic Era","au":"Neal J.","y":2024,"vn":"Journal of Clinical and Translational Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/cts.2024.215","cby":0,"nref":0,"oast":"gold","cx":"Grant","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"-1","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"BS9CHRDT","t":"Three years of quality assurance data assessing the performance of over 4000 grant peer review contributions to the Canadian Institutes of Health Research Project Grant Competition","au":"Ardern, Clare L.","y":2023,"vn":"FACETS","ty":"journalArticle","tyl":"Journal article","doi":"10.1139/facets-2022-0175","cby":4,"nref":13,"oast":"gold","cx":"Grant","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"BSI9PZ5P","t":"Not all peer reviews are significant: a dataset of exhaustive vs. trivial scientific peer reviews leveraging chain-of-thought reasoning","au":"Bharti, Prabhat Kumar","y":2026,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-025-05435-7","cby":1,"nref":18,"oast":"closed","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"6","cc":0,"cp":1,"self":false,"ncoefs":0},{"id":"BTE5MMT9","t":"The changing forms and expectations of peer review","au":"Horbach, Serge P. J. M.","y":2018,"vn":"Research Integrity and Peer Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s41073-018-0051-5","cby":176,"nref":110,"oast":"gold","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"BTNL5XNW","t":"The effect of public funding on research output: the New Zealand Marsden Fund","au":"Gush, Jason","y":2017,"vn":"New Zealand Economic Papers","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/00779954.2017.1325921","cby":38,"nref":24,"oast":"green","cx":"Grant","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["post-award-validity"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"BU8EHDHM","t":"Expert decision support system as a means of improving the quality of reviewing articles in a scientific journal","au":"Spasennikov, Valeriy","y":2022,"vn":"Ergodesign","ty":"journalArticle","tyl":"Journal article","doi":"10.30987/2658-4026-2022-2-128-136","cby":4,"nref":17,"oast":"closed","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"-1","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"BVGT7SQ9","t":"Post-Publication Peer Review for Real","au":"Ikeda, Koki","y":2020,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31234/osf.io/sp3j5","cby":6,"nref":29,"oast":"gold","cx":"General","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":5,"cp":1,"self":false,"ncoefs":0},{"id":"BX3VF92R","t":"Large sample variance of kappa in the case of different sets of raters.","au":"Fleiss, Joseph L.","y":1979,"vn":"Psychological Bulletin","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/0033-2909.86.5.974","cby":391,"nref":10,"oast":"closed","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"4","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"BZ2KEKP2","t":"Blinding in peer review: the preferences of reviewers for nursing journals","au":"Baggs, Judith Gedney","y":2008,"vn":"Journal of Advanced Nursing","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/j.1365-2648.2008.04816.x","cby":89,"nref":35,"oast":"closed","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"10","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"BZ4JJDAM","t":"Peer Review: A System under Stress","au":"Gropp, Robert E.","y":2017,"vn":"BioScience","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/biosci/bix034","cby":53,"nref":12,"oast":"closed","cx":"General","co":["Reliability-noise","Bias","Validity","Interventions"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":["level","factors","value"],"sg":["whole-process"],"st":"conceptual-commentary","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"BZDWP8J8","t":"How much would each researcher receive if competitive government research funding were distributed equally among researchers?","au":"Vaesen, Krist","y":2017,"vn":"PLoS ONE","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0183967","cby":64,"nref":25,"oast":"gold","cx":"Grant","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["funding-decision"],"st":"observational-quantitative","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"BZFYKEBB","t":"The Novelty Paradox & Bias for Normal Science: Evidence from Randomized Medical Grant Proposal Evaluations","au":"Boudreau, Kevin","y":2012,"vn":"Harvard Business School working paper series # 13-053","ty":"journalArticle","tyl":"Journal article","doi":"10.2139/ssrn.2184791","cby":17,"nref":28,"oast":"green","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["proposal-quality","individual-scoring"],"st":"experimental-rct","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"C2RR4L79","t":"Bias and truth in science evaluation: a simulation model of grant review panel discussions.","au":"Bethencourt, Adrián Martín","y":2021,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"","cby":0,"nref":0,"oast":"","cx":"Grant","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["panel-consensus"],"st":"simulation-methodological","tp":"1","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"C3BI4GWS","t":"How professors think: inside the curious world of academic judgment","au":"Álvaro, SANTANA ACUÑA","y":2009,"vn":"Choice Reviews Online","ty":"journalArticle","tyl":"Journal article","doi":"10.5860/choice.47-1492","cby":612,"nref":0,"oast":"green","cx":"Grant","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"-1","cc":3,"cp":0,"self":false,"ncoefs":0},{"id":"C3F8YBY5","t":"Heuristics, not plumage: A response to Osterloh and Frey's discussion paper on ‘Borrowed plumes’","au":"Wooding, Steven","y":2019,"vn":"Research Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.respol.2019.103871","cby":5,"nref":30,"oast":"green","cx":"General","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"C3QMZY4M","t":"Peer Reviewed Evaluation of Registered End-Points of Randomised Trials (the PRE-REPORT study): a stepped wedge, cluster-randomised trial","au":"Jones, Christopher W.","y":2022,"vn":"BMJ Open","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmjopen-2022-066624","cby":11,"nref":32,"oast":"gold","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"experimental-rct","tp":"11","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"C3XF2ZR4","t":"Reminding Peer Reviewers of Reporting Guideline Items to Improve Completeness in Published Articles","au":"Speich, Benjamin","y":2023,"vn":"JAMA Network Open","ty":"journalArticle","tyl":"Journal article","doi":"10.1001/jamanetworkopen.2023.17651","cby":44,"nref":25,"oast":"gold","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"experimental-rct","tp":"11","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"C3XQ6W24","t":"Gender differences in grant peer review: A meta-analysis","au":"Bornmann, L","y":2007,"vn":"Journal of Informetrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.joi.2007.03.001","cby":347,"nref":31,"oast":"closed","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["funding-decision"],"st":"systematic-review-meta-analysis","tp":"2","cc":4,"cp":4,"self":false,"ncoefs":0},{"id":"C5N7Q8TK","t":"Symbolic scarcity and epistemic waste: a systems theory of academic publishing failure.","au":"Hansen, Trym","y":2026,"vn":"Research integrity and peer review","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s41073-026-00193-3","cby":0,"nref":11,"oast":"gold","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"C5SPC6BS","t":"Funding lotteries for research grant allocation: An extended taxonomy and evaluation of their fairness","au":"Feliciani, Thomas","y":2024,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvae025","cby":4,"nref":59,"oast":"hybrid","cx":"Grant","co":["Bias","Validity"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["funding-decision"],"st":"simulation-methodological","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"C7YCC4A7","t":"Variability and negligence: grant peer review panels evaluating impact ex ante","au":"Oxley, Kristin","y":2025,"vn":"Science and Public Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/scipol/scae081","cby":5,"nref":82,"oast":"hybrid","cx":"Grant","co":["Reliability-noise","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["panel-consensus"],"st":"qualitative","tp":"13","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"C8THDZNN","t":"A randomized controlled study of reviewer bias against an unconventional therapy","au":"Resch, K I","y":2000,"vn":"Journal of the Royal Society of Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/014107680009300402","cby":94,"nref":17,"oast":"bronze","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"experimental-rct","tp":"11","cc":2,"cp":0,"self":false,"ncoefs":0},{"id":"C9INWUTB","t":"Can journal reviewers dependably assess rigour, significance, and originality in theoretical papers? Evidence from physics","au":"Thelwall, Mike","y":2023,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvad018","cby":5,"nref":75,"oast":"green","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level","factors"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"3","cc":5,"cp":1,"self":false,"ncoefs":0},{"id":"CAM2G6QZ","t":"The Peer Review Process and Publishing Academic Work","au":"Reyes, Andres De Los","y":2024,"vn":"","ty":"book","tyl":"Book","doi":"10.1007/978-3-031-54284-8_7","cby":0,"nref":23,"oast":"closed","cx":"Journal","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"book-monograph","tp":"0","cc":2,"cp":2,"self":false,"ncoefs":0},{"id":"CBSAJ6NJ","t":"PP021 peer review innovations for grant applications: efficient and effective?","au":"Shepherd, Jonathan","y":2017,"vn":"International Journal of Technology Assessment in Health Care","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/S0266462317002124","cby":0,"nref":1,"oast":"bronze","cx":"Grant","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"systematic-review-meta-analysis","tp":"7","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"CDSKP5Q8","t":"Funding grant proposals for scientific research: retrospective analysis of scores by members of grant review panel.","au":"Graves, Nicholas","y":2011,"vn":"BMJ (Clinical Research Ed.)","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmj.d4797","cby":138,"nref":28,"oast":"hybrid","cx":"Grant","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"CM2NSUHH","t":"And the winner is … : inter-rater reliability among scholarship assessors","au":"Johnston, Lucy","y":2015,"vn":"Studies in Higher Education","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/03075079.2015.1124849","cby":4,"nref":24,"oast":"closed","cx":"Grant","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"2","cc":5,"cp":4,"self":false,"ncoefs":1},{"id":"CM3DJF35","t":"Is Living Easier With Eyes Closed?","au":"Wien, Charlotte","y":2024,"vn":"European Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/s1062798723000558","cby":1,"nref":22,"oast":"hybrid","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"10","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"CNGBYJUU","t":"Threats to grant peer review: a qualitative study","au":"Sims‐Gould, Joanie","y":2025,"vn":"BMJ Open","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmjopen-2024-091666","cby":4,"nref":34,"oast":"gold","cx":"Grant","co":["Reliability-noise","Bias","Interventions"],"pco":"Reliability-noise","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["reviewer-selection","individual-scoring","panel-consensus"],"st":"qualitative","tp":"8","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"CPIAV23Z","t":"Do bibliometrics introduce gender, institutional or interdisciplinary biases into research evaluations?","au":"Thelwall, Mike","y":2023,"vn":"Research Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.respol.2023.104829","cby":29,"nref":88,"oast":"hybrid","cx":"General","co":["Bias"],"pco":"Bias","bs":["demographic","prestige-halo"],"pbs":"demographic","rs":[],"sg":["funding-decision"],"st":"observational-quantitative","tp":"9","cc":0,"cp":1,"self":false,"ncoefs":0},{"id":"CQUBJBF5","t":"Reading Peer Review","au":"Eve, Martin Paul","y":2020,"vn":"Cambridge University Press eBooks","ty":"book","tyl":"Book","doi":"10.1017/9781108783521","cby":13,"nref":327,"oast":"hybrid","cx":"Journal","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"book-monograph","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"CSUMH6XZ","t":"Comparison of the Null Distributions of Weighted Kappa and the C Ordinal Statistic","au":"Cicchetti, Dominic","y":1977,"vn":"Applied Psychological Measurement","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/014662167700100206","cby":206,"nref":6,"oast":"green","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"4","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"CSVNG9TD","t":"Double-blind review in software engineering venues","au":"Beller, Moritz","y":2016,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.7287/peerj.preprints.1757v2","cby":1,"nref":34,"oast":"gold","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"10","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"CUA96JA7","t":"Ex ante evaluation of interdisciplinary research projects: A literature review","au":"Oviedo‐García, M. Ángeles","y":2016,"vn":"Social Science Information","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/0539018416658147","cby":5,"nref":45,"oast":"closed","cx":"Grant","co":["Reliability-noise","Bias","Validity","Interventions"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"CUHSPUCH","t":"Textual analysis of artificial intelligence manuscripts reveals features associated with peer review outcome","au":"Vincent‐Lamarre, Philippe","y":2021,"vn":"Quantitative Science Studies","ty":"journalArticle","tyl":"Journal article","doi":"10.1162/qss_a_00125","cby":3,"nref":62,"oast":"gold","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["ai-llm"],"pbs":"ai-llm","rs":[],"sg":["funding-decision"],"st":"observational-quantitative","tp":"6","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"CUKLB2A9","t":"Anosmic peer review: A rose by another name is evidently not a rose","au":"Scarr, Sandra","y":1982,"vn":"Behavioral and Brain Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/s0140525x0001164x","cby":50,"nref":201,"oast":"closed","cx":"Journal","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"12","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"CWDVSG5K","t":"The limitations to our understanding of peer review.","au":"Tennant, Jonathan P","y":2020,"vn":"Research Integrity and Peer Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s41073-020-00092-1","cby":355,"nref":120,"oast":"gold","cx":"Journal","co":["Bias","Validity"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"CWZ994PR","t":"How to sponsor ground-breaking research: a comparison of funding schemes","au":"Heinze, Thomas","y":2008,"vn":"Science and Public Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.3152/030234208X317151","cby":125,"nref":27,"oast":"green","cx":"Grant","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"qualitative","tp":"1","cc":4,"cp":2,"self":false,"ncoefs":0},{"id":"CY45T4US","t":"The AI Review Lottery: Widespread AI-Assisted Peer Reviews Boost Paper Scores and Acceptance Rates","au":"Russo, Giuseppe","y":2025,"vn":"Proceedings of the ACM on Human-Computer Interaction","ty":"journalArticle","tyl":"Journal article","doi":"10.1145/3757667","cby":5,"nref":66,"oast":"green","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["ai-llm"],"pbs":"ai-llm","rs":[],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"6","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"CYVRJWAL","t":"Peer review","au":"Hyland, Ken","y":2020,"vn":"Journal of English for Research Publication Purposes","ty":"journalArticle","tyl":"Journal article","doi":"10.1075/jerpp.19010.hyl","cby":11,"nref":56,"oast":"green","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"D2ITMW42","t":"The Cultivation of Social Work Knowledge: Towards a More Robust System of Peer Review","au":"Dunleavy, Daniel J.","y":2020,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31235/osf.io/rvqze","cby":4,"nref":83,"oast":"gold","cx":"Journal","co":["Reliability-noise","Validity","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"D2RXKQ8L","t":"A Coefficient of Agreement for Nominal Scales","au":"Cohen, Jacob","y":1960,"vn":"Educational and Psychological Measurement","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/001316446002000104","cby":41373,"nref":6,"oast":"closed","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["whole-process"],"st":"simulation-methodological","tp":"4","cc":5,"cp":0,"self":false,"ncoefs":0},{"id":"D69EUCWP","t":"The Review Process at PSPB: Correlates of Interreviewer Agreement and Manuscript Acceptance","au":"Petty, Richard E.","y":1999,"vn":"Personality and Social Psychology Bulletin","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/0146167299025002005","cby":62,"nref":44,"oast":"closed","cx":"Journal","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["prestige-halo","demographic"],"pbs":"demographic","rs":["level","factors"],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"5","cc":3,"cp":3,"self":false,"ncoefs":0},{"id":"D8EVAPCQ","t":"Multiple evaluations of grant proposals by independent assessors: confirmatory factor analysis evaluations of reliability, validity, and structure.","au":"Marsh, H W","y":1999,"vn":"Multivariate behavioral research","ty":"journalArticle","tyl":"Journal article","doi":"10.1207/s15327906mbr3401_1","cby":16,"nref":0,"oast":"closed","cx":"Grant","co":["Reliability-noise","Validity"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["factors"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"8","cc":4,"cp":0,"self":false,"ncoefs":0},{"id":"DBHUKEVE","t":"Evaluating the Pros and Cons of Different Peer Review Policies via Simulation","au":"Zhu, Jia","y":2015,"vn":"Science and Engineering Ethics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11948-015-9683-8","cby":6,"nref":41,"oast":"closed","cx":"Journal","co":["Bias","Validity","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"simulation-methodological","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"DBUHQVVR","t":"Does the pressure to fill journal quotas bias evaluation?: Evidence from publication delays and rejection rates","au":"Park, Brian","y":2020,"vn":"PLoS ONE","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0236927","cby":7,"nref":22,"oast":"gold","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["funding-decision"],"st":"observational-quantitative","tp":"-1","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"DBXX2Q3H","t":"Gender Gaps in the Evaluation of Research: Evidence from Submissions to Economics Conferences*","au":"Hospido, Laura","y":2020,"vn":"Oxford Bulletin of Economics and Statistics","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/obes.12409","cby":57,"nref":48,"oast":"bronze","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["funding-decision"],"st":"observational-quantitative","tp":"5","cc":4,"cp":5,"self":false,"ncoefs":0},{"id":"DM28BPQZ","t":"Grant review feedback: appropriateness and usefulness.","au":"Gallo, Stephen A","y":2021,"vn":"Science and engineering ethics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11948-021-00295-9","cby":20,"nref":21,"oast":"hybrid","cx":"Grant","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["proposal-quality","individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"DNH4PGL6","t":"Can transparency undermine peer review? A simulation model of scientist behavior under open peer review","au":"Bianchi, Federico","y":2022,"vn":"Science and Public Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/scipol/scac027","cby":21,"nref":69,"oast":"hybrid","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"DRPY6FB5","t":"Contributions of reviewer judgements to editorial decision-making for the Canadian Journal of Behavioural Science: 1985–1986.","au":"Linden, Wolfgang","y":1992,"vn":"Canadian Journal of Behavioural Science/Revue canadienne des sciences du comportement","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/h0078757","cby":2,"nref":9,"oast":"closed","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"3","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"DS5G6JKI","t":"International peer review and decision-making practices, bias, and burden: The funders' perspective","au":"Meadmore K.","y":2019,"vn":"BMJ Evidence-Based Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmjebm-2019-EBMLive.19","cby":0,"nref":0,"oast":"bronze","cx":"Grant","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"7","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"DSQ6UWH2","t":"Who stands to win from double-blind peer review?","au":"Garvalov, Boyan K.","y":2015,"vn":"Advances in Regenerative Biology","ty":"journalArticle","tyl":"Journal article","doi":"10.3402/arb.v2.26879","cby":7,"nref":23,"oast":"hybrid","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"10","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"DTFX59UM","t":"What Factors are Important to the Success of Resubmitted Grant Applications in Health Research? A Retrospective Study of Over 20,000 Applications to the Canadian Institutes of Health Research","au":"Wrightson, James G.","y":2024,"vn":"medRxiv","ty":"journalArticle","tyl":"Journal article","doi":"10.1101/2024.05.29.24308137","cby":4,"nref":36,"oast":"green","cx":"Grant","co":["Bias","Validity"],"pco":"Bias","bs":["prestige-halo"],"pbs":"prestige-halo","rs":[],"sg":["proposal-quality","funding-decision"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"DU3KIZEC","t":"Journal Peer Review and Editorial Evaluation: Cautious Innovator or Sleepy Giant?","au":"Horbach, Serge P. J. M.","y":2019,"vn":"Minerva","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11024-019-09388-z","cby":48,"nref":61,"oast":"hybrid","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"DUZT4KCB","t":"Feedback practices in journal peer-review: a systematic literature review","au":"Chong, Sin Wang","y":2023,"vn":"Assessment & Evaluation in Higher Education","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/02602938.2022.2164757","cby":21,"nref":44,"oast":"hybrid","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"systematic-review-meta-analysis","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"DVDHBSTH","t":"An experimental study of simulated grant peer review: Gender differences and psychometric characteristics of proposal scores","au":"Schmaling, Karen B.","y":2024,"vn":"PLOS ONE","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0315567","cby":0,"nref":23,"oast":"gold","cx":"Grant","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["demographic"],"pbs":"demographic","rs":["level","factors"],"sg":["individual-scoring"],"st":"experimental-rct","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"DVXN4LHZ","t":"Measuring bias, burden and conservatism in research funding processes","au":"Guthrie, Susan","y":2019,"vn":"F1000Research","ty":"journalArticle","tyl":"Journal article","doi":"10.12688/f1000research.19156.1","cby":36,"nref":65,"oast":"gold","cx":"Grant","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"systematic-review-meta-analysis","tp":"1","cc":4,"cp":2,"self":false,"ncoefs":0},{"id":"DX8A4GJ8","t":"Commensuration Bias in Peer Review","au":"Lee, Carole J.","y":2015,"vn":"Philosophy of Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1086/683652","cby":88,"nref":56,"oast":"closed","cx":"General","co":["Bias","Interventions"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"DYZ8TVFC","t":"A maturity model for the scientific review of clinical trial designs and their informativeness","au":"Dolley, Shawn","y":2024,"vn":"Trials","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s13063-024-08099-5","cby":6,"nref":26,"oast":"gold","cx":"Grant","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["proposal-quality"],"st":"conceptual-commentary","tp":"11","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"DZ9QX9UK","t":"European Research Council: excellence and leadership over time from a gender perspective","au":"Bautista‐Puig, Núria","y":2019,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvz023","cby":22,"nref":91,"oast":"bronze","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["reviewer-selection","funding-decision"],"st":"observational-quantitative","tp":"2","cc":4,"cp":5,"self":false,"ncoefs":0},{"id":"DZKVKM62","t":"Gender bias in funding evaluation: A randomized experiment","au":"Castro, Laura Cruz","y":2023,"vn":"Quantitative Science Studies","ty":"journalArticle","tyl":"Journal article","doi":"10.1162/qss_a_00263","cby":13,"nref":108,"oast":"gold","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring"],"st":"experimental-rct","tp":"2","cc":4,"cp":5,"self":false,"ncoefs":0},{"id":"E28UBWHZ","t":"Double‐blind peer review—An experiment","au":"Fox, Charles W.","y":2019,"vn":"Functional Ecology","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/1365-2435.13269","cby":10,"nref":12,"oast":"bronze","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"5","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"E3U9A6YC","t":"A protocol of a cross-sectional study evaluating an online tool for early career peer reviewers assessing reports of randomised controlled trials.","au":"Chauvin, Anthony","y":2017,"vn":"BMJ Open","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmjopen-2017-017462","cby":10,"nref":26,"oast":"gold","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"11","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"E4VMBDGP","t":"The jury is out: a new approach to awarding science prizes","au":"Hill, Michael","y":2021,"vn":"F1000Research","ty":"journalArticle","tyl":"Journal article","doi":"10.12688/f1000research.75098.1","cby":3,"nref":30,"oast":"gold","cx":"General","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["panel-consensus","funding-decision"],"st":"conceptual-commentary","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"E4ZVSVLA","t":"A study of criteria for journal manuscripts.","au":"Wolff, Wirt M.","y":1970,"vn":"American Psychologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/h0029770","cby":114,"nref":12,"oast":"closed","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"3","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"E5YQG6G8","t":"Development of the review quality instrument (RQI) for assessing peer reviews of manuscripts.","au":"van Rooyen, S","y":1999,"vn":"Journal of Clinical Epidemiology","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/s0895-4356(99)00047-5","cby":177,"nref":14,"oast":"closed","cx":"Journal","co":["Reliability-noise","Validity"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"3","cc":2,"cp":1,"self":false,"ncoefs":5},{"id":"E69UQ2W5","t":"Are reviewer scores consistent with citations?","au":"Xie, Weixi","y":2024,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-024-05103-2","cby":2,"nref":63,"oast":"closed","cx":"Journal","co":["Reliability-noise","Validity"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring","post-award-validity"],"st":"observational-quantitative","tp":"9","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"E6NDGFIC","t":"Personal attributes of authors and reviewers, social bias and the outcomes of peer review: a case study.","au":"Walker, Richard","y":2015,"vn":"F1000Research","ty":"journalArticle","tyl":"Journal article","doi":"10.12688/f1000research.6012.2","cby":55,"nref":51,"oast":"gold","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["demographic","prestige-halo"],"pbs":"demographic","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"E6QJ72I8","t":"Prospective analyses of sex/gender-related publication decisions in general medical journals: editorial rejection of population-based women's reproductive physiology.","au":"Kalidasan, Dharani","y":2022,"vn":"BMJ open","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmjopen-2021-057854","cby":11,"nref":51,"oast":"gold","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["funding-decision"],"st":"observational-quantitative","tp":"5","cc":4,"cp":5,"self":false,"ncoefs":0},{"id":"E7836ZR3","t":"Using ChatGPT to Evaluate the Methodological Components of Research Proposals: An Experimental Study on Undergraduate English Majors in Vietnam","au":"Tran, The Phi","y":2025,"vn":"Computer-Assisted Language Learning Electronic Journal","ty":"journalArticle","tyl":"Journal article","doi":"10.54855/callej.252637","cby":1,"nref":41,"oast":"hybrid","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level","factors"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"6","cc":5,"cp":0,"self":false,"ncoefs":4},{"id":"E7IAN24K","t":"A year in metascience (2025): the past, present and future of UK metascience","au":"UK Metascience Unit","y":2025,"vn":"","ty":"report","tyl":"Report","doi":"","cby":0,"nref":0,"oast":"","cx":"Grant","co":["Reliability-noise","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"7","cc":0,"cp":0,"self":false,"ncoefs":2},{"id":"E7VGIDPE","t":"Automated scholarly paper review: Concepts, technologies, and challenges","au":"Lin, Jialiang","y":2023,"vn":"Information Fusion","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.inffus.2023.101830","cby":31,"nref":192,"oast":"closed","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"6","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"E88CKB29","t":"Pork Barrel or Barrel of Gold? Examining the performance implications of earmarking in public R&D grants","au":"Kok, Holmer","y":2022,"vn":"Research Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.respol.2022.104514","cby":3,"nref":89,"oast":"hybrid","cx":"Grant","co":["Bias","Validity"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"1","cc":4,"cp":2,"self":false,"ncoefs":0},{"id":"E8UCA4GJ","t":"Author‐suggested reviewers: gender differences and influences on the peer review process at an ecology journal","au":"Fox, Charles W.","y":2016,"vn":"Functional Ecology","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/1365-2435.12665","cby":43,"nref":39,"oast":"bronze","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["reviewer-selection"],"st":"observational-quantitative","tp":"5","cc":2,"cp":5,"self":false,"ncoefs":0},{"id":"E93QFXS4","t":"Trust, transparency, and disciplinary alignment in research funding evaluations: an illustrative case from the Norwegian AI Center call (2024–2025)","au":"Krumsvik, Rune Johan","y":2026,"vn":"Frontiers in Research Metrics and Analytics","ty":"journalArticle","tyl":"Journal article","doi":"10.3389/frma.2026.1874986","cby":0,"nref":23,"oast":"gold","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["panel-consensus","funding-decision"],"st":"conceptual-commentary","tp":"6","cc":4,"cp":2,"self":false,"ncoefs":0},{"id":"E96MVBF3","t":"Negotiating science funding: The interplay of merit, bias, and administrative discretion in grant allocation in Kazakhstan","au":"Medeuov, Darkhan","y":2025,"vn":"PLoS ONE","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0318875","cby":1,"nref":53,"oast":"gold","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["demographic","cognitive-COI"],"pbs":"demographic","rs":[],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"2","cc":4,"cp":2,"self":false,"ncoefs":0},{"id":"E9LBD95C","t":"Why the referees’ reports I receive as an editor are so much better than the reports I receive as an author?","au":"García, J. A.","y":2016,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-015-1827-8","cby":7,"nref":31,"oast":"green","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"EBBEPJCN","t":"Single- versus double-blind reviewing","au":"Snodgrass, Richard T.","y":2006,"vn":"ACM SIGMOD Record","ty":"journalArticle","tyl":"Journal article","doi":"10.1145/1168092.1168094","cby":123,"nref":48,"oast":"bronze","cx":"Journal","co":["Bias","Validity","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"10","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"EBE9WAEU","t":"Strategyproof peer selection using randomization, partitioning, and apportionment","au":"Aziz, Haris","y":2019,"vn":"Artificial Intelligence","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.artint.2019.06.004","cby":10,"nref":68,"oast":"green","cx":"General","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["individual-scoring","funding-decision"],"st":"simulation-methodological","tp":"-1","cc":4,"cp":2,"self":false,"ncoefs":0},{"id":"EBJ79HYD","t":"Talent Identification at the limits of Peer Review: an analysis of the EMBO Postdoctoral Fellowships Selection Process","au":"Klaus, Bernd","y":2018,"vn":"bioRxiv (Cold Spring Harbor Laboratory)","ty":"journalArticle","tyl":"Journal article","doi":"10.1101/481655","cby":12,"nref":87,"oast":"green","cx":"Grant","co":["Bias","Validity"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["funding-decision","post-award-validity"],"st":"observational-quantitative","tp":"2","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"EC8W23YY","t":"Gender, race, and grant reviews: Translating and responding to research feedback.","au":"Biernat, Monica","y":2020,"vn":"Personality and Social Psychology Bulletin","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/0146167219845921","cby":23,"nref":50,"oast":"bronze","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring"],"st":"experimental-rct","tp":"2","cc":4,"cp":5,"self":false,"ncoefs":0},{"id":"ED6VSDD5","t":"Exploring Bias in Scientific Peer Review: An ASCO Initiative","au":"McKenzie, Nathalie D.","y":2022,"vn":"JCO Oncology Practice","ty":"journalArticle","tyl":"Journal article","doi":"10.1200/op.22.00275","cby":7,"nref":22,"oast":"closed","cx":"General","co":["Bias","Interventions"],"pco":"Bias","bs":["demographic","prestige-halo","cognitive-COI"],"pbs":"demographic","rs":[],"sg":["individual-scoring","funding-decision"],"st":"qualitative","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"EFBDCCW7","t":"Estimating Interrater Agreement with the Average Deviation Index: A User’s Guide","au":"Burke, Michael J.","y":2002,"vn":"Organizational Research Methods","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/1094428102005002002","cby":443,"nref":16,"oast":"closed","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"4","cc":6,"cp":3,"self":false,"ncoefs":0},{"id":"EFWSGJML","t":"A comparison of Cohen's Kappa and Gwet's AC1 when calculating inter-rater reliability coefficients: a study conducted with personality disorder samples.","au":"Wongpakaran, Nahathai","y":2013,"vn":"BMC Medical Research Methodology","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/1471-2288-13-61","cby":983,"nref":27,"oast":"gold","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level","factors"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"4","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"EGVIHGTT","t":"Peer review reduces spin in PCORI research reports","au":"Mayo-Wilson, Evan","y":2021,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31222/osf.io/xknhp","cby":0,"nref":18,"oast":"gold","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"11","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"EIPTF2DN","t":"Conservatism Gets Funded? A Field Experiment on the Role of Negative Information in Novel Project Evaluation","au":"Lane, Jacqueline N.","y":2021,"vn":"Management Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1287/mnsc.2021.4107","cby":55,"nref":66,"oast":"green","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["panel-consensus","individual-scoring"],"st":"experimental-rct","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"EJFLDVJU","t":"Retrospective analysis of the quality of reports by author-suggested and non-author-suggested reviewers in journals operating on open or single-blind peer review models","au":"Kowalczuk, Maria","y":2015,"vn":"BMJ Open","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmjopen-2015-008707","cby":61,"nref":17,"oast":"gold","cx":"Journal","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["reviewer-selection","individual-scoring"],"st":"observational-quantitative","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":1},{"id":"EJTCGZTW","t":"A Generalized Kappa Coefficient","au":"Uebersax, John S.","y":1982,"vn":"Educational and Psychological Measurement","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/0013164482421018","cby":76,"nref":5,"oast":"closed","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"4","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"EK4WFSX9","t":"Is the quality of reviews reflected in editors' and authors' satisfaction with peer review? A cross‐sectional study in 12 journals across four research fields","au":"Pranić, Shelly Melissa","y":2020,"vn":"Learned Publishing","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/leap.1344","cby":18,"nref":41,"oast":"closed","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"3","cc":2,"cp":1,"self":false,"ncoefs":3},{"id":"EQNEHVU3","t":"Does the Gender Composition of Scientific Committees Matter?","au":"Bagues, Manuel","y":2017,"vn":"American Economic Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1257/aer.20151211","cby":274,"nref":59,"oast":"bronze","cx":"General","co":["Bias","Validity"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["panel-consensus"],"st":"observational-quantitative","tp":"2","cc":4,"cp":5,"self":false,"ncoefs":0},{"id":"ES4ZGEIL","t":"Peer-review for the peer-review system","au":"Ruhli F.J.","y":2009,"vn":"Human Ontogenetics","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/huon.200900004","cby":8,"nref":22,"oast":"hybrid","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"-1","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"EZ496VHZ","t":"Peer review across borders: benefits and challenges of international review panels in research funding organizations","au":"Peterson, Helen","y":2024,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvaf030","cby":1,"nref":45,"oast":"hybrid","cx":"Grant","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":["factors","value"],"sg":["reviewer-selection","individual-scoring"],"st":"qualitative","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"EZA7L96Y","t":"Nobel and novice: Author prominence affects peer review.","au":"Huber, Jürgen","y":2022,"vn":"Proceedings of the National Academy of Sciences of the United States of America","ty":"journalArticle","tyl":"Journal article","doi":"10.1073/pnas.2205779119","cby":127,"nref":25,"oast":"hybrid","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["prestige-halo"],"pbs":"prestige-halo","rs":[],"sg":["individual-scoring"],"st":"experimental-rct","tp":"-1","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"EZNFKUCC","t":"Is There Gender Bias in the Peer Review Process at Journal of Neurophysiology?","au":"Lane, John A.S.","y":2009,"vn":"Journal of Neurophysiology","ty":"journalArticle","tyl":"Journal article","doi":"10.1152/jn.00196.2009","cby":16,"nref":8,"oast":"closed","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"5","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"F29KWSCM","t":"Gender-equal funding rates conceal unequal evaluations","au":"Bol, Thijs","y":2021,"vn":"Research Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.respol.2021.104399","cby":68,"nref":59,"oast":"hybrid","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring","panel-consensus","funding-decision"],"st":"observational-quantitative","tp":"2","cc":4,"cp":5,"self":false,"ncoefs":0},{"id":"F2AW6R6R","t":"The validation of peer review through research impact measures and the implications for funding strategies.","au":"Gallo, Stephen A","y":2014,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0106474","cby":49,"nref":30,"oast":"gold","cx":"Grant","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["funding-decision","post-award-validity"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"F2TL4U3X","t":"Peer review and publication. Presidential address before the 70th annual meeting of the American Society for Clinical Investigation, San Francisco, California, 30 April 1978.","au":"Wilson, Jess","y":1978,"vn":"Journal of Clinical Investigation","ty":"journalArticle","tyl":"Journal article","doi":"10.1172/jci109091","cby":105,"nref":15,"oast":"bronze","cx":"Journal","co":["Reliability-noise","Bias","Validity"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"0","cc":3,"cp":3,"self":false,"ncoefs":0},{"id":"F46K5IYY","t":"Gatekeeping Without Peer Review","au":"Tarr, Michael","y":2024,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31234/osf.io/g75c4","cby":0,"nref":7,"oast":"gold","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["funding-decision"],"st":"conceptual-commentary","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"F4C6TEFK","t":"Meta-research: justifying career disruption in funding applications, a survey of Australian researchers","au":"Barnett, Adrian","y":2022,"vn":"eLife","ty":"journalArticle","tyl":"Journal article","doi":"10.7554/elife.76123","cby":8,"nref":27,"oast":"gold","cx":"Grant","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring","proposal-quality"],"st":"observational-quantitative","tp":"2","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"F68GYWKS","t":"Improving the peer-review process for grant applications: reliability, validity, bias, and generalizability.","au":"Marsh, Herbert W","y":2008,"vn":"The American Psychologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/0003-066X.63.3.160","cby":300,"nref":35,"oast":"closed","cx":"Grant","co":["Reliability-noise","Bias","Validity","Interventions"],"pco":"Reliability-noise","bs":["demographic","prestige-halo"],"pbs":"demographic","rs":["factors"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"8","cc":4,"cp":4,"self":false,"ncoefs":4},{"id":"F6AMXHF7","t":"Gender differences in peer reviews of grant applications: A substantive-methodological synergy in support of the null hypothesis model","au":"Marsh, Herbert W.","y":2011,"vn":"Journal of Informetrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.joi.2010.10.004","cby":50,"nref":66,"oast":"closed","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"2","cc":4,"cp":4,"self":false,"ncoefs":0},{"id":"F6KINVIB","t":"Suggesting or Excluding Reviewers Can Help Get Your Paper Published","au":"Grimm, David","y":2005,"vn":"Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1126/science.309.5743.1974","cby":40,"nref":0,"oast":"closed","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["reviewer-selection"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":0,"self":false,"ncoefs":0},{"id":"F83SPSMN","t":"Peer review and innovation","au":"Spier, R.E.","y":2002,"vn":"Science and Engineering Ethics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11948-002-0035-0","cby":48,"nref":17,"oast":"bronze","cx":"General","co":["Bias","Validity"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"1","cc":4,"cp":4,"self":false,"ncoefs":0},{"id":"F8KRUZLI","t":"Reviewing peer review at the NIH.","au":"Lauer, Michael S","y":2015,"vn":"The New England Journal of Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.1056/NEJMp1507427","cby":21,"nref":5,"oast":"closed","cx":"Grant","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["post-award-validity"],"st":"conceptual-commentary","tp":"7","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"F9GMEZ7U","t":"AI-assisted prescreening of biomedical research proposals: ethical considerations and the pilot case of “la Caixa” Foundation","au":"Cortes, Carla M.","y":2024,"vn":"Data & Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/dap.2024.41","cby":12,"nref":40,"oast":"gold","cx":"Grant","co":["Bias","Interventions"],"pco":"Bias","bs":["ai-llm"],"pbs":"ai-llm","rs":[],"sg":["proposal-quality"],"st":"observational-quantitative","tp":"-1","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"F9HSQVFX","t":"Is there a gender gap in chemical sciences scholarly communication?","au":"Day, A E","y":2020,"vn":"Chemical science","ty":"journalArticle","tyl":"Journal article","doi":"10.1039/c9sc04090k","cby":56,"nref":23,"oast":"diamond","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"5","cc":3,"cp":5,"self":false,"ncoefs":0},{"id":"F9U369U4","t":"Potential sources of bias in research fellowship assessments: effects of university prestige and field of study","au":"Bornmann, Lutz","y":2006,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.3152/147154406781775850","cby":41,"nref":0,"oast":"closed","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["prestige-halo"],"pbs":"prestige-halo","rs":[],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"2","cc":4,"cp":0,"self":false,"ncoefs":0},{"id":"F9YGPA3S","t":"‘Science by consensus’ impedes scientific creativity and progress: An alternative to funding biomedical research","au":"Düzgüneş, Nejat","y":2022,"vn":"F1000Research","ty":"journalArticle","tyl":"Journal article","doi":"10.12688/f1000research.124082.1","cby":3,"nref":9,"oast":"gold","cx":"Grant","co":["Reliability-noise","Bias","Validity","Interventions"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"FAWCL6KR","t":"Development of a rating form to evaluate grant applications to the Hogg Foundation for Mental Health.","au":"Whaley, Arthur L","y":2006,"vn":"Evaluation Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/0193841X05275586","cby":5,"nref":19,"oast":"closed","cx":"Grant","co":["Reliability-noise","Validity"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level","factors"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":6,"cp":0,"self":false,"ncoefs":6},{"id":"FBSYGXK4","t":"Development and assessment of criteria to select projects for funding in the EU health programme.","au":"Van den Broucke, Stephan","y":2012,"vn":"European journal of public health","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/eurpub/ckr066","cby":7,"nref":7,"oast":"bronze","cx":"Grant","co":["Validity","Interventions"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["individual-scoring","panel-consensus"],"st":"observational-quantitative","tp":"7","cc":1,"cp":0,"self":false,"ncoefs":0},{"id":"FDNSXEWP","t":"Selection of research fellowship recipients by committee peer review. Reliability, fairness and predictive validity of Board of Trustees' decisions","au":"Bornmann, Lutz","y":2005,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-005-0214-2","cby":128,"nref":64,"oast":"green","cx":"Grant","co":["Reliability-noise","Bias","Validity"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring","funding-decision","post-award-validity"],"st":"observational-quantitative","tp":"-1","cc":4,"cp":4,"self":false,"ncoefs":0},{"id":"FE53H48F","t":"Questionable research practices in competitive grant funding: A survey.","au":"Conix, Stijn","y":2023,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0293310","cby":2,"nref":60,"oast":"gold","cx":"Grant","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"FEDSMXXE","t":"Ladies, Gentlemen, and Scientific Publication at the Royal Society, 1945–1990","au":"Røstvik, Camilla Mørk","y":2018,"vn":"Open Library of Humanities","ty":"journalArticle","tyl":"Journal article","doi":"10.16995/olh.265","cby":11,"nref":60,"oast":"diamond","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["reviewer-selection"],"st":"observational-quantitative","tp":"5","cc":4,"cp":1,"self":false,"ncoefs":0},{"id":"FEGJPLXX","t":"Developing a rubric to distinguish translational science from translational research in CTSA pilot projects","au":"Dillon P.","y":2023,"vn":"Journal of Clinical and Translational Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/cts.2023.437","cby":1,"nref":0,"oast":"gold","cx":"Grant","co":["Reliability-noise","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"FFGAWNZF","t":"Tres siglos después. . . ¿Es vigente el arbitraje por pares en las publicaciones científicas?","au":"Sepúlveda-Vildósola, Ana Carolina","y":2015,"vn":"Investigación en Educación Médica","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.riem.2015.09.001","cby":3,"nref":36,"oast":"diamond","cx":"Journal","co":["Reliability-noise","Bias","Validity","Interventions"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"-1","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"FGY2CT6A","t":"Does double-blind peer review effectively correct for gender disparities in research funding?","au":"Madsen, Emil Bargmann","y":2024,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31235/osf.io/uw3xe","cby":1,"nref":98,"oast":"gold","cx":"Grant","co":["Bias","Interventions"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"2","cc":4,"cp":5,"self":false,"ncoefs":0},{"id":"FJLLQTJJ","t":"Measuring nominal scale agreement among many raters.","au":"Fleiss, Joseph L.","y":1971,"vn":"Psychological Bulletin","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/h0031619","cby":8552,"nref":5,"oast":"closed","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"4","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"FLGSGDMR","t":"A randomized trial of an editorial intervention to reduce spin in the abstract's conclusion of manuscripts showed no significant effect.","au":"Ghannad, Mona","y":2021,"vn":"Journal of Clinical Epidemiology","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.jclinepi.2020.10.014","cby":26,"nref":27,"oast":"hybrid","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["proposal-quality"],"st":"experimental-rct","tp":"11","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"FM5RIB4P","t":"An Overview of Post-Publication Peer Review","au":"O’Sullivan, Lydia","y":2021,"vn":"Scholarly Assessment Reports","ty":"journalArticle","tyl":"Journal article","doi":"10.29024/sar.26","cby":26,"nref":49,"oast":"diamond","cx":"General","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"FPITCKY4","t":"Evaluation of stakeholder views on peer review of NIHR applications for funding: a qualitative study.","au":"Turner, Sheila","y":2018,"vn":"BMJ Open","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmjopen-2018-022548","cby":16,"nref":20,"oast":"gold","cx":"Grant","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"qualitative","tp":"7","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"FPWBZM9M","t":"Influence of peer review on the reporting of primary outcome(s) and statistical analyses of randomised trials.","au":"Hopewell, Sally","y":2018,"vn":"Trials","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s13063-017-2395-4","cby":10,"nref":19,"oast":"gold","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"11","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"FQMUE9X2","t":"Targeted, actionable and fair: Reviewer reports as feedback and its effect on ECR career choices.","au":"Derrick, Gemma Elizabeth","y":2023,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvad034","cby":7,"nref":42,"oast":"hybrid","cx":"Grant","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"qualitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"FS9SHXLZ","t":"The consequences of competition: simulating the effects of research grant allocation strategies","au":"Höylä, Tuomas","y":2016,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-016-1940-3","cby":16,"nref":28,"oast":"closed","cx":"Grant","co":["Reliability-noise","Validity"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["funding-decision"],"st":"simulation-methodological","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"FSW6MJXT","t":"Quantitative evaluation of peer review of program project and center applications in allergy and immunology.","au":"Das, N K","y":1985,"vn":"Journal of clinical immunology","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/BF00929456","cby":4,"nref":5,"oast":"closed","cx":"Grant","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":0,"self":false,"ncoefs":0},{"id":"FT37VU8M","t":"Evaluation for the allocation of university research project funding: Can rules improve the peer review?","au":"Reale, Emanuela","y":2017,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvx019","cby":29,"nref":24,"oast":"closed","cx":"Grant","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"qualitative","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"FUM3UBJI","t":"Problems with Peer Review Shine a Light on Gaps in Scientific Training","au":"Proctor, Diana M.","y":2023,"vn":"mBio","ty":"journalArticle","tyl":"Journal article","doi":"10.1128/mbio.03183-22","cby":21,"nref":18,"oast":"gold","cx":"Journal","co":["Reliability-noise","Bias","Interventions"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"FV9M3PCW","t":"Gatekeeping should be conserved in the open science era","au":"Desmond, Hugh","y":2024,"vn":"Synthese","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11229-024-04559-2","cby":10,"nref":61,"oast":"hybrid","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"1","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"FXQ5RBYN","t":"Ideology Critique, Representational Pluralism, and the Abolition of Peer Review. A Marxist Addendum to McKeown’s Criticism of Analytic Philosophy","au":"Jeuk, Alexander Albert","y":2022,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31219/osf.io/x4b2e","cby":1,"nref":86,"oast":"gold","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":4,"cp":2,"self":false,"ncoefs":0},{"id":"G2B4D2J6","t":"Peer reviewers’ dilemmas: a qualitative exploration of decisional conflict in the evaluation of grant applications in the medical humanities and social sciences","au":"Vallée-Tourangeau, Gaëlle","y":2022,"vn":"Humanities and Social Sciences Communications","ty":"journalArticle","tyl":"Journal article","doi":"10.1057/s41599-022-01050-6","cby":12,"nref":50,"oast":"gold","cx":"Grant","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["prestige-halo"],"pbs":"prestige-halo","rs":[],"sg":["individual-scoring"],"st":"qualitative","tp":"8","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"G526NSHB","t":"The role of metrics in peer assessments","au":"Langfeldt, Liv","y":2020,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvaa032","cby":46,"nref":65,"oast":"hybrid","cx":"General","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"9","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"G6BW4RFI","t":"Using peer review to evaluate the societal relevance of humanities research","au":"Conix, Stijn","y":2025,"vn":"Quantitative Science Studies","ty":"journalArticle","tyl":"Journal article","doi":"10.1162/qss.a.19","cby":0,"nref":46,"oast":"gold","cx":"General","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":["factors"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"0","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"G6K4RHJ6","t":"Grant Peer Review: Improving Inter-Rater Reliability with Training.","au":"Sattler, David N","y":2015,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0130450","cby":86,"nref":24,"oast":"gold","cx":"Grant","co":["Reliability-noise","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"experimental-rct","tp":"-1","cc":1,"cp":4,"self":false,"ncoefs":0},{"id":"G7V7A6A4","t":"Bias in peer review: a case study","au":"Walker, Richard","y":2015,"vn":"F1000Research","ty":"journalArticle","tyl":"Journal article","doi":"10.12688/f1000research.6012.1","cby":16,"nref":49,"oast":"gold","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["demographic","prestige-halo"],"pbs":"demographic","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"5","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"G87QSGJQ","t":"Comparing human and AI expertise in the academic peer review process: towards a hybrid approach","au":"Farber, Shai","y":2025,"vn":"Higher Education Research & Development","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/07294360.2024.2445575","cby":22,"nref":36,"oast":"closed","cx":"Journal","co":["Reliability-noise","Bias","Interventions"],"pco":"Reliability-noise","bs":["ai-llm"],"pbs":"ai-llm","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"6","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"G8RWYA8C","t":"What professors do in peer review: Interrogating assessment practices in the recruitment of professors in Sweden.","au":"Mahlck, Paula","y":2020,"vn":"Gender, Work and Organization","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/gwao.12500","cby":9,"nref":31,"oast":"hybrid","cx":"General","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["demographic","prestige-halo"],"pbs":"demographic","rs":[],"sg":["individual-scoring","funding-decision"],"st":"qualitative","tp":"2","cc":4,"cp":1,"self":false,"ncoefs":0},{"id":"G9XZ5UV7","t":"A Proposal for More Informative Abstracts of Clinical Articles","au":"LITERATURE, AD HOC WORKING GROUP FOR CRITICAL APPRAISAL OF THE MEDICAL","y":1987,"vn":"Annals of Internal Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.7326/0003-4819-106-4-598","cby":236,"nref":15,"oast":"closed","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["proposal-quality"],"st":"guideline-policy","tp":"0","cc":5,"cp":0,"self":false,"ncoefs":0},{"id":"GB3BX4WQ","t":"Standardising Peer Review in Paleontology journals","au":"Tennant, Jonathan","y":2020,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31233/osf.io/qzycs","cby":1,"nref":6,"oast":"gold","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"guideline-policy","tp":"0","cc":2,"cp":0,"self":false,"ncoefs":0},{"id":"GDWBNVPZ","t":"Assessment of gender and geographical bias in the editorial decision-making process of biomedical journals: A Case-Control study","au":"Gayet-Ageron A.","y":2024,"vn":"medRxiv","ty":"journalArticle","tyl":"Journal article","doi":"10.1101/2024.03.15.24304220","cby":1,"nref":39,"oast":"green","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["funding-decision"],"st":"observational-quantitative","tp":"5","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"GE2AQA7U","t":"An observational study of the Patient-Centered Outcomes Research Institute’s switch from in-person to virtual research application review panels during the COVID-19 pandemic","au":"Bloodworth, Robin","y":2026,"vn":"Research Integrity and Peer Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s41073-026-00198-y","cby":0,"nref":13,"oast":"gold","cx":"Grant","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["panel-consensus"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"GEWF5FVV","t":"Do Readers and Peer Reviewers Agree on Manuscript Quality?","au":"Justice, Amy C.","y":1994,"vn":"JAMA","ty":"journalArticle","tyl":"Journal article","doi":"10.1001/jama.1994.03520020043011","cby":43,"nref":2,"oast":"closed","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":5,"cp":0,"self":false,"ncoefs":0},{"id":"GFRCZP3P","t":"Journal editors’ perspectives on the roles and tasks of peer reviewers in biomedical journals: a qualitative study","au":"Glonti, Ketevan","y":2019,"vn":"BMJ Open","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmjopen-2019-033421","cby":74,"nref":38,"oast":"gold","cx":"Journal","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"qualitative","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"GGQQTLWV","t":"Language and socioeconomics predict geographic variation in peer review outcomes at an ecology journal","au":"Burns, C. Sean","y":2017,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-017-2517-5","cby":20,"nref":64,"oast":"closed","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"5","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"GHKXSBKL","t":"Influence of reviewer interaction network on long-term citations: a case study of the scientific peer-review system of the journal of high energy physics","au":"Sikdar, Sandipan","y":2017,"vn":"arXiv (Cornell University)","ty":"journalArticle","tyl":"Journal article","doi":"10.5555/3200334.3200354","cby":5,"nref":29,"oast":"green","cx":"Journal","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["reviewer-selection","post-award-validity"],"st":"observational-quantitative","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"GHRIZ2A3","t":"UK Doubles Its “World-Leading” Research in Life Sciences and Medicine in Six Years: Testing the Claim?","au":"Wooding, Steven","y":2015,"vn":"PLoS ONE","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0132990","cby":16,"nref":13,"oast":"gold","cx":"General","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["post-award-validity"],"st":"observational-quantitative","tp":"9","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"GHRV2IBK","t":"A Social Epistemological Inquiry into Biases in Journal Peer Review","au":"Jukola, Saana","y":2016,"vn":"Perspectives on Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1162/posc_a_00237","cby":16,"nref":46,"oast":"closed","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":3,"cp":4,"self":false,"ncoefs":0},{"id":"GIY7LUIX","t":"Development and Validation of a Scoring Rubric for Editorial Evaluation of Peer-review Quality: A Pilot Study.","au":"Love, Jeffrey N.","y":2024,"vn":"PubMed","ty":"journalArticle","tyl":"Journal article","doi":"10.5811/westjem.18432","cby":1,"nref":32,"oast":"green","cx":"Journal","co":["Reliability-noise","Validity","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level","factors"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"3","cc":3,"cp":1,"self":false,"ncoefs":3},{"id":"GKF84MXT","t":"Sceptics and champions: participant insights on the use of partial randomization to allocate research culture funding","au":"Davies, Catherine","y":2024,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvaf006","cby":0,"nref":14,"oast":"hybrid","cx":"Grant","co":["Bias","Validity","Interventions"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["funding-decision"],"st":"qualitative","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"GL6NPLR6","t":"Open science and modified funding lotteries can impede the natural selection of bad science","au":"Smaldino, Paul E.","y":2019,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31219/osf.io/zvkwq","cby":18,"nref":66,"oast":"gold","cx":"General","co":["Validity","Interventions"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["funding-decision"],"st":"simulation-methodological","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"GLBUJJ2K","t":"The Reliability of Peer Assessments","au":"Goldman, Ronald L.","y":1994,"vn":"Evaluation & the Health Professions","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/016327879401700101","cby":62,"nref":60,"oast":"closed","cx":"General","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"systematic-review-meta-analysis","tp":"15","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"GNW32DYJ","t":"But the reviewers are making different criticisms of my paper! Diversity and uniqueness in reviewer comments.","au":"Fiske, Donald W.","y":1990,"vn":"American Psychologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/0003-066x.45.5.591","cby":144,"nref":5,"oast":"closed","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"3","cc":3,"cp":0,"self":false,"ncoefs":0},{"id":"GPQAWJJ9","t":"An analysis of rater effects in reviews of scientific manuscripts","au":"Turner, D. P.","y":2017,"vn":"undefined","ty":"journalArticle","tyl":"Journal article","doi":"","cby":0,"nref":0,"oast":"","cx":"Journal","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"3","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"GPUHMSPG","t":"Double anonymity in peer review within the chemistry periodicals community","au":"Brown, Richard J. C.","y":2007,"vn":"Learned Publishing","ty":"journalArticle","tyl":"Journal article","doi":"10.1087/174148507x185108","cby":11,"nref":0,"oast":"closed","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"0","cc":2,"cp":0,"self":false,"ncoefs":0},{"id":"GRDE4DDS","t":"Mentored peer review of standardized manuscripts as a teaching tool for residents: a pilot randomized controlled multi-center study.","au":"Wong, Victoria S S","y":2017,"vn":"Research Integrity and Peer Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s41073-017-0032-0","cby":21,"nref":38,"oast":"gold","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"experimental-rct","tp":"-1","cc":2,"cp":1,"self":false,"ncoefs":1},{"id":"GS27TS6D","t":"Responses by four Local Research Ethics Committees to submitted proposals.","au":"Kent, G","y":1999,"vn":"Journal of medical ethics","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/jme.25.3.274","cby":23,"nref":17,"oast":"bronze","cx":"General","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["panel-consensus"],"st":"observational-quantitative","tp":"7","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"GSEWAUVH","t":"Peer review for funding decisions","au":"Liu, Yuxian","y":2025,"vn":"Journal of Data and Information Science","ty":"journalArticle","tyl":"Journal article","doi":"10.2478/jdis-2025-0050","cby":1,"nref":75,"oast":"diamond","cx":"Grant","co":["Reliability-noise","Bias","Validity","Interventions"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":["level"],"sg":["whole-process"],"st":"conceptual-commentary","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"GSWIY3RL","t":"Reimagining Peer Review Needs Publishers and Institutions to Collaborate More","au":"King, Stuart","y":2024,"vn":"Exchanges The Interdisciplinary Research Journal","ty":"journalArticle","tyl":"Journal article","doi":"10.31273/eirj.v11i3.1519","cby":1,"nref":18,"oast":"diamond","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"GUAUHBD5","t":"Peer review procedures as practice, decision, and governance – the road to theories of peer review","au":"Reinhart, Martin","y":2021,"vn":"Science and Public Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/scipol/scad089","cby":12,"nref":48,"oast":"hybrid","cx":"General","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"GUJQFPNY","t":"What Are Reviewers Looking For? A Qualitative Analysis of Open-Ended Responses from a Questionnaire Sent to Faculty in Agricultural Communications","au":"Settle, Quisto","y":2020,"vn":"Journal of Applied Communications","ty":"journalArticle","tyl":"Journal article","doi":"10.4148/1051-0834.2318","cby":1,"nref":43,"oast":"diamond","cx":"Journal","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"qualitative","tp":"-1","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"GUX5QL5F","t":"Criticism and Judgment: A Critical Look at Scientific Peer Review","au":"Hope, Aluko A.","y":2019,"vn":"American Journal of Critical Care","ty":"journalArticle","tyl":"Journal article","doi":"10.4037/ajcc2019152","cby":14,"nref":5,"oast":"bronze","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"GVVSKS4V","t":"Peer review reliability: The hierarchy of the sciences","au":"Crothers, Charles","y":1993,"vn":"Behavioral and Brain Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/s0140525x00030764","cby":25,"nref":26,"oast":"closed","cx":"General","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"12","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"GZBYTT5I","t":"The time has come for big changes to improve research funding","au":"Kolarz, Peter","y":2026,"vn":"PLoS Biology","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pbio.3003754","cby":0,"nref":10,"oast":"gold","cx":"Grant","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"H3V2822W","t":"Does the use of open, non-anonymous peer review in scholarly publishing introduce bias? Evidence from the F1000 post-publication open peer review publishing model","au":"Thelwall, Mike","y":2019,"vn":"arXiv (Cornell University)","ty":"journalArticle","tyl":"Journal article","doi":"10.48550/arxiv.1911.03379","cby":0,"nref":24,"oast":"green","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["demographic","cognitive-COI"],"pbs":"demographic","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"0","cc":4,"cp":1,"self":false,"ncoefs":0},{"id":"H4CUS6M8","t":"An examination of the federal psychiatric training grant peer review process.","au":"Wilkinson, Charles B","y":1977,"vn":"The American Journal of Psychiatry","ty":"journalArticle","tyl":"Journal article","doi":"10.1176/ajp.134.6.637","cby":2,"nref":3,"oast":"closed","cx":"Grant","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["reviewer-selection"],"st":"observational-quantitative","tp":"8","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"H73S9JI4","t":"Peer Advocacy: Expressions of Loyalty in Peer Review","au":"Geschwind, Lars","y":2022,"vn":"","ty":"book","tyl":"Book","doi":"10.1007/978-3-030-75263-7_9","cby":4,"nref":25,"oast":"gold","cx":"General","co":["Bias"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["individual-scoring"],"st":"book-monograph","tp":"0","cc":4,"cp":1,"self":false,"ncoefs":0},{"id":"H8TDR5AF","t":"Are female applicants disadvantaged in National Institutes of Health peer review? Combining algorithmic text mining and qualitative methods to detect evaluative differences in R01 reviewers' critiques.","au":"Magua, Wairimu","y":2017,"vn":"Journal of Women's Health","ty":"journalArticle","tyl":"Journal article","doi":"10.1089/jwh.2016.6021","cby":127,"nref":82,"oast":"closed","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"2","cc":4,"cp":5,"self":false,"ncoefs":0},{"id":"H9B5WQZN","t":"Funding decision-making systems: An empirical comparison of continuous and dichotomous approaches based on psychometric theory","au":"Mutz, Rüdiger","y":2016,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvw002","cby":5,"nref":37,"oast":"bronze","cx":"Grant","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":["level"],"sg":["funding-decision"],"st":"simulation-methodological","tp":"1","cc":4,"cp":2,"self":false,"ncoefs":0},{"id":"HAKSM7H8","t":"Comments on Single-Blind Reviewing from the Editorial Staff","au":"Atkeson, Lonna Rae","y":2018,"vn":"Political Analysis","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/pan.2018.34","cby":0,"nref":4,"oast":"bronze","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"10","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"HBP5E7SN","t":"Do funding applications where peer reviewers disagree have higher citations? A cross-sectional study.","au":"Barnett, Adrian G","y":2018,"vn":"F1000Research","ty":"journalArticle","tyl":"Journal article","doi":"10.12688/f1000research.15479.2","cby":3,"nref":43,"oast":"gold","cx":"Grant","co":["Reliability-noise","Validity"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["factors"],"sg":["individual-scoring","post-award-validity"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"HBXRKG4U","t":"‘Fund people, not projects’: From narrative CVs to lotteries in science funding policy","au":"Shaw, Jamie","y":2024,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvae035","cby":1,"nref":87,"oast":"closed","cx":"Grant","co":["Bias","Validity","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["funding-decision"],"st":"conceptual-commentary","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"HC9VZTTZ","t":"Patterns of evaluation in science: Institutionalisation, structure and functions of the referee system","au":"Zuckerman, Harriet","y":1971,"vn":"Minerva","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/bf01553188","cby":800,"nref":0,"oast":"closed","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"-1","cc":3,"cp":0,"self":false,"ncoefs":0},{"id":"HE2HZJ8P","t":"Quality assessment of scientific manuscripts in peer review and education","au":"Augusteijn, Hilde Elisabeth Maria","y":2023,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31219/osf.io/7dc6a","cby":5,"nref":62,"oast":"gold","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["individual-scoring"],"st":"experimental-rct","tp":"3","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"HEYTAC8Z","t":"Evaluation of Large Language Model Performance in Assessing Health Economic Study Quality","au":"Dun, Chen","y":2025,"vn":"Journal of health economics and outcomes research","ty":"journalArticle","tyl":"Journal article","doi":"10.36469/jheor.2025.145214","cby":0,"nref":14,"oast":"gold","cx":"General","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["ai-llm"],"pbs":"ai-llm","rs":["level"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"11","cc":5,"cp":0,"self":false,"ncoefs":3},{"id":"HF9NCMFH","t":"An exploration of referees’ comments published in open peer review journals: The characteristics of review language and the association between review scrutiny and citations","au":"Wolfram, Dietmar","y":2021,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvab005","cby":26,"nref":28,"oast":"closed","cx":"Journal","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["individual-scoring","post-award-validity"],"st":"observational-quantitative","tp":"-1","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"HFSYDHZ3","t":"Topic choice contributes to the lower rate of NIH awards to African-American/black scientists","au":"Hoppe, Travis","y":2019,"vn":"Science Advances","ty":"journalArticle","tyl":"Journal article","doi":"10.1126/sciadv.aaw7238","cby":629,"nref":27,"oast":"gold","cx":"Grant","co":["Bias","Validity"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"14","cc":4,"cp":2,"self":false,"ncoefs":0},{"id":"HH53MPTF","t":"Evaluating psychological research reports: Dimensions, reliability, and correlates of quality judgments.","au":"Gottfredson, Stephen D.","y":1978,"vn":"American Psychologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/0003-066x.33.10.920","cby":256,"nref":42,"oast":"closed","cx":"Journal","co":["Reliability-noise","Validity"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["factors"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"3","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"HI2UVM24","t":"Effect of an editorial intervention to improve the completeness of reporting of randomised trials: a randomised controlled trial","au":"Blanco, David","y":2020,"vn":"BMJ Open","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmjopen-2020-036799","cby":69,"nref":41,"oast":"gold","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"experimental-rct","tp":"11","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"HIEHTPKP","t":"What factors are important to the success of resubmitted grant applications in biomedical and health research? A retrospective study of over 20 000 applications to the Canadian Institutes of Health Research","au":"Wrightson, James G.","y":2025,"vn":"FACETS","ty":"journalArticle","tyl":"Journal article","doi":"10.1139/facets-2024-0116","cby":0,"nref":34,"oast":"gold","cx":"Grant","co":["Bias","Validity"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["proposal-quality","funding-decision"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"HMW89QW7","t":"Male applicants are more likely to be awarded fellowships than female applicants: A case study of a Japanese national funding agency.","au":"Kyogoku, Daisuke","y":2023,"vn":"PloS one","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0291372","cby":3,"nref":45,"oast":"gold","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["funding-decision"],"st":"observational-quantitative","tp":"2","cc":4,"cp":5,"self":false,"ncoefs":0},{"id":"HNZ2E7JB","t":"Does double‐blind peer review reduce bias? Evidence from a top computer science conference","au":"Sun, Mengyi","y":2021,"vn":"Journal of the Association for Information Science and Technology","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/asi.24582","cby":3,"nref":32,"oast":"hybrid","cx":"Journal","co":["Reliability-noise","Bias","Validity","Interventions"],"pco":"Reliability-noise","bs":["prestige-halo"],"pbs":"prestige-halo","rs":["factors","value"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"10","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"HPC6SYSG","t":"An Examination of Sources of Peer-Review Bias","au":"Blackburn, Jessica L.","y":2006,"vn":"Psychological Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/j.1467-9280.2006.01715.x","cby":57,"nref":17,"oast":"closed","cx":"Journal","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":["factors"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":3,"cp":4,"self":false,"ncoefs":3},{"id":"HPEHPE8N","t":"Academic judgments under uncertainty: A study of collective anchoring effects in Swedish Research Council panel groups","au":"Roumbanis, Lambros","y":2016,"vn":"Social Studies of Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/0306312716659789","cby":60,"nref":62,"oast":"closed","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["panel-consensus"],"st":"qualitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"HPGFXDLZ","t":"Relationship between journal impact factor and the thoroughness and helpfulness of peer reviews","au":"Severin, Anna","y":2023,"vn":"PLoS Biology","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pbio.3002238","cby":38,"nref":43,"oast":"gold","cx":"Journal","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"6","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"HQHC5EYY","t":"Centralized Funding and Epistemic Exploration","au":"Avin, Shahar","y":2017,"vn":"The British Journal for the Philosophy of Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/bjps/axx059","cby":34,"nref":41,"oast":"green","cx":"Grant","co":["Bias","Validity"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["funding-decision"],"st":"simulation-methodological","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"HQXJUIA5","t":"Journal and disciplinary variations in academic open peer review anonymity, outcomes, and length","au":"Thelwall, Mike","y":2022,"vn":"Journal of Librarianship and Information Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/09610006221079345","cby":21,"nref":30,"oast":"green","cx":"Journal","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"HT7RTIKQ","t":"Prior publication productivity, grant percentile ranking, and topic-normalized citation impact of NHLBI cardiovascular R01 grants.","au":"Kaltman, Jonathan R","y":2014,"vn":"Circulation Research","ty":"journalArticle","tyl":"Journal article","doi":"10.1161/CIRCRESAHA.115.304766","cby":52,"nref":12,"oast":"closed","cx":"Grant","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["post-award-validity"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"HUD5RNL8","t":"Peer Review Quality and Transparency of the Peer-Review Process in Open Access and Subscription Journals.","au":"Wicherts, Jelte M","y":2016,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0147913","cby":167,"nref":26,"oast":"gold","cx":"Journal","co":["Reliability-noise","Validity"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["factors"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"HUMIF4QI","t":"How do NIHR peer review panels use bibliometric information to support their decisions?","au":"Gunashekar, Salil","y":2017,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-017-2417-8","cby":34,"nref":9,"oast":"hybrid","cx":"Grant","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["panel-consensus"],"st":"qualitative","tp":"9","cc":4,"cp":0,"self":false,"ncoefs":0},{"id":"HUZ86E9X","t":"Is the Lack of Inter-referee Reliability a Threat to the Effectiveness of Peer Review in Science?","au":"Wray, K. Brad","y":2025,"vn":"Perspectives on Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1162/posc_a_00639","cby":0,"nref":30,"oast":"closed","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"HVR47UFQ","t":"Metascience on Peer Review: Testing the Effects of a Study’s Originality and Statistical Significance in a Field Experiment","au":"Elson, Malte","y":2020,"vn":"Advances in Methods and Practices in Psychological Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/2515245919895419","cby":22,"nref":43,"oast":"bronze","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["individual-scoring"],"st":"experimental-rct","tp":"-1","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"HY8N87UZ","t":"Clinical trials proposed for the VA Cooperative Studies Program: Success rates and factors impacting approval","au":"Burnaska, David R.","y":2021,"vn":"Contemporary Clinical Trials Communications","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.conctc.2021.100811","cby":2,"nref":60,"oast":"gold","cx":"Grant","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["funding-decision"],"st":"guideline-policy","tp":"7","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"HYTZEAF9","t":"MERIT (Manuscript Evaluation Reflecting Intrinsic Tenets): A Framework for Efficient and Equitable Academic Peer Review","au":"Arboleda‐Velásquez, Joseph F.","y":2024,"vn":"ScienceBank","ty":"journalArticle","tyl":"Journal article","doi":"10.61340/rwng4653","cby":0,"nref":9,"oast":"hybrid","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"HZ4YRKUY","t":"The modified lottery: Formalizing the intrinsic randomness of research funding.","au":"De Peuter, Steven","y":2022,"vn":"Accountability in research","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/08989621.2021.1927727","cby":15,"nref":85,"oast":"green","cx":"Grant","co":["Reliability-noise","Bias","Validity","Interventions"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["funding-decision"],"st":"conceptual-commentary","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"I358ZRGB","t":"Peer Review Under Review — a Statistical Study on Proposal Ranking at ESO. Part I: the Pre-meeting Phase","au":"Patat, Ferdinando","y":2018,"vn":"Publications of the Astronomical Society of the Pacific","ty":"journalArticle","tyl":"Journal article","doi":"10.1088/1538-3873/aac463","cby":2,"nref":11,"oast":"bronze","cx":"Grant","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":3},{"id":"I42AJ26U","t":"What makes an effective grants peer reviewer? An exploratory study of the necessary skills.","au":"Steiner Davis, Miriam L E","y":2020,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0232327","cby":14,"nref":61,"oast":"gold","cx":"Grant","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"qualitative","tp":"8","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"I4LKXWXM","t":"Communicating and compromising on disciplinary expertise in the peer review of research proposals","au":"Huutoniemi, Katri Ilona","y":2012,"vn":"Social Studies of Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/0306312712458478","cby":72,"nref":47,"oast":"closed","cx":"Grant","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["panel-consensus"],"st":"qualitative","tp":"9","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"I4PYTQC4","t":"Peer-review in a world with rational scientists: Toward selection of the average","au":"Thurner, S.","y":2011,"vn":"The European Physical Journal B","ty":"journalArticle","tyl":"Journal article","doi":"10.1140/epjb/e2011-20545-7","cby":57,"nref":12,"oast":"green","cx":"Journal","co":["Bias","Validity"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"I4ZU3XFU","t":"The reliability of reviews for the American Psychologist.","au":"Scarr, Sandra","y":1978,"vn":"American Psychologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/h0078544","cby":151,"nref":1,"oast":"closed","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"3","cc":5,"cp":0,"self":false,"ncoefs":0},{"id":"I5Y9JHPK","t":"Predicting productivity returns on investment: thirty years of peer review, grant funding, and publication of highly cited papers at the national heart, lung, and blood institute.","au":"Lauer, Michael S","y":2015,"vn":"Circulation Research","ty":"journalArticle","tyl":"Journal article","doi":"10.1161/CIRCRESAHA.115.306830","cby":55,"nref":29,"oast":"closed","cx":"Grant","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["post-award-validity"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"I7KMMCXK","t":"What works for peer review and decision-making in research funding: a realist synthesis.","au":"Recio-Saucedo, Alejandra","y":2022,"vn":"Research Integrity and Peer Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s41073-022-00120-2","cby":51,"nref":70,"oast":"gold","cx":"Grant","co":["Reliability-noise","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"systematic-review-meta-analysis","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"I8D6WFWV","t":"On the effectiveness of the scientific peer-review system: a case study of the Journal of High Energy Physics","au":"Sikdar, Sandipan","y":2018,"vn":"International Journal on Digital Libraries","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s00799-018-0247-9","cby":4,"nref":31,"oast":"closed","cx":"Journal","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["whole-process","post-award-validity"],"st":"observational-quantitative","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"I8SJR4X6","t":"UnWEIRDing Peer Review in Human-Computer Interaction","au":"Nigatu, Hellina Hailu","y":2026,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.1145/3772318.3791553","cby":1,"nref":106,"oast":"gold","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring"],"st":"qualitative","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"IBJ7WKTZ","t":"Is novel research worth doing? Evidence from peer review at 49 journals","au":"Teplitskiy, Misha","y":2022,"vn":"Proceedings of the National Academy of Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1073/pnas.2118046119","cby":38,"nref":71,"oast":"hybrid","cx":"Journal","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"0","cc":4,"cp":2,"self":false,"ncoefs":0},{"id":"IBR4QCFH","t":"Journal policies and editors’ opinions on peer review","au":"Hamilton, Daniel G","y":2020,"vn":"eLife","ty":"journalArticle","tyl":"Journal article","doi":"10.7554/eLife.62529","cby":68,"nref":55,"oast":"gold","cx":"Journal","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"IBUMSYH9","t":"Determinants of quality of research environment: An assessment of the environment submissions in the UK’s Research Excellence Framework in 2014","au":"Pinar, Mehmet","y":2020,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvaa003","cby":16,"nref":28,"oast":"hybrid","cx":"General","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["prestige-halo"],"pbs":"prestige-halo","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"9","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"ICFNMK39","t":"Attitudes of referees in a multidisciplinary journal: An empirical analysis","au":"Casnici, Niccolò","y":2016,"vn":"Journal of the Association for Information Science and Technology","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/asi.23665","cby":59,"nref":50,"oast":"bronze","cx":"Journal","co":["Reliability-noise","Bias","Validity"],"pco":"Reliability-noise","bs":["demographic"],"pbs":"demographic","rs":["level"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"ID6UT6PK","t":"AI in the gatekeeper’s chair: elite researchers’ perceptions of AI-assisted feedback in journal peer review","au":"Li, Heng","y":2026,"vn":"Assessment & Evaluation in Higher Education","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/02602938.2026.2691499","cby":0,"nref":35,"oast":"closed","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["ai-llm"],"pbs":"ai-llm","rs":[],"sg":["individual-scoring"],"st":"experimental-rct","tp":"6","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"IDTRS5C7","t":"Illusory reliability in journal reviewing.","au":"Gilmore, J. Barnard","y":1979,"vn":"Canadian Psychological Review/Psychologie canadienne","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/h0081518","cby":36,"nref":0,"oast":"closed","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"3","cc":5,"cp":0,"self":false,"ncoefs":0},{"id":"IFPUHR5K","t":"Biases in grant proposal success rates, funding rates and award sizes affect the geographical distribution of funding for biomedical research.","au":"Wahls, Wayne P","y":2016,"vn":"PeerJ","ty":"journalArticle","tyl":"Journal article","doi":"10.7717/peerj.1917","cby":26,"nref":14,"oast":"gold","cx":"Grant","co":["Bias","Validity"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["funding-decision"],"st":"observational-quantitative","tp":"14","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"IGGDUDQL","t":"Does citation context information enhance the validity of citation analysis for measuring research quality? An empirical comparison of peer assessments and enriched citations","au":"Donner, Paul","y":2025,"vn":"Quantitative Science Studies","ty":"journalArticle","tyl":"Journal article","doi":"10.1162/qss.a.15","cby":1,"nref":36,"oast":"gold","cx":"General","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"9","cc":3,"cp":4,"self":false,"ncoefs":0},{"id":"IH5URG3N","t":"The Game Between a Biased Reviewer and His Editor","au":"García, J. A.","y":2017,"vn":"Science and Engineering Ethics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11948-017-9998-8","cby":13,"nref":24,"oast":"green","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring","funding-decision"],"st":"simulation-methodological","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"II3R27M2","t":"The Bias of Sighted Reviewers in Research Proposal Evaluation: A Comparative Analysis of Blind and Open Review in Korea","au":"Lee, Mushin","y":2000,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1023/a:1005636503358","cby":14,"nref":19,"oast":"closed","cx":"Grant","co":["Bias","Interventions"],"pco":"Bias","bs":["prestige-halo"],"pbs":"prestige-halo","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"10","cc":4,"cp":4,"self":false,"ncoefs":0},{"id":"IJWHZBDG","t":"Grant reviewer perceptions of the quality, effectiveness, and influence of panel discussion.","au":"Gallo, Stephen A","y":2020,"vn":"Research Integrity and Peer Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s41073-020-00093-0","cby":18,"nref":27,"oast":"gold","cx":"Grant","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["panel-consensus"],"st":"observational-quantitative","tp":"13","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"IKBM99JX","t":"Effective Peer Review: Who, Where, or What?","au":"Hall, Russell P.","y":2022,"vn":"JID Innovations","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.xjidi.2022.100162","cby":4,"nref":27,"oast":"gold","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"ILFZLJIP","t":"Associations of topic-specific peer review outcomes and institute and center award rates with funding disparities at the National Institutes of Health.","au":"Lauer, Michael S","y":2021,"vn":"eLife","ty":"journalArticle","tyl":"Journal article","doi":"10.7554/eLife.67173","cby":39,"nref":9,"oast":"gold","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["funding-decision","individual-scoring"],"st":"observational-quantitative","tp":"14","cc":4,"cp":2,"self":false,"ncoefs":0},{"id":"IPM7YXWJ","t":"Anonymous reviewing and the peer-review process.","au":"Surwillo, Walter W.","y":1986,"vn":"American Psychologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/0003-066x.41.2.218.a","cby":50,"nref":2,"oast":"closed","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"-1","cc":5,"cp":0,"self":false,"ncoefs":0},{"id":"IQ5FTA4C","t":"Double-blind reviewing and gender biases at EvoLang conferences: An update","au":"Cuskley, Christine","y":2019,"vn":"Journal of Language Evolution","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/jole/lzz007","cby":8,"nref":32,"oast":"green","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"5","cc":4,"cp":1,"self":false,"ncoefs":0},{"id":"IQURWTM7","t":"A primer on different models of peer review","au":"Alfred, Jane","y":2023,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31219/osf.io/jnehu","cby":0,"nref":18,"oast":"gold","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"IRL44KLX","t":"FENS‐Kavli Network of Excellence: Postponed, non‐competitive peer review for research funding","au":"Dresler, Martin","y":2022,"vn":"European Journal of Neuroscience","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/ejn.15818","cby":6,"nref":39,"oast":"bronze","cx":"Grant","co":["Reliability-noise","Validity","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"ITEBWPJP","t":"Unravelling the panel contribution upon peer review evaluation of numerous, unstructured and highly interdisciplinary research proposals","au":"Baimpos, Theodoros","y":2020,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvz013","cby":2,"nref":27,"oast":"hybrid","cx":"Grant","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["panel-consensus","individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":6,"cp":2,"self":false,"ncoefs":0},{"id":"ITKLS9MM","t":"From funding agencies to scientific agency","au":"Bollen, Johan","y":2014,"vn":"EMBO Reports","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/embr.201338068","cby":61,"nref":9,"oast":"bronze","cx":"Grant","co":["Validity","Interventions"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"ITXJM6F2","t":"Bias in awarding research grants.","au":"Martin, B","y":1986,"vn":"British medical journal (Clinical research ed.)","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmj.293.6546.550","cby":6,"nref":9,"oast":"bronze","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["funding-decision"],"st":"conceptual-commentary","tp":"-1","cc":3,"cp":0,"self":false,"ncoefs":0},{"id":"IV59GNII","t":"'Your comments are meaner than your score': score calibration talk influences intra- and inter-panel variability during scientific grant peer review.","au":"Pier, Elizabeth L","y":2017,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvw025","cby":49,"nref":62,"oast":"bronze","cx":"Grant","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["factors"],"sg":["panel-consensus"],"st":"observational-quantitative","tp":"13","cc":1,"cp":2,"self":false,"ncoefs":8},{"id":"IVG78IE3","t":"Statistical reviewers improve reporting in biomedical articles: a randomized trial.","au":"Cobo, Erik","y":2007,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0000332","cby":116,"nref":46,"oast":"gold","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"experimental-rct","tp":"11","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"IWCKCW6T","t":"The forced battle between peer-review and scientometric research assessment: Why the CoARA initiative is unsound","au":"Abramo, Giovanni","y":2024,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvae021","cby":30,"nref":39,"oast":"hybrid","cx":"General","co":["Reliability-noise","Bias","Validity"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"-1","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"IWQZXBVF","t":"Randomised controlled trial of the READER method of critical appraisal in general practice","au":"MacAuley, Domhnall","y":1998,"vn":"BMJ","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmj.316.7138.1134","cby":30,"nref":15,"oast":"bronze","cx":"Analogue","co":["Reliability-noise","Validity","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["factors"],"sg":["individual-scoring"],"st":"experimental-rct","tp":"11","cc":2,"cp":0,"self":false,"ncoefs":0},{"id":"IX2FS8HU","t":"Blinded versus unblinded review: A field study comparing the equity of peer review","au":"Pleskac, Timothy Joseph","y":2025,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31234/osf.io/q2tkw_v2","cby":0,"nref":0,"oast":"gold","cx":"Journal","co":["Reliability-noise","Bias","Validity"],"pco":"Reliability-noise","bs":["demographic","cognitive-COI"],"pbs":"demographic","rs":["factors","value"],"sg":["individual-scoring","post-award-validity"],"st":"experimental-rct","tp":"10","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"IXVXHAXS","t":"An Objective Rating Form to Evaluate Grant Proposals to the Hogg Foundation for Mental Health","au":"Whaley, Arthur L.","y":2006,"vn":"Evaluation Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/0193841x06288737","cby":3,"nref":11,"oast":"closed","cx":"Grant","co":["Reliability-noise","Validity","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":6,"cp":0,"self":false,"ncoefs":0},{"id":"IZAQ9Y4J","t":"role of peer review in the evaluation of research in Italy. Some remarks on the evaluation of PRINs","au":"Vivarelli, Maurizio","y":2022,"vn":"JLIS it","ty":"journalArticle","tyl":"Journal article","doi":"10.36253/jlis.it-500","cby":0,"nref":22,"oast":"diamond","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"9","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"J2S8SMJC","t":"Alternatives to Peer Review: Novel Approaches for Research Evaluation","au":"Birukou, Aliaksandr","y":2011,"vn":"Frontiers in Computational Neuroscience","ty":"journalArticle","tyl":"Journal article","doi":"10.3389/fncom.2011.00056","cby":67,"nref":90,"oast":"gold","cx":"General","co":["Reliability-noise","Bias","Validity","Interventions"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"J2WJBPIT","t":"Scientific publishing without gatekeeping: an empirical investigation of eLife’s new peer review process","au":"Mutz, Rüdiger","y":2025,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-025-05422-y","cby":3,"nref":61,"oast":"hybrid","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":3,"cp":1,"self":false,"ncoefs":4},{"id":"J4LTG2X6","t":"Putting Journal-based Peer Review in Its Place: Transparency, Post-publication Review, and Systems for Self-correction","au":"Higgins, Wendy C","y":2025,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31234/osf.io/g6fut_v1","cby":0,"nref":0,"oast":"gold","cx":"Journal","co":["Validity","Interventions"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"J583ZHPU","t":"Weighted kappa: Nominal scale agreement provision for scaled disagreement or partial credit.","au":"Cohen, Jacob","y":1968,"vn":"Psychological Bulletin","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/h0026256","cby":8557,"nref":10,"oast":"closed","cx":"Analogue","co":["Reliability-noise","Validity"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"4","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"J5BUPAHX","t":"Administrative discretion in scientific funding: Evidence from a prestigious postdoctoral training program✰","au":"Ginther, Donna K.","y":2020,"vn":"Research Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.respol.2020.103953","cby":21,"nref":59,"oast":"green","cx":"Grant","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["funding-decision"],"st":"observational-quantitative","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"J5PCTC3D","t":"Quantification of Agreement in Psychiatric Diagnosis Revisited","au":"Shrout, Patrick E.","y":1987,"vn":"Archives of General Psychiatry","ty":"journalArticle","tyl":"Journal article","doi":"10.1001/archpsyc.1987.01800140084013","cby":351,"nref":35,"oast":"closed","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["whole-process"],"st":"conceptual-commentary","tp":"4","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"J6L3SI4Q","t":"Fair or flawed? Rethinking grant review with generative AI","au":"Sikimić, Vlasta","y":2025,"vn":"Synthese","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11229-025-05366-z","cby":0,"nref":66,"oast":"green","cx":"Grant","co":["Bias","Validity","Interventions"],"pco":"Bias","bs":["ai-llm"],"pbs":"ai-llm","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"6","cc":4,"cp":2,"self":false,"ncoefs":0},{"id":"J92QMRFW","t":"A manifesto for fair and equitable research funding in ecology","au":"Winder, Linton","y":2017,"vn":"Rethinking Ecology","ty":"journalArticle","tyl":"Journal article","doi":"10.3897/rethinkingecology.2.21798","cby":1,"nref":38,"oast":"gold","cx":"Grant","co":["Reliability-noise","Bias","Validity","Interventions"],"pco":"Reliability-noise","bs":["demographic","prestige-halo"],"pbs":"demographic","rs":[],"sg":["funding-decision"],"st":"simulation-methodological","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"JAKKEBE4","t":"ASSESSMENT OF SUPPORT OF COMPETITIVE SELECTION PROCESSES FOR FUNDING OF SCHOLARLY RESEARCH (R&D) PROJECTS PROVIDED BY THE NATIONAL ELECTRONIC SCIENTIFIC INFORMATION SYSTEM","au":"Жарінова, Алла","y":2026,"vn":"Science and innovation","ty":"journalArticle","tyl":"Journal article","doi":"10.15407/scine22.03.023","cby":0,"nref":9,"oast":"diamond","cx":"Grant","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"JBIBRH23","t":"Features of successful bids for funding of applied health research: a cohort study","au":"Turner, Sheila","y":2014,"vn":"Health Research Policy and Systems","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/1478-4505-12-54","cby":7,"nref":39,"oast":"gold","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["proposal-quality","funding-decision"],"st":"observational-quantitative","tp":"7","cc":1,"cp":5,"self":false,"ncoefs":0},{"id":"JCNY992P","t":"Panel discussion does not improve reliability of peer review for medical research grant proposals.","au":"Fogelholm, Mikael","y":2012,"vn":"Journal of Clinical Epidemiology","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.jclinepi.2011.05.001","cby":87,"nref":9,"oast":"closed","cx":"Grant","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring","panel-consensus"],"st":"experimental-rct","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":4},{"id":"JD3VGE3T","t":"Bias and Groupthink in Science’s Peer-Review System","au":"Resnik, David B.","y":2020,"vn":"Groupthink in science: greed, pathological altruism, ideology, competition, and culture","ty":"bookSection","tyl":"Book chapter","doi":"10.1007/978-3-030-36822-7_9","cby":25,"nref":78,"oast":"closed","cx":"General","co":["Bias","Interventions"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["individual-scoring"],"st":"book-monograph","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"JEMVX4GK","t":"Dear Reviewer 2: Go F’ Yourself","au":"Peterson, David A.","y":2020,"vn":"Social Science Quarterly","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/ssqu.12824","cby":50,"nref":6,"oast":"closed","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"3","cc":5,"cp":1,"self":false,"ncoefs":0},{"id":"JEXNP37Z","t":"The need for a research agenda.","au":"Bailar, J C","y":1985,"vn":"The New England Journal of Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.1056/NEJM198503073121023","cby":163,"nref":20,"oast":"closed","cx":"Journal","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"JFCN34TN","t":"NIH Peer Review of Grant Applications for Clinical Research","au":"Kotchen, Theodore A.","y":2004,"vn":"JAMA","ty":"journalArticle","tyl":"Journal article","doi":"10.1001/jama.291.7.836","cby":104,"nref":25,"oast":"closed","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["funding-decision"],"st":"observational-quantitative","tp":"7","cc":4,"cp":0,"self":false,"ncoefs":0},{"id":"JGLEEC55","t":"An experimental test of the effects of redacting grant applicant identifiers on peer review outcomes","au":"Nakamura, Richard","y":2021,"vn":"eLife","ty":"journalArticle","tyl":"Journal article","doi":"10.7554/eLife.71368","cby":38,"nref":47,"oast":"gold","cx":"Grant","co":["Bias","Interventions"],"pco":"Bias","bs":["demographic","prestige-halo"],"pbs":"demographic","rs":[],"sg":["individual-scoring"],"st":"experimental-rct","tp":"14","cc":4,"cp":5,"self":false,"ncoefs":0},{"id":"JGPW43LL","t":"Erosion of peer review?","au":"van der Linden, Wim","y":2014,"vn":"Educational Measurement: Issues and Practice","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/emip.12044","cby":0,"nref":2,"oast":"closed","cx":"Journal","co":["Validity","Interventions"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"-1","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"JHIX7RTB","t":"Does single blind peer review hinder newcomers?","au":"Seeber, Marco","y":2017,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-017-2264-7","cby":64,"nref":81,"oast":"hybrid","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["prestige-halo"],"pbs":"prestige-halo","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"10","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"JITSB8WY","t":"Evolution of Peer Review in Scientific Communication","au":"Kochetkov, Dmitry","y":2024,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31235/osf.io/b2ra3_v1","cby":0,"nref":0,"oast":"gold","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"JIUWDBMD","t":"An end to ‘God-like’ scientific knowledge? How non-anonymous referees and open review alter meanings for scientific knowledge","au":"Gaudet, Joanne","y":2017,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31219/osf.io/6ypuw","cby":0,"nref":42,"oast":"gold","cx":"Journal","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"JJYTBKG4","t":"Assessing health research grant applications: A retrospective comparative review of a one-stage versus a two-stage application assessment process.","au":"Morgan, Ben","y":2020,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0230118","cby":5,"nref":9,"oast":"gold","cx":"Grant","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"7","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"JMK4CLUJ","t":"Conditions that do or do not disadvantage interdisciplinary research proposals in project evaluation","au":"Seeber, Marco","y":2022,"vn":"Journal of the Association for Information Science and Technology","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/asi.24617","cby":26,"nref":63,"oast":"closed","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"9","cc":4,"cp":2,"self":false,"ncoefs":0},{"id":"JPKCD6M8","t":"Statistical notes for clinical researchers: Evaluation of measurement error 2: Dahlberg's error, Bland-Altman method, and Kappa coefficient","au":"Kim, Hae‐Yeong","y":2013,"vn":"Restorative Dentistry & Endodontics","ty":"journalArticle","tyl":"Journal article","doi":"10.5395/rde.2013.38.3.182","cby":87,"nref":6,"oast":"diamond","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"4","cc":5,"cp":0,"self":false,"ncoefs":0},{"id":"JRSMWBU4","t":"Peer review and panel decisions in the assessment of Australian Research Council project grant applicants: what counts in a highly competitive context?","au":"Bazeley, Pat","y":1998,"vn":"Higher Education","ty":"journalArticle","tyl":"Journal article","doi":"10.1023/a:1003118502318","cby":38,"nref":27,"oast":"closed","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"8","cc":4,"cp":4,"self":false,"ncoefs":0},{"id":"JTR3R42M","t":"Bayesian Nonparametric Models for Multiple Raters: A General Statistical Framework","au":"Mignemi, Giuseppe","y":2025,"vn":"Psychometrika","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/psy.2025.10035","cby":2,"nref":119,"oast":"hybrid","cx":"Analogue","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":["level","factors"],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"4","cc":6,"cp":2,"self":false,"ncoefs":0},{"id":"JU4PBD6P","t":"Policy Considerations for Random Allocation of Research Funds","au":"Avin, Shahar","y":2017,"vn":"PhilSci-Archive (University of Pittsburgh)","ty":"journalArticle","tyl":"Journal article","doi":"10.13130/2282-5398/8626","cby":27,"nref":37,"oast":"green","cx":"Grant","co":["Reliability-noise","Validity","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["funding-decision"],"st":"conceptual-commentary","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"JX8TSFKY","t":"Chance and interrater agreement on manuscripts.","au":"Watkins, Marley W.","y":1979,"vn":"American Psychologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/0003-066x.34.9.796","cby":115,"nref":7,"oast":"closed","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"4","cc":5,"cp":3,"self":false,"ncoefs":3},{"id":"JXSMF6RN","t":"Gender differences in peer reviewed grant applications, awards, and amounts: a systematic review and meta-analysis","au":"Schmaling, Karen B.","y":2023,"vn":"Research Integrity and Peer Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s41073-023-00127-3","cby":70,"nref":80,"oast":"gold","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["funding-decision"],"st":"systematic-review-meta-analysis","tp":"2","cc":4,"cp":5,"self":false,"ncoefs":0},{"id":"JXTFVYUU","t":"Funding Science: The Real Defects of Peer Review and An Alternative To It","au":"Roy, Rustum","y":1985,"vn":"Science Technology & Human Values","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/016224398501000309","cby":141,"nref":12,"oast":"closed","cx":"Grant","co":["Validity","Interventions"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"1","cc":1,"cp":0,"self":false,"ncoefs":0},{"id":"JZW58HZM","t":"Sunlight not shadows: Double-anonymized peer review is not the progressive answer to status bias","au":"Horbach, Serge P. J. M.","y":2022,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31219/osf.io/fqb5c","cby":4,"nref":7,"oast":"gold","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["prestige-halo"],"pbs":"prestige-halo","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"K3DZVZXI","t":"A New Way to Distribute Research Seed Funding: Peer Review Without Formal Review","au":"Burns, Mark A.","y":2021,"vn":"Change The Magazine of Higher Learning","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/00091383.2021.1883976","cby":0,"nref":8,"oast":"closed","cx":"Grant","co":["Reliability-noise","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["funding-decision"],"st":"observational-quantitative","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"K4EVXX88","t":"Retractions and Rewards in Science: An Open Question for Reviewers and Funders","au":"Ribeiro, Mariana Dias","y":2023,"vn":"Science and Engineering Ethics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11948-023-00446-0","cby":6,"nref":25,"oast":"closed","cx":"Grant","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"1","cc":6,"cp":1,"self":false,"ncoefs":0},{"id":"K5N2TIMX","t":"ChatGPT identifies gender disparities in scientific peer review","au":"Verharen, Jeroen P. H.","y":2023,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.7554/elife.90230.2","cby":1,"nref":25,"oast":"gold","cx":"Journal","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["ai-llm","demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"6","cc":2,"cp":1,"self":false,"ncoefs":1},{"id":"K64P7MNV","t":"Potential bias in peer review of grant applications at the Swiss National Science Foundation","au":"Severin, Anna","y":2019,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.7287/peerj.preprints.27587v1","cby":3,"nref":0,"oast":"gold","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["demographic","prestige-halo"],"pbs":"demographic","rs":[],"sg":["individual-scoring","reviewer-selection"],"st":"observational-quantitative","tp":"2","cc":6,"cp":0,"self":false,"ncoefs":0},{"id":"KARAZVW9","t":"Assessment of potential bias in research grant peer review in Canada.","au":"Tamblyn, Robyn","y":2018,"vn":"Canadian Medical Association Journal","ty":"journalArticle","tyl":"Journal article","doi":"10.1503/cmaj.170901","cby":156,"nref":50,"oast":"gold","cx":"Grant","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["demographic"],"pbs":"demographic","rs":["level","factors"],"sg":["reviewer-selection","individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"2","cc":4,"cp":2,"self":false,"ncoefs":6},{"id":"KC5HMJS8","t":"A scoping review on biomedical journal peer review guides for reviewers","au":"Song, Eunhye","y":2021,"vn":"PLoS ONE","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0251440","cby":40,"nref":98,"oast":"gold","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"systematic-review-meta-analysis","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"KFAYWFPV","t":"Does the committee peer review select the best applicants for funding? An investigation of the selection process for two European molecular biology organization programmes.","au":"Bornmann, Lutz","y":2008,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0003480","cby":116,"nref":62,"oast":"gold","cx":"Grant","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["post-award-validity","funding-decision"],"st":"observational-quantitative","tp":"2","cc":1,"cp":4,"self":false,"ncoefs":0},{"id":"KIXZ6B2J","t":"Grant Research Support in Russia: What Can We Learn from the Russian Science Foundation’s First Grant Competition?","au":"Simachev, Yuri","y":2017,"vn":"Foresight-Russia","ty":"journalArticle","tyl":"Journal article","doi":"10.17323/2500-2597.2017.4.74.83","cby":5,"nref":12,"oast":"green","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["prestige-halo"],"pbs":"prestige-halo","rs":[],"sg":["funding-decision"],"st":"observational-quantitative","tp":"-1","cc":4,"cp":2,"self":false,"ncoefs":0},{"id":"KMEYIQ9Y","t":"Does Masking Author Identity Improve Peer Review Quality?","au":"Justice, Amy C.","y":1998,"vn":"The Journal of the American Medical Association","ty":"journalArticle","tyl":"Journal article","doi":"10.1001/jama.280.3.240","cby":280,"nref":6,"oast":"closed","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["prestige-halo"],"pbs":"prestige-halo","rs":[],"sg":["individual-scoring"],"st":"experimental-rct","tp":"10","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"KNJE6VS9","t":"A Bayesian approach to ranking and rater evaluation: An application to grant reviews.","au":"Cao, Jing","y":2010,"vn":"Journal of Educational and Behavioral Statistics","ty":"journalArticle","tyl":"Journal article","doi":"10.3102/1076998609353116","cby":16,"nref":36,"oast":"closed","cx":"Grant","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":["factors"],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"8","cc":1,"cp":8,"self":false,"ncoefs":0},{"id":"KPLP8QZE","t":"Gender Bias Impacts Top-Merited Candidates.","au":"Andersson, Emma Rachel","y":2021,"vn":"Frontiers in research metrics and analytics","ty":"journalArticle","tyl":"Journal article","doi":"10.3389/frma.2021.594424","cby":20,"nref":23,"oast":"gold","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"2","cc":4,"cp":5,"self":false,"ncoefs":0},{"id":"KPSCZ7V2","t":"Intraclass correlations: Uses in assessing rater reliability.","au":"Shrout, Patrick E.","y":1979,"vn":"Psychological Bulletin","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/0033-2909.86.2.420","cby":23060,"nref":20,"oast":"closed","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"4","cc":6,"cp":3,"self":false,"ncoefs":0},{"id":"KRNLC6TD","t":"A multi-disciplinary perspective on emergent and future innovations in peer review","au":"Tennant, Jonathan","y":2017,"vn":"F1000Research","ty":"journalArticle","tyl":"Journal article","doi":"10.12688/f1000research.12037.2","cby":40,"nref":277,"oast":"gold","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"KRP5YXFM","t":"Where do field-specific notions of research quality come from?","au":"Laudel, Grit","y":2024,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvae027","cby":7,"nref":36,"oast":"hybrid","cx":"Journal","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"qualitative","tp":"9","cc":3,"cp":2,"self":false,"ncoefs":0},{"id":"KVALUL6S","t":"The Necessity of Commensuration Bias in Grant Peer Review","au":"Heesen, Remco","y":2022,"vn":"Ergo an Open Access Journal of Philosophy","ty":"journalArticle","tyl":"Journal article","doi":"10.3998/ergo.2240","cby":4,"nref":38,"oast":"diamond","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"8","cc":3,"cp":2,"self":false,"ncoefs":0},{"id":"KWATA3RR","t":"A new reader trial approach to peer review in funding research grants: An Australian experiment","au":"Jayasinghe, Upali W.","y":2006,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-006-0171-4","cby":38,"nref":36,"oast":"closed","cx":"Grant","co":["Reliability-noise","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"8","cc":4,"cp":4,"self":false,"ncoefs":4},{"id":"KXATMKHW","t":"Impact of interventions to improve the quality of peer review of biomedical journals: a systematic review and meta-analysis.","au":"Bruce, Rachel","y":2016,"vn":"BMC Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s12916-016-0631-5","cby":230,"nref":57,"oast":"gold","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"systematic-review-meta-analysis","tp":"11","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"KXCEL6DG","t":"Does Peer Review Identify the Best Papers? A Simulation Study of Editors, Reviewers, and the Scientific Publication Process","au":"Esarey, Justin","y":2017,"vn":"PS Political Science & Politics","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/s1049096517001081","cby":23,"nref":34,"oast":"bronze","cx":"Journal","co":["Reliability-noise","Validity"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"simulation-methodological","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"KXEIIMC8","t":"A survey of how biology researchers assess credibility when serving on grant and hiring committees","au":"Hrynaszkiewicz I.","y":2026,"vn":"PeerJ","ty":"journalArticle","tyl":"Journal article","doi":"10.7717/peerj.20502","cby":0,"nref":30,"oast":"gold","cx":"General","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["reviewer-selection","individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":4,"cp":2,"self":false,"ncoefs":0},{"id":"KY6KDPFD","t":"The peer review process for awarding funds to international science research consortia: a qualitative developmental evaluation","au":"Gregorius, Stefanie","y":2018,"vn":"F1000Research","ty":"journalArticle","tyl":"Journal article","doi":"10.12688/f1000research.12496.3","cby":9,"nref":24,"oast":"gold","cx":"Grant","co":["Reliability-noise","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["panel-consensus"],"st":"qualitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"L32ZULYV","t":"Enhancing Consistency in Peer Review: A Statistical Analysis of Discrepancies and Proposals for Improvement","au":"Alarfaj, Maher Mohammed","y":2025,"vn":"Learned Publishing","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/leap.2033","cby":1,"nref":13,"oast":"hybrid","cx":"General","co":["Reliability-noise","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"L3X5I6CT","t":"Evaluating high risk research: an assessment of the Wellcome Trust's Sir Henry Wellcome Commemorative Awards for Innovative Research","au":"Grant, Julius","y":1999,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.3152/147154499781777513","cby":19,"nref":0,"oast":"closed","cx":"Grant","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"experimental-rct","tp":"-1","cc":4,"cp":0,"self":false,"ncoefs":0},{"id":"L6CC3BZQ","t":"How pre-publication journal peer review (re)produces ignorance at scientific and medical journals: a case study","au":"Gaudet, Joanne","y":2017,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31219/osf.io/3dq5j","cby":3,"nref":128,"oast":"gold","cx":"Journal","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"qualitative","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"L6N4AXR4","t":"“It takes a village to write a really good paper”: A normative framework for peer reviewing in philosophy","au":"Copeland, Samantha","y":2024,"vn":"Metaphilosophy","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/meta.12670","cby":3,"nref":24,"oast":"hybrid","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"L6S33TPR","t":"Imperfect referees: Reducing the impact of multiple biases in peer review","au":"Yun, Zhao","y":2015,"vn":"Journal of the Association for Information Science and Technology","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/asi.23317","cby":8,"nref":46,"oast":"closed","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["prestige-halo","cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"-1","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"L7CAZX6J","t":"The practicalities of a partial lottery to allocate research funding","au":"Meer, Martijn van der","y":2024,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvae023","cby":2,"nref":32,"oast":"hybrid","cx":"Grant","co":["Validity","Interventions"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["funding-decision"],"st":"conceptual-commentary","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"L7XCUZU9","t":"Effect of structured workshop training on subsequent performance of journal peer reviewers.","au":"Callaham, Michael L","y":2002,"vn":"Annals of Emergency Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.1067/mem.2002.127121","cby":52,"nref":12,"oast":"bronze","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"experimental-rct","tp":"-1","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"LAHV4J24","t":"Strengthening Community Involvement in Grant Review: Insights from the Community–University Research Partnership (CURES) Pilot Review Process","au":"Paberzs, Adam","y":2014,"vn":"Clinical and Translational Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/cts.12141","cby":16,"nref":20,"oast":"green","cx":"Grant","co":["Reliability-noise","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["panel-consensus","reviewer-selection"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"LAPY8Q5J","t":"Improving editorial procedures.","au":"Crandall, Rick","y":1990,"vn":"American Psychologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/0003-066x.45.5.665","cby":38,"nref":11,"oast":"closed","cx":"Journal","co":["Reliability-noise","Bias","Interventions"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"3","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"LB5K9AI7","t":"Networking and knowledge exchange to promote the formation of transdisciplinary coalitions and levels of agreement among transdisciplinary peer reviewers.","au":"Lobb, Rebecca","y":2013,"vn":"Journal of public health management and practice : JPHMP","ty":"journalArticle","tyl":"Journal article","doi":"10.1097/PHH.0b013e31823991c2","cby":13,"nref":25,"oast":"closed","cx":"Grant","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"7","cc":1,"cp":4,"self":false,"ncoefs":8},{"id":"LCGWFM9G","t":"Peer review: the attitudes and behaviours of Covid-19 pandemic-era early career researchers","au":"Nicholas, David","y":2023,"vn":"El Profesional de la Informacion","ty":"journalArticle","tyl":"Journal article","doi":"10.3145/epi.2023.may.06","cby":9,"nref":79,"oast":"diamond","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"-1","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"LDQ9B77W","t":"The relationship of previous training and experience of journal peer reviewers to subsequent review quality.","au":"Callaham, Michael L","y":2007,"vn":"PLoS Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pmed.0040040","cby":140,"nref":16,"oast":"gold","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"LEGPWSWB","t":"The validity of peer review in a general medicine journal.","au":"Jackson, Jeffrey L","y":2011,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0022475","cby":74,"nref":18,"oast":"gold","cx":"Journal","co":["Reliability-noise","Validity"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level","factors","value"],"sg":["individual-scoring","funding-decision","post-award-validity"],"st":"observational-quantitative","tp":"3","cc":3,"cp":1,"self":false,"ncoefs":4},{"id":"LFCNU42R","t":"Peer review for improving the quality of grant applications.","au":"Demicheli, V","y":2007,"vn":"Cochrane Database of Systematic Reviews","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/14651858.MR000003.pub2","cby":100,"nref":37,"oast":"green","cx":"Grant","co":["Reliability-noise","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["whole-process"],"st":"systematic-review-meta-analysis","tp":"8","cc":1,"cp":4,"self":false,"ncoefs":0},{"id":"LFVVACBY","t":"Reviewer ratings and agreement on manuscripts reviewed for the Journal of Counseling Psychology..","au":"Munley, Patrick H.","y":1988,"vn":"Journal of Counseling Psychology","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/0022-0167.35.2.198","cby":17,"nref":13,"oast":"closed","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"3","cc":5,"cp":3,"self":false,"ncoefs":3},{"id":"LHCCRG7C","t":"Are gender gaps due to evaluations of the applicant or the science? A natural experiment at a national funding agency.","au":"Witteman, Holly O","y":2019,"vn":"The Lancet","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/S0140-6736(18)32611-4","cby":561,"nref":70,"oast":"closed","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"2","cc":4,"cp":5,"self":false,"ncoefs":0},{"id":"LHXMSP4I","t":"Perspective: Is NIH Funding the “Best Science by the Best Scientists”? A Critique of the NIH R01 Research Grant Review Policies","au":"Costello, Leslie C.","y":2010,"vn":"Academic Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.1097/acm.0b013e3181d74256","cby":33,"nref":3,"oast":"bronze","cx":"Grant","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":0,"self":false,"ncoefs":0},{"id":"LIB9SBY5","t":"Is Peer Review a Good Idea?","au":"Heesen, Remco","y":2019,"vn":"The British Journal for the Philosophy of Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/bjps/axz029","cby":129,"nref":87,"oast":"hybrid","cx":"Journal","co":["Validity","Interventions"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"-1","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"LJCKBTRE","t":"What is submitted and what gets accepted in Indian Pediatrics: Analysis of submissions, review process, decision making, and criteria for rejection","au":"Gupta, Piyush","y":2013,"vn":"Indian Pediatrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s13312-013-0001-8","cby":34,"nref":14,"oast":"closed","cx":"Journal","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"3","cc":3,"cp":1,"self":false,"ncoefs":3},{"id":"LKWH7CPF","t":"Corrigendum: Prestigious Science Journals Struggle to Reach Even Average Reliability (Frontiers in Human Neuroscience, (2018), 12, 10.3389/fnhum.2018.00037)","au":"Brembs B.","y":2018,"vn":"Frontiers in Human Neuroscience","ty":"journalArticle","tyl":"Journal article","doi":"10.3389/fnhum.2018.00376","cby":0,"nref":0,"oast":"closed","cx":"Journal","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"-1","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"LLMTWE7V","t":"Partial and particularistic: grant peer review panels evaluating interdisciplinarity","au":"Oxley, Kristin","y":2024,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvaf037","cby":1,"nref":49,"oast":"hybrid","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["panel-consensus"],"st":"qualitative","tp":"13","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"LLTEQ7U5","t":"Peer Review, Program Officers and Science Funding","au":"Roebber, Paul J.","y":2011,"vn":"PLoS ONE","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0018680","cby":35,"nref":15,"oast":"gold","cx":"Grant","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"simulation-methodological","tp":"1","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"LNLZUDIW","t":"Gender differences in Dutch research funding over time: A statistical investigation of the innovation scheme 2012-2021.","au":"Albers, Casper","y":2024,"vn":"PloS one","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0297311","cby":7,"nref":24,"oast":"gold","cx":"Grant","co":["Bias","Interventions"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["funding-decision"],"st":"observational-quantitative","tp":"2","cc":4,"cp":5,"self":false,"ncoefs":0},{"id":"LP9VFUWK","t":"Frequency and Type of Conflicts of Interest in the Peer Review of Basic Biomedical Research Funding Applications: Self-Reporting Versus Manual Detection","au":"Gallo, Stephen A.","y":2015,"vn":"Science and Engineering Ethics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11948-015-9631-7","cby":7,"nref":19,"oast":"green","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["reviewer-selection"],"st":"observational-quantitative","tp":"0","cc":1,"cp":4,"self":false,"ncoefs":0},{"id":"LPDZSVPC","t":"First Comprehensive Survey of NSF Applicants Focuses on Their Concerns About Proposal Review","au":"McCullough, Jim","y":1989,"vn":"Science Technology & Human Values","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/016224398901400107","cby":38,"nref":0,"oast":"closed","cx":"Grant","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":0,"self":false,"ncoefs":0},{"id":"LR2JK33N","t":"Back to the roots: reimagining scientific evaluation of research without peer review","au":"Sallam, Malik","y":2026,"vn":"Insights the UKSG journal","ty":"journalArticle","tyl":"Journal article","doi":"10.1629/uksg.714","cby":0,"nref":31,"oast":"diamond","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"LVKA3ZWD","t":"Individual versus general structured feedback to improve agreement in grant peer review: a randomized controlled trial","au":"Hesselberg, Jan-Ole","y":2021,"vn":"Research Integrity and Peer Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s41073-021-00115-5","cby":8,"nref":17,"oast":"gold","cx":"Grant","co":["Reliability-noise","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level","factors"],"sg":["individual-scoring"],"st":"experimental-rct","tp":"-1","cc":1,"cp":2,"self":true,"ncoefs":7},{"id":"LW6BBXR7","t":"Effect of Blinding and Unmasking on the Quality of Peer Review","au":"Rooyen, Susan van","y":1998,"vn":"JAMA","ty":"journalArticle","tyl":"Journal article","doi":"10.1001/jama.280.3.234","cby":288,"nref":20,"oast":"bronze","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"experimental-rct","tp":"10","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"LYM4S2ML","t":"Positive bias for European men in peer reviewed applications for faculty position at Karolinska Institutet","au":"Hagg S.","y":2018,"vn":"F1000Research","ty":"journalArticle","tyl":"Journal article","doi":"10.12688/f1000research.13030.2","cby":5,"nref":13,"oast":"gold","cx":"General","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"2","cc":4,"cp":5,"self":false,"ncoefs":0},{"id":"LZBRQ7CQ","t":"PREPRINT: Open science and modified funding lotteries can impede the natural selection of bad science","au":"Smaldino","y":2019,"vn":"Open Science Framework","ty":"journalArticle","tyl":"Journal article","doi":"10.1098/rsos.190194","cby":55,"nref":83,"oast":"gold","cx":"General","co":["Validity","Interventions"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["funding-decision"],"st":"simulation-methodological","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"LZDVVTHX","t":"Using Peer Review to Improve Research and Promote Collaboration","au":"Kupfer, David J.","y":2014,"vn":"Academic Psychiatry","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s40596-013-0027-1","cby":9,"nref":2,"oast":"green","cx":"Grant","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["panel-consensus"],"st":"observational-quantitative","tp":"7","cc":2,"cp":0,"self":false,"ncoefs":0},{"id":"LZLZ77Q3","t":"Changing peer review practices: transforming roles and future challenges","au":"Horbach, Serge P. J. M.","y":2025,"vn":"Acta Orthopaedica","ty":"journalArticle","tyl":"Journal article","doi":"10.2340/17453674.2025.44353","cby":2,"nref":25,"oast":"gold","cx":"Journal","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"M4FB9M3J","t":"Use of check lists in assessing the statistical content of medical studies.","au":"Gardner, M J","y":1986,"vn":"BMJ","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmj.292.6523.810","cby":141,"nref":27,"oast":"bronze","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":2,"cp":0,"self":false,"ncoefs":0},{"id":"M4FE5YNE","t":"Involving clinical experts in prioritising topics for health technology assessment: a randomised controlled trial","au":"Cook, Andrew","y":2017,"vn":"BMJ Open","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmjopen-2017-016104","cby":1,"nref":24,"oast":"gold","cx":"Grant","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"experimental-rct","tp":"11","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"M4I4S2CC","t":"Reimagining peer review: the emergence of peer community in registered reports system.","au":"Zoccali, Carmine","y":2023,"vn":"Journal of nephrology","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s40620-023-01709-6","cby":10,"nref":0,"oast":"closed","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"M653B5D9","t":"Cognitive bias in peer review: A new approach","au":"Ström","y":2009,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"","cby":10,"nref":20,"oast":"closed","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":4,"cp":4,"self":false,"ncoefs":0},{"id":"M72NE68B","t":"Evaluating science: A comparison of human and AI reviewers","au":"Shcherbiak, Anna","y":2024,"vn":"Judgment and Decision Making","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/jdm.2024.24","cby":11,"nref":31,"oast":"diamond","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"6","cc":2,"cp":1,"self":false,"ncoefs":4},{"id":"M8NNE35R","t":"The Costs of Competition in Distributing Scarce Research Funds","au":"Schweiger, Gerald","y":2024,"vn":"PNAS","ty":"journalArticle","tyl":"Journal article","doi":"10.1073/pnas.2407644121","cby":18,"nref":109,"oast":"hybrid","cx":"Grant","co":["Reliability-noise","Bias","Validity"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"MAPYT9ZR","t":"The effectiveness of the peer review process: inter-referee agreement and predictive validity of manuscript refereeing at Angewandte Chemie.","au":"Bornmann, Lutz","y":2008,"vn":"Angewandte Chemie International Edition","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/anie.200800513","cby":69,"nref":44,"oast":"closed","cx":"Journal","co":["Reliability-noise","Validity"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring","post-award-validity"],"st":"observational-quantitative","tp":"-1","cc":3,"cp":4,"self":false,"ncoefs":3},{"id":"MBW2C84N","t":"An assessment of selection processes among candidates for public research grants: the case of the Ramón y Cajal Programme in Spain","au":"Cañibano, Carolina","y":2009,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.3152/095820209x444968","cby":15,"nref":0,"oast":"closed","cx":"Grant","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["funding-decision"],"st":"observational-quantitative","tp":"2","cc":4,"cp":0,"self":false,"ncoefs":0},{"id":"MC7H39UQ","t":"The Decision-Making Constraints and Processes of Grant Peer Review, and Their Effects on the Review Outcome","au":"Langfeldt, Liv","y":2001,"vn":"Social studies of science","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/030631201031006002","cby":206,"nref":4,"oast":"green","cx":"Grant","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["factors"],"sg":["individual-scoring","panel-consensus","funding-decision"],"st":"qualitative","tp":"8","cc":1,"cp":1,"self":false,"ncoefs":0},{"id":"MFK36ARS","t":"Selecting manuscripts for a high‐impact journal through peer review: A citation analysis of communications that were accepted by Angewandte Chemie International Edition, or rejected but published elsewhere","au":"Bornmann, Lutz","y":2008,"vn":"Journal of the American Society for Information Science and Technology","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/asi.20901","cby":69,"nref":50,"oast":"green","cx":"Journal","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["funding-decision","post-award-validity"],"st":"observational-quantitative","tp":"-1","cc":3,"cp":4,"self":false,"ncoefs":0},{"id":"MGSKAUXW","t":"No evidence that gender contributes to personal research funding success in The Netherlands: A reaction to van der Lee and Ellemers","au":"Völker, Beate","y":2015,"vn":"Proceedings of the National Academy of Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1073/pnas.1519046112","cby":36,"nref":2,"oast":"green","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["funding-decision"],"st":"observational-quantitative","tp":"2","cc":4,"cp":0,"self":false,"ncoefs":0},{"id":"MHK2CREE","t":"Comparing ChatGPT and human expertise: exploring new avenues in peer review of single-case experimental research","au":"Rakap, Salih","y":2025,"vn":"European Journal of Special Needs Education","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/08856257.2025.2533555","cby":0,"nref":33,"oast":"closed","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"6","cc":2,"cp":1,"self":false,"ncoefs":1},{"id":"MIAXJ264","t":"Scientific peer review","au":"Bornmann, Lutz","y":2011,"vn":"Annual Review of Information Science and Technology","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/aris.2011.1440450112","cby":566,"nref":266,"oast":"closed","cx":"General","co":["Reliability-noise","Bias","Validity"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"systematic-review-meta-analysis","tp":"0","cc":3,"cp":4,"self":false,"ncoefs":0},{"id":"MJUP8VVE","t":"Peer Review at the Crossroads","au":"Kochetkov, Dmitry","y":2025,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31235/osf.io/b2ra3_v3","cby":0,"nref":0,"oast":"gold","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"MLXXE7WL","t":"Improving peer review of systematic reviews and related review types by involving librarians and information specialists as methodological peer reviewers: a randomized controlled trial","au":"Rethlefsen, Melissa L.","y":2024,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31222/osf.io/7hu3v","cby":1,"nref":0,"oast":"closed","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["reviewer-selection","individual-scoring"],"st":"experimental-rct","tp":"11","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"MM2N6SHA","t":"Opening Pandora’s box: Developing reviewer rhetorical sensitivity through retracted articles","au":"Khuder, Baraa","y":2025,"vn":"Accountability in Research","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/08989621.2025.2607681","cby":0,"nref":23,"oast":"hybrid","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"qualitative","tp":"0","cc":2,"cp":0,"self":false,"ncoefs":0},{"id":"MM5CTRWM","t":"Editorial peer reviewers' recommendations at a general medical journal: are they reliable and do editors care?","au":"Kravitz, Richard L","y":2010,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0010072","cby":176,"nref":29,"oast":"gold","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"3","cc":3,"cp":1,"self":false,"ncoefs":3},{"id":"MMDJHRAP","t":"Distrust in grant peer review—reasons and remedies","au":"Langfeldt, Liv","y":2023,"vn":"Science and Public Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/scipol/scad051","cby":14,"nref":59,"oast":"hybrid","cx":"Grant","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"1","cc":4,"cp":2,"self":false,"ncoefs":0},{"id":"MQ4IPI6N","t":"Surveys of current status in biomedical science grant review: funding organisations' and grant reviewers' perspectives.","au":"Schroter, Sara","y":2010,"vn":"BMC Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/1741-7015-8-62","cby":48,"nref":11,"oast":"gold","cx":"Grant","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"8","cc":1,"cp":4,"self":false,"ncoefs":0},{"id":"MRNKJKY2","t":"Peer review in an Era of Evaluation","au":"Forsberg, Eva","y":2022,"vn":"","ty":"book","tyl":"Book","doi":"10.1007/978-3-030-75263-7","cby":26,"nref":488,"oast":"gold","cx":"Grant","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"book-monograph","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"MTAZBL3E","t":"Large-scale language analysis of peer review reports","au":"Buljan, Ivan","y":2020,"vn":"eLife","ty":"journalArticle","tyl":"Journal article","doi":"10.7554/elife.53249","cby":35,"nref":33,"oast":"gold","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"6","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"MTIHLHV3","t":"Is the expertise of evaluation panels congruent with the research interests of the research groups: A quantitative approach based on barycenters","au":"Rahman, A.I.M. Jakaria","y":2015,"vn":"Journal of Informetrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.joi.2015.07.009","cby":11,"nref":40,"oast":"closed","cx":"General","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["reviewer-selection"],"st":"simulation-methodological","tp":"9","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"MTLGNRD3","t":"Flipping the grant application review process","au":"Dinov, Ivo D.","y":2019,"vn":"Studies in Higher Education","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/03075079.2019.1628201","cby":3,"nref":31,"oast":"green","cx":"Grant","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"MVK2CRH7","t":"Inference About Weighted Kappa in the Non-Null Case","au":"Fleiss, Joseph L.","y":1978,"vn":"Applied Psychological Measurement","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/014662167800200111","cby":142,"nref":6,"oast":"green","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level","factors"],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"4","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"MVKIF9TD","t":"Gender bias in scholarly peer review.","au":"Helmer, Markus","y":2017,"vn":"eLife","ty":"journalArticle","tyl":"Journal article","doi":"10.7554/eLife.21718","cby":380,"nref":106,"oast":"gold","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["reviewer-selection","individual-scoring"],"st":"observational-quantitative","tp":"5","cc":2,"cp":5,"self":false,"ncoefs":0},{"id":"MY2NAXXH","t":"Why can't we make research grant allocation systems more consistent? A personal opinion","au":"Cousens, Roger","y":2019,"vn":"Ecology and Evolution","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/ece3.4855","cby":2,"nref":44,"oast":"gold","cx":"Grant","co":["Reliability-noise","Validity"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"N2LFYALW","t":"Emerging new methods of peer review in scholarly journals","au":"Björk, Bo‐Christer","y":2015,"vn":"Learned Publishing","ty":"journalArticle","tyl":"Journal article","doi":"10.1087/20150202","cby":24,"nref":2,"oast":"bronze","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":3,"cp":0,"self":false,"ncoefs":0},{"id":"N4W7NGXC","t":"Selecting competent referees to assess research projects proposals: A study of referees' registers","au":"Abramo, Giovanni","y":2012,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvs031","cby":6,"nref":12,"oast":"closed","cx":"Grant","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["reviewer-selection"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":4,"self":false,"ncoefs":0},{"id":"N4Z5ARMM","t":"For what it’s worth – the open peer review landscape","au":"Tattersall, Andy","y":2015,"vn":"Online Information Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1108/oir-06-2015-0182","cby":37,"nref":24,"oast":"closed","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"N5TQTHKV","t":"A radical change in peer review","au":"Mervis, Jeffrey","y":2014,"vn":"Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1126/science.345.6194.248","cby":10,"nref":0,"oast":"closed","cx":"Grant","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"-1","cc":1,"cp":0,"self":false,"ncoefs":0},{"id":"N8M2AUCU","t":"Peer Review Interrater Reliability of Scientific Abstracts: A Study of an Anesthesia Subspecialty Society","au":"Cohen, Ira Todd","y":2005,"vn":"Journal of Education in Perioperative Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.46374/volvii-issue2-cohen2","cby":2,"nref":22,"oast":"bronze","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level","value"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"15","cc":5,"cp":6,"self":false,"ncoefs":2},{"id":"N8W8PGRJ","t":"Percentile Ranking and Citation Impact of a Large Cohort of National Heart, Lung, and Blood Institute–Funded Cardiovascular R01 Grants","au":"Danthi, Narasimhan","y":2014,"vn":"Circulation Research","ty":"journalArticle","tyl":"Journal article","doi":"10.1161/circresaha.114.302656","cby":84,"nref":20,"oast":"bronze","cx":"Grant","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["post-award-validity"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"N99QJWN2","t":"Peer review: Risk and risk tolerance","au":"Gallo, Stephen A.","y":2022,"vn":"PLOS ONE","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0273813","cby":6,"nref":35,"oast":"gold","cx":"Grant","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"experimental-rct","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":1},{"id":"NAZXFWMK","t":"A statistical approach to calibrating the scores of biased reviewers of scientific papers","au":"Kuhlisch, Wiltrud","y":2015,"vn":"Metrika","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s00184-015-0542-z","cby":3,"nref":35,"oast":"closed","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"3","cc":4,"cp":4,"self":false,"ncoefs":0},{"id":"NB4IK4KH","t":"Design and Analysis of the NIPS 2016 Review Process","au":"Shah, Nihar B.","y":2017,"vn":"arXiv (Cornell University)","ty":"journalArticle","tyl":"Journal article","doi":"10.48550/arxiv.1708.09794","cby":48,"nref":16,"oast":"green","cx":"Journal","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"0","cc":1,"cp":0,"self":false,"ncoefs":0},{"id":"NCCTU8QK","t":"How to best evaluate applications for junior fellowships? Remote evaluation and face-to-face panel meetings compared","au":"Bieri, Marco","y":2020,"vn":"bioRxiv (Cold Spring Harbor Laboratory)","ty":"journalArticle","tyl":"Journal article","doi":"10.1101/2020.11.26.400028","cby":0,"nref":24,"oast":"green","cx":"Grant","co":["Reliability-noise","Validity","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level","value"],"sg":["individual-scoring","panel-consensus","funding-decision"],"st":"observational-quantitative","tp":"2","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"NCMPJGAC","t":"Review for \"That's a Lot to Process! Pitfalls of Popular Path Models\"","au":"Montoya, Amanda Kay","y":2021,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/25152459221095827/v1/review3","cby":0,"nref":57,"oast":"gold","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"guideline-policy","tp":"-1","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"NG8FPPXE","t":"Does the use of open, nonanonymous peer review in scholarly publishing introduce bias? Evidence from the F1000 postpublication open peer review publishing model","au":"Thelwall, Mike","y":2019,"vn":"Journal of Information Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/0165551520938678","cby":41,"nref":31,"oast":"green","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"NGZJ7MXU","t":"Assessing inter-rater reliability for rating scales: resolving some basic issues.","au":"Cicchetti, D V","y":1976,"vn":"The British Journal of Psychiatry","ty":"journalArticle","tyl":"Journal article","doi":"10.1192/bjp.129.5.452","cby":322,"nref":8,"oast":"closed","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"4","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"NJCR7T7F","t":"Towards improving peer review: Crowd-sourced insights from Twitter","au":"Allen, Kelly‐Ann","y":2022,"vn":"Journal of University Teaching and Learning Practice","ty":"journalArticle","tyl":"Journal article","doi":"10.53761/1.19.3.2","cby":7,"nref":33,"oast":"diamond","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"NL2R3R8Q","t":"Reflections on a mentored group peer review process","au":"Auby, Harpreet","y":2024,"vn":"Journal of Engineering Education","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/jee.20616","cby":2,"nref":17,"oast":"bronze","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":7,"cp":0,"self":false,"ncoefs":0},{"id":"NLNWACRS","t":"Foucault, the “Facts,” and the Fiction of Neutrality: Neutrality in Librarianship and Peer Review","au":"Johnson, Heidi R.","y":2016,"vn":"Canadian Journal of Academic Librarianship","ty":"journalArticle","tyl":"Journal article","doi":"10.33137/cjal-rcbu.v1.24304","cby":11,"nref":22,"oast":"diamond","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"NN8254C6","t":"Large sample standard errors of kappa and weighted kappa.","au":"Fleiss, Joseph L.","y":1969,"vn":"Psychological Bulletin","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/h0028106","cby":1525,"nref":4,"oast":"closed","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"4","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"NND4U3YX","t":"A physics editor comments on Peters and Ceci's peer-review study","au":"Adair, Robert K.","y":1982,"vn":"Behavioral and Brain Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/s0140525x00011195","cby":57,"nref":201,"oast":"closed","cx":"Journal","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"12","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"NNIEFSDW","t":"Unique review criteria and patient and stakeholder reviewers: analysis of pcori's approach to research funding.","au":"Forsythe, Laura P","y":2018,"vn":"Value in Health","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.jval.2018.03.017","cby":11,"nref":37,"oast":"bronze","cx":"Grant","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level","factors"],"sg":["individual-scoring","panel-consensus","funding-decision"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"NPECNPPD","t":"Arbitrariness in the peer review process","au":"Brezis, Elise S.","y":2020,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-020-03348-1","cby":81,"nref":26,"oast":"hybrid","cx":"General","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"0","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"NPXHHPC4","t":"Extraneous factors in judicial decisions","au":"Danziger, Shai","y":2011,"vn":"Proceedings of the National Academy of Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1073/pnas.1018033108","cby":1411,"nref":26,"oast":"closed","cx":"Analogue","co":["Bias"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":0,"self":false,"ncoefs":0},{"id":"NRNXMEJZ","t":"The Role of Peer Review in the Scientific Process","au":"Garza, Henriette De La","y":2022,"vn":"","ty":"book","tyl":"Book","doi":"10.1007/978-3-030-99680-2_41","cby":4,"nref":23,"oast":"closed","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"book-monograph","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"NSYAMRTC","t":"Perceptions of grant peer reviewers: a mixed methods study","au":"Schmaling, Karen B.","y":2024,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvae050","cby":3,"nref":37,"oast":"closed","cx":"Grant","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"8","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"NUWKCHMA","t":"Is Peer Review a Game of Chance?","au":"Neff, Bryan D.","y":2006,"vn":"BioScience","ty":"journalArticle","tyl":"Journal article","doi":"10.1641/0006-3568(2006)56[333:iprago]2.0.co;2","cby":82,"nref":31,"oast":"bronze","cx":"Journal","co":["Reliability-noise","Validity"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"simulation-methodological","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"NYR9UWFI","t":"Fast, Furious and Dubious? MDPI and the Depth of Peer Review Reports","au":"Maddi, Abdelghani","y":2023,"vn":"Research Square","ty":"journalArticle","tyl":"Journal article","doi":"10.21203/rs.3.rs-3027724/v1","cby":1,"nref":18,"oast":"green","cx":"Journal","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"0","cc":0,"cp":1,"self":false,"ncoefs":0},{"id":"P2DZ89LR","t":"Research funding: the case for a modified lottery.","au":"Fang, Ferric C","y":2016,"vn":"mBio","ty":"journalArticle","tyl":"Journal article","doi":"10.1128/mBio.00422-16","cby":163,"nref":57,"oast":"gold","cx":"Grant","co":["Reliability-noise","Bias","Validity","Interventions"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["funding-decision"],"st":"conceptual-commentary","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"P5PUM6V5","t":"Peer review’s irremediable flaws: Scientists’ perspectives on grant evaluation in Germany","au":"Barlösius, Eva","y":2023,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvad032","cby":7,"nref":44,"oast":"hybrid","cx":"Grant","co":["Bias","Validity"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"0","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"P5TFHNG5","t":"Zoom fatigue saps grant reviewers’ attention","au":"Chawla, Dalmeet Singh","y":2021,"vn":"Nature","ty":"journalArticle","tyl":"Journal article","doi":"10.1038/d41586-021-00161-5","cby":17,"nref":2,"oast":"closed","cx":"Grant","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["panel-consensus"],"st":"conceptual-commentary","tp":"13","cc":1,"cp":0,"self":false,"ncoefs":0},{"id":"P64HCKBT","t":"Reimagining the peer‐review system for translational health science journals","au":"Smith, Elise","y":2021,"vn":"Clinical and Translational Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/cts.13050","cby":24,"nref":65,"oast":"gold","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"P6D79UMP","t":"Medical Journal Peer Review: Process and Bias","au":"Manchikanti, Laxmaiah","y":2015,"vn":"Pain Physician","ty":"journalArticle","tyl":"Journal article","doi":"10.36076/ppj/2015.18.e1","cby":129,"nref":101,"oast":"hybrid","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"P7XBS7BU","t":"Peer‐reviewing geen valide basis voor erkennen en waarderen van onderzoeksprestaties","au":"Breetvelt, Iris","y":2021,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31234/osf.io/9n5mq","cby":0,"nref":13,"oast":"gold","cx":"General","co":["Reliability-noise","Bias","Validity"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"-1","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"PAEJWKA6","t":"Gender differences in grant and personnel award funding rates at the Canadian Institutes of Health Research based on research content area: A retrospective analysis.","au":"Burns, Karen E A","y":2019,"vn":"PLoS Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pmed.1002935","cby":110,"nref":50,"oast":"gold","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["funding-decision"],"st":"observational-quantitative","tp":"2","cc":4,"cp":5,"self":false,"ncoefs":0},{"id":"PB3TYFEA","t":"Improving manuscript evaluation procedures.","au":"Bowen, Donald D.","y":1972,"vn":"American Psychologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/h0033849","cby":160,"nref":16,"oast":"closed","cx":"Journal","co":["Reliability-noise","Bias","Interventions"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"3","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"PBECA3WZ","t":"Grant application review: The case of transparency.","au":"Gurwitz, David","y":2014,"vn":"PLoS Biology","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pbio.1002010","cby":30,"nref":30,"oast":"gold","cx":"Grant","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"PBV9GKQU","t":"Rational Disagreement in Peer Review","au":"Harnad, Stevan","y":1985,"vn":"Science Technology & Human Values","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/016224398501000307","cby":74,"nref":0,"oast":"closed","cx":"Grant","co":["Reliability-noise","Validity"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"0","cc":5,"cp":0,"self":false,"ncoefs":0},{"id":"PEMI66JZ","t":"An intervention to improve the reliability of manuscript reviews for the Journal of the American Academy of Child and Adolescent Psychiatry.","au":"Strayhorn, J","y":1993,"vn":"The American Journal of Psychiatry","ty":"journalArticle","tyl":"Journal article","doi":"10.1176/ajp.150.6.947","cby":53,"nref":30,"oast":"closed","cx":"Journal","co":["Reliability-noise","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"3","cc":5,"cp":3,"self":false,"ncoefs":7},{"id":"PFRFLMJE","t":"Open Science, Closed Peer Review?","au":"Evans, Daniel","y":2025,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31222/osf.io/dmgex_v2","cby":0,"nref":33,"oast":"gold","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"PH5T9KLF","t":"Peer-review for selection of oral presentations for conferences: Are we reliable?","au":"Deveugele, Myriam","y":2017,"vn":"Patient Education and Counseling","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.pec.2017.06.007","cby":21,"nref":11,"oast":"closed","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"15","cc":3,"cp":2,"self":false,"ncoefs":0},{"id":"PKDGKHJ5","t":"Rethinking (un)blinding in biomedical proposal peer review: A multi-stakeholder qualitative study","au":"Qussini, Seba","y":2025,"vn":"Accountability in Research","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/08989621.2025.2593625","cby":0,"nref":45,"oast":"hybrid","cx":"Grant","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"qualitative","tp":"7","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"PP7EU66X","t":"Lack of peer reviewer diversity advantages scientists from wealthier countries","au":"Zumel Dumlao, James M.","y":2025,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31235/osf.io/754e3_v3","cby":1,"nref":0,"oast":"gold","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["demographic","cognitive-COI"],"pbs":"demographic","rs":[],"sg":["reviewer-selection","individual-scoring"],"st":"observational-quantitative","tp":"5","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"PQZETV2H","t":"Building a Data-Driven Model of Peer Review: The Case of Science Foundation Ireland","au":"Feliciani, Thomas","y":2021,"vn":"Springer proceedings in complexity","ty":"book","tyl":"Book","doi":"10.1007/978-3-030-61503-1_21","cby":0,"nref":15,"oast":"closed","cx":"Grant","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"book-monograph","tp":"0","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"PR6SUMXI","t":"Researchers’ Goals When Assessing Credibility and Impact in Committees and in Their Own Work","au":"Harney, James","y":2021,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31219/osf.io/ryds4","cby":6,"nref":24,"oast":"gold","cx":"General","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["panel-consensus"],"st":"qualitative","tp":"-1","cc":4,"cp":2,"self":false,"ncoefs":0},{"id":"PSBEMHHH","t":"An efficient system to fund science: from proposal review to peer-to-peer distributions.","au":"Bollen, Johan","y":2017,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-016-2110-3","cby":49,"nref":32,"oast":"green","cx":"Grant","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["funding-decision"],"st":"simulation-methodological","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"PUG2N7TW","t":"NSF's ‘Big Pitch’ Tests Anonymized Grant Reviews","au":"Bhattacharjee, Yudhijit","y":2012,"vn":"Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1126/science.336.6084.969","cby":24,"nref":0,"oast":"closed","cx":"Grant","co":["Reliability-noise","Bias","Interventions"],"pco":"Reliability-noise","bs":["demographic","prestige-halo"],"pbs":"demographic","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"-1","cc":4,"cp":0,"self":false,"ncoefs":0},{"id":"PWQWRMFY","t":"Peer review of health research funding proposals: A systematic map and systematic review of innovations for effectiveness and efficiency.","au":"Shepherd, Jonathan","y":2018,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0196914","cby":41,"nref":105,"oast":"gold","cx":"Grant","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"systematic-review-meta-analysis","tp":"7","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"PYVKKJGJ","t":"A harm reduction approach to improving peer review by acknowledging its imperfections","au":"Cooke, Steven J.","y":2024,"vn":"FACETS","ty":"journalArticle","tyl":"Journal article","doi":"10.1139/facets-2024-0102","cby":8,"nref":195,"oast":"gold","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"PZBSQXF9","t":"Engineering Features From Advanced Medical Technology Initiative Submissions to Enable Predictive Modeling for Proposal Success.","au":"Pavliscsak, Holly","y":2024,"vn":"Military medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/milmed/usae063","cby":0,"nref":26,"oast":"hybrid","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["ai-llm"],"pbs":"ai-llm","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":4,"cp":0,"self":false,"ncoefs":0},{"id":"Q3AG2M2K","t":"Envisioning the Future of Peer Review: Investigating LLM-Assisted Reviewing Using ChatGPT as a Case Study","au":"Chen, Shiping","y":2025,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.1145/3729176.3729196","cby":2,"nref":60,"oast":"gold","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"experimental-rct","tp":"6","cc":2,"cp":1,"self":false,"ncoefs":5},{"id":"Q4YMV96Z","t":"Reviewer training for improving grant and journal peer review.","au":"Hesselberg, Jan-Ole","y":2023,"vn":"Cochrane Database of Systematic Reviews","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/14651858.MR000056.pub2","cby":14,"nref":67,"oast":"green","cx":"General","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"systematic-review-meta-analysis","tp":"-1","cc":2,"cp":2,"self":true,"ncoefs":0},{"id":"Q5HY3MN9","t":"Enhancing peer review skills in higher education: A mixed-methods study on challenges and training needs","au":"Stupacher, Jan","y":2025,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31219/osf.io/89xju_v2","cby":0,"nref":0,"oast":"gold","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"0","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"Q6KQ5BNR","t":"Publication problems in psychology and an explicit evaluation schema for manuscripts.","au":"Wolff, Wirt M.","y":1973,"vn":"American Psychologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/h0034678","cby":133,"nref":10,"oast":"closed","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"-1","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"Q9DUXILF","t":"How reliable is peer review of scientific abstracts?","au":"Rubin, Haya R.","y":1993,"vn":"Journal of General Internal Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/bf02600092","cby":63,"nref":37,"oast":"closed","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"15","cc":5,"cp":3,"self":false,"ncoefs":4},{"id":"QA4UL4N5","t":"Pride and prejudice – What can we learn from peer review?","au":"Sueur, Helen Le","y":2020,"vn":"Medical Teacher","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/0142159x.2020.1774527","cby":28,"nref":26,"oast":"hybrid","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"QCWPTK8N","t":"Can small and reasoning large language models score journal articles for research quality and do averaging and few-shot help?","au":"Thelwall, Mike","y":2026,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-026-05585-2","cby":3,"nref":18,"oast":"hybrid","cx":"General","co":["Validity","Interventions"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"6","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"QFEN9AY7","t":"An evaluation of the process of peer review","au":"Riding, James B.","y":2022,"vn":"Palynology","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/01916122.2022.2151052","cby":5,"nref":37,"oast":"closed","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"QFLU6VYM","t":"Confirmatory bias in peer review","au":"García, J. A.","y":2020,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-020-03357-0","cby":28,"nref":30,"oast":"closed","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"QFPVXIEC","t":"Unpacking the black box - a questionnaire study of how medical journals identify and manage conflicts of interest of editors and peer reviewers","au":"Korfitsen, Christoffer Bruun","y":2026,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31222/osf.io/g4v2n_v1","cby":0,"nref":0,"oast":"gold","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["reviewer-selection"],"st":"observational-quantitative","tp":"0","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"QI9FZXDI","t":"Insiders, outsiders, and efficiency in a National Science Foundation panel.","au":"Klahr, David","y":1985,"vn":"American Psychologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/0003-066x.40.2.148","cby":25,"nref":10,"oast":"closed","cx":"Grant","co":["Reliability-noise","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["panel-consensus","individual-scoring"],"st":"observational-quantitative","tp":"13","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"QIPW3GDB","t":"Reviewer training to assess knowledge translation in funding applications is long overdue","au":"Scarrow, Gayle","y":2017,"vn":"Research Integrity and Peer Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s41073-017-0037-8","cby":8,"nref":25,"oast":"gold","cx":"Grant","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"QJ4FU8AL","t":"Impact of a short version of the CONSORT checklist for peer reviewers to improve the reporting of randomised controlled trials published in biomedical journals: study protocol for a randomised controlled trial","au":"Speich, Benjamin","y":2020,"vn":"BMJ Open","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmjopen-2019-035114","cby":33,"nref":38,"oast":"gold","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["proposal-quality","individual-scoring"],"st":"experimental-rct","tp":"11","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"QJGGX98U","t":"Peer review: A flawed process at the heart of science and journals","au":"Smith, Richard","y":2006,"vn":"Journal of the Royal Society of Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/014107680609900414","cby":848,"nref":14,"oast":"bronze","cx":"Journal","co":["Reliability-noise","Bias","Validity","Interventions"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"QM7APGKP","t":"Reliability, fairness, objectivity and other inappropriate goals in peer review","au":"Bailar, John C.","y":1991,"vn":"Behavioral and Brain Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/S0140525X00065705","cby":54,"nref":363,"oast":"closed","cx":"General","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"12","cc":5,"cp":3,"self":false,"ncoefs":2},{"id":"QN8S7TQ4","t":"Communicating the Significance of Research Questions: Insights from Peer Review at a Flagship Journal","au":"Cai, Jinfa","y":2020,"vn":"International Journal of Science and Mathematics Education","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s10763-020-10073-x","cby":5,"nref":34,"oast":"closed","cx":"Journal","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"0","cc":5,"cp":1,"self":false,"ncoefs":0},{"id":"QPFJ9ABU","t":"Risking Groupthink in Impact Assessment","au":"Derrick, Gemma","y":2018,"vn":"","ty":"book","tyl":"Book","doi":"10.1007/978-3-319-63627-6_6","cby":1,"nref":48,"oast":"closed","cx":"General","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["panel-consensus"],"st":"book-monograph","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"QPIKNW58","t":"Blinding applicants in a first-stage peer-review process of biomedical research grants: An observational study","au":"Solans‐Domènech, Maite","y":2017,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvx021","cby":16,"nref":30,"oast":"bronze","cx":"Grant","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["prestige-halo"],"pbs":"prestige-halo","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":4,"cp":2,"self":false,"ncoefs":0},{"id":"QQRFQCNU","t":"A critique of Whitehurst's \"Interrater agreement for journal manuscript reviews\": De omnibus, disputandem est.","au":"Cicchetti, Domenic V.","y":1985,"vn":"American Psychologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/0003-066x.40.5.563","cby":43,"nref":8,"oast":"closed","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"-1","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"QR68P67D","t":"Improving the efficiency of research proposals evaluation: A two-stage procedure","au":"Seeber, Marco","y":2024,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvae020","cby":1,"nref":77,"oast":"hybrid","cx":"Grant","co":["Reliability-noise","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["proposal-quality","individual-scoring"],"st":"observational-quantitative","tp":"7","cc":4,"cp":2,"self":false,"ncoefs":6},{"id":"QWNRA5FG","t":"Comparison of the effects of using the Crowe Critical Appraisal Tool versus informal appraisal in assessing health research: a randomised trial.","au":"Crowe, Michael","y":2011,"vn":"International journal of evidence-based healthcare","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/j.1744-1609.2011.00237.x","cby":157,"nref":0,"oast":"closed","cx":"General","co":["Reliability-noise","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"experimental-rct","tp":"-1","cc":2,"cp":0,"self":false,"ncoefs":0},{"id":"QXKD6QZG","t":"A&A community survey on the future of scientific publishing","au":"Alves, J.","y":2026,"vn":"Astronomy and Astrophysics","ty":"journalArticle","tyl":"Journal article","doi":"10.1051/0004-6361/202661365","cby":0,"nref":8,"oast":"diamond","cx":"Journal","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["ai-llm","bias-general"],"pbs":"ai-llm","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"R2SIKQTM","t":"Healing the NIH-Funded Biomedical Research Enterprise","au":"Germain, Ronald N.","y":2015,"vn":"Cell","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.cell.2015.05.052","cby":26,"nref":8,"oast":"bronze","cx":"Grant","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"R3CGAELC","t":"Grant writing and grant peer review as questionable research practices","au":"Conix, Stijn","y":2021,"vn":"F1000Research","ty":"journalArticle","tyl":"Journal article","doi":"10.12688/f1000research.73893.1","cby":15,"nref":59,"oast":"gold","cx":"Grant","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["proposal-quality","whole-process"],"st":"conceptual-commentary","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"R4APKMSR","t":"Citizen Scholar Chapter 5: Peer Review, Unleashed","au":"Cohen, Philip N.","y":2023,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31235/osf.io/nax2p","cby":0,"nref":0,"oast":"gold","cx":"General","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"book-monograph","tp":"-1","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"R4QME8PM","t":"Comparing 2 approaches to residency application file review","au":"Younan J.","y":2019,"vn":"CMAJ. Canadian Medical Association Journal","ty":"journalArticle","tyl":"Journal article","doi":"10.1503/cjs.011719","cby":1,"nref":0,"oast":"hybrid","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":0,"cp":0,"self":false,"ncoefs":2},{"id":"R733ES7Q","t":"Rethink funding by putting the lottery first","au":"Luebber, Finn","y":2023,"vn":"Nature Human Behaviour","ty":"journalArticle","tyl":"Journal article","doi":"10.1038/s41562-023-01649-y","cby":9,"nref":9,"oast":"closed","cx":"Grant","co":["Bias","Validity","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["funding-decision"],"st":"simulation-methodological","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"R8HWP4XI","t":"‘Science by consensus’ impedes scientific creativity and progress: A simple alternative to funding biomedical research","au":"Düzgüneş, Nejat","y":2023,"vn":"F1000Research","ty":"journalArticle","tyl":"Journal article","doi":"10.12688/f1000research.124082.2","cby":0,"nref":14,"oast":"gold","cx":"Grant","co":["Reliability-noise","Bias","Validity","Interventions"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["funding-decision"],"st":"conceptual-commentary","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"R8Q4R9V6","t":"Manuscript review in psychology: An alternative model.","au":"Bornstein, Robert F.","y":1990,"vn":"American Psychologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/0003-066x.45.5.672","cby":41,"nref":0,"oast":"closed","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"3","cc":5,"cp":0,"self":false,"ncoefs":0},{"id":"RA5UQF8G","t":"When journal editors play favorites","au":"Heesen, Remco","y":2017,"vn":"Philosophical Studies","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11098-017-0895-4","cby":14,"nref":59,"oast":"hybrid","cx":"Journal","co":["Bias","Validity"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["individual-scoring","funding-decision"],"st":"simulation-methodological","tp":"10","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"RATFKYTW","t":"Exploring virtual funding committee practices in the allocation of National Institute for Health and Care Research funding: A netnographic study","au":"Blatch-Jones, Amanda","y":2024,"vn":"F1000Research","ty":"journalArticle","tyl":"Journal article","doi":"10.12688/f1000research.145582.1","cby":1,"nref":31,"oast":"gold","cx":"Grant","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["panel-consensus"],"st":"qualitative","tp":"7","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"RB5785EC","t":"Peer review process: Assessments by applicant-nominated referees are biased, inflated, unreliable and invalid","au":"Marsh, Herbert W.","y":2007,"vn":"Australian psychologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/00050060600823275","cby":37,"nref":14,"oast":"closed","cx":"Grant","co":["Reliability-noise","Bias","Validity"],"pco":"Reliability-noise","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["reviewer-selection","individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"-1","cc":4,"cp":4,"self":false,"ncoefs":0},{"id":"RB6NP5W7","t":"Assessing precision in the manuscript review process: A little better than a dice roll","au":"Lindsey, Duncan","y":1988,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/bf02020243","cby":98,"nref":30,"oast":"closed","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["funding-decision"],"st":"conceptual-commentary","tp":"3","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"RBI5CT7L","t":"Peer Review in Academia","au":"Forsberg, Eva","y":2022,"vn":"","ty":"book","tyl":"Book","doi":"10.1007/978-3-030-75263-7_1","cby":8,"nref":72,"oast":"gold","cx":"General","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"book-monograph","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"RCUHNQYG","t":"Double‐blind peer review affects reviewer ratings and editor decisions at an ecology journal","au":"Fox, Charles W.","y":2023,"vn":"Functional Ecology","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/1365-2435.14259","cby":105,"nref":64,"oast":"bronze","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring","funding-decision"],"st":"experimental-rct","tp":"10","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"RFR8CR44","t":"How Masked Is the “Masked Peer Review” of Abstracts Submitted to International Medical Conferences?","au":"Falagas, Matthew E.","y":2006,"vn":"Mayo Clinic Proceedings","ty":"journalArticle","tyl":"Journal article","doi":"10.4065/81.5.705","cby":6,"nref":6,"oast":"bronze","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"RGKZZYMF","t":"Fairness of Peer Review Systems in Hospitality and Tourism Journals","au":"Cheong, Fan","y":2024,"vn":"Journal of China Tourism Research","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/19388160.2024.2441362","cby":0,"nref":73,"oast":"closed","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"qualitative","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"RJEGBDFX","t":"A game theoretic approach to peer review of grant proposals","au":"Bayindir, Esra Eren","y":2019,"vn":"Journal of Informetrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.joi.2019.100981","cby":3,"nref":25,"oast":"closed","cx":"Grant","co":["Reliability-noise","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["panel-consensus"],"st":"simulation-methodological","tp":"-1","cc":6,"cp":2,"self":false,"ncoefs":5},{"id":"RJLFSWWX","t":"Standardizing an approach to the evaluation of implementation science proposals.","au":"Crable, Erika L","y":2018,"vn":"Implementation Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s13012-018-0770-5","cby":38,"nref":28,"oast":"gold","cx":"Grant","co":["Reliability-noise","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"RJR9QVGF","t":"Wie zuverlässig ist das Peer-Review-Verfahren? Eine Untersuchung der Interrater-Reliabilität von Gutachter*innen auf DGPuK-Tagungen","au":"Koch, Thomas","y":2019,"vn":"Studies in Communication and Media","ty":"journalArticle","tyl":"Journal article","doi":"10.5771/2192-4007-2019-2-203","cby":2,"nref":58,"oast":"diamond","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":5,"cp":4,"self":false,"ncoefs":4},{"id":"RMNMTAKV","t":"Gender differences in peer review outcomes and manuscript impact at six journals of ecology and evolution.","au":"Fox, Charles W","y":2019,"vn":"Ecology and Evolution","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/ece3.4993","cby":190,"nref":59,"oast":"gold","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"5","cc":4,"cp":5,"self":false,"ncoefs":0},{"id":"RN4Z3CSB","t":"Defining the role of cognitive distance in the peer review process with an explorative study of a grant scheme in infection biology","au":"Wang, Qi","y":2015,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvv009","cby":31,"nref":59,"oast":"green","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["reviewer-selection"],"st":"observational-quantitative","tp":"-1","cc":4,"cp":4,"self":false,"ncoefs":0},{"id":"RP9WIA7I","t":"Differences in Editorial Board Reviewer Behavior Based on Gender","au":"Wing, Deborah A.","y":2010,"vn":"Journal of Women s Health","ty":"journalArticle","tyl":"Journal article","doi":"10.1089/jwh.2009.1904","cby":76,"nref":16,"oast":"closed","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"5","cc":3,"cp":5,"self":false,"ncoefs":1},{"id":"RQS39XWL","t":"Impact from the Evaluators’ Eye","au":"Derrick, Gemma","y":2018,"vn":"","ty":"book","tyl":"Book","doi":"10.1007/978-3-319-63627-6_1","cby":1,"nref":49,"oast":"closed","cx":"General","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["panel-consensus"],"st":"book-monograph","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"RR7TA29K","t":"Revealing Reviewers’ Identities as Part of Open Peer Review and Analysis of the Review Reports","au":"Bolek, Cezary","y":2020,"vn":"LIBER Quarterly The Journal of the Association of European Research Libraries","ty":"journalArticle","tyl":"Journal article","doi":"10.18352/lq.10347","cby":6,"nref":18,"oast":"diamond","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"RR8Z3GMV","t":"Is Lottery a Better Way of Resource Distribution Than Baseline Funding?","au":"Sakamoto, Takahiro","y":2023,"vn":"Philosophy of Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/psa.2023.49","cby":2,"nref":32,"oast":"bronze","cx":"Grant","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["funding-decision"],"st":"simulation-methodological","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"RRJ5XCFM","t":"Reliability of reviews for the American Psychologist: A biostatistical assessment of the data.","au":"Cicchetti, Domenic V.","y":1980,"vn":"American Psychologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/0003-066x.35.3.300","cby":160,"nref":26,"oast":"closed","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"3","cc":5,"cp":3,"self":false,"ncoefs":7},{"id":"RRPR8N5D","t":"Development and evaluation of a quality score for abstracts","au":"Timmer, Antje","y":2003,"vn":"BMC Medical Research Methodology","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/1471-2288-3-2","cby":121,"nref":41,"oast":"gold","cx":"Journal","co":["Reliability-noise","Validity"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level","factors"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"15","cc":5,"cp":6,"self":false,"ncoefs":4},{"id":"RRRS56JK","t":"Role, function, and expectations of research funding committees: Perspectives from committee members.","au":"Blatch-Jones, Amanda","y":2024,"vn":"F1000Research","ty":"journalArticle","tyl":"Journal article","doi":"10.12688/f1000research.154665.3","cby":2,"nref":36,"oast":"gold","cx":"Grant","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["panel-consensus","funding-decision"],"st":"qualitative","tp":"7","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"RUSREZRN","t":"Exploring the potential role of community engagement in evaluating clinical and translational science grant proposals.","au":"Treem, Jeffrey W","y":2018,"vn":"Journal of clinical and translational science","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/cts.2018.311","cby":4,"nref":17,"oast":"gold","cx":"Grant","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring","panel-consensus"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":0,"self":false,"ncoefs":0},{"id":"RV5VHIXB","t":"Uncertainty and Individual Discretion in Allocating Research Funds","au":"Goldstein, Anna","y":2017,"vn":"SSRN Electronic Journal","ty":"journalArticle","tyl":"Journal article","doi":"10.2139/ssrn.3012169","cby":13,"nref":49,"oast":"green","cx":"Grant","co":["Reliability-noise","Validity"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["funding-decision"],"st":"observational-quantitative","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"RVJKZWBR","t":"The Selection of Scientific Talent in the Allocation of Research Grants","au":"Arensbergen, P. van","y":2012,"vn":"Higher Education Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1057/hep.2012.15","cby":44,"nref":54,"oast":"closed","cx":"Grant","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring","panel-consensus","funding-decision"],"st":"observational-quantitative","tp":"2","cc":4,"cp":2,"self":false,"ncoefs":0},{"id":"RYZBUT3Q","t":"Cost of the NSERC Science Grant Peer Review System Exceeds the Cost of Giving Every Qualified Researcher a Baseline Grant","au":"Gordon, Richard","y":2009,"vn":"Accountability in Research","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/08989620802689821","cby":56,"nref":45,"oast":"closed","cx":"Grant","co":["Reliability-noise","Validity","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["funding-decision"],"st":"observational-quantitative","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"RZQ4LY8Z","t":"Dutch research funding, gender bias, and Simpson's paradox.","au":"Albers, Casper J","y":2015,"vn":"Proceedings of the National Academy of Sciences of the United States of America","ty":"journalArticle","tyl":"Journal article","doi":"10.1073/pnas.1518936112","cby":58,"nref":5,"oast":"bronze","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["funding-decision"],"st":"conceptual-commentary","tp":"2","cc":4,"cp":5,"self":false,"ncoefs":0},{"id":"S2F2F3ZG","t":"Funding by lottery: political problems and research opportunities.","au":"Barnett, Adrian G","y":2016,"vn":"mBio","ty":"journalArticle","tyl":"Journal article","doi":"10.1128/mBio.01369-16","cby":24,"nref":2,"oast":"gold","cx":"Grant","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["funding-decision"],"st":"conceptual-commentary","tp":"1","cc":1,"cp":0,"self":false,"ncoefs":0},{"id":"S2HNC4AW","t":"Blinded vs. unblinded peer review of manuscripts submitted to a dermatology journal: a randomized multi-rater study","au":"Alam, M Fasihul","y":2011,"vn":"British Journal of Dermatology","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/j.1365-2133.2011.10432.x","cby":57,"nref":10,"oast":"closed","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["prestige-halo"],"pbs":"prestige-halo","rs":[],"sg":["individual-scoring"],"st":"experimental-rct","tp":"10","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"S36DXTBV","t":"Anti-Asian biases in federal grant reviews: Commentary on Yip et al. (2021).","au":"Yip, Tiffany","y":2024,"vn":"American Psychologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/amp0001337","cby":9,"nref":10,"oast":"green","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring"],"st":"qualitative","tp":"14","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"S38N4UBK","t":"How qualitative criteria can improve the assessment process of interdisciplinary research proposals","au":"Schölvinck, Anne-Floor M","y":2024,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvae049","cby":5,"nref":51,"oast":"hybrid","cx":"Grant","co":["Validity","Interventions"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"qualitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"S49RVGAY","t":"Toward a New Model of Scientific Publishing: Discussion and a Proposal","au":"Kravitz, Dwight J.","y":2011,"vn":"Frontiers in Computational Neuroscience","ty":"journalArticle","tyl":"Journal article","doi":"10.3389/fncom.2011.00055","cby":62,"nref":10,"oast":"gold","cx":"Journal","co":["Validity","Interventions"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"-1","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"S4CDNSIP","t":"Research on research funding: an imperative for science and society","au":"Severin, Anna","y":2020,"vn":"British Journal of Sports Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bjsports-2020-103340","cby":11,"nref":9,"oast":"green","cx":"Grant","co":["Bias","Validity"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"7","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"S4D7BM4N","t":"Scientific Grant Funding","au":"Azoulay, Pierre","y":2020,"vn":"National Bureau of Economic Research","ty":"journalArticle","tyl":"Journal article","doi":"10.3386/w26889","cby":46,"nref":114,"oast":"green","cx":"Grant","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"S59DARDU","t":"A Kuhnian Critique of Psychometric Research on Peer Review","au":"Lee, Carole J.","y":2012,"vn":"Philosophy of Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1086/667841","cby":43,"nref":61,"oast":"closed","cx":"General","co":["Reliability-noise","Validity"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"3","cc":3,"cp":4,"self":false,"ncoefs":0},{"id":"S5ZNPKAX","t":"How reliable is peer review? An examination of operating grant proposals simultaneously submitted to two similar peer review systems.","au":"Hodgson, C","y":1997,"vn":"Journal of Clinical Epidemiology","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/S0895-4356(97)00167-4","cby":48,"nref":24,"oast":"closed","cx":"Grant","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level","factors"],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"8","cc":1,"cp":4,"self":false,"ncoefs":4},{"id":"S7AYT2Z6","t":"Is peer review useful in assessing research proposals in Indigenous health? A case study","au":"Street, Jackie","y":2009,"vn":"Health Research Policy and Systems","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/1478-4505-7-2","cby":16,"nref":13,"oast":"gold","cx":"Grant","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"qualitative","tp":"-1","cc":1,"cp":4,"self":false,"ncoefs":0},{"id":"S9EHKIC4","t":"Frame Search and Re-search: How Quantitative Sociological Articles Change During Peer Review","au":"Teplitskiy, Misha","y":2015,"vn":"The American Sociologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s12108-015-9288-3","cby":29,"nref":65,"oast":"closed","cx":"Journal","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"0","cc":3,"cp":2,"self":false,"ncoefs":0},{"id":"SA7AMBC5","t":"Reliability of editors' subjective quality ratings of peer reviews of manuscripts.","au":"Callaham, M L","y":1998,"vn":"The Journal of the American Medical Association","ty":"journalArticle","tyl":"Journal article","doi":"10.1001/jama.280.3.229","cby":123,"nref":9,"oast":"closed","cx":"Journal","co":["Reliability-noise","Validity"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level","factors"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"3","cc":2,"cp":1,"self":false,"ncoefs":4},{"id":"SA84R68P","t":"Inter-Rater Reliability in Assessing the Methodological Quality of Research Papers in Psychology","au":"Etzel, Franka Tabitha","y":2025,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31234/osf.io/4w7rb_v2","cby":0,"nref":0,"oast":"gold","cx":"General","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level","factors"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":0,"cp":0,"self":false,"ncoefs":4},{"id":"SAJ2V3PH","t":"An Analysis of Preliminary and Post-Discussion Priority Scores for Grant Applications Peer Reviewed by the Center for Scientific Review at the NIH","au":"Martin, Michael R.","y":2010,"vn":"PLoS ONE","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0013526","cby":31,"nref":6,"oast":"gold","cx":"Grant","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring","panel-consensus"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":5,"self":false,"ncoefs":0},{"id":"SAUTPX5E","t":"Peer review in funding-by-lottery: A systematic overview and expansion","au":"Shaw, Jamie","y":2023,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvac022","cby":29,"nref":115,"oast":"closed","cx":"Grant","co":["Reliability-noise","Bias","Validity"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["funding-decision"],"st":"conceptual-commentary","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"SBVJDN7K","t":"A randomized trial of fellowships for early career researchers finds a high reliability in funding decisions.","au":"Clarke, Philip","y":2016,"vn":"Journal of Clinical Epidemiology","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.jclinepi.2015.04.010","cby":21,"nref":15,"oast":"closed","cx":"Grant","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level","factors"],"sg":["panel-consensus","funding-decision"],"st":"experimental-rct","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":1},{"id":"SC63XAI2","t":"Journal ratings as predictors of articles quality in Arts, Humanities and Social Sciences: an analysis based on the Italian Research Evaluation Exercise","au":"Bonaccorsi, Andrea","y":2015,"vn":"F1000Research","ty":"journalArticle","tyl":"Journal article","doi":"10.12688/f1000research.6478.1","cby":15,"nref":64,"oast":"gold","cx":"General","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"9","cc":3,"cp":4,"self":false,"ncoefs":0},{"id":"SCBRUUIQ","t":"The sociology of scientific validity: How professional networks shape judgement in peer review","au":"Teplitskiy, Misha","y":2018,"vn":"Research Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.respol.2018.06.014","cby":112,"nref":123,"oast":"green","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["reviewer-selection","individual-scoring"],"st":"observational-quantitative","tp":"0","cc":3,"cp":2,"self":false,"ncoefs":5},{"id":"SDFDFFNQ","t":"Contest models highlight inherent inefficiencies of scientific funding competitions","au":"Gross, Kevin","y":2019,"vn":"PLoS Biology","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pbio.3000065","cby":122,"nref":34,"oast":"gold","cx":"Grant","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["proposal-quality","funding-decision"],"st":"simulation-methodological","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"SEPHFSGV","t":"Using simplified peer review processes to fund research: a prospective study","au":"Herbert, Danielle L.","y":2015,"vn":"BMJ Open","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmjopen-2015-008380","cby":32,"nref":21,"oast":"gold","cx":"Grant","co":["Reliability-noise","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level","value"],"sg":["funding-decision","panel-consensus"],"st":"observational-quantitative","tp":"7","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"SHG9FRIZ","t":"Funding Risky Research","au":"Franzoni, Chiara","y":2022,"vn":"Entrepreneurship and Innovation Policy and the Economy","ty":"journalArticle","tyl":"Journal article","doi":"10.1086/719252","cby":0,"nref":45,"oast":"green","cx":"Grant","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["funding-decision"],"st":"conceptual-commentary","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"SHHLHEPD","t":"Reliability of reviewers' ratings when using public peer review: a case study","au":"Bornmann, Lutz","y":2010,"vn":"Learned Publishing","ty":"journalArticle","tyl":"Journal article","doi":"10.1087/20100207","cby":50,"nref":0,"oast":"closed","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":2,"cp":0,"self":false,"ncoefs":6},{"id":"SI6TYDVR","t":"Testing Heterogeneity in Inter-Rater Reliability","au":"Bartos̆, Frantis̆ek","y":2020,"vn":"Springer proceedings in mathematics & statistics","ty":"book","tyl":"Book","doi":"10.1007/978-3-030-43469-4_26","cby":5,"nref":24,"oast":"closed","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"book-monograph","tp":"4","cc":6,"cp":2,"self":false,"ncoefs":0},{"id":"SIN7UYQK","t":"Scientists from Minority-Serving Institutions and Their Participation in Grant Peer Review","au":"Gallo, Stephen A.","y":2021,"vn":"BioScience","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/biosci/biab130","cby":4,"nref":27,"oast":"bronze","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["reviewer-selection"],"st":"observational-quantitative","tp":"14","cc":4,"cp":2,"self":false,"ncoefs":0},{"id":"SL7FVHLY","t":"Improving Scientific Judgments in Law and Government: A Field Experiment of Patent Peer Review","au":"Ho, Daniel E.","y":2020,"vn":"Journal of Empirical Legal Studies","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/jels.12249","cby":10,"nref":49,"oast":"hybrid","cx":"General","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["funding-decision"],"st":"experimental-rct","tp":"1","cc":3,"cp":2,"self":false,"ncoefs":0},{"id":"SLY82ACP","t":"Should Authors be Requested to Suggest Peer Reviewers?","au":"Silva, Jaime A. Teixeira da","y":2017,"vn":"Science and Engineering Ethics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11948-016-9842-6","cby":23,"nref":34,"oast":"closed","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["reviewer-selection"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"SNXCY9C9","t":"The iSchool Community: A Case Study of iConference Reviews","au":"Bogers, Toine","y":2016,"vn":"iConference 2016 Proceedings","ty":"journalArticle","tyl":"Journal article","doi":"10.9776/16247","cby":8,"nref":19,"oast":"green","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["reviewer-selection","individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"SQH3JZM2","t":"A Computer Program for Assessing Specific Category Rater Agreement for Qualitative Data","au":"Cicchetti, Domenic V.","y":1978,"vn":"Educational and Psychological Measurement","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/001316447803800322","cby":93,"nref":12,"oast":"closed","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"4","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"SQSVEPGJ","t":"Peer review--the newcomers' perspective.","au":"Mainguy, Gaell","y":2005,"vn":"PLoS biology","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pbio.0030326","cby":27,"nref":7,"oast":"gold","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"10","cc":4,"cp":1,"self":false,"ncoefs":0},{"id":"SR9TSCSZ","t":"2  Does end-user-engaged peer review improve the relevance, replicability, and accessibility of research evidence?","au":"Li, Edward W","y":2022,"vn":"BMJ Publishing Group Ltd","ty":"conferencePaper","tyl":"Conference paper","doi":"10.1136/ebm-2022-EBMLive.24","cby":0,"nref":0,"oast":"closed","cx":"General","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["reviewer-selection","individual-scoring"],"st":"systematic-review-meta-analysis","tp":"-1","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"SS4GAFL8","t":"Extending ArXiv.org to Achieve Open Peer Review and Publishing","au":"Boldt, Axel","y":2010,"vn":"Journal of Scholarly Publishing","ty":"journalArticle","tyl":"Journal article","doi":"10.3138/jsp.42.2.238","cby":23,"nref":0,"oast":"closed","cx":"Journal","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":0,"self":false,"ncoefs":0},{"id":"SSVEGHMV","t":"Innovations in peer review in scholarly publishing: a meta-summary","au":"Woods, Helen Buckley","y":2022,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31235/osf.io/qaksd","cby":6,"nref":25,"oast":"gold","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"systematic-review-meta-analysis","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"STUQWYY7","t":"Agreeing to disagree: A study of the reliability of manuscript reviews","au":"Kirk, S. A.","y":1997,"vn":"Social Work Research","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/swr/21.2.121","cby":37,"nref":0,"oast":"closed","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"3","cc":2,"cp":0,"self":false,"ncoefs":5},{"id":"SUR92HZC","t":"A Quantitative Linguistic Analysis of National Institutes of Health R01 Application Critiques From Investigators at One Institution","au":"Kaatz, Anna","y":2014,"vn":"Academic Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.1097/acm.0000000000000442","cby":83,"nref":38,"oast":"green","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"-1","cc":4,"cp":5,"self":false,"ncoefs":0},{"id":"SUY5YMZQ","t":"Analysis of National Institutes of Health R01 Application Critiques, Impact, and Criteria Scores: Does the Sex of the Principal Investigator Make a Difference?","au":"Kaatz, Anna","y":2016,"vn":"Academic medicine : journal of the Association of American Medical Colleges","ty":"journalArticle","tyl":"Journal article","doi":"10.1097/ACM.0000000000001272","cby":157,"nref":56,"oast":"bronze","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":4,"cp":5,"self":false,"ncoefs":0},{"id":"SVBDILLU","t":"A unified reviewing format for grant applications and evaluations.","au":"Janke, Carsten","y":2018,"vn":"EMBO reports","ty":"journalArticle","tyl":"Journal article","doi":"10.15252/embr.201745611","cby":2,"nref":0,"oast":"bronze","cx":"Grant","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["proposal-quality","individual-scoring"],"st":"conceptual-commentary","tp":"8","cc":6,"cp":0,"self":false,"ncoefs":0},{"id":"SWHA4YCC","t":"Peer Review or Lottery? A Critical Analysis of Two Different Forms of Decision-making Mechanisms for Allocation of Research Grants","au":"Roumbanis, Lambros","y":2019,"vn":"Science, Technology, & Human Values","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/0162243918822744","cby":83,"nref":59,"oast":"closed","cx":"Grant","co":["Reliability-noise","Bias","Validity"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":["factors"],"sg":["funding-decision"],"st":"conceptual-commentary","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"SXIK8HML","t":"Reviewer recommendations and editorial outcomes in peer review: a longitudinal analysis of agreement and disagreement across review rounds","au":"Cuevas, Carlos De las","y":2026,"vn":"Research Integrity and Peer Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s41073-026-00200-7","cby":0,"nref":13,"oast":"gold","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"3","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"SYCLI3BC","t":"Towards a More Structured Peer Review Process with Empirical Standards","au":"Arshad, Arham","y":2021,"vn":"Evaluation and Assessment in Software Engineering","ty":"journalArticle","tyl":"Journal article","doi":"10.1145/3463274.3463359","cby":6,"nref":12,"oast":"closed","cx":"Journal","co":["Reliability-noise","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"-1","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"SYDFSMXV","t":"Assessing interrater agreement via the average deviation index given a variety of theoretical and methodological problems","au":"Smith-Crowe, Kristin","y":2013,"vn":"Organizational Research Methods","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/1094428112465898","cby":29,"nref":59,"oast":"closed","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"4","cc":6,"cp":3,"self":false,"ncoefs":0},{"id":"SYUM6PRB","t":"Examining the predictive validity of NIH peer review scores.","au":"Lindner, Mark D","y":2015,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0126938","cby":27,"nref":118,"oast":"gold","cx":"Grant","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["post-award-validity"],"st":"observational-quantitative","tp":"9","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"SZ554632","t":"Assessing Inter-rater Reliability With Heterogeneous Variance Components Models: Flexible Approach Accounting for Contextual Variables","au":"Martinková, Patrícia","y":2023,"vn":"Journal of Educational and Behavioral Statistics","ty":"journalArticle","tyl":"Journal article","doi":"10.3102/10769986221150517","cby":7,"nref":55,"oast":"closed","cx":"Grant","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level","factors"],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"4","cc":6,"cp":2,"self":false,"ncoefs":0},{"id":"T275N5RX","t":"Can Oncologists Predict the Efficacy of Treatments in Randomized Trials?","au":"Benjamin, D. M.","y":2020,"vn":"The Oncologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1634/theoncologist.2020-0054","cby":15,"nref":13,"oast":"bronze","cx":"Analogue","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"11","cc":1,"cp":0,"self":false,"ncoefs":0},{"id":"T3A36PFH","t":"Interrater agreement for journal manuscript reviews.","au":"Whitehurst, Grover J.","y":1984,"vn":"American Psychologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/0003-066x.39.1.22","cby":190,"nref":12,"oast":"closed","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"3","cc":5,"cp":3,"self":false,"ncoefs":4},{"id":"T4CIJ7Y9","t":"Hidden Inequities in Peer Review: Integrating Geographic, Ethnic, and Linguistic Bias","au":"Machado, Maria","y":2026,"vn":"Trends in Scholarly Publishing","ty":"journalArticle","tyl":"Journal article","doi":"10.21124/tsp.2026.37.42","cby":0,"nref":23,"oast":"diamond","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["demographic","bias-general"],"pbs":"demographic","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"5","cc":4,"cp":5,"self":false,"ncoefs":0},{"id":"T584RU39","t":"Gender bias in the allocation of student grants","au":"Wijnen, Marjolijn N.","y":2021,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-021-03985-0","cby":22,"nref":35,"oast":"hybrid","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"2","cc":4,"cp":5,"self":false,"ncoefs":0},{"id":"T5FYTZ8E","t":"Structured peer review","au":"Malički, Mario","y":2023,"vn":"PUBMET","ty":"journalArticle","tyl":"Journal article","doi":"10.15291/pubmet.4268","cby":0,"nref":2,"oast":"diamond","cx":"Journal","co":["Reliability-noise","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":0,"cp":0,"self":false,"ncoefs":2},{"id":"T67A4S8E","t":"Review panel consensus and post-decision commercial performance: a study of early stage technologies","au":"Galbraith, Craig S.","y":2010,"vn":"The Journal of technology transfer","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s10961-009-9122-6","cby":12,"nref":87,"oast":"closed","cx":"Grant","co":["Reliability-noise","Validity"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["panel-consensus","post-award-validity"],"st":"observational-quantitative","tp":"13","cc":4,"cp":2,"self":false,"ncoefs":0},{"id":"T6YQ33G6","t":"Inter-rater agreement in the scoring of abstracts submitted to a primary care research conference","au":"Montgomery, Alan","y":2002,"vn":"BMC Health Services Research","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/1472-6963-2-8","cby":147,"nref":5,"oast":"gold","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"15","cc":5,"cp":0,"self":false,"ncoefs":4},{"id":"T6Z9QVYH","t":"How Gendered Is the Peer-Review Process? A Mixed-Design Analysis of Reviewer Feedback","au":"König, Thomas","y":2021,"vn":"PS Political Science & Politics","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/s1049096521000937","cby":7,"nref":31,"oast":"closed","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"5","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"TA7PAIQN","t":"Lottery before peer review is associated with increased female representation and reduced estimated economic cost in a German funding line.","au":"Luebber, Finn","y":2025,"vn":"Nature communications","ty":"journalArticle","tyl":"Journal article","doi":"10.1038/s41467-025-65660-9","cby":2,"nref":56,"oast":"gold","cx":"Grant","co":["Bias","Validity","Interventions"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["funding-decision"],"st":"observational-quantitative","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"TCT2KQAT","t":"Similar‐to‐me effects in the grant application process: Applicants, panellists, and the likelihood of obtaining funds","au":"Banal‐Estañol, Albert","y":2023,"vn":"R and D Management","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/radm.12601","cby":6,"nref":46,"oast":"hybrid","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["cognitive-COI","demographic"],"pbs":"demographic","rs":[],"sg":["panel-consensus","funding-decision"],"st":"observational-quantitative","tp":"2","cc":4,"cp":2,"self":false,"ncoefs":0},{"id":"TE7UT3JG","t":"Looking Across and Looking Beyond the Knowledge Frontier: Intellectual Distance, Novelty, and Resource Allocation in Science.","au":"Boudreau, Kevin J","y":2016,"vn":"Management science","ty":"journalArticle","tyl":"Journal article","doi":"10.1287/mnsc.2015.2207","cby":1,"nref":60,"oast":"green","cx":"Grant","co":["Reliability-noise","Validity"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["factors","value"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":5,"cp":2,"self":false,"ncoefs":6},{"id":"THI5QKB2","t":"How to improve scientific peer review: Four schools of thought","au":"Waltman, Ludo","y":2023,"vn":"Learned Publishing","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/leap.1544","cby":65,"nref":66,"oast":"hybrid","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"TIAHXM4B","t":"Interjudgmental reliability of reviews for the Journal of Educational Psychology..","au":"Marsh, Herbert W.","y":1981,"vn":"Journal of Educational Psychology","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/0022-0663.73.6.872","cby":89,"nref":7,"oast":"closed","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"3","cc":5,"cp":3,"self":false,"ncoefs":2},{"id":"TLKPSVAY","t":"Review of Peer Review Final report","au":"Kolarz, Peter","y":null,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"","cby":0,"nref":0,"oast":"","cx":"Grant","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"systematic-review-meta-analysis","tp":"11","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"TLPKXX2Q","t":"Cognitive Bias in the Peer Review Process","au":"Street, Chris","y":2019,"vn":"ACM SIGMIS Database the DATABASE for Advances in Information Systems","ty":"journalArticle","tyl":"Journal article","doi":"10.1145/3371041.3371046","cby":11,"nref":91,"oast":"closed","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["individual-scoring"],"st":"experimental-rct","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"TLSP5JN5","t":"Do Author-Suggested Reviewers Rate Submissions More Favorably than Editor-Suggested Reviewers? A Study on Atmospheric Chemistry and Physics","au":"Bornmann, Lutz","y":2010,"vn":"PLoS ONE","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0013345","cby":43,"nref":52,"oast":"gold","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["reviewer-selection","individual-scoring"],"st":"observational-quantitative","tp":"3","cc":2,"cp":4,"self":false,"ncoefs":0},{"id":"TTLBYQFY","t":"Citation Impact of NHLBI R01 Grants Funded Through the American Recovery and Reinvestment Act as Compared to R01 Grants Funded Through a Standard Payline","au":"Danthi, Narasimhan","y":2015,"vn":"Circulation Research","ty":"journalArticle","tyl":"Journal article","doi":"10.1161/circresaha.116.305894","cby":21,"nref":9,"oast":"bronze","cx":"Grant","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["funding-decision","post-award-validity"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"TTVVB5VB","t":"Factors influencing the scientific performance of Momentum grant holders: an evaluation of the first 117 research groups","au":"Győrffy, Balázs","y":2018,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-018-2852-1","cby":24,"nref":61,"oast":"hybrid","cx":"Grant","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["post-award-validity"],"st":"observational-quantitative","tp":"-1","cc":4,"cp":2,"self":false,"ncoefs":0},{"id":"TVAKSECW","t":"Supporting robust, rigorous, and reliable reviewing as the cornerstone of our profession: Introducing a competency framework for peer review","au":"Köhler, Tine","y":2020,"vn":"Industrial and organizational psychology","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/iop.2019.121","cby":58,"nref":57,"oast":"green","cx":"General","co":["Reliability-noise","Bias","Interventions"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"0","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"TZXYKKCC","t":"A retrospective analysis of the effect of discussion in teleconference and face-to-face scientific peer-review panels.","au":"Carpenter, Afton S","y":2015,"vn":"BMJ Open","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmjopen-2015-009138","cby":32,"nref":17,"oast":"gold","cx":"Grant","co":["Reliability-noise","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["panel-consensus"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"U2VS5L4D","t":"Does the NIH Fund Edge Science?","au":"Packalén, Mikko","y":2018,"vn":"National Bureau of Economic Research","ty":"journalArticle","tyl":"Journal article","doi":"10.3386/w24860","cby":12,"nref":52,"oast":"green","cx":"Grant","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["funding-decision"],"st":"observational-quantitative","tp":"-1","cc":4,"cp":2,"self":false,"ncoefs":0},{"id":"U3FISC8H","t":"Peer review reports of randomized controlled trials in oncology can be short and superficial","au":"Logullo, Patrícia","y":2025,"vn":"Journal of Clinical Epidemiology","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.jclinepi.2025.111893","cby":2,"nref":69,"oast":"hybrid","cx":"Journal","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"11","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"U43AIBHF","t":"Working double-blind","au":"Nature","y":2008,"vn":"Nature","ty":"journalArticle","tyl":"Journal article","doi":"10.1038/451605b","cby":26,"nref":3,"oast":"bronze","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"-1","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"U4JKK6XI","t":"How do authors’ perceptions of their papers compare with co-authors’ perceptions and peer-review decisions?","au":"Rastogi, Charvi","y":2024,"vn":"PLoS ONE","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0300710","cby":4,"nref":45,"oast":"gold","cx":"Journal","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":["level"],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"5","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"U4LVMCYZ","t":"Peer review of grant applications: what do we know?","au":"Wessely, S","y":1998,"vn":"The Lancet","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/S0140-6736(97)11129-1","cby":143,"nref":100,"oast":"closed","cx":"Grant","co":["Reliability-noise","Bias","Validity","Interventions"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["reviewer-selection","individual-scoring"],"st":"systematic-review-meta-analysis","tp":"-1","cc":1,"cp":4,"self":false,"ncoefs":0},{"id":"U5BTC2NZ","t":"Ejes de discusión en la evaluación de la ciencia: revisión por pares, bibliometría y pertinencia","au":"Sarthou, Nerina Fernanda","y":2016,"vn":"Revista de Estudios Sociales","ty":"journalArticle","tyl":"Journal article","doi":"10.7440/res58.2016.06","cby":14,"nref":41,"oast":"diamond","cx":"General","co":["Reliability-noise","Bias","Validity","Interventions"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"9","cc":3,"cp":2,"self":false,"ncoefs":0},{"id":"U6HNRM36","t":"Scientists’ perception of pilot study quality was influenced by statistical significance and study design","au":"Klinggraeff, Lauren von","y":2023,"vn":"Journal of Clinical Epidemiology","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.jclinepi.2023.05.011","cby":5,"nref":30,"oast":"hybrid","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"experimental-rct","tp":"11","cc":4,"cp":1,"self":false,"ncoefs":0},{"id":"UAGF8GCJ","t":"Grant reviewer perceptions of the quality and effectiveness of panel discussion","au":"Gallo, Stephen A.","y":2019,"vn":"bioRxiv (Cold Spring Harbor Laboratory)","ty":"journalArticle","tyl":"Journal article","doi":"10.1101/586685","cby":2,"nref":35,"oast":"green","cx":"Grant","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["panel-consensus"],"st":"observational-quantitative","tp":"13","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"UEURBNET","t":"The troubles with peer review for allocating research funding: Funders need to experiment with versions of peer review and decision-making.","au":"Bendiscioli, Sandra","y":2019,"vn":"EMBO Reports","ty":"journalArticle","tyl":"Journal article","doi":"10.15252/embr.201949472","cby":51,"nref":13,"oast":"hybrid","cx":"Grant","co":["Bias","Validity","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"UF4JGDEA","t":"Peer review practices for evaluating biomedical research grants: A scientific statement from the american heart association.","au":"Liaw, Lucy","y":2017,"vn":"Circulation Research","ty":"journalArticle","tyl":"Journal article","doi":"10.1161/RES.0000000000000158","cby":15,"nref":46,"oast":"closed","cx":"Grant","co":["Reliability-noise","Bias","Validity","Interventions"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"guideline-policy","tp":"7","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"UFFKJYI4","t":"Reliability of ratings of research papers.","au":"McReynolds, Paul","y":1971,"vn":"American Psychologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/h0037937","cby":109,"nref":0,"oast":"closed","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"3","cc":5,"cp":0,"self":false,"ncoefs":0},{"id":"UJX27BA2","t":"Peer-Review and Academic Archaeology: Quality, Epistemology and Science Policies","au":"Salazar, Diego","y":2019,"vn":"Archaeologies","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11759-019-09367-6","cby":4,"nref":59,"oast":"closed","cx":"Journal","co":["Validity","Interventions"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"UKMK3ZNF","t":"Evaluation of scientific research projects on the basis of evidential reasoning approach under the perspective of expert reliability","au":"Zhu, Weidong","y":2021,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-021-04201-9","cby":11,"nref":58,"oast":"closed","cx":"Grant","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"-1","cc":3,"cp":2,"self":false,"ncoefs":0},{"id":"UKWAWEDT","t":"Interreferee agreement on some characteristics of manuscripts submitted to the Journal of Personality and Social Psychology.","au":"Scott, William A.","y":1974,"vn":"American Psychologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/h0037631","cby":167,"nref":7,"oast":"closed","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level","factors"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"3","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"ULDI5EEF","t":"How to Fix Peer Review: Separating Its Two Functions--Improving Manuscripts and Judging Their Scientific Merit--Would Help.","au":"Kaplan, David","y":2005,"vn":"Journal of Child and Family Studies","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s10826-005-6845-3","cby":19,"nref":6,"oast":"closed","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"UPSXNUDQ","t":"The present and future of peer review: Ideas, interventions, and evidence","au":"Aczél, Balázs","y":2025,"vn":"Proceedings of the National Academy of Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1073/pnas.2401232121","cby":72,"nref":88,"oast":"hybrid","cx":"General","co":["Reliability-noise","Bias","Validity","Interventions"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":["level"],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"UQ4VV8EV","t":"Reviewer bias in single- versus double-blind peer review","au":"Tomkins, Andrew","y":2017,"vn":"Proceedings of the National Academy of Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1073/pnas.1707323114","cby":608,"nref":27,"oast":"bronze","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["prestige-halo"],"pbs":"prestige-halo","rs":[],"sg":["individual-scoring"],"st":"experimental-rct","tp":"10","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"UQL4XDCH","t":"Interdisciplinarity can aid the spread of better methods between scientific communities","au":"Smaldino, Paul E.","y":2022,"vn":"Collective Intelligence","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/26339137221131816","cby":23,"nref":66,"oast":"diamond","cx":"General","co":["Bias"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"-1","cc":3,"cp":2,"self":false,"ncoefs":0},{"id":"USAZDF6Y","t":"Authors and reviewers who suffer from confirmatory bias","au":"García, J. A.","y":2016,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-016-2079-y","cby":16,"nref":25,"oast":"green","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"USLLK4XA","t":"Double entropy inter-rater agreement indices.","au":"Olenko, Andriy","y":2016,"vn":"Applied Psychological Measurement","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/0146621615592718","cby":12,"nref":29,"oast":"closed","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"4","cc":6,"cp":3,"self":false,"ncoefs":0},{"id":"UTBUWH8P","t":"Context-Aware Reviewer Assignment for Trust Enhanced Peer Review","au":"Li, Lei","y":2015,"vn":"PLoS ONE","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0130493","cby":27,"nref":24,"oast":"gold","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["reviewer-selection"],"st":"simulation-methodological","tp":"0","cc":3,"cp":4,"self":false,"ncoefs":0},{"id":"UTCS2K7S","t":"Double-blind peer review of manuscripts: opportunities, challenges, and way forward","au":"Arumugam, Ashokan","y":2020,"vn":"Physical Therapy Reviews","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/10833196.2019.1698161","cby":13,"nref":39,"oast":"bronze","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"10","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"UUUT86RV","t":"Female-dominated disciplines have lower evaluated research quality and funding success rates, for men and women.","au":"James, Alex","y":2024,"vn":"eLife","ty":"journalArticle","tyl":"Journal article","doi":"10.7554/eLife.97613","cby":6,"nref":41,"oast":"gold","cx":"General","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["funding-decision"],"st":"observational-quantitative","tp":"2","cc":4,"cp":5,"self":false,"ncoefs":0},{"id":"UX8S6N6I","t":"The future of societal impact assessment using peer review: pre-evaluation training, consensus building and inter-reviewer reliability","au":"Derrick, Gemma","y":2017,"vn":"Palgrave Communications","ty":"journalArticle","tyl":"Journal article","doi":"10.1057/palcomms.2017.40","cby":30,"nref":50,"oast":"gold","cx":"General","co":["Reliability-noise","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring","panel-consensus"],"st":"qualitative","tp":"0","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"UXZF2E9W","t":"Bias, Lotteries, and Affirmative Action in Science Funding Policy","au":"Shaw, Jamie","y":2024,"vn":"The British Journal for the Philosophy of Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1086/730218","cby":3,"nref":79,"oast":"closed","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["funding-decision"],"st":"conceptual-commentary","tp":"1","cc":4,"cp":2,"self":false,"ncoefs":0},{"id":"UYD3UVJU","t":"Engaging doctoral students in peer review: a pre-post study evaluating the effectiveness of the “Peerspectives” course on review quality, knowledge and skills","au":"Rohmann, Jessica L.","y":2025,"vn":"medRxiv","ty":"journalArticle","tyl":"Journal article","doi":"10.1101/2025.02.11.25322060","cby":2,"nref":42,"oast":"green","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":2,"cp":1,"self":false,"ncoefs":3},{"id":"UYGWPDNF","t":"The Reliability and Influence of Peer Reviews","au":"Chibnik, Michael","y":2016,"vn":"American Anthropologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/aman.12523","cby":0,"nref":9,"oast":"closed","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"3","cc":5,"cp":1,"self":false,"ncoefs":0},{"id":"V2ANRITC","t":"Open peer review: promoting transparency in open science","au":"Wolfram, Dietmar","y":2020,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-020-03488-4","cby":165,"nref":35,"oast":"hybrid","cx":"Journal","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"V5I4EX6D","t":"A Robust Community-Based Credit System to Enhance Peer Review in Scientific Research","au":"Tan, Wanpeng","y":2023,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31222/osf.io/y9qh6","cby":1,"nref":0,"oast":"gold","cx":"General","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"1","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"V5LUUSHS","t":"When zero may not be zero: A cautionary note on the use of inter‐rater reliability in evaluating grant peer review","au":"Erosheva, Elena A.","y":2021,"vn":"Journal of the Royal Statistical Society: Series A (Statistics in Society)","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/rssa.12681","cby":30,"nref":63,"oast":"hybrid","cx":"Grant","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level","value"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"V6JGCBSK","t":"What do we know about grant peer review in the health sciences?","au":"Guthrie, Susan","y":2017,"vn":"F1000Research","ty":"journalArticle","tyl":"Journal article","doi":"10.12688/f1000research.11917.1","cby":68,"nref":111,"oast":"gold","cx":"Grant","co":["Reliability-noise","Bias","Validity"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":["factors"],"sg":["whole-process"],"st":"systematic-review-meta-analysis","tp":"7","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"V6S3FITK","t":"Criteria for assessing grant applications: a systematic review","au":"Hug, Sven E.","y":2020,"vn":"Palgrave Communications","ty":"journalArticle","tyl":"Journal article","doi":"10.1057/s41599-020-0412-9","cby":47,"nref":137,"oast":"gold","cx":"Grant","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"systematic-review-meta-analysis","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":2},{"id":"V73EMDT7","t":"Peer Review and Bibliometric: Potentials and Problems","au":"Bornmann, Lutz","y":2011,"vn":"","ty":"book","tyl":"Book","doi":"10.1007/978-94-007-1116-7_8","cby":16,"nref":176,"oast":"closed","cx":"General","co":["Reliability-noise","Bias","Validity"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"book-monograph","tp":"9","cc":3,"cp":4,"self":false,"ncoefs":0},{"id":"V74GUKBP","t":"Science Evaluation: Peer Review, Bibliometrics, and Research Impact Assessment","au":"Redman, Barbara K.","y":2023,"vn":"","ty":"book","tyl":"Book","doi":"10.1007/978-3-031-27111-3_8","cby":2,"nref":71,"oast":"closed","cx":"General","co":["Reliability-noise","Bias","Validity","Interventions"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"book-monograph","tp":"9","cc":3,"cp":2,"self":false,"ncoefs":0},{"id":"V75ZGITL","t":"Computing Inter-Rater Reliability for Observational Data: An Overview and Tutorial.","au":"Hallgren, Kevin A","y":2012,"vn":"Tutorials in quantitative methods for psychology","ty":"journalArticle","tyl":"Journal article","doi":"10.20982/tqmp.08.1.p023","cby":3872,"nref":24,"oast":"diamond","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"4","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"V7CCKM45","t":"Ranking versus rating in peer review of research grant applications","au":"Tamblyn, Robyn","y":2023,"vn":"PLOS ONE","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0292306","cby":5,"nref":41,"oast":"gold","cx":"Grant","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["demographic"],"pbs":"demographic","rs":["factors"],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":6},{"id":"V8UFU356","t":"Meetings that matter: the dual benefits of panel peer review","au":"Oxley, Kristin","y":2024,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvaf047","cby":1,"nref":50,"oast":"hybrid","cx":"Grant","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["panel-consensus"],"st":"qualitative","tp":"13","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"VAJCBBM4","t":"Distortions from Unbiased Noise in Sequential Review: Why the Research Excellence Framework Needs a Noise Audit","au":"Ward, Robert","y":2026,"vn":"Quarterly Journal of Experimental Psychology","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/17470218261446939","cby":0,"nref":21,"oast":"bronze","cx":"General","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":["factors","value"],"sg":["individual-scoring","panel-consensus"],"st":"simulation-methodological","tp":"13","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"VDGEQNPW","t":"Scientific autonomy in the structural bubble: from institutional bias to AI-mediated consensus","au":"Rubenstein, Elias","y":2026,"vn":"Frontiers in Research Metrics and Analytics","ty":"journalArticle","tyl":"Journal article","doi":"10.3389/frma.2026.1766504","cby":0,"nref":34,"oast":"gold","cx":"General","co":["Bias"],"pco":"Bias","bs":["ai-llm","prestige-halo"],"pbs":"prestige-halo","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"-1","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"VDYRFLE8","t":"Ten considerations for open peer review","au":"Schmidt, Birgit","y":2018,"vn":"F1000Research","ty":"journalArticle","tyl":"Journal article","doi":"10.12688/f1000research.15334.1","cby":80,"nref":25,"oast":"gold","cx":"Journal","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"VFV5RJU5","t":"Are peer reviewers influenced by their work being cited?","au":"Barnett, Adrian","y":2025,"vn":"eLife","ty":"journalArticle","tyl":"Journal article","doi":"10.7554/elife.108748.2","cby":0,"nref":56,"oast":"gold","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"GETDEYXE","t":"Scholars as government-appointed research evaluators: Do they create congruence between their professional quality standards and political demands?","au":"Woiwode, Hendrik","y":2020,"vn":"PLoS ONE","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0239336","cby":0,"nref":53,"oast":"gold","cx":"General","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"qualitative","tp":"9","cc":3,"cp":2,"self":false,"ncoefs":0},{"id":"45K94TWP","t":"Applying psychometric theory and research to developing a continuously distributed approach to making research funding decisions.","au":"De Los Reyes, Andres","y":2012,"vn":"Review of General Psychology","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/a0027250","cby":0,"nref":10,"oast":"closed","cx":"Grant","co":["Reliability-noise","Validity","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["funding-decision"],"st":"conceptual-commentary","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"49ZXXL9X","t":"Latent Markov modeling applied to grant peer review","au":"Bornmann, Lutz","y":2008,"vn":"Journal of Informetrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.joi.2008.05.003","cby":18,"nref":40,"oast":"closed","cx":"Grant","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"2","cc":4,"cp":4,"self":false,"ncoefs":0},{"id":"5CTCSBI3","t":"Gatekeepers of science—Effects of external reviewers’ attributes on the assessments of fellowship applications","au":"Bornmann, Lutz","y":2006,"vn":"Journal of Informetrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.joi.2006.09.005","cby":44,"nref":39,"oast":"closed","cx":"Grant","co":["Bias","Validity"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["reviewer-selection","individual-scoring"],"st":"observational-quantitative","tp":"2","cc":4,"cp":4,"self":false,"ncoefs":1},{"id":"6LMXE72N","t":"Measuring Agreement for Multinomial Data","au":"Davies, Mark","y":1982,"vn":"Biometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.2307/2529886","cby":353,"nref":10,"oast":"closed","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"4","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"836GCTFJ","t":"Interrater agreement for reviews for Developmental Review","au":"Whitehurst, Grover J.","y":1983,"vn":"Developmental Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/0273-2297(83)90009-6","cby":47,"nref":13,"oast":"closed","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"3","cc":5,"cp":3,"self":false,"ncoefs":3},{"id":"9GSQFNSJ","t":"Extension of the Kappa Coefficient","au":"Kraemer, Helena C.","y":1980,"vn":"Biometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.2307/2529972","cby":237,"nref":23,"oast":"closed","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"4","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"ABE5N6J3","t":"Agreement among reviewers of review articles","au":"Oxman, Andrew D","y":1991,"vn":"Journal of Clinical Epidemiology","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/0895-4356(91)90205-n","cby":286,"nref":51,"oast":"closed","cx":"General","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"15","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"ADI4QBWF","t":"A proposed index for measuring agreement in test-retest studies","au":"Rogot, Eugene","y":1966,"vn":"Journal of Chronic Diseases","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/0021-9681(66)90032-4","cby":113,"nref":8,"oast":"closed","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"4","cc":5,"cp":0,"self":false,"ncoefs":0},{"id":"CLMQHE39","t":"Editorial Peer Review: Its Strengths and Weaknesses","au":"Hepfer, Cindy","y":2002,"vn":"Serials Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/s0098-7913(02)00181-8","cby":82,"nref":0,"oast":"closed","cx":"Journal","co":["Reliability-noise","Bias","Validity"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"-1","cc":3,"cp":0,"self":false,"ncoefs":0},{"id":"EAXQ23NK","t":"A proposal for an 'equal peer-review' statement.","au":"Moustafa, Khaled","y":2015,"vn":"Trends in pharmacological sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.tips.2015.06.001","cby":8,"nref":7,"oast":"green","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["prestige-halo"],"pbs":"prestige-halo","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"-1","cc":4,"cp":0,"self":false,"ncoefs":0},{"id":"G4TTQ7ZR","t":"High agreement but low Kappa: I. the problems of two paradoxes","au":"Feinstein, Alvan R.","y":1990,"vn":"Journal of Clinical Epidemiology","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/0895-4356(90)90158-l","cby":2903,"nref":11,"oast":"closed","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"4","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"JN8MZ48Y","t":"Peer Review Interrater Concordance of Scientific Abstracts: A Study of Anesthesiology Subspecialty and Component Societies","au":"Cohen, Ira Todd","y":2006,"vn":"Anesthesia & Analgesia","ty":"journalArticle","tyl":"Journal article","doi":"10.1213/01.ane.0000200314.73035.4d","cby":6,"nref":13,"oast":"closed","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level","factors"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"15","cc":5,"cp":6,"self":false,"ncoefs":8},{"id":"JZNXVKWD","t":"Reviewer bias against the unconventional? A randomized double-blind study of peer review.","au":"Ernst, E","y":1999,"vn":"Complementary therapies in medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/S0965-2299(99)80054-5","cby":17,"nref":31,"oast":"closed","cx":"Journal","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":["level"],"sg":["individual-scoring"],"st":"experimental-rct","tp":"11","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"K52L9EZ2","t":"Research funding: past performance is a stronger predictor of future scientific output than reviewer scores","au":"Győrffy, Balázs","y":2020,"vn":"Journal of Informetrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.joi.2020.101050","cby":70,"nref":27,"oast":"hybrid","cx":"Grant","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["individual-scoring","post-award-validity"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"KCX9AZ3T","t":"Implicit criteria weighting and evaluators' backgrounds in grant peer review","au":"Franzoni, Chiara","y":2026,"vn":"Research Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.respol.2026.105535","cby":0,"nref":39,"oast":"hybrid","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["prestige-halo"],"pbs":"prestige-halo","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":4,"cp":2,"self":false,"ncoefs":0},{"id":"KULA287G","t":"Editorial Comment: Agreement between reviewers of Journal of Economic Psychology submissions","au":"Kemp, Simon","y":2005,"vn":"Journal of Economic Psychology","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.joep.2005.05.004","cby":6,"nref":19,"oast":"closed","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"3","cc":5,"cp":3,"self":false,"ncoefs":4},{"id":"KZDA7KMW","t":"Measuring Agreement between Two Judges on the Presence or Absence of a Trait","au":"Fleiss, Joseph L.","y":1975,"vn":"Biometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.2307/2529549","cby":506,"nref":17,"oast":"closed","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"4","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"LJ3DDLY7","t":"A new approach to referees' assessments of manuscripts","au":"Hargens, Lowell L.","y":1990,"vn":"Social Science Research","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/0049-089x(90)90012-8","cby":66,"nref":27,"oast":"closed","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"3","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"MMQAABEQ","t":"Artificial intelligence in scholarly peer review: a scoping review of applications, risks, and governance challenges","au":"Nabavi, Ali","y":2026,"vn":"International Journal of Medical Informatics","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.ijmedinf.2026.106418","cby":2,"nref":112,"oast":"closed","cx":"Journal","co":["Reliability-noise","Bias","Interventions"],"pco":"Reliability-noise","bs":["ai-llm"],"pbs":"ai-llm","rs":[],"sg":["whole-process"],"st":"systematic-review-meta-analysis","tp":"6","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"P2AKBBFQ","t":"Peer review: and the winner is …","au":"Greenberg, Daniel S.","y":1999,"vn":"The Lancet","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/s0140-6736(05)76844-6","cby":5,"nref":0,"oast":"closed","cx":"Grant","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"11","cc":1,"cp":0,"self":false,"ncoefs":0},{"id":"QUNM7567","t":"Peer review in biomedical publication","au":"Ingelfinger, Franz J.","y":1974,"vn":"The American Journal of Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/0002-9343(74)90635-4","cby":243,"nref":3,"oast":"closed","cx":"Journal","co":["Reliability-noise","Bias","Validity","Interventions"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"0","cc":5,"cp":0,"self":false,"ncoefs":0},{"id":"TPJDUPCA","t":"Evaluating Grant Peer Review in the Health Sciences: A review of the literature","au":"Ismail, Sharif","y":2009,"vn":"RAND Corporation eBooks","ty":"report","tyl":"Report","doi":"10.7249/tr742","cby":19,"nref":0,"oast":"closed","cx":"Grant","co":["Reliability-noise","Bias","Validity","Interventions"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"7","cc":1,"cp":0,"self":false,"ncoefs":0},{"id":"TT68QNR5","t":"John Snow’s Grant Application","au":"Rothman, Kenneth J.","y":2016,"vn":"Epidemiology","ty":"journalArticle","tyl":"Journal article","doi":"10.1097/ede.0000000000000453","cby":6,"nref":0,"oast":"closed","cx":"Grant","co":["Bias","Validity"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["proposal-quality"],"st":"conceptual-commentary","tp":"-1","cc":1,"cp":0,"self":false,"ncoefs":0},{"id":"TUSX26VN","t":"High agreement but low kappa: II. Resolving the paradoxes","au":"Cicchetti, Domenic V.","y":1990,"vn":"Journal of Clinical Epidemiology","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/0895-4356(90)90159-m","cby":1840,"nref":17,"oast":"closed","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"4","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"5PN6GE4C","t":"Relationship between peer review quality and scientific impact: Insights from LLMs-assessed reviews","au":"Sun, Zhuanlan","y":2026,"vn":"Journal of Informetrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.joi.2026.101801","cby":0,"nref":54,"oast":"hybrid","cx":"Journal","co":["Reliability-noise","Validity"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring","post-award-validity"],"st":"observational-quantitative","tp":"6","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"667Z2XRJ","t":"The predictive validity of peer review: A selective review of the judgmental forecasting qualities of peers, and implications for innovation in science","au":"Benda, Wim G.G.","y":2010,"vn":"International Journal of Forecasting","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.ijforecast.2010.03.003","cby":58,"nref":119,"oast":"closed","cx":"General","co":["Reliability-noise","Bias","Validity"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring","post-award-validity"],"st":"conceptual-commentary","tp":"0","cc":3,"cp":4,"self":false,"ncoefs":0},{"id":"8DLAPIKQ","t":"Threats to objectivity in peer review: the case of gender","au":"Kaatz, Anna","y":2014,"vn":"Trends in Pharmacological Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.tips.2014.06.005","cby":139,"nref":21,"oast":"closed","cx":"General","co":["Bias","Interventions"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"5","cc":4,"cp":5,"self":false,"ncoefs":0},{"id":"A2LG7MLV","t":"How to detect indications of potential sources of bias in peer review: A generalized latent variable modeling approach exemplified by a gender study","au":"Bornmann, Lutz","y":2008,"vn":"Journal of Informetrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.joi.2008.09.003","cby":25,"nref":28,"oast":"closed","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring","funding-decision"],"st":"simulation-methodological","tp":"2","cc":4,"cp":4,"self":false,"ncoefs":0},{"id":"BIYZNW7P","t":"Unpacking the predictive power of peer review under scientific disruption: Evidence from AI research","au":"Zhang, Yujie","y":2026,"vn":"Journal of Informetrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.joi.2026.101811","cby":0,"nref":74,"oast":"closed","cx":"Journal","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["individual-scoring","post-award-validity"],"st":"observational-quantitative","tp":"-1","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"CN7CIUKK","t":"The big consequences of small biases: A simulation of peer review","au":"Day, Theodore E.","y":2015,"vn":"Research Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.respol.2015.01.006","cby":40,"nref":20,"oast":"closed","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring","funding-decision"],"st":"simulation-methodological","tp":"-1","cc":4,"cp":5,"self":false,"ncoefs":0},{"id":"CQTBQ2KA","t":"Selection committee membership: Service or self-service","au":"Besselaar, Peter van den","y":2012,"vn":"Journal of Informetrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.joi.2012.05.003","cby":17,"nref":18,"oast":"closed","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["reviewer-selection"],"st":"observational-quantitative","tp":"1","cc":4,"cp":2,"self":false,"ncoefs":0},{"id":"DLJNLM4I","t":"The fading of status bias during the open peer review process","au":"Sun, Zhuanlan","y":2024,"vn":"Journal of Informetrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.joi.2024.101528","cby":9,"nref":62,"oast":"closed","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["prestige-halo"],"pbs":"prestige-halo","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"EY4N66YR","t":"Large language models for peer review in biotechnology.","au":"Li, Wenyu","y":2026,"vn":"New biotechnology","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.nbt.2026.03.007","cby":1,"nref":23,"oast":"gold","cx":"General","co":["Reliability-noise","Bias","Validity"],"pco":"Reliability-noise","bs":["ai-llm"],"pbs":"ai-llm","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"6","cc":2,"cp":0,"self":false,"ncoefs":0},{"id":"FNNGDXAV","t":"Generative artificial intelligence for project evaluation: comparing large language models and human experts","au":"Santos, José M.R.C.A.","y":2026,"vn":"Procedia Computer Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.procs.2026.03.136","cby":0,"nref":26,"oast":"diamond","cx":"Grant","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["ai-llm"],"pbs":"ai-llm","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"6","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"KNAEUISZ","t":"Procedural problems with peer review, impact factor and citation index: A response to Borja and Elliott (2024)","au":"Bianchi, Carlo Nike","y":2025,"vn":"Estuarine Coastal and Shelf Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.ecss.2025.109184","cby":1,"nref":31,"oast":"closed","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["reviewer-selection","whole-process"],"st":"conceptual-commentary","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"PSNKTKAU","t":"The “invisible hand” of peer review: The implications of author-referee networks on peer review in a scholarly journal","au":"Dondio, Pierpaolo","y":2019,"vn":"Journal of Informetrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.joi.2019.03.018","cby":49,"nref":45,"oast":"hybrid","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["individual-scoring","reviewer-selection"],"st":"observational-quantitative","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"U5A9M4EQ","t":"Development and Implementation of a Peer Review Training Program","au":"Thigpen, Jonathan","y":2025,"vn":"American Journal of Pharmaceutical Education","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.ajpe.2025.101884","cby":2,"nref":12,"oast":"gold","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"U8R56WTD","t":"The effectiveness of peer review in identifying issues leading to retractions","au":"Zheng, Xiang","y":2023,"vn":"Journal of Informetrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.joi.2023.101423","cby":16,"nref":61,"oast":"closed","cx":"Journal","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"29KVHR3C","t":"A model of the editorial process in academic journals","au":"Bayar, Onur","y":2021,"vn":"Research Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.respol.2021.104339","cby":14,"nref":33,"oast":"closed","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring","funding-decision"],"st":"simulation-methodological","tp":"-1","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"JU6II4Y3","t":"Journal review and gender equality: a critical comment on Budden et al.","au":"Whittaker, Robert J.","y":2008,"vn":"Trends in Ecology & Evolution","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.tree.2008.06.003","cby":36,"nref":4,"oast":"closed","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"5","cc":4,"cp":1,"self":false,"ncoefs":0},{"id":"KFD5UHRM","t":"Comparative outcomes of face-to-face and virtual review meetings","au":"Vo, Nghia M.","y":2016,"vn":"International Journal of Surgery Open","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.ijso.2016.07.002","cby":5,"nref":5,"oast":"gold","cx":"Grant","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["panel-consensus"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"DK7Z8PE6","t":"Who Benefits from Peer Review? An Analysis of the Outcome of 100 Requests for Review by Plastic and Reconstructive Surgery","au":"Loonen, Martijn P. J.","y":2005,"vn":"Plastic & Reconstructive Surgery","ty":"journalArticle","tyl":"Journal article","doi":"10.1097/01.prs.0000178796.82273.7c","cby":27,"nref":28,"oast":"closed","cx":"Journal","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":4},{"id":"F2Z5HBK3","t":"Empowering peer reviewers with a checklist to improve transparency","au":"Parker, Timothy","y":2018,"vn":"Nature Ecology & Evolution","ty":"journalArticle","tyl":"Journal article","doi":"10.1038/s41559-018-0545-z","cby":49,"nref":32,"oast":"closed","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"guideline-policy","tp":"5","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"II5SPEMP","t":"Chance and consensus in peer review.","au":"Cole, S","y":1981,"vn":"Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1126/science.7302566","cby":624,"nref":3,"oast":"closed","cx":"Grant","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":0,"self":false,"ncoefs":0},{"id":"L47VGY36","t":"Advancing nursing education science: an analysis of the NLN's Grants Program 2008-2010.","au":"Duffy, Joanne R","y":2011,"vn":"Nursing education perspectives","ty":"journalArticle","tyl":"Journal article","doi":"10.5480/1536-5026-32.1.10","cby":4,"nref":3,"oast":"closed","cx":"Grant","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"9ARNWGFR","t":"Halo Effect in Peer Review: Exploring the Possibility of Bias Associated with the Feeling of Belonging to a Group","au":"Sordi, José Osvaldo De","y":2019,"vn":"Perspectivas em Ciência da Informação","ty":"journalArticle","tyl":"Journal article","doi":"10.1590/1981-5344/3860","cby":2,"nref":31,"oast":"diamond","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["prestige-halo"],"pbs":"prestige-halo","rs":[],"sg":["individual-scoring"],"st":"qualitative","tp":"-1","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"38BW3IRN","t":"What does better peer review look like? Definitions, essential areas, and recommendations for better practice","au":"Allen, Heidi","y":2018,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31219/osf.io/4mfk2","cby":4,"nref":0,"oast":"gold","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"guideline-policy","tp":"0","cc":2,"cp":0,"self":false,"ncoefs":0},{"id":"4EXX72HE","t":"A Multilevel Modelling Approach to Investigating the Predictive Validity of Editorial Decisions: Do the Editors of a High Profile Journal Select Manuscripts that are Highly Cited After Publication?","au":"Bornmann, Lutz","y":2011,"vn":"Journal of the Royal Statistical Society Series A (Statistics in Society)","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/j.1467-985x.2011.00689.x","cby":89,"nref":92,"oast":"bronze","cx":"Journal","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["funding-decision","post-award-validity"],"st":"observational-quantitative","tp":"-1","cc":3,"cp":4,"self":false,"ncoefs":0},{"id":"5QKAGHIT","t":"Experimenting with Grant Peer Review: A Mixed Methods Case Study of the Effects on Time Use and the Quality of Reviewing","au":"Besselaar, Peter van den","y":2026,"vn":"Publications","ty":"journalArticle","tyl":"Journal article","doi":"10.3390/publications14020033","cby":0,"nref":50,"oast":"gold","cx":"Grant","co":["Validity","Interventions"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["panel-consensus","individual-scoring"],"st":"observational-quantitative","tp":"8","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"648JUZU3","t":"Moving to the Double-Blind Review System","au":"Boeck, Paul De","y":2015,"vn":"Journal of Intelligence","ty":"journalArticle","tyl":"Journal article","doi":"10.3390/jintelligence3040158","cby":0,"nref":1,"oast":"gold","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["prestige-halo"],"pbs":"prestige-halo","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"10","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"6NFKSA2S","t":"Peer review in megajournals compared with traditional scholarly journals: Does it make a difference?","au":"Björk, Bo‐Christer","y":2016,"vn":"Learned Publishing","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/leap.1007","cby":60,"nref":8,"oast":"hybrid","cx":"Journal","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"-1","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"85KL54GC","t":"Judging research papers for research excellence","au":"Tymms, Peter","y":2017,"vn":"Studies in Higher Education","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/03075079.2016.1266609","cby":9,"nref":19,"oast":"green","cx":"General","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"9","cc":5,"cp":4,"self":false,"ncoefs":0},{"id":"8JJIF6SZ","t":"Where next for partial randomisation of research funding? The feasibility of RCTs and alternatives","au":"Stafford, Tom","y":2024,"vn":"Wellcome Open Research","ty":"journalArticle","tyl":"Journal article","doi":"10.12688/wellcomeopenres.19565.2","cby":0,"nref":41,"oast":"gold","cx":"Grant","co":["Validity","Interventions"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["funding-decision"],"st":"conceptual-commentary","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"9CDDJRXJ","t":"The association of gender, experience, and academic rank in peer-reviewed manuscript evaluation","au":"García, J. A.","y":2024,"vn":"Accountability in Research","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/08989621.2024.2440098","cby":5,"nref":63,"oast":"green","cx":"Journal","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["demographic","prestige-halo"],"pbs":"demographic","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"5","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"A8TZ4TZK","t":"A Psychometric Analysis and Primer for Decision Making in Public Third-Party Funding Allocation","au":"Rahal, Rima-Maria","y":2025,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31234/osf.io/mp8y2_v3","cby":0,"nref":0,"oast":"gold","cx":"Grant","co":["Bias","Validity"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"1","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"BGW9Q5PA","t":"Applicants as reviewers: A mixed methods evaluation of the risks, benefits, and potential of distributed peer review for grant funding allocations","au":"Butters, Anna","y":2026,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31222/osf.io/t2p56_v2","cby":1,"nref":0,"oast":"gold","cx":"Grant","co":["Reliability-noise","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level","factors"],"sg":["reviewer-selection","panel-consensus","funding-decision"],"st":"observational-quantitative","tp":"8","cc":1,"cp":0,"self":false,"ncoefs":2},{"id":"C5VVQXIS","t":"Where are the carrots? A proposal to start crediting the peer reviewers for their contribution to science","au":"Laxdal, Aron Gauti","y":2023,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31219/osf.io/7c5xr","cby":4,"nref":0,"oast":"gold","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"CSH8NIEK","t":"Little race or gender bias in an experiment of initial review of NIH R01 grant proposals.","au":"Forscher, Patrick S","y":2019,"vn":"Nature Human Behaviour","ty":"journalArticle","tyl":"Journal article","doi":"10.1038/s41562-018-0517-y","cby":44,"nref":27,"oast":"green","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring"],"st":"experimental-rct","tp":"14","cc":4,"cp":5,"self":false,"ncoefs":0},{"id":"F8ULRBVB","t":"Open Peer Review: Fast Forward for a New Science","au":"Hachani, Samir","y":2015,"vn":"","ty":"book","tyl":"Book","doi":"10.1108/s0065-283020150000039012","cby":8,"nref":38,"oast":"green","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"book-monograph","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"GPSLVGVP","t":"Mitigating Negativity in Science Funding: The Role of Three-Step Procedures and Group Decision-Making","au":"Hottenrott, Hanna","y":2025,"vn":"SSRN Electronic Journal","ty":"journalArticle","tyl":"Journal article","doi":"10.2139/ssrn.5379818","cby":0,"nref":32,"oast":"green","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring","panel-consensus"],"st":"observational-quantitative","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"GWDJVJX5","t":"When do experts listen to other experts? the role of negative information in expert evaluations for novel projects","au":"Lane, Jacqueline","y":2020,"vn":"SSRN Electronic Journal","ty":"journalArticle","tyl":"Journal article","doi":"10.2139/ssrn.3656495","cby":4,"nref":82,"oast":"green","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["panel-consensus"],"st":"experimental-rct","tp":"-1","cc":3,"cp":2,"self":false,"ncoefs":0},{"id":"H9DZ6W5V","t":"Research Conducted in Women Was Deemed More Impactful but Less Publishable than the Same Research Conducted in Men","au":"Murrar, Sohad","y":2021,"vn":"Journal of Women s Health","ty":"journalArticle","tyl":"Journal article","doi":"10.1089/jwh.2020.8666","cby":37,"nref":55,"oast":"green","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring"],"st":"experimental-rct","tp":"5","cc":4,"cp":5,"self":false,"ncoefs":0},{"id":"IMRJ7IJ3","t":"Organization of research project competitions in the Republic of Moldova: lottery is better than incompetence and partiality","au":"Moldova, Agentia Nationala de Asigurare a Calitatii in Educatie si Cercetare, Republica","y":2021,"vn":"Akademos","ty":"journalArticle","tyl":"Journal article","doi":"10.52673/18570461.21.2-61.01","cby":0,"nref":72,"oast":"hybrid","cx":"Grant","co":["Reliability-noise","Bias","Validity","Interventions"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["funding-decision"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"J4QQRCKM","t":"Variables Affecting Evaluation and Publication of Oncology Case Reports: A Systematic Analysis","au":"Florita, Antonio","y":2016,"vn":"The International Journal of Biological Markers","ty":"journalArticle","tyl":"Journal article","doi":"10.5301/jbm.5000237","cby":1,"nref":51,"oast":"hybrid","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["funding-decision"],"st":"observational-quantitative","tp":"11","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"MRCX6QN8","t":"Publications as a measure of scientific advancement and of scientists' productivity","au":"Daniel, Hans‐Dieter","y":2005,"vn":"Learned Publishing","ty":"journalArticle","tyl":"Journal article","doi":"10.1087/0953151053584939","cby":53,"nref":21,"oast":"bronze","cx":"Journal","co":["Reliability-noise","Bias","Validity"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":3,"cp":4,"self":false,"ncoefs":0},{"id":"Q5KUVRG8","t":"Why Not Open the Black Box of Journal Editing in Philosophy? Make Peer Reviews of Published Papers Available","au":"Muckadell, Caroline Schaffalitzky de","y":2017,"vn":"Metaphilosophy","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/meta.12240","cby":6,"nref":15,"oast":"green","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"QMJ45GTT","t":"Peer review training: A catalyst for quality Physiotherapy research","au":"Kasi, Madhuri","y":2025,"vn":"Physiotherapy - The Journal of Indian Association of Physiotherapists","ty":"journalArticle","tyl":"Journal article","doi":"10.4103/pjiap.pjiap_48_25","cby":0,"nref":9,"oast":"hybrid","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["reviewer-selection","individual-scoring"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"U5QL8TKC","t":"Rubrics, Not Vibes: Structured AI Integration as Quality Control for Peer Review","au":"Turner, Stephen D.","y":2026,"vn":"SSRN Electronic Journal","ty":"journalArticle","tyl":"Journal article","doi":"10.2139/ssrn.6314421","cby":0,"nref":17,"oast":"green","cx":"General","co":["Reliability-noise","Bias","Interventions"],"pco":"Reliability-noise","bs":["ai-llm"],"pbs":"ai-llm","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"6","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"UBXPPIM6","t":"Ensuring Sustainable Evaluation: How to Improve Quality of Evaluating Grant Proposals?","au":"Wieczorkowska, Grażyna","y":2021,"vn":"Sustainability","ty":"journalArticle","tyl":"Journal article","doi":"10.3390/su13052842","cby":4,"nref":45,"oast":"gold","cx":"Grant","co":["Reliability-noise","Bias","Interventions"],"pco":"Reliability-noise","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":["level","factors"],"sg":["individual-scoring"],"st":"experimental-rct","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"UUS7MZZX","t":"Reviewers’ Perceptions of the Peer Review Process in Journalism and Mass Communication","au":"Curtin, Patricia A.","y":2017,"vn":"Journalism & Mass Communication Quarterly","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/1077699017736031","cby":16,"nref":37,"oast":"bronze","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"CTHBPG4W","t":"Gender and Geographic Origin as Determinants of Manuscript Publication Outcomes: JBMR® Bibliometric Analysis from 2017 to 2019","au":"Rivadeneira, Fernando","y":2020,"vn":"Journal of Bone and Mineral Research","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/jbmr.4696","cby":3,"nref":39,"oast":"green","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"5","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"F7BJWUEL","t":"Peer Review Agreement or Peer Review Disagreement: Which Is Better?","au":"Hemlin, Sven","y":2009,"vn":"Journal of Psychology of Science and Technology","ty":"journalArticle","tyl":"Journal article","doi":"10.1891/1939-7054.2.1.5","cby":11,"nref":0,"oast":"closed","cx":"General","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":["level"],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"0","cc":4,"cp":0,"self":false,"ncoefs":0},{"id":"LLZCSWHH","t":"A New Paradigm for Scientific Publishing, Peer Review, and Impact Assessment","au":"Argibay, Nicolas","y":2026,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31222/osf.io/gcdqf_v1","cby":0,"nref":0,"oast":"gold","cx":"General","co":["Bias","Validity","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"-1","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"4BPNLHNQ","t":"Validation of the CoRE Questionnaire for a Medical Journal Peer Review","au":"Doi, Suhail A.R.","y":2015,"vn":"Accountability in Research","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/08989621.2014.1002835","cby":2,"nref":3,"oast":"closed","cx":"Journal","co":["Reliability-noise","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level","value"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":3,"cp":0,"self":false,"ncoefs":2},{"id":"6S73VTHR","t":"Promoting equity in the peer review process of journal publication","au":"Bancroft, Senetta F.","y":2022,"vn":"Science Education","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/sce.21733","cby":19,"nref":36,"oast":"closed","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"5","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"7GZFFKLQ","t":"Triple-blind review as a solution to gender bias in academic publishing, a theoretical approach","au":"Conklin, Michael","y":2022,"vn":"Studies in Higher Education","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/03075079.2022.2081681","cby":17,"nref":31,"oast":"closed","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"5","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"8IUUWWR8","t":"Ein Blick in die Black Box des Review-Verfahrens","au":"Otte, Gunnar","y":2019,"vn":"Zeitschrift für Soziologie","ty":"journalArticle","tyl":"Journal article","doi":"10.1515/zfsoz-2019-0001","cby":5,"nref":8,"oast":"closed","cx":"Journal","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"-1","cc":0,"cp":1,"self":false,"ncoefs":0},{"id":"96KKLHBV","t":"Battling bias at NIH.","au":"Sherley, James L","y":2012,"vn":"Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1126/science.337.6094.523-b","cby":0,"nref":0,"oast":"closed","cx":"Grant","co":["Bias","Interventions"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring","funding-decision"],"st":"conceptual-commentary","tp":"14","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"97SX9MRQ","t":"Peer review at National Institutes of Health: Small steps forward.","au":"Johnston, S. Claiborne","y":2008,"vn":"Annals of Neurology","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/ana.21260","cby":2,"nref":0,"oast":"closed","cx":"Grant","co":["Reliability-noise","Bias","Validity","Interventions"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":["factors"],"sg":["whole-process"],"st":"conceptual-commentary","tp":"7","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"BDU9ZWUC","t":"The bibliometric indicators as predictors of the final decision of the peer review","au":"Vieira, Elizabeth S.","y":2015,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvv037","cby":7,"nref":38,"oast":"closed","cx":"Grant","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["funding-decision"],"st":"observational-quantitative","tp":"9","cc":4,"cp":4,"self":false,"ncoefs":0},{"id":"BZH4FMTG","t":"Does “Decision Fatigue” Impact Manuscript Acceptance? An Analysis of Editorial Decisions by the American Journal of Gastroenterology","au":"Kwan, James","y":2016,"vn":"The American Journal of Gastroenterology","ty":"journalArticle","tyl":"Journal article","doi":"10.1038/ajg.2016.452","cby":6,"nref":11,"oast":"closed","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"3","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"DRNLYALD","t":"The Air We Breathe: A Critical Look at Practices and Alternatives in the Peer-Review Process","au":"Suls, Jerry","y":2009,"vn":"Perspectives on Psychological Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/j.1745-6924.2009.01105.x","cby":85,"nref":80,"oast":"closed","cx":"Journal","co":["Reliability-noise","Bias","Validity","Interventions"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"-1","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"DTEQFJLY","t":"Double-Blind Review in Software Engineering Venues: The Community's Perspective","au":"Bacchelli, Alberto","y":2017,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.1109/icse-c.2017.49","cby":13,"nref":57,"oast":"green","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"10","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"HHTIHQYC","t":"Peer review in the development of academic articles: Experiences of Finnish authors in the educational sciences","au":"Atjonen, Päivi","y":2018,"vn":"Learned Publishing","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/leap.1204","cby":7,"nref":44,"oast":"closed","cx":"Journal","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"IZUPZGW2","t":"Rebound Peer Review: A Viable Recourse for Aggrieved Authors?","au":"Sen, Chandan K.","y":2011,"vn":"Antioxidants and Redox Signaling","ty":"journalArticle","tyl":"Journal article","doi":"10.1089/ars.2011.4424","cby":10,"nref":23,"oast":"closed","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["prestige-halo"],"pbs":"prestige-halo","rs":[],"sg":["funding-decision"],"st":"conceptual-commentary","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"LFSAIIW3","t":"The policy challenges of peer review: managing bias, conflict of interests and interdisciplinary assessments","au":"Langfeldt, Liv","y":2006,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.3152/147154406781776039","cby":180,"nref":0,"oast":"closed","cx":"Grant","co":["Reliability-noise","Bias","Interventions"],"pco":"Reliability-noise","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":["factors"],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":1,"cp":0,"self":false,"ncoefs":0},{"id":"LIK4IE3M","t":"Refinement: Measuring informativeness of ratings in the absence of a gold standard","au":"Grant, Sheridan","y":2022,"vn":"British Journal of Mathematical and Statistical Psychology","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/bmsp.12268","cby":2,"nref":65,"oast":"closed","cx":"Grant","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"8","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"N77BA9EY","t":"Editorial Judgments","au":"Hirschauer, Stefan","y":2009,"vn":"Social Studies of Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/0306312709335405","cby":116,"nref":48,"oast":"closed","cx":"Journal","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["funding-decision"],"st":"qualitative","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"P9WMK7AM","t":"National institutes of health: The lottery.","au":"Heilman, Kenneth M","y":2007,"vn":"Annals of Neurology","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/ana.21133","cby":1,"nref":6,"oast":"closed","cx":"Grant","co":["Reliability-noise","Bias","Validity"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"7","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"PH3ATIG8","t":"Is three better than one? simulating the effect of reviewer selection and behavior on the quality and efficiency of peer review","au":"Bianchi, Federico","y":2015,"vn":"2015 Winter Simulation Conference (WSC)","ty":"journalArticle","tyl":"Journal article","doi":"10.1109/wsc.2015.7408561","cby":12,"nref":19,"oast":"green","cx":"Journal","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["reviewer-selection","individual-scoring"],"st":"simulation-methodological","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"PI3PZFJZ","t":"What We Still Don't Know About Peer Review","au":"Sabaj, Omar","y":2016,"vn":"Journal of Scholarly Publishing","ty":"journalArticle","tyl":"Journal article","doi":"10.3138/jsp.47.2.180","cby":40,"nref":142,"oast":"closed","cx":"General","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"systematic-review-meta-analysis","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"PL3YJB2Z","t":"Peer review research assessment: are the reviewers really experts?","au":"Abramo, Giovanni","y":2024,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvaf043","cby":3,"nref":28,"oast":"closed","cx":"General","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["reviewer-selection"],"st":"observational-quantitative","tp":"9","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"QMMNS2MV","t":"Research Note—Making Peer Review Evidence-Based: It’s Time to Open the “Black Box”","au":"Dunleavy, Daniel J.","y":2024,"vn":"Journal of Social Work Education","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/10437797.2024.2378685","cby":1,"nref":103,"oast":"closed","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"QWVEZ7QZ","t":"On the Nature and Role of Peer Review in Mathematics","au":"Andersen, Line Edslev","y":2016,"vn":"Accountability in Research","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/08989621.2016.1274885","cby":29,"nref":42,"oast":"closed","cx":"Journal","co":["Reliability-noise","Validity"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring","reviewer-selection"],"st":"qualitative","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"2CHZ5LDK","t":"Guardians of Science: Fairness and Reliability of Peer Review","au":"Fletcher, Sara","y":1994,"vn":"BMJ","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmj.309.6952.488","cby":28,"nref":0,"oast":"closed","cx":"Journal","co":["Reliability-noise","Bias","Validity"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":3,"cp":0,"self":false,"ncoefs":0},{"id":"2IQ63KYT","t":"New Light on Old Boys: Cognitive and Institutional Particularism in the Peer Review System","au":"Travis, G.D.L.","y":1991,"vn":"Science, technology & human values","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/016224399101600303","cby":247,"nref":20,"oast":"closed","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["reviewer-selection","individual-scoring"],"st":"qualitative","tp":"1","cc":3,"cp":3,"self":false,"ncoefs":0},{"id":"3MQGCJCA","t":"How blind is blind review?","au":"Yankauer, Alfred","y":1991,"vn":"American Journal of Public Health","ty":"journalArticle","tyl":"Journal article","doi":"10.2105/ajph.81.7.843","cby":65,"nref":11,"oast":"green","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"10","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"3PY7SBW2","t":"Peer Review for Journals as it Stands Today—Part 1","au":"Campanario, Juan Miguel","y":1998,"vn":"Science Communication","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/1075547098019003002","cby":200,"nref":217,"oast":"closed","cx":"Journal","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["reviewer-selection","individual-scoring","whole-process"],"st":"conceptual-commentary","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"8TI3P7Z6","t":"Variation in journal peer review systems. Possible causes and consequences.","au":"Hargens, Lowell L.","y":1990,"vn":"PubMed","ty":"journalArticle","tyl":"Journal article","doi":"","cby":81,"nref":12,"oast":"closed","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["funding-decision"],"st":"observational-quantitative","tp":"-1","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"B4FTVF6S","t":"Do Journal Rejection Rates Index Consensus?","au":"Cole, Stephen","y":1988,"vn":"American Sociological Review","ty":"journalArticle","tyl":"Journal article","doi":"10.2307/2095740","cby":66,"nref":11,"oast":"closed","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["funding-decision"],"st":"conceptual-commentary","tp":"-1","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"BCKFAVIX","t":"Scholarly Consensus and Journal Rejection Rates","au":"Hargens, Lowell L.","y":1988,"vn":"American Sociological Review","ty":"journalArticle","tyl":"Journal article","doi":"10.2307/2095739","cby":239,"nref":21,"oast":"closed","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["funding-decision"],"st":"observational-quantitative","tp":"-1","cc":3,"cp":3,"self":false,"ncoefs":1},{"id":"CG7WES8M","t":"The nonuse of psychological research at two federal agencies.","au":"Arkes, Hal R","y":2003,"vn":"Psychological science","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/1467-9280.01410","cby":58,"nref":28,"oast":"closed","cx":"Grant","co":["Reliability-noise","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"7","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"E3GA6AEC","t":"What is the source of bias in peer review?","au":"Over, Ray","y":1982,"vn":"Behavioral and Brain Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/s0140525x00011559","cby":47,"nref":198,"oast":"closed","cx":"Journal","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"12","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"EIL6RVZV","t":"Peer reviewing: Improve or be rejected","au":"Howe, Michael J. A.","y":1982,"vn":"Behavioral and Brain Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/s0140525x00011456","cby":41,"nref":199,"oast":"closed","cx":"Journal","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"12","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"FPDF4D3R","t":"Peer review and editorial decision-making","au":"Howard, Louise M.","y":1998,"vn":"The British Journal of Psychiatry","ty":"journalArticle","tyl":"Journal article","doi":"10.1192/bjp.173.2.110","cby":42,"nref":26,"oast":"closed","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level","factors"],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"3","cc":3,"cp":1,"self":false,"ncoefs":3},{"id":"HP47KBFE","t":"On peer review: “We have met the enemy and he is us”","au":"Cicchetti, Domenic V.","y":1982,"vn":"Behavioral and Brain Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/s0140525x00011286","cby":47,"nref":198,"oast":"closed","cx":"Journal","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"12","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"IBG58DEY","t":"Theoretical implications of failure to detect prepublished submissions","au":"Eckberg, Douglas Lee","y":1982,"vn":"Behavioral and Brain Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/s0140525x0001133x","cby":38,"nref":199,"oast":"closed","cx":"Journal","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"12","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"KKY4QUVR","t":"The Manuscript Review and Decision-Making Process","au":"Bakanic, Von","y":1987,"vn":"American Sociological Review","ty":"journalArticle","tyl":"Journal article","doi":"10.2307/2095599","cby":203,"nref":22,"oast":"closed","cx":"Journal","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"0","cc":3,"cp":3,"self":false,"ncoefs":0},{"id":"QTGZWS97","t":"Reflections on the peer review process","au":"Marsh, Herbert W.","y":1991,"vn":"Behavioral and Brain Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/s0140525x00065912","cby":19,"nref":364,"oast":"closed","cx":"General","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"12","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"RXDUVK5E","t":"Manuscript evaluation by journal referees and editors: Randomness or bias?","au":"Colman, Andrew M.","y":1982,"vn":"Behavioral and Brain Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/s0140525x00011298","cby":50,"nref":199,"oast":"closed","cx":"Journal","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"12","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"SDDU5JNV","t":"Confirmational Response Bias Among Social Work Journals","au":"Epstein, William","y":1990,"vn":"Science Technology & Human Values","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/016224399001500102","cby":143,"nref":18,"oast":"closed","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"experimental-rct","tp":"0","cc":3,"cp":3,"self":false,"ncoefs":0},{"id":"U2FK4Y5H","t":"US and Non-US Submissions","au":"Link, Ann Marie","y":1998,"vn":"JAMA","ty":"journalArticle","tyl":"Journal article","doi":"10.1001/jama.280.3.246","cby":323,"nref":3,"oast":"closed","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"3","cc":2,"cp":0,"self":false,"ncoefs":0},{"id":"8GRXGMM9","t":"The Effects of Blinding on the Quality of Peer Review","au":"Mcnutt, Robert","y":1990,"vn":"JAMA","ty":"journalArticle","tyl":"Journal article","doi":"10.1001/jama.1990.03440100079012","cby":258,"nref":0,"oast":"closed","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"experimental-rct","tp":"10","cc":2,"cp":0,"self":false,"ncoefs":0},{"id":"QIRA4ABW","t":"Reviewer Bias","au":"Ernst, E","y":1992,"vn":"Annals of Internal Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.7326/0003-4819-116-11-958_2","cby":22,"nref":5,"oast":"closed","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":2,"cp":0,"self":false,"ncoefs":0},{"id":"2PDM7FYG","t":"Peer-review research: Objections and obligations","au":"Peters, Douglas","y":1982,"vn":"Behavioral and Brain Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/s0140525x00011754","cby":27,"nref":199,"oast":"closed","cx":"Journal","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"12","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"6BRBUR5I","t":"Peer Review and Manuscript Management in Scientific Journals","au":"Irene, Hames","y":2007,"vn":"","ty":"book","tyl":"Book","doi":"10.1002/9780470750803","cby":184,"nref":0,"oast":"closed","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"book-monograph","tp":"0","cc":3,"cp":0,"self":false,"ncoefs":0},{"id":"848B4KFS","t":"Estimation of the Reliability of Ratings","au":"Ebel, Robert L.","y":1951,"vn":"Psychometrika","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/bf02288803","cby":986,"nref":8,"oast":"closed","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"simulation-methodological","tp":"4","cc":5,"cp":0,"self":false,"ncoefs":0},{"id":"8RNI7EWX","t":"Taming complexity: narrative CVs in grant funding evaluations (RoRI Working Paper No. 14)","au":"Research on Research Institute (RoRI)","y":2025,"vn":"Figshare","ty":"report","tyl":"Report","doi":"10.6084/m9.figshare.27291537.v4","cby":0,"nref":0,"oast":"green","cx":"Grant","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["proposal-quality","panel-consensus"],"st":"qualitative","tp":"-1","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"92UR7R8Y","t":"Publish or Politic: Referee Bias in Manuscript Review1","au":"Abramowitz, Stephen I.","y":1975,"vn":"Journal of Applied Social Psychology","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/j.1559-1816.1975.tb00675.x","cby":137,"nref":29,"oast":"closed","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["individual-scoring"],"st":"experimental-rct","tp":"-1","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"9C2AQMRD","t":"Statistical Methods for Rates and Proportions","au":"Fleiss, Joseph L.","y":2003,"vn":"Wiley series in probability and statistics","ty":"book","tyl":"Book","doi":"10.1002/0471445428","cby":18058,"nref":0,"oast":"closed","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"book-monograph","tp":"4","cc":5,"cp":0,"self":false,"ncoefs":0},{"id":"BMWJHC4K","t":"Peer-review practices of psychological journals: The fate of published articles, submitted again","au":"Peters, Douglas P.","y":1982,"vn":"Behavioral and Brain Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/S0140525X00011183","cby":1083,"nref":204,"oast":"closed","cx":"Journal","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"3","cc":3,"cp":3,"self":false,"ncoefs":0},{"id":"D32A3NXV","t":"The experimental research funder’s handbook (RoRI Working Paper No.6)","au":"Bendiscioli, Sandra","y":2021,"vn":"Open MIND","ty":"report","tyl":"Report","doi":"10.6084/m9.figshare.17102426.v1","cby":4,"nref":0,"oast":"green","cx":"Grant","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"guideline-policy","tp":"-1","cc":1,"cp":0,"self":false,"ncoefs":0},{"id":"FZ3ZTGAR","t":"Questioning peer review: Understanding peer review system-wide (RoRI Working Paper No. 21)","au":"Gregory, Kathleen","y":2026,"vn":"Figshare","ty":"report","tyl":"Report","doi":"10.6084/m9.figshare.32685546.v4","cby":0,"nref":0,"oast":"green","cx":"General","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"qualitative","tp":"0","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"H7D6ZSQU","t":"The Marks of Examiners.","au":"S., F.","y":1937,"vn":"Journal Of The Royal Statistical Society","ty":"journalArticle","tyl":"Journal article","doi":"10.2307/2980288","cby":98,"nref":0,"oast":"closed","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"-1","cc":5,"cp":0,"self":false,"ncoefs":0},{"id":"K6QJQFGS","t":"Editorial : Pourquoi l'évaluation par les pairs s’impose - Why peer review is needed","au":"This, Hervé","y":2023,"vn":"Notes Académiques de l Académie d agriculture de France / Academic Notes of the French Academy of Agriculture","ty":"journalArticle","tyl":"Journal article","doi":"10.58630/pubac.not.690522","cby":0,"nref":12,"oast":"bronze","cx":"Journal","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"-1","cc":4,"cp":0,"self":false,"ncoefs":0},{"id":"MBCQSMRJ","t":"Applicants as Reviewers: Evaluating the Risks, Benefits, and Potential of Distributed Peer Review for Grant Funding Allocations  (RoRI Working Paper No. 17)","au":"Research on Research Institute (RoRI)","y":2026,"vn":"Figshare","ty":"report","tyl":"Report","doi":"10.6084/m9.figshare.29994841.v4","cby":0,"nref":0,"oast":"green","cx":"Grant","co":["Reliability-noise","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["reviewer-selection","funding-decision"],"st":"observational-quantitative","tp":"-1","cc":0,"cp":0,"self":false,"ncoefs":2},{"id":"N6R93HK3","t":"Manuscript Characteristics Which Influence Acceptance for Management and Social Science Journals.","au":"Kerr, S. J.","y":1977,"vn":"Academy of Management Journal","ty":"journalArticle","tyl":"Journal article","doi":"10.2307/255467","cby":239,"nref":0,"oast":"closed","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"-1","cc":5,"cp":0,"self":false,"ncoefs":0},{"id":"NZSRHRPT","t":"Peer Review: Quality Control of Applied Social Research","au":"Noble, John H.","y":1974,"vn":"Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1126/science.185.4155.916","cby":55,"nref":16,"oast":"closed","cx":"Grant","co":["Reliability-noise","Bias","Interventions"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":5,"cp":0,"self":false,"ncoefs":0},{"id":"QR6S3V6J","t":"Rejection, rebuttal, revision: Some flexible features of peer review","au":"Rubin, Donald B.","y":1982,"vn":"Behavioral and Brain Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/s0140525x00011638","cby":37,"nref":200,"oast":"closed","cx":"Journal","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"12","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"RIVJNS9P","t":"Where next for partial randomisation of research funding? The feasibility of RCTs and alternatives (RoRI Working Paper No.9)","au":"Stafford, Tom","y":2022,"vn":"Figshare","ty":"report","tyl":"Report","doi":"10.6084/m9.figshare.20101118.v1","cby":0,"nref":0,"oast":"green","cx":"Grant","co":["Validity","Interventions"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["funding-decision","whole-process"],"st":"conceptual-commentary","tp":"1","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"S4NYFMCH","t":"Reviewer “bias”: Do Peters and Ceci protest too much?","au":"Perlman, Daniel","y":1982,"vn":"Behavioral and Brain Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/s0140525x00011572","cby":17,"nref":199,"oast":"closed","cx":"Journal","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"12","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"SWUKRL6L","t":"Peer review in funding organizations: An analytical literature review (RoRI Working Paper No.11)","au":"Gregory, Kathleen","y":2024,"vn":"Figshare","ty":"report","tyl":"Report","doi":"10.6084/m9.figshare.26861680.v5","cby":0,"nref":0,"oast":"green","cx":"Grant","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"systematic-review-meta-analysis","tp":"0","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"T2KC2YGE","t":"Why draw lots? Funder motivations for using partial randomisation to allocate research grants (RoRI Working Paper No.7)","au":"Woods, Helen Buckley","y":2021,"vn":"Figshare","ty":"report","tyl":"Report","doi":"10.6084/m9.figshare.17102495.v2","cby":0,"nref":0,"oast":"green","cx":"Grant","co":["Validity","Interventions"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["funding-decision"],"st":"qualitative","tp":"1","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"TKIAN9Z9","t":"Why Are Research Grant Applications Disapproved?","au":"Allen, Ernest M.","y":1960,"vn":"Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1126/science.132.3439.1532","cby":77,"nref":0,"oast":"closed","cx":"Grant","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["proposal-quality","funding-decision"],"st":"observational-quantitative","tp":"-1","cc":5,"cp":0,"self":false,"ncoefs":0},{"id":"VAJTKYSK","t":"Perhaps it was right to reject the resubmitted manuscripts","au":"Thomas, Garth J.","y":1982,"vn":"Behavioral and Brain Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1017/s0140525x00011675","cby":41,"nref":201,"oast":"closed","cx":"Journal","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"12","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"WR6XY2LQ","t":"Statistical significance criteria for the rWG and average deviation interrater agreement indices.","au":"Smith-Crowe, Kristin","y":2014,"vn":"The Journal of applied psychology","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/a0034556","cby":66,"nref":65,"oast":"closed","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"4","cc":6,"cp":3,"self":false,"ncoefs":0},{"id":"W3TP344G","t":"Rating intervals: an experiment in peer review.","au":"Green, J G","y":1989,"vn":"The FASEB Journal","ty":"journalArticle","tyl":"Journal article","doi":"10.1096/fasebj.3.8.2721858","cby":13,"nref":0,"oast":"closed","cx":"Grant","co":["Reliability-noise","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring","panel-consensus"],"st":"experimental-rct","tp":"-1","cc":1,"cp":0,"self":false,"ncoefs":0},{"id":"WU4XSEWV","t":"Effect of using reporting guidelines during peer review on quality of final manuscripts submitted to a biomedical journal: masked randomised trial.","au":"Cobo, E","y":2011,"vn":"BMJ (Clinical Research Ed.)","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmj.d6783","cby":297,"nref":39,"oast":"hybrid","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["proposal-quality","individual-scoring"],"st":"experimental-rct","tp":"11","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"XUCCE9SV","t":"Calibration with confidence: a principled method for panel assessment.","au":"MacKay, R S","y":2017,"vn":"Royal Society Open Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1098/rsos.160760","cby":16,"nref":25,"oast":"gold","cx":"General","co":["Reliability-noise","Bias","Interventions"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring","panel-consensus"],"st":"simulation-methodological","tp":"4","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"XIFAVINM","t":"Tools used to assess the quality of peer review reports: a methodological systematic review.","au":"Superchi, Cecilia","y":2019,"vn":"BMC Medical Research Methodology","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s12874-019-0688-x","cby":111,"nref":62,"oast":"gold","cx":"Journal","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"systematic-review-meta-analysis","tp":"-1","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"WXYAW7F6","t":"Examining the value added by committee discussion in the review of applications for research awards","au":"Obrecht, Michael","y":2007,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.3152/095820207X223785","cby":42,"nref":0,"oast":"closed","cx":"Grant","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring","panel-consensus"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":0,"self":false,"ncoefs":0},{"id":"VQLA65TQ","t":"On average deviation indices for estimating interrater agreement","au":"Burke, Michael J.","y":1999,"vn":"Organizational Research Methods","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/109442819921004","cby":457,"nref":33,"oast":"closed","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"4","cc":6,"cp":3,"self":false,"ncoefs":0},{"id":"Y26RHK69","t":"Effects of training on quality of peer review: randomised controlled trial.","au":"Schroter, Sara","y":2004,"vn":"BMJ (Clinical Research Ed.)","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmj.38023.700775.AE","cby":269,"nref":18,"oast":"bronze","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":["factors"],"sg":["individual-scoring"],"st":"experimental-rct","tp":"-1","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"WV6LSXZU","t":"Disparities in ratings of internal and external applicants: A case for model-based inter-rater reliability.","au":"Martinková, Patrícia","y":2018,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0203002","cby":17,"nref":57,"oast":"gold","cx":"Analogue","co":["Reliability-noise","Bias","Validity"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":["level","factors"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":6,"cp":2,"self":false,"ncoefs":0},{"id":"WSUJ7BB2","t":"How Criterion Scores Predict the Overall Impact Score and Funding Outcomes for National Institutes of Health Peer-Reviewed Applications.","au":"Eblen, Matthew K","y":2016,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0155060","cby":43,"nref":17,"oast":"gold","cx":"Grant","co":["Bias","Validity"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"-1","cc":4,"cp":2,"self":false,"ncoefs":0},{"id":"VXQ8SIJC","t":"The leaky pipeline in research grant peer review and funding decisions: challenges and future directions.","au":"Sato, Sayaka","y":2021,"vn":"Higher Education","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s10734-020-00626-y","cby":87,"nref":120,"oast":"hybrid","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring","funding-decision"],"st":"conceptual-commentary","tp":"2","cc":4,"cp":5,"self":false,"ncoefs":0},{"id":"ZY2PTU76","t":"Answers to 20 Questions About Interrater Reliability and Interrater Agreement","au":"LeBreton, J. M.","y":2007,"vn":"Organizational Research Methods","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/1094428106296642","cby":3524,"nref":64,"oast":"closed","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"4","cc":6,"cp":3,"self":false,"ncoefs":0},{"id":"YYCAX8PN","t":"Are peer-reviews of grant proposals reliable? An analysis of Economic and Social Research Council (ESRC) funding applications","au":"Jerrim, John","y":2020,"vn":"The Social science journal","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/03623319.2020.1728506","cby":31,"nref":25,"oast":"closed","cx":"Grant","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":["level"],"sg":["reviewer-selection","individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"8","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"WY96SPAI","t":"NIH peer review: scored review criteria and overall impact.","au":"Lindner, Mark D","y":2016,"vn":"The American journal of evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/1098214015582049","cby":21,"nref":36,"oast":"hybrid","cx":"Grant","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"7","cc":4,"cp":2,"self":false,"ncoefs":0},{"id":"YSPY8PEM","t":"Funding science by lottery","au":"Avin, Shahar","y":2015,"vn":"Recent developments in the philosophy of science: EPSA13 helsinki","ty":"bookSection","tyl":"Book chapter","doi":"10.1007/978-3-319-23015-3_9","cby":27,"nref":18,"oast":"green","cx":"Grant","co":["Reliability-noise","Validity"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["funding-decision"],"st":"book-monograph","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"XZ8J64TH","t":"Peer review in the funding of research in higher education: the australian experience","au":"Jayasinghe, U. W.","y":2001,"vn":"Educational evaluation and policy analysis","ty":"journalArticle","tyl":"Journal article","doi":"10.3102/01623737023004343","cby":57,"nref":44,"oast":"closed","cx":"Grant","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["demographic","prestige-halo"],"pbs":"demographic","rs":["level","factors"],"sg":["reviewer-selection","individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"-1","cc":4,"cp":4,"self":false,"ncoefs":0},{"id":"YXJVI9WJ","t":"Peer Review Evaluation Process of Marie Curie Actions under EU's Seventh Framework Programme for Research.","au":"Pina, David G","y":2015,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0130753","cby":28,"nref":35,"oast":"gold","cx":"Grant","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring","panel-consensus"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"XTPUUZW7","t":"Peer review of grant applications: criteria used and qualitative study of reviewer practices.","au":"Abdoul, Hendy","y":2012,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0046054","cby":76,"nref":50,"oast":"gold","cx":"Grant","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["proposal-quality","individual-scoring"],"st":"qualitative","tp":"8","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"WS9B8LJP","t":"Early career grants, performance, and careers: A study on predictive validity of grant decisions","au":"van den Besselaar, Peter","y":2015,"vn":"Journal of Informetrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.joi.2015.07.011","cby":92,"nref":47,"oast":"closed","cx":"Grant","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["funding-decision","post-award-validity"],"st":"observational-quantitative","tp":"2","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"WNJ55J6C","t":"Influence of external peer reviewer scores for funding applications on funding board decisions: a retrospective analysis of 1561 reviews.","au":"Sorrell, Lexy","y":2018,"vn":"BMJ Open","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmjopen-2018-022547","cby":5,"nref":29,"oast":"gold","cx":"Grant","co":["Reliability-noise","Validity"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["factors","value"],"sg":["reviewer-selection","individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"7","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"VHL3ICQ9","t":"The acceptability of using a lottery to allocate research funding: a survey of applicants.","au":"Liu, Mengyao","y":2020,"vn":"Research Integrity and Peer Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s41073-019-0089-z","cby":70,"nref":29,"oast":"gold","cx":"Grant","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["funding-decision"],"st":"observational-quantitative","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"WH32N544","t":"Publication prejudices: An experimental study of confirmatory bias in the peer review system.","au":"Mahoney, MJ","y":1977,"vn":"Cognitive therapy and research","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/BF01173636","cby":1035,"nref":29,"oast":"closed","cx":"Journal","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":["level"],"sg":["individual-scoring"],"st":"experimental-rct","tp":"0","cc":3,"cp":3,"self":false,"ncoefs":0},{"id":"WIVHR46Q","t":"Effect on the Quality of Peer Review of Blinding Reviewers and Asking Them to Sign Their Reports","au":"Godlee, Fiona","y":1998,"vn":"The Journal of the American Medical Association","ty":"journalArticle","tyl":"Journal article","doi":"10.1001/jama.280.3.237","cby":338,"nref":11,"oast":"closed","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["prestige-halo"],"pbs":"prestige-halo","rs":[],"sg":["individual-scoring"],"st":"experimental-rct","tp":"10","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"ZCR8KFF5","t":"Extent of type I and type II errors in editorial decisions: A case study on Angewandte Chemie International Edition","au":"Bornmann, Lutz","y":2009,"vn":"Journal of Informetrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.joi.2009.05.002","cby":32,"nref":31,"oast":"closed","cx":"Journal","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["funding-decision","post-award-validity"],"st":"observational-quantitative","tp":"-1","cc":3,"cp":4,"self":false,"ncoefs":0},{"id":"XHLJRX3Y","t":"The peer review process used to evaluate manuscripts submitted to academic journals","au":"Marsh, Herbert W.","y":1989,"vn":"The Journal of Experimental Education","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/00220973.1989.10806503","cby":119,"nref":21,"oast":"closed","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level","factors"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"3","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"ZN6UACIU","t":"Risk evaluation in peer review of grant applications","au":"Gallo, Stephen","y":2018,"vn":"Environment Systems and Decisions","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s10669-018-9677-6","cby":36,"nref":32,"oast":"hybrid","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"YNHSVDSY","t":"Improving the Peer review process: Capturing more information and enabling high-risk/high-return research","au":"Linton, Jonathan D.","y":2016,"vn":"Research Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.respol.2016.07.004","cby":31,"nref":14,"oast":"closed","cx":"Grant","co":["Reliability-noise","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["panel-consensus","funding-decision"],"st":"conceptual-commentary","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"VPIZJDYQ","t":"How do journals of different rank instruct peer reviewers? Reviewer guidelines in the field of management","au":"Seeber, Marco","y":2020,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-019-03343-1","cby":38,"nref":78,"oast":"green","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["proposal-quality","individual-scoring"],"st":"observational-quantitative","tp":"3","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"WC3AX6UH","t":"Inter-rater reliability and validity of peer reviews in an interdisciplinary field","au":"Jirschitzka, Jens","y":2017,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-017-2516-6","cby":43,"nref":149,"oast":"closed","cx":"Journal","co":["Reliability-noise","Validity"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level","factors"],"sg":["reviewer-selection","individual-scoring"],"st":"observational-quantitative","tp":"3","cc":5,"cp":4,"self":false,"ncoefs":0},{"id":"VP2W7UDL","t":"Bias in research grant evaluation has dire consequences for small universities.","au":"Murray, Dennis L","y":2016,"vn":"Plos One","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0155876","cby":89,"nref":45,"oast":"gold","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["prestige-halo"],"pbs":"prestige-halo","rs":[],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"-1","cc":4,"cp":2,"self":false,"ncoefs":0},{"id":"XUEM9UZN","t":"Alternatives to Peer Review in Research Project Funding: 2013 Update","au":"Guthrie, Susan","y":2013,"vn":"","ty":"webpage","tyl":"Webpage","doi":"","cby":6,"nref":0,"oast":"closed","cx":"Grant","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"1","cc":1,"cp":0,"self":false,"ncoefs":0},{"id":"Y5JCHZCD","t":"Engaging Patients and Stakeholders in Research Proposal Review: The Patient-Centered Outcomes Research Institute","au":"Fleurence, Rachael","y":2014,"vn":"Annals of Internal Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.7326/m13-2412","cby":82,"nref":8,"oast":"closed","cx":"Grant","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["reviewer-selection","individual-scoring","panel-consensus"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"YXXSYUJR","t":"Assessing the potential of a Bayesian ranking as an alternative to consensus meetings for decision making in research funding: A case study of Marie Skłodowska-Curie actions","au":"Heyard, Rachel","y":2025,"vn":"PLoS ONE","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0317772","cby":0,"nref":26,"oast":"gold","cx":"Grant","co":["Reliability-noise","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring","panel-consensus","funding-decision"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"YSVZ7IT3","t":"Do Grant Proposal Texts Matter for Funding Decisions? A Field Experiment","au":"Şimşek, Müge","y":2023,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31219/osf.io/yhs2e","cby":1,"nref":45,"oast":"gold","cx":"Grant","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["proposal-quality","individual-scoring"],"st":"experimental-rct","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"ZPZW8PR3","t":"The reliability of peer review in anthrozoology","au":"Herzog, Harold A.","y":2005,"vn":"Anthrozoös","ty":"journalArticle","tyl":"Journal article","doi":"10.2752/089279305785594180","cby":4,"nref":21,"oast":"closed","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"15","cc":5,"cp":4,"self":false,"ncoefs":0},{"id":"YBHRWT7Z","t":"Gender contributes to personal research funding success in The Netherlands","au":"Lee, Romy van der","y":2015,"vn":"Proceedings of the National Academy of Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1073/pnas.1510159112","cby":320,"nref":43,"oast":"bronze","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"2","cc":4,"cp":5,"self":false,"ncoefs":0},{"id":"YJ6FIFYN","t":"Association of percentile ranking with citation impact and productivity in a large cohort of de novo NIMH-funded R01 grants","au":"Doyle, Jamie Mihoko","y":2015,"vn":"Molecular Psychiatry","ty":"journalArticle","tyl":"Journal article","doi":"10.1038/mp.2015.71","cby":41,"nref":13,"oast":"green","cx":"Grant","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["post-award-validity"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"VPW3P5QI","t":"Face-to-face panel meetings versus remote evaluation of fellowship applications: simulation study at the Swiss National Science Foundation","au":"Bieri, Marco","y":2021,"vn":"BMJ Open","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmjopen-2020-047386","cby":10,"nref":26,"oast":"gold","cx":"Grant","co":["Reliability-noise","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level","value"],"sg":["panel-consensus","funding-decision"],"st":"simulation-methodological","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"W53JXMKE","t":"Making peer review evidence-based: It’s time to open the \"black box\"","au":"Dunleavy, Daniel J.","y":2022,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31222/osf.io/9ka8q","cby":2,"nref":106,"oast":"gold","cx":"Journal","co":["Reliability-noise","Bias","Interventions"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"VKJXIFFN","t":"Addressing Racial Disparities in NIH Funding","au":"Comfort, Nicole","y":2021,"vn":"Journal of Science Policy & Governance","ty":"journalArticle","tyl":"Journal article","doi":"10.38126/jspg180408","cby":3,"nref":39,"oast":"diamond","cx":"Grant","co":["Bias","Interventions"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["funding-decision"],"st":"guideline-policy","tp":"14","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"ZE8HW5W5","t":"Quality and peer review of research: an adjudicating role for editors.","au":"Newton, Douglas P","y":2010,"vn":"Accountability in research","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/08989621003791945","cby":78,"nref":61,"oast":"green","cx":"Journal","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring","funding-decision"],"st":"qualitative","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"VN36QJYZ","t":"The foundation and consequences of gender bias in grant peer review processes.","au":"Morgan, Rosemary","y":2018,"vn":"CMAJ : Canadian Medical Association journal = journal de l'Association medicale canadienne","ty":"journalArticle","tyl":"Journal article","doi":"10.1503/cmaj.180188","cby":24,"nref":11,"oast":"gold","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"2","cc":4,"cp":5,"self":false,"ncoefs":0},{"id":"YR6REJFY","t":"Gender inequities in medical research funding is driving an exodus of women from Australian STEMM academia.","au":"Borger, Jessica G","y":2022,"vn":"Immunology and cell biology","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/imcb.12568","cby":20,"nref":14,"oast":"hybrid","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["funding-decision"],"st":"observational-quantitative","tp":"2","cc":4,"cp":5,"self":false,"ncoefs":0},{"id":"ZSH388T5","t":"Gender and geographical bias in the editorial decision-making process of biomedical journals: a case-control study.","au":"Gayet-Ageron, Angèle","y":2025,"vn":"BMJ evidence-based medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmjebm-2024-113083","cby":3,"nref":45,"oast":"hybrid","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["funding-decision"],"st":"observational-quantitative","tp":"5","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"WCVGJTSZ","t":"Geographical diversity of peer reviewers shapes author success.","au":"Zumel Dumlao, James M","y":2025,"vn":"Proceedings of the National Academy of Sciences of the United States of America","ty":"journalArticle","tyl":"Journal article","doi":"10.1073/pnas.2507394122","cby":10,"nref":46,"oast":"hybrid","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["demographic","cognitive-COI"],"pbs":"demographic","rs":[],"sg":["reviewer-selection","individual-scoring"],"st":"observational-quantitative","tp":"5","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"W24XKFMR","t":"Does a paper's country of origin affect the length of the review process?","au":"Valkonen, Laura","y":2012,"vn":"Cortex: A Journal Devoted to the Study of the Nervous System and Behavior","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.cortex.2012.05.015","cby":0,"nref":141,"oast":"closed","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"-1","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"YIN5T6G3","t":"Bibliometrics as a performance measurement tool for research evaluation: The case of research funded by the National Cancer Institute of Canada.","au":"Campbell, David","y":2010,"vn":"American Journal of Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/1098214009354774","cby":158,"nref":25,"oast":"closed","cx":"Grant","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["post-award-validity"],"st":"observational-quantitative","tp":"9","cc":1,"cp":1,"self":false,"ncoefs":0},{"id":"ZYMGMAKP","t":"Criteria Used by a Peer Review Committee for Selection of Research Fellows: A Boolean Probit Analysis.","au":"Bornmann, Lutz","y":2005,"vn":"International Journal of Selection and Assessment","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/j.1468-2389.2005.00326.x","cby":17,"nref":40,"oast":"bronze","cx":"Grant","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["panel-consensus","funding-decision"],"st":"observational-quantitative","tp":"2","cc":4,"cp":4,"self":false,"ncoefs":0},{"id":"XJZJLZUX","t":"Evaluating scientific papers and research proposals.","au":"Disney, Henry","y":2001,"vn":"Cortex: A Journal Devoted to the Study of the Nervous System and Behavior","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/S0010-9452(08)70600-2","cby":1,"nref":0,"oast":"closed","cx":"General","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"-1","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"YSR8YJQI","t":"Reexamining the NSF GRFP review process","au":"Evande R.","y":2023,"vn":"Journal of Biological Chemistry","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.jbc.2023.103515","cby":0,"nref":0,"oast":"gold","cx":"Grant","co":["Bias","Interventions"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"-1","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"WC3QYWZ4","t":"Predominance of animal-based expertise may bias NIH neuroscience grant review: A pilot study with implications for non-animal methodologies","au":"Trunnell E.R.","y":2025,"vn":"bioRxiv","ty":"journalArticle","tyl":"Journal article","doi":"10.1101/2025.02.28.640877","cby":3,"nref":8,"oast":"green","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["reviewer-selection","funding-decision"],"st":"observational-quantitative","tp":"-1","cc":4,"cp":0,"self":false,"ncoefs":0},{"id":"YM46CVWT","t":"Research funders should be more transparent: a plea for open applications","au":"","y":2022,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31219/osf.io/37tsu","cby":1,"nref":0,"oast":"gold","cx":"Grant","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"1","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"ZWJJHGJC","t":"An ethnographic analysis virtual peer review panels","au":"","y":2023,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31235/osf.io/fbkr6","cby":0,"nref":0,"oast":"closed","cx":"Grant","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["panel-consensus"],"st":"qualitative","tp":"13","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"Y34R4SM5","t":"The Case for Using Educational Scholarship to Improve Peer Review","au":"","y":2024,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31234/osf.io/zak4f","cby":0,"nref":0,"oast":"gold","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"0","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"XWC5YURW","t":"Professionalising peer reviewers: maturation of an old idea","au":"","y":2024,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31219/osf.io/zcgvr","cby":0,"nref":0,"oast":"gold","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":["level"],"sg":["reviewer-selection"],"st":"conceptual-commentary","tp":"0","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"VR97RI38","t":"Integrating Artificial Intelligence into Scholarly Peer Review: A Framework for Enhancing Efficiency and Quality","au":"","y":2025,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31219/osf.io/s764u_v1","cby":0,"nref":0,"oast":"gold","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"6","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"WMHFBK5H","t":"Hidden Prompts in Manuscripts Exploit AI-Assisted Peer Review","au":"","y":2025,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31234/osf.io/nea5u_v1","cby":8,"nref":0,"oast":"gold","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["ai-llm"],"pbs":"ai-llm","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"6","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"Z24XRKEM","t":"Good practice in the use of machine learning & AI by research funding organisations: insights from a workshop series","au":"Holm, Jon","y":2022,"vn":"Figshare","ty":"report","tyl":"Report","doi":"10.6084/m9.figshare.21710015.v1","cby":0,"nref":0,"oast":"green","cx":"Grant","co":["Bias","Interventions"],"pco":"Bias","bs":["ai-llm"],"pbs":"ai-llm","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"6","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"VYVGQ37Z","t":"The Future of Peer Review: A system-wide perspective","au":"Pinfield, Stephen","y":2026,"vn":"Figshare","ty":"report","tyl":"Report","doi":"10.6084/m9.figshare.32685351.v1","cby":0,"nref":0,"oast":"green","cx":"General","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"qualitative","tp":"0","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"ZPUABMMS","t":"Policy proposals to promote inclusion of caregivers in the research funding system","au":"Torres, Isabel","y":2024,"vn":"Frontiers in Education","ty":"journalArticle","tyl":"Journal article","doi":"10.3389/feduc.2024.1472517","cby":3,"nref":111,"oast":"gold","cx":"Grant","co":["Bias","Interventions"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["whole-process"],"st":"guideline-policy","tp":"2","cc":4,"cp":5,"self":false,"ncoefs":0},{"id":"Y7ME8VV7","t":"Double blind reviews in software engineering venues: Practicability, promises and perils","au":"Beller, Moritz","y":2016,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.7287/peerj.preprints.1757v1","cby":0,"nref":43,"oast":"gold","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"10","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"ZA5D24NV","t":"A new approach to peer review assessments: Score, then rank","au":"Gallo, Stephen A.","y":2022,"vn":"Research Square","ty":"journalArticle","tyl":"Journal article","doi":"10.21203/rs.3.rs-2198949/v1","cby":0,"nref":32,"oast":"green","cx":"Grant","co":["Reliability-noise","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring","funding-decision"],"st":"simulation-methodological","tp":"8","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"VUDVZI2F","t":"Are numerical scores important for grant proposals' evaluation? A cross sectional study","au":"Buljan, Ivan","y":2023,"vn":"F1000Research","ty":"journalArticle","tyl":"Journal article","doi":"10.12688/f1000research.139743.1","cby":0,"nref":24,"oast":"gold","cx":"Grant","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["individual-scoring","panel-consensus"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"YD9UFWAY","t":"NIH's peer review stands up to scrutiny","au":"Mervis, Jeffrey","y":2015,"vn":"Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1126/science.348.6233.384","cby":5,"nref":0,"oast":"closed","cx":"Grant","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["post-award-validity"],"st":"conceptual-commentary","tp":"7","cc":1,"cp":0,"self":false,"ncoefs":0},{"id":"Z2G4LSCA","t":"Chapter 5: Reviewing the Reviewers: Training Neural Networks to Read Peer Review Reports","au":"Eve, Martin Paul","y":2022,"vn":"Bielefeld University Press eBooks","ty":"book","tyl":"Book","doi":"10.1515/9783839455845-006","cby":0,"nref":89,"oast":"closed","cx":"Journal","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"book-monograph","tp":"6","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"XVXGZW69","t":"Rethinking resource allocation in science","au":"Bollen, Johan","y":2019,"vn":"Ecology and Society","ty":"journalArticle","tyl":"Journal article","doi":"10.5751/es-11005-240329","cby":0,"nref":9,"oast":"gold","cx":"Grant","co":["Reliability-noise","Bias","Validity","Interventions"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"1","cc":1,"cp":0,"self":false,"ncoefs":0},{"id":"VU5PYHEV","t":"The Many Dilemmas of Grant Peer Review","au":"Langfeldt, Liv","y":2024,"vn":"SpringerBriefs in political science","ty":"book","tyl":"Book","doi":"10.1007/978-3-031-69580-3_5","cby":1,"nref":17,"oast":"hybrid","cx":"Grant","co":["Reliability-noise","Bias","Interventions"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"book-monograph","tp":"8","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"YKBFB3V5","t":"Science rules! A qualitative study of scientists’ approaches to grant lottery","au":"Philipps, Axel","y":2020,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvaa027","cby":17,"nref":64,"oast":"hybrid","cx":"Grant","co":["Validity","Interventions"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["funding-decision"],"st":"qualitative","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"ZYWQI8FV","t":"Objective and Numerical Method of Finding Importance of Medical Research and Prioritizing Grants and Publications","au":"Dyczkowski, Jerzy","y":2020,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"","cby":0,"nref":0,"oast":"","cx":"General","co":["Validity","Interventions"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["proposal-quality","funding-decision"],"st":"simulation-methodological","tp":"-1","cc":0,"cp":0,"self":false,"ncoefs":0},{"id":"X8NPAVC9","t":"Prediction of SMEs’ R&D performances by machine learning for project selection","au":"Yoo, Hyoung Sun","y":2023,"vn":"Scientific Reports","ty":"journalArticle","tyl":"Journal article","doi":"10.1038/s41598-023-34684-w","cby":15,"nref":32,"oast":"gold","cx":"Grant","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["funding-decision","post-award-validity"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"XEDBKFQC","t":"Toward predicting research proposal success","au":"Boyack, Kevin W.","y":2017,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-017-2609-2","cby":39,"nref":45,"oast":"green","cx":"Grant","co":["Bias","Validity"],"pco":"Bias","bs":["prestige-halo"],"pbs":"prestige-halo","rs":[],"sg":["proposal-quality","funding-decision"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"VWS5K2BN","t":"Social Valuation Across Multiple Audiences: The Interplay of Ability and Identity Judgments","au":"Fini, Riccardo","y":2017,"vn":"Academy of Management Journal","ty":"journalArticle","tyl":"Journal article","doi":"10.5465/amj.2016.0661","cby":51,"nref":125,"oast":"green","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["prestige-halo"],"pbs":"prestige-halo","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"1","cc":4,"cp":2,"self":false,"ncoefs":0},{"id":"W68NZPTA","t":"The peer review process: a primer forJNISreaders","au":"Hirsch, Joshua A","y":2015,"vn":"Journal of NeuroInterventional Surgery","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/neurintsurg-2015-011781","cby":11,"nref":32,"oast":"closed","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"WN5CRXC2","t":"Modern applications of cross-classified random effects models in social and behavioral research: Illustration with R package PLmixed","au":"Huang, Sijia","y":2022,"vn":"Frontiers in Psychology","ty":"journalArticle","tyl":"Journal article","doi":"10.3389/fpsyg.2022.976964","cby":5,"nref":54,"oast":"gold","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"simulation-methodological","tp":"4","cc":0,"cp":4,"self":false,"ncoefs":0},{"id":"VPZMMF5B","t":"Cross-classified models in I/O psychology","au":"Claus, Anna M.","y":2020,"vn":"Journal of Vocational Behavior","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.jvb.2020.103447","cby":20,"nref":45,"oast":"closed","cx":"Analogue","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"4","cc":6,"cp":4,"self":false,"ncoefs":0},{"id":"VZZHSWUF","t":"Gender inequality in applying research project and funding","au":"Liao, Chien Hsiang","y":2022,"vn":"Journal of Information Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/01655515221097861","cby":14,"nref":54,"oast":"closed","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["demographic","prestige-halo"],"pbs":"demographic","rs":[],"sg":["funding-decision"],"st":"observational-quantitative","tp":"2","cc":4,"cp":5,"self":false,"ncoefs":0},{"id":"XHKV6QPM","t":"Grants Peer Review in Theory and Practice","au":"Chubin, Daryl E.","y":1994,"vn":"Evaluation Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/0193841x9401800103","cby":51,"nref":20,"oast":"closed","cx":"Grant","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"1","cc":4,"cp":4,"self":false,"ncoefs":0},{"id":"WWP3Q9HN","t":"Arbitrariness of peer review: A Bayesian analysis of the NIPS experiment","au":"François, Olivier","y":2015,"vn":"arXiv (Cornell University)","ty":"journalArticle","tyl":"Journal article","doi":"10.48550/arxiv.1507.06411","cby":12,"nref":13,"oast":"green","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["panel-consensus","funding-decision"],"st":"simulation-methodological","tp":"0","cc":6,"cp":2,"self":false,"ncoefs":0},{"id":"XTSCEH5T","t":"Reviewing the Test Reviews: Quality Judgments and Reviewer Agreements in the Mental Measurements Yearbook","au":"Hogan, Thomas P.","y":2021,"vn":"Applied Measurement in Education","ty":"journalArticle","tyl":"Journal article","doi":"10.1080/08957347.2021.1890742","cby":1,"nref":27,"oast":"closed","cx":"General","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"3","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"YNVIY6T2","t":"Peer review in practice","au":"Dorothy, Butchard","y":2017,"vn":"UCL Press eBooks","ty":"book","tyl":"Book","doi":"10.14324/111.9781911307679.15","cby":6,"nref":44,"oast":"green","cx":"General","co":["Reliability-noise","Bias","Interventions"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"book-monograph","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"ZZLRBXG8","t":"Gender Differences in Grant Submissions across Science and Engineering Fields at the NSF","au":"Rissler, Leslie J.","y":2020,"vn":"BioScience","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/biosci/biaa072","cby":47,"nref":54,"oast":"hybrid","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["funding-decision"],"st":"observational-quantitative","tp":"2","cc":4,"cp":5,"self":false,"ncoefs":0},{"id":"XKNN8HWX","t":"Authors anonymous?","au":"Benwell, R","y":1979,"vn":"Physics Bulletin","ty":"journalArticle","tyl":"Journal article","doi":"10.1088/0031-9112/30/7/006","cby":95,"nref":0,"oast":"closed","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["prestige-halo"],"pbs":"prestige-halo","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"-1","cc":5,"cp":0,"self":false,"ncoefs":0},{"id":"XXJERAKS","t":"A Computer Program for Determining the Reliability of Dimensionally Scaled Data when the Numbers and Specific Sets of Examiners may Vary at Each Assessment","au":"Ciccheti, Domenic V.","y":1988,"vn":"Educational and Psychological Measurement","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/0013164488483020","cby":51,"nref":7,"oast":"closed","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"4","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"ZHGU8LLV","t":"Testing the Normal Approximation and Minimal Sample Size Requirements of Weighted Kappa When the Number of Categories is Large","au":"Cicchetti, Domenic V.","y":1981,"vn":"Applied Psychological Measurement","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/014662168100500114","cby":187,"nref":8,"oast":"green","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"4","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"WAA8LCI8","t":"The Reliability of Dichotomous Judgments: Unequal Numbers of Judges per Subject","au":"Fleiss, Joseph L.","y":1979,"vn":"Applied Psychological Measurement","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/014662167900300410","cby":190,"nref":12,"oast":"green","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"4","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"XTWK9EL7","t":"Interrater reliability and agreement of subjective judgments.","au":"Tinsley, Howard E. A.","y":1975,"vn":"Journal of Counseling Psychology","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/h0076640","cby":861,"nref":52,"oast":"closed","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"4","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"XVMP7GIU","t":"The Kappa Coefficient of Agreement for Multiple Observers When the Number of Subjects is Small","au":"Gross, Shulamith T.","y":1986,"vn":"Biometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.2307/2530702","cby":80,"nref":9,"oast":"closed","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"4","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"Z7YJFU77","t":"Paradigm Development and Particularism: Journal Publication in Three Scientific Disciplines","au":"Pfeffer, Jeffrey","y":1977,"vn":"Social Forces","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/sf/55.4.938","cby":195,"nref":9,"oast":"closed","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["prestige-halo"],"pbs":"prestige-halo","rs":[],"sg":["reviewer-selection","funding-decision"],"st":"observational-quantitative","tp":"0","cc":5,"cp":0,"self":false,"ncoefs":0},{"id":"ZG9A9VH7","t":"Guarding the Guardians: A Conference on Editorial Peer Review","au":"Rennie, Drummond","y":1986,"vn":"JAMA","ty":"journalArticle","tyl":"Journal article","doi":"10.1001/jama.1986.03380170107031","cby":143,"nref":4,"oast":"closed","cx":"Journal","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":5,"cp":0,"self":false,"ncoefs":0},{"id":"ZH29M6TV","t":"Bias, Controversy, and Abuse in the Study of the Scientific Publication System","au":"Mahoney, M. J.","y":1990,"vn":"Science Technology & Human Values","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/016224399001500105","cby":47,"nref":5,"oast":"closed","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":5,"cp":0,"self":false,"ncoefs":0},{"id":"WJIKJ2M5","t":"Problems Identified by Secondary Review of Accepted Manuscripts","au":"Garfunkel, Joseph M.","y":1990,"vn":"JAMA","ty":"journalArticle","tyl":"Journal article","doi":"10.1001/jama.1990.03440100077011","cby":72,"nref":4,"oast":"closed","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"3","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"ZE7G284F","t":"Judgment of counseling process: Reliability, agreement, and error.","au":"Lawlis, G. Frank","y":1972,"vn":"Psychological Bulletin","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/h0032935","cby":174,"nref":0,"oast":"closed","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"4","cc":5,"cp":0,"self":false,"ncoefs":0},{"id":"WV99LFB4","t":"2 x 2 Kappa Coefficients: Measures of Agreement or Association","au":"Blöch, D.","y":1989,"vn":"Biometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.2307/2532052","cby":293,"nref":17,"oast":"closed","cx":"Analogue","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"4","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"W48D45BZ","t":"We need research on what constitutes good journal papers--and good editing--not guesswork on how to improve manuscriptsp.","au":"Crandall, Rick","y":1987,"vn":"American Psychologist","ty":"journalArticle","tyl":"Journal article","doi":"10.1037//0003-066x.42.4.407.b","cby":36,"nref":4,"oast":"closed","cx":"Journal","co":["Reliability-noise","Validity"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["proposal-quality","individual-scoring"],"st":"conceptual-commentary","tp":"3","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"YGF3URKN","t":"Issues Surrounding Publishing","au":"Coniam, David","y":2022,"vn":"Springer briefs in education","ty":"book","tyl":"Book","doi":"10.1007/978-981-19-3065-2_2","cby":0,"nref":53,"oast":"closed","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"book-monograph","tp":"-1","cc":3,"cp":4,"self":false,"ncoefs":0},{"id":"ZCSN6RNZ","t":"External Reviewers’ Influence on Citations of Articles Published in Pedagogical Business Journals: Holding Acceptance Rate Constant","au":"Alam, Arshad","y":2016,"vn":"SSRN Electronic Journal","ty":"journalArticle","tyl":"Journal article","doi":"","cby":0,"nref":29,"oast":"green","cx":"Journal","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["reviewer-selection","post-award-validity"],"st":"observational-quantitative","tp":"0","cc":0,"cp":4,"self":false,"ncoefs":0},{"id":"WZVF5TQJ","t":"Do grant proposal texts matter for funding decisions? A field experiment","au":"Şimşek, Müge","y":2024,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-024-04968-7","cby":9,"nref":33,"oast":"hybrid","cx":"Grant","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["proposal-quality","individual-scoring"],"st":"experimental-rct","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"XY9T7EXT","t":"How to avoid borrowed plumes in academia","au":"Osterloh, Margit","y":2019,"vn":"Research Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.respol.2019.103831","cby":98,"nref":128,"oast":"closed","cx":"Journal","co":["Reliability-noise","Validity","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"YWD3GJ8R","t":"An Evaluation of the Peer Review Process at Angewandte Chemie","au":"Daniel, Hans‐Dieter","y":1993,"vn":"Angewandte Chemie International Edition in English","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/anie.199302341","cby":21,"nref":17,"oast":"closed","cx":"Journal","co":["Reliability-noise","Bias","Validity"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":3,"cp":3,"self":false,"ncoefs":0},{"id":"ZX28CA2C","t":"The Measurement of Observer Agreement for Categorical Data","au":"Landis, J. Richard","y":1977,"vn":"Biometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.2307/2529310","cby":79617,"nref":94,"oast":"green","cx":"Analogue","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":["level"],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"4","cc":5,"cp":3,"self":false,"ncoefs":0},{"id":"ZGLDA6ZW","t":"Reliability of a Structured Method of Selecting Abstracts for a Plastic Surgical Scientific Meeting","au":"Steen, Lydia P. E. van der","y":2003,"vn":"Plastic & Reconstructive Surgery","ty":"journalArticle","tyl":"Journal article","doi":"10.1097/01.prs.0000061092.88629.82","cby":28,"nref":26,"oast":"closed","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring","post-award-validity"],"st":"observational-quantitative","tp":"15","cc":5,"cp":1,"self":false,"ncoefs":0},{"id":"YQ73QQX4","t":"A review of ideas and strategies to improve scientific research and its dissemination in life and social sciences","au":"Almgren, Hannes","y":2020,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31222/osf.io/9hg3j","cby":0,"nref":140,"oast":"gold","cx":"General","co":["Bias","Interventions"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"VYR3JTDF","t":"A Website System for Communicating Psychological Science","au":"Diener, Ed","y":2017,"vn":"Perspectives on Psychological Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/1745691616685864","cby":0,"nref":14,"oast":"bronze","cx":"Journal","co":["Bias","Validity","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"-1","cc":5,"cp":1,"self":false,"ncoefs":0},{"id":"XV9APMTQ","t":"Publizieren in der Deutschdidaktik","au":"Funke, Reinold","y":2021,"vn":"Zeitschrift für Sprachlich-Literarisches Lernen und Deutschdidaktik","ty":"journalArticle","tyl":"Journal article","doi":"10.46586/slld.z.2021.8770","cby":1,"nref":18,"oast":"diamond","cx":"Journal","co":["Reliability-noise","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["factors"],"sg":["whole-process"],"st":"conceptual-commentary","tp":"-1","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"Y3RTHFB6","t":"Improving what is published: Toward an evidence-based framework for manuscript review.","au":"Bornstein, Robert F.","y":2021,"vn":"Archives of Scientific Psychology","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/arc0000076","cby":4,"nref":89,"oast":"gold","cx":"Journal","co":["Reliability-noise","Bias","Validity"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":["value"],"sg":["whole-process"],"st":"conceptual-commentary","tp":"-1","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"XIB72TAG","t":"Is Peer Review Fit for Purpose?","au":"Tight, Malcolm","y":2022,"vn":"","ty":"book","tyl":"Book","doi":"10.1007/978-3-030-75263-7_10","cby":6,"nref":26,"oast":"gold","cx":"General","co":["Reliability-noise","Validity"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"book-monograph","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"WGB2HQAZ","t":"Difficulty of recruiting reviewers predicts review scores and editorial decisions at six journals of ecology and evolution","au":"Fox, Charles W.","y":2017,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-017-2489-5","cby":28,"nref":27,"oast":"closed","cx":"Journal","co":["Reliability-noise","Bias","Validity"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["reviewer-selection","individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"5","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"W5SHTMNM","t":"ACADEMIC TALENT SELECTION IN GRANT REVIEW PANELS","au":"Arensbergen, P. van","y":2014,"vn":"Data Archiving and Networked Services (DANS)","ty":"book","tyl":"Book","doi":"","cby":7,"nref":53,"oast":"closed","cx":"Grant","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring","panel-consensus","funding-decision"],"st":"book-monograph","tp":"2","cc":4,"cp":2,"self":false,"ncoefs":0},{"id":"X8UKNNYN","t":"Behind Door #3: The Hard-to-Please Grant Reviewer.","au":"Miner, Jeremy T.","y":2011,"vn":"Research management review","ty":"journalArticle","tyl":"Journal article","doi":"","cby":3,"nref":20,"oast":"closed","cx":"Grant","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"8","cc":1,"cp":4,"self":false,"ncoefs":0},{"id":"YJW7DQHS","t":"Institutionalized biases in the award of research grants: a preliminary analysis revisiting the principle of accumulative advantage","au":"Viner, Neil","y":2003,"vn":"Research Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.respol.2003.09.005","cby":101,"nref":32,"oast":"closed","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["demographic","prestige-halo"],"pbs":"demographic","rs":[],"sg":["funding-decision"],"st":"observational-quantitative","tp":"-1","cc":4,"cp":4,"self":false,"ncoefs":0},{"id":"ZPYJF2B9","t":"Assessing the potential of a Bayesian ranking as an alternative to consensus meetings for decision making in research funding: A Case Study of Marie Skłodowska-Curie Actions","au":"Heyard, Rachel","y":2024,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31222/osf.io/c4ujg","cby":0,"nref":20,"oast":"gold","cx":"Grant","co":["Reliability-noise","Interventions"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level"],"sg":["individual-scoring","panel-consensus","funding-decision"],"st":"observational-quantitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"X6QKSCWD","t":"Ponderings on peer review: Part I. Basic principles","au":"Seals, Douglas R.","y":2023,"vn":"American Journal of Physiology-Regulatory, Integrative and Comparative Physiology","ty":"journalArticle","tyl":"Journal article","doi":"10.1152/ajpregu.00062.2023","cby":8,"nref":24,"oast":"closed","cx":"General","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":1,"cp":1,"self":false,"ncoefs":0},{"id":"VQRAA6KJ","t":"Peer Review for Journals as it Stands Today—Part 2","au":"Campanario, Juan Miguel","y":1998,"vn":"Science Communication","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/1075547098019004002","cby":134,"nref":139,"oast":"closed","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"WVBAFXRY","t":"Editors’ Requests of Peer Reviewers: A Study and a Proposal","au":"Frank, Erica","y":1996,"vn":"Preventive Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.1006/pmed.1996.0035","cby":21,"nref":0,"oast":"closed","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"0","cc":3,"cp":0,"self":false,"ncoefs":0},{"id":"XIQ6JFMS","t":"Fairness as Appropriateness","au":"Mallard, Grégoire","y":2009,"vn":"Science Technology & Human Values","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/0162243908329381","cby":123,"nref":70,"oast":"green","cx":"Grant","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["panel-consensus"],"st":"qualitative","tp":"9","cc":3,"cp":4,"self":false,"ncoefs":0},{"id":"Y2YMGN82","t":"An Exploratory Study of Statistical Assessment of Papers Published in the British Medical Journal","au":"Gardner, Martin J.","y":1990,"vn":"JAMA","ty":"journalArticle","tyl":"Journal article","doi":"10.1001/jama.1990.03440100061010","cby":130,"nref":17,"oast":"closed","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["proposal-quality","individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":2,"cp":0,"self":false,"ncoefs":0},{"id":"WVWHEAHL","t":"Nepotism and sexism in peer-review","au":"Wennerås, Christine","y":1997,"vn":"Nature","ty":"journalArticle","tyl":"Journal article","doi":"10.1038/387341a0","cby":1359,"nref":16,"oast":"closed","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["demographic","cognitive-COI"],"pbs":"demographic","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"2","cc":4,"cp":1,"self":false,"ncoefs":0},{"id":"VIZQNEHX","t":"Are three heads better than two? How the number of reviewers and editor behavior affect the rejection rate","au":"Schultz, David M.","y":2009,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-009-0084-0","cby":40,"nref":13,"oast":"green","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["value"],"sg":["reviewer-selection","funding-decision"],"st":"observational-quantitative","tp":"3","cc":3,"cp":3,"self":false,"ncoefs":0},{"id":"XPFIBE54","t":"Influence of interdisciplinarity on peer-review and bibliometric evaluations in physics research","au":"Rinia, Ed J.","y":2001,"vn":"Research Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/s0048-7333(00)00082-2","cby":108,"nref":6,"oast":"closed","cx":"General","co":["Bias"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"9","cc":4,"cp":0,"self":false,"ncoefs":0},{"id":"YBW8PCPY","t":"Effect of open peer review on quality of reviews and on reviewers'recommendations: a randomised trial","au":"Rooyen, Susan van","y":1999,"vn":"BMJ","ty":"journalArticle","tyl":"Journal article","doi":"10.1136/bmj.318.7175.23","cby":422,"nref":14,"oast":"bronze","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"experimental-rct","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"ZI6TF964","t":"Equal opportunities in Canada","au":"Friesen, Henry G.","y":1998,"vn":"Nature","ty":"journalArticle","tyl":"Journal article","doi":"10.1038/34768","cby":11,"nref":1,"oast":"bronze","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["funding-decision"],"st":"observational-quantitative","tp":"2","cc":4,"cp":0,"self":false,"ncoefs":0},{"id":"ZVGM62KZ","t":"Referees’ Attitudes toward Open Peer Review and Electronic Transmission of Papers","au":"Melero, R.","y":2001,"vn":"Food Science and Technology International","ty":"journalArticle","tyl":"Journal article","doi":"10.1106/0mxd-ym6f-3lm6-g9eb","cby":21,"nref":17,"oast":"closed","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":["level","factors"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"Y2UWHWNU","t":"Citation gamesmanship: testing for evidence of ego bias in peer review","au":"Sugimoto, Cassidy R.","y":2012,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-012-0845-z","cby":41,"nref":49,"oast":"closed","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"-1","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"W78VMZZS","t":"Journal peer review in context: A qualitative study of the social and subjective dimensions of manuscript review in biomedical publishing","au":"Lipworth, Wendy","y":2011,"vn":"Social Science & Medicine","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.socscimed.2011.02.002","cby":70,"nref":23,"oast":"green","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["individual-scoring"],"st":"qualitative","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"Y325AUKI","t":"The Gender Gap in NIH Grant Applications","au":"Ley, Timothy J.","y":2008,"vn":"Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1126/science.1165878","cby":279,"nref":11,"oast":"closed","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["funding-decision"],"st":"observational-quantitative","tp":"2","cc":4,"cp":0,"self":false,"ncoefs":0},{"id":"WNWHJLDM","t":"Estimating within-group interrater reliability with and without response bias.","au":"James, Lawrence R.","y":1984,"vn":"Journal of Applied Psychology","ty":"journalArticle","tyl":"Journal article","doi":"10.1037/0021-9010.69.1.85","cby":4663,"nref":37,"oast":"closed","cx":"Analogue","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":["level"],"sg":["individual-scoring"],"st":"simulation-methodological","tp":"4","cc":6,"cp":3,"self":false,"ncoefs":0},{"id":"WEW94PCC","t":"Institutions of performance: The journal as a theater of power and legitimacy","au":"Khaled, Ahmed","y":2026,"vn":"Political Studies Review","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/14789299261454217","cby":0,"nref":11,"oast":"closed","cx":"Journal","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"-1","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"YHAILXFM","t":"Does output-driven research culture hinder social outcome-focused funding? Insights from the STI hub program of DST, India","au":"Stanley, Ansel Elias","y":2026,"vn":"Research Evaluation","ty":"journalArticle","tyl":"Journal article","doi":"10.1093/reseval/rvag029","cby":0,"nref":63,"oast":"closed","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["prestige-halo"],"pbs":"prestige-halo","rs":[],"sg":["proposal-quality","individual-scoring"],"st":"qualitative","tp":"1","cc":4,"cp":2,"self":false,"ncoefs":0},{"id":"WZH5R7CJ","t":"Peer Reviewing as a Distributed Cognitive Validation System: Reconceptualizing Peer Review in Networked Knowledge Systems","au":"KWONGO, JOEL MANTSHIUMBA","y":2026,"vn":"SSRN Electronic Journal","ty":"journalArticle","tyl":"Journal article","doi":"10.2139/ssrn.6362173","cby":0,"nref":36,"oast":"green","cx":"Journal","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"W7GNC85E","t":"All that Glitters: A Perspective on Peer Reviewed Publications","au":"Jameison, Danielle","y":2020,"vn":"Harvard Medical Student Review","ty":"journalArticle","tyl":"Journal article","doi":"10.65539/4yj42w18","cby":0,"nref":5,"oast":"hybrid","cx":"Journal","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":0,"self":false,"ncoefs":0},{"id":"XJVRNWAZ","t":"Is there gender gap in research grant success in economics? Case study of Slovakia","au":"Némethová, Valéria","y":2025,"vn":"Sociologija","ty":"journalArticle","tyl":"Journal article","doi":"10.2298/soc2502304n","cby":0,"nref":39,"oast":"diamond","cx":"Grant","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["funding-decision","post-award-validity"],"st":"observational-quantitative","tp":"2","cc":4,"cp":5,"self":false,"ncoefs":0},{"id":"XJ9E8VL4","t":"A Conceptual Framework for AI- and Blockchain-Enabled Research Project Evaluation Systems","au":"Amanzholova, Saule","y":2026,"vn":"Information","ty":"journalArticle","tyl":"Journal article","doi":"10.3390/info17020151","cby":0,"nref":11,"oast":"gold","cx":"Grant","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"6","cc":4,"cp":1,"self":false,"ncoefs":0},{"id":"XKIJUH93","t":"O processo editorial de revistas científicas: o longo caminho entre a submissão e a publicação","au":"Almeida, Débora Rezende de","y":2026,"vn":"Revista de Sociologia e Política","ty":"journalArticle","tyl":"Journal article","doi":"10.1590/1678-98732434e004","cby":0,"nref":11,"oast":"diamond","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"-1","cc":0,"cp":1,"self":false,"ncoefs":0},{"id":"VTMMCRS3","t":"Academic Peer Review: Eliminating the Option of ‘Reject’ Without Author Response","au":"Arabi, Alya A.","y":2026,"vn":"Learned Publishing","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/leap.2038","cby":0,"nref":39,"oast":"hybrid","cx":"Journal","co":["Reliability-noise","Bias","Interventions"],"pco":"Reliability-noise","bs":["ai-llm"],"pbs":"ai-llm","rs":[],"sg":["funding-decision"],"st":"conceptual-commentary","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"WETTECGT","t":"Published peer review reports have higher informative content than unpublished reports","au":"Álvarez-García, Elena","y":2025,"vn":"Journal of Informetrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.joi.2025.101760","cby":0,"nref":40,"oast":"hybrid","cx":"Journal","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"VRIHCI6Q","t":"Funding big science: managing diversity, social responsibility, and limited resources","au":"Shaw, Jamie","y":2025,"vn":"European Journal for Philosophy of Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s13194-025-00698-6","cby":0,"nref":121,"oast":"closed","cx":"Grant","co":["Validity","Interventions"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["funding-decision"],"st":"conceptual-commentary","tp":"1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"YUHH5UFA","t":"An expert criticism on post-publication peer review platforms: the case of pubpeer","au":"Tsatsakis, Aristidis","y":2025,"vn":"DARU Journal of Pharmaceutical Sciences","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s40199-025-00580-z","cby":0,"nref":8,"oast":"green","cx":"General","co":["Bias","Interventions"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"VW2V5MMQ","t":"How Transport Journals Weed Out Qualitative Research and Why It Matters","au":"Klein, Nicholas J.","y":2025,"vn":"Transportation","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11116-026-10777-z","cby":0,"nref":30,"oast":"hybrid","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring","funding-decision"],"st":"qualitative","tp":"-1","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"VHDT8P7V","t":"When Cognitive Proximity Leads to Higher Evaluation Decision Quality: A Study of Public Funding Allocation","au":"Zhang, Chuqing","y":2021,"vn":"Frontiers in Psychology","ty":"journalArticle","tyl":"Journal article","doi":"10.3389/fpsyg.2021.697989","cby":0,"nref":50,"oast":"gold","cx":"Grant","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["reviewer-selection","individual-scoring"],"st":"observational-quantitative","tp":"1","cc":4,"cp":2,"self":false,"ncoefs":0},{"id":"ZW2W6EES","t":"What Can Journals Do to Increase the Reliability of Scientific Research?","au":"Lee, W. Robert","y":2020,"vn":"Practical Radiation Oncology","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.prro.2020.03.001","cby":0,"nref":9,"oast":"closed","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":0,"cp":1,"self":false,"ncoefs":0},{"id":"ZCRB5IEL","t":"Does Unblinding Make One King in the Land of the Blind(Ed)?","au":"Hee, Ng Heok","y":2015,"vn":"Proceedings of Singapore Healthcare","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/201010581502400201","cby":0,"nref":42,"oast":"diamond","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"10","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"WVZ6EEK5","t":"Greenlighting Innovative Projects: How Evaluation Structure Shapes the Perceived Feasibility of Early-Stage Ideas","au":"Lane, Jacqueline N.","y":2024,"vn":"SSRN Electronic Journal","ty":"journalArticle","tyl":"Journal article","doi":"10.2139/ssrn.4769282","cby":3,"nref":156,"oast":"green","cx":"Grant","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring","funding-decision"],"st":"experimental-rct","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"Y99Z2GUP","t":"Author-Reviewer Homophily in Peer Review","au":"Murray, Dakota","y":2018,"vn":"bioRxiv (Cold Spring Harbor Laboratory)","ty":"journalArticle","tyl":"Journal article","doi":"10.1101/400515","cby":179,"nref":130,"oast":"green","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["demographic","cognitive-COI"],"pbs":"demographic","rs":[],"sg":["reviewer-selection","funding-decision"],"st":"observational-quantitative","tp":"5","cc":3,"cp":5,"self":false,"ncoefs":0},{"id":"ZZYPVDYT","t":"Behavioral Optimization in Scientific Publishing","au":"Watve, Milind","y":2023,"vn":"Qeios","ty":"journalArticle","tyl":"Journal article","doi":"10.32388/8w10nd.3","cby":2,"nref":65,"oast":"diamond","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["demographic","prestige-halo","cognitive-COI"],"pbs":"demographic","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"-1","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"WDD7VXQ5","t":"Identifying emerging scholars: seeing through the crystal ball of scholarship selection committees","au":"Chandler, Vincent","y":2019,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-019-03120-0","cby":2,"nref":12,"oast":"closed","cx":"Grant","co":["Bias","Validity"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring","post-award-validity"],"st":"observational-quantitative","tp":"2","cc":4,"cp":2,"self":false,"ncoefs":0},{"id":"WCX3TBVD","t":"Barriers to Publication in the Middle East: A Categorization of Rejection Reasons among Arab Researchers","au":"Elgamri, Alya","y":2025,"vn":"Journal of Academic Ethics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s10805-025-09680-6","cby":1,"nref":45,"oast":"closed","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["proposal-quality","individual-scoring"],"st":"observational-quantitative","tp":"5","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"WNMMH2FJ","t":"A duoethnographic exploration of what peer reviewing teaches us about peer review","au":"Holly, James","y":2025,"vn":"Journal of Engineering Education","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/jee.70013","cby":1,"nref":75,"oast":"hybrid","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring"],"st":"qualitative","tp":"0","cc":7,"cp":0,"self":false,"ncoefs":0},{"id":"XNSNZATC","t":"Critical Contextual Empiricism for Busy People: Scientific Argumentation as Epistemic Exchange","au":"Novaes, Catarina Dutilh","y":2025,"vn":"Topoi","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11245-025-10198-0","cby":1,"nref":73,"oast":"hybrid","cx":"Journal","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"-1","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"YH4UKLA2","t":"Scientific Peer Review in the Modern Era: A Comprehensive Guide","au":"Sandhu, Nimrat Kaur","y":2022,"vn":"Indian Journal of Rheumatology","ty":"journalArticle","tyl":"Journal article","doi":"10.4103/0973-3698.364681","cby":2,"nref":23,"oast":"diamond","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"XFRJ7EYE","t":"A Case Study for the Interrater Reliability of Journal Referees","au":"Marson, Stephen M.","y":2021,"vn":"Research on Social Work Practice","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/10497315211052456","cby":4,"nref":45,"oast":"closed","cx":"Journal","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":["level","factors"],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"ZHTYCRJS","t":"Peer Review May Not Be Such a Bad Idea: Response to Heesen and Bright","au":"Rowbottom, Darrell P.","y":2021,"vn":"The British Journal for the Philosophy of Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1086/714787","cby":9,"nref":24,"oast":"closed","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"-1","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"VJGI4LJQ","t":"Review of interactive open-access publishing with community-based open peer review for improved scientific discourse and quality assurance","au":"Ervens, Barbara","y":2025,"vn":"Atmospheric chemistry and physics","ty":"journalArticle","tyl":"Journal article","doi":"10.5194/acp-25-13903-2025","cby":1,"nref":191,"oast":"gold","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"VLAGCTV8","t":"On arXiv moderation system","au":"Силагадзе, З. К.","y":2023,"vn":"Journal of Informetrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.joi.2023.101433","cby":3,"nref":34,"oast":"closed","cx":"General","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":0,"cp":1,"self":false,"ncoefs":0},{"id":"YHI9F8UZ","t":"Reducing bias in the peer‐review process","au":"Frair, Jacqueline L.","y":2025,"vn":"Journal of Wildlife Management","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/jwmg.70019","cby":2,"nref":32,"oast":"bronze","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring","funding-decision"],"st":"conceptual-commentary","tp":"-1","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"WYCLR47W","t":"Impact of author characteristics on outcomes of single- versus double-blind peer review: a systematic review of comparative studies in scientific abstracts and publications","au":"Giannakakos, Vasiliki P.","y":2024,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-024-05213-x","cby":2,"nref":50,"oast":"hybrid","cx":"General","co":["Bias"],"pco":"Bias","bs":["demographic","prestige-halo"],"pbs":"demographic","rs":[],"sg":["individual-scoring","funding-decision"],"st":"systematic-review-meta-analysis","tp":"10","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"ZM57TQNS","t":"Cognitive bias undermining intellectual capital development in the scientific contexts: a study to understand origins, effects and solutions","au":"Palmucci, Dario Natale","y":2024,"vn":"Journal of Intellectual Capital","ty":"journalArticle","tyl":"Journal article","doi":"10.1108/jic-07-2024-0199","cby":3,"nref":88,"oast":"closed","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["demographic","prestige-halo","cognitive-COI"],"pbs":"demographic","rs":[],"sg":["individual-scoring"],"st":"qualitative","tp":"5","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"YDF4F296","t":"The role of non-scientific factors vis-à-vis the quality of publications in determining their scholarly impact","au":"Abramo, Giovanni","y":2024,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-024-05106-z","cby":3,"nref":96,"oast":"hybrid","cx":"General","co":["Validity"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["post-award-validity"],"st":"observational-quantitative","tp":"9","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"ZN37E6CT","t":"Preprint review services: Disrupting the scholarly communication landscape?","au":"Henriques, Susana Oliveira","y":2023,"vn":"","ty":"journalArticle","tyl":"Journal article","doi":"10.31235/osf.io/8c6xm","cby":6,"nref":17,"oast":"gold","cx":"General","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"qualitative","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"Y4Z4GDFY","t":"Let’s agree to agree: The situational academic quality of the UK REF as consensual public knowledge","au":"Milyaeva, Sveta","y":2023,"vn":"Social Studies of Science","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/03063127231152915","cby":6,"nref":63,"oast":"hybrid","cx":"General","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["proposal-quality","panel-consensus"],"st":"qualitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"Y6MKSCBH","t":"Editors are biased too: An extension of Fox et al. (2023)'s analysis makes the case for triple‐blind review","au":"Srivastava, Diane S.","y":2024,"vn":"Functional Ecology","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/1365-2435.14483","cby":7,"nref":16,"oast":"hybrid","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["funding-decision"],"st":"observational-quantitative","tp":"-1","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"WIFBZF3Z","t":"Reforms in Academic Publishing: Should Behavioral Disorders and Special Education Journals Embrace Them?","au":"Cook, Bryan G.","y":2016,"vn":"Behavioral Disorders","ty":"journalArticle","tyl":"Journal article","doi":"10.17988/0198-7429-41.3.161","cby":9,"nref":77,"oast":"closed","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"-1","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"YNWHEN66","t":"What influences the regional diversity of reviewers: A study of medical and agricultural/biological sciences journals","au":"Gaston, Thomas","y":2018,"vn":"Learned Publishing","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/leap.1155","cby":14,"nref":11,"oast":"bronze","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["reviewer-selection","individual-scoring"],"st":"observational-quantitative","tp":"5","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"ZAW6VU37","t":"Evaluation, ranking and selection of R&D projects by multiple experts: an evidential reasoning rule based approach","au":"Liu, Fang","y":2017,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-017-2278-1","cby":21,"nref":42,"oast":"green","cx":"Grant","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["panel-consensus","funding-decision"],"st":"simulation-methodological","tp":"-1","cc":3,"cp":2,"self":false,"ncoefs":0},{"id":"XRW5XR99","t":"Nurturing ideas, cultivating reviewers: The C.E.D.A.R. tree of generative peer review","au":"Cristofaro, Matteo","y":2025,"vn":"European Management Journal","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.emj.2025.10.011","cby":2,"nref":25,"oast":"hybrid","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"YARRVZJ4","t":"Investigating the effect of publication text similarity between reviewers and authors on the rigor of peer review: An intellectual proximity perspective","au":"Kang, Yanlan","y":2025,"vn":"Journal of Informetrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.joi.2025.101709","cby":2,"nref":70,"oast":"closed","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["reviewer-selection","individual-scoring"],"st":"observational-quantitative","tp":"-1","cc":4,"cp":1,"self":false,"ncoefs":0},{"id":"ZEDT76D3","t":"Social behavioural epistemology and the scientific community","au":"Watve, Milind","y":2017,"vn":"Journal of Genetics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s12041-017-0790-y","cby":7,"nref":59,"oast":"closed","cx":"General","co":["Bias"],"pco":"Bias","bs":["prestige-halo"],"pbs":"prestige-halo","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"-1","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"Y3NCX4KP","t":"Technology assisted research assessment: algorithmic bias and transparency issues","au":"Thelwall, Mike","y":2023,"vn":"Aslib Journal of Information Management","ty":"journalArticle","tyl":"Journal article","doi":"10.1108/ajim-04-2023-0119","cby":15,"nref":67,"oast":"green","cx":"General","co":["Bias"],"pco":"Bias","bs":["ai-llm"],"pbs":"ai-llm","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"6","cc":4,"cp":1,"self":false,"ncoefs":0},{"id":"WTZECG89","t":"The academic status of reviewers predicts their language use","au":"Sun, Zhuanlan","y":2023,"vn":"Journal of Informetrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.joi.2023.101449","cby":13,"nref":46,"oast":"closed","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["prestige-halo"],"pbs":"prestige-halo","rs":[],"sg":["individual-scoring"],"st":"observational-quantitative","tp":"6","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"ZR8TTA2L","t":"PEERRec: An AI-based approach to automatically generate recommendations and predict decisions in peer review","au":"Bharti, Prabhat Kumar","y":2023,"vn":"International Journal on Digital Libraries","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s00799-023-00375-0","cby":13,"nref":37,"oast":"closed","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["individual-scoring","funding-decision"],"st":"simulation-methodological","tp":"6","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"Z89VP5AE","t":"All-pervading insider bias alters review time in Turkish university journals","au":"Tütüncü, Lokman","y":2023,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-023-04724-3","cby":13,"nref":66,"oast":"green","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["cognitive-COI","prestige-halo"],"pbs":"cognitive-COI","rs":[],"sg":["whole-process"],"st":"observational-quantitative","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"Z475W66K","t":"Editorial: Editorial Practice at the International Journal of Primatology: the Roles of Gender and Country of Affiliation in Participation in Scientific Publication","au":"Setchell, Joanna M.","y":2018,"vn":"International Journal of Primatology","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s10764-018-0067-1","cby":15,"nref":37,"oast":"bronze","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["reviewer-selection","funding-decision"],"st":"observational-quantitative","tp":"5","cc":4,"cp":5,"self":false,"ncoefs":0},{"id":"XML2CJ6J","t":"Preserving the Quality of Scientific Research: Peer Review of Research Articles","au":"Silva, Pali U. K. De","y":2017,"vn":"Fascinating life sciences","ty":"book","tyl":"Book","doi":"10.1007/978-3-319-50627-2_6","cby":17,"nref":118,"oast":"closed","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"book-monograph","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"XFLLARLX","t":"Critical essay: Blinding faith – Paradoxes and pathologies of opacity in peer review","au":"Willmott, Hugh","y":2021,"vn":"Human Relations","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/00187267211016752","cby":21,"nref":122,"oast":"hybrid","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["individual-scoring"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"WZTKU9QM","t":"Disagreement and Agonistic Chance in Peer Review","au":"Roumbanis, Lambros","y":2021,"vn":"Science Technology & Human Values","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/01622439211026016","cby":29,"nref":49,"oast":"hybrid","cx":"Grant","co":["Reliability-noise"],"pco":"Reliability-noise","bs":[],"pbs":"","rs":[],"sg":["panel-consensus"],"st":"qualitative","tp":"-1","cc":1,"cp":2,"self":false,"ncoefs":0},{"id":"VTZWHCCL","t":"How peer-review constrains cognition: on the frontline in the knowledge sector","au":"Cowley, Stephen J.","y":2015,"vn":"Frontiers in Psychology","ty":"journalArticle","tyl":"Journal article","doi":"10.3389/fpsyg.2015.01706","cby":29,"nref":107,"oast":"gold","cx":"Journal","co":["Reliability-noise","Bias","Validity"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"YKTWXZ98","t":"Megajournal mismanagement: Manuscript decision bias and anomalous editor activity at PLOS ONE","au":"Petersen, Alexander M.","y":2019,"vn":"Journal of Informetrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.joi.2019.100974","cby":31,"nref":70,"oast":"hybrid","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["reviewer-selection","funding-decision"],"st":"observational-quantitative","tp":"-1","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"W2VF7UBP","t":"Reviewer recommendations and editors’ decisions for a conservation journal: Is it just a crapshoot? And do Chinese authors get a fair shot?","au":"Campos‐Arceiz, Ahimsa","y":2015,"vn":"Biological Conservation","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.biocon.2015.02.025","cby":34,"nref":18,"oast":"closed","cx":"Journal","co":["Reliability-noise","Bias"],"pco":"Reliability-noise","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"5","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"YJIXI5AV","t":"Managing the Growth of Peer Review at the Royal Society Journals, 1865-1965","au":"Fyfe, Aileen","y":2019,"vn":"Science Technology & Human Values","ty":"journalArticle","tyl":"Journal article","doi":"10.1177/0162243919862868","cby":36,"nref":61,"oast":"hybrid","cx":"Journal","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["reviewer-selection"],"st":"observational-quantitative","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"WVHDHIT8","t":"Gender differences in patterns of authorship do not affect peer review outcomes at an ecology journal","au":"Fox, Charles W.","y":2015,"vn":"Functional Ecology","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/1365-2435.12587","cby":64,"nref":81,"oast":"bronze","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["demographic"],"pbs":"demographic","rs":[],"sg":["reviewer-selection","individual-scoring","funding-decision"],"st":"observational-quantitative","tp":"5","cc":4,"cp":5,"self":false,"ncoefs":0},{"id":"XTFW3RID","t":"Bias against scientific novelty: A prepublication perspective","au":"Liang, Zhentao","y":2022,"vn":"Journal of the Association for Information Science and Technology","ty":"journalArticle","tyl":"Journal article","doi":"10.1002/asi.24725","cby":32,"nref":57,"oast":"closed","cx":"Journal","co":["Bias"],"pco":"Bias","bs":["prestige-halo"],"pbs":"prestige-halo","rs":[],"sg":["proposal-quality","funding-decision"],"st":"observational-quantitative","tp":"0","cc":4,"cp":1,"self":false,"ncoefs":0},{"id":"ZSKAUZH4","t":"The crisis of peer review: Part of the evolution of science","au":"Horta, Hugo","y":2024,"vn":"Higher Education Quarterly","ty":"journalArticle","tyl":"Journal article","doi":"10.1111/hequ.12511","cby":63,"nref":107,"oast":"hybrid","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"XIBX7QW6","t":"The ability of different peer review procedures to flag problematic publications","au":"Horbach, Serge P. J. M.","y":2018,"vn":"Scientometrics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11192-018-2969-2","cby":73,"nref":80,"oast":"hybrid","cx":"Journal","co":["Validity","Interventions"],"pco":"Validity","bs":[],"pbs":"","rs":[],"sg":["post-award-validity"],"st":"observational-quantitative","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"ZHEM7BQQ","t":"Can editors save peer review from peer reviewers?","au":"D’Andrea, Rafael","y":2017,"vn":"PLoS ONE","ty":"journalArticle","tyl":"Journal article","doi":"10.1371/journal.pone.0186111","cby":79,"nref":39,"oast":"gold","cx":"Journal","co":["Bias","Interventions"],"pco":"Bias","bs":["cognitive-COI"],"pbs":"cognitive-COI","rs":[],"sg":["individual-scoring","reviewer-selection","funding-decision"],"st":"simulation-methodological","tp":"0","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"X8TU7KXM","t":"Is the future of peer review automated?","au":"Schulz, Robert","y":2022,"vn":"BMC Research Notes","ty":"journalArticle","tyl":"Journal article","doi":"10.1186/s13104-022-06080-6","cby":87,"nref":36,"oast":"gold","cx":"Journal","co":["Interventions"],"pco":"Interventions","bs":[],"pbs":"","rs":[],"sg":["proposal-quality","individual-scoring"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"X3JD6ZBB","t":"What Do Editors Maximize? Evidence from Four Economics Journals","au":"Card, David","y":2019,"vn":"The Review of Economics and Statistics","ty":"journalArticle","tyl":"Journal article","doi":"10.1162/rest_a_00839","cby":124,"nref":43,"oast":"closed","cx":"Journal","co":["Bias","Validity"],"pco":"Bias","bs":["prestige-halo"],"pbs":"prestige-halo","rs":[],"sg":["individual-scoring","funding-decision","post-award-validity"],"st":"observational-quantitative","tp":"-1","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"ZV4UN4ZA","t":"Scientific Autonomy, Public Accountability, and the Rise of “Peer Review” in the Cold War United States","au":"Baldwin, Melinda","y":2018,"vn":"Isis","ty":"journalArticle","tyl":"Journal article","doi":"10.1086/700070","cby":161,"nref":58,"oast":"closed","cx":"General","co":["general-systemic"],"pco":"general-systemic","bs":[],"pbs":"","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"-1","cc":3,"cp":1,"self":false,"ncoefs":0},{"id":"Z7KLFNBP","t":"Ensuring the Quality, Fairness, and Integrity of Journal Peer Review: A Possible Role of Editors","au":"Resnik, David B.","y":2015,"vn":"Science and Engineering Ethics","ty":"journalArticle","tyl":"Journal article","doi":"10.1007/s11948-015-9625-5","cby":161,"nref":98,"oast":"closed","cx":"Journal","co":["Reliability-noise","Bias","Interventions"],"pco":"Reliability-noise","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["whole-process"],"st":"conceptual-commentary","tp":"0","cc":2,"cp":1,"self":false,"ncoefs":0},{"id":"W8TKHTI7","t":"Project selection in NIH: A natural experiment from ARRA","au":"Park, Hyunwoo","y":2015,"vn":"Research Policy","ty":"journalArticle","tyl":"Journal article","doi":"10.1016/j.respol.2015.03.004","cby":40,"nref":41,"oast":"closed","cx":"Grant","co":["Bias","Validity"],"pco":"Bias","bs":["bias-general"],"pbs":"bias-general","rs":[],"sg":["funding-decision","post-award-validity"],"st":"observational-quantitative","tp":"-1","cc":4,"cp":2,"self":false,"ncoefs":0}]