[{"data":1,"prerenderedAt":892},["ShallowReactive",2],{"navigation_docs_en":3,"\u002Fen\u002Fai-engineering\u002Fevaluate-ai-systems\u002Fch04-2-model-selection":141,"\u002Fen\u002Fai-engineering\u002Fevaluate-ai-systems\u002Fch04-2-model-selection-surround":887},[4],{"title":5,"icon":6,"path":7,"stem":8,"children":9,"page":45},"AI Engineering",null,"\u002Fen\u002Fai-engineering","en\u002F1.ai-engineering",[10,46,77,114],{"title":11,"icon":12,"path":13,"stem":14,"children":15,"page":45},"Introduction to Building AI Applications with Foundation Models","i-lucide-brain-circuit","\u002Fen\u002Fai-engineering\u002Fintro","en\u002F1.ai-engineering\u002F1.intro",[16,20,25,30,35,40],{"title":11,"path":17,"stem":18,"icon":19},"\u002Fen\u002Fai-engineering\u002Fintro\u002Fch01","en\u002F1.ai-engineering\u002F1.intro\u002Fch01","i-lucide-sparkles",{"title":21,"path":22,"stem":23,"icon":24},"The Rise of AI Engineering","\u002Fen\u002Fai-engineering\u002Fintro\u002Fch011-the-rise-of-ai-engineering","en\u002F1.ai-engineering\u002F1.intro\u002Fch011-the-rise-of-ai-engineering","i-lucide-history",{"title":26,"path":27,"stem":28,"icon":29},"Foundation Model Use Cases","\u002Fen\u002Fai-engineering\u002Fintro\u002Fch012-foundation-model-use-cases","en\u002F1.ai-engineering\u002F1.intro\u002Fch012-foundation-model-use-cases","i-lucide-layout-grid",{"title":31,"path":32,"stem":33,"icon":34},"Planning AI Applications","\u002Fen\u002Fai-engineering\u002Fintro\u002Fch013-planning-ai-applications","en\u002F1.ai-engineering\u002F1.intro\u002Fch013-planning-ai-applications","i-lucide-clipboard-list",{"title":36,"path":37,"stem":38,"icon":39},"The AI Engineering Stack","\u002Fen\u002Fai-engineering\u002Fintro\u002Fch014-the-ai-engineering-stack","en\u002F1.ai-engineering\u002F1.intro\u002Fch014-the-ai-engineering-stack","i-lucide-layers",{"title":41,"path":42,"stem":43,"icon":44},"Summary","\u002Fen\u002Fai-engineering\u002Fintro\u002Fch015-summary","en\u002F1.ai-engineering\u002F1.intro\u002Fch015-summary","i-lucide-flag",false,{"title":47,"icon":6,"path":48,"stem":49,"children":50,"page":45},"Understanding Foundation Models","\u002Fen\u002Fai-engineering\u002Funderstanding-foundation-models","en\u002F1.ai-engineering\u002F2.understanding-foundation-models",[51,54,59,64,69,74],{"title":47,"path":52,"stem":53,"icon":12},"\u002Fen\u002Fai-engineering\u002Funderstanding-foundation-models\u002Fch02","en\u002F1.ai-engineering\u002F2.understanding-foundation-models\u002Fch02",{"title":55,"path":56,"stem":57,"icon":58},"Training Data","\u002Fen\u002Fai-engineering\u002Funderstanding-foundation-models\u002Fch02-1-training-data","en\u002F1.ai-engineering\u002F2.understanding-foundation-models\u002Fch02-1-training-data","i-lucide-database",{"title":60,"path":61,"stem":62,"icon":63},"Modeling","\u002Fen\u002Fai-engineering\u002Funderstanding-foundation-models\u002Fch02-2-modeling","en\u002F1.ai-engineering\u002F2.understanding-foundation-models\u002Fch02-2-modeling","i-lucide-network",{"title":65,"path":66,"stem":67,"icon":68},"Post-Training","\u002Fen\u002Fai-engineering\u002Funderstanding-foundation-models\u002Fch02-3-post-training","en\u002F1.ai-engineering\u002F2.understanding-foundation-models\u002Fch02-3-post-training","i-lucide-sliders-horizontal",{"title":70,"path":71,"stem":72,"icon":73},"Sampling","\u002Fen\u002Fai-engineering\u002Funderstanding-foundation-models\u002Fch02-4-sampling","en\u002F1.ai-engineering\u002F2.understanding-foundation-models\u002Fch02-4-sampling","i-lucide-dices",{"title":41,"path":75,"stem":76,"icon":44},"\u002Fen\u002Fai-engineering\u002Funderstanding-foundation-models\u002Fch02-5-summary","en\u002F1.ai-engineering\u002F2.understanding-foundation-models\u002Fch02-5-summary",{"title":78,"path":79,"stem":80,"children":81,"page":45},"Evaluation Methodology","\u002Fen\u002Fai-engineering\u002Fevaluation-methodology","en\u002F1.ai-engineering\u002F3.evaluation-methodology",[82,86,91,96,101,106,111],{"title":78,"path":83,"stem":84,"icon":85},"\u002Fen\u002Fai-engineering\u002Fevaluation-methodology\u002Fch03","en\u002F1.ai-engineering\u002F3.evaluation-methodology\u002Fch03","i-lucide-clipboard-check",{"title":87,"path":88,"stem":89,"icon":90},"Challenges of Evaluating Foundation Models","\u002Fen\u002Fai-engineering\u002Fevaluation-methodology\u002Fch03-1-challenges-of-evaluating-foundation-models","en\u002F1.ai-engineering\u002F3.evaluation-methodology\u002Fch03-1-challenges-of-evaluating-foundation-models","i-lucide-shield-alert",{"title":92,"path":93,"stem":94,"icon":95},"Understanding Language Modeling Metrics","\u002Fen\u002Fai-engineering\u002Fevaluation-methodology\u002Fch03-2-understanding-language-modeling-metrics","en\u002F1.ai-engineering\u002F3.evaluation-methodology\u002Fch03-2-understanding-language-modeling-metrics","i-lucide-sigma",{"title":97,"path":98,"stem":99,"icon":100},"Exact Evaluation","\u002Fen\u002Fai-engineering\u002Fevaluation-methodology\u002Fch03-3-exact-evaluation","en\u002F1.ai-engineering\u002F3.evaluation-methodology\u002Fch03-3-exact-evaluation","i-lucide-check-check",{"title":102,"path":103,"stem":104,"icon":105},"AI as a Judge","\u002Fen\u002Fai-engineering\u002Fevaluation-methodology\u002Fch03-4-ai-as-a-judge","en\u002F1.ai-engineering\u002F3.evaluation-methodology\u002Fch03-4-ai-as-a-judge","i-lucide-scale",{"title":107,"path":108,"stem":109,"icon":110},"Ranking Models with Comparative Evaluation","\u002Fen\u002Fai-engineering\u002Fevaluation-methodology\u002Fch03-5-ranking-models-with-comparative-evaluation","en\u002F1.ai-engineering\u002F3.evaluation-methodology\u002Fch03-5-ranking-models-with-comparative-evaluation","i-lucide-trophy",{"title":41,"path":112,"stem":113,"icon":44},"\u002Fen\u002Fai-engineering\u002Fevaluation-methodology\u002Fch03-6-summary","en\u002F1.ai-engineering\u002F3.evaluation-methodology\u002Fch03-6-summary",{"title":115,"icon":116,"path":117,"stem":118,"children":119,"page":45},"Evaluate AI Systems","i-lucide-binary","\u002Fen\u002Fai-engineering\u002Fevaluate-ai-systems","en\u002F1.ai-engineering\u002F4.evaluate-ai-systems",[120,123,128,133,138],{"title":115,"path":121,"stem":122,"icon":116},"\u002Fen\u002Fai-engineering\u002Fevaluate-ai-systems\u002Fch04","en\u002F1.ai-engineering\u002F4.evaluate-ai-systems\u002Fch04",{"title":124,"path":125,"stem":126,"icon":127},"Evaluation Criteria","\u002Fen\u002Fai-engineering\u002Fevaluate-ai-systems\u002Fch04-1-evaluation-criteria","en\u002F1.ai-engineering\u002F4.evaluate-ai-systems\u002Fch04-1-evaluation-criteria","i-lucide-check-circle-2",{"title":129,"path":130,"stem":131,"icon":132},"Model Selection","\u002Fen\u002Fai-engineering\u002Fevaluate-ai-systems\u002Fch04-2-model-selection","en\u002F1.ai-engineering\u002F4.evaluate-ai-systems\u002Fch04-2-model-selection","i-lucide-cpu",{"title":134,"path":135,"stem":136,"icon":137},"Design Your Evaluation Pipeline","\u002Fen\u002Fai-engineering\u002Fevaluate-ai-systems\u002Fch04-3-design-your-evaluation-pipeline","en\u002F1.ai-engineering\u002F4.evaluate-ai-systems\u002Fch04-3-design-your-evaluation-pipeline","i-lucide-workflow",{"title":41,"path":139,"stem":140,"icon":44},"\u002Fen\u002Fai-engineering\u002Fevaluate-ai-systems\u002Fch04-4-summary","en\u002F1.ai-engineering\u002F4.evaluate-ai-systems\u002Fch04-4-summary",{"id":142,"title":129,"body":143,"description":881,"extension":882,"links":6,"meta":883,"navigation":884,"path":130,"seo":885,"stem":131,"__hash__":886},"docs_en\u002Fen\u002F1.ai-engineering\u002F4.evaluate-ai-systems\u002Fch04-2-model-selection.md",{"type":144,"value":145,"toc":859},"minimark",[146,161,166,170,181,194,199,206,234,237,241,248,254,258,273,297,301,331,337,343,347,449,451,455,458,478,482,485,506,669,673,676,772,776,787,793,800,802,806,818,827,831,845,851],[147,148,149,153],"u-page-hero",{},[150,151,129],"template",{"v-slot:title":152},"",[150,154,155,156,160],{"v-slot:description":152},"At the end of the day, you don't care which model is universally \"the best\" — you care which model is best ",[157,158,159],"strong",{},"for your application",". Model selection maps candidate models along capability, latency, and cost frontiers to find the optimal trade-off.",[162,163,165],"h2",{"id":164},"the-selection-workflow","The Selection Workflow",[167,168,169],"p",{},"Model selection is not a one-time decision; it is an iterative discipline revisited across prompt engineering, retrieval tuning, and fine-tuning.",[167,171,172,173,176,177,180],{},"When evaluating candidate models, distinguish between ",[157,174,175],{},"hard attributes"," and ",[157,178,179],{},"soft attributes",":",[182,183,184,189],"card-group",{},[185,186,188],"card",{"icon":90,"title":187},"Hard Attributes (Fixed Constraints)","Attributes governed by organizational policy or model architecture that are impractical or impossible to alter: commercial licenses, data privacy requirements, model size, and hardware limits.",[185,190,193],{"icon":191,"title":192},"i-lucide-wrench","Soft Attributes (Malleable)","Attributes that can be improved through engineering effort: accuracy, factual consistency, output format, or latency (when self-hosting).",[195,196,198],"h3",{"id":197},"four-step-evaluation-funnel","Four-Step Evaluation Funnel",[167,200,201],{},[202,203],"img",{"alt":204,"src":205},"Figure 4-5. An overview of the evaluation workflow to evaluate models for your application.","\u002Fmedia\u002Ffig-4-5.png",[207,208,209,216,222,228],"ol",{},[210,211,212,215],"li",{},[157,213,214],{},"Filter by Hard Constraints",": Eliminate models violating legal, compliance, privacy, or infrastructure requirements.",[210,217,218,221],{},[157,219,220],{},"Narrow via Public Benchmarks",": Leverage public benchmarks and leaderboards to identify 3–5 top candidate models.",[210,223,224,227],{},[157,225,226],{},"Internal Pipeline Evaluation",": Replay representative traffic through custom evaluation suites to determine domain-specific performance.",[210,229,230,233],{},[157,231,232],{},"Production Observability",": Continuously monitor model drift, error modes, and user feedback post-deployment.",[235,236],"hr",{},[162,238,240],{"id":239},"model-build-vs-buy-self-hosting-vs-model-apis","Model Build vs. Buy: Self-Hosting vs. Model APIs",[167,242,243,244,247],{},"Deploying a model requires an ",[157,245,246],{},"inference service"," that receives user prompts, executes forward passes, and returns completions.",[167,249,250],{},[202,251],{"alt":252,"src":253},"Figure 4-6. An inference service runs the model and provides an interface for users to access the model.","\u002Fmedia\u002Ffig-4-6.png",[195,255,257],{"id":256},"open-weights-vs-truly-open-models","Open Weights vs. Truly Open Models",[259,260,261,267],"ul",{},[210,262,263,266],{},[157,264,265],{},"Open Weight",": Model weights are publicly downloadable, but training code, curation recipes, and pre-training datasets remain proprietary (e.g., Llama 3, Mistral 7B).",[210,268,269,272],{},[157,270,271],{},"Open Model \u002F Open Data",": Both weights and full training datasets are publicly auditable (e.g., OLMo).",[274,275,276,280,283],"warning",{},[195,277,279],{"id":278},"commercial-license-traps","Commercial License Traps",[167,281,282],{},"Open-weight licenses often contain strict usage conditions:",[259,284,285,291],{},[210,286,287,290],{},[157,288,289],{},"User Thresholds",": Llama 2\u002F3 licenses mandate explicit commercial agreements for products exceeding 700M monthly active users.",[210,292,293,296],{},[157,294,295],{},"Distillation Bans",": Many providers forbid using model completions to train or fine-tune competing models.",[195,298,300],{"id":299},"seven-critical-decision-axes","Seven Critical Decision Axes",[182,302,303,308,312,317,322,327],{},[185,304,307],{"icon":305,"title":306},"i-lucide-lock","1. Data Privacy & Sovereignty","Sending data to external APIs is unacceptable for organizations with strict compliance mandates or air-gapped systems (e.g., Samsung's proprietary data leak via ChatGPT).",[185,309,311],{"icon":105,"title":310},"2. Data Lineage & Legal Risk","Training data opacity creates copyright exposure. Commercial contracts often provide copyright indemnification; open-source models place legal liability squarely on the user.",[185,313,316],{"icon":314,"title":315},"i-lucide-trending-up","3. Capability Frontier","While open models are closing the gap, frontier closed models retain advantages in complex reasoning and tool use due to massive compute investments.",[185,318,321],{"icon":319,"title":320},"i-lucide-sliders","4. Features vs. Transparency","Commercial APIs provide out-of-the-box function calling and JSON modes, but rarely expose token logprobs. Self-hosted models grant full access to logits, embeddings, and weights.",[185,323,326],{"icon":324,"title":325},"i-lucide-coins","5. API Cost vs. Engineering Cost","APIs charge linearly per token; self-hosting involves fixed GPU clusters and specialized MLOps talent. At high sustained volume, self-hosting becomes drastically cheaper.",[185,328,330],{"icon":68,"title":329},"6. Version Control & Lock-in","Commercial APIs frequently update or deprecate model snapshots without notice, altering prompt behaviors. Self-hosting allows freezing exact weights indefinitely.",[167,332,333],{},[202,334],{"alt":335,"src":336},"Figure 4-7. The gap between open source models and proprietary models is decreasing on the MMLU benchmark. Image by Maxime Labonne.","\u002Fmedia\u002Ffig-4-7.png",[167,338,339],{},[202,340],{"alt":341,"src":342},"Figure 4-8. Why enterprises care about open source models. Image from the 2024 study by a16z.","\u002Fmedia\u002Ffig-4-8.png",[195,344,346],{"id":345},"comprehensive-comparison-matrix","Comprehensive Comparison Matrix",[348,349,350,367],"table",{},[351,352,353],"thead",{},[354,355,356,361,364],"tr",{},[357,358,360],"th",{"align":359},"left","Dimension",[357,362,363],{"align":359},"Managed Model APIs",[357,365,366],{"align":359},"Self-Hosted Open Weights",[368,369,370,384,397,410,423,436],"tbody",{},[354,371,372,378,381],{},[373,374,375],"td",{"align":359},[157,376,377],{},"Data Privacy",[373,379,380],{"align":359},"Sensitive data leaves infrastructure; risk of provider training on prompts",[373,382,383],{"align":359},"Zero data egress; runs entirely within private VPC or air-gapped on-premise",[354,385,386,391,394],{},[373,387,388],{"align":359},[157,389,390],{},"Capabilities",[373,392,393],{"align":359},"Frontier reasoning, multimodal inputs, massive context windows",[373,395,396],{"align":359},"Competitive for 80% of tasks; slightly lags frontier models",[354,398,399,404,407],{},[373,400,401],{"align":359},[157,402,403],{},"Functionality",[373,405,406],{"align":359},"Turnkey tool use, structured outputs, managed guardrails",[373,408,409],{"align":359},"Full logprob access, custom logits processors, arbitrary fine-tuning",[354,411,412,417,420],{},[373,413,414],{"align":359},[157,415,416],{},"Financials",[373,418,419],{"align":359},"Pay-as-you-go per token; high variable cost at massive scale",[373,421,422],{"align":359},"High capital expenditure\u002Ffixed compute, but near-zero marginal token cost",[354,424,425,430,433],{},[373,426,427],{"align":359},[157,428,429],{},"Operational Burden",[373,431,432],{"align":359},"Zero infrastructure maintenance; dependent on third-party uptime\u002FSLA",[373,434,435],{"align":359},"Requires GPU provisioning, cluster autoscaling, quantization, and monitoring",[354,437,438,443,446],{},[373,439,440],{"align":359},[157,441,442],{},"Deployment Edge",[373,444,445],{"align":359},"Requires active internet connection; subject to global API latency",[373,447,448],{"align":359},"Supports local, offline, on-device execution (smartphones, IoT, local desktops)",[235,450],{},[162,452,454],{"id":453},"navigating-public-benchmarks-leaderboards","Navigating Public Benchmarks & Leaderboards",[167,456,457],{},"Standardized benchmark suites measure broad capabilities across standardized tasks:",[259,459,460,466,472],{},[210,461,462,465],{},[157,463,464],{},"BIG-bench",": 200+ multi-task capabilities curated by Google Research.",[210,467,468,471],{},[157,469,470],{},"lm-evaluation-harness",": EleutherAI's framework unifying 400+ evaluation suites.",[210,473,474,477],{},[157,475,476],{},"OpenAI Evals",": Automated framework for evaluating closed models on standard suites.",[195,479,481],{"id":480},"the-illusion-of-simple-averages","The Illusion of Simple Averages",[167,483,484],{},"Public leaderboards (such as Hugging Face's Open LLM Leaderboard or Stanford HELM) aggregate multiple benchmarks to create unified rankings.",[486,487,488,492],"caution",{},[195,489,491],{"id":490},"flaws-in-public-leaderboard-rankings","Flaws in Public Leaderboard Rankings",[259,493,494,500],{},[210,495,496,499],{},[157,497,498],{},"Unweighted Averaging",": Simple averages treat an 80% score on math the same as an 80% score on common sense, regardless of difficulty.",[210,501,502,505],{},[157,503,504],{},"Benchmark Correlation",": As shown in Table 4-5, MMLU, WinoGrande, and ARC-C have Pearson correlations > 0.85. Averaging them overweights general reasoning while under-representing truthfulness or coding.",[348,507,508,533],{},[351,509,510],{},[354,511,512,515,518,521,524,527,530],{},[357,513,514],{"align":359},"Benchmark",[357,516,517],{"align":359},"ARC-C",[357,519,520],{"align":359},"HellaSwag",[357,522,523],{"align":359},"MMLU",[357,525,526],{"align":359},"TruthfulQA",[357,528,529],{"align":359},"WinoGrande",[357,531,532],{"align":359},"GSM-8K",[368,534,535,563,585,608,628,651],{},[354,536,537,541,544,547,552,555,560],{},[373,538,539],{"align":359},[157,540,517],{},[373,542,543],{"align":359},"1.0000",[373,545,546],{"align":359},"0.4812",[373,548,549],{"align":359},[157,550,551],{},"0.8672",[373,553,554],{"align":359},"0.4809",[373,556,557],{"align":359},[157,558,559],{},"0.8856",[373,561,562],{"align":359},"0.7438",[354,564,565,569,571,573,576,579,582],{},[373,566,567],{"align":359},[157,568,520],{},[373,570,546],{"align":359},[373,572,543],{"align":359},[373,574,575],{"align":359},"0.6105",[373,577,578],{"align":359},"0.4228",[373,580,581],{"align":359},"0.4842",[373,583,584],{"align":359},"0.3547",[354,586,587,591,593,595,597,600,605],{},[373,588,589],{"align":359},[157,590,523],{},[373,592,551],{"align":359},[373,594,575],{"align":359},[373,596,543],{"align":359},[373,598,599],{"align":359},"0.5507",[373,601,602],{"align":359},[157,603,604],{},"0.9011",[373,606,607],{"align":359},"0.7936",[354,609,610,614,616,618,620,622,625],{},[373,611,612],{"align":359},[157,613,526],{},[373,615,554],{"align":359},[373,617,578],{"align":359},[373,619,599],{"align":359},[373,621,543],{"align":359},[373,623,624],{"align":359},"0.4550",[373,626,627],{"align":359},"0.5009",[354,629,630,634,638,640,644,646,648],{},[373,631,632],{"align":359},[157,633,529],{},[373,635,636],{"align":359},[157,637,559],{},[373,639,581],{"align":359},[373,641,642],{"align":359},[157,643,604],{},[373,645,624],{"align":359},[373,647,543],{"align":359},[373,649,650],{"align":359},"0.7979",[354,652,653,657,659,661,663,665,667],{},[373,654,655],{"align":359},[157,656,532],{},[373,658,562],{"align":359},[373,660,584],{"align":359},[373,662,607],{"align":359},[373,664,627],{"align":359},[373,666,650],{"align":359},[373,668,543],{"align":359},[195,670,672],{"id":671},"benchmark-saturation-modern-suites","Benchmark Saturation & Modern Suites",[167,674,675],{},"As models conquer legacy benchmarks, evaluations rapidly shift from grade-school math to graduate-level problem solving:",[259,677,678,731,766],{},[210,679,680,730],{},[157,681,682,683,729],{},"MMLU ",[684,685,688,711],"span",{"className":686},[687],"katex",[684,689,692],{"className":690},[691],"katex-mathml",[693,694,696],"math",{"xmlns":695},"http:\u002F\u002Fwww.w3.org\u002F1998\u002FMath\u002FMathML",[697,698,699,706],"semantics",{},[700,701,702],"mrow",{},[703,704,705],"mo",{},"→",[707,708,710],"annotation",{"encoding":709},"application\u002Fx-tex","\\rightarrow",[684,712,716],{"className":713,"ariaHidden":715},[714],"katex-html","true",[684,717,720,725],{"className":718},[719],"base",[684,721],{"className":722,"style":724},[723],"strut","height:0.3669em;",[684,726,705],{"className":727},[728],"mrel"," MMLU-PRO",": Expanded distractors, multi-step chain-of-thought requirements.",[210,732,733,765],{},[157,734,735,736,764],{},"GSM-8K ",[684,737,739,752],{"className":738},[687],[684,740,742],{"className":741},[691],[693,743,744],{"xmlns":695},[697,745,746,750],{},[700,747,748],{},[703,749,705],{},[707,751,710],{"encoding":709},[684,753,755],{"className":754,"ariaHidden":715},[714],[684,756,758,761],{"className":757},[719],[684,759],{"className":760,"style":724},[723],[684,762,705],{"className":763},[728]," MATH Level 5",": Advanced competition-grade mathematics.",[210,767,768,771],{},[157,769,770],{},"GPQA",": Graduate-level physics, chemistry, and biology questions validated by PhD experts.",[195,773,775],{"id":774},"are-frontier-models-getting-worse","Are Frontier Models Getting Worse?",[167,777,778,779,786],{},"Users frequently report that models degrade after provider updates. A Stanford\u002FUC Berkeley study (",[780,781,785],"a",{"href":782,"rel":783},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2307.09009",[784],"nofollow","Chen et al., 2023",") confirmed measurable shifts in performance across versions:",[167,788,789],{},[202,790],{"alt":791,"src":792},"Figure 4-9. Changes in the performances of GPT-3.5 and GPT-4 from March 2023 to June 2023 on certain benchmarks (Chen et al., 2023).","\u002Fmedia\u002Ffig-4-9.png",[794,795,796,797],"tip",{},"Model updates are rarely strictly superior across all dimensions. Fine-tuning to improve safety or instruction adherence frequently causes regressions in coding syntax or niche reasoning. ",[157,798,799],{},"Prompts are not portable across model snapshots.",[235,801],{},[162,803,805],{"id":804},"data-contamination-in-public-benchmarks","Data Contamination in Public Benchmarks",[167,807,808,809,813,814,817],{},"Data contamination (",[810,811,812],"em",{},"data leakage"," or ",[810,815,816],{},"training on the test set",") occurs when evaluation prompts exist inside the model's pre-training web scrape.",[167,819,820,821,826],{},"In a famous demonstration, Rylan Schaeffer (",[780,822,825],{"href":823,"rel":824},"https:\u002F\u002Farxiv.org\u002Fabs\u002F2309.08632",[784],"2023",") trained a tiny 1-million parameter model exclusively on public benchmark test sets, achieving near-perfect scores that humiliated massive models.",[195,828,830],{"id":829},"detection-and-decontamination","Detection and Decontamination",[207,832,833,839],{},[210,834,835,838],{},[157,836,837],{},"N-gram Overlap",": Scans the pre-training corpus for exact 8–13 token overlaps with test sets. Highly accurate, but requires access to raw pre-training text dumps.",[210,840,841,844],{},[157,842,843],{},"Perplexity Probing",": Identifies test samples where the model exhibits anomalously low perplexity, indicating rote memorization.",[167,846,847],{},[202,848],{"alt":849,"src":850},"Figure 4-10. Relative difference in GPT-3's performance when evaluating using only the clean sample compared to evaluating using the whole benchmark.","\u002Fmedia\u002Ffig-4-10.png",[852,853,854,855,858],"note",{},"Public benchmarks filter out bad models, but they cannot identify the best model for your unique workload. You must engineer a ",[157,856,857],{},"custom, private evaluation pipeline",".",{"title":152,"searchDepth":860,"depth":860,"links":861},2,[862,866,872,878],{"id":164,"depth":860,"text":165,"children":863},[864],{"id":197,"depth":865,"text":198},3,{"id":239,"depth":860,"text":240,"children":867},[868,869,870,871],{"id":256,"depth":865,"text":257},{"id":278,"depth":865,"text":279},{"id":299,"depth":865,"text":300},{"id":345,"depth":865,"text":346},{"id":453,"depth":860,"text":454,"children":873},[874,875,876,877],{"id":480,"depth":865,"text":481},{"id":490,"depth":865,"text":491},{"id":671,"depth":865,"text":672},{"id":774,"depth":865,"text":775},{"id":804,"depth":860,"text":805,"children":879},[880],{"id":829,"depth":865,"text":830},"A strategic guide to selecting foundation models, comparing self-hosting vs model APIs, and critically navigating public benchmarks and leaderboards.","md",{},{"icon":132},{"title":129,"description":881},"Jhj4x59RfOhtnhHW0yJd3b9E22pbMOSJlFHZhBkZ97Y",[888,890],{"title":124,"path":125,"stem":126,"description":889,"icon":127,"children":-1},"How to define and calculate criteria for evaluating AI applications, including domain capabilities, factual consistency, safety, instruction-following, and cost-latency tradeoffs.",{"title":134,"path":135,"stem":136,"description":891,"icon":137,"children":-1},"A practical guide to architecting production evaluation pipelines, defining scoring rubrics, slicing datasets, and avoiding Simpson's paradox.",1789413992443]