{"config":"agent_bash_reliability","id":"arena-agent-bash-reliability","title":"Agents · bash reliability","category":"Coding and agents","metric":"IPS score","direction":"higher","precision":6,"publishedAt":"2026-11-04","interval":"95% confidence interval","observationLabel":"Observations","description":"Agent Arena signal scores from the official dataset. Each signal is a separate measurement; it is not a success percentage.","limitations":["The population is the models in this source board, including variants and models without an available API price.","Ranks use the published confidence bounds: one plus the number of models whose lower bound exceeds this model’s upper bound. Shared ranks are not proof of equal performance."],"omissions":[],"rows":[{"name":"Claude Sonnet 5 (High)","organization":"anthropic","score":0.1508228006557272,"low":0.14521544847403287,"high":0.15643015283742154,"observations":231347,"sourceRank":1,"publishedAt":"2026-11-04","rank":1},{"name":"Claude Opus 5.5 (Max)","organization":"anthropic","score":0.15006150155739292,"low":0.1457867943105792,"high":0.15433620880420665,"observations":264620,"sourceRank":2,"publishedAt":"2026-11-04","rank":1},{"name":"GPT-6.1 Terra (Max)","organization":"openai","score":0.13519894497043167,"low":0.1313777310773117,"high":0.13902015886355162,"observations":528425,"sourceRank":3,"publishedAt":"2026-11-04","rank":3},{"name":"Claude Sonnet 5.5 (Max)","organization":"anthropic","score":0.12645116230327919,"low":0.11968644401357194,"high":0.13321588059298642,"observations":140397,"sourceRank":4,"publishedAt":"2026-11-04","rank":3},{"name":"GPT-5.6 Sol (xHigh)","organization":"openai","score":0.11314201452552014,"low":0.10650923510851848,"high":0.1197747939425218,"observations":213048,"sourceRank":5,"publishedAt":"2026-11-04","rank":4},{"name":"GPT-6 Sol (Max)","organization":"openai","score":0.1126766987284395,"low":0.10716691135659374,"high":0.11818648610028525,"observations":227574,"sourceRank":6,"publishedAt":"2026-11-04","rank":5},{"name":"Claude Fable 5 (High)","organization":"anthropic","score":0.1019304008256458,"low":0.09316186062442952,"high":0.11069894102686209,"observations":125868,"sourceRank":7,"publishedAt":"2026-11-04","rank":5},{"name":"Claude Opus 5 (Max)","organization":"anthropic","score":0.10026049976757415,"low":0.09449691735860154,"high":0.10602408217654676,"observations":243903,"sourceRank":8,"publishedAt":"2026-11-04","rank":7},{"name":"Claude Fable 5.1 (Max)","organization":"anthropic","score":0.10021575155052387,"low":0.0913072942599801,"high":0.10912420884106763,"observations":114914,"sourceRank":9,"publishedAt":"2026-11-04","rank":5},{"name":"Gemini 3.8 Flash (High)","organization":"google","score":0.09969412697854246,"low":0.09578005582191572,"high":0.10360819813516921,"observations":846362,"sourceRank":10,"publishedAt":"2026-11-04","rank":7},{"name":"Grok 4.6 (xHigh)","organization":"xai","score":0.09816708307943925,"low":0.0879998480152325,"high":0.108334318143646,"observations":72255,"sourceRank":11,"publishedAt":"2026-11-04","rank":5},{"name":"GPT-6 Luna (Max)","organization":"openai","score":0.09595902935490164,"low":0.0931781801284115,"high":0.09873987858139177,"observations":2849515,"sourceRank":12,"publishedAt":"2026-11-04","rank":7},{"name":"Grok 4.7 (xHigh)","organization":"xai","score":0.09442562364210391,"low":0.08818539734603348,"high":0.10066584993817435,"observations":256072,"sourceRank":13,"publishedAt":"2026-11-04","rank":7},{"name":"Muse Wonder Full (Max)","organization":"meta","score":0.08401119731523302,"low":0.07962843379882455,"high":0.08839396083164149,"observations":563468,"sourceRank":14,"publishedAt":"2026-11-04","rank":12},{"name":"GPT-6 Astra (Max)","organization":"openai","score":0.08003329514903396,"low":0.0685250977164708,"high":0.0915414925815971,"observations":42658,"sourceRank":15,"publishedAt":"2026-11-04","rank":11},{"name":"Kimi K3.1 Pro (Max)","organization":"moonshot","score":0.07794588433184671,"low":0.07233981392031609,"high":0.08355195474337733,"observations":376791,"sourceRank":16,"publishedAt":"2026-11-04","rank":14},{"name":"Claude Haiku 5.5 (High)","organization":"anthropic","score":0.0736200850543069,"low":0.06797702209346354,"high":0.07926314801515025,"observations":471690,"sourceRank":17,"publishedAt":"2026-11-04","rank":15},{"name":"GPT-6 Luna (Medium)","organization":"openai","score":0.06934343344417906,"low":0.06510144494275368,"high":0.07358542194560445,"observations":715939,"sourceRank":18,"publishedAt":"2026-11-04","rank":15},{"name":"Hy4","organization":"tencent","score":0.0661550685070105,"low":0.05465018220887098,"high":0.07765995480515002,"observations":117034,"sourceRank":19,"publishedAt":"2026-11-04","rank":15},{"name":"Gemini 3.7 Flash (High)","organization":"google","score":0.06260792219261105,"low":0.054019253443538424,"high":0.07119659094168367,"observations":257912,"sourceRank":20,"publishedAt":"2026-11-04","rank":16},{"name":"GPT-6.1 Sol (Max)","organization":"openai","score":0.05051596565205088,"low":0.04357483343865495,"high":0.05745709786544681,"observations":240656,"sourceRank":21,"publishedAt":"2026-11-04","rank":19},{"name":"Muse Wonder (Max)","organization":"meta","score":0.04973851110097044,"low":0.040897029126197484,"high":0.058579993075743395,"observations":221013,"sourceRank":22,"publishedAt":"2026-11-04","rank":19},{"name":"Qwen3.7-Max","organization":"alibaba","score":0.035785046749171046,"low":0.011339900022766802,"high":0.06023019347557529,"observations":42865,"sourceRank":23,"publishedAt":"2026-11-04","rank":19},{"name":"DeepSeek-V4.1-Flash-20261017 (Max)","organization":"deepseek","score":0.03204124891338392,"low":0.028929523256948954,"high":0.03515297456981888,"observations":2502923,"sourceRank":24,"publishedAt":"2026-11-04","rank":23},{"name":"Beam 1.0 501B A23B","organization":"reflection","score":0.03067998150274953,"low":0.002456205935570315,"high":0.058903757069928744,"observations":93649,"sourceRank":25,"publishedAt":"2026-11-04","rank":19},{"name":"MiMo-V2.6-Pro","organization":"xiaomi","score":0.02227593726946403,"low":0.015637871479810304,"high":0.028914003059117756,"observations":621400,"sourceRank":26,"publishedAt":"2026-11-04","rank":24},{"name":"Inkling-Small","organization":"thinky","score":0.0211740013312193,"low":-0.008848105275441328,"high":0.05119610793787993,"observations":16293,"sourceRank":27,"publishedAt":"2026-11-04","rank":21},{"name":"GPT-6 Luna (Low)","organization":"openai","score":0.01937155089566736,"low":0.012094858129879805,"high":0.026648243661454917,"observations":275807,"sourceRank":28,"publishedAt":"2026-11-04","rank":24},{"name":"Muse Spark 1.3 (Max)","organization":"meta","score":0.018689522433702477,"low":0.010873259156009668,"high":0.026505785711395287,"observations":532135,"sourceRank":29,"publishedAt":"2026-11-04","rank":24},{"name":"DeepSeek-V4-Pro (High, 20260813)","organization":"deepseek","score":0.017976244055739776,"low":0.005068898358150114,"high":0.030883589753329437,"observations":86502,"sourceRank":30,"publishedAt":"2026-11-04","rank":23},{"name":"Gemini 3.1 Pro Preview","organization":"google","score":0.016872090261136424,"low":-0.015214562191167388,"high":0.04895874271344024,"observations":35300,"sourceRank":31,"publishedAt":"2026-11-04","rank":21},{"name":"Inkling","organization":"thinky","score":0.013742137782286001,"low":-0.015964380170681556,"high":0.043448655735253555,"observations":17111,"sourceRank":32,"publishedAt":"2026-11-04","rank":22},{"name":"GPT-5.6 Terra (xHigh)","organization":"openai","score":0.008166428813090182,"low":0.0004487614119364613,"high":0.0158840962142439,"observations":211380,"sourceRank":33,"publishedAt":"2026-11-04","rank":24},{"name":"Qwen3.7-Plus","organization":"alibaba","score":0.00814525401874397,"low":-0.020271494462508567,"high":0.036562002499996506,"observations":32326,"sourceRank":34,"publishedAt":"2026-11-04","rank":23},{"name":"MiMo-V2.5-Pro","organization":"xiaomi","score":-0.008051783039461219,"low":-0.0284641182240613,"high":0.012360552145138863,"observations":55140,"sourceRank":35,"publishedAt":"2026-11-04","rank":25},{"name":"MiniMax-M3","organization":"minimax","score":-0.020824014905200167,"low":-0.03883822124307383,"high":-0.0028098085673265057,"observations":80936,"sourceRank":36,"publishedAt":"2026-11-04","rank":31},{"name":"Muse Spark 1.1","organization":"meta","score":-0.024235672156713323,"low":-0.038725216619166974,"high":-0.00974612769425967,"observations":291738,"sourceRank":37,"publishedAt":"2026-11-04","rank":32},{"name":"Qwen3.8-Max","organization":"alibaba","score":-0.028594353419823776,"low":-0.03540155980174671,"high":-0.02178714703790084,"observations":594068,"sourceRank":38,"publishedAt":"2026-11-04","rank":35},{"name":"Hy4 preview","organization":"tencent","score":-0.030979303026754362,"low":-0.03600632779537412,"high":-0.025952278258134606,"observations":1101920,"sourceRank":39,"publishedAt":"2026-11-04","rank":35},{"name":"Muse Spark 1.2 (xHigh)","organization":"meta","score":-0.037641865595629015,"low":-0.04881583101842689,"high":-0.02646790017283114,"observations":212382,"sourceRank":40,"publishedAt":"2026-11-04","rank":35},{"name":"MiMo-V2.6-Flash","organization":"xiaomi","score":-0.041423985714488715,"low":-0.0508713480137472,"high":-0.03197662341523023,"observations":379502,"sourceRank":41,"publishedAt":"2026-11-04","rank":36},{"name":"GLM-5.2 (Max)","organization":"zai","score":-0.045286386409343765,"low":-0.05300634994050308,"high":-0.03756642287818445,"observations":468965,"sourceRank":42,"publishedAt":"2026-11-04","rank":38},{"name":"Kimi K3 (Max)","organization":"moonshot","score":-0.046981915281220615,"low":-0.05339883274902969,"high":-0.04056499781341154,"observations":682226,"sourceRank":43,"publishedAt":"2026-11-04","rank":40},{"name":"MiniMax-M3.1-Flash","organization":"minimax","score":-0.049564501523616994,"low":-0.059181244053592506,"high":-0.03994775899364148,"observations":310814,"sourceRank":44,"publishedAt":"2026-11-04","rank":40},{"name":"GLM-5.3 (Max)","organization":"zai","score":-0.0539018311421737,"low":-0.06224958426823677,"high":-0.04555407801611063,"observations":521367,"sourceRank":45,"publishedAt":"2026-11-04","rank":40},{"name":"Gemini 4 Carbon (High)","organization":"google","score":-0.05498270400050406,"low":-0.0624238100159011,"high":-0.04754159798510702,"observations":476207,"sourceRank":46,"publishedAt":"2026-11-04","rank":40},{"name":"Gemini 4 Argon (High)","organization":"google","score":-0.056096162471543615,"low":-0.06756904815410757,"high":-0.04462327678897966,"observations":478113,"sourceRank":47,"publishedAt":"2026-11-04","rank":40},{"name":"Mistral Large 4","organization":"mistral","score":-0.05897129223842287,"low":-0.08030325116977093,"high":-0.03763933330707482,"observations":36597,"sourceRank":48,"publishedAt":"2026-11-04","rank":38},{"name":"Mistral Medium 3.5","organization":"mistral","score":-0.07318496743833822,"low":-0.11230698804495914,"high":-0.034062946831717296,"observations":32021,"sourceRank":49,"publishedAt":"2026-11-04","rank":36},{"name":"Spark-X2.6","organization":"synlan","score":-0.08660866091977862,"low":-0.10985112576797922,"high":-0.06336619607157802,"observations":119972,"sourceRank":50,"publishedAt":"2026-11-04","rank":47},{"name":"GLM-5.3-Flash","organization":"zai","score":-0.11254340916739577,"low":-0.12105199181571248,"high":-0.10403482651907907,"observations":962019,"sourceRank":51,"publishedAt":"2026-11-04","rank":49},{"name":"Step 5 Preview","organization":"stepfun","score":-0.11686295148647055,"low":-0.1308162534176445,"high":-0.1029096495552966,"observations":310483,"sourceRank":52,"publishedAt":"2026-11-04","rank":49},{"name":"Qwen3.8-27B","organization":"alibaba","score":-0.11709749102275543,"low":-0.12948716995399448,"high":-0.10470781209151639,"observations":359452,"sourceRank":53,"publishedAt":"2026-11-04","rank":49},{"name":"NVIDIA-Nemotron-3.5-Super-VL-120B-A12B (NVFP4)","organization":"nvidia","score":-0.12788637838373842,"low":-0.14349361443614894,"high":-0.11227914233132791,"observations":147074,"sourceRank":54,"publishedAt":"2026-11-04","rank":50},{"name":"Hy3","organization":"tencent","score":-0.13668291936822444,"low":-0.17214769921147907,"high":-0.1012181395249698,"observations":13169,"sourceRank":55,"publishedAt":"2026-11-04","rank":49},{"name":"Nemotron 3 Ultra","organization":"nvidia","score":-0.15384991878715074,"low":-0.2740689100210913,"high":-0.03363092755321018,"observations":5428,"sourceRank":56,"publishedAt":"2026-11-04","rank":36},{"name":"Qwen3.8-Flash-Next","organization":"alibaba","score":-0.21783810379257554,"low":-0.2254252555237149,"high":-0.21025095206143618,"observations":1174420,"sourceRank":57,"publishedAt":"2026-11-04","rank":56},{"name":"Solar Pro 4","organization":"upstage","score":-0.4482038603362475,"low":-0.5250015500941241,"high":-0.3714061705783709,"observations":17227,"sourceRank":58,"publishedAt":"2026-11-04","rank":58}],"source":{"id":"arena","publisher":"Arena (lmarena-ai)","href":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset","licence":"CC-BY-4.0","licenceUrl":"https://creativecommons.org/licenses/by/4.0/","revision":"8b6deb69ace01ee8dcda9e46eba7a5e8a808ec53","fetchedAt":"2026-11-05T06:35:05.520Z","evidence":[{"file":"a7a159f557a1e594a5d41218fe8d53c63c63daa39b655e51d6fc5b482a17d059.json","sha256":"a7a159f557a1e594a5d41218fe8d53c63c63daa39b655e51d6fc5b482a17d059","url":"https://huggingface.co/api/datasets/lmarena-ai/leaderboard-dataset","revision":null,"role":"metadata"},{"file":"88a46d472f35274fb6b7a852c3d16c8ee8007d51b37f290161537bec26895ee4.txt","sha256":"88a46d472f35274fb6b7a852c3d16c8ee8007d51b37f290161537bec26895ee4","url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/raw/8b6deb69ace01ee8dcda9e46eba7a5e8a808ec53/README.md","revision":"8b6deb69ace01ee8dcda9e46eba7a5e8a808ec53","role":"licence"},{"file":"d1ac4050ea00c1be6589371c86e83e5ecfe9ee016362dea6985b8e94703cc150.parquet","sha256":"d1ac4050ea00c1be6589371c86e83e5ecfe9ee016362dea6985b8e94703cc150","url":"https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset/resolve/8b6deb69ace01ee8dcda9e46eba7a5e8a808ec53/agent_bash_reliability/latest-00000-of-00001.parquet","revision":"8b6deb69ace01ee8dcda9e46eba7a5e8a808ec53","role":"rows"}]},"status":"fresh"}