{ "run_id": "run-20260910T064154-bada0b", "started_at": 1789022514, "official": true, "page": "https://ranking.ai-roundtable.de/runs/run-20260910T064154-bada0b", "raw_jsonl_url": "https://licenses.ai-roundtable.de/r2/runs/run-20260910T064154-bada0b/raw.jsonl", "progress_jsonl_url": "https://licenses.ai-roundtable.de/r2/runs/run-20260910T064154-bada0b/progress.jsonl", "summary": { "run_id": "run-20260910T064154-bada0b", "question_bank_version": "2026-09-09-v5", "code_version": "0.2.13", "official": true, "started_at": 1789022514.8059044, "judges_active": [ "gpt-5", "claude-opus-4-8", "claude-opus-4-7", "mistral-large-2" ], "by_model_domain": [ { "model_id": "claude-fable-5-1", "domain": "jura", "reasoning_mode": "on", "reasoning_level": "adaptive", "reasoning_vendor_default": true, "score_mean": 0.9074007936507936, "score_ci_lo": 0.8799603174603174, "score_ci_hi": 0.9336507936507936, "top100": 91, "n_cells": 35, "closed_accuracy": null, "open_rubric_score": 0.9074007936507936, "hallu_rate_mean": 1.6857142857142857, "hallu_prevalence": 1, "hallu_per_1k_tokens": 0.5991490053111005, "contradicted_rate_mean": 0.02857142857142857, "extra_claims_rate_mean": 1.6571428571428573, "extra_false_rate_mean": 0.5642857142857143, "extra_unsupported_rate_mean": 1.0928571428571427, "hallu_top100": 57, "inter_rater_agreement": 0.8689002267573696, "n_judges": 4, "score_mean_ex_family": 0.898344671201814, "self_judge_delta": 0.009056122448979576, "n_cells_ex_family": 35, "false_claim_types": { "az": 16, "paragraph": 125, "study": 0, "number": 2, "other": 47 }, "tokens_in_total": 22926, "tokens_out_total": 98473, "n_failed": 0, "n_timeout": 0, "duration_median_s": 41, "duration_p90_s": 56.9, "reliability_source": "derived-from-raw" }, { "model_id": "claude-fable-5-1", "domain": "medizin", "reasoning_mode": "on", "reasoning_level": "adaptive", "reasoning_vendor_default": true, "score_mean": 0.8767625661375662, "score_ci_lo": 0.8290740740740741, "score_ci_hi": 0.9164417989417989, "top100": 88, "n_cells": 30, "closed_accuracy": null, "open_rubric_score": 0.8767625661375662, "hallu_rate_mean": 1.5, "hallu_prevalence": 1, "hallu_per_1k_tokens": 0.7092422140965831, "contradicted_rate_mean": 0.16666666666666666, "extra_claims_rate_mean": 1.3333333333333333, "extra_false_rate_mean": 0.30833333333333335, "extra_unsupported_rate_mean": 1.025, "hallu_top100": 62, "inter_rater_agreement": 0.8400132275132275, "n_judges": 4, "score_mean_ex_family": 0.8566269841269841, "self_judge_delta": 0.020135582010582076, "n_cells_ex_family": 30, "false_claim_types": { "az": 0, "paragraph": 0, "study": 1, "number": 51, "other": 91 }, "tokens_in_total": 17883, "tokens_out_total": 63448, "n_failed": 0, "n_timeout": 0, "duration_median_s": 35, "duration_p90_s": 47.6, "reliability_source": "derived-from-raw" }, { "model_id": "claude-fable-5-1", "domain": "steuerrecht", "reasoning_mode": "on", "reasoning_level": "adaptive", "reasoning_vendor_default": true, "score_mean": 0.8091674230787134, "score_ci_lo": 0.740820648885165, "score_ci_hi": 0.866442652329749, "top100": 81, "n_cells": 31, "closed_accuracy": null, "open_rubric_score": 0.8091674230787134, "hallu_rate_mean": 4, "hallu_prevalence": 1, "hallu_per_1k_tokens": 0.6659148273454703, "contradicted_rate_mean": 0.3790322580645161, "extra_claims_rate_mean": 3.620967741935484, "extra_false_rate_mean": 1.7258064516129032, "extra_unsupported_rate_mean": 1.8951612903225807, "hallu_top100": 0, "inter_rater_agreement": 0.8319508448540707, "n_judges": 4, "score_mean_ex_family": 0.7878956616859842, "self_judge_delta": 0.021271761392729127, "n_cells_ex_family": 31, "false_claim_types": { "az": 11, "paragraph": 223, "study": 0, "number": 44, "other": 94 }, "tokens_in_total": 20250, "tokens_out_total": 186210, "n_failed": 0, "n_timeout": 0, "duration_median_s": 77.4, "duration_p90_s": 113.9, "reliability_source": "derived-from-raw" }, { "model_id": "claude-fable-5-1", "domain": "wirtschaftsrecht", "reasoning_mode": "on", "reasoning_level": "adaptive", "reasoning_vendor_default": true, "score_mean": 0.7334093915343914, "score_ci_lo": 0.6546064814814815, "score_ci_hi": 0.8086805555555556, "top100": 73, "n_cells": 30, "closed_accuracy": null, "open_rubric_score": 0.7334093915343914, "hallu_rate_mean": 3.7444444444444445, "hallu_prevalence": 1, "hallu_per_1k_tokens": 0.8136146460293432, "contradicted_rate_mean": 0.5694444444444444, "extra_claims_rate_mean": 3.175, "extra_false_rate_mean": 1.5527777777777778, "extra_unsupported_rate_mean": 1.6222222222222222, "hallu_top100": 6, "inter_rater_agreement": 0.7120899470899471, "n_judges": 4, "score_mean_ex_family": 0.679702380952381, "self_judge_delta": 0.05370701058201044, "n_cells_ex_family": 30, "false_claim_types": { "az": 4, "paragraph": 231, "study": 0, "number": 12, "other": 72 }, "tokens_in_total": 22171, "tokens_out_total": 138067, "n_failed": 0, "n_timeout": 0, "duration_median_s": 64.8, "duration_p90_s": 106.3, "reliability_source": "derived-from-raw" }, { "model_id": "claude-opus-4-7", "domain": "jura", "reasoning_mode": "off", "reasoning_level": "off", "reasoning_vendor_default": true, "score_mean": 0.7913775510204082, "score_ci_lo": 0.7303174603174603, "score_ci_hi": 0.8492261904761905, "top100": 79, "n_cells": 35, "closed_accuracy": null, "open_rubric_score": 0.7913775510204082, "hallu_rate_mean": 1.3071428571428572, "hallu_prevalence": 0.9142857142857143, "hallu_per_1k_tokens": 1.4779996123279704, "contradicted_rate_mean": 0.2, "extra_claims_rate_mean": 1.1071428571428572, "extra_false_rate_mean": 0.6285714285714286, "extra_unsupported_rate_mean": 0.4785714285714286, "hallu_top100": 67, "inter_rater_agreement": 0.8209297052154194, "n_judges": 4, "score_mean_ex_family": 0.8017970521541951, "self_judge_delta": -0.010419501133786868, "n_cells_ex_family": 35, "false_claim_types": { "az": 2, "paragraph": 90, "study": 0, "number": 5, "other": 29 }, "tokens_in_total": 23031, "tokens_out_total": 30954, "n_failed": 0, "n_timeout": 0, "duration_median_s": 15.5, "duration_p90_s": 19.3, "reliability_source": "derived-from-raw" }, { "model_id": "claude-opus-4-7", "domain": "medizin", "reasoning_mode": "off", "reasoning_level": "off", "reasoning_vendor_default": true, "score_mean": 0.8509160052910053, "score_ci_lo": 0.803670634920635, "score_ci_hi": 0.8937037037037037, "top100": 85, "n_cells": 30, "closed_accuracy": null, "open_rubric_score": 0.8509160052910053, "hallu_rate_mean": 1.25, "hallu_prevalence": 1, "hallu_per_1k_tokens": 1.2481278082875686, "contradicted_rate_mean": 0.10833333333333334, "extra_claims_rate_mean": 1.1416666666666666, "extra_false_rate_mean": 0.26666666666666666, "extra_unsupported_rate_mean": 0.875, "hallu_top100": 68, "inter_rater_agreement": 0.8476455026455026, "n_judges": 4, "score_mean_ex_family": 0.822063492063492, "self_judge_delta": 0.028852513227513255, "n_cells_ex_family": 30, "false_claim_types": { "az": 0, "paragraph": 0, "study": 2, "number": 37, "other": 85 }, "tokens_in_total": 17973, "tokens_out_total": 30045, "n_failed": 0, "n_timeout": 0, "duration_median_s": 18.6, "duration_p90_s": 23.5, "reliability_source": "derived-from-raw" }, { "model_id": "claude-opus-4-7", "domain": "steuerrecht", "reasoning_mode": "off", "reasoning_level": "off", "reasoning_vendor_default": true, "score_mean": 0.7321847507331378, "score_ci_lo": 0.6933630661453242, "score_ci_hi": 0.7685035842293906, "top100": 73, "n_cells": 31, "closed_accuracy": null, "open_rubric_score": 0.7321847507331378, "hallu_rate_mean": 3.0483870967741935, "hallu_prevalence": 1, "hallu_per_1k_tokens": 2.3084815321477428, "contradicted_rate_mean": 0.3870967741935484, "extra_claims_rate_mean": 2.661290322580645, "extra_false_rate_mean": 1.4516129032258065, "extra_unsupported_rate_mean": 1.2096774193548387, "hallu_top100": 23, "inter_rater_agreement": 0.7766117395149653, "n_judges": 4, "score_mean_ex_family": 0.7271866126704836, "self_judge_delta": 0.004998138062654256, "n_cells_ex_family": 31, "false_claim_types": { "az": 11, "paragraph": 143, "study": 0, "number": 40, "other": 68 }, "tokens_in_total": 20343, "tokens_out_total": 40936, "n_failed": 0, "n_timeout": 0, "duration_median_s": 22.5, "duration_p90_s": 29, "reliability_source": "derived-from-raw" }, { "model_id": "claude-opus-4-7", "domain": "wirtschaftsrecht", "reasoning_mode": "off", "reasoning_level": "off", "reasoning_vendor_default": true, "score_mean": 0.7806084656084656, "score_ci_lo": 0.7346230158730158, "score_ci_hi": 0.8243088624338625, "top100": 78, "n_cells": 30, "closed_accuracy": null, "open_rubric_score": 0.7806084656084656, "hallu_rate_mean": 2.9916666666666667, "hallu_prevalence": 1, "hallu_per_1k_tokens": 2.0591946770677985, "contradicted_rate_mean": 0.24166666666666667, "extra_claims_rate_mean": 2.75, "extra_false_rate_mean": 1.525, "extra_unsupported_rate_mean": 1.225, "hallu_top100": 25, "inter_rater_agreement": 0.7299338624338624, "n_judges": 4, "score_mean_ex_family": 0.7458068783068783, "self_judge_delta": 0.03480158730158733, "n_cells_ex_family": 30, "false_claim_types": { "az": 10, "paragraph": 185, "study": 0, "number": 25, "other": 62 }, "tokens_in_total": 22261, "tokens_out_total": 43585, "n_failed": 0, "n_timeout": 0, "duration_median_s": 25.7, "duration_p90_s": 37.6, "reliability_source": "derived-from-raw" }, { "model_id": "claude-opus-4-8", "domain": "jura", "reasoning_mode": "off", "reasoning_level": "off", "reasoning_vendor_default": true, "score_mean": 0.7978883219954648, "score_ci_lo": 0.7592375283446712, "score_ci_hi": 0.835249433106576, "top100": 80, "n_cells": 35, "closed_accuracy": null, "open_rubric_score": 0.7978883219954648, "hallu_rate_mean": 0.7357142857142858, "hallu_prevalence": 0.8, "hallu_per_1k_tokens": 0.7992178528197648, "contradicted_rate_mean": 0.04285714285714286, "extra_claims_rate_mean": 0.6928571428571428, "extra_false_rate_mean": 0.2785714285714286, "extra_unsupported_rate_mean": 0.4142857142857143, "hallu_top100": 81, "inter_rater_agreement": 0.7739455782312925, "n_judges": 4, "score_mean_ex_family": 0.809767573696145, "self_judge_delta": -0.011879251700680271, "n_cells_ex_family": 35, "false_claim_types": { "az": 0, "paragraph": 51, "study": 0, "number": 7, "other": 29 }, "tokens_in_total": 22856, "tokens_out_total": 32219, "n_failed": 0, "n_timeout": 0, "duration_median_s": 15, "duration_p90_s": 18.9, "reliability_source": "derived-from-raw" }, { "model_id": "claude-opus-4-8", "domain": "medizin", "reasoning_mode": "off", "reasoning_level": "off", "reasoning_vendor_default": true, "score_mean": 0.8063624338624339, "score_ci_lo": 0.7477314814814815, "score_ci_hi": 0.861111111111111, "top100": 81, "n_cells": 30, "closed_accuracy": null, "open_rubric_score": 0.8063624338624339, "hallu_rate_mean": 0.85, "hallu_prevalence": 0.9666666666666667, "hallu_per_1k_tokens": 0.8728094194961665, "contradicted_rate_mean": 0.15, "extra_claims_rate_mean": 0.7, "extra_false_rate_mean": 0.1, "extra_unsupported_rate_mean": 0.6, "hallu_top100": 78, "inter_rater_agreement": 0.7923677248677249, "n_judges": 4, "score_mean_ex_family": 0.7773941798941799, "self_judge_delta": 0.028968253968253976, "n_cells_ex_family": 30, "false_claim_types": { "az": 0, "paragraph": 0, "study": 1, "number": 16, "other": 64 }, "tokens_in_total": 17823, "tokens_out_total": 29216, "n_failed": 0, "n_timeout": 0, "duration_median_s": 17.2, "duration_p90_s": 20.1, "reliability_source": "derived-from-raw" }, { "model_id": "claude-opus-4-8", "domain": "steuerrecht", "reasoning_mode": "off", "reasoning_level": "off", "reasoning_vendor_default": true, "score_mean": 0.7106107154494252, "score_ci_lo": 0.6589550458502071, "score_ci_hi": 0.7590126495368431, "top100": 71, "n_cells": 31, "closed_accuracy": null, "open_rubric_score": 0.7106107154494252, "hallu_rate_mean": 1.9516129032258065, "hallu_prevalence": 1, "hallu_per_1k_tokens": 1.3765329571568339, "contradicted_rate_mean": 0.27419354838709675, "extra_claims_rate_mean": 1.6774193548387097, "extra_false_rate_mean": 0.9112903225806451, "extra_unsupported_rate_mean": 0.7661290322580645, "hallu_top100": 51, "inter_rater_agreement": 0.7760496671786995, "n_judges": 4, "score_mean_ex_family": 0.6949041102266909, "self_judge_delta": 0.01570660522273437, "n_cells_ex_family": 31, "false_claim_types": { "az": 1, "paragraph": 77, "study": 0, "number": 34, "other": 67 }, "tokens_in_total": 20188, "tokens_out_total": 43951, "n_failed": 0, "n_timeout": 0, "duration_median_s": 22.8, "duration_p90_s": 27.8, "reliability_source": "derived-from-raw" }, { "model_id": "claude-opus-4-8", "domain": "wirtschaftsrecht", "reasoning_mode": "off", "reasoning_level": "off", "reasoning_vendor_default": true, "score_mean": 0.6732936507936508, "score_ci_lo": 0.6181944444444445, "score_ci_hi": 0.7353869047619047, "top100": 67, "n_cells": 30, "closed_accuracy": null, "open_rubric_score": 0.6732936507936508, "hallu_rate_mean": 2.35, "hallu_prevalence": 1, "hallu_per_1k_tokens": 1.5578733371635658, "contradicted_rate_mean": 0.475, "extra_claims_rate_mean": 1.875, "extra_false_rate_mean": 1.0083333333333333, "extra_unsupported_rate_mean": 0.8666666666666667, "hallu_top100": 41, "inter_rater_agreement": 0.6863756613756614, "n_judges": 4, "score_mean_ex_family": 0.624378306878307, "self_judge_delta": 0.0489153439153438, "n_cells_ex_family": 30, "false_claim_types": { "az": 0, "paragraph": 136, "study": 0, "number": 10, "other": 57 }, "tokens_in_total": 22111, "tokens_out_total": 45254, "n_failed": 0, "n_timeout": 0, "duration_median_s": 24.9, "duration_p90_s": 33.5, "reliability_source": "derived-from-raw" }, { "model_id": "claude-opus-5", "domain": "jura", "reasoning_mode": "on", "reasoning_level": "adaptive", "reasoning_vendor_default": true, "score_mean": 0.9106632653061225, "score_ci_lo": 0.8756632653061225, "score_ci_hi": 0.9394302721088434, "top100": 91, "n_cells": 35, "closed_accuracy": null, "open_rubric_score": 0.9106632653061225, "hallu_rate_mean": 2.05, "hallu_prevalence": 1, "hallu_per_1k_tokens": 0.6017578878507808, "contradicted_rate_mean": 0.12142857142857143, "extra_claims_rate_mean": 1.9285714285714286, "extra_false_rate_mean": 0.55, "extra_unsupported_rate_mean": 1.3785714285714286, "hallu_top100": 48, "inter_rater_agreement": 0.8553741496598639, "n_judges": 4, "score_mean_ex_family": 0.8980839002267573, "self_judge_delta": 0.01257936507936519, "n_cells_ex_family": 35, "false_claim_types": { "az": 10, "paragraph": 162, "study": 0, "number": 8, "other": 54 }, "tokens_in_total": 22856, "tokens_out_total": 119234, "n_failed": 0, "n_timeout": 0, "duration_median_s": 49.1, "duration_p90_s": 63.6, "reliability_source": "derived-from-raw" }, { "model_id": "claude-opus-5", "domain": "medizin", "reasoning_mode": "on", "reasoning_level": "adaptive", "reasoning_vendor_default": true, "score_mean": 0.9200231481481481, "score_ci_lo": 0.8680092592592593, "score_ci_hi": 0.9584027777777777, "top100": 92, "n_cells": 30, "closed_accuracy": null, "open_rubric_score": 0.9200231481481481, "hallu_rate_mean": 2.05, "hallu_prevalence": 1, "hallu_per_1k_tokens": 0.6523676171079429, "contradicted_rate_mean": 0.19166666666666668, "extra_claims_rate_mean": 1.8583333333333334, "extra_false_rate_mean": 0.325, "extra_unsupported_rate_mean": 1.5333333333333334, "hallu_top100": 48, "inter_rater_agreement": 0.9001851851851852, "n_judges": 4, "score_mean_ex_family": 0.9076388888888889, "self_judge_delta": 0.012384259259259234, "n_cells_ex_family": 30, "false_claim_types": { "az": 0, "paragraph": 0, "study": 1, "number": 66, "other": 139 }, "tokens_in_total": 17823, "tokens_out_total": 94272, "n_failed": 0, "n_timeout": 0, "duration_median_s": 45.4, "duration_p90_s": 61.1, "reliability_source": "derived-from-raw" }, { "model_id": "claude-opus-5", "domain": "steuerrecht", "reasoning_mode": "on", "reasoning_level": "adaptive", "reasoning_vendor_default": true, "score_mean": 0.8899467020434763, "score_ci_lo": 0.8527684680910488, "score_ci_hi": 0.9222964087883443, "top100": 89, "n_cells": 31, "closed_accuracy": null, "open_rubric_score": 0.8899467020434763, "hallu_rate_mean": 4.25, "hallu_prevalence": 1, "hallu_per_1k_tokens": 0.5896648644777829, "contradicted_rate_mean": 0.3225806451612903, "extra_claims_rate_mean": 3.9274193548387095, "extra_false_rate_mean": 1.8387096774193548, "extra_unsupported_rate_mean": 2.088709677419355, "hallu_top100": 0, "inter_rater_agreement": 0.8625168738071963, "n_judges": 4, "score_mean_ex_family": 0.8579335986594051, "self_judge_delta": 0.03201310338407115, "n_cells_ex_family": 31, "false_claim_types": { "az": 32, "paragraph": 238, "study": 0, "number": 72, "other": 76 }, "tokens_in_total": 20188, "tokens_out_total": 223432, "n_failed": 0, "n_timeout": 0, "duration_median_s": 91.5, "duration_p90_s": 135, "reliability_source": "derived-from-raw" }, { "model_id": "claude-opus-5", "domain": "wirtschaftsrecht", "reasoning_mode": "on", "reasoning_level": "adaptive", "reasoning_vendor_default": true, "score_mean": 0.7988756613756614, "score_ci_lo": 0.7377116402116403, "score_ci_hi": 0.8581746031746033, "top100": 80, "n_cells": 30, "closed_accuracy": null, "open_rubric_score": 0.7988756613756614, "hallu_rate_mean": 3.7333333333333334, "hallu_prevalence": 1, "hallu_per_1k_tokens": 0.47434311948364366, "contradicted_rate_mean": 0.5083333333333333, "extra_claims_rate_mean": 3.225, "extra_false_rate_mean": 1.375, "extra_unsupported_rate_mean": 1.85, "hallu_top100": 6, "inter_rater_agreement": 0.7937698412698412, "n_judges": 4, "score_mean_ex_family": 0.7484060846560846, "self_judge_delta": 0.05046957671957675, "n_cells_ex_family": 30, "false_claim_types": { "az": 8, "paragraph": 209, "study": 0, "number": 23, "other": 102 }, "tokens_in_total": 22111, "tokens_out_total": 236116, "n_failed": 0, "n_timeout": 0, "duration_median_s": 97.1, "duration_p90_s": 164.6, "reliability_source": "derived-from-raw" }, { "model_id": "deepseek-v4-pro", "domain": "jura", "reasoning_mode": "on", "reasoning_level": "standard", "reasoning_vendor_default": true, "score_mean": 0.6103769841269842, "score_ci_lo": 0.5350396825396825, "score_ci_hi": 0.6817261904761904, "top100": 61, "n_cells": 35, "closed_accuracy": null, "open_rubric_score": 0.6103769841269842, "hallu_rate_mean": 0.7357142857142858, "hallu_prevalence": 0.6285714285714286, "hallu_per_1k_tokens": 0.11004320531284321, "contradicted_rate_mean": 0.34285714285714286, "extra_claims_rate_mean": 0.39285714285714285, "extra_false_rate_mean": 0.18571428571428572, "extra_unsupported_rate_mean": 0.20714285714285716, "hallu_top100": 81, "inter_rater_agreement": 0.7232993197278912, "n_judges": 4, "score_mean_ex_family": 0.6103769841269842, "self_judge_delta": 0, "n_cells_ex_family": 35, "false_claim_types": { "az": 0, "paragraph": 21, "study": 0, "number": 3, "other": 19 }, "tokens_in_total": 16575, "tokens_out_total": 233999, "n_failed": 0, "n_timeout": 0, "duration_median_s": 118.1, "duration_p90_s": 263.8, "reliability_source": "derived-from-raw" }, { "model_id": "deepseek-v4-pro", "domain": "medizin", "reasoning_mode": "on", "reasoning_level": "standard", "reasoning_vendor_default": true, "score_mean": 0.6796329365079364, "score_ci_lo": 0.5998611111111111, "score_ci_hi": 0.7420403439153439, "top100": 68, "n_cells": 30, "closed_accuracy": null, "open_rubric_score": 0.6796329365079364, "hallu_rate_mean": 0.5333333333333333, "hallu_prevalence": 0.8, "hallu_per_1k_tokens": 0.243605359317905, "contradicted_rate_mean": 0.19166666666666668, "extra_claims_rate_mean": 0.3416666666666667, "extra_false_rate_mean": 0.125, "extra_unsupported_rate_mean": 0.21666666666666667, "hallu_top100": 86, "inter_rater_agreement": 0.7192195767195767, "n_judges": 4, "score_mean_ex_family": 0.6796329365079364, "self_judge_delta": 0, "n_cells_ex_family": 30, "false_claim_types": { "az": 0, "paragraph": 0, "study": 0, "number": 8, "other": 32 }, "tokens_in_total": 13028, "tokens_out_total": 65680, "n_failed": 0, "n_timeout": 0, "duration_median_s": 44.4, "duration_p90_s": 79.2, "reliability_source": "derived-from-raw" }, { "model_id": "deepseek-v4-pro", "domain": "steuerrecht", "reasoning_mode": "on", "reasoning_level": "standard", "reasoning_vendor_default": true, "score_mean": 0.48025851836335703, "score_ci_lo": 0.41330441511893123, "score_ci_hi": 0.5442402131918261, "top100": 48, "n_cells": 31, "closed_accuracy": null, "open_rubric_score": 0.48025851836335703, "hallu_rate_mean": 2.2983870967741935, "hallu_prevalence": 0.9354838709677419, "hallu_per_1k_tokens": 0.25702999585144565, "contradicted_rate_mean": 0.8467741935483871, "extra_claims_rate_mean": 1.4516129032258065, "extra_false_rate_mean": 0.967741935483871, "extra_unsupported_rate_mean": 0.4838709677419355, "hallu_top100": 42, "inter_rater_agreement": 0.648220686123912, "n_judges": 4, "score_mean_ex_family": 0.48025851836335703, "self_judge_delta": 0, "n_cells_ex_family": 31, "false_claim_types": { "az": 0, "paragraph": 43, "study": 0, "number": 52, "other": 44 }, "tokens_in_total": 14442, "tokens_out_total": 277205, "n_failed": 0, "n_timeout": 0, "duration_median_s": 154.9, "duration_p90_s": 254.8, "reliability_source": "derived-from-raw" }, { "model_id": "deepseek-v4-pro", "domain": "wirtschaftsrecht", "reasoning_mode": "on", "reasoning_level": "standard", "reasoning_vendor_default": true, "score_mean": 0.5910119047619048, "score_ci_lo": 0.5153075396825397, "score_ci_hi": 0.6657837301587303, "top100": 59, "n_cells": 30, "closed_accuracy": null, "open_rubric_score": 0.5910119047619048, "hallu_rate_mean": 1.9416666666666667, "hallu_prevalence": 0.9666666666666667, "hallu_per_1k_tokens": 0.2517231693351483, "contradicted_rate_mean": 0.5, "extra_claims_rate_mean": 1.4416666666666667, "extra_false_rate_mean": 0.9, "extra_unsupported_rate_mean": 0.5416666666666666, "hallu_top100": 51, "inter_rater_agreement": 0.6271957671957672, "n_judges": 4, "score_mean_ex_family": 0.5910119047619048, "self_judge_delta": 0, "n_cells_ex_family": 30, "false_claim_types": { "az": 0, "paragraph": 69, "study": 0, "number": 15, "other": 59 }, "tokens_in_total": 15332, "tokens_out_total": 231405, "n_failed": 0, "n_timeout": 0, "duration_median_s": 147.5, "duration_p90_s": 281.1, "reliability_source": "derived-from-raw" }, { "model_id": "gemini-3-1-pro", "domain": "jura", "reasoning_mode": "on", "reasoning_level": "adaptive", "reasoning_vendor_default": true, "score_mean": 0.5862018140589569, "score_ci_lo": 0.5202579365079365, "score_ci_hi": 0.6447732426303855, "top100": 59, "n_cells": 35, "closed_accuracy": null, "open_rubric_score": 0.5862018140589569, "hallu_rate_mean": 0.5785714285714286, "hallu_prevalence": 0.5428571428571428, "hallu_per_1k_tokens": 3.118263012011087, "contradicted_rate_mean": 0.24285714285714285, "extra_claims_rate_mean": 0.3357142857142857, "extra_false_rate_mean": 0.15714285714285714, "extra_unsupported_rate_mean": 0.17857142857142858, "hallu_top100": 85, "inter_rater_agreement": 0.6827097505668934, "n_judges": 4, "score_mean_ex_family": 0.5862018140589569, "self_judge_delta": 0, "n_cells_ex_family": 35, "false_claim_types": { "az": 0, "paragraph": 15, "study": 0, "number": 6, "other": 17 }, "tokens_in_total": 12953, "tokens_out_total": 6494, "n_failed": 0, "n_timeout": 0, "duration_median_s": 23.6, "duration_p90_s": 34.9, "reliability_source": "derived-from-raw" }, { "model_id": "gemini-3-1-pro", "domain": "medizin", "reasoning_mode": "on", "reasoning_level": "adaptive", "reasoning_vendor_default": true, "score_mean": 0.5962863756613757, "score_ci_lo": 0.5426322751322752, "score_ci_hi": 0.6498974867724867, "top100": 60, "n_cells": 30, "closed_accuracy": null, "open_rubric_score": 0.5962863756613757, "hallu_rate_mean": 0.4, "hallu_prevalence": 0.5666666666666667, "hallu_per_1k_tokens": 1.9014419267944858, "contradicted_rate_mean": 0.11666666666666667, "extra_claims_rate_mean": 0.2833333333333333, "extra_false_rate_mean": 0.09166666666666666, "extra_unsupported_rate_mean": 0.19166666666666668, "hallu_top100": 90, "inter_rater_agreement": 0.7326587301587302, "n_judges": 4, "score_mean_ex_family": 0.5962863756613757, "self_judge_delta": 0, "n_cells_ex_family": 30, "false_claim_types": { "az": 0, "paragraph": 0, "study": 0, "number": 3, "other": 28 }, "tokens_in_total": 10224, "tokens_out_total": 6311, "n_failed": 0, "n_timeout": 0, "duration_median_s": 21.3, "duration_p90_s": 30, "reliability_source": "derived-from-raw" }, { "model_id": "gemini-3-1-pro", "domain": "steuerrecht", "reasoning_mode": "on", "reasoning_level": "adaptive", "reasoning_vendor_default": true, "score_mean": 0.48600870455709166, "score_ci_lo": 0.43226766513056836, "score_ci_hi": 0.5432219662058372, "top100": 49, "n_cells": 31, "closed_accuracy": null, "open_rubric_score": 0.48600870455709166, "hallu_rate_mean": 1.0403225806451613, "hallu_prevalence": 0.7419354838709677, "hallu_per_1k_tokens": 4.217340133385641, "contradicted_rate_mean": 0.4112903225806452, "extra_claims_rate_mean": 0.6290322580645161, "extra_false_rate_mean": 0.3790322580645161, "extra_unsupported_rate_mean": 0.25, "hallu_top100": 73, "inter_rater_agreement": 0.674204021784667, "n_judges": 4, "score_mean_ex_family": 0.48600870455709166, "self_judge_delta": 0, "n_cells_ex_family": 31, "false_claim_types": { "az": 0, "paragraph": 14, "study": 0, "number": 10, "other": 36 }, "tokens_in_total": 11090, "tokens_out_total": 7647, "n_failed": 0, "n_timeout": 0, "duration_median_s": 27.8, "duration_p90_s": 36.2, "reliability_source": "derived-from-raw" }, { "model_id": "gemini-3-1-pro", "domain": "wirtschaftsrecht", "reasoning_mode": "on", "reasoning_level": "adaptive", "reasoning_vendor_default": true, "score_mean": 0.48924272486772485, "score_ci_lo": 0.4187962962962963, "score_ci_hi": 0.5568022486772487, "top100": 49, "n_cells": 30, "closed_accuracy": null, "open_rubric_score": 0.48924272486772485, "hallu_rate_mean": 1.5416666666666667, "hallu_prevalence": 0.9666666666666667, "hallu_per_1k_tokens": 5.070160052620039, "contradicted_rate_mean": 0.6611111111111111, "extra_claims_rate_mean": 0.8805555555555556, "extra_false_rate_mean": 0.5166666666666667, "extra_unsupported_rate_mean": 0.3638888888888889, "hallu_top100": 61, "inter_rater_agreement": 0.5750925925925926, "n_judges": 4, "score_mean_ex_family": 0.48924272486772485, "self_judge_delta": 0, "n_cells_ex_family": 30, "false_claim_types": { "az": 0, "paragraph": 27, "study": 0, "number": 14, "other": 45 }, "tokens_in_total": 11544, "tokens_out_total": 9122, "n_failed": 0, "n_timeout": 0, "duration_median_s": 27.2, "duration_p90_s": 37.8, "reliability_source": "derived-from-raw" }, { "model_id": "glm-5.3", "domain": "jura", "reasoning_mode": "on", "reasoning_level": "standard", "reasoning_vendor_default": true, "score_mean": 0.7036892361111111, "score_ci_lo": 0.5969184027777777, "score_ci_hi": 0.7900607638888889, "top100": 70, "n_cells": 32, "closed_accuracy": null, "open_rubric_score": 0.7036892361111111, "hallu_rate_mean": 2.7421875, "hallu_prevalence": 1, "hallu_per_1k_tokens": 0.32208808512668796, "contradicted_rate_mean": 0.7109375, "extra_claims_rate_mean": 2.03125, "extra_false_rate_mean": 1.0859375, "extra_unsupported_rate_mean": 0.9453125, "hallu_top100": 31, "inter_rater_agreement": 0.7931423611111111, "n_judges": 4, "score_mean_ex_family": 0.7036892361111111, "self_judge_delta": 0, "n_cells_ex_family": 32, "false_claim_types": { "az": 4, "paragraph": 148, "study": 0, "number": 9, "other": 57 }, "tokens_in_total": 12069, "tokens_out_total": 272441, "n_failed": 3, "n_timeout": 2, "duration_median_s": 148.4, "duration_p90_s": 482.8, "reliability_source": "derived-from-raw" }, { "model_id": "glm-5.3", "domain": "medizin", "reasoning_mode": "on", "reasoning_level": "standard", "reasoning_vendor_default": true, "score_mean": 0.8677281746031745, "score_ci_lo": 0.819212962962963, "score_ci_hi": 0.907691798941799, "top100": 87, "n_cells": 30, "closed_accuracy": null, "open_rubric_score": 0.8677281746031745, "hallu_rate_mean": 1.5916666666666666, "hallu_prevalence": 1, "hallu_per_1k_tokens": 0.34690941849989826, "contradicted_rate_mean": 0.18333333333333332, "extra_claims_rate_mean": 1.4083333333333334, "extra_false_rate_mean": 0.44166666666666665, "extra_unsupported_rate_mean": 0.9666666666666667, "hallu_top100": 60, "inter_rater_agreement": 0.7932142857142856, "n_judges": 4, "score_mean_ex_family": 0.8677281746031745, "self_judge_delta": 0, "n_cells_ex_family": 30, "false_claim_types": { "az": 0, "paragraph": 0, "study": 1, "number": 51, "other": 101 }, "tokens_in_total": 10855, "tokens_out_total": 137644, "n_failed": 0, "n_timeout": 0, "duration_median_s": 83.2, "duration_p90_s": 157.2, "reliability_source": "derived-from-raw" }, { "model_id": "glm-5.3", "domain": "steuerrecht", "reasoning_mode": "on", "reasoning_level": "standard", "reasoning_vendor_default": true, "score_mean": 0.5850308641975308, "score_ci_lo": 0.4219907407407407, "score_ci_hi": 0.7095679012345679, "top100": 59, "n_cells": 9, "closed_accuracy": null, "open_rubric_score": 0.5850308641975308, "hallu_rate_mean": 5.083333333333333, "hallu_prevalence": 1, "hallu_per_1k_tokens": 0.5374828181722059, "contradicted_rate_mean": 1.2777777777777777, "extra_claims_rate_mean": 3.8055555555555554, "extra_false_rate_mean": 2.9722222222222223, "extra_unsupported_rate_mean": 0.8333333333333334, "hallu_top100": 0, "inter_rater_agreement": 0.7266273849607183, "n_judges": 4, "score_mean_ex_family": 0.5850308641975308, "self_judge_delta": 0, "n_cells_ex_family": 9, "false_claim_types": { "az": 13, "paragraph": 43, "study": 0, "number": 19, "other": 34 }, "tokens_in_total": 3054, "tokens_out_total": 85119, "n_failed": 22, "n_timeout": 16, "duration_median_s": 226.1, "duration_p90_s": 864.6, "reliability_source": "derived-from-raw" }, { "model_id": "glm-5.3", "domain": "wirtschaftsrecht", "reasoning_mode": "on", "reasoning_level": "standard", "reasoning_vendor_default": true, "score_mean": 0.6755140692640693, "score_ci_lo": 0.5471861471861472, "score_ci_hi": 0.78125, "top100": 68, "n_cells": 11, "closed_accuracy": null, "open_rubric_score": 0.6755140692640693, "hallu_rate_mean": 4.204545454545454, "hallu_prevalence": 1, "hallu_per_1k_tokens": 0.35317475468672443, "contradicted_rate_mean": 0.6363636363636364, "extra_claims_rate_mean": 3.5681818181818183, "extra_false_rate_mean": 2.3636363636363638, "extra_unsupported_rate_mean": 1.2045454545454546, "hallu_top100": 0, "inter_rater_agreement": 0.728030303030303, "n_judges": 4, "score_mean_ex_family": 0.6755140692640693, "self_judge_delta": 0, "n_cells_ex_family": 11, "false_claim_types": { "az": 1, "paragraph": 96, "study": 0, "number": 7, "other": 18 }, "tokens_in_total": 4133, "tokens_out_total": 130955, "n_failed": 19, "n_timeout": 6, "duration_median_s": 244, "duration_p90_s": 501, "reliability_source": "derived-from-raw" }, { "model_id": "gpt-5", "domain": "jura", "reasoning_mode": "on", "reasoning_level": "medium", "reasoning_vendor_default": true, "score_mean": 0.7691213151927437, "score_ci_lo": 0.7040419501133787, "score_ci_hi": 0.832687074829932, "top100": 77, "n_cells": 35, "closed_accuracy": null, "open_rubric_score": 0.7691213151927437, "hallu_rate_mean": 1.2, "hallu_prevalence": 0.8571428571428571, "hallu_per_1k_tokens": 0.2814070351758794, "contradicted_rate_mean": 0.22142857142857142, "extra_claims_rate_mean": 0.9785714285714285, "extra_false_rate_mean": 0.42142857142857143, "extra_unsupported_rate_mean": 0.5571428571428572, "hallu_top100": 70, "inter_rater_agreement": 0.7687755102040816, "n_judges": 4, "score_mean_ex_family": 0.7741836734693878, "self_judge_delta": -0.00506235827664403, "n_cells_ex_family": 35, "false_claim_types": { "az": 2, "paragraph": 65, "study": 0, "number": 6, "other": 40 }, "tokens_in_total": 11892, "tokens_out_total": 149250, "n_failed": 0, "n_timeout": 0, "duration_median_s": 62.9, "duration_p90_s": 98, "reliability_source": "derived-from-raw" }, { "model_id": "gpt-5", "domain": "medizin", "reasoning_mode": "on", "reasoning_level": "medium", "reasoning_vendor_default": true, "score_mean": 0.8404431216931217, "score_ci_lo": 0.7750033068783069, "score_ci_hi": 0.8958630952380953, "top100": 84, "n_cells": 30, "closed_accuracy": null, "open_rubric_score": 0.8404431216931217, "hallu_rate_mean": 1.3, "hallu_prevalence": 1, "hallu_per_1k_tokens": 0.407255411797876, "contradicted_rate_mean": 0.25833333333333336, "extra_claims_rate_mean": 1.0416666666666667, "extra_false_rate_mean": 0.19166666666666668, "extra_unsupported_rate_mean": 0.85, "hallu_top100": 67, "inter_rater_agreement": 0.8324206349206349, "n_judges": 4, "score_mean_ex_family": 0.8704629629629629, "self_judge_delta": -0.03001984126984114, "n_cells_ex_family": 30, "false_claim_types": { "az": 0, "paragraph": 0, "study": 0, "number": 56, "other": 60 }, "tokens_in_total": 9789, "tokens_out_total": 95763, "n_failed": 0, "n_timeout": 0, "duration_median_s": 47.5, "duration_p90_s": 68.4, "reliability_source": "derived-from-raw" }, { "model_id": "gpt-5", "domain": "steuerrecht", "reasoning_mode": "on", "reasoning_level": "medium", "reasoning_vendor_default": true, "score_mean": 0.6502202322766839, "score_ci_lo": 0.5660580458967556, "score_ci_hi": 0.733786016850533, "top100": 65, "n_cells": 31, "closed_accuracy": null, "open_rubric_score": 0.6502202322766839, "hallu_rate_mean": 2.725806451612903, "hallu_prevalence": 1, "hallu_per_1k_tokens": 0.4467259481692166, "contradicted_rate_mean": 0.7661290322580645, "extra_claims_rate_mean": 1.9596774193548387, "extra_false_rate_mean": 1.1290322580645162, "extra_unsupported_rate_mean": 0.8306451612903226, "hallu_top100": 31, "inter_rater_agreement": 0.7209642508029604, "n_judges": 4, "score_mean_ex_family": 0.677467066983196, "self_judge_delta": -0.027246834706512124, "n_cells_ex_family": 31, "false_claim_types": { "az": 1, "paragraph": 83, "study": 0, "number": 63, "other": 50 }, "tokens_in_total": 10385, "tokens_out_total": 189154, "n_failed": 0, "n_timeout": 0, "duration_median_s": 81.8, "duration_p90_s": 196.2, "reliability_source": "derived-from-raw" }, { "model_id": "gpt-5", "domain": "wirtschaftsrecht", "reasoning_mode": "on", "reasoning_level": "medium", "reasoning_vendor_default": true, "score_mean": 0.7965806878306878, "score_ci_lo": 0.721468253968254, "score_ci_hi": 0.8598048941798943, "top100": 80, "n_cells": 30, "closed_accuracy": null, "open_rubric_score": 0.7965806878306878, "hallu_rate_mean": 1.825, "hallu_prevalence": 0.9666666666666667, "hallu_per_1k_tokens": 0.33019123952886686, "contradicted_rate_mean": 0.3416666666666667, "extra_claims_rate_mean": 1.4833333333333334, "extra_false_rate_mean": 0.5916666666666667, "extra_unsupported_rate_mean": 0.8916666666666667, "hallu_top100": 54, "inter_rater_agreement": 0.7485449735449735, "n_judges": 4, "score_mean_ex_family": 0.8354938271604938, "self_judge_delta": -0.038913139329806, "n_cells_ex_family": 30, "false_claim_types": { "az": 0, "paragraph": 82, "study": 0, "number": 15, "other": 60 }, "tokens_in_total": 11128, "tokens_out_total": 165813, "n_failed": 0, "n_timeout": 0, "duration_median_s": 97.3, "duration_p90_s": 155.7, "reliability_source": "derived-from-raw" }, { "model_id": "gpt-5.6-sol", "domain": "jura", "reasoning_mode": "on", "reasoning_level": "medium", "reasoning_vendor_default": true, "score_mean": 0.7746995464852607, "score_ci_lo": 0.71906179138322, "score_ci_hi": 0.8182539682539682, "top100": 77, "n_cells": 35, "closed_accuracy": null, "open_rubric_score": 0.7746995464852607, "hallu_rate_mean": 0.9214285714285714, "hallu_prevalence": 0.8285714285714286, "hallu_per_1k_tokens": 0.5453069782384471, "contradicted_rate_mean": 0.12857142857142856, "extra_claims_rate_mean": 0.7928571428571428, "extra_false_rate_mean": 0.3, "extra_unsupported_rate_mean": 0.4928571428571429, "hallu_top100": 76, "inter_rater_agreement": 0.7580725623582766, "n_judges": 4, "score_mean_ex_family": 0.7944217687074829, "self_judge_delta": -0.019722222222222197, "n_cells_ex_family": 35, "false_claim_types": { "az": 0, "paragraph": 68, "study": 0, "number": 4, "other": 17 }, "tokens_in_total": 11892, "tokens_out_total": 59141, "n_failed": 0, "n_timeout": 0, "duration_median_s": 45.3, "duration_p90_s": 85.3, "reliability_source": "derived-from-raw" }, { "model_id": "gpt-5.6-sol", "domain": "medizin", "reasoning_mode": "on", "reasoning_level": "medium", "reasoning_vendor_default": true, "score_mean": 0.8326951058201059, "score_ci_lo": 0.7862632275132275, "score_ci_hi": 0.8765013227513226, "top100": 83, "n_cells": 30, "closed_accuracy": null, "open_rubric_score": 0.8326951058201059, "hallu_rate_mean": 0.75, "hallu_prevalence": 0.9, "hallu_per_1k_tokens": 0.7755678880424666, "contradicted_rate_mean": 0.175, "extra_claims_rate_mean": 0.575, "extra_false_rate_mean": 0.125, "extra_unsupported_rate_mean": 0.45, "hallu_top100": 81, "inter_rater_agreement": 0.7788095238095238, "n_judges": 4, "score_mean_ex_family": 0.8774911816578483, "self_judge_delta": -0.04479607583774248, "n_cells_ex_family": 30, "false_claim_types": { "az": 0, "paragraph": 0, "study": 1, "number": 16, "other": 46 }, "tokens_in_total": 9789, "tokens_out_total": 29011, "n_failed": 0, "n_timeout": 0, "duration_median_s": 26.2, "duration_p90_s": 53.6, "reliability_source": "derived-from-raw" }, { "model_id": "gpt-5.6-sol", "domain": "steuerrecht", "reasoning_mode": "on", "reasoning_level": "medium", "reasoning_vendor_default": true, "score_mean": 0.7316765000232742, "score_ci_lo": 0.6751504096262161, "score_ci_hi": 0.780130801098543, "top100": 73, "n_cells": 31, "closed_accuracy": null, "open_rubric_score": 0.7316765000232742, "hallu_rate_mean": 1.7741935483870968, "hallu_prevalence": 1, "hallu_per_1k_tokens": 0.524173949507753, "contradicted_rate_mean": 0.3064516129032258, "extra_claims_rate_mean": 1.467741935483871, "extra_false_rate_mean": 0.7258064516129032, "extra_unsupported_rate_mean": 0.7419354838709677, "hallu_top100": 55, "inter_rater_agreement": 0.780657962109575, "n_judges": 4, "score_mean_ex_family": 0.7575800245155084, "self_judge_delta": -0.02590352449223421, "n_cells_ex_family": 31, "false_claim_types": { "az": 0, "paragraph": 64, "study": 0, "number": 40, "other": 49 }, "tokens_in_total": 10385, "tokens_out_total": 104927, "n_failed": 0, "n_timeout": 0, "duration_median_s": 90.4, "duration_p90_s": 133.5, "reliability_source": "derived-from-raw" }, { "model_id": "gpt-5.6-sol", "domain": "wirtschaftsrecht", "reasoning_mode": "on", "reasoning_level": "medium", "reasoning_vendor_default": true, "score_mean": 0.7791997354497354, "score_ci_lo": 0.7217195767195768, "score_ci_hi": 0.8358862433862434, "top100": 78, "n_cells": 30, "closed_accuracy": null, "open_rubric_score": 0.7791997354497354, "hallu_rate_mean": 1.4666666666666666, "hallu_prevalence": 1, "hallu_per_1k_tokens": 0.557145389621901, "contradicted_rate_mean": 0.3416666666666667, "extra_claims_rate_mean": 1.125, "extra_false_rate_mean": 0.5833333333333334, "extra_unsupported_rate_mean": 0.5416666666666666, "hallu_top100": 63, "inter_rater_agreement": 0.7302380952380952, "n_judges": 4, "score_mean_ex_family": 0.8321957671957672, "self_judge_delta": -0.052996031746031735, "n_cells_ex_family": 30, "false_claim_types": { "az": 0, "paragraph": 55, "study": 0, "number": 10, "other": 57 }, "tokens_in_total": 11128, "tokens_out_total": 78974, "n_failed": 0, "n_timeout": 0, "duration_median_s": 76.9, "duration_p90_s": 136.7, "reliability_source": "derived-from-raw" }, { "model_id": "gpt-6-astra", "domain": "jura", "reasoning_mode": "on", "reasoning_level": "medium", "reasoning_vendor_default": true, "score_mean": 0.7343820861678004, "score_ci_lo": 0.6677380952380952, "score_ci_hi": 0.7941468253968255, "top100": 73, "n_cells": 35, "closed_accuracy": null, "open_rubric_score": 0.7343820861678004, "hallu_rate_mean": 1.2857142857142858, "hallu_prevalence": 0.9142857142857143, "hallu_per_1k_tokens": 1.2257572455872738, "contradicted_rate_mean": 0.2571428571428571, "extra_claims_rate_mean": 1.0285714285714285, "extra_false_rate_mean": 0.5071428571428571, "extra_unsupported_rate_mean": 0.5214285714285715, "hallu_top100": 67, "inter_rater_agreement": 0.6884353741496598, "n_judges": 4, "score_mean_ex_family": 0.7718820861678004, "self_judge_delta": -0.03749999999999998, "n_cells_ex_family": 35, "false_claim_types": { "az": 0, "paragraph": 82, "study": 0, "number": 13, "other": 30 }, "tokens_in_total": 11892, "tokens_out_total": 36712, "n_failed": 0, "n_timeout": 0, "duration_median_s": 33.8, "duration_p90_s": 56.3, "reliability_source": "derived-from-raw" }, { "model_id": "gpt-6-astra", "domain": "medizin", "reasoning_mode": "on", "reasoning_level": "medium", "reasoning_vendor_default": true, "score_mean": 0.81676917989418, "score_ci_lo": 0.7808597883597883, "score_ci_hi": 0.8532539682539683, "top100": 82, "n_cells": 30, "closed_accuracy": null, "open_rubric_score": 0.81676917989418, "hallu_rate_mean": 0.9666666666666667, "hallu_prevalence": 0.9, "hallu_per_1k_tokens": 1.1398474962660168, "contradicted_rate_mean": 0.26666666666666666, "extra_claims_rate_mean": 0.7, "extra_false_rate_mean": 0.175, "extra_unsupported_rate_mean": 0.525, "hallu_top100": 75, "inter_rater_agreement": 0.7779761904761905, "n_judges": 4, "score_mean_ex_family": 0.8605776014109348, "self_judge_delta": -0.04380842151675479, "n_cells_ex_family": 30, "false_claim_types": { "az": 0, "paragraph": 0, "study": 0, "number": 21, "other": 57 }, "tokens_in_total": 9789, "tokens_out_total": 25442, "n_failed": 0, "n_timeout": 0, "duration_median_s": 26.8, "duration_p90_s": 37.9, "reliability_source": "derived-from-raw" }, { "model_id": "gpt-6-astra", "domain": "steuerrecht", "reasoning_mode": "on", "reasoning_level": "medium", "reasoning_vendor_default": true, "score_mean": 0.736117220593027, "score_ci_lo": 0.6750692407950473, "score_ci_hi": 0.7930840664711633, "top100": 74, "n_cells": 31, "closed_accuracy": null, "open_rubric_score": 0.736117220593027, "hallu_rate_mean": 1.8790322580645162, "hallu_prevalence": 1, "hallu_per_1k_tokens": 0.9758426589827782, "contradicted_rate_mean": 0.31451612903225806, "extra_claims_rate_mean": 1.564516129032258, "extra_false_rate_mean": 0.6612903225806451, "extra_unsupported_rate_mean": 0.9032258064516129, "hallu_top100": 53, "inter_rater_agreement": 0.7220150816925011, "n_judges": 4, "score_mean_ex_family": 0.7723591521978619, "self_judge_delta": -0.03624193160483491, "n_cells_ex_family": 31, "false_claim_types": { "az": 0, "paragraph": 43, "study": 0, "number": 50, "other": 76 }, "tokens_in_total": 10385, "tokens_out_total": 59692, "n_failed": 0, "n_timeout": 0, "duration_median_s": 57.1, "duration_p90_s": 85.2, "reliability_source": "derived-from-raw" }, { "model_id": "gpt-6-astra", "domain": "wirtschaftsrecht", "reasoning_mode": "on", "reasoning_level": "medium", "reasoning_vendor_default": true, "score_mean": 0.7544775132275132, "score_ci_lo": 0.6872718253968254, "score_ci_hi": 0.824785052910053, "top100": 75, "n_cells": 30, "closed_accuracy": null, "open_rubric_score": 0.7544775132275132, "hallu_rate_mean": 1.575, "hallu_prevalence": 0.9666666666666667, "hallu_per_1k_tokens": 0.9308143887159687, "contradicted_rate_mean": 0.3333333333333333, "extra_claims_rate_mean": 1.2416666666666667, "extra_false_rate_mean": 0.6333333333333333, "extra_unsupported_rate_mean": 0.6083333333333333, "hallu_top100": 60, "inter_rater_agreement": 0.7002777777777778, "n_judges": 4, "score_mean_ex_family": 0.8068121693121693, "self_judge_delta": -0.05233465608465604, "n_cells_ex_family": 30, "false_claim_types": { "az": 0, "paragraph": 63, "study": 0, "number": 17, "other": 59 }, "tokens_in_total": 11128, "tokens_out_total": 50762, "n_failed": 0, "n_timeout": 0, "duration_median_s": 56.8, "duration_p90_s": 94.2, "reliability_source": "derived-from-raw" }, { "model_id": "grok-4.5", "domain": "jura", "reasoning_mode": "on", "reasoning_level": "high", "reasoning_vendor_default": true, "score_mean": 0.652562358276644, "score_ci_lo": 0.5891496598639455, "score_ci_hi": 0.7033985260770975, "top100": 65, "n_cells": 35, "closed_accuracy": null, "open_rubric_score": 0.652562358276644, "hallu_rate_mean": 0.5642857142857143, "hallu_prevalence": 0.5142857142857142, "hallu_per_1k_tokens": 2.8364210828665803, "contradicted_rate_mean": 0.20714285714285716, "extra_claims_rate_mean": 0.35714285714285715, "extra_false_rate_mean": 0.2, "extra_unsupported_rate_mean": 0.15714285714285714, "hallu_top100": 85, "inter_rater_agreement": 0.7015192743764173, "n_judges": 4, "score_mean_ex_family": 0.652562358276644, "self_judge_delta": 0, "n_cells_ex_family": 35, "false_claim_types": { "az": 0, "paragraph": 22, "study": 0, "number": 3, "other": 14 }, "tokens_in_total": 28950, "tokens_out_total": 6963, "n_failed": 0, "n_timeout": 0, "duration_median_s": 30.5, "duration_p90_s": 45, "reliability_source": "derived-from-raw" }, { "model_id": "grok-4.5", "domain": "medizin", "reasoning_mode": "on", "reasoning_level": "high", "reasoning_vendor_default": true, "score_mean": 0.7016798941798942, "score_ci_lo": 0.6465674603174603, "score_ci_hi": 0.7590906084656085, "top100": 70, "n_cells": 30, "closed_accuracy": null, "open_rubric_score": 0.7016798941798942, "hallu_rate_mean": 0.45, "hallu_prevalence": 0.7666666666666667, "hallu_per_1k_tokens": 1.5919811320754718, "contradicted_rate_mean": 0.13333333333333333, "extra_claims_rate_mean": 0.31666666666666665, "extra_false_rate_mean": 0.058333333333333334, "extra_unsupported_rate_mean": 0.25833333333333336, "hallu_top100": 88, "inter_rater_agreement": 0.7005423280423281, "n_judges": 4, "score_mean_ex_family": 0.7016798941798942, "self_judge_delta": 0, "n_cells_ex_family": 30, "false_claim_types": { "az": 0, "paragraph": 0, "study": 0, "number": 7, "other": 28 }, "tokens_in_total": 24284, "tokens_out_total": 8480, "n_failed": 0, "n_timeout": 0, "duration_median_s": 20.4, "duration_p90_s": 27.4, "reliability_source": "derived-from-raw" }, { "model_id": "grok-4.5", "domain": "steuerrecht", "reasoning_mode": "on", "reasoning_level": "high", "reasoning_vendor_default": true, "score_mean": 0.5796818996415771, "score_ci_lo": 0.5270833333333333, "score_ci_hi": 0.6291890681003585, "top100": 58, "n_cells": 31, "closed_accuracy": null, "open_rubric_score": 0.5796818996415771, "hallu_rate_mean": 1.25, "hallu_prevalence": 0.967741935483871, "hallu_per_1k_tokens": 3.613727501632006, "contradicted_rate_mean": 0.3548387096774194, "extra_claims_rate_mean": 0.8951612903225806, "extra_false_rate_mean": 0.4596774193548387, "extra_unsupported_rate_mean": 0.43548387096774194, "hallu_top100": 68, "inter_rater_agreement": 0.7045547642321837, "n_judges": 4, "score_mean_ex_family": 0.5796818996415771, "self_judge_delta": 0, "n_cells_ex_family": 31, "false_claim_types": { "az": 0, "paragraph": 29, "study": 0, "number": 9, "other": 56 }, "tokens_in_total": 25335, "tokens_out_total": 10723, "n_failed": 0, "n_timeout": 0, "duration_median_s": 40.9, "duration_p90_s": 52.9, "reliability_source": "derived-from-raw" }, { "model_id": "grok-4.5", "domain": "wirtschaftsrecht", "reasoning_mode": "on", "reasoning_level": "high", "reasoning_vendor_default": true, "score_mean": 0.6658498677248678, "score_ci_lo": 0.6113293650793651, "score_ci_hi": 0.7097420634920635, "top100": 67, "n_cells": 30, "closed_accuracy": null, "open_rubric_score": 0.6658498677248678, "hallu_rate_mean": 1, "hallu_prevalence": 0.9, "hallu_per_1k_tokens": 2.279981760145919, "contradicted_rate_mean": 0.225, "extra_claims_rate_mean": 0.775, "extra_false_rate_mean": 0.30833333333333335, "extra_unsupported_rate_mean": 0.4666666666666667, "hallu_top100": 75, "inter_rater_agreement": 0.6003042328042327, "n_judges": 4, "score_mean_ex_family": 0.6658498677248678, "self_judge_delta": 0, "n_cells_ex_family": 30, "false_claim_types": { "az": 0, "paragraph": 31, "study": 0, "number": 7, "other": 44 }, "tokens_in_total": 25514, "tokens_out_total": 13158, "n_failed": 0, "n_timeout": 0, "duration_median_s": 29.6, "duration_p90_s": 45.3, "reliability_source": "derived-from-raw" }, { "model_id": "grok-4.6", "domain": "jura", "reasoning_mode": "on", "reasoning_level": "high", "reasoning_vendor_default": true, "score_mean": 0.6437018140589569, "score_ci_lo": 0.5582341269841269, "score_ci_hi": 0.7121626984126984, "top100": 64, "n_cells": 35, "closed_accuracy": null, "open_rubric_score": 0.6437018140589569, "hallu_rate_mean": 0.65, "hallu_prevalence": 0.5142857142857142, "hallu_per_1k_tokens": 3.360413589364845, "contradicted_rate_mean": 0.2857142857142857, "extra_claims_rate_mean": 0.36428571428571427, "extra_false_rate_mean": 0.22857142857142856, "extra_unsupported_rate_mean": 0.1357142857142857, "hallu_top100": 83, "inter_rater_agreement": 0.7156462585034014, "n_judges": 4, "score_mean_ex_family": 0.6437018140589569, "self_judge_delta": 0, "n_cells_ex_family": 35, "false_claim_types": { "az": 0, "paragraph": 18, "study": 0, "number": 3, "other": 20 }, "tokens_in_total": 33920, "tokens_out_total": 6770, "n_failed": 0, "n_timeout": 0, "duration_median_s": 60.9, "duration_p90_s": 106.5, "reliability_source": "derived-from-raw" }, { "model_id": "grok-4.6", "domain": "medizin", "reasoning_mode": "on", "reasoning_level": "high", "reasoning_vendor_default": true, "score_mean": 0.6444246031746032, "score_ci_lo": 0.5780357142857143, "score_ci_hi": 0.7027579365079365, "top100": 64, "n_cells": 30, "closed_accuracy": null, "open_rubric_score": 0.6444246031746032, "hallu_rate_mean": 0.5583333333333333, "hallu_prevalence": 0.7333333333333333, "hallu_per_1k_tokens": 2.4156331122007497, "contradicted_rate_mean": 0.15833333333333333, "extra_claims_rate_mean": 0.4, "extra_false_rate_mean": 0.075, "extra_unsupported_rate_mean": 0.325, "hallu_top100": 86, "inter_rater_agreement": 0.7429365079365079, "n_judges": 4, "score_mean_ex_family": 0.6444246031746032, "self_judge_delta": 0, "n_cells_ex_family": 30, "false_claim_types": { "az": 0, "paragraph": 0, "study": 0, "number": 6, "other": 38 }, "tokens_in_total": 28544, "tokens_out_total": 6934, "n_failed": 0, "n_timeout": 0, "duration_median_s": 38.7, "duration_p90_s": 59, "reliability_source": "derived-from-raw" }, { "model_id": "grok-4.6", "domain": "steuerrecht", "reasoning_mode": "on", "reasoning_level": "high", "reasoning_vendor_default": true, "score_mean": 0.5252894730717311, "score_ci_lo": 0.46132697947214074, "score_ci_hi": 0.5838002723083369, "top100": 53, "n_cells": 31, "closed_accuracy": null, "open_rubric_score": 0.5252894730717311, "hallu_rate_mean": 1.9112903225806452, "hallu_prevalence": 0.967741935483871, "hallu_per_1k_tokens": 4.723750298971538, "contradicted_rate_mean": 0.6451612903225806, "extra_claims_rate_mean": 1.2661290322580645, "extra_false_rate_mean": 0.7419354838709677, "extra_unsupported_rate_mean": 0.5241935483870968, "hallu_top100": 52, "inter_rater_agreement": 0.6860587441232603, "n_judges": 4, "score_mean_ex_family": 0.5252894730717311, "self_judge_delta": 0, "n_cells_ex_family": 31, "false_claim_types": { "az": 0, "paragraph": 60, "study": 0, "number": 16, "other": 55 }, "tokens_in_total": 29737, "tokens_out_total": 12543, "n_failed": 0, "n_timeout": 0, "duration_median_s": 93.4, "duration_p90_s": 173.2, "reliability_source": "derived-from-raw" }, { "model_id": "grok-4.6", "domain": "wirtschaftsrecht", "reasoning_mode": "on", "reasoning_level": "high", "reasoning_vendor_default": true, "score_mean": 0.6898148148148149, "score_ci_lo": 0.6242890211640212, "score_ci_hi": 0.7511607142857143, "top100": 69, "n_cells": 30, "closed_accuracy": null, "open_rubric_score": 0.6898148148148149, "hallu_rate_mean": 1.4166666666666667, "hallu_prevalence": 0.9666666666666667, "hallu_per_1k_tokens": 2.7708958143173814, "contradicted_rate_mean": 0.375, "extra_claims_rate_mean": 1.0416666666666667, "extra_false_rate_mean": 0.5333333333333333, "extra_unsupported_rate_mean": 0.5083333333333333, "hallu_top100": 64, "inter_rater_agreement": 0.659589947089947, "n_judges": 4, "score_mean_ex_family": 0.6898148148148149, "self_judge_delta": 0, "n_cells_ex_family": 30, "false_claim_types": { "az": 0, "paragraph": 58, "study": 0, "number": 4, "other": 50 }, "tokens_in_total": 29774, "tokens_out_total": 15338, "n_failed": 0, "n_timeout": 0, "duration_median_s": 76.8, "duration_p90_s": 128.3, "reliability_source": "derived-from-raw" }, { "model_id": "kimi-k3", "domain": "jura", "reasoning_mode": "on", "reasoning_level": "standard", "reasoning_vendor_default": true, "score_mean": 0.8114408263305322, "score_ci_lo": 0.7615254435107376, "score_ci_hi": 0.8600898692810458, "top100": 81, "n_cells": 34, "closed_accuracy": null, "open_rubric_score": 0.8114408263305322, "hallu_rate_mean": 1.9705882352941178, "hallu_prevalence": 0.8823529411764706, "hallu_per_1k_tokens": 0.3951869765247139, "contradicted_rate_mean": 0.2377450980392157, "extra_claims_rate_mean": 1.7328431372549018, "extra_false_rate_mean": 0.9191176470588235, "extra_unsupported_rate_mean": 0.8137254901960784, "hallu_top100": 50, "inter_rater_agreement": 0.7906979458450047, "n_judges": 4, "score_mean_ex_family": 0.8114408263305322, "self_judge_delta": 0, "n_cells_ex_family": 34, "false_claim_types": { "az": 3, "paragraph": 150, "study": 0, "number": 6, "other": 38 }, "tokens_in_total": 17543, "tokens_out_total": 169540, "n_failed": 1, "n_timeout": 1, "duration_median_s": 143.4, "duration_p90_s": 243.2, "reliability_source": "derived-from-raw" }, { "model_id": "kimi-k3", "domain": "medizin", "reasoning_mode": "on", "reasoning_level": "standard", "reasoning_vendor_default": true, "score_mean": 0.8658796296296296, "score_ci_lo": 0.8127314814814814, "score_ci_hi": 0.9106018518518518, "top100": 87, "n_cells": 30, "closed_accuracy": null, "open_rubric_score": 0.8658796296296296, "hallu_rate_mean": 1.5583333333333333, "hallu_prevalence": 1, "hallu_per_1k_tokens": 0.4993644452514981, "contradicted_rate_mean": 0.225, "extra_claims_rate_mean": 1.3333333333333333, "extra_false_rate_mean": 0.225, "extra_unsupported_rate_mean": 1.1083333333333334, "hallu_top100": 61, "inter_rater_agreement": 0.836574074074074, "n_judges": 4, "score_mean_ex_family": 0.8658796296296296, "self_judge_delta": 0, "n_cells_ex_family": 30, "false_claim_types": { "az": 0, "paragraph": 0, "study": 1, "number": 50, "other": 91 }, "tokens_in_total": 14166, "tokens_out_total": 93619, "n_failed": 0, "n_timeout": 0, "duration_median_s": 88.1, "duration_p90_s": 142.2, "reliability_source": "derived-from-raw" }, { "model_id": "kimi-k3", "domain": "steuerrecht", "reasoning_mode": "on", "reasoning_level": "standard", "reasoning_vendor_default": true, "score_mean": 0.5776292688792689, "score_ci_lo": 0.4326785714285714, "score_ci_hi": 0.7121464646464646, "top100": 58, "n_cells": 15, "closed_accuracy": null, "open_rubric_score": 0.5776292688792689, "hallu_rate_mean": 4.216666666666667, "hallu_prevalence": 1, "hallu_per_1k_tokens": 0.4896041366711563, "contradicted_rate_mean": 1.1666666666666667, "extra_claims_rate_mean": 3.05, "extra_false_rate_mean": 1.9166666666666667, "extra_unsupported_rate_mean": 1.1333333333333333, "hallu_top100": 0, "inter_rater_agreement": 0.7193650793650794, "n_judges": 4, "score_mean_ex_family": 0.5776292688792689, "self_judge_delta": 0, "n_cells_ex_family": 15, "false_claim_types": { "az": 6, "paragraph": 71, "study": 0, "number": 35, "other": 36 }, "tokens_in_total": 7520, "tokens_out_total": 129186, "n_failed": 16, "n_timeout": 16, "duration_median_s": 246, "duration_p90_s": 572, "reliability_source": "derived-from-raw" }, { "model_id": "kimi-k3", "domain": "wirtschaftsrecht", "reasoning_mode": "on", "reasoning_level": "standard", "reasoning_vendor_default": true, "score_mean": 0.738558201058201, "score_ci_lo": 0.652980324074074, "score_ci_hi": 0.8154513888888889, "top100": 74, "n_cells": 24, "closed_accuracy": null, "open_rubric_score": 0.738558201058201, "hallu_rate_mean": 3.59375, "hallu_prevalence": 1, "hallu_per_1k_tokens": 0.5029301145806001, "contradicted_rate_mean": 0.5208333333333334, "extra_claims_rate_mean": 3.0729166666666665, "extra_false_rate_mean": 1.875, "extra_unsupported_rate_mean": 1.1979166666666667, "hallu_top100": 10, "inter_rater_agreement": 0.7278935185185186, "n_judges": 4, "score_mean_ex_family": 0.738558201058201, "self_judge_delta": 0, "n_cells_ex_family": 24, "false_claim_types": { "az": 0, "paragraph": 176, "study": 0, "number": 17, "other": 66 }, "tokens_in_total": 12966, "tokens_out_total": 171495, "n_failed": 6, "n_timeout": 6, "duration_median_s": 215.8, "duration_p90_s": 274, "reliability_source": "derived-from-raw" }, { "model_id": "mistral-large-2", "domain": "jura", "reasoning_mode": "off", "reasoning_level": "off", "reasoning_vendor_default": true, "score_mean": 0.4273044217687075, "score_ci_lo": 0.3487131519274376, "score_ci_hi": 0.5079790249433107, "top100": 43, "n_cells": 35, "closed_accuracy": null, "open_rubric_score": 0.4273044217687075, "hallu_rate_mean": 2.05, "hallu_prevalence": 0.8857142857142857, "hallu_per_1k_tokens": 7.147838214783821, "contradicted_rate_mean": 0.9714285714285714, "extra_claims_rate_mean": 1.0785714285714285, "extra_false_rate_mean": 0.8357142857142857, "extra_unsupported_rate_mean": 0.24285714285714285, "hallu_top100": 48, "inter_rater_agreement": 0.6412018140589569, "n_judges": 4, "score_mean_ex_family": 0.3830801209372638, "self_judge_delta": 0.044224300831443686, "n_cells_ex_family": 35, "false_claim_types": { "az": 0, "paragraph": 58, "study": 0, "number": 3, "other": 47 }, "tokens_in_total": 13389, "tokens_out_total": 10038, "n_failed": 0, "n_timeout": 0, "duration_median_s": 4.9, "duration_p90_s": 64.8, "reliability_source": "derived-from-raw" }, { "model_id": "mistral-large-2", "domain": "medizin", "reasoning_mode": "off", "reasoning_level": "off", "reasoning_vendor_default": true, "score_mean": 0.6523908730158731, "score_ci_lo": 0.6025859788359789, "score_ci_hi": 0.7019378306878307, "top100": 65, "n_cells": 30, "closed_accuracy": null, "open_rubric_score": 0.6523908730158731, "hallu_rate_mean": 1.4583333333333333, "hallu_prevalence": 1, "hallu_per_1k_tokens": 2.8549986948577395, "contradicted_rate_mean": 0.25833333333333336, "extra_claims_rate_mean": 1.2, "extra_false_rate_mean": 0.3, "extra_unsupported_rate_mean": 0.9, "hallu_top100": 63, "inter_rater_agreement": 0.693968253968254, "n_judges": 4, "score_mean_ex_family": 0.6189550264550264, "self_judge_delta": 0.03343584656084664, "n_cells_ex_family": 30, "false_claim_types": { "az": 0, "paragraph": 0, "study": 1, "number": 27, "other": 96 }, "tokens_in_total": 10461, "tokens_out_total": 15324, "n_failed": 0, "n_timeout": 0, "duration_median_s": 8.2, "duration_p90_s": 12.2, "reliability_source": "derived-from-raw" }, { "model_id": "mistral-large-2", "domain": "steuerrecht", "reasoning_mode": "off", "reasoning_level": "off", "reasoning_vendor_default": true, "score_mean": 0.27157956291020807, "score_ci_lo": 0.20617814085556022, "score_ci_hi": 0.3329338895871154, "top100": 27, "n_cells": 31, "closed_accuracy": null, "open_rubric_score": 0.27157956291020807, "hallu_rate_mean": 3.2661290322580645, "hallu_prevalence": 1, "hallu_per_1k_tokens": 7.222341108495613, "contradicted_rate_mean": 1.403225806451613, "extra_claims_rate_mean": 1.8629032258064515, "extra_false_rate_mean": 1.3951612903225807, "extra_unsupported_rate_mean": 0.46774193548387094, "hallu_top100": 18, "inter_rater_agreement": 0.5291416468835825, "n_judges": 4, "score_mean_ex_family": 0.22399804496578687, "self_judge_delta": 0.0475815179444212, "n_cells_ex_family": 31, "false_claim_types": { "az": 0, "paragraph": 60, "study": 0, "number": 38, "other": 86 }, "tokens_in_total": 11379, "tokens_out_total": 14019, "n_failed": 0, "n_timeout": 0, "duration_median_s": 7.6, "duration_p90_s": 10.1, "reliability_source": "derived-from-raw" }, { "model_id": "mistral-large-2", "domain": "wirtschaftsrecht", "reasoning_mode": "off", "reasoning_level": "off", "reasoning_vendor_default": true, "score_mean": 0.5887962962962963, "score_ci_lo": 0.5191269841269841, "score_ci_hi": 0.6550099206349207, "top100": 59, "n_cells": 30, "closed_accuracy": null, "open_rubric_score": 0.5887962962962963, "hallu_rate_mean": 2.441666666666667, "hallu_prevalence": 1, "hallu_per_1k_tokens": 3.9263507718696395, "contradicted_rate_mean": 0.35833333333333334, "extra_claims_rate_mean": 2.0833333333333335, "extra_false_rate_mean": 1.3666666666666667, "extra_unsupported_rate_mean": 0.7166666666666667, "hallu_top100": 38, "inter_rater_agreement": 0.5571031746031746, "n_judges": 4, "score_mean_ex_family": 0.5389638447971782, "self_judge_delta": 0.04983245149911808, "n_cells_ex_family": 30, "false_claim_types": { "az": 0, "paragraph": 102, "study": 0, "number": 25, "other": 77 }, "tokens_in_total": 11869, "tokens_out_total": 18656, "n_failed": 0, "n_timeout": 0, "duration_median_s": 10.7, "duration_p90_s": 18.7, "reliability_source": "derived-from-raw" } ], "by_model_overall": [ { "model_id": "claude-fable-5-1", "reasoning_mode": "on", "reasoning_level": "adaptive", "reasoning_vendor_default": true, "score_mean_across_domains": 0.8345109255823542, "score_ci_lo": 0.8036186696900982, "score_ci_hi": 0.8626193911908198, "top100": 83, "n_cells": 126, "hallu_rate_mean_across_domains": 2.701058201058201, "hallu_top100_across_domains": 32, "hallu_per_1k_tokens": 0.6999891676504908, "contradicted_rate_mean": 0.27645502645502645, "extra_claims_rate_mean": 2.4246031746031744, "extra_false_rate_mean": 1.0244708994708995, "extra_unsupported_rate_mean": 1.4001322751322751, "score_mean_ex_family": 0.8091802377516664, "self_judge_delta": 0.025330687830687815, "n_cells_ex_family": 126, "false_claim_types": { "az": 31, "paragraph": 579, "study": 1, "number": 109, "other": 304 }, "n_failed": 0, "n_timeout": 0, "duration_median_s": 49.2, "duration_p90_s": 91.3, "reliability_source": "derived-from-raw" }, { "model_id": "claude-opus-4-7", "reasoning_mode": "off", "reasoning_level": "off", "reasoning_vendor_default": true, "score_mean_across_domains": 0.7884259975033785, "score_ci_lo": 0.7623575608694657, "score_ci_hi": 0.8135047327011613, "top100": 79, "n_cells": 126, "hallu_rate_mean_across_domains": 2.123015873015873, "hallu_top100_across_domains": 46, "hallu_per_1k_tokens": 1.838235294117647, "contradicted_rate_mean": 0.23412698412698413, "extra_claims_rate_mean": 1.8888888888888888, "extra_false_rate_mean": 0.9583333333333334, "extra_unsupported_rate_mean": 0.9305555555555556, "score_mean_ex_family": 0.7749348645181979, "self_judge_delta": 0.013491132985180565, "n_cells_ex_family": 126, "false_claim_types": { "az": 23, "paragraph": 418, "study": 2, "number": 107, "other": 244 }, "n_failed": 0, "n_timeout": 0, "duration_median_s": 19.3, "duration_p90_s": 29.4, "reliability_source": "derived-from-raw" }, { "model_id": "claude-opus-4-8", "reasoning_mode": "off", "reasoning_level": "off", "reasoning_vendor_default": true, "score_mean_across_domains": 0.7487675078448888, "score_ci_lo": 0.7210482804232805, "score_ci_hi": 0.7747367381295953, "top100": 75, "n_cells": 126, "hallu_rate_mean_across_domains": 1.4464285714285714, "hallu_top100_across_domains": 63, "hallu_per_1k_tokens": 1.2098380244291025, "contradicted_rate_mean": 0.22817460317460317, "extra_claims_rate_mean": 1.2182539682539681, "extra_false_rate_mean": 0.5654761904761905, "extra_unsupported_rate_mean": 0.6527777777777778, "score_mean_ex_family": 0.729659262694977, "self_judge_delta": 0.019108245149911718, "n_cells_ex_family": 126, "false_claim_types": { "az": 1, "paragraph": 264, "study": 1, "number": 67, "other": 217 }, "n_failed": 0, "n_timeout": 0, "duration_median_s": 18.9, "duration_p90_s": 27.8, "reliability_source": "derived-from-raw" }, { "model_id": "claude-opus-5", "reasoning_mode": "on", "reasoning_level": "adaptive", "reasoning_vendor_default": true, "score_mean_across_domains": 0.8811787804347327, "score_ci_lo": 0.8575373204837491, "score_ci_hi": 0.9029813440527726, "top100": 88, "n_cells": 126, "hallu_rate_mean_across_domains": 2.992063492063492, "hallu_top100_across_domains": 25, "hallu_per_1k_tokens": 0.5601333622562231, "contradicted_rate_mean": 0.27976190476190477, "extra_claims_rate_mean": 2.7123015873015874, "extra_false_rate_mean": 1.0099206349206349, "extra_unsupported_rate_mean": 1.7023809523809523, "score_mean_ex_family": 0.8548430735930735, "self_judge_delta": 0.0263357068416592, "n_cells_ex_family": 126, "false_claim_types": { "az": 50, "paragraph": 609, "study": 1, "number": 169, "other": 371 }, "n_failed": 0, "n_timeout": 0, "duration_median_s": 62.9, "duration_p90_s": 128.5, "reliability_source": "derived-from-raw" }, { "model_id": "deepseek-v4-pro", "reasoning_mode": "on", "reasoning_level": "standard", "reasoning_vendor_default": true, "score_mean_across_domains": 0.5902424900936806, "score_ci_lo": 0.5524864575459814, "score_ci_hi": 0.6268118400558876, "top100": 59, "n_cells": 126, "hallu_rate_mean_across_domains": 1.3591269841269842, "hallu_top100_across_domains": 66, "hallu_per_1k_tokens": 0.21186729004106206, "contradicted_rate_mean": 0.46825396825396826, "extra_claims_rate_mean": 0.8908730158730159, "extra_false_rate_mean": 0.5337301587301587, "extra_unsupported_rate_mean": 0.35714285714285715, "score_mean_ex_family": 0.5902424900936806, "self_judge_delta": 0, "n_cells_ex_family": 126, "false_claim_types": { "az": 0, "paragraph": 133, "study": 0, "number": 78, "other": 154 }, "n_failed": 0, "n_timeout": 0, "duration_median_s": 118.1, "duration_p90_s": 256.4, "reliability_source": "derived-from-raw" }, { "model_id": "gemini-3-1-pro", "reasoning_mode": "on", "reasoning_level": "adaptive", "reasoning_vendor_default": true, "score_mean_across_domains": 0.5408667170571932, "score_ci_lo": 0.5121488410178886, "score_ci_hi": 0.5729292642983119, "top100": 54, "n_cells": 126, "hallu_rate_mean_across_domains": 0.878968253968254, "hallu_top100_across_domains": 78, "hallu_per_1k_tokens": 3.7448434435652938, "contradicted_rate_mean": 0.35383597883597884, "extra_claims_rate_mean": 0.5251322751322751, "extra_false_rate_mean": 0.28174603174603174, "extra_unsupported_rate_mean": 0.2433862433862434, "score_mean_ex_family": 0.5408667170571932, "self_judge_delta": 0, "n_cells_ex_family": 126, "false_claim_types": { "az": 0, "paragraph": 56, "study": 0, "number": 33, "other": 126 }, "n_failed": 0, "n_timeout": 0, "duration_median_s": 24.1, "duration_p90_s": 35.6, "reliability_source": "derived-from-raw" }, { "model_id": "glm-5.3", "reasoning_mode": "on", "reasoning_level": "standard", "reasoning_vendor_default": true, "score_mean_across_domains": 0.746900406504065, "score_ci_lo": 0.6975754936120789, "score_ci_hi": 0.7941298877274487, "top100": 75, "n_cells": 82, "hallu_rate_mean_across_domains": 2.774390243902439, "hallu_top100_across_domains": 30, "hallu_per_1k_tokens": 0.3633262478060684, "contradicted_rate_mean": 0.5701219512195121, "extra_claims_rate_mean": 2.2042682926829267, "extra_false_rate_mean": 1.228658536585366, "extra_unsupported_rate_mean": 0.975609756097561, "score_mean_ex_family": 0.746900406504065, "self_judge_delta": 0, "n_cells_ex_family": 82, "false_claim_types": { "az": 18, "paragraph": 287, "study": 1, "number": 86, "other": 210 }, "n_failed": 44, "n_timeout": 24, "duration_median_s": 125.8, "duration_p90_s": 408.6, "reliability_source": "derived-from-raw" }, { "model_id": "gpt-5", "reasoning_mode": "on", "reasoning_level": "medium", "reasoning_vendor_default": true, "score_mean_across_domains": 0.7633872025241073, "score_ci_lo": 0.7245212899974806, "score_ci_hi": 0.8004344751368561, "top100": 76, "n_cells": 126, "hallu_rate_mean_across_domains": 1.748015873015873, "hallu_top100_across_domains": 56, "hallu_per_1k_tokens": 0.36709556985232844, "contradicted_rate_mean": 0.39285714285714285, "extra_claims_rate_mean": 1.3551587301587302, "extra_false_rate_mean": 0.5813492063492064, "extra_unsupported_rate_mean": 0.7738095238095238, "score_mean_ex_family": 0.7879096139016774, "self_judge_delta": -0.024522411377570053, "n_cells_ex_family": 126, "false_claim_types": { "az": 3, "paragraph": 230, "study": 0, "number": 140, "other": 210 }, "n_failed": 0, "n_timeout": 0, "duration_median_s": 66.8, "duration_p90_s": 137.3, "reliability_source": "derived-from-raw" }, { "model_id": "gpt-5.6-sol", "reasoning_mode": "on", "reasoning_level": "medium", "reasoning_vendor_default": true, "score_mean_across_domains": 0.77899445131588, "score_ci_lo": 0.7514158735884926, "score_ci_hi": 0.8045219771410247, "top100": 78, "n_cells": 126, "hallu_rate_mean_across_domains": 1.2202380952380953, "hallu_top100_across_domains": 69, "hallu_per_1k_tokens": 0.5651472323407571, "contradicted_rate_mean": 0.23412698412698413, "extra_claims_rate_mean": 0.9861111111111112, "extra_false_rate_mean": 0.4305555555555556, "extra_unsupported_rate_mean": 0.5555555555555556, "score_mean_ex_family": 0.8141297708758026, "self_judge_delta": -0.03513531955992266, "n_cells_ex_family": 126, "false_claim_types": { "az": 0, "paragraph": 187, "study": 1, "number": 70, "other": 169 }, "n_failed": 0, "n_timeout": 0, "duration_median_s": 55.8, "duration_p90_s": 114.6, "reliability_source": "derived-from-raw" }, { "model_id": "gpt-6-astra", "reasoning_mode": "on", "reasoning_level": "medium", "reasoning_vendor_default": true, "score_mean_across_domains": 0.7592095845072035, "score_ci_lo": 0.7308137354268306, "score_ci_hi": 0.7875133134061706, "top100": 76, "n_cells": 126, "hallu_rate_mean_across_domains": 1.4246031746031746, "hallu_top100_across_domains": 64, "hallu_per_1k_tokens": 1.0399286243974786, "contradicted_rate_mean": 0.2916666666666667, "extra_claims_rate_mean": 1.132936507936508, "extra_false_rate_mean": 0.49603174603174605, "extra_unsupported_rate_mean": 0.6369047619047619, "score_mean_ex_family": 0.8014341258388877, "self_judge_delta": -0.042224541331684184, "n_cells_ex_family": 126, "false_claim_types": { "az": 0, "paragraph": 188, "study": 0, "number": 101, "other": 222 }, "n_failed": 0, "n_timeout": 0, "duration_median_s": 39.7, "duration_p90_s": 76.3, "reliability_source": "derived-from-raw" }, { "model_id": "grok-4.5", "reasoning_mode": "on", "reasoning_level": "high", "reasoning_vendor_default": true, "score_mean_across_domains": 0.6494897959183674, "score_ci_lo": 0.6216939090450995, "score_ci_hi": 0.6787863756613757, "top100": 65, "n_cells": 126, "hallu_rate_mean_across_domains": 0.8095238095238095, "hallu_top100_across_domains": 79, "hallu_per_1k_tokens": 2.593835825450107, "contradicted_rate_mean": 0.23015873015873015, "extra_claims_rate_mean": 0.5793650793650794, "extra_false_rate_mean": 0.25595238095238093, "extra_unsupported_rate_mean": 0.32341269841269843, "score_mean_ex_family": 0.6494897959183674, "self_judge_delta": 0, "n_cells_ex_family": 126, "false_claim_types": { "az": 0, "paragraph": 82, "study": 0, "number": 26, "other": 142 }, "n_failed": 0, "n_timeout": 0, "duration_median_s": 30, "duration_p90_s": 48.6, "reliability_source": "derived-from-raw" }, { "model_id": "grok-4.6", "reasoning_mode": "on", "reasoning_level": "high", "reasoning_vendor_default": true, "score_mean_across_domains": 0.6257199975949976, "score_ci_lo": 0.5894202798964704, "score_ci_hi": 0.6615236119700405, "top100": 63, "n_cells": 126, "hallu_rate_mean_across_domains": 1.121031746031746, "hallu_top100_across_domains": 71, "hallu_per_1k_tokens": 3.3966574485992544, "contradicted_rate_mean": 0.36507936507936506, "extra_claims_rate_mean": 0.7559523809523809, "extra_false_rate_mean": 0.39087301587301587, "extra_unsupported_rate_mean": 0.36507936507936506, "score_mean_ex_family": 0.6257199975949976, "self_judge_delta": 0, "n_cells_ex_family": 126, "false_claim_types": { "az": 0, "paragraph": 136, "study": 0, "number": 29, "other": 163 }, "n_failed": 0, "n_timeout": 0, "duration_median_s": 64.9, "duration_p90_s": 123.6, "reliability_source": "derived-from-raw" }, { "model_id": "kimi-k3", "reasoning_mode": "on", "reasoning_level": "standard", "reasoning_vendor_default": true, "score_mean_across_domains": 0.7762642023564353, "score_ci_lo": 0.7354826874851147, "score_ci_hi": 0.8141816920943133, "top100": 78, "n_cells": 103, "hallu_rate_mean_across_domains": 2.5558252427184467, "hallu_top100_across_domains": 36, "hallu_per_1k_tokens": 0.4668877695800227, "contradicted_rate_mean": 0.4352750809061489, "extra_claims_rate_mean": 2.1205501618122975, "extra_false_rate_mean": 1.0849514563106797, "extra_unsupported_rate_mean": 1.0355987055016183, "score_mean_ex_family": 0.7762642023564353, "self_judge_delta": 0, "n_cells_ex_family": 103, "false_claim_types": { "az": 9, "paragraph": 397, "study": 1, "number": 108, "other": 231 }, "n_failed": 23, "n_timeout": 23, "duration_median_s": 145.5, "duration_p90_s": 272.1, "reliability_source": "derived-from-raw" }, { "model_id": "mistral-large-2", "reasoning_mode": "off", "reasoning_level": "off", "reasoning_vendor_default": true, "score_mean_across_domains": 0.4810336213610023, "score_ci_lo": 0.43992676195057145, "score_ci_hi": 0.5229719244600196, "top100": 48, "n_cells": 126, "hallu_rate_mean_across_domains": 2.3015873015873014, "hallu_top100_across_domains": 42, "hallu_per_1k_tokens": 4.996812378310388, "contradicted_rate_mean": 0.7619047619047619, "extra_claims_rate_mean": 1.5396825396825398, "extra_false_rate_mean": 0.9722222222222222, "extra_unsupported_rate_mean": 0.5674603174603174, "score_mean_ex_family": 0.43721674416118855, "self_judge_delta": 0.04381687719981375, "n_cells_ex_family": 126, "false_claim_types": { "az": 0, "paragraph": 220, "study": 1, "number": 93, "other": 306 }, "n_failed": 0, "n_timeout": 0, "duration_median_s": 7.7, "duration_p90_s": 12.7, "reliability_source": "derived-from-raw" } ], "oracle_by_domain": [ { "domain": "jura", "n_items": 35, "oracle_mean": 0.9536, "oracle_top100": 95, "best_single_model_id": "claude-opus-5", "best_single_model_mean": 0.9107, "best_single_model_top100": 91 }, { "domain": "medizin", "n_items": 30, "oracle_mean": 0.9604, "oracle_top100": 96, "best_single_model_id": "claude-opus-5", "best_single_model_mean": 0.92, "best_single_model_top100": 92 }, { "domain": "steuerrecht", "n_items": 31, "oracle_mean": 0.9175, "oracle_top100": 92, "best_single_model_id": "claude-opus-5", "best_single_model_mean": 0.8899, "best_single_model_top100": 89 }, { "domain": "wirtschaftsrecht", "n_items": 30, "oracle_mean": 0.8905, "oracle_top100": 89, "best_single_model_id": "claude-opus-5", "best_single_model_mean": 0.7989, "best_single_model_top100": 80 } ], "cost": { "total_usd": 283.5037 } } }