{
  "written_on": "2026-10-10",
  "model": "mlx-community/Qwen3-1.7B-4bit",
  "head_parameters": 1318657,
  "backbone_trainable_parameters": 0,
  "v2_checkpoint_sha256": "0e8f91e97b1ac2e309463fbb8b0e10c9d165bf4961bd40613aaa0603830e2988",
  "public_panels": {
    "bfcl_irrelevance": {
      "count": 100,
      "correct": 42,
      "accuracy": 0.42,
      "no_tool_precision": 1.0,
      "no_tool_recall": 0.42,
      "false_abstentions": 0,
      "false_tool_routes_on_irrelevance": 58
    },
    "bfcl_multiple": {
      "count": 100,
      "correct": 89,
      "accuracy": 0.89,
      "no_tool_precision": null,
      "no_tool_recall": null,
      "false_abstentions": 0,
      "false_tool_routes_on_irrelevance": 0
    },
    "toolret_apigen": {
      "count": 96,
      "correct": 75,
      "accuracy": 0.78125,
      "no_tool_precision": 0.0,
      "no_tool_recall": null,
      "false_abstentions": 2,
      "false_tool_routes_on_irrelevance": 0
    },
    "bfcl_combined": {
      "count": 200,
      "correct": 131,
      "accuracy": 0.655,
      "no_tool_precision": 1.0,
      "no_tool_recall": 0.42,
      "false_abstentions": 0,
      "false_tool_routes_on_irrelevance": 58
    }
  },
  "public_order_diagnostics": {
    "bfcl_irrelevance": {
      "order_disagreements": 29,
      "all_orders_correct": 31,
      "accuracy_by_order": {
        "0": 0.42,
        "1": 0.49
      }
    },
    "bfcl_multiple": {
      "order_disagreements": 23,
      "all_orders_correct": 77,
      "accuracy_by_order": {
        "0": 0.89,
        "1": 0.91,
        "2": 0.87,
        "3": 0.91
      }
    },
    "toolret_apigen": {
      "order_disagreements": 37,
      "all_orders_correct": 54,
      "accuracy_by_order": {
        "0": 0.78125,
        "1": 0.7291666666666666,
        "2": 0.65625,
        "3": 0.78125
      }
    }
  },
  "confirmation_416": {
    "v2": {
      "count": 416,
      "correct": 312,
      "accuracy": 0.75,
      "no_tool": {
        "precision": 0.7465753424657534,
        "recall": 0.68125,
        "false_abstentions": 37
      },
      "unsupported_action_false_executions": {
        "count": 24,
        "denominator": 48,
        "rate": 0.5
      },
      "order_sensitivity": {
        "requests": 416,
        "disagreements": 143,
        "rate": 0.34375,
        "ids": [
          "g3-scenario-16-00-pair-02-1",
          "g3-scenario-16-00-pair-04-1",
          "g3-scenario-16-00-pair-06-0",
          "g3-scenario-16-00-pair-09-0",
          "g3-scenario-16-00-pair-11-0",
          "g3-scenario-16-00-pair-12-0",
          "g3-scenario-16-00-pair-12-1",
          "g3-scenario-16-01-pair-02-0",
          "g3-scenario-16-01-pair-05-0",
          "g3-scenario-16-01-pair-06-1",
          "g3-scenario-16-01-pair-08-1",
          "g3-scenario-16-01-pair-09-0",
          "g3-scenario-16-01-pair-09-1",
          "g3-scenario-16-01-pair-11-1",
          "g3-scenario-16-02-pair-00-0",
          "g3-scenario-16-02-pair-00-1",
          "g3-scenario-16-02-pair-04-1",
          "g3-scenario-16-02-pair-05-1",
          "g3-scenario-16-02-pair-06-0",
          "g3-scenario-16-02-pair-08-1",
          "g3-scenario-16-02-pair-10-0",
          "g3-scenario-16-02-pair-10-1",
          "g3-scenario-16-02-pair-11-0",
          "g3-scenario-16-02-pair-12-0",
          "g3-scenario-16-02-pair-12-1",
          "g3-scenario-16-03-pair-00-0",
          "g3-scenario-16-03-pair-02-1",
          "g3-scenario-16-03-pair-04-1",
          "g3-scenario-16-03-pair-07-0",
          "g3-scenario-16-03-pair-07-1",
          "g3-scenario-16-03-pair-09-0",
          "g3-scenario-16-03-pair-11-0",
          "g3-scenario-16-03-pair-11-1",
          "g3-scenario-16-03-pair-12-0",
          "g3-scenario-17-00-pair-00-1",
          "g3-scenario-17-00-pair-02-0",
          "g3-scenario-17-00-pair-02-1",
          "g3-scenario-17-00-pair-04-1",
          "g3-scenario-17-00-pair-05-0",
          "g3-scenario-17-00-pair-05-1",
          "g3-scenario-17-00-pair-06-0",
          "g3-scenario-17-00-pair-06-1",
          "g3-scenario-17-00-pair-07-0",
          "g3-scenario-17-00-pair-09-0",
          "g3-scenario-17-00-pair-09-1",
          "g3-scenario-17-00-pair-10-0",
          "g3-scenario-17-00-pair-11-0",
          "g3-scenario-17-00-pair-11-1",
          "g3-scenario-17-01-pair-00-1",
          "g3-scenario-17-01-pair-01-0",
          "g3-scenario-17-01-pair-01-1",
          "g3-scenario-17-01-pair-04-1",
          "g3-scenario-17-01-pair-07-0",
          "g3-scenario-17-01-pair-08-1",
          "g3-scenario-17-01-pair-11-1",
          "g3-scenario-17-02-pair-00-1",
          "g3-scenario-17-02-pair-01-1",
          "g3-scenario-17-02-pair-02-0",
          "g3-scenario-17-02-pair-06-0",
          "g3-scenario-17-02-pair-06-1",
          "g3-scenario-17-02-pair-10-0",
          "g3-scenario-17-02-pair-10-1",
          "g3-scenario-17-02-pair-11-0",
          "g3-scenario-17-02-pair-12-0",
          "g3-scenario-17-02-pair-12-1",
          "g3-scenario-17-03-pair-00-1",
          "g3-scenario-17-03-pair-01-1",
          "g3-scenario-17-03-pair-04-1",
          "g3-scenario-17-03-pair-06-1",
          "g3-scenario-17-03-pair-07-0",
          "g3-scenario-17-03-pair-09-1",
          "g3-scenario-17-03-pair-11-1",
          "g3-scenario-17-03-pair-12-1",
          "g3-scenario-18-00-pair-00-0",
          "g3-scenario-18-00-pair-00-1",
          "g3-scenario-18-00-pair-04-1",
          "g3-scenario-18-00-pair-05-0",
          "g3-scenario-18-00-pair-05-1",
          "g3-scenario-18-00-pair-06-0",
          "g3-scenario-18-00-pair-07-0",
          "g3-scenario-18-00-pair-09-0",
          "g3-scenario-18-00-pair-10-0",
          "g3-scenario-18-00-pair-10-1",
          "g3-scenario-18-00-pair-11-0",
          "g3-scenario-18-00-pair-12-0",
          "g3-scenario-18-01-pair-00-0",
          "g3-scenario-18-01-pair-00-1",
          "g3-scenario-18-01-pair-02-0",
          "g3-scenario-18-01-pair-04-1",
          "g3-scenario-18-01-pair-07-0",
          "g3-scenario-18-01-pair-07-1",
          "g3-scenario-18-01-pair-08-1",
          "g3-scenario-18-02-pair-00-0",
          "g3-scenario-18-02-pair-02-0",
          "g3-scenario-18-02-pair-02-1",
          "g3-scenario-18-02-pair-03-1",
          "g3-scenario-18-02-pair-05-0",
          "g3-scenario-18-02-pair-06-0",
          "g3-scenario-18-02-pair-06-1",
          "g3-scenario-18-02-pair-07-0",
          "g3-scenario-18-02-pair-07-1",
          "g3-scenario-18-02-pair-09-0",
          "g3-scenario-18-02-pair-09-1",
          "g3-scenario-18-02-pair-10-1",
          "g3-scenario-18-02-pair-12-0",
          "g3-scenario-18-02-pair-12-1",
          "g3-scenario-18-03-pair-00-0",
          "g3-scenario-18-03-pair-02-0",
          "g3-scenario-18-03-pair-05-1",
          "g3-scenario-18-03-pair-07-0",
          "g3-scenario-18-03-pair-08-1",
          "g3-scenario-18-03-pair-11-1",
          "g3-scenario-19-00-pair-00-1",
          "g3-scenario-19-00-pair-04-1",
          "g3-scenario-19-00-pair-05-1",
          "g3-scenario-19-00-pair-06-0",
          "g3-scenario-19-00-pair-10-1",
          "g3-scenario-19-01-pair-00-0",
          "g3-scenario-19-01-pair-00-1",
          "g3-scenario-19-01-pair-02-0",
          "g3-scenario-19-01-pair-03-1",
          "g3-scenario-19-01-pair-06-1",
          "g3-scenario-19-01-pair-07-0",
          "g3-scenario-19-01-pair-08-1",
          "g3-scenario-19-01-pair-09-1",
          "g3-scenario-19-01-pair-11-1",
          "g3-scenario-19-02-pair-02-0",
          "g3-scenario-19-02-pair-05-1",
          "g3-scenario-19-02-pair-06-0",
          "g3-scenario-19-02-pair-07-1",
          "g3-scenario-19-02-pair-08-1",
          "g3-scenario-19-02-pair-11-0",
          "g3-scenario-19-03-pair-00-0",
          "g3-scenario-19-03-pair-00-1",
          "g3-scenario-19-03-pair-01-0",
          "g3-scenario-19-03-pair-01-1",
          "g3-scenario-19-03-pair-04-1",
          "g3-scenario-19-03-pair-06-0",
          "g3-scenario-19-03-pair-06-1",
          "g3-scenario-19-03-pair-07-0",
          "g3-scenario-19-03-pair-07-1",
          "g3-scenario-19-03-pair-11-1",
          "g3-scenario-19-03-pair-12-1"
        ],
        "overlap_incorrect_abstention_routes": 27,
        "overlap_capability_boundary_errors": 52
      }
    },
    "v3-seed-42": {
      "count": 416,
      "correct": 305,
      "accuracy": 0.7331730769230769,
      "no_tool": {
        "precision": 0.6909090909090909,
        "recall": 0.7125,
        "false_abstentions": 51
      },
      "unsupported_action_false_executions": {
        "count": 22,
        "denominator": 48,
        "rate": 0.4583333333333333
      },
      "order_sensitivity": {
        "requests": 416,
        "disagreements": 186,
        "rate": 0.44711538461538464,
        "ids": [
          "g3-scenario-16-00-pair-01-0",
          "g3-scenario-16-00-pair-02-1",
          "g3-scenario-16-00-pair-05-1",
          "g3-scenario-16-00-pair-06-0",
          "g3-scenario-16-00-pair-08-0",
          "g3-scenario-16-00-pair-09-0",
          "g3-scenario-16-00-pair-09-1",
          "g3-scenario-16-00-pair-10-1",
          "g3-scenario-16-00-pair-11-0",
          "g3-scenario-16-00-pair-12-0",
          "g3-scenario-16-00-pair-12-1",
          "g3-scenario-16-01-pair-00-1",
          "g3-scenario-16-01-pair-02-0",
          "g3-scenario-16-01-pair-02-1",
          "g3-scenario-16-01-pair-03-0",
          "g3-scenario-16-01-pair-03-1",
          "g3-scenario-16-01-pair-05-0",
          "g3-scenario-16-01-pair-05-1",
          "g3-scenario-16-01-pair-06-0",
          "g3-scenario-16-01-pair-06-1",
          "g3-scenario-16-01-pair-08-1",
          "g3-scenario-16-01-pair-09-0",
          "g3-scenario-16-01-pair-09-1",
          "g3-scenario-16-01-pair-10-0",
          "g3-scenario-16-01-pair-11-0",
          "g3-scenario-16-01-pair-11-1",
          "g3-scenario-16-01-pair-12-0",
          "g3-scenario-16-02-pair-00-0",
          "g3-scenario-16-02-pair-01-0",
          "g3-scenario-16-02-pair-01-1",
          "g3-scenario-16-02-pair-02-0",
          "g3-scenario-16-02-pair-02-1",
          "g3-scenario-16-02-pair-06-0",
          "g3-scenario-16-02-pair-06-1",
          "g3-scenario-16-02-pair-07-0",
          "g3-scenario-16-02-pair-08-1",
          "g3-scenario-16-02-pair-09-0",
          "g3-scenario-16-02-pair-10-0",
          "g3-scenario-16-02-pair-10-1",
          "g3-scenario-16-02-pair-11-1",
          "g3-scenario-16-02-pair-12-0",
          "g3-scenario-16-02-pair-12-1",
          "g3-scenario-16-03-pair-00-1",
          "g3-scenario-16-03-pair-01-1",
          "g3-scenario-16-03-pair-02-1",
          "g3-scenario-16-03-pair-03-0",
          "g3-scenario-16-03-pair-04-1",
          "g3-scenario-16-03-pair-05-0",
          "g3-scenario-16-03-pair-07-1",
          "g3-scenario-16-03-pair-09-0",
          "g3-scenario-16-03-pair-10-0",
          "g3-scenario-16-03-pair-10-1",
          "g3-scenario-16-03-pair-11-0",
          "g3-scenario-17-00-pair-00-1",
          "g3-scenario-17-00-pair-01-1",
          "g3-scenario-17-00-pair-02-0",
          "g3-scenario-17-00-pair-02-1",
          "g3-scenario-17-00-pair-05-0",
          "g3-scenario-17-00-pair-07-0",
          "g3-scenario-17-00-pair-09-0",
          "g3-scenario-17-00-pair-09-1",
          "g3-scenario-17-00-pair-10-0",
          "g3-scenario-17-00-pair-10-1",
          "g3-scenario-17-00-pair-11-0",
          "g3-scenario-17-00-pair-11-1",
          "g3-scenario-17-00-pair-12-0",
          "g3-scenario-17-00-pair-12-1",
          "g3-scenario-17-01-pair-00-1",
          "g3-scenario-17-01-pair-01-1",
          "g3-scenario-17-01-pair-04-1",
          "g3-scenario-17-01-pair-05-0",
          "g3-scenario-17-01-pair-05-1",
          "g3-scenario-17-01-pair-07-1",
          "g3-scenario-17-01-pair-08-1",
          "g3-scenario-17-01-pair-10-0",
          "g3-scenario-17-01-pair-11-1",
          "g3-scenario-17-02-pair-00-0",
          "g3-scenario-17-02-pair-00-1",
          "g3-scenario-17-02-pair-05-1",
          "g3-scenario-17-02-pair-06-0",
          "g3-scenario-17-02-pair-07-0",
          "g3-scenario-17-02-pair-07-1",
          "g3-scenario-17-02-pair-09-0",
          "g3-scenario-17-02-pair-10-0",
          "g3-scenario-17-02-pair-10-1",
          "g3-scenario-17-02-pair-11-0",
          "g3-scenario-17-02-pair-11-1",
          "g3-scenario-17-02-pair-12-0",
          "g3-scenario-17-02-pair-12-1",
          "g3-scenario-17-03-pair-00-1",
          "g3-scenario-17-03-pair-01-1",
          "g3-scenario-17-03-pair-03-1",
          "g3-scenario-17-03-pair-07-1",
          "g3-scenario-17-03-pair-09-0",
          "g3-scenario-17-03-pair-09-1",
          "g3-scenario-17-03-pair-11-1",
          "g3-scenario-18-00-pair-00-0",
          "g3-scenario-18-00-pair-00-1",
          "g3-scenario-18-00-pair-01-1",
          "g3-scenario-18-00-pair-02-0",
          "g3-scenario-18-00-pair-02-1",
          "g3-scenario-18-00-pair-05-0",
          "g3-scenario-18-00-pair-05-1",
          "g3-scenario-18-00-pair-07-0",
          "g3-scenario-18-00-pair-08-1",
          "g3-scenario-18-00-pair-09-0",
          "g3-scenario-18-00-pair-10-1",
          "g3-scenario-18-00-pair-11-0",
          "g3-scenario-18-00-pair-12-0",
          "g3-scenario-18-01-pair-00-0",
          "g3-scenario-18-01-pair-00-1",
          "g3-scenario-18-01-pair-02-0",
          "g3-scenario-18-01-pair-02-1",
          "g3-scenario-18-01-pair-04-1",
          "g3-scenario-18-01-pair-05-0",
          "g3-scenario-18-01-pair-05-1",
          "g3-scenario-18-01-pair-07-1",
          "g3-scenario-18-01-pair-09-0",
          "g3-scenario-18-01-pair-10-1",
          "g3-scenario-18-01-pair-11-0",
          "g3-scenario-18-01-pair-12-0",
          "g3-scenario-18-02-pair-00-0",
          "g3-scenario-18-02-pair-01-1",
          "g3-scenario-18-02-pair-02-0",
          "g3-scenario-18-02-pair-02-1",
          "g3-scenario-18-02-pair-03-1",
          "g3-scenario-18-02-pair-05-0",
          "g3-scenario-18-02-pair-06-0",
          "g3-scenario-18-02-pair-06-1",
          "g3-scenario-18-02-pair-07-0",
          "g3-scenario-18-02-pair-09-0",
          "g3-scenario-18-02-pair-09-1",
          "g3-scenario-18-02-pair-10-0",
          "g3-scenario-18-02-pair-10-1",
          "g3-scenario-18-02-pair-11-0",
          "g3-scenario-18-02-pair-12-0",
          "g3-scenario-18-03-pair-00-1",
          "g3-scenario-18-03-pair-01-1",
          "g3-scenario-18-03-pair-02-0",
          "g3-scenario-18-03-pair-05-1",
          "g3-scenario-18-03-pair-07-0",
          "g3-scenario-18-03-pair-07-1",
          "g3-scenario-18-03-pair-11-0",
          "g3-scenario-18-03-pair-11-1",
          "g3-scenario-19-00-pair-00-0",
          "g3-scenario-19-00-pair-00-1",
          "g3-scenario-19-00-pair-01-1",
          "g3-scenario-19-00-pair-02-0",
          "g3-scenario-19-00-pair-02-1",
          "g3-scenario-19-00-pair-04-1",
          "g3-scenario-19-00-pair-07-0",
          "g3-scenario-19-00-pair-09-0",
          "g3-scenario-19-00-pair-10-1",
          "g3-scenario-19-00-pair-11-0",
          "g3-scenario-19-00-pair-12-0",
          "g3-scenario-19-01-pair-00-0",
          "g3-scenario-19-01-pair-00-1",
          "g3-scenario-19-01-pair-01-1",
          "g3-scenario-19-01-pair-02-0",
          "g3-scenario-19-01-pair-03-1",
          "g3-scenario-19-01-pair-05-0",
          "g3-scenario-19-01-pair-05-1",
          "g3-scenario-19-01-pair-06-0",
          "g3-scenario-19-01-pair-07-0",
          "g3-scenario-19-01-pair-10-0",
          "g3-scenario-19-01-pair-11-1",
          "g3-scenario-19-01-pair-12-0",
          "g3-scenario-19-02-pair-00-0",
          "g3-scenario-19-02-pair-01-1",
          "g3-scenario-19-02-pair-02-1",
          "g3-scenario-19-02-pair-05-0",
          "g3-scenario-19-02-pair-07-0",
          "g3-scenario-19-02-pair-12-0",
          "g3-scenario-19-02-pair-12-1",
          "g3-scenario-19-03-pair-00-0",
          "g3-scenario-19-03-pair-00-1",
          "g3-scenario-19-03-pair-01-0",
          "g3-scenario-19-03-pair-01-1",
          "g3-scenario-19-03-pair-02-0",
          "g3-scenario-19-03-pair-04-1",
          "g3-scenario-19-03-pair-05-0",
          "g3-scenario-19-03-pair-06-0",
          "g3-scenario-19-03-pair-07-0",
          "g3-scenario-19-03-pair-07-1",
          "g3-scenario-19-03-pair-10-0",
          "g3-scenario-19-03-pair-12-1"
        ],
        "overlap_incorrect_abstention_routes": 36,
        "overlap_capability_boundary_errors": 68
      }
    },
    "v3-seed-7": {
      "count": 416,
      "correct": 286,
      "accuracy": 0.6875,
      "no_tool": {
        "precision": 0.7976190476190477,
        "recall": 0.41875,
        "false_abstentions": 17
      },
      "unsupported_action_false_executions": {
        "count": 42,
        "denominator": 48,
        "rate": 0.875
      },
      "order_sensitivity": {
        "requests": 416,
        "disagreements": 138,
        "rate": 0.3317307692307692,
        "ids": [
          "g3-scenario-16-00-pair-03-1",
          "g3-scenario-16-00-pair-07-0",
          "g3-scenario-16-00-pair-07-1",
          "g3-scenario-16-00-pair-09-1",
          "g3-scenario-16-00-pair-12-0",
          "g3-scenario-16-00-pair-12-1",
          "g3-scenario-16-01-pair-03-1",
          "g3-scenario-16-01-pair-05-0",
          "g3-scenario-16-01-pair-05-1",
          "g3-scenario-16-01-pair-06-0",
          "g3-scenario-16-01-pair-06-1",
          "g3-scenario-16-01-pair-09-1",
          "g3-scenario-16-01-pair-10-0",
          "g3-scenario-16-01-pair-11-0",
          "g3-scenario-16-01-pair-11-1",
          "g3-scenario-16-01-pair-12-0",
          "g3-scenario-16-02-pair-00-0",
          "g3-scenario-16-02-pair-00-1",
          "g3-scenario-16-02-pair-01-0",
          "g3-scenario-16-02-pair-01-1",
          "g3-scenario-16-02-pair-04-1",
          "g3-scenario-16-02-pair-05-1",
          "g3-scenario-16-02-pair-06-0",
          "g3-scenario-16-02-pair-07-0",
          "g3-scenario-16-02-pair-07-1",
          "g3-scenario-16-02-pair-08-1",
          "g3-scenario-16-02-pair-11-0",
          "g3-scenario-16-02-pair-11-1",
          "g3-scenario-16-03-pair-00-0",
          "g3-scenario-16-03-pair-00-1",
          "g3-scenario-16-03-pair-04-1",
          "g3-scenario-16-03-pair-05-0",
          "g3-scenario-16-03-pair-05-1",
          "g3-scenario-16-03-pair-06-1",
          "g3-scenario-16-03-pair-07-0",
          "g3-scenario-16-03-pair-07-1",
          "g3-scenario-16-03-pair-09-1",
          "g3-scenario-16-03-pair-10-0",
          "g3-scenario-16-03-pair-10-1",
          "g3-scenario-16-03-pair-11-1",
          "g3-scenario-16-03-pair-12-0",
          "g3-scenario-17-00-pair-00-1",
          "g3-scenario-17-00-pair-03-1",
          "g3-scenario-17-00-pair-05-0",
          "g3-scenario-17-00-pair-06-0",
          "g3-scenario-17-00-pair-07-0",
          "g3-scenario-17-00-pair-09-1",
          "g3-scenario-17-00-pair-10-0",
          "g3-scenario-17-00-pair-10-1",
          "g3-scenario-17-00-pair-11-0",
          "g3-scenario-17-00-pair-11-1",
          "g3-scenario-17-00-pair-12-0",
          "g3-scenario-17-00-pair-12-1",
          "g3-scenario-17-01-pair-01-1",
          "g3-scenario-17-01-pair-03-1",
          "g3-scenario-17-01-pair-05-0",
          "g3-scenario-17-01-pair-05-1",
          "g3-scenario-17-01-pair-08-1",
          "g3-scenario-17-01-pair-10-0",
          "g3-scenario-17-01-pair-11-1",
          "g3-scenario-17-02-pair-00-0",
          "g3-scenario-17-02-pair-00-1",
          "g3-scenario-17-02-pair-04-1",
          "g3-scenario-17-02-pair-06-0",
          "g3-scenario-17-02-pair-07-1",
          "g3-scenario-17-02-pair-10-0",
          "g3-scenario-17-02-pair-11-0",
          "g3-scenario-17-03-pair-03-1",
          "g3-scenario-17-03-pair-05-1",
          "g3-scenario-17-03-pair-06-1",
          "g3-scenario-17-03-pair-09-1",
          "g3-scenario-17-03-pair-11-1",
          "g3-scenario-18-00-pair-00-1",
          "g3-scenario-18-00-pair-04-1",
          "g3-scenario-18-00-pair-05-1",
          "g3-scenario-18-00-pair-07-0",
          "g3-scenario-18-00-pair-07-1",
          "g3-scenario-18-00-pair-08-1",
          "g3-scenario-18-00-pair-10-1",
          "g3-scenario-18-00-pair-11-0",
          "g3-scenario-18-01-pair-00-0",
          "g3-scenario-18-01-pair-00-1",
          "g3-scenario-18-01-pair-03-1",
          "g3-scenario-18-01-pair-04-1",
          "g3-scenario-18-01-pair-06-1",
          "g3-scenario-18-01-pair-07-0",
          "g3-scenario-18-01-pair-07-1",
          "g3-scenario-18-01-pair-10-0",
          "g3-scenario-18-01-pair-10-1",
          "g3-scenario-18-01-pair-11-0",
          "g3-scenario-18-01-pair-12-0",
          "g3-scenario-18-02-pair-00-0",
          "g3-scenario-18-02-pair-03-1",
          "g3-scenario-18-02-pair-05-0",
          "g3-scenario-18-02-pair-06-0",
          "g3-scenario-18-02-pair-06-1",
          "g3-scenario-18-02-pair-07-0",
          "g3-scenario-18-02-pair-08-1",
          "g3-scenario-18-02-pair-09-1",
          "g3-scenario-18-02-pair-11-0",
          "g3-scenario-18-03-pair-00-1",
          "g3-scenario-18-03-pair-05-0",
          "g3-scenario-18-03-pair-05-1",
          "g3-scenario-18-03-pair-07-0",
          "g3-scenario-18-03-pair-07-1",
          "g3-scenario-18-03-pair-11-1",
          "g3-scenario-19-00-pair-00-0",
          "g3-scenario-19-00-pair-00-1",
          "g3-scenario-19-00-pair-04-1",
          "g3-scenario-19-00-pair-07-0",
          "g3-scenario-19-00-pair-07-1",
          "g3-scenario-19-00-pair-10-1",
          "g3-scenario-19-00-pair-11-0",
          "g3-scenario-19-01-pair-03-1",
          "g3-scenario-19-01-pair-05-1",
          "g3-scenario-19-01-pair-06-0",
          "g3-scenario-19-01-pair-08-1",
          "g3-scenario-19-01-pair-09-1",
          "g3-scenario-19-01-pair-10-0",
          "g3-scenario-19-01-pair-11-1",
          "g3-scenario-19-01-pair-12-0",
          "g3-scenario-19-02-pair-00-0",
          "g3-scenario-19-02-pair-05-1",
          "g3-scenario-19-02-pair-07-0",
          "g3-scenario-19-02-pair-08-1",
          "g3-scenario-19-02-pair-09-1",
          "g3-scenario-19-02-pair-11-0",
          "g3-scenario-19-02-pair-12-1",
          "g3-scenario-19-03-pair-02-0",
          "g3-scenario-19-03-pair-03-1",
          "g3-scenario-19-03-pair-06-0",
          "g3-scenario-19-03-pair-07-0",
          "g3-scenario-19-03-pair-07-1",
          "g3-scenario-19-03-pair-09-1",
          "g3-scenario-19-03-pair-10-0",
          "g3-scenario-19-03-pair-11-1",
          "g3-scenario-19-03-pair-12-0",
          "g3-scenario-19-03-pair-12-1"
        ],
        "overlap_incorrect_abstention_routes": 40,
        "overlap_capability_boundary_errors": 58
      }
    },
    "v3-seed-123": {
      "count": 416,
      "correct": 303,
      "accuracy": 0.7283653846153846,
      "no_tool": {
        "precision": 0.935064935064935,
        "recall": 0.45,
        "false_abstentions": 5
      },
      "unsupported_action_false_executions": {
        "count": 37,
        "denominator": 48,
        "rate": 0.7708333333333334
      },
      "order_sensitivity": {
        "requests": 416,
        "disagreements": 144,
        "rate": 0.34615384615384615,
        "ids": [
          "g3-scenario-16-00-pair-00-0",
          "g3-scenario-16-00-pair-01-1",
          "g3-scenario-16-00-pair-04-1",
          "g3-scenario-16-00-pair-06-1",
          "g3-scenario-16-00-pair-07-0",
          "g3-scenario-16-00-pair-07-1",
          "g3-scenario-16-00-pair-09-1",
          "g3-scenario-16-00-pair-12-1",
          "g3-scenario-16-01-pair-00-1",
          "g3-scenario-16-01-pair-01-1",
          "g3-scenario-16-01-pair-02-0",
          "g3-scenario-16-01-pair-03-0",
          "g3-scenario-16-01-pair-03-1",
          "g3-scenario-16-01-pair-05-0",
          "g3-scenario-16-01-pair-05-1",
          "g3-scenario-16-01-pair-06-1",
          "g3-scenario-16-01-pair-09-0",
          "g3-scenario-16-01-pair-09-1",
          "g3-scenario-16-01-pair-11-1",
          "g3-scenario-16-01-pair-12-0",
          "g3-scenario-16-02-pair-00-0",
          "g3-scenario-16-02-pair-00-1",
          "g3-scenario-16-02-pair-01-0",
          "g3-scenario-16-02-pair-01-1",
          "g3-scenario-16-02-pair-04-1",
          "g3-scenario-16-02-pair-05-1",
          "g3-scenario-16-02-pair-07-0",
          "g3-scenario-16-02-pair-08-0",
          "g3-scenario-16-02-pair-08-1",
          "g3-scenario-16-02-pair-10-0",
          "g3-scenario-16-02-pair-11-0",
          "g3-scenario-16-02-pair-11-1",
          "g3-scenario-16-03-pair-00-1",
          "g3-scenario-16-03-pair-01-1",
          "g3-scenario-16-03-pair-02-0",
          "g3-scenario-16-03-pair-04-1",
          "g3-scenario-16-03-pair-10-0",
          "g3-scenario-16-03-pair-11-1",
          "g3-scenario-16-03-pair-12-0",
          "g3-scenario-17-00-pair-00-1",
          "g3-scenario-17-00-pair-01-1",
          "g3-scenario-17-00-pair-02-0",
          "g3-scenario-17-00-pair-06-1",
          "g3-scenario-17-00-pair-07-0",
          "g3-scenario-17-00-pair-08-1",
          "g3-scenario-17-00-pair-09-1",
          "g3-scenario-17-00-pair-11-0",
          "g3-scenario-17-00-pair-11-1",
          "g3-scenario-17-00-pair-12-0",
          "g3-scenario-17-00-pair-12-1",
          "g3-scenario-17-01-pair-00-1",
          "g3-scenario-17-01-pair-01-0",
          "g3-scenario-17-01-pair-01-1",
          "g3-scenario-17-01-pair-04-1",
          "g3-scenario-17-01-pair-05-0",
          "g3-scenario-17-01-pair-05-1",
          "g3-scenario-17-01-pair-08-1",
          "g3-scenario-17-01-pair-10-0",
          "g3-scenario-17-01-pair-11-1",
          "g3-scenario-17-02-pair-00-0",
          "g3-scenario-17-02-pair-00-1",
          "g3-scenario-17-02-pair-01-1",
          "g3-scenario-17-02-pair-04-1",
          "g3-scenario-17-02-pair-07-1",
          "g3-scenario-17-02-pair-11-0",
          "g3-scenario-17-03-pair-01-1",
          "g3-scenario-17-03-pair-03-1",
          "g3-scenario-17-03-pair-09-1",
          "g3-scenario-17-03-pair-10-0",
          "g3-scenario-17-03-pair-11-1",
          "g3-scenario-18-00-pair-00-0",
          "g3-scenario-18-00-pair-00-1",
          "g3-scenario-18-00-pair-01-1",
          "g3-scenario-18-00-pair-05-0",
          "g3-scenario-18-00-pair-05-1",
          "g3-scenario-18-00-pair-06-1",
          "g3-scenario-18-00-pair-07-0",
          "g3-scenario-18-00-pair-10-0",
          "g3-scenario-18-00-pair-11-0",
          "g3-scenario-18-01-pair-00-0",
          "g3-scenario-18-01-pair-00-1",
          "g3-scenario-18-01-pair-04-1",
          "g3-scenario-18-01-pair-05-0",
          "g3-scenario-18-01-pair-07-1",
          "g3-scenario-18-01-pair-11-0",
          "g3-scenario-18-01-pair-12-0",
          "g3-scenario-18-02-pair-00-0",
          "g3-scenario-18-02-pair-00-1",
          "g3-scenario-18-02-pair-03-1",
          "g3-scenario-18-02-pair-04-1",
          "g3-scenario-18-02-pair-06-0",
          "g3-scenario-18-02-pair-07-0",
          "g3-scenario-18-02-pair-09-1",
          "g3-scenario-18-02-pair-11-0",
          "g3-scenario-18-03-pair-00-0",
          "g3-scenario-18-03-pair-00-1",
          "g3-scenario-18-03-pair-01-1",
          "g3-scenario-18-03-pair-02-0",
          "g3-scenario-18-03-pair-05-0",
          "g3-scenario-18-03-pair-05-1",
          "g3-scenario-18-03-pair-07-0",
          "g3-scenario-18-03-pair-08-0",
          "g3-scenario-18-03-pair-09-0",
          "g3-scenario-18-03-pair-09-1",
          "g3-scenario-18-03-pair-10-0",
          "g3-scenario-18-03-pair-11-1",
          "g3-scenario-19-00-pair-00-0",
          "g3-scenario-19-00-pair-00-1",
          "g3-scenario-19-00-pair-04-1",
          "g3-scenario-19-00-pair-07-0",
          "g3-scenario-19-00-pair-07-1",
          "g3-scenario-19-00-pair-09-1",
          "g3-scenario-19-00-pair-10-1",
          "g3-scenario-19-01-pair-03-1",
          "g3-scenario-19-01-pair-05-1",
          "g3-scenario-19-01-pair-06-0",
          "g3-scenario-19-01-pair-07-0",
          "g3-scenario-19-01-pair-10-0",
          "g3-scenario-19-01-pair-11-1",
          "g3-scenario-19-01-pair-12-0",
          "g3-scenario-19-02-pair-01-1",
          "g3-scenario-19-02-pair-02-0",
          "g3-scenario-19-02-pair-04-1",
          "g3-scenario-19-02-pair-05-0",
          "g3-scenario-19-02-pair-05-1",
          "g3-scenario-19-02-pair-07-0",
          "g3-scenario-19-02-pair-08-1",
          "g3-scenario-19-02-pair-11-0",
          "g3-scenario-19-02-pair-12-1",
          "g3-scenario-19-03-pair-00-0",
          "g3-scenario-19-03-pair-00-1",
          "g3-scenario-19-03-pair-01-0",
          "g3-scenario-19-03-pair-01-1",
          "g3-scenario-19-03-pair-02-0",
          "g3-scenario-19-03-pair-03-1",
          "g3-scenario-19-03-pair-04-1",
          "g3-scenario-19-03-pair-06-0",
          "g3-scenario-19-03-pair-07-0",
          "g3-scenario-19-03-pair-07-1",
          "g3-scenario-19-03-pair-08-1",
          "g3-scenario-19-03-pair-09-1",
          "g3-scenario-19-03-pair-10-0",
          "g3-scenario-19-03-pair-11-1",
          "g3-scenario-19-03-pair-12-1"
        ],
        "overlap_incorrect_abstention_routes": 51,
        "overlap_capability_boundary_errors": 65
      }
    }
  },
  "public_memory": {
    "mlx_peak_allocated_bytes": 1663682166,
    "mlx_active_allocated_bytes": 973882452,
    "process_max_rss_bytes_macos": 1337262080
  },
  "public_qwen_forwards": 984,
  "source_hashes": {
    "decision-model/evaluations/public-benchmarks-v1/runs/v2-results-v1/summary.json": "4bf9377f6fba600ccaec90295e82fc3fbe625d6b95b27ac1a88c2b7778a3ddf9",
    "decision-model/evaluations/public-benchmarks-v1/runs/v2-results-v1/predictions.jsonl": "4372e0012472cbcbc5a55f93c0fa39ba71326466170ea7c33561f8811d39b327",
    "decision-model/evaluations/public-benchmarks-v1/runs/v2-results-v1/report.md": "103473c5eb6118b0ffc632d1ac048c016101198a080fcf40a9c97026153d08ce",
    "decision-model/protocols/joint-schema-v3/runs/confirmation-v3/summary.json": "0eba8f9b8cd685501c017fa867f9a5f8dc1517e2c8e8c8de9fc0baec960dee78",
    "decision-model/protocols/joint-schema-v3/runs/confirmation-v3/predictions.jsonl": "ee9d74931778a893a993aec2f6af8f09e003c2810836dfdcce028596c1fe1b6b",
    "decision-model/protocols/joint-schema-v3/runs/confirmation-v3/report.md": "138bcc383f404c011c7bdfc80c7e30212e04232dc43acf5a98374c7653b67176"
  },
  "limitations": [
    "Adapted bounded-choice routing diagnostics, not official ToolRet retrieval or BFCL leaderboard scores.",
    "ToolRet is APIGen only, four real candidates plus no_tool; two lexical-hard and one seeded-random distractor. Not full-corpus retrieval.",
    "A singleton relevance judgment does not guarantee an unambiguous executable call. Unjudged distractors may also be relevant; public labels can be noisy.",
    "ToolRet native names retained as native_name; unique corpus IDs replace name. Other documentation fields are preserved.",
    "BFCL uses single-user-message multiple/irrelevance only; single-call name selection, no argument scoring, execution, parallel or multi-turn evaluation.",
    "V2 prompt is unchanged: missing inputs, drafting and multiple-tool requests map to no_tool. Retrieval relevance and executability can conflict.",
    "No schema or description truncation. Length-compatible subset excludes examples over 1536 tokens in any diagnostic order; exclusions saved.",
    "Fixed synthetic date/time context; source queries are not rewritten. Public benchmark pretraining contamination is unknown.",
    "No training, model selection, threshold fitting or temperature fitting. Scores reported per panel; no pooled cross-benchmark score.",
    "BFCL panels may reuse request wording with different candidate sets; these are correlated availability contrasts, not independent requests.",
    "Canonical latency excludes loading/serialization/order diagnostics and includes tokenization, frozen Qwen and synchronized head. First request retained.",
    "One authored template dataset; pairs/family instances are correlated.",
    "Held-out frame/schema families share task grammar and eight capabilities, not unseen domains.",
    "Three seeds quantify training variation, not independent dataset replication.",
    "Two-choice cases have two unique orders; other sets have four.",
    "No thresholds, fitted temperatures, generation or LoRA."
  ]
}