{
  "version": 2,
  "benchmark": "decisions-v3",
  "status": "complete",
  "started_at": "2026-09-20T15:48:52.820Z",
  "finished_at": "2026-09-20T15:52:04.611Z",
  "model": "qwen3.5-4b",
  "source": {
    "commit": "476a2a04140344c01d0797c0fc250823d394605b",
    "relevant_worktree_modified": false,
    "harness_sha256": "aedaf7e29b46ac0122f44233415d966eec0998a4b4fcf34bd8f32a8d8f21f515",
    "shared_harness_sha256": "9c363a7d135c9dfe97797eddb54cbb56826913dbcb9919e7bddaa03b0aeed162",
    "fixture_id": "decisions-v3",
    "fixture_sha256": "992d0078faff0f781d87be0755828b48689345e4a8a68d20b25f12b7a0fa87cc",
    "warmup_fixture_sha256": "8e3073ac4ad1fb460fe574084b40631577ece6a49e190fca2d89659cee15604c",
    "package_sha256": "a63e63026627c5d5178496768e9642dde3e31d114225643ce4410cbb5cb932b8",
    "runtime_source_sha256": "2dbe7f535f0091929cc24f4c9b6f021434ac7b8a57d6420416881a8c07adc68d",
    "lockfile_sha256": "2641927e4207467ad05df6fb30afa5384d68385b8c6271576ff7bf544c9fb178"
  },
  "environment": {
    "platform": "darwin",
    "arch": "arm64",
    "os": "25.5.0",
    "bun": "1.3.14",
    "cpu": "Apple M5 Max",
    "memory_bytes": 38654705664,
    "native": {
      "ok": true,
      "elapsed_ms": 205,
      "backend": "metal",
      "gpu_offloading": true,
      "supported_backends": [
        "metal"
      ]
    },
    "weight": {
      "id": "qwen3.5-4b",
      "sha256": "00fe7986ff5f6b463e62455821146049db6f9313603938a70800d1fb69ef11a4",
      "bytes": 2740937888
    },
    "client_region": "Not independently verified",
    "provider": null
  },
  "methodology": {
    "unique_cases": 72,
    "cases_per_type": 24,
    "initial_calls": 1,
    "warmups": 1,
    "measured_repetitions": 3,
    "measured_calls": 216,
    "permutation_calls": 144,
    "planned_calls": 362,
    "concurrency": 1,
    "context_tokens": 2048,
    "percentile_method": "nearest rank, ceil(p*n)-1 in sorted elapsed times",
    "request_timeout_ms": 60000,
    "run_timeout_ms": 1200000,
    "stop_rule": "Stop after three consecutive invalid/error responses; no retries.",
    "latency_boundary": "LocalRunner through complete decision, including model/IPC; excludes HTTP and response validation",
    "throughput_boundary": "Measured loop wall time including validation, excluding report persistence and permutations.",
    "grading": "Choice exact label. Noul >0.5 is true, <0.5 false, tie incorrect. Score unique maximum-probability level; ties incorrect; weighted-score absolute error reported separately. Failures count as incorrect.",
    "quality_scope": "72 authored synthetic cases in nine low-risk workflow families. Fixture and labels frozen before model execution. Not a production benchmark, calibration study or independent third-party evaluation. Repeats/permutations are not independent cases.",
    "permutations": "All six orders for each of 24 choice cases. Invariance requires six valid identical semantic choices; it does not mean correct.",
    "initial_scope": "Initial and warmup use prior forms-v1 cases, not cases from the selected qualification fixture. One initial request, not an isolated cold-machine/server measurement. Weight verification primes filesystem caches.",
    "usage": "Actual wrapped local prompt tokens or provider-reported counters. Missing usage fails validation. Local output zero means no generated answer text; not no computation.",
    "input_usd_per_million": null,
    "output_usd_per_million": null,
    "price_source": null,
    "price_checked": "2026-09-20",
    "cost_scope": "Known reported usage estimate, not invoice. All calls include warmups/permutations; unknown failed-call billing remains unknown."
  },
  "samples": [
    {
      "phase": "initial",
      "repetition": 0,
      "case_id": "warmup-form-01",
      "family": "warmup-forms",
      "type": "choice",
      "order": [
        "submit",
        "correct",
        "wait"
      ],
      "elapsed_ms": 1667.2693330000002,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "submit",
            "probabilities": {
              "submit": 1,
              "correct": 0,
              "wait": 0
            },
            "confidence": 1
          }
        },
        "usage": {
          "input_tokens": 136,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "submit"
      },
      "grade": {
        "correct": true,
        "predicted": "submit",
        "score_absolute_error": null,
        "confidence": 1,
        "tie": false
      }
    },
    {
      "phase": "warmup",
      "repetition": 0,
      "case_id": "warmup-form-02",
      "family": "warmup-forms",
      "type": "choice",
      "order": [
        "submit",
        "correct",
        "wait"
      ],
      "elapsed_ms": 455.60612500000025,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "correct",
            "probabilities": {
              "submit": 0,
              "correct": 1,
              "wait": 0
            },
            "confidence": 1
          }
        },
        "usage": {
          "input_tokens": 131,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "correct"
      },
      "grade": {
        "correct": true,
        "predicted": "correct",
        "score_absolute_error": null,
        "confidence": 1,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "image-layout-01",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "landscape",
        "balanced",
        "portrait"
      ],
      "elapsed_ms": 427.38358300000027,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "landscape": 0.763,
              "balanced": 0.23,
              "portrait": 0.007
            },
            "confidence": 0.645
          }
        },
        "usage": {
          "input_tokens": 203,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "grade": {
        "correct": true,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.645,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "image-layout-02",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "portrait",
        "landscape"
      ],
      "elapsed_ms": 420.3925420000005,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.282,
              "portrait": 0.268,
              "landscape": 0.45
            },
            "confidence": 0.175
          }
        },
        "usage": {
          "input_tokens": 203,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "portrait"
      },
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.175,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "image-layout-03",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "landscape",
        "portrait",
        "balanced"
      ],
      "elapsed_ms": 434.2880409999998,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "balanced",
            "probabilities": {
              "landscape": 0.162,
              "portrait": 0.061,
              "balanced": 0.777
            },
            "confidence": 0.665
          }
        },
        "usage": {
          "input_tokens": 203,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "grade": {
        "correct": true,
        "predicted": "balanced",
        "score_absolute_error": null,
        "confidence": 0.665,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "image-layout-04",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "landscape",
        "portrait"
      ],
      "elapsed_ms": 429.1167920000007,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.267,
              "landscape": 0.727,
              "portrait": 0.006
            },
            "confidence": 0.59
          }
        },
        "usage": {
          "input_tokens": 205,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.59,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "image-layout-05",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "portrait",
        "landscape",
        "balanced"
      ],
      "elapsed_ms": 435.70437500000025,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "portrait": 0.065,
              "landscape": 0.861,
              "balanced": 0.074
            },
            "confidence": 0.792
          }
        },
        "usage": {
          "input_tokens": 204,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "grade": {
        "correct": true,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.792,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "image-layout-06",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "landscape",
        "portrait"
      ],
      "elapsed_ms": 429.990417,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.254,
              "landscape": 0.423,
              "portrait": 0.323
            },
            "confidence": 0.135
          }
        },
        "usage": {
          "input_tokens": 204,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "portrait"
      },
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.135,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "image-layout-07",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "portrait",
        "landscape"
      ],
      "elapsed_ms": 441.5448749999996,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "portrait",
            "probabilities": {
              "balanced": 0.281,
              "portrait": 0.637,
              "landscape": 0.082
            },
            "confidence": 0.456
          }
        },
        "usage": {
          "input_tokens": 215,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "grade": {
        "correct": false,
        "predicted": "portrait",
        "score_absolute_error": null,
        "confidence": 0.456,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "image-layout-08",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "portrait",
        "landscape",
        "balanced"
      ],
      "elapsed_ms": 436.17362500000036,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "portrait": 0.154,
              "landscape": 0.462,
              "balanced": 0.384
            },
            "confidence": 0.193
          }
        },
        "usage": {
          "input_tokens": 202,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "grade": {
        "correct": true,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.193,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "notification-delivery-01",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "digest",
        "email"
      ],
      "elapsed_ms": 435.20754099999976,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "in_app": 0.142,
              "digest": 0.118,
              "email": 0.74
            },
            "confidence": 0.61
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "grade": {
        "correct": true,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.61,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "notification-delivery-02",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "digest",
        "email"
      ],
      "elapsed_ms": 427.01558300000033,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "in_app": 0.344,
              "digest": 0.201,
              "email": 0.455
            },
            "confidence": 0.182
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "in_app"
      },
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.182,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "notification-delivery-03",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "email",
        "digest",
        "in_app"
      ],
      "elapsed_ms": 430.1415829999996,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "digest",
            "probabilities": {
              "email": 0.18,
              "digest": 0.663,
              "in_app": 0.157
            },
            "confidence": 0.494
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "grade": {
        "correct": true,
        "predicted": "digest",
        "score_absolute_error": null,
        "confidence": 0.494,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "notification-delivery-04",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "in_app",
        "email"
      ],
      "elapsed_ms": 421.94537500000024,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "digest": 0.025,
              "in_app": 0.199,
              "email": 0.776
            },
            "confidence": 0.664
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "grade": {
        "correct": true,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.664,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "notification-delivery-05",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "digest",
        "email"
      ],
      "elapsed_ms": 444.98154199999954,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "digest",
            "probabilities": {
              "in_app": 0.29,
              "digest": 0.402,
              "email": 0.308
            },
            "confidence": 0.103
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "in_app"
      },
      "grade": {
        "correct": false,
        "predicted": "digest",
        "score_absolute_error": null,
        "confidence": 0.103,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "notification-delivery-06",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "email",
        "digest",
        "in_app"
      ],
      "elapsed_ms": 430.9445830000004,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "digest",
            "probabilities": {
              "email": 0.189,
              "digest": 0.594,
              "in_app": 0.217
            },
            "confidence": 0.391
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "grade": {
        "correct": true,
        "predicted": "digest",
        "score_absolute_error": null,
        "confidence": 0.391,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "notification-delivery-07",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "in_app",
        "email"
      ],
      "elapsed_ms": 472.8114170000008,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "digest": 0.11,
              "in_app": 0.362,
              "email": 0.528
            },
            "confidence": 0.292
          }
        },
        "usage": {
          "input_tokens": 219,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "grade": {
        "correct": true,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.292,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "notification-delivery-08",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "email",
        "in_app"
      ],
      "elapsed_ms": 480.9132920000011,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "digest": 0.369,
              "email": 0.42,
              "in_app": 0.211
            },
            "confidence": 0.13
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.13,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "release-note-group-01",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "fix",
        "breaking",
        "addition"
      ],
      "elapsed_ms": 491.0450000000001,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "breaking",
            "probabilities": {
              "fix": 0.01,
              "breaking": 0.988,
              "addition": 0.002
            },
            "confidence": 0.983
          }
        },
        "usage": {
          "input_tokens": 225,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "breaking"
      },
      "grade": {
        "correct": true,
        "predicted": "breaking",
        "score_absolute_error": null,
        "confidence": 0.983,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "release-note-group-02",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "breaking",
        "fix"
      ],
      "elapsed_ms": 451.4007500000007,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "addition": 0.002,
              "breaking": 0.023,
              "fix": 0.975
            },
            "confidence": 0.963
          }
        },
        "usage": {
          "input_tokens": 218,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "grade": {
        "correct": true,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.963,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "release-note-group-03",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "breaking",
        "fix"
      ],
      "elapsed_ms": 458.3531669999993,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "addition": 0.992,
              "breaking": 0.001,
              "fix": 0.006
            },
            "confidence": 0.989
          }
        },
        "usage": {
          "input_tokens": 214,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "grade": {
        "correct": true,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.989,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "release-note-group-04",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "fix",
        "breaking",
        "addition"
      ],
      "elapsed_ms": 451.2602909999987,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "breaking",
            "probabilities": {
              "fix": 0.055,
              "breaking": 0.942,
              "addition": 0.002
            },
            "confidence": 0.914
          }
        },
        "usage": {
          "input_tokens": 222,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "breaking"
      },
      "grade": {
        "correct": true,
        "predicted": "breaking",
        "score_absolute_error": null,
        "confidence": 0.914,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "release-note-group-05",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "addition",
        "fix"
      ],
      "elapsed_ms": 467.1935830000002,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "breaking": 0,
              "addition": 0.003,
              "fix": 0.997
            },
            "confidence": 0.995
          }
        },
        "usage": {
          "input_tokens": 220,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "grade": {
        "correct": true,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.995,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "release-note-group-06",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "fix",
        "breaking"
      ],
      "elapsed_ms": 471.74066700000003,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "addition": 0.937,
              "fix": 0.012,
              "breaking": 0.051
            },
            "confidence": 0.905
          }
        },
        "usage": {
          "input_tokens": 226,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "grade": {
        "correct": true,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.905,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "release-note-group-07",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "fix",
        "addition"
      ],
      "elapsed_ms": 463.0936669999992,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "breaking": 0.083,
              "fix": 0.184,
              "addition": 0.733
            },
            "confidence": 0.6
          }
        },
        "usage": {
          "input_tokens": 220,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "grade": {
        "correct": false,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.6,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "release-note-group-08",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "fix",
        "addition"
      ],
      "elapsed_ms": 490.26129200000105,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "breaking": 0.005,
              "fix": 0,
              "addition": 0.995
            },
            "confidence": 0.992
          }
        },
        "usage": {
          "input_tokens": 216,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "grade": {
        "correct": true,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.992,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "demo-discount-01",
      "family": "demo-discount",
      "type": "noul",
      "order": null,
      "elapsed_ms": 463.5462920000009,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.986
          }
        },
        "usage": {
          "input_tokens": 201,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": true
      },
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.986,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "demo-discount-02",
      "family": "demo-discount",
      "type": "noul",
      "order": null,
      "elapsed_ms": 467.69195800000125,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.168
          }
        },
        "usage": {
          "input_tokens": 201,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": false
      },
      "grade": {
        "correct": true,
        "predicted": false,
        "score_absolute_error": null,
        "confidence": 0.832,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "demo-discount-03",
      "family": "demo-discount",
      "type": "noul",
      "order": null,
      "elapsed_ms": 469.5309589999997,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.992
          }
        },
        "usage": {
          "input_tokens": 201,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": true
      },
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.992,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "demo-discount-04",
      "family": "demo-discount",
      "type": "noul",
      "order": null,
      "elapsed_ms": 469.33087499999965,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.952
          }
        },
        "usage": {
          "input_tokens": 201,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": false
      },
      "grade": {
        "correct": false,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.952,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "demo-discount-05",
      "family": "demo-discount",
      "type": "noul",
      "order": null,
      "elapsed_ms": 467.57083299999977,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.958
          }
        },
        "usage": {
          "input_tokens": 203,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": true
      },
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.958,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "demo-discount-06",
      "family": "demo-discount",
      "type": "noul",
      "order": null,
      "elapsed_ms": 457.81437500000175,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.089
          }
        },
        "usage": {
          "input_tokens": 201,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": false
      },
      "grade": {
        "correct": true,
        "predicted": false,
        "score_absolute_error": null,
        "confidence": 0.911,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "demo-discount-07",
      "family": "demo-discount",
      "type": "noul",
      "order": null,
      "elapsed_ms": 470.28545900000245,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.985
          }
        },
        "usage": {
          "input_tokens": 202,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": true
      },
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.985,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "demo-discount-08",
      "family": "demo-discount",
      "type": "noul",
      "order": null,
      "elapsed_ms": 477.04049999999916,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.066
          }
        },
        "usage": {
          "input_tokens": 202,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": false
      },
      "grade": {
        "correct": true,
        "predicted": false,
        "score_absolute_error": null,
        "confidence": 0.9339999999999999,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "overnight-counter-01",
      "family": "overnight-counter",
      "type": "noul",
      "order": null,
      "elapsed_ms": 431.50066700000025,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.958
          }
        },
        "usage": {
          "input_tokens": 186,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": true
      },
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.958,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "overnight-counter-02",
      "family": "overnight-counter",
      "type": "noul",
      "order": null,
      "elapsed_ms": 452.69695800000045,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.745
          }
        },
        "usage": {
          "input_tokens": 186,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": false
      },
      "grade": {
        "correct": false,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.745,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "overnight-counter-03",
      "family": "overnight-counter",
      "type": "noul",
      "order": null,
      "elapsed_ms": 465.2954170000012,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.956
          }
        },
        "usage": {
          "input_tokens": 186,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": true
      },
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.956,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "overnight-counter-04",
      "family": "overnight-counter",
      "type": "noul",
      "order": null,
      "elapsed_ms": 438.92054199999984,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.528
          }
        },
        "usage": {
          "input_tokens": 186,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": false
      },
      "grade": {
        "correct": false,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.528,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "overnight-counter-05",
      "family": "overnight-counter",
      "type": "noul",
      "order": null,
      "elapsed_ms": 420.68629099999816,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.809
          }
        },
        "usage": {
          "input_tokens": 186,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": true
      },
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.809,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "overnight-counter-06",
      "family": "overnight-counter",
      "type": "noul",
      "order": null,
      "elapsed_ms": 407.3545830000003,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.263
          }
        },
        "usage": {
          "input_tokens": 186,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": false
      },
      "grade": {
        "correct": true,
        "predicted": false,
        "score_absolute_error": null,
        "confidence": 0.737,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "overnight-counter-07",
      "family": "overnight-counter",
      "type": "noul",
      "order": null,
      "elapsed_ms": 422.51666699999987,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.82
          }
        },
        "usage": {
          "input_tokens": 186,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": true
      },
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.82,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "overnight-counter-08",
      "family": "overnight-counter",
      "type": "noul",
      "order": null,
      "elapsed_ms": 436.0667910000011,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.721
          }
        },
        "usage": {
          "input_tokens": 186,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": false
      },
      "grade": {
        "correct": false,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.721,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "tag-filter-01",
      "family": "tag-filter",
      "type": "noul",
      "order": null,
      "elapsed_ms": 414.97279100000014,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.881
          }
        },
        "usage": {
          "input_tokens": 180,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": true
      },
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.881,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "tag-filter-02",
      "family": "tag-filter",
      "type": "noul",
      "order": null,
      "elapsed_ms": 400.3118340000001,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.863
          }
        },
        "usage": {
          "input_tokens": 177,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": false
      },
      "grade": {
        "correct": false,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.863,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "tag-filter-03",
      "family": "tag-filter",
      "type": "noul",
      "order": null,
      "elapsed_ms": 439.4837090000001,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.95
          }
        },
        "usage": {
          "input_tokens": 181,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": true
      },
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.95,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "tag-filter-04",
      "family": "tag-filter",
      "type": "noul",
      "order": null,
      "elapsed_ms": 431.2820410000022,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.644
          }
        },
        "usage": {
          "input_tokens": 180,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": false
      },
      "grade": {
        "correct": false,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.644,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "tag-filter-05",
      "family": "tag-filter",
      "type": "noul",
      "order": null,
      "elapsed_ms": 464.35633400000006,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.985
          }
        },
        "usage": {
          "input_tokens": 174,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": true
      },
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.985,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "tag-filter-06",
      "family": "tag-filter",
      "type": "noul",
      "order": null,
      "elapsed_ms": 456.62445899999875,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.876
          }
        },
        "usage": {
          "input_tokens": 179,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": false
      },
      "grade": {
        "correct": false,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.876,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "tag-filter-07",
      "family": "tag-filter",
      "type": "noul",
      "order": null,
      "elapsed_ms": 459.3152090000003,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.928
          }
        },
        "usage": {
          "input_tokens": 180,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": true
      },
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.928,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "tag-filter-08",
      "family": "tag-filter",
      "type": "noul",
      "order": null,
      "elapsed_ms": 426.47620800000004,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.845
          }
        },
        "usage": {
          "input_tokens": 188,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": false
      },
      "grade": {
        "correct": false,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.845,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "mini-game-points-01",
      "family": "mini-game-points",
      "type": "score",
      "order": null,
      "elapsed_ms": 494.3042090000017,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 0.245,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.793,
              "1": 0.183,
              "2": 0.01,
              "3": 0.014
            },
            "confidence": 0.724
          }
        },
        "usage": {
          "input_tokens": 199,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 0
      },
      "grade": {
        "correct": true,
        "predicted": 0,
        "score_absolute_error": 0.245,
        "confidence": 0.724,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "mini-game-points-02",
      "family": "mini-game-points",
      "type": "score",
      "order": null,
      "elapsed_ms": 490.83329199999935,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.025,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.199,
              "1": 0.628,
              "2": 0.123,
              "3": 0.05
            },
            "confidence": 0.504
          }
        },
        "usage": {
          "input_tokens": 199,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 1
      },
      "grade": {
        "correct": true,
        "predicted": 1,
        "score_absolute_error": 0.02499999999999991,
        "confidence": 0.504,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "mini-game-points-03",
      "family": "mini-game-points",
      "type": "score",
      "order": null,
      "elapsed_ms": 476.6175000000003,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 0.875,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.32,
              "1": 0.53,
              "2": 0.104,
              "3": 0.045
            },
            "confidence": 0.374
          }
        },
        "usage": {
          "input_tokens": 199,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 2
      },
      "grade": {
        "correct": false,
        "predicted": 1,
        "score_absolute_error": 1.125,
        "confidence": 0.374,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "mini-game-points-04",
      "family": "mini-game-points",
      "type": "score",
      "order": null,
      "elapsed_ms": 484.8821250000001,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.135,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.183,
              "1": 0.571,
              "2": 0.173,
              "3": 0.072
            },
            "confidence": 0.428
          }
        },
        "usage": {
          "input_tokens": 199,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 3
      },
      "grade": {
        "correct": false,
        "predicted": 1,
        "score_absolute_error": 1.865,
        "confidence": 0.428,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "mini-game-points-05",
      "family": "mini-game-points",
      "type": "score",
      "order": null,
      "elapsed_ms": 500.5121670000008,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.012,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.309,
              "1": 0.45,
              "2": 0.161,
              "3": 0.08
            },
            "confidence": 0.266
          }
        },
        "usage": {
          "input_tokens": 199,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 0
      },
      "grade": {
        "correct": false,
        "predicted": 1,
        "score_absolute_error": 1.012,
        "confidence": 0.266,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "mini-game-points-06",
      "family": "mini-game-points",
      "type": "score",
      "order": null,
      "elapsed_ms": 483.5600000000013,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.459,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.159,
              "1": 0.411,
              "2": 0.243,
              "3": 0.187
            },
            "confidence": 0.214
          }
        },
        "usage": {
          "input_tokens": 199,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 1
      },
      "grade": {
        "correct": true,
        "predicted": 1,
        "score_absolute_error": 0.4590000000000001,
        "confidence": 0.214,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "mini-game-points-07",
      "family": "mini-game-points",
      "type": "score",
      "order": null,
      "elapsed_ms": 487.9890000000014,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.227,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.308,
              "1": 0.357,
              "2": 0.135,
              "3": 0.2
            },
            "confidence": 0.143
          }
        },
        "usage": {
          "input_tokens": 207,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 2
      },
      "grade": {
        "correct": false,
        "predicted": 1,
        "score_absolute_error": 0.7729999999999999,
        "confidence": 0.143,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "mini-game-points-08",
      "family": "mini-game-points",
      "type": "score",
      "order": null,
      "elapsed_ms": 475.94187500000044,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.613,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.087,
              "1": 0.398,
              "2": 0.329,
              "3": 0.185
            },
            "confidence": 0.198
          }
        },
        "usage": {
          "input_tokens": 199,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 3
      },
      "grade": {
        "correct": false,
        "predicted": 1,
        "score_absolute_error": 1.387,
        "confidence": 0.198,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "display-detail-01",
      "family": "display-detail",
      "type": "score",
      "order": null,
      "elapsed_ms": 506.7021659999991,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.097,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.133,
              "1": 0.674,
              "2": 0.157,
              "3": 0.037
            },
            "confidence": 0.565
          }
        },
        "usage": {
          "input_tokens": 226,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 0
      },
      "grade": {
        "correct": false,
        "predicted": 1,
        "score_absolute_error": 1.097,
        "confidence": 0.565,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "display-detail-02",
      "family": "display-detail",
      "type": "score",
      "order": null,
      "elapsed_ms": 507.94937500000015,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.035,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.171,
              "1": 0.658,
              "2": 0.135,
              "3": 0.036
            },
            "confidence": 0.544
          }
        },
        "usage": {
          "input_tokens": 226,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 1
      },
      "grade": {
        "correct": true,
        "predicted": 1,
        "score_absolute_error": 0.03499999999999992,
        "confidence": 0.544,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "display-detail-03",
      "family": "display-detail",
      "type": "score",
      "order": null,
      "elapsed_ms": 503.9039589999993,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.741,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.023,
              "1": 0.347,
              "2": 0.498,
              "3": 0.133
            },
            "confidence": 0.331
          }
        },
        "usage": {
          "input_tokens": 227,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 2
      },
      "grade": {
        "correct": true,
        "predicted": 2,
        "score_absolute_error": 0.2589999999999999,
        "confidence": 0.331,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "display-detail-04",
      "family": "display-detail",
      "type": "score",
      "order": null,
      "elapsed_ms": 502.87995800000135,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 2.08,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.029,
              "1": 0.246,
              "2": 0.342,
              "3": 0.383
            },
            "confidence": 0.178
          }
        },
        "usage": {
          "input_tokens": 227,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 3
      },
      "grade": {
        "correct": true,
        "predicted": 3,
        "score_absolute_error": 0.9199999999999999,
        "confidence": 0.178,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "display-detail-05",
      "family": "display-detail",
      "type": "score",
      "order": null,
      "elapsed_ms": 531.3502919999992,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 0.953,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.331,
              "1": 0.44,
              "2": 0.175,
              "3": 0.054
            },
            "confidence": 0.253
          }
        },
        "usage": {
          "input_tokens": 226,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 0
      },
      "grade": {
        "correct": false,
        "predicted": 1,
        "score_absolute_error": 0.953,
        "confidence": 0.253,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "display-detail-06",
      "family": "display-detail",
      "type": "score",
      "order": null,
      "elapsed_ms": 484.66241699999955,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.175,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.22,
              "1": 0.451,
              "2": 0.263,
              "3": 0.066
            },
            "confidence": 0.268
          }
        },
        "usage": {
          "input_tokens": 226,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 1
      },
      "grade": {
        "correct": true,
        "predicted": 1,
        "score_absolute_error": 0.17500000000000004,
        "confidence": 0.268,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "display-detail-07",
      "family": "display-detail",
      "type": "score",
      "order": null,
      "elapsed_ms": 482.0637910000005,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.246,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.226,
              "1": 0.404,
              "2": 0.268,
              "3": 0.102
            },
            "confidence": 0.205
          }
        },
        "usage": {
          "input_tokens": 227,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 2
      },
      "grade": {
        "correct": false,
        "predicted": 1,
        "score_absolute_error": 0.754,
        "confidence": 0.205,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "display-detail-08",
      "family": "display-detail",
      "type": "score",
      "order": null,
      "elapsed_ms": 489.14229200000045,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 2.765,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.009,
              "1": 0.05,
              "2": 0.107,
              "3": 0.833
            },
            "confidence": 0.778
          }
        },
        "usage": {
          "input_tokens": 232,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 3
      },
      "grade": {
        "correct": true,
        "predicted": 3,
        "score_absolute_error": 0.23499999999999988,
        "confidence": 0.778,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "glossary-strength-01",
      "family": "glossary-strength",
      "type": "score",
      "order": null,
      "elapsed_ms": 502.5108330000003,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 0.98,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.494,
              "1": 0.231,
              "2": 0.078,
              "3": 0.197
            },
            "confidence": 0.325
          }
        },
        "usage": {
          "input_tokens": 224,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 0
      },
      "grade": {
        "correct": true,
        "predicted": 0,
        "score_absolute_error": 0.98,
        "confidence": 0.325,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "glossary-strength-02",
      "family": "glossary-strength",
      "type": "score",
      "order": null,
      "elapsed_ms": 474.5620410000047,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.219,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.387,
              "1": 0.229,
              "2": 0.162,
              "3": 0.222
            },
            "confidence": 0.183
          }
        },
        "usage": {
          "input_tokens": 230,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 1
      },
      "grade": {
        "correct": false,
        "predicted": 0,
        "score_absolute_error": 0.21900000000000008,
        "confidence": 0.183,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "glossary-strength-03",
      "family": "glossary-strength",
      "type": "score",
      "order": null,
      "elapsed_ms": 446.8800839999967,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.845,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.06,
              "1": 0.306,
              "2": 0.362,
              "3": 0.271
            },
            "confidence": 0.15
          }
        },
        "usage": {
          "input_tokens": 224,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 2
      },
      "grade": {
        "correct": true,
        "predicted": 2,
        "score_absolute_error": 0.15500000000000003,
        "confidence": 0.15,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "glossary-strength-04",
      "family": "glossary-strength",
      "type": "score",
      "order": null,
      "elapsed_ms": 481.5972499999989,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 2.852,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.007,
              "1": 0.006,
              "2": 0.113,
              "3": 0.874
            },
            "confidence": 0.831
          }
        },
        "usage": {
          "input_tokens": 233,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 3
      },
      "grade": {
        "correct": true,
        "predicted": 3,
        "score_absolute_error": 0.14800000000000013,
        "confidence": 0.831,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "glossary-strength-05",
      "family": "glossary-strength",
      "type": "score",
      "order": null,
      "elapsed_ms": 475.0971669999999,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 2.786,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.021,
              "1": 0.019,
              "2": 0.111,
              "3": 0.848
            },
            "confidence": 0.797
          }
        },
        "usage": {
          "input_tokens": 235,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 0
      },
      "grade": {
        "correct": false,
        "predicted": 3,
        "score_absolute_error": 2.786,
        "confidence": 0.797,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "glossary-strength-06",
      "family": "glossary-strength",
      "type": "score",
      "order": null,
      "elapsed_ms": 518.2700409999961,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 2.109,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.087,
              "1": 0.182,
              "2": 0.265,
              "3": 0.465
            },
            "confidence": 0.287
          }
        },
        "usage": {
          "input_tokens": 235,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 1
      },
      "grade": {
        "correct": false,
        "predicted": 3,
        "score_absolute_error": 1.109,
        "confidence": 0.287,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "glossary-strength-07",
      "family": "glossary-strength",
      "type": "score",
      "order": null,
      "elapsed_ms": 484.9689999999973,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.359,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.279,
              "1": 0.286,
              "2": 0.232,
              "3": 0.203
            },
            "confidence": 0.049
          }
        },
        "usage": {
          "input_tokens": 234,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 2
      },
      "grade": {
        "correct": false,
        "predicted": 1,
        "score_absolute_error": 0.641,
        "confidence": 0.049,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "glossary-strength-08",
      "family": "glossary-strength",
      "type": "score",
      "order": null,
      "elapsed_ms": 494.4502499999944,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 2.907,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.008,
              "1": 0.005,
              "2": 0.059,
              "3": 0.928
            },
            "confidence": 0.904
          }
        },
        "usage": {
          "input_tokens": 236,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 3
      },
      "grade": {
        "correct": true,
        "predicted": 3,
        "score_absolute_error": 0.09299999999999997,
        "confidence": 0.904,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "image-layout-01",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "landscape",
        "balanced",
        "portrait"
      ],
      "elapsed_ms": 434.27475000000413,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "landscape": 0.763,
              "balanced": 0.23,
              "portrait": 0.007
            },
            "confidence": 0.645
          }
        },
        "usage": {
          "input_tokens": 203,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "grade": {
        "correct": true,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.645,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "image-layout-02",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "portrait",
        "landscape"
      ],
      "elapsed_ms": 462.676916000004,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.282,
              "portrait": 0.268,
              "landscape": 0.45
            },
            "confidence": 0.175
          }
        },
        "usage": {
          "input_tokens": 203,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "portrait"
      },
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.175,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "image-layout-03",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "landscape",
        "portrait",
        "balanced"
      ],
      "elapsed_ms": 457.14304200000333,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "balanced",
            "probabilities": {
              "landscape": 0.162,
              "portrait": 0.061,
              "balanced": 0.777
            },
            "confidence": 0.665
          }
        },
        "usage": {
          "input_tokens": 203,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "grade": {
        "correct": true,
        "predicted": "balanced",
        "score_absolute_error": null,
        "confidence": 0.665,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "image-layout-04",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "landscape",
        "portrait"
      ],
      "elapsed_ms": 438.8373750000028,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.267,
              "landscape": 0.727,
              "portrait": 0.006
            },
            "confidence": 0.59
          }
        },
        "usage": {
          "input_tokens": 205,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.59,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "image-layout-05",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "portrait",
        "landscape",
        "balanced"
      ],
      "elapsed_ms": 458.8756249999933,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "portrait": 0.065,
              "landscape": 0.861,
              "balanced": 0.074
            },
            "confidence": 0.792
          }
        },
        "usage": {
          "input_tokens": 204,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "grade": {
        "correct": true,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.792,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "image-layout-06",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "landscape",
        "portrait"
      ],
      "elapsed_ms": 469.307624999994,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.254,
              "landscape": 0.423,
              "portrait": 0.323
            },
            "confidence": 0.135
          }
        },
        "usage": {
          "input_tokens": 204,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "portrait"
      },
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.135,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "image-layout-07",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "portrait",
        "landscape"
      ],
      "elapsed_ms": 441.663959000005,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "portrait",
            "probabilities": {
              "balanced": 0.281,
              "portrait": 0.637,
              "landscape": 0.082
            },
            "confidence": 0.456
          }
        },
        "usage": {
          "input_tokens": 215,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "grade": {
        "correct": false,
        "predicted": "portrait",
        "score_absolute_error": null,
        "confidence": 0.456,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "image-layout-08",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "portrait",
        "landscape",
        "balanced"
      ],
      "elapsed_ms": 449.8546669999996,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "portrait": 0.154,
              "landscape": 0.462,
              "balanced": 0.384
            },
            "confidence": 0.193
          }
        },
        "usage": {
          "input_tokens": 202,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "grade": {
        "correct": true,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.193,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "notification-delivery-01",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "digest",
        "email"
      ],
      "elapsed_ms": 456.30916699999943,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "in_app": 0.142,
              "digest": 0.118,
              "email": 0.74
            },
            "confidence": 0.61
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "grade": {
        "correct": true,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.61,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "notification-delivery-02",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "digest",
        "email"
      ],
      "elapsed_ms": 435.5625,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "in_app": 0.344,
              "digest": 0.201,
              "email": 0.455
            },
            "confidence": 0.182
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "in_app"
      },
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.182,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "notification-delivery-03",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "email",
        "digest",
        "in_app"
      ],
      "elapsed_ms": 440.53958299999795,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "digest",
            "probabilities": {
              "email": 0.18,
              "digest": 0.663,
              "in_app": 0.157
            },
            "confidence": 0.494
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "grade": {
        "correct": true,
        "predicted": "digest",
        "score_absolute_error": null,
        "confidence": 0.494,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "notification-delivery-04",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "in_app",
        "email"
      ],
      "elapsed_ms": 443.97925000000396,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "digest": 0.025,
              "in_app": 0.199,
              "email": 0.776
            },
            "confidence": 0.664
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "grade": {
        "correct": true,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.664,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "notification-delivery-05",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "digest",
        "email"
      ],
      "elapsed_ms": 465.43308299999626,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "digest",
            "probabilities": {
              "in_app": 0.29,
              "digest": 0.402,
              "email": 0.308
            },
            "confidence": 0.103
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "in_app"
      },
      "grade": {
        "correct": false,
        "predicted": "digest",
        "score_absolute_error": null,
        "confidence": 0.103,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "notification-delivery-06",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "email",
        "digest",
        "in_app"
      ],
      "elapsed_ms": 447.4899590000059,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "digest",
            "probabilities": {
              "email": 0.189,
              "digest": 0.594,
              "in_app": 0.217
            },
            "confidence": 0.391
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "grade": {
        "correct": true,
        "predicted": "digest",
        "score_absolute_error": null,
        "confidence": 0.391,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "notification-delivery-07",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "in_app",
        "email"
      ],
      "elapsed_ms": 456.27475000000413,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "digest": 0.11,
              "in_app": 0.362,
              "email": 0.528
            },
            "confidence": 0.292
          }
        },
        "usage": {
          "input_tokens": 219,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "grade": {
        "correct": true,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.292,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "notification-delivery-08",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "email",
        "in_app"
      ],
      "elapsed_ms": 449.7881669999988,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "digest": 0.369,
              "email": 0.42,
              "in_app": 0.211
            },
            "confidence": 0.13
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.13,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "release-note-group-01",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "fix",
        "breaking",
        "addition"
      ],
      "elapsed_ms": 463.8003749999989,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "breaking",
            "probabilities": {
              "fix": 0.01,
              "breaking": 0.988,
              "addition": 0.002
            },
            "confidence": 0.983
          }
        },
        "usage": {
          "input_tokens": 225,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "breaking"
      },
      "grade": {
        "correct": true,
        "predicted": "breaking",
        "score_absolute_error": null,
        "confidence": 0.983,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "release-note-group-02",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "breaking",
        "fix"
      ],
      "elapsed_ms": 447.0410839999968,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "addition": 0.002,
              "breaking": 0.023,
              "fix": 0.975
            },
            "confidence": 0.963
          }
        },
        "usage": {
          "input_tokens": 218,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "grade": {
        "correct": true,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.963,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "release-note-group-03",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "breaking",
        "fix"
      ],
      "elapsed_ms": 452.28583300000173,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "addition": 0.992,
              "breaking": 0.001,
              "fix": 0.006
            },
            "confidence": 0.989
          }
        },
        "usage": {
          "input_tokens": 214,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "grade": {
        "correct": true,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.989,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "release-note-group-04",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "fix",
        "breaking",
        "addition"
      ],
      "elapsed_ms": 441.5318750000006,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "breaking",
            "probabilities": {
              "fix": 0.055,
              "breaking": 0.942,
              "addition": 0.002
            },
            "confidence": 0.914
          }
        },
        "usage": {
          "input_tokens": 222,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "breaking"
      },
      "grade": {
        "correct": true,
        "predicted": "breaking",
        "score_absolute_error": null,
        "confidence": 0.914,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "release-note-group-05",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "addition",
        "fix"
      ],
      "elapsed_ms": 444.1817080000037,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "breaking": 0,
              "addition": 0.003,
              "fix": 0.997
            },
            "confidence": 0.995
          }
        },
        "usage": {
          "input_tokens": 220,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "grade": {
        "correct": true,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.995,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "release-note-group-06",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "fix",
        "breaking"
      ],
      "elapsed_ms": 479.1859170000025,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "addition": 0.937,
              "fix": 0.012,
              "breaking": 0.051
            },
            "confidence": 0.905
          }
        },
        "usage": {
          "input_tokens": 226,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "grade": {
        "correct": true,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.905,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "release-note-group-07",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "fix",
        "addition"
      ],
      "elapsed_ms": 495.1465410000019,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "breaking": 0.083,
              "fix": 0.184,
              "addition": 0.733
            },
            "confidence": 0.6
          }
        },
        "usage": {
          "input_tokens": 220,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "grade": {
        "correct": false,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.6,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "release-note-group-08",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "fix",
        "addition"
      ],
      "elapsed_ms": 444.1980410000033,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "breaking": 0.005,
              "fix": 0,
              "addition": 0.995
            },
            "confidence": 0.992
          }
        },
        "usage": {
          "input_tokens": 216,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "grade": {
        "correct": true,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.992,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "demo-discount-01",
      "family": "demo-discount",
      "type": "noul",
      "order": null,
      "elapsed_ms": 449.31550000000425,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.986
          }
        },
        "usage": {
          "input_tokens": 201,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": true
      },
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.986,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "demo-discount-02",
      "family": "demo-discount",
      "type": "noul",
      "order": null,
      "elapsed_ms": 446.3230840000033,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.168
          }
        },
        "usage": {
          "input_tokens": 201,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": false
      },
      "grade": {
        "correct": true,
        "predicted": false,
        "score_absolute_error": null,
        "confidence": 0.832,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "demo-discount-03",
      "family": "demo-discount",
      "type": "noul",
      "order": null,
      "elapsed_ms": 444.29699999999866,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.992
          }
        },
        "usage": {
          "input_tokens": 201,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": true
      },
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.992,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "demo-discount-04",
      "family": "demo-discount",
      "type": "noul",
      "order": null,
      "elapsed_ms": 429.78279199999815,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.952
          }
        },
        "usage": {
          "input_tokens": 201,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": false
      },
      "grade": {
        "correct": false,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.952,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "demo-discount-05",
      "family": "demo-discount",
      "type": "noul",
      "order": null,
      "elapsed_ms": 430.6077920000025,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.958
          }
        },
        "usage": {
          "input_tokens": 203,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": true
      },
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.958,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "demo-discount-06",
      "family": "demo-discount",
      "type": "noul",
      "order": null,
      "elapsed_ms": 450.18358300000546,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.089
          }
        },
        "usage": {
          "input_tokens": 201,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": false
      },
      "grade": {
        "correct": true,
        "predicted": false,
        "score_absolute_error": null,
        "confidence": 0.911,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "demo-discount-07",
      "family": "demo-discount",
      "type": "noul",
      "order": null,
      "elapsed_ms": 433.44770900000003,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.985
          }
        },
        "usage": {
          "input_tokens": 202,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": true
      },
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.985,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "demo-discount-08",
      "family": "demo-discount",
      "type": "noul",
      "order": null,
      "elapsed_ms": 441.8242079999982,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.066
          }
        },
        "usage": {
          "input_tokens": 202,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": false
      },
      "grade": {
        "correct": true,
        "predicted": false,
        "score_absolute_error": null,
        "confidence": 0.9339999999999999,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "overnight-counter-01",
      "family": "overnight-counter",
      "type": "noul",
      "order": null,
      "elapsed_ms": 414.435166999996,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.958
          }
        },
        "usage": {
          "input_tokens": 186,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": true
      },
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.958,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "overnight-counter-02",
      "family": "overnight-counter",
      "type": "noul",
      "order": null,
      "elapsed_ms": 431.29037499999686,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.745
          }
        },
        "usage": {
          "input_tokens": 186,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": false
      },
      "grade": {
        "correct": false,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.745,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "overnight-counter-03",
      "family": "overnight-counter",
      "type": "noul",
      "order": null,
      "elapsed_ms": 432.09808299999713,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.956
          }
        },
        "usage": {
          "input_tokens": 186,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": true
      },
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.956,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "overnight-counter-04",
      "family": "overnight-counter",
      "type": "noul",
      "order": null,
      "elapsed_ms": 419.27929199999926,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.528
          }
        },
        "usage": {
          "input_tokens": 186,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": false
      },
      "grade": {
        "correct": false,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.528,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "overnight-counter-05",
      "family": "overnight-counter",
      "type": "noul",
      "order": null,
      "elapsed_ms": 410.50091699999757,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.809
          }
        },
        "usage": {
          "input_tokens": 186,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": true
      },
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.809,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "overnight-counter-06",
      "family": "overnight-counter",
      "type": "noul",
      "order": null,
      "elapsed_ms": 417.21291600000404,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.263
          }
        },
        "usage": {
          "input_tokens": 186,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": false
      },
      "grade": {
        "correct": true,
        "predicted": false,
        "score_absolute_error": null,
        "confidence": 0.737,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "overnight-counter-07",
      "family": "overnight-counter",
      "type": "noul",
      "order": null,
      "elapsed_ms": 421.1207080000022,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.82
          }
        },
        "usage": {
          "input_tokens": 186,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": true
      },
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.82,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "overnight-counter-08",
      "family": "overnight-counter",
      "type": "noul",
      "order": null,
      "elapsed_ms": 440.32454199999484,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.721
          }
        },
        "usage": {
          "input_tokens": 186,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": false
      },
      "grade": {
        "correct": false,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.721,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "tag-filter-01",
      "family": "tag-filter",
      "type": "noul",
      "order": null,
      "elapsed_ms": 410.4528749999954,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.881
          }
        },
        "usage": {
          "input_tokens": 180,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": true
      },
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.881,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "tag-filter-02",
      "family": "tag-filter",
      "type": "noul",
      "order": null,
      "elapsed_ms": 409.617290999995,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.863
          }
        },
        "usage": {
          "input_tokens": 177,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": false
      },
      "grade": {
        "correct": false,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.863,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "tag-filter-03",
      "family": "tag-filter",
      "type": "noul",
      "order": null,
      "elapsed_ms": 421.53024999999616,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.95
          }
        },
        "usage": {
          "input_tokens": 181,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": true
      },
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.95,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "tag-filter-04",
      "family": "tag-filter",
      "type": "noul",
      "order": null,
      "elapsed_ms": 407.17766700000357,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.644
          }
        },
        "usage": {
          "input_tokens": 180,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": false
      },
      "grade": {
        "correct": false,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.644,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "tag-filter-05",
      "family": "tag-filter",
      "type": "noul",
      "order": null,
      "elapsed_ms": 432.1254590000026,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.985
          }
        },
        "usage": {
          "input_tokens": 174,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": true
      },
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.985,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "tag-filter-06",
      "family": "tag-filter",
      "type": "noul",
      "order": null,
      "elapsed_ms": 428.8956249999974,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.876
          }
        },
        "usage": {
          "input_tokens": 179,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": false
      },
      "grade": {
        "correct": false,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.876,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "tag-filter-07",
      "family": "tag-filter",
      "type": "noul",
      "order": null,
      "elapsed_ms": 419.6512090000033,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.928
          }
        },
        "usage": {
          "input_tokens": 180,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": true
      },
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.928,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "tag-filter-08",
      "family": "tag-filter",
      "type": "noul",
      "order": null,
      "elapsed_ms": 407.12104199999885,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.845
          }
        },
        "usage": {
          "input_tokens": 188,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": false
      },
      "grade": {
        "correct": false,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.845,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "mini-game-points-01",
      "family": "mini-game-points",
      "type": "score",
      "order": null,
      "elapsed_ms": 460.37004100000195,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 0.245,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.793,
              "1": 0.183,
              "2": 0.01,
              "3": 0.014
            },
            "confidence": 0.724
          }
        },
        "usage": {
          "input_tokens": 199,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 0
      },
      "grade": {
        "correct": true,
        "predicted": 0,
        "score_absolute_error": 0.245,
        "confidence": 0.724,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "mini-game-points-02",
      "family": "mini-game-points",
      "type": "score",
      "order": null,
      "elapsed_ms": 468.6236250000002,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.025,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.199,
              "1": 0.628,
              "2": 0.123,
              "3": 0.05
            },
            "confidence": 0.504
          }
        },
        "usage": {
          "input_tokens": 199,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 1
      },
      "grade": {
        "correct": true,
        "predicted": 1,
        "score_absolute_error": 0.02499999999999991,
        "confidence": 0.504,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "mini-game-points-03",
      "family": "mini-game-points",
      "type": "score",
      "order": null,
      "elapsed_ms": 491.59554099999514,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 0.875,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.32,
              "1": 0.53,
              "2": 0.104,
              "3": 0.045
            },
            "confidence": 0.374
          }
        },
        "usage": {
          "input_tokens": 199,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 2
      },
      "grade": {
        "correct": false,
        "predicted": 1,
        "score_absolute_error": 1.125,
        "confidence": 0.374,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "mini-game-points-04",
      "family": "mini-game-points",
      "type": "score",
      "order": null,
      "elapsed_ms": 503.5292909999989,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.135,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.183,
              "1": 0.571,
              "2": 0.173,
              "3": 0.072
            },
            "confidence": 0.428
          }
        },
        "usage": {
          "input_tokens": 199,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 3
      },
      "grade": {
        "correct": false,
        "predicted": 1,
        "score_absolute_error": 1.865,
        "confidence": 0.428,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "mini-game-points-05",
      "family": "mini-game-points",
      "type": "score",
      "order": null,
      "elapsed_ms": 548.3499169999996,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.012,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.309,
              "1": 0.45,
              "2": 0.161,
              "3": 0.08
            },
            "confidence": 0.266
          }
        },
        "usage": {
          "input_tokens": 199,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 0
      },
      "grade": {
        "correct": false,
        "predicted": 1,
        "score_absolute_error": 1.012,
        "confidence": 0.266,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "mini-game-points-06",
      "family": "mini-game-points",
      "type": "score",
      "order": null,
      "elapsed_ms": 507.37741700000333,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.459,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.159,
              "1": 0.411,
              "2": 0.243,
              "3": 0.187
            },
            "confidence": 0.214
          }
        },
        "usage": {
          "input_tokens": 199,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 1
      },
      "grade": {
        "correct": true,
        "predicted": 1,
        "score_absolute_error": 0.4590000000000001,
        "confidence": 0.214,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "mini-game-points-07",
      "family": "mini-game-points",
      "type": "score",
      "order": null,
      "elapsed_ms": 494.18933399999514,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.227,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.308,
              "1": 0.357,
              "2": 0.135,
              "3": 0.2
            },
            "confidence": 0.143
          }
        },
        "usage": {
          "input_tokens": 207,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 2
      },
      "grade": {
        "correct": false,
        "predicted": 1,
        "score_absolute_error": 0.7729999999999999,
        "confidence": 0.143,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "mini-game-points-08",
      "family": "mini-game-points",
      "type": "score",
      "order": null,
      "elapsed_ms": 480.5487080000021,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.613,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.087,
              "1": 0.398,
              "2": 0.329,
              "3": 0.185
            },
            "confidence": 0.198
          }
        },
        "usage": {
          "input_tokens": 199,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 3
      },
      "grade": {
        "correct": false,
        "predicted": 1,
        "score_absolute_error": 1.387,
        "confidence": 0.198,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "display-detail-01",
      "family": "display-detail",
      "type": "score",
      "order": null,
      "elapsed_ms": 478.3839169999992,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.097,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.133,
              "1": 0.674,
              "2": 0.157,
              "3": 0.037
            },
            "confidence": 0.565
          }
        },
        "usage": {
          "input_tokens": 226,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 0
      },
      "grade": {
        "correct": false,
        "predicted": 1,
        "score_absolute_error": 1.097,
        "confidence": 0.565,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "display-detail-02",
      "family": "display-detail",
      "type": "score",
      "order": null,
      "elapsed_ms": 481.31587500000023,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.035,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.171,
              "1": 0.658,
              "2": 0.135,
              "3": 0.036
            },
            "confidence": 0.544
          }
        },
        "usage": {
          "input_tokens": 226,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 1
      },
      "grade": {
        "correct": true,
        "predicted": 1,
        "score_absolute_error": 0.03499999999999992,
        "confidence": 0.544,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "display-detail-03",
      "family": "display-detail",
      "type": "score",
      "order": null,
      "elapsed_ms": 499.109000000004,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.741,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.023,
              "1": 0.347,
              "2": 0.498,
              "3": 0.133
            },
            "confidence": 0.331
          }
        },
        "usage": {
          "input_tokens": 227,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 2
      },
      "grade": {
        "correct": true,
        "predicted": 2,
        "score_absolute_error": 0.2589999999999999,
        "confidence": 0.331,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "display-detail-04",
      "family": "display-detail",
      "type": "score",
      "order": null,
      "elapsed_ms": 465.78741699999955,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 2.08,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.029,
              "1": 0.246,
              "2": 0.342,
              "3": 0.383
            },
            "confidence": 0.178
          }
        },
        "usage": {
          "input_tokens": 227,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 3
      },
      "grade": {
        "correct": true,
        "predicted": 3,
        "score_absolute_error": 0.9199999999999999,
        "confidence": 0.178,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "display-detail-05",
      "family": "display-detail",
      "type": "score",
      "order": null,
      "elapsed_ms": 474.299665999999,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 0.953,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.331,
              "1": 0.44,
              "2": 0.175,
              "3": 0.054
            },
            "confidence": 0.253
          }
        },
        "usage": {
          "input_tokens": 226,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 0
      },
      "grade": {
        "correct": false,
        "predicted": 1,
        "score_absolute_error": 0.953,
        "confidence": 0.253,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "display-detail-06",
      "family": "display-detail",
      "type": "score",
      "order": null,
      "elapsed_ms": 499.3032919999969,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.175,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.22,
              "1": 0.451,
              "2": 0.263,
              "3": 0.066
            },
            "confidence": 0.268
          }
        },
        "usage": {
          "input_tokens": 226,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 1
      },
      "grade": {
        "correct": true,
        "predicted": 1,
        "score_absolute_error": 0.17500000000000004,
        "confidence": 0.268,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "display-detail-07",
      "family": "display-detail",
      "type": "score",
      "order": null,
      "elapsed_ms": 495.13620799999626,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.246,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.226,
              "1": 0.404,
              "2": 0.268,
              "3": 0.102
            },
            "confidence": 0.205
          }
        },
        "usage": {
          "input_tokens": 227,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 2
      },
      "grade": {
        "correct": false,
        "predicted": 1,
        "score_absolute_error": 0.754,
        "confidence": 0.205,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "display-detail-08",
      "family": "display-detail",
      "type": "score",
      "order": null,
      "elapsed_ms": 519.1827499999999,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 2.765,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.009,
              "1": 0.05,
              "2": 0.107,
              "3": 0.833
            },
            "confidence": 0.778
          }
        },
        "usage": {
          "input_tokens": 232,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 3
      },
      "grade": {
        "correct": true,
        "predicted": 3,
        "score_absolute_error": 0.23499999999999988,
        "confidence": 0.778,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "glossary-strength-01",
      "family": "glossary-strength",
      "type": "score",
      "order": null,
      "elapsed_ms": 509.02704200000153,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 0.98,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.494,
              "1": 0.231,
              "2": 0.078,
              "3": 0.197
            },
            "confidence": 0.325
          }
        },
        "usage": {
          "input_tokens": 224,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 0
      },
      "grade": {
        "correct": true,
        "predicted": 0,
        "score_absolute_error": 0.98,
        "confidence": 0.325,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "glossary-strength-02",
      "family": "glossary-strength",
      "type": "score",
      "order": null,
      "elapsed_ms": 501.56941700000607,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.219,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.387,
              "1": 0.229,
              "2": 0.162,
              "3": 0.222
            },
            "confidence": 0.183
          }
        },
        "usage": {
          "input_tokens": 230,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 1
      },
      "grade": {
        "correct": false,
        "predicted": 0,
        "score_absolute_error": 0.21900000000000008,
        "confidence": 0.183,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "glossary-strength-03",
      "family": "glossary-strength",
      "type": "score",
      "order": null,
      "elapsed_ms": 467.78670799999963,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.845,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.06,
              "1": 0.306,
              "2": 0.362,
              "3": 0.271
            },
            "confidence": 0.15
          }
        },
        "usage": {
          "input_tokens": 224,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 2
      },
      "grade": {
        "correct": true,
        "predicted": 2,
        "score_absolute_error": 0.15500000000000003,
        "confidence": 0.15,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "glossary-strength-04",
      "family": "glossary-strength",
      "type": "score",
      "order": null,
      "elapsed_ms": 495.12075000000186,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 2.852,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.007,
              "1": 0.006,
              "2": 0.113,
              "3": 0.874
            },
            "confidence": 0.831
          }
        },
        "usage": {
          "input_tokens": 233,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 3
      },
      "grade": {
        "correct": true,
        "predicted": 3,
        "score_absolute_error": 0.14800000000000013,
        "confidence": 0.831,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "glossary-strength-05",
      "family": "glossary-strength",
      "type": "score",
      "order": null,
      "elapsed_ms": 504.79612500000803,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 2.786,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.021,
              "1": 0.019,
              "2": 0.111,
              "3": 0.848
            },
            "confidence": 0.797
          }
        },
        "usage": {
          "input_tokens": 235,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 0
      },
      "grade": {
        "correct": false,
        "predicted": 3,
        "score_absolute_error": 2.786,
        "confidence": 0.797,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "glossary-strength-06",
      "family": "glossary-strength",
      "type": "score",
      "order": null,
      "elapsed_ms": 496.9113750000106,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 2.109,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.087,
              "1": 0.182,
              "2": 0.265,
              "3": 0.465
            },
            "confidence": 0.287
          }
        },
        "usage": {
          "input_tokens": 235,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 1
      },
      "grade": {
        "correct": false,
        "predicted": 3,
        "score_absolute_error": 1.109,
        "confidence": 0.287,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "glossary-strength-07",
      "family": "glossary-strength",
      "type": "score",
      "order": null,
      "elapsed_ms": 523.4562500000029,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.359,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.279,
              "1": 0.286,
              "2": 0.232,
              "3": 0.203
            },
            "confidence": 0.049
          }
        },
        "usage": {
          "input_tokens": 234,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 2
      },
      "grade": {
        "correct": false,
        "predicted": 1,
        "score_absolute_error": 0.641,
        "confidence": 0.049,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "glossary-strength-08",
      "family": "glossary-strength",
      "type": "score",
      "order": null,
      "elapsed_ms": 496.4602499999892,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 2.907,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.008,
              "1": 0.005,
              "2": 0.059,
              "3": 0.928
            },
            "confidence": 0.904
          }
        },
        "usage": {
          "input_tokens": 236,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 3
      },
      "grade": {
        "correct": true,
        "predicted": 3,
        "score_absolute_error": 0.09299999999999997,
        "confidence": 0.904,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "image-layout-01",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "landscape",
        "balanced",
        "portrait"
      ],
      "elapsed_ms": 486.10108399999444,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "landscape": 0.763,
              "balanced": 0.23,
              "portrait": 0.007
            },
            "confidence": 0.645
          }
        },
        "usage": {
          "input_tokens": 203,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "grade": {
        "correct": true,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.645,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "image-layout-02",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "portrait",
        "landscape"
      ],
      "elapsed_ms": 456.5048749999987,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.282,
              "portrait": 0.268,
              "landscape": 0.45
            },
            "confidence": 0.175
          }
        },
        "usage": {
          "input_tokens": 203,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "portrait"
      },
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.175,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "image-layout-03",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "landscape",
        "portrait",
        "balanced"
      ],
      "elapsed_ms": 502.8633339999942,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "balanced",
            "probabilities": {
              "landscape": 0.162,
              "portrait": 0.061,
              "balanced": 0.777
            },
            "confidence": 0.665
          }
        },
        "usage": {
          "input_tokens": 203,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "grade": {
        "correct": true,
        "predicted": "balanced",
        "score_absolute_error": null,
        "confidence": 0.665,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "image-layout-04",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "landscape",
        "portrait"
      ],
      "elapsed_ms": 457.2727089999971,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.267,
              "landscape": 0.727,
              "portrait": 0.006
            },
            "confidence": 0.59
          }
        },
        "usage": {
          "input_tokens": 205,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.59,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "image-layout-05",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "portrait",
        "landscape",
        "balanced"
      ],
      "elapsed_ms": 479.9860000000044,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "portrait": 0.065,
              "landscape": 0.861,
              "balanced": 0.074
            },
            "confidence": 0.792
          }
        },
        "usage": {
          "input_tokens": 204,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "grade": {
        "correct": true,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.792,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "image-layout-06",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "landscape",
        "portrait"
      ],
      "elapsed_ms": 439.6912080000038,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.254,
              "landscape": 0.423,
              "portrait": 0.323
            },
            "confidence": 0.135
          }
        },
        "usage": {
          "input_tokens": 204,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "portrait"
      },
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.135,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "image-layout-07",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "portrait",
        "landscape"
      ],
      "elapsed_ms": 470.8621669999993,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "portrait",
            "probabilities": {
              "balanced": 0.281,
              "portrait": 0.637,
              "landscape": 0.082
            },
            "confidence": 0.456
          }
        },
        "usage": {
          "input_tokens": 215,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "grade": {
        "correct": false,
        "predicted": "portrait",
        "score_absolute_error": null,
        "confidence": 0.456,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "image-layout-08",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "portrait",
        "landscape",
        "balanced"
      ],
      "elapsed_ms": 454.3121669999964,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "portrait": 0.154,
              "landscape": 0.462,
              "balanced": 0.384
            },
            "confidence": 0.193
          }
        },
        "usage": {
          "input_tokens": 202,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "grade": {
        "correct": true,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.193,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "notification-delivery-01",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "digest",
        "email"
      ],
      "elapsed_ms": 471.57320900000923,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "in_app": 0.142,
              "digest": 0.118,
              "email": 0.74
            },
            "confidence": 0.61
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "grade": {
        "correct": true,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.61,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "notification-delivery-02",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "digest",
        "email"
      ],
      "elapsed_ms": 474.0854580000014,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "in_app": 0.344,
              "digest": 0.201,
              "email": 0.455
            },
            "confidence": 0.182
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "in_app"
      },
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.182,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "notification-delivery-03",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "email",
        "digest",
        "in_app"
      ],
      "elapsed_ms": 462.99499999999534,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "digest",
            "probabilities": {
              "email": 0.18,
              "digest": 0.663,
              "in_app": 0.157
            },
            "confidence": 0.494
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "grade": {
        "correct": true,
        "predicted": "digest",
        "score_absolute_error": null,
        "confidence": 0.494,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "notification-delivery-04",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "in_app",
        "email"
      ],
      "elapsed_ms": 550.9112500000047,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "digest": 0.025,
              "in_app": 0.199,
              "email": 0.776
            },
            "confidence": 0.664
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "grade": {
        "correct": true,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.664,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "notification-delivery-05",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "digest",
        "email"
      ],
      "elapsed_ms": 529.0924170000071,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "digest",
            "probabilities": {
              "in_app": 0.29,
              "digest": 0.402,
              "email": 0.308
            },
            "confidence": 0.103
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "in_app"
      },
      "grade": {
        "correct": false,
        "predicted": "digest",
        "score_absolute_error": null,
        "confidence": 0.103,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "notification-delivery-06",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "email",
        "digest",
        "in_app"
      ],
      "elapsed_ms": 557.3595000000059,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "digest",
            "probabilities": {
              "email": 0.189,
              "digest": 0.594,
              "in_app": 0.217
            },
            "confidence": 0.391
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "grade": {
        "correct": true,
        "predicted": "digest",
        "score_absolute_error": null,
        "confidence": 0.391,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "notification-delivery-07",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "in_app",
        "email"
      ],
      "elapsed_ms": 532.7665410000045,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "digest": 0.11,
              "in_app": 0.362,
              "email": 0.528
            },
            "confidence": 0.292
          }
        },
        "usage": {
          "input_tokens": 219,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "grade": {
        "correct": true,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.292,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "notification-delivery-08",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "email",
        "in_app"
      ],
      "elapsed_ms": 507.53245899999456,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "digest": 0.369,
              "email": 0.42,
              "in_app": 0.211
            },
            "confidence": 0.13
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.13,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "release-note-group-01",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "fix",
        "breaking",
        "addition"
      ],
      "elapsed_ms": 522.8424999999988,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "breaking",
            "probabilities": {
              "fix": 0.01,
              "breaking": 0.988,
              "addition": 0.002
            },
            "confidence": 0.983
          }
        },
        "usage": {
          "input_tokens": 225,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "breaking"
      },
      "grade": {
        "correct": true,
        "predicted": "breaking",
        "score_absolute_error": null,
        "confidence": 0.983,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "release-note-group-02",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "breaking",
        "fix"
      ],
      "elapsed_ms": 484.9159590000054,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "addition": 0.002,
              "breaking": 0.023,
              "fix": 0.975
            },
            "confidence": 0.963
          }
        },
        "usage": {
          "input_tokens": 218,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "grade": {
        "correct": true,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.963,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "release-note-group-03",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "breaking",
        "fix"
      ],
      "elapsed_ms": 468.7890000000043,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "addition": 0.992,
              "breaking": 0.001,
              "fix": 0.006
            },
            "confidence": 0.989
          }
        },
        "usage": {
          "input_tokens": 214,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "grade": {
        "correct": true,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.989,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "release-note-group-04",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "fix",
        "breaking",
        "addition"
      ],
      "elapsed_ms": 482.56149999999616,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "breaking",
            "probabilities": {
              "fix": 0.055,
              "breaking": 0.942,
              "addition": 0.002
            },
            "confidence": 0.914
          }
        },
        "usage": {
          "input_tokens": 222,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "breaking"
      },
      "grade": {
        "correct": true,
        "predicted": "breaking",
        "score_absolute_error": null,
        "confidence": 0.914,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "release-note-group-05",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "addition",
        "fix"
      ],
      "elapsed_ms": 451.9236669999955,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "breaking": 0,
              "addition": 0.003,
              "fix": 0.997
            },
            "confidence": 0.995
          }
        },
        "usage": {
          "input_tokens": 220,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "grade": {
        "correct": true,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.995,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "release-note-group-06",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "fix",
        "breaking"
      ],
      "elapsed_ms": 498.0384999999951,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "addition": 0.937,
              "fix": 0.012,
              "breaking": 0.051
            },
            "confidence": 0.905
          }
        },
        "usage": {
          "input_tokens": 226,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "grade": {
        "correct": true,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.905,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "release-note-group-07",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "fix",
        "addition"
      ],
      "elapsed_ms": 477.3383329999924,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "breaking": 0.083,
              "fix": 0.184,
              "addition": 0.733
            },
            "confidence": 0.6
          }
        },
        "usage": {
          "input_tokens": 220,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "grade": {
        "correct": false,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.6,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "release-note-group-08",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "fix",
        "addition"
      ],
      "elapsed_ms": 512.750583999994,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "breaking": 0.005,
              "fix": 0,
              "addition": 0.995
            },
            "confidence": 0.992
          }
        },
        "usage": {
          "input_tokens": 216,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "grade": {
        "correct": true,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.992,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "demo-discount-01",
      "family": "demo-discount",
      "type": "noul",
      "order": null,
      "elapsed_ms": 475.32125000000815,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.986
          }
        },
        "usage": {
          "input_tokens": 201,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": true
      },
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.986,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "demo-discount-02",
      "family": "demo-discount",
      "type": "noul",
      "order": null,
      "elapsed_ms": 499.0575830000016,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.168
          }
        },
        "usage": {
          "input_tokens": 201,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": false
      },
      "grade": {
        "correct": true,
        "predicted": false,
        "score_absolute_error": null,
        "confidence": 0.832,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "demo-discount-03",
      "family": "demo-discount",
      "type": "noul",
      "order": null,
      "elapsed_ms": 454.11387499999546,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.992
          }
        },
        "usage": {
          "input_tokens": 201,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": true
      },
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.992,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "demo-discount-04",
      "family": "demo-discount",
      "type": "noul",
      "order": null,
      "elapsed_ms": 461.2483329999959,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.952
          }
        },
        "usage": {
          "input_tokens": 201,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": false
      },
      "grade": {
        "correct": false,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.952,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "demo-discount-05",
      "family": "demo-discount",
      "type": "noul",
      "order": null,
      "elapsed_ms": 456.42637499999546,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.958
          }
        },
        "usage": {
          "input_tokens": 203,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": true
      },
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.958,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "demo-discount-06",
      "family": "demo-discount",
      "type": "noul",
      "order": null,
      "elapsed_ms": 456.8281670000142,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.089
          }
        },
        "usage": {
          "input_tokens": 201,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": false
      },
      "grade": {
        "correct": true,
        "predicted": false,
        "score_absolute_error": null,
        "confidence": 0.911,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "demo-discount-07",
      "family": "demo-discount",
      "type": "noul",
      "order": null,
      "elapsed_ms": 439.4632500000007,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.985
          }
        },
        "usage": {
          "input_tokens": 202,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": true
      },
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.985,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "demo-discount-08",
      "family": "demo-discount",
      "type": "noul",
      "order": null,
      "elapsed_ms": 468.99679200000537,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.066
          }
        },
        "usage": {
          "input_tokens": 202,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": false
      },
      "grade": {
        "correct": true,
        "predicted": false,
        "score_absolute_error": null,
        "confidence": 0.9339999999999999,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "overnight-counter-01",
      "family": "overnight-counter",
      "type": "noul",
      "order": null,
      "elapsed_ms": 440.26966599999287,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.958
          }
        },
        "usage": {
          "input_tokens": 186,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": true
      },
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.958,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "overnight-counter-02",
      "family": "overnight-counter",
      "type": "noul",
      "order": null,
      "elapsed_ms": 437.1674580000108,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.745
          }
        },
        "usage": {
          "input_tokens": 186,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": false
      },
      "grade": {
        "correct": false,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.745,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "overnight-counter-03",
      "family": "overnight-counter",
      "type": "noul",
      "order": null,
      "elapsed_ms": 464.7084579999937,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.956
          }
        },
        "usage": {
          "input_tokens": 186,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": true
      },
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.956,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "overnight-counter-04",
      "family": "overnight-counter",
      "type": "noul",
      "order": null,
      "elapsed_ms": 450.278999999995,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.528
          }
        },
        "usage": {
          "input_tokens": 186,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": false
      },
      "grade": {
        "correct": false,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.528,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "overnight-counter-05",
      "family": "overnight-counter",
      "type": "noul",
      "order": null,
      "elapsed_ms": 429.1721249999973,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.809
          }
        },
        "usage": {
          "input_tokens": 186,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": true
      },
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.809,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "overnight-counter-06",
      "family": "overnight-counter",
      "type": "noul",
      "order": null,
      "elapsed_ms": 434.8951669999951,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.263
          }
        },
        "usage": {
          "input_tokens": 186,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": false
      },
      "grade": {
        "correct": true,
        "predicted": false,
        "score_absolute_error": null,
        "confidence": 0.737,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "overnight-counter-07",
      "family": "overnight-counter",
      "type": "noul",
      "order": null,
      "elapsed_ms": 429.1547920000012,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.82
          }
        },
        "usage": {
          "input_tokens": 186,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": true
      },
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.82,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "overnight-counter-08",
      "family": "overnight-counter",
      "type": "noul",
      "order": null,
      "elapsed_ms": 433.8086670000048,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.721
          }
        },
        "usage": {
          "input_tokens": 186,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": false
      },
      "grade": {
        "correct": false,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.721,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "tag-filter-01",
      "family": "tag-filter",
      "type": "noul",
      "order": null,
      "elapsed_ms": 435.0436250000057,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.881
          }
        },
        "usage": {
          "input_tokens": 180,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": true
      },
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.881,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "tag-filter-02",
      "family": "tag-filter",
      "type": "noul",
      "order": null,
      "elapsed_ms": 432.7168329999986,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.863
          }
        },
        "usage": {
          "input_tokens": 177,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": false
      },
      "grade": {
        "correct": false,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.863,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "tag-filter-03",
      "family": "tag-filter",
      "type": "noul",
      "order": null,
      "elapsed_ms": 443.83462500000314,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.95
          }
        },
        "usage": {
          "input_tokens": 181,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": true
      },
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.95,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "tag-filter-04",
      "family": "tag-filter",
      "type": "noul",
      "order": null,
      "elapsed_ms": 437.77779100000043,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.644
          }
        },
        "usage": {
          "input_tokens": 180,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": false
      },
      "grade": {
        "correct": false,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.644,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "tag-filter-05",
      "family": "tag-filter",
      "type": "noul",
      "order": null,
      "elapsed_ms": 441.734832999995,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.985
          }
        },
        "usage": {
          "input_tokens": 174,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": true
      },
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.985,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "tag-filter-06",
      "family": "tag-filter",
      "type": "noul",
      "order": null,
      "elapsed_ms": 458.52995799999917,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.876
          }
        },
        "usage": {
          "input_tokens": 179,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": false
      },
      "grade": {
        "correct": false,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.876,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "tag-filter-07",
      "family": "tag-filter",
      "type": "noul",
      "order": null,
      "elapsed_ms": 441.32624999999825,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.928
          }
        },
        "usage": {
          "input_tokens": 180,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": true
      },
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.928,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "tag-filter-08",
      "family": "tag-filter",
      "type": "noul",
      "order": null,
      "elapsed_ms": 447.5437499999971,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.845
          }
        },
        "usage": {
          "input_tokens": 188,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "noul",
        "noul": false
      },
      "grade": {
        "correct": false,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.845,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "mini-game-points-01",
      "family": "mini-game-points",
      "type": "score",
      "order": null,
      "elapsed_ms": 504.8511670000007,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 0.245,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.793,
              "1": 0.183,
              "2": 0.01,
              "3": 0.014
            },
            "confidence": 0.724
          }
        },
        "usage": {
          "input_tokens": 199,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 0
      },
      "grade": {
        "correct": true,
        "predicted": 0,
        "score_absolute_error": 0.245,
        "confidence": 0.724,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "mini-game-points-02",
      "family": "mini-game-points",
      "type": "score",
      "order": null,
      "elapsed_ms": 485.8182910000032,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.025,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.199,
              "1": 0.628,
              "2": 0.123,
              "3": 0.05
            },
            "confidence": 0.504
          }
        },
        "usage": {
          "input_tokens": 199,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 1
      },
      "grade": {
        "correct": true,
        "predicted": 1,
        "score_absolute_error": 0.02499999999999991,
        "confidence": 0.504,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "mini-game-points-03",
      "family": "mini-game-points",
      "type": "score",
      "order": null,
      "elapsed_ms": 491.31162499998754,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 0.875,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.32,
              "1": 0.53,
              "2": 0.104,
              "3": 0.045
            },
            "confidence": 0.374
          }
        },
        "usage": {
          "input_tokens": 199,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 2
      },
      "grade": {
        "correct": false,
        "predicted": 1,
        "score_absolute_error": 1.125,
        "confidence": 0.374,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "mini-game-points-04",
      "family": "mini-game-points",
      "type": "score",
      "order": null,
      "elapsed_ms": 509.0800840000011,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.135,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.183,
              "1": 0.571,
              "2": 0.173,
              "3": 0.072
            },
            "confidence": 0.428
          }
        },
        "usage": {
          "input_tokens": 199,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 3
      },
      "grade": {
        "correct": false,
        "predicted": 1,
        "score_absolute_error": 1.865,
        "confidence": 0.428,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "mini-game-points-05",
      "family": "mini-game-points",
      "type": "score",
      "order": null,
      "elapsed_ms": 535.2255839999998,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.012,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.309,
              "1": 0.45,
              "2": 0.161,
              "3": 0.08
            },
            "confidence": 0.266
          }
        },
        "usage": {
          "input_tokens": 199,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 0
      },
      "grade": {
        "correct": false,
        "predicted": 1,
        "score_absolute_error": 1.012,
        "confidence": 0.266,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "mini-game-points-06",
      "family": "mini-game-points",
      "type": "score",
      "order": null,
      "elapsed_ms": 537.2035000000033,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.459,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.159,
              "1": 0.411,
              "2": 0.243,
              "3": 0.187
            },
            "confidence": 0.214
          }
        },
        "usage": {
          "input_tokens": 199,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 1
      },
      "grade": {
        "correct": true,
        "predicted": 1,
        "score_absolute_error": 0.4590000000000001,
        "confidence": 0.214,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "mini-game-points-07",
      "family": "mini-game-points",
      "type": "score",
      "order": null,
      "elapsed_ms": 492.8136249999952,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.227,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.308,
              "1": 0.357,
              "2": 0.135,
              "3": 0.2
            },
            "confidence": 0.143
          }
        },
        "usage": {
          "input_tokens": 207,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 2
      },
      "grade": {
        "correct": false,
        "predicted": 1,
        "score_absolute_error": 0.7729999999999999,
        "confidence": 0.143,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "mini-game-points-08",
      "family": "mini-game-points",
      "type": "score",
      "order": null,
      "elapsed_ms": 480.26679200000945,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.613,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.087,
              "1": 0.398,
              "2": 0.329,
              "3": 0.185
            },
            "confidence": 0.198
          }
        },
        "usage": {
          "input_tokens": 199,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 3
      },
      "grade": {
        "correct": false,
        "predicted": 1,
        "score_absolute_error": 1.387,
        "confidence": 0.198,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "display-detail-01",
      "family": "display-detail",
      "type": "score",
      "order": null,
      "elapsed_ms": 510.16441700000723,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.097,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.133,
              "1": 0.674,
              "2": 0.157,
              "3": 0.037
            },
            "confidence": 0.565
          }
        },
        "usage": {
          "input_tokens": 226,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 0
      },
      "grade": {
        "correct": false,
        "predicted": 1,
        "score_absolute_error": 1.097,
        "confidence": 0.565,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "display-detail-02",
      "family": "display-detail",
      "type": "score",
      "order": null,
      "elapsed_ms": 522.4159999999974,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.035,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.171,
              "1": 0.658,
              "2": 0.135,
              "3": 0.036
            },
            "confidence": 0.544
          }
        },
        "usage": {
          "input_tokens": 226,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 1
      },
      "grade": {
        "correct": true,
        "predicted": 1,
        "score_absolute_error": 0.03499999999999992,
        "confidence": 0.544,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "display-detail-03",
      "family": "display-detail",
      "type": "score",
      "order": null,
      "elapsed_ms": 524.6553330000024,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.741,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.023,
              "1": 0.347,
              "2": 0.498,
              "3": 0.133
            },
            "confidence": 0.331
          }
        },
        "usage": {
          "input_tokens": 227,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 2
      },
      "grade": {
        "correct": true,
        "predicted": 2,
        "score_absolute_error": 0.2589999999999999,
        "confidence": 0.331,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "display-detail-04",
      "family": "display-detail",
      "type": "score",
      "order": null,
      "elapsed_ms": 505.47325000001,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 2.08,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.029,
              "1": 0.246,
              "2": 0.342,
              "3": 0.383
            },
            "confidence": 0.178
          }
        },
        "usage": {
          "input_tokens": 227,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 3
      },
      "grade": {
        "correct": true,
        "predicted": 3,
        "score_absolute_error": 0.9199999999999999,
        "confidence": 0.178,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "display-detail-05",
      "family": "display-detail",
      "type": "score",
      "order": null,
      "elapsed_ms": 560.0491670000047,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 0.953,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.331,
              "1": 0.44,
              "2": 0.175,
              "3": 0.054
            },
            "confidence": 0.253
          }
        },
        "usage": {
          "input_tokens": 226,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 0
      },
      "grade": {
        "correct": false,
        "predicted": 1,
        "score_absolute_error": 0.953,
        "confidence": 0.253,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "display-detail-06",
      "family": "display-detail",
      "type": "score",
      "order": null,
      "elapsed_ms": 546.4098749999976,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.175,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.22,
              "1": 0.451,
              "2": 0.263,
              "3": 0.066
            },
            "confidence": 0.268
          }
        },
        "usage": {
          "input_tokens": 226,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 1
      },
      "grade": {
        "correct": true,
        "predicted": 1,
        "score_absolute_error": 0.17500000000000004,
        "confidence": 0.268,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "display-detail-07",
      "family": "display-detail",
      "type": "score",
      "order": null,
      "elapsed_ms": 591.8641669999924,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.246,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.226,
              "1": 0.404,
              "2": 0.268,
              "3": 0.102
            },
            "confidence": 0.205
          }
        },
        "usage": {
          "input_tokens": 227,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 2
      },
      "grade": {
        "correct": false,
        "predicted": 1,
        "score_absolute_error": 0.754,
        "confidence": 0.205,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "display-detail-08",
      "family": "display-detail",
      "type": "score",
      "order": null,
      "elapsed_ms": 585.2541250000068,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 2.765,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.009,
              "1": 0.05,
              "2": 0.107,
              "3": 0.833
            },
            "confidence": 0.778
          }
        },
        "usage": {
          "input_tokens": 232,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 3
      },
      "grade": {
        "correct": true,
        "predicted": 3,
        "score_absolute_error": 0.23499999999999988,
        "confidence": 0.778,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "glossary-strength-01",
      "family": "glossary-strength",
      "type": "score",
      "order": null,
      "elapsed_ms": 558.2430420000019,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 0.98,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.494,
              "1": 0.231,
              "2": 0.078,
              "3": 0.197
            },
            "confidence": 0.325
          }
        },
        "usage": {
          "input_tokens": 224,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 0
      },
      "grade": {
        "correct": true,
        "predicted": 0,
        "score_absolute_error": 0.98,
        "confidence": 0.325,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "glossary-strength-02",
      "family": "glossary-strength",
      "type": "score",
      "order": null,
      "elapsed_ms": 592.4981249999983,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.219,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.387,
              "1": 0.229,
              "2": 0.162,
              "3": 0.222
            },
            "confidence": 0.183
          }
        },
        "usage": {
          "input_tokens": 230,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 1
      },
      "grade": {
        "correct": false,
        "predicted": 0,
        "score_absolute_error": 0.21900000000000008,
        "confidence": 0.183,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "glossary-strength-03",
      "family": "glossary-strength",
      "type": "score",
      "order": null,
      "elapsed_ms": 540.7696250000008,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.845,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.06,
              "1": 0.306,
              "2": 0.362,
              "3": 0.271
            },
            "confidence": 0.15
          }
        },
        "usage": {
          "input_tokens": 224,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 2
      },
      "grade": {
        "correct": true,
        "predicted": 2,
        "score_absolute_error": 0.15500000000000003,
        "confidence": 0.15,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "glossary-strength-04",
      "family": "glossary-strength",
      "type": "score",
      "order": null,
      "elapsed_ms": 559.3737089999922,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 2.852,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.007,
              "1": 0.006,
              "2": 0.113,
              "3": 0.874
            },
            "confidence": 0.831
          }
        },
        "usage": {
          "input_tokens": 233,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 3
      },
      "grade": {
        "correct": true,
        "predicted": 3,
        "score_absolute_error": 0.14800000000000013,
        "confidence": 0.831,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "glossary-strength-05",
      "family": "glossary-strength",
      "type": "score",
      "order": null,
      "elapsed_ms": 536.6177080000052,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 2.786,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.021,
              "1": 0.019,
              "2": 0.111,
              "3": 0.848
            },
            "confidence": 0.797
          }
        },
        "usage": {
          "input_tokens": 235,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 0
      },
      "grade": {
        "correct": false,
        "predicted": 3,
        "score_absolute_error": 2.786,
        "confidence": 0.797,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "glossary-strength-06",
      "family": "glossary-strength",
      "type": "score",
      "order": null,
      "elapsed_ms": 502.7108329999901,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 2.109,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.087,
              "1": 0.182,
              "2": 0.265,
              "3": 0.465
            },
            "confidence": 0.287
          }
        },
        "usage": {
          "input_tokens": 235,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 1
      },
      "grade": {
        "correct": false,
        "predicted": 3,
        "score_absolute_error": 1.109,
        "confidence": 0.287,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "glossary-strength-07",
      "family": "glossary-strength",
      "type": "score",
      "order": null,
      "elapsed_ms": 512.5536670000001,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.359,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.279,
              "1": 0.286,
              "2": 0.232,
              "3": 0.203
            },
            "confidence": 0.049
          }
        },
        "usage": {
          "input_tokens": 234,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 2
      },
      "grade": {
        "correct": false,
        "predicted": 1,
        "score_absolute_error": 0.641,
        "confidence": 0.049,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "glossary-strength-08",
      "family": "glossary-strength",
      "type": "score",
      "order": null,
      "elapsed_ms": 488.9108750000014,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "score",
            "score": 2.907,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.008,
              "1": 0.005,
              "2": 0.059,
              "3": 0.928
            },
            "confidence": 0.904
          }
        },
        "usage": {
          "input_tokens": 236,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "score",
        "level": 3
      },
      "grade": {
        "correct": true,
        "predicted": 3,
        "score_absolute_error": 0.09299999999999997,
        "confidence": 0.904,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 0,
      "case_id": "image-layout-01",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "landscape",
        "balanced",
        "portrait"
      ],
      "elapsed_ms": 471.3029589999933,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "landscape": 0.763,
              "balanced": 0.23,
              "portrait": 0.007
            },
            "confidence": 0.645
          }
        },
        "usage": {
          "input_tokens": 203,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "grade": {
        "correct": true,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.645,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 1,
      "case_id": "image-layout-01",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "landscape",
        "portrait",
        "balanced"
      ],
      "elapsed_ms": 476.5095000000001,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "landscape": 0.645,
              "portrait": 0.042,
              "balanced": 0.313
            },
            "confidence": 0.467
          }
        },
        "usage": {
          "input_tokens": 203,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "grade": {
        "correct": true,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.467,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 2,
      "case_id": "image-layout-01",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "landscape",
        "portrait"
      ],
      "elapsed_ms": 460.47608399999444,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.358,
              "landscape": 0.558,
              "portrait": 0.084
            },
            "confidence": 0.337
          }
        },
        "usage": {
          "input_tokens": 203,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "grade": {
        "correct": true,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.337,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 3,
      "case_id": "image-layout-01",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "portrait",
        "landscape"
      ],
      "elapsed_ms": 462.5859999999957,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.145,
              "portrait": 0.074,
              "landscape": 0.782
            },
            "confidence": 0.672
          }
        },
        "usage": {
          "input_tokens": 203,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "grade": {
        "correct": true,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.672,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 4,
      "case_id": "image-layout-01",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "portrait",
        "landscape",
        "balanced"
      ],
      "elapsed_ms": 458.4372089999961,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "portrait": 0.077,
              "landscape": 0.692,
              "balanced": 0.231
            },
            "confidence": 0.538
          }
        },
        "usage": {
          "input_tokens": 203,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "grade": {
        "correct": true,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.538,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 5,
      "case_id": "image-layout-01",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "portrait",
        "balanced",
        "landscape"
      ],
      "elapsed_ms": 475.18395799999416,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "portrait": 0.109,
              "balanced": 0.239,
              "landscape": 0.653
            },
            "confidence": 0.479
          }
        },
        "usage": {
          "input_tokens": 203,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "grade": {
        "correct": true,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.479,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 0,
      "case_id": "image-layout-02",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "portrait",
        "landscape"
      ],
      "elapsed_ms": 470.91154099999403,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.282,
              "portrait": 0.268,
              "landscape": 0.45
            },
            "confidence": 0.175
          }
        },
        "usage": {
          "input_tokens": 203,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "portrait"
      },
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.175,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 1,
      "case_id": "image-layout-02",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "landscape",
        "portrait"
      ],
      "elapsed_ms": 481.59666700000525,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.346,
              "landscape": 0.438,
              "portrait": 0.217
            },
            "confidence": 0.156
          }
        },
        "usage": {
          "input_tokens": 203,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "portrait"
      },
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.156,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 2,
      "case_id": "image-layout-02",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "portrait",
        "balanced",
        "landscape"
      ],
      "elapsed_ms": 471.55083300000115,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "balanced",
            "probabilities": {
              "portrait": 0.16,
              "balanced": 0.456,
              "landscape": 0.384
            },
            "confidence": 0.184
          }
        },
        "usage": {
          "input_tokens": 203,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "portrait"
      },
      "grade": {
        "correct": false,
        "predicted": "balanced",
        "score_absolute_error": null,
        "confidence": 0.184,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 3,
      "case_id": "image-layout-02",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "portrait",
        "landscape",
        "balanced"
      ],
      "elapsed_ms": 476.75500000000466,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "balanced",
            "probabilities": {
              "portrait": 0.099,
              "landscape": 0.318,
              "balanced": 0.582
            },
            "confidence": 0.374
          }
        },
        "usage": {
          "input_tokens": 203,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "portrait"
      },
      "grade": {
        "correct": false,
        "predicted": "balanced",
        "score_absolute_error": null,
        "confidence": 0.374,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 4,
      "case_id": "image-layout-02",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "landscape",
        "balanced",
        "portrait"
      ],
      "elapsed_ms": 483.4073749999952,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "balanced",
            "probabilities": {
              "landscape": 0.313,
              "balanced": 0.537,
              "portrait": 0.15
            },
            "confidence": 0.305
          }
        },
        "usage": {
          "input_tokens": 203,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "portrait"
      },
      "grade": {
        "correct": false,
        "predicted": "balanced",
        "score_absolute_error": null,
        "confidence": 0.305,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 5,
      "case_id": "image-layout-02",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "landscape",
        "portrait",
        "balanced"
      ],
      "elapsed_ms": 474.1525419999962,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "balanced",
            "probabilities": {
              "landscape": 0.174,
              "portrait": 0.194,
              "balanced": 0.633
            },
            "confidence": 0.449
          }
        },
        "usage": {
          "input_tokens": 203,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "portrait"
      },
      "grade": {
        "correct": false,
        "predicted": "balanced",
        "score_absolute_error": null,
        "confidence": 0.449,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 0,
      "case_id": "image-layout-03",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "landscape",
        "portrait",
        "balanced"
      ],
      "elapsed_ms": 473.40524999999616,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "balanced",
            "probabilities": {
              "landscape": 0.162,
              "portrait": 0.061,
              "balanced": 0.777
            },
            "confidence": 0.665
          }
        },
        "usage": {
          "input_tokens": 203,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "grade": {
        "correct": true,
        "predicted": "balanced",
        "score_absolute_error": null,
        "confidence": 0.665,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 1,
      "case_id": "image-layout-03",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "landscape",
        "balanced",
        "portrait"
      ],
      "elapsed_ms": 474.8767919999955,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "balanced",
            "probabilities": {
              "landscape": 0.449,
              "balanced": 0.505,
              "portrait": 0.047
            },
            "confidence": 0.257
          }
        },
        "usage": {
          "input_tokens": 203,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "grade": {
        "correct": true,
        "predicted": "balanced",
        "score_absolute_error": null,
        "confidence": 0.257,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 2,
      "case_id": "image-layout-03",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "portrait",
        "landscape",
        "balanced"
      ],
      "elapsed_ms": 506.4178329999995,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "balanced",
            "probabilities": {
              "portrait": 0.059,
              "landscape": 0.234,
              "balanced": 0.707
            },
            "confidence": 0.561
          }
        },
        "usage": {
          "input_tokens": 203,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "grade": {
        "correct": true,
        "predicted": "balanced",
        "score_absolute_error": null,
        "confidence": 0.561,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 3,
      "case_id": "image-layout-03",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "portrait",
        "balanced",
        "landscape"
      ],
      "elapsed_ms": 504.8865829999995,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "balanced",
            "probabilities": {
              "portrait": 0.125,
              "balanced": 0.443,
              "landscape": 0.432
            },
            "confidence": 0.164
          }
        },
        "usage": {
          "input_tokens": 203,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "grade": {
        "correct": true,
        "predicted": "balanced",
        "score_absolute_error": null,
        "confidence": 0.164,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 4,
      "case_id": "image-layout-03",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "landscape",
        "portrait"
      ],
      "elapsed_ms": 497.5141250000015,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "balanced",
            "probabilities": {
              "balanced": 0.527,
              "landscape": 0.329,
              "portrait": 0.143
            },
            "confidence": 0.291
          }
        },
        "usage": {
          "input_tokens": 203,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "grade": {
        "correct": true,
        "predicted": "balanced",
        "score_absolute_error": null,
        "confidence": 0.291,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 5,
      "case_id": "image-layout-03",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "portrait",
        "landscape"
      ],
      "elapsed_ms": 491.0011249999952,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.32,
              "portrait": 0.131,
              "landscape": 0.549
            },
            "confidence": 0.324
          }
        },
        "usage": {
          "input_tokens": 203,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.324,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 0,
      "case_id": "image-layout-04",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "landscape",
        "portrait"
      ],
      "elapsed_ms": 517.1764169999951,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.267,
              "landscape": 0.727,
              "portrait": 0.006
            },
            "confidence": 0.59
          }
        },
        "usage": {
          "input_tokens": 205,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.59,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 1,
      "case_id": "image-layout-04",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "portrait",
        "landscape"
      ],
      "elapsed_ms": 479.3715419999935,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.05,
              "portrait": 0.02,
              "landscape": 0.93
            },
            "confidence": 0.895
          }
        },
        "usage": {
          "input_tokens": 205,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.895,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 2,
      "case_id": "image-layout-04",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "landscape",
        "balanced",
        "portrait"
      ],
      "elapsed_ms": 497.0922920000012,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "landscape": 0.906,
              "balanced": 0.092,
              "portrait": 0.002
            },
            "confidence": 0.858
          }
        },
        "usage": {
          "input_tokens": 205,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.858,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 3,
      "case_id": "image-layout-04",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "landscape",
        "portrait",
        "balanced"
      ],
      "elapsed_ms": 492.951166999992,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "landscape": 0.907,
              "portrait": 0.016,
              "balanced": 0.078
            },
            "confidence": 0.86
          }
        },
        "usage": {
          "input_tokens": 205,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.86,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 4,
      "case_id": "image-layout-04",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "portrait",
        "balanced",
        "landscape"
      ],
      "elapsed_ms": 507.3782499999943,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "portrait": 0.049,
              "balanced": 0.1,
              "landscape": 0.851
            },
            "confidence": 0.776
          }
        },
        "usage": {
          "input_tokens": 205,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.776,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 5,
      "case_id": "image-layout-04",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "portrait",
        "landscape",
        "balanced"
      ],
      "elapsed_ms": 524.0148330000084,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "portrait": 0.038,
              "landscape": 0.952,
              "balanced": 0.01
            },
            "confidence": 0.928
          }
        },
        "usage": {
          "input_tokens": 205,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.928,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 0,
      "case_id": "image-layout-05",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "portrait",
        "landscape",
        "balanced"
      ],
      "elapsed_ms": 547.1127919999999,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "portrait": 0.065,
              "landscape": 0.861,
              "balanced": 0.074
            },
            "confidence": 0.792
          }
        },
        "usage": {
          "input_tokens": 204,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "grade": {
        "correct": true,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.792,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 1,
      "case_id": "image-layout-05",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "portrait",
        "balanced",
        "landscape"
      ],
      "elapsed_ms": 533.8298330000107,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "portrait": 0.087,
              "balanced": 0.208,
              "landscape": 0.706
            },
            "confidence": 0.558
          }
        },
        "usage": {
          "input_tokens": 204,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "grade": {
        "correct": true,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.558,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 2,
      "case_id": "image-layout-05",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "landscape",
        "portrait",
        "balanced"
      ],
      "elapsed_ms": 513.8660419999942,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "landscape": 0.728,
              "portrait": 0.039,
              "balanced": 0.234
            },
            "confidence": 0.591
          }
        },
        "usage": {
          "input_tokens": 204,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "grade": {
        "correct": true,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.591,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 3,
      "case_id": "image-layout-05",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "landscape",
        "balanced",
        "portrait"
      ],
      "elapsed_ms": 507.638666999992,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "landscape": 0.763,
              "balanced": 0.229,
              "portrait": 0.007
            },
            "confidence": 0.645
          }
        },
        "usage": {
          "input_tokens": 204,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "grade": {
        "correct": true,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.645,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 4,
      "case_id": "image-layout-05",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "portrait",
        "landscape"
      ],
      "elapsed_ms": 506.135374999998,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.102,
              "portrait": 0.062,
              "landscape": 0.836
            },
            "confidence": 0.754
          }
        },
        "usage": {
          "input_tokens": 204,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "grade": {
        "correct": true,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.754,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 5,
      "case_id": "image-layout-05",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "landscape",
        "portrait"
      ],
      "elapsed_ms": 579.5654579999973,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.313,
              "landscape": 0.638,
              "portrait": 0.05
            },
            "confidence": 0.457
          }
        },
        "usage": {
          "input_tokens": 204,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "grade": {
        "correct": true,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.457,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 0,
      "case_id": "image-layout-06",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "landscape",
        "portrait"
      ],
      "elapsed_ms": 631.0561250000028,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.254,
              "landscape": 0.423,
              "portrait": 0.323
            },
            "confidence": 0.135
          }
        },
        "usage": {
          "input_tokens": 204,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "portrait"
      },
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.135,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 1,
      "case_id": "image-layout-06",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "portrait",
        "landscape"
      ],
      "elapsed_ms": 604.8410830000066,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "portrait",
            "probabilities": {
              "balanced": 0.269,
              "portrait": 0.46,
              "landscape": 0.271
            },
            "confidence": 0.189
          }
        },
        "usage": {
          "input_tokens": 204,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "portrait"
      },
      "grade": {
        "correct": true,
        "predicted": "portrait",
        "score_absolute_error": null,
        "confidence": 0.189,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 2,
      "case_id": "image-layout-06",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "landscape",
        "balanced",
        "portrait"
      ],
      "elapsed_ms": 612.6558329999971,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "balanced",
            "probabilities": {
              "landscape": 0.225,
              "balanced": 0.477,
              "portrait": 0.298
            },
            "confidence": 0.215
          }
        },
        "usage": {
          "input_tokens": 204,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "portrait"
      },
      "grade": {
        "correct": false,
        "predicted": "balanced",
        "score_absolute_error": null,
        "confidence": 0.215,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 3,
      "case_id": "image-layout-06",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "landscape",
        "portrait",
        "balanced"
      ],
      "elapsed_ms": 551.1040409999987,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "portrait",
            "probabilities": {
              "landscape": 0.067,
              "portrait": 0.808,
              "balanced": 0.125
            },
            "confidence": 0.712
          }
        },
        "usage": {
          "input_tokens": 204,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "portrait"
      },
      "grade": {
        "correct": true,
        "predicted": "portrait",
        "score_absolute_error": null,
        "confidence": 0.712,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 4,
      "case_id": "image-layout-06",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "portrait",
        "balanced",
        "landscape"
      ],
      "elapsed_ms": 507.35058300000674,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "balanced",
            "probabilities": {
              "portrait": 0.27,
              "balanced": 0.479,
              "landscape": 0.251
            },
            "confidence": 0.218
          }
        },
        "usage": {
          "input_tokens": 204,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "portrait"
      },
      "grade": {
        "correct": false,
        "predicted": "balanced",
        "score_absolute_error": null,
        "confidence": 0.218,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 5,
      "case_id": "image-layout-06",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "portrait",
        "landscape",
        "balanced"
      ],
      "elapsed_ms": 515.3884589999943,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "portrait": 0.23,
              "landscape": 0.386,
              "balanced": 0.384
            },
            "confidence": 0.079
          }
        },
        "usage": {
          "input_tokens": 204,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "portrait"
      },
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.079,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 0,
      "case_id": "image-layout-07",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "portrait",
        "landscape"
      ],
      "elapsed_ms": 505.61599999999453,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "portrait",
            "probabilities": {
              "balanced": 0.281,
              "portrait": 0.637,
              "landscape": 0.082
            },
            "confidence": 0.456
          }
        },
        "usage": {
          "input_tokens": 215,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "grade": {
        "correct": false,
        "predicted": "portrait",
        "score_absolute_error": null,
        "confidence": 0.456,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 1,
      "case_id": "image-layout-07",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "landscape",
        "portrait"
      ],
      "elapsed_ms": 581.677291999993,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "portrait",
            "probabilities": {
              "balanced": 0.155,
              "landscape": 0.292,
              "portrait": 0.553
            },
            "confidence": 0.33
          }
        },
        "usage": {
          "input_tokens": 215,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "grade": {
        "correct": false,
        "predicted": "portrait",
        "score_absolute_error": null,
        "confidence": 0.33,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 2,
      "case_id": "image-layout-07",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "portrait",
        "balanced",
        "landscape"
      ],
      "elapsed_ms": 627.741750000001,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "portrait",
            "probabilities": {
              "portrait": 0.548,
              "balanced": 0.308,
              "landscape": 0.144
            },
            "confidence": 0.322
          }
        },
        "usage": {
          "input_tokens": 215,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "grade": {
        "correct": false,
        "predicted": "portrait",
        "score_absolute_error": null,
        "confidence": 0.322,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 3,
      "case_id": "image-layout-07",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "portrait",
        "landscape",
        "balanced"
      ],
      "elapsed_ms": 593.2969169999997,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "portrait",
            "probabilities": {
              "portrait": 0.498,
              "landscape": 0.167,
              "balanced": 0.335
            },
            "confidence": 0.247
          }
        },
        "usage": {
          "input_tokens": 215,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "grade": {
        "correct": false,
        "predicted": "portrait",
        "score_absolute_error": null,
        "confidence": 0.247,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 4,
      "case_id": "image-layout-07",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "landscape",
        "balanced",
        "portrait"
      ],
      "elapsed_ms": 558.0877909999981,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "portrait",
            "probabilities": {
              "landscape": 0.179,
              "balanced": 0.397,
              "portrait": 0.424
            },
            "confidence": 0.136
          }
        },
        "usage": {
          "input_tokens": 215,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "grade": {
        "correct": false,
        "predicted": "portrait",
        "score_absolute_error": null,
        "confidence": 0.136,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 5,
      "case_id": "image-layout-07",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "landscape",
        "portrait",
        "balanced"
      ],
      "elapsed_ms": 571.0437910000037,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "portrait",
            "probabilities": {
              "landscape": 0.138,
              "portrait": 0.701,
              "balanced": 0.16
            },
            "confidence": 0.552
          }
        },
        "usage": {
          "input_tokens": 215,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "grade": {
        "correct": false,
        "predicted": "portrait",
        "score_absolute_error": null,
        "confidence": 0.552,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 0,
      "case_id": "image-layout-08",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "portrait",
        "landscape",
        "balanced"
      ],
      "elapsed_ms": 540.1922919999924,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "portrait": 0.154,
              "landscape": 0.462,
              "balanced": 0.384
            },
            "confidence": 0.193
          }
        },
        "usage": {
          "input_tokens": 202,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "grade": {
        "correct": true,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.193,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 1,
      "case_id": "image-layout-08",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "portrait",
        "balanced",
        "landscape"
      ],
      "elapsed_ms": 533.8944159999955,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "portrait": 0.313,
              "balanced": 0.314,
              "landscape": 0.372
            },
            "confidence": 0.059
          }
        },
        "usage": {
          "input_tokens": 202,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "grade": {
        "correct": true,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.059,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 2,
      "case_id": "image-layout-08",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "landscape",
        "portrait",
        "balanced"
      ],
      "elapsed_ms": 576.4170409999933,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "balanced",
            "probabilities": {
              "landscape": 0.265,
              "portrait": 0.328,
              "balanced": 0.406
            },
            "confidence": 0.11
          }
        },
        "usage": {
          "input_tokens": 202,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "grade": {
        "correct": false,
        "predicted": "balanced",
        "score_absolute_error": null,
        "confidence": 0.11,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 3,
      "case_id": "image-layout-08",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "landscape",
        "balanced",
        "portrait"
      ],
      "elapsed_ms": 568.2704999999987,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "balanced",
            "probabilities": {
              "landscape": 0.331,
              "balanced": 0.405,
              "portrait": 0.264
            },
            "confidence": 0.107
          }
        },
        "usage": {
          "input_tokens": 202,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "grade": {
        "correct": false,
        "predicted": "balanced",
        "score_absolute_error": null,
        "confidence": 0.107,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 4,
      "case_id": "image-layout-08",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "portrait",
        "landscape"
      ],
      "elapsed_ms": 527.0979579999985,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.199,
              "portrait": 0.35,
              "landscape": 0.45
            },
            "confidence": 0.176
          }
        },
        "usage": {
          "input_tokens": 202,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "grade": {
        "correct": true,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.176,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 5,
      "case_id": "image-layout-08",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "landscape",
        "portrait"
      ],
      "elapsed_ms": 562.2099589999998,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.25,
              "landscape": 0.497,
              "portrait": 0.252
            },
            "confidence": 0.246
          }
        },
        "usage": {
          "input_tokens": 202,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "grade": {
        "correct": true,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.246,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 0,
      "case_id": "notification-delivery-01",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "digest",
        "email"
      ],
      "elapsed_ms": 575.8713330000028,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "in_app": 0.142,
              "digest": 0.118,
              "email": 0.74
            },
            "confidence": 0.61
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "grade": {
        "correct": true,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.61,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 1,
      "case_id": "notification-delivery-01",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "email",
        "digest"
      ],
      "elapsed_ms": 567.2190410000039,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "in_app": 0.237,
              "email": 0.653,
              "digest": 0.11
            },
            "confidence": 0.48
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "grade": {
        "correct": true,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.48,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 2,
      "case_id": "notification-delivery-01",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "in_app",
        "email"
      ],
      "elapsed_ms": 573.4376249999914,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "digest": 0.011,
              "in_app": 0.172,
              "email": 0.817
            },
            "confidence": 0.726
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "grade": {
        "correct": true,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.726,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 3,
      "case_id": "notification-delivery-01",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "email",
        "in_app"
      ],
      "elapsed_ms": 552.80258399999,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "digest": 0.096,
              "email": 0.674,
              "in_app": 0.231
            },
            "confidence": 0.511
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "grade": {
        "correct": true,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.511,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 4,
      "case_id": "notification-delivery-01",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "email",
        "in_app",
        "digest"
      ],
      "elapsed_ms": 574.3421669999952,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "email": 0.63,
              "in_app": 0.2,
              "digest": 0.17
            },
            "confidence": 0.445
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "grade": {
        "correct": true,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.445,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 5,
      "case_id": "notification-delivery-01",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "email",
        "digest",
        "in_app"
      ],
      "elapsed_ms": 510.97066600000835,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "email": 0.596,
              "digest": 0.158,
              "in_app": 0.246
            },
            "confidence": 0.394
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "grade": {
        "correct": true,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.394,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 0,
      "case_id": "notification-delivery-02",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "digest",
        "email"
      ],
      "elapsed_ms": 528.1392920000071,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "in_app": 0.344,
              "digest": 0.201,
              "email": 0.455
            },
            "confidence": 0.182
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "in_app"
      },
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.182,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 1,
      "case_id": "notification-delivery-02",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "email",
        "digest"
      ],
      "elapsed_ms": 542.0481660000223,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "digest",
            "probabilities": {
              "in_app": 0.386,
              "email": 0.174,
              "digest": 0.441
            },
            "confidence": 0.161
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "in_app"
      },
      "grade": {
        "correct": false,
        "predicted": "digest",
        "score_absolute_error": null,
        "confidence": 0.161,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 2,
      "case_id": "notification-delivery-02",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "in_app",
        "email"
      ],
      "elapsed_ms": 601.5014580000134,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "digest": 0.05,
              "in_app": 0.352,
              "email": 0.598
            },
            "confidence": 0.398
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "in_app"
      },
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.398,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 3,
      "case_id": "notification-delivery-02",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "email",
        "in_app"
      ],
      "elapsed_ms": 557.2050410000084,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "digest": 0.107,
              "email": 0.455,
              "in_app": 0.437
            },
            "confidence": 0.183
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "in_app"
      },
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.183,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 4,
      "case_id": "notification-delivery-02",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "email",
        "in_app",
        "digest"
      ],
      "elapsed_ms": 559.1116249999905,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "email": 0.389,
              "in_app": 0.259,
              "digest": 0.352
            },
            "confidence": 0.083
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "in_app"
      },
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.083,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 5,
      "case_id": "notification-delivery-02",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "email",
        "digest",
        "in_app"
      ],
      "elapsed_ms": 527.5939170000202,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "in_app",
            "probabilities": {
              "email": 0.341,
              "digest": 0.222,
              "in_app": 0.437
            },
            "confidence": 0.156
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "in_app"
      },
      "grade": {
        "correct": true,
        "predicted": "in_app",
        "score_absolute_error": null,
        "confidence": 0.156,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 0,
      "case_id": "notification-delivery-03",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "email",
        "digest",
        "in_app"
      ],
      "elapsed_ms": 569.9625839999935,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "digest",
            "probabilities": {
              "email": 0.18,
              "digest": 0.663,
              "in_app": 0.157
            },
            "confidence": 0.494
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "grade": {
        "correct": true,
        "predicted": "digest",
        "score_absolute_error": null,
        "confidence": 0.494,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 1,
      "case_id": "notification-delivery-03",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "email",
        "in_app",
        "digest"
      ],
      "elapsed_ms": 554.4603749999951,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "digest",
            "probabilities": {
              "email": 0.056,
              "in_app": 0.006,
              "digest": 0.938
            },
            "confidence": 0.906
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "grade": {
        "correct": true,
        "predicted": "digest",
        "score_absolute_error": null,
        "confidence": 0.906,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 2,
      "case_id": "notification-delivery-03",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "email",
        "in_app"
      ],
      "elapsed_ms": 566.4689159999834,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "digest": 0.281,
              "email": 0.455,
              "in_app": 0.264
            },
            "confidence": 0.182
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.182,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 3,
      "case_id": "notification-delivery-03",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "in_app",
        "email"
      ],
      "elapsed_ms": 545.4092919999966,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "digest": 0.074,
              "in_app": 0.306,
              "email": 0.621
            },
            "confidence": 0.431
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.431,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 4,
      "case_id": "notification-delivery-03",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "email",
        "digest"
      ],
      "elapsed_ms": 610.758542000025,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "digest",
            "probabilities": {
              "in_app": 0.03,
              "email": 0.003,
              "digest": 0.967
            },
            "confidence": 0.95
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "grade": {
        "correct": true,
        "predicted": "digest",
        "score_absolute_error": null,
        "confidence": 0.95,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 5,
      "case_id": "notification-delivery-03",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "digest",
        "email"
      ],
      "elapsed_ms": 559.9693330000155,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "digest",
            "probabilities": {
              "in_app": 0.199,
              "digest": 0.769,
              "email": 0.032
            },
            "confidence": 0.654
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "grade": {
        "correct": true,
        "predicted": "digest",
        "score_absolute_error": null,
        "confidence": 0.654,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 0,
      "case_id": "notification-delivery-04",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "in_app",
        "email"
      ],
      "elapsed_ms": 584.2925830000022,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "digest": 0.025,
              "in_app": 0.199,
              "email": 0.776
            },
            "confidence": 0.664
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "grade": {
        "correct": true,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.664,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 1,
      "case_id": "notification-delivery-04",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "email",
        "in_app"
      ],
      "elapsed_ms": 594.1483749999898,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "digest": 0.213,
              "email": 0.635,
              "in_app": 0.152
            },
            "confidence": 0.453
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "grade": {
        "correct": true,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.453,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 2,
      "case_id": "notification-delivery-04",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "digest",
        "email"
      ],
      "elapsed_ms": 572.1424579999875,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "digest",
            "probabilities": {
              "in_app": 0.113,
              "digest": 0.529,
              "email": 0.358
            },
            "confidence": 0.293
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "grade": {
        "correct": false,
        "predicted": "digest",
        "score_absolute_error": null,
        "confidence": 0.293,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 3,
      "case_id": "notification-delivery-04",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "email",
        "digest"
      ],
      "elapsed_ms": 525.2739590000128,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "digest",
            "probabilities": {
              "in_app": 0.086,
              "email": 0.324,
              "digest": 0.591
            },
            "confidence": 0.386
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "grade": {
        "correct": false,
        "predicted": "digest",
        "score_absolute_error": null,
        "confidence": 0.386,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 4,
      "case_id": "notification-delivery-04",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "email",
        "digest",
        "in_app"
      ],
      "elapsed_ms": 514.8980840000149,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "digest",
            "probabilities": {
              "email": 0.379,
              "digest": 0.465,
              "in_app": 0.155
            },
            "confidence": 0.198
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "grade": {
        "correct": false,
        "predicted": "digest",
        "score_absolute_error": null,
        "confidence": 0.198,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 5,
      "case_id": "notification-delivery-04",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "email",
        "in_app",
        "digest"
      ],
      "elapsed_ms": 594.4690830000036,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "digest",
            "probabilities": {
              "email": 0.141,
              "in_app": 0.034,
              "digest": 0.825
            },
            "confidence": 0.738
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "grade": {
        "correct": false,
        "predicted": "digest",
        "score_absolute_error": null,
        "confidence": 0.738,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 0,
      "case_id": "notification-delivery-05",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "digest",
        "email"
      ],
      "elapsed_ms": 629.2717080000148,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "digest",
            "probabilities": {
              "in_app": 0.29,
              "digest": 0.402,
              "email": 0.308
            },
            "confidence": 0.103
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "in_app"
      },
      "grade": {
        "correct": false,
        "predicted": "digest",
        "score_absolute_error": null,
        "confidence": 0.103,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 1,
      "case_id": "notification-delivery-05",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "email",
        "digest"
      ],
      "elapsed_ms": 584.0129579999775,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "digest",
            "probabilities": {
              "in_app": 0.323,
              "email": 0.195,
              "digest": 0.482
            },
            "confidence": 0.222
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "in_app"
      },
      "grade": {
        "correct": false,
        "predicted": "digest",
        "score_absolute_error": null,
        "confidence": 0.222,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 2,
      "case_id": "notification-delivery-05",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "in_app",
        "email"
      ],
      "elapsed_ms": 547.1787079999922,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "digest": 0.064,
              "in_app": 0.436,
              "email": 0.499
            },
            "confidence": 0.249
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "in_app"
      },
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.249,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 3,
      "case_id": "notification-delivery-05",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "email",
        "in_app"
      ],
      "elapsed_ms": 541.2597920000262,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "in_app",
            "probabilities": {
              "digest": 0.095,
              "email": 0.42,
              "in_app": 0.485
            },
            "confidence": 0.227
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "in_app"
      },
      "grade": {
        "correct": true,
        "predicted": "in_app",
        "score_absolute_error": null,
        "confidence": 0.227,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 4,
      "case_id": "notification-delivery-05",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "email",
        "in_app",
        "digest"
      ],
      "elapsed_ms": 552.9211249999935,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "digest",
            "probabilities": {
              "email": 0.281,
              "in_app": 0.346,
              "digest": 0.372
            },
            "confidence": 0.058
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "in_app"
      },
      "grade": {
        "correct": false,
        "predicted": "digest",
        "score_absolute_error": null,
        "confidence": 0.058,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 5,
      "case_id": "notification-delivery-05",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "email",
        "digest",
        "in_app"
      ],
      "elapsed_ms": 562.5492080000113,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "in_app",
            "probabilities": {
              "email": 0.224,
              "digest": 0.301,
              "in_app": 0.475
            },
            "confidence": 0.213
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "in_app"
      },
      "grade": {
        "correct": true,
        "predicted": "in_app",
        "score_absolute_error": null,
        "confidence": 0.213,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 0,
      "case_id": "notification-delivery-06",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "email",
        "digest",
        "in_app"
      ],
      "elapsed_ms": 563.7713329999824,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "digest",
            "probabilities": {
              "email": 0.189,
              "digest": 0.594,
              "in_app": 0.217
            },
            "confidence": 0.391
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "grade": {
        "correct": true,
        "predicted": "digest",
        "score_absolute_error": null,
        "confidence": 0.391,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 1,
      "case_id": "notification-delivery-06",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "email",
        "in_app",
        "digest"
      ],
      "elapsed_ms": 522.3269589999982,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "digest",
            "probabilities": {
              "email": 0.064,
              "in_app": 0.022,
              "digest": 0.915
            },
            "confidence": 0.872
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "grade": {
        "correct": true,
        "predicted": "digest",
        "score_absolute_error": null,
        "confidence": 0.872,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 2,
      "case_id": "notification-delivery-06",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "email",
        "in_app"
      ],
      "elapsed_ms": 628.3148749999818,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "digest": 0.311,
              "email": 0.44,
              "in_app": 0.249
            },
            "confidence": 0.16
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.16,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 3,
      "case_id": "notification-delivery-06",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "in_app",
        "email"
      ],
      "elapsed_ms": 619.2253329999803,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "digest": 0.136,
              "in_app": 0.362,
              "email": 0.502
            },
            "confidence": 0.253
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.253,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 4,
      "case_id": "notification-delivery-06",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "email",
        "digest"
      ],
      "elapsed_ms": 625.7792499999923,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "digest",
            "probabilities": {
              "in_app": 0.059,
              "email": 0.056,
              "digest": 0.885
            },
            "confidence": 0.827
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "grade": {
        "correct": true,
        "predicted": "digest",
        "score_absolute_error": null,
        "confidence": 0.827,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 5,
      "case_id": "notification-delivery-06",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "digest",
        "email"
      ],
      "elapsed_ms": 579.4604999999865,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "digest",
            "probabilities": {
              "in_app": 0.167,
              "digest": 0.702,
              "email": 0.131
            },
            "confidence": 0.553
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "grade": {
        "correct": true,
        "predicted": "digest",
        "score_absolute_error": null,
        "confidence": 0.553,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 0,
      "case_id": "notification-delivery-07",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "in_app",
        "email"
      ],
      "elapsed_ms": 571.5280419999908,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "digest": 0.11,
              "in_app": 0.362,
              "email": 0.528
            },
            "confidence": 0.292
          }
        },
        "usage": {
          "input_tokens": 219,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "grade": {
        "correct": true,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.292,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 1,
      "case_id": "notification-delivery-07",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "email",
        "in_app"
      ],
      "elapsed_ms": 601.1334170000046,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "digest": 0.329,
              "email": 0.503,
              "in_app": 0.168
            },
            "confidence": 0.254
          }
        },
        "usage": {
          "input_tokens": 219,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "grade": {
        "correct": true,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.254,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 2,
      "case_id": "notification-delivery-07",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "digest",
        "email"
      ],
      "elapsed_ms": 608.791333000001,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "digest",
            "probabilities": {
              "in_app": 0.134,
              "digest": 0.475,
              "email": 0.391
            },
            "confidence": 0.213
          }
        },
        "usage": {
          "input_tokens": 219,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "grade": {
        "correct": false,
        "predicted": "digest",
        "score_absolute_error": null,
        "confidence": 0.213,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 3,
      "case_id": "notification-delivery-07",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "email",
        "digest"
      ],
      "elapsed_ms": 807.1163749999832,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "digest",
            "probabilities": {
              "in_app": 0.116,
              "email": 0.235,
              "digest": 0.649
            },
            "confidence": 0.473
          }
        },
        "usage": {
          "input_tokens": 219,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "grade": {
        "correct": false,
        "predicted": "digest",
        "score_absolute_error": null,
        "confidence": 0.473,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 4,
      "case_id": "notification-delivery-07",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "email",
        "digest",
        "in_app"
      ],
      "elapsed_ms": 676.7870829999738,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "digest",
            "probabilities": {
              "email": 0.298,
              "digest": 0.529,
              "in_app": 0.174
            },
            "confidence": 0.293
          }
        },
        "usage": {
          "input_tokens": 219,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "grade": {
        "correct": false,
        "predicted": "digest",
        "score_absolute_error": null,
        "confidence": 0.293,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 5,
      "case_id": "notification-delivery-07",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "email",
        "in_app",
        "digest"
      ],
      "elapsed_ms": 670.617041999998,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "digest",
            "probabilities": {
              "email": 0.164,
              "in_app": 0.06,
              "digest": 0.776
            },
            "confidence": 0.664
          }
        },
        "usage": {
          "input_tokens": 219,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "grade": {
        "correct": false,
        "predicted": "digest",
        "score_absolute_error": null,
        "confidence": 0.664,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 0,
      "case_id": "notification-delivery-08",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "email",
        "in_app"
      ],
      "elapsed_ms": 698.6777090000105,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "digest": 0.369,
              "email": 0.42,
              "in_app": 0.211
            },
            "confidence": 0.13
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.13,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 1,
      "case_id": "notification-delivery-08",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "in_app",
        "email"
      ],
      "elapsed_ms": 699.9406249999884,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "in_app",
            "probabilities": {
              "digest": 0.187,
              "in_app": 0.41,
              "email": 0.402
            },
            "confidence": 0.115
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "grade": {
        "correct": false,
        "predicted": "in_app",
        "score_absolute_error": null,
        "confidence": 0.115,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 2,
      "case_id": "notification-delivery-08",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "email",
        "digest",
        "in_app"
      ],
      "elapsed_ms": 635.7564169999969,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "digest",
            "probabilities": {
              "email": 0.15,
              "digest": 0.701,
              "in_app": 0.148
            },
            "confidence": 0.552
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "grade": {
        "correct": true,
        "predicted": "digest",
        "score_absolute_error": null,
        "confidence": 0.552,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 3,
      "case_id": "notification-delivery-08",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "email",
        "in_app",
        "digest"
      ],
      "elapsed_ms": 620.7121249999909,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "digest",
            "probabilities": {
              "email": 0.045,
              "in_app": 0.021,
              "digest": 0.935
            },
            "confidence": 0.902
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "grade": {
        "correct": true,
        "predicted": "digest",
        "score_absolute_error": null,
        "confidence": 0.902,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 4,
      "case_id": "notification-delivery-08",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "digest",
        "email"
      ],
      "elapsed_ms": 608.2350830000069,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "digest",
            "probabilities": {
              "in_app": 0.118,
              "digest": 0.836,
              "email": 0.046
            },
            "confidence": 0.754
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "grade": {
        "correct": true,
        "predicted": "digest",
        "score_absolute_error": null,
        "confidence": 0.754,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 5,
      "case_id": "notification-delivery-08",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "email",
        "digest"
      ],
      "elapsed_ms": 585.5409590000054,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "digest",
            "probabilities": {
              "in_app": 0.037,
              "email": 0.031,
              "digest": 0.932
            },
            "confidence": 0.898
          }
        },
        "usage": {
          "input_tokens": 206,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "grade": {
        "correct": true,
        "predicted": "digest",
        "score_absolute_error": null,
        "confidence": 0.898,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 0,
      "case_id": "release-note-group-01",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "fix",
        "breaking",
        "addition"
      ],
      "elapsed_ms": 605.080667000002,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "breaking",
            "probabilities": {
              "fix": 0.01,
              "breaking": 0.988,
              "addition": 0.002
            },
            "confidence": 0.983
          }
        },
        "usage": {
          "input_tokens": 225,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "breaking"
      },
      "grade": {
        "correct": true,
        "predicted": "breaking",
        "score_absolute_error": null,
        "confidence": 0.983,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 1,
      "case_id": "release-note-group-01",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "fix",
        "addition",
        "breaking"
      ],
      "elapsed_ms": 626.9162079999805,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "breaking",
            "probabilities": {
              "fix": 0.012,
              "addition": 0.013,
              "breaking": 0.975
            },
            "confidence": 0.963
          }
        },
        "usage": {
          "input_tokens": 225,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "breaking"
      },
      "grade": {
        "correct": true,
        "predicted": "breaking",
        "score_absolute_error": null,
        "confidence": 0.963,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 2,
      "case_id": "release-note-group-01",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "fix",
        "addition"
      ],
      "elapsed_ms": 617.7464589999872,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "breaking",
            "probabilities": {
              "breaking": 0.994,
              "fix": 0.002,
              "addition": 0.004
            },
            "confidence": 0.992
          }
        },
        "usage": {
          "input_tokens": 225,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "breaking"
      },
      "grade": {
        "correct": true,
        "predicted": "breaking",
        "score_absolute_error": null,
        "confidence": 0.992,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 3,
      "case_id": "release-note-group-01",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "addition",
        "fix"
      ],
      "elapsed_ms": 660.8518330000225,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "breaking",
            "probabilities": {
              "breaking": 0.99,
              "addition": 0.01,
              "fix": 0
            },
            "confidence": 0.984
          }
        },
        "usage": {
          "input_tokens": 225,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "breaking"
      },
      "grade": {
        "correct": true,
        "predicted": "breaking",
        "score_absolute_error": null,
        "confidence": 0.984,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 4,
      "case_id": "release-note-group-01",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "fix",
        "breaking"
      ],
      "elapsed_ms": 651.1104160000104,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "breaking",
            "probabilities": {
              "addition": 0.027,
              "fix": 0.003,
              "breaking": 0.97
            },
            "confidence": 0.956
          }
        },
        "usage": {
          "input_tokens": 225,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "breaking"
      },
      "grade": {
        "correct": true,
        "predicted": "breaking",
        "score_absolute_error": null,
        "confidence": 0.956,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 5,
      "case_id": "release-note-group-01",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "breaking",
        "fix"
      ],
      "elapsed_ms": 627.0697500000242,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "breaking",
            "probabilities": {
              "addition": 0.05,
              "breaking": 0.949,
              "fix": 0.001
            },
            "confidence": 0.924
          }
        },
        "usage": {
          "input_tokens": 225,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "breaking"
      },
      "grade": {
        "correct": true,
        "predicted": "breaking",
        "score_absolute_error": null,
        "confidence": 0.924,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 0,
      "case_id": "release-note-group-02",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "breaking",
        "fix"
      ],
      "elapsed_ms": 665.3232089999947,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "addition": 0.002,
              "breaking": 0.023,
              "fix": 0.975
            },
            "confidence": 0.963
          }
        },
        "usage": {
          "input_tokens": 218,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "grade": {
        "correct": true,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.963,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 1,
      "case_id": "release-note-group-02",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "fix",
        "breaking"
      ],
      "elapsed_ms": 748.5907920000027,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "addition": 0.004,
              "fix": 0.995,
              "breaking": 0.001
            },
            "confidence": 0.992
          }
        },
        "usage": {
          "input_tokens": 218,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "grade": {
        "correct": true,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.992,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 2,
      "case_id": "release-note-group-02",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "addition",
        "fix"
      ],
      "elapsed_ms": 931.1280839999963,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "breaking": 0.003,
              "addition": 0.004,
              "fix": 0.993
            },
            "confidence": 0.989
          }
        },
        "usage": {
          "input_tokens": 218,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "grade": {
        "correct": true,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.989,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 3,
      "case_id": "release-note-group-02",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "fix",
        "addition"
      ],
      "elapsed_ms": 794.4542920000094,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "breaking": 0.002,
              "fix": 0.997,
              "addition": 0.001
            },
            "confidence": 0.995
          }
        },
        "usage": {
          "input_tokens": 218,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "grade": {
        "correct": true,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.995,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 4,
      "case_id": "release-note-group-02",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "fix",
        "addition",
        "breaking"
      ],
      "elapsed_ms": 814.3570839999884,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "fix": 0.998,
              "addition": 0.002,
              "breaking": 0
            },
            "confidence": 0.996
          }
        },
        "usage": {
          "input_tokens": 218,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "grade": {
        "correct": true,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.996,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 5,
      "case_id": "release-note-group-02",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "fix",
        "breaking",
        "addition"
      ],
      "elapsed_ms": 828.599749999994,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "fix": 0.998,
              "breaking": 0.002,
              "addition": 0
            },
            "confidence": 0.997
          }
        },
        "usage": {
          "input_tokens": 218,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "grade": {
        "correct": true,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.997,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 0,
      "case_id": "release-note-group-03",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "breaking",
        "fix"
      ],
      "elapsed_ms": 806.8937499999884,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "addition": 0.992,
              "breaking": 0.001,
              "fix": 0.006
            },
            "confidence": 0.989
          }
        },
        "usage": {
          "input_tokens": 214,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "grade": {
        "correct": true,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.989,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 1,
      "case_id": "release-note-group-03",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "fix",
        "breaking"
      ],
      "elapsed_ms": 792.5169999999925,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "addition": 0.973,
              "fix": 0.001,
              "breaking": 0.026
            },
            "confidence": 0.96
          }
        },
        "usage": {
          "input_tokens": 214,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "grade": {
        "correct": true,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.96,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 2,
      "case_id": "release-note-group-03",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "addition",
        "fix"
      ],
      "elapsed_ms": 819.707458000019,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "breaking": 0.009,
              "addition": 0.99,
              "fix": 0.001
            },
            "confidence": 0.985
          }
        },
        "usage": {
          "input_tokens": 214,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "grade": {
        "correct": true,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.985,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 3,
      "case_id": "release-note-group-03",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "fix",
        "addition"
      ],
      "elapsed_ms": 801.0267079999903,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "breaking": 0.002,
              "fix": 0,
              "addition": 0.998
            },
            "confidence": 0.997
          }
        },
        "usage": {
          "input_tokens": 214,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "grade": {
        "correct": true,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.997,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 4,
      "case_id": "release-note-group-03",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "fix",
        "addition",
        "breaking"
      ],
      "elapsed_ms": 808.8830420000013,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "fix": 0.003,
              "addition": 0.995,
              "breaking": 0.002
            },
            "confidence": 0.993
          }
        },
        "usage": {
          "input_tokens": 214,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "grade": {
        "correct": true,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.993,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 5,
      "case_id": "release-note-group-03",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "fix",
        "breaking",
        "addition"
      ],
      "elapsed_ms": 819.9213330000057,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "fix": 0,
              "breaking": 0.001,
              "addition": 0.999
            },
            "confidence": 0.998
          }
        },
        "usage": {
          "input_tokens": 214,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "grade": {
        "correct": true,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.998,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 0,
      "case_id": "release-note-group-04",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "fix",
        "breaking",
        "addition"
      ],
      "elapsed_ms": 906.4212089999928,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "breaking",
            "probabilities": {
              "fix": 0.055,
              "breaking": 0.942,
              "addition": 0.002
            },
            "confidence": 0.914
          }
        },
        "usage": {
          "input_tokens": 222,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "breaking"
      },
      "grade": {
        "correct": true,
        "predicted": "breaking",
        "score_absolute_error": null,
        "confidence": 0.914,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 1,
      "case_id": "release-note-group-04",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "fix",
        "addition",
        "breaking"
      ],
      "elapsed_ms": 852.1987080000108,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "breaking",
            "probabilities": {
              "fix": 0.08,
              "addition": 0.045,
              "breaking": 0.875
            },
            "confidence": 0.813
          }
        },
        "usage": {
          "input_tokens": 222,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "breaking"
      },
      "grade": {
        "correct": true,
        "predicted": "breaking",
        "score_absolute_error": null,
        "confidence": 0.813,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 2,
      "case_id": "release-note-group-04",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "fix",
        "addition"
      ],
      "elapsed_ms": 845.0505410000042,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "breaking",
            "probabilities": {
              "breaking": 0.967,
              "fix": 0.03,
              "addition": 0.003
            },
            "confidence": 0.95
          }
        },
        "usage": {
          "input_tokens": 222,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "breaking"
      },
      "grade": {
        "correct": true,
        "predicted": "breaking",
        "score_absolute_error": null,
        "confidence": 0.95,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 3,
      "case_id": "release-note-group-04",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "addition",
        "fix"
      ],
      "elapsed_ms": 673.7820829999982,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "breaking",
            "probabilities": {
              "breaking": 0.918,
              "addition": 0.043,
              "fix": 0.039
            },
            "confidence": 0.877
          }
        },
        "usage": {
          "input_tokens": 222,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "breaking"
      },
      "grade": {
        "correct": true,
        "predicted": "breaking",
        "score_absolute_error": null,
        "confidence": 0.877,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 4,
      "case_id": "release-note-group-04",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "fix",
        "breaking"
      ],
      "elapsed_ms": 775.5763750000042,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "breaking",
            "probabilities": {
              "addition": 0.076,
              "fix": 0.09,
              "breaking": 0.834
            },
            "confidence": 0.751
          }
        },
        "usage": {
          "input_tokens": 222,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "breaking"
      },
      "grade": {
        "correct": true,
        "predicted": "breaking",
        "score_absolute_error": null,
        "confidence": 0.751,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 5,
      "case_id": "release-note-group-04",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "breaking",
        "fix"
      ],
      "elapsed_ms": 704.8071249999921,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "breaking",
            "probabilities": {
              "addition": 0.066,
              "breaking": 0.887,
              "fix": 0.047
            },
            "confidence": 0.83
          }
        },
        "usage": {
          "input_tokens": 222,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "breaking"
      },
      "grade": {
        "correct": true,
        "predicted": "breaking",
        "score_absolute_error": null,
        "confidence": 0.83,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 0,
      "case_id": "release-note-group-05",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "addition",
        "fix"
      ],
      "elapsed_ms": 772.9963339999958,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "breaking": 0,
              "addition": 0.003,
              "fix": 0.997
            },
            "confidence": 0.995
          }
        },
        "usage": {
          "input_tokens": 220,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "grade": {
        "correct": true,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.995,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 1,
      "case_id": "release-note-group-05",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "fix",
        "addition"
      ],
      "elapsed_ms": 756.2110829999729,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "breaking": 0.001,
              "fix": 0.998,
              "addition": 0
            },
            "confidence": 0.997
          }
        },
        "usage": {
          "input_tokens": 220,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "grade": {
        "correct": true,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.997,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 2,
      "case_id": "release-note-group-05",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "breaking",
        "fix"
      ],
      "elapsed_ms": 834.5875410000153,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "addition": 0.001,
              "breaking": 0.022,
              "fix": 0.978
            },
            "confidence": 0.966
          }
        },
        "usage": {
          "input_tokens": 220,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "grade": {
        "correct": true,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.966,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 3,
      "case_id": "release-note-group-05",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "fix",
        "breaking"
      ],
      "elapsed_ms": 991.9394999999786,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "addition": 0.002,
              "fix": 0.998,
              "breaking": 0
            },
            "confidence": 0.997
          }
        },
        "usage": {
          "input_tokens": 220,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "grade": {
        "correct": true,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.997,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 4,
      "case_id": "release-note-group-05",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "fix",
        "breaking",
        "addition"
      ],
      "elapsed_ms": 818.7170409999962,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "fix": 0.996,
              "breaking": 0.004,
              "addition": 0
            },
            "confidence": 0.994
          }
        },
        "usage": {
          "input_tokens": 220,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "grade": {
        "correct": true,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.994,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 5,
      "case_id": "release-note-group-05",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "fix",
        "addition",
        "breaking"
      ],
      "elapsed_ms": 810.2534589999996,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "fix": 0.997,
              "addition": 0.003,
              "breaking": 0
            },
            "confidence": 0.996
          }
        },
        "usage": {
          "input_tokens": 220,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "grade": {
        "correct": true,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.996,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 0,
      "case_id": "release-note-group-06",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "fix",
        "breaking"
      ],
      "elapsed_ms": 771.3573749999923,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "addition": 0.937,
              "fix": 0.012,
              "breaking": 0.051
            },
            "confidence": 0.905
          }
        },
        "usage": {
          "input_tokens": 226,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "grade": {
        "correct": true,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.905,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 1,
      "case_id": "release-note-group-06",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "breaking",
        "fix"
      ],
      "elapsed_ms": 710.3940000000002,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "addition": 0.932,
              "breaking": 0.011,
              "fix": 0.057
            },
            "confidence": 0.898
          }
        },
        "usage": {
          "input_tokens": 226,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "grade": {
        "correct": true,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.898,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 2,
      "case_id": "release-note-group-06",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "fix",
        "addition",
        "breaking"
      ],
      "elapsed_ms": 682.8541670000122,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "fix": 0.03,
              "addition": 0.962,
              "breaking": 0.007
            },
            "confidence": 0.943
          }
        },
        "usage": {
          "input_tokens": 226,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "grade": {
        "correct": true,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.943,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 3,
      "case_id": "release-note-group-06",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "fix",
        "breaking",
        "addition"
      ],
      "elapsed_ms": 687.2331669999985,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "fix": 0.01,
              "breaking": 0.005,
              "addition": 0.986
            },
            "confidence": 0.978
          }
        },
        "usage": {
          "input_tokens": 226,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "grade": {
        "correct": true,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.978,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 4,
      "case_id": "release-note-group-06",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "addition",
        "fix"
      ],
      "elapsed_ms": 703.5145840000187,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "breaking": 0.031,
              "addition": 0.96,
              "fix": 0.01
            },
            "confidence": 0.94
          }
        },
        "usage": {
          "input_tokens": 226,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "grade": {
        "correct": true,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.94,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 5,
      "case_id": "release-note-group-06",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "fix",
        "addition"
      ],
      "elapsed_ms": 700.8880409999983,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "breaking": 0.013,
              "fix": 0.009,
              "addition": 0.978
            },
            "confidence": 0.967
          }
        },
        "usage": {
          "input_tokens": 226,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "grade": {
        "correct": true,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.967,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 0,
      "case_id": "release-note-group-07",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "fix",
        "addition"
      ],
      "elapsed_ms": 622.400708000001,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "breaking": 0.083,
              "fix": 0.184,
              "addition": 0.733
            },
            "confidence": 0.6
          }
        },
        "usage": {
          "input_tokens": 220,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "grade": {
        "correct": false,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.6,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 1,
      "case_id": "release-note-group-07",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "addition",
        "fix"
      ],
      "elapsed_ms": 616.6751249999797,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "breaking": 0.093,
              "addition": 0.709,
              "fix": 0.199
            },
            "confidence": 0.563
          }
        },
        "usage": {
          "input_tokens": 220,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "grade": {
        "correct": false,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.563,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 2,
      "case_id": "release-note-group-07",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "fix",
        "breaking",
        "addition"
      ],
      "elapsed_ms": 619.1937499999767,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "fix": 0.208,
              "breaking": 0.052,
              "addition": 0.74
            },
            "confidence": 0.61
          }
        },
        "usage": {
          "input_tokens": 220,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "grade": {
        "correct": false,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.61,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 3,
      "case_id": "release-note-group-07",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "fix",
        "addition",
        "breaking"
      ],
      "elapsed_ms": 653.4820830000099,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "fix": 0.261,
              "addition": 0.712,
              "breaking": 0.027
            },
            "confidence": 0.567
          }
        },
        "usage": {
          "input_tokens": 220,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "grade": {
        "correct": false,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.567,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 4,
      "case_id": "release-note-group-07",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "breaking",
        "fix"
      ],
      "elapsed_ms": 589.5197919999773,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "addition": 0.563,
              "breaking": 0.055,
              "fix": 0.381
            },
            "confidence": 0.345
          }
        },
        "usage": {
          "input_tokens": 220,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "grade": {
        "correct": false,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.345,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 5,
      "case_id": "release-note-group-07",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "fix",
        "breaking"
      ],
      "elapsed_ms": 631.3097919999855,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "addition": 0.568,
              "fix": 0.362,
              "breaking": 0.07
            },
            "confidence": 0.352
          }
        },
        "usage": {
          "input_tokens": 220,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "grade": {
        "correct": false,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.352,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 0,
      "case_id": "release-note-group-08",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "fix",
        "addition"
      ],
      "elapsed_ms": 614.2130830000096,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "breaking": 0.005,
              "fix": 0,
              "addition": 0.995
            },
            "confidence": 0.992
          }
        },
        "usage": {
          "input_tokens": 216,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "grade": {
        "correct": true,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.992,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 1,
      "case_id": "release-note-group-08",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "addition",
        "fix"
      ],
      "elapsed_ms": 597.9444579999836,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "breaking": 0.024,
              "addition": 0.975,
              "fix": 0.001
            },
            "confidence": 0.962
          }
        },
        "usage": {
          "input_tokens": 216,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "grade": {
        "correct": true,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.962,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 2,
      "case_id": "release-note-group-08",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "fix",
        "breaking",
        "addition"
      ],
      "elapsed_ms": 623.8314170000085,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "fix": 0.001,
              "breaking": 0.001,
              "addition": 0.998
            },
            "confidence": 0.997
          }
        },
        "usage": {
          "input_tokens": 216,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "grade": {
        "correct": true,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.997,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 3,
      "case_id": "release-note-group-08",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "fix",
        "addition",
        "breaking"
      ],
      "elapsed_ms": 636.8105000000214,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "fix": 0.004,
              "addition": 0.995,
              "breaking": 0.001
            },
            "confidence": 0.993
          }
        },
        "usage": {
          "input_tokens": 216,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "grade": {
        "correct": true,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.993,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 4,
      "case_id": "release-note-group-08",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "breaking",
        "fix"
      ],
      "elapsed_ms": 652.0380000000005,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "addition": 0.981,
              "breaking": 0.002,
              "fix": 0.018
            },
            "confidence": 0.971
          }
        },
        "usage": {
          "input_tokens": 216,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "grade": {
        "correct": true,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.971,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 5,
      "case_id": "release-note-group-08",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "fix",
        "breaking"
      ],
      "elapsed_ms": 664.1136669999978,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "qwen3.5-4b",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "addition": 0.946,
              "fix": 0.002,
              "breaking": 0.052
            },
            "confidence": 0.919
          }
        },
        "usage": {
          "input_tokens": 216,
          "output_tokens": 0
        }
      },
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "grade": {
        "correct": true,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.919,
        "tie": false
      }
    }
  ],
  "summary": {
    "attempted_calls": 362,
    "skipped_calls": 0,
    "first_pass_planned": 72,
    "first_pass_observed": 72,
    "first_pass_skipped": 0,
    "first_pass_correct": 44,
    "by_type": {
      "choice": {
        "planned": 24,
        "observed": 24,
        "skipped": 0,
        "correct": 16,
        "valid": 24,
        "ties": 0
      },
      "noul": {
        "planned": 24,
        "observed": 24,
        "skipped": 0,
        "correct": 16,
        "valid": 24,
        "ties": 0
      },
      "score": {
        "planned": 24,
        "observed": 24,
        "skipped": 0,
        "correct": 12,
        "valid": 24,
        "ties": 0
      }
    },
    "by_family": {
      "image-layout": {
        "observed": 8,
        "correct": 4
      },
      "notification-delivery": {
        "observed": 8,
        "correct": 5
      },
      "release-note-group": {
        "observed": 8,
        "correct": 7
      },
      "demo-discount": {
        "observed": 8,
        "correct": 7
      },
      "overnight-counter": {
        "observed": 8,
        "correct": 5
      },
      "tag-filter": {
        "observed": 8,
        "correct": 4
      },
      "mini-game-points": {
        "observed": 8,
        "correct": 3
      },
      "display-detail": {
        "observed": 8,
        "correct": 5
      },
      "glossary-strength": {
        "observed": 8,
        "correct": 4
      }
    },
    "measured_attempts": 216,
    "measured_valid": 216,
    "measured_failures": 0,
    "measured_correct": 132,
    "p50_ms": 464.7084579999937,
    "p95_ms": 540.7696250000008,
    "p50_valid_ms": 464.7084579999937,
    "p95_valid_ms": 540.7696250000008,
    "measured_elapsed_ms": 101433.692542,
    "valid_decisions_per_second": 2.1294699481689703,
    "first_score_mae": 0.7270833333333333,
    "score_mae_observed": 24,
    "permutations": {
      "attempts": 144,
      "valid": 144,
      "correct": 90,
      "ties": 0,
      "complete_case_groups": 24,
      "invariant_case_groups": 13,
      "by_correct_option_position": [
        {
          "position": 0,
          "observed": 48,
          "valid": 48,
          "correct": 21
        },
        {
          "position": 1,
          "observed": 48,
          "valid": 48,
          "correct": 34
        },
        {
          "position": 2,
          "observed": 48,
          "valid": 48,
          "correct": 35
        }
      ]
    },
    "measured_usage": {
      "known_calls": 216,
      "unknown_calls": 0,
      "input_tokens": 44604,
      "output_tokens": 0
    },
    "all_usage": {
      "known_calls": 362,
      "unknown_calls": 0,
      "input_tokens": 75237,
      "output_tokens": 0
    },
    "estimated_measured_input_cost_usd": null,
    "estimated_all_input_cost_usd": null
  }
}