{
  "version": 1,
  "benchmark": "decisions-v3",
  "status": "complete",
  "started_at": "2026-09-20T16:19:35.724Z",
  "finished_at": "2026-09-20T16:19:43.168Z",
  "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
  "upstream_response_model": "laya-rl-agent",
  "qualification_scope": "Direct pinned Laya MLX upstream candidate; not Sys1 runtime integration or default qualification.",
  "source": {
    "sys1_commit": "476a2a04140344c01d0797c0fc250823d394605b",
    "fixture_sha256": "992d0078faff0f781d87be0755828b48689345e4a8a68d20b25f12b7a0fa87cc",
    "warmup_fixture_sha256": "8e3073ac4ad1fb460fe574084b40631577ece6a49e190fca2d89659cee15604c",
    "grader_sha256": "aedaf7e29b46ac0122f44233415d966eec0998a4b4fcf34bd8f32a8d8f21f515",
    "shared_harness_sha256": "9c363a7d135c9dfe97797eddb54cbb56826913dbcb9919e7bddaa03b0aeed162",
    "response_validator_sha256": "040188479f202689f591cd04ee70b5186c7434a0c13cda27db5168b24b3ade73",
    "protocol_sha256": "e546f90c7d84b83f440eeaf147999b09ac80c59704486aee5917f48789146ef4",
    "laya_commit": "fc1df62828a3fedf4d8229fdac1cbd85f1cdf337",
    "model_repository": "aac6fef/laya-typed-decisions-mlx",
    "model_revision": "28416e78cb26a239a4eabaa2e084904ec5e6cacb",
    "schedule_sha256": "919745861e9f02931e5e079395279449defe9b5baa8649cb02b231749fee4cd9",
    "preflight_sha256": "b3de0522bc5892454b8902726e0775992f025f789347d9df927906c91bb03115",
    "harness_sha256": "d0b1336c62eb1dfcab80cc593c96bd170db4c29ad186e5bf47f84cc53dd69205",
    "worker_sha256": "bde021779a6f6076d79ca963833847392fcea9726a623464f009b3d69f846c8f",
    "exporter_sha256": "bbe7e7868a620f2ea9c664f781d00167c5d86b0af5ce4d06f5904e9311732ecb",
    "requirements_sha256": "f3b58b4fcd66e08daf5c0165dcccd169f93b31eb41042d137bf1f6f721030f52",
    "artifacts": [
      {
        "file": "LICENSE",
        "bytes": 10173,
        "sha256": "a6cba85bc92e0cff7a450b1d873c0eaa2e9fc96bf472df0247a26bec77bf3ff9",
        "git_blob_sha1": "d9a10c0d8e868ebf8da0b3dc95bb0be634c34bfe"
      },
      {
        "file": "NOTICE",
        "bytes": 698,
        "sha256": "f23ae8ae701f5a43e186182e387aeef00b666daed18a946fc7c9ea827b00a8fa",
        "git_blob_sha1": "a41f006a83bff3f324dfbe8b3d9963f9b39063d6"
      },
      {
        "file": "README.md",
        "bytes": 3699,
        "sha256": "3e462f1cd9a711f2d10cf7508b5cf98737eda70c4833fd168c993501420679ef",
        "git_blob_sha1": "1f8767bd3e29593501ab7df82ed90f7b7d963ff8"
      },
      {
        "file": "encoder/config.json",
        "bytes": 2084,
        "sha256": "5268d24ad3b77c8151de5dcb0762ba4391619aad9ab0bda33e36fb083cfeae6d",
        "git_blob_sha1": "d4be4829750fb04c0aa8b9897c3ea827f76c0109"
      },
      {
        "file": "manifest.json",
        "bytes": 1778,
        "sha256": "3cba9de548ccbfd05dde36365ec2ffdafce9e79b68b1b435d41343ef1afd7d70",
        "git_blob_sha1": "363631152f707e5148462ec9a92b11a99051ba9a"
      },
      {
        "file": "mlx_config.json",
        "bytes": 322,
        "sha256": "fe0d126c46e70e28e47767801b15e3219280484ce298cab6aadcd339ca242cae",
        "git_blob_sha1": "47516cb633704ddc43a121566e00b925133c2669"
      },
      {
        "file": "rl_agent_config.json",
        "bytes": 848,
        "sha256": "d99c800b818cb1a60734adba961c27abeaff2d142bb9fc473edb34b7de32fde9",
        "git_blob_sha1": "d0293fddc8b337c9bc651f2a7b95c0848fbbb488"
      },
      {
        "file": "tokenizer/tokenizer.json",
        "bytes": 3583228,
        "sha256": "6c8aaa9a542084f2457eab775d4eeb51f92a70c0fd9de28d5edb0ddec3c08d30",
        "git_blob_sha1": "2f4d8583e507b7466d2490e2d6c045647a822698"
      },
      {
        "file": "tokenizer/tokenizer_config.json",
        "bytes": 337,
        "sha256": "08d4cf3ac4dca381759441b85b91a6d40e688471dcd33d15d6649eb0a9a854d1",
        "git_blob_sha1": "ed1ffabc2ce11120754705709569e365e46da71a"
      },
      {
        "file": "validation.json",
        "bytes": 18275,
        "sha256": "3b3c53564fd89f431252518733b4f70b856c1b911bcdf3a271d8ca1ec7a1b60c",
        "git_blob_sha1": "532e2fe0a7363f3d0ee5eec641e3db39500ac01c"
      },
      {
        "file": "model.safetensors",
        "bytes": 842609225,
        "sha256": "804ef8802b4cac7a67913b0cfb8448659e934a50284aaa867b98d7d9a6e7d1e0",
        "lfs_verified": true
      }
    ]
  },
  "environment": {
    "cpu": "Apple M5 Max",
    "platform": "darwin",
    "arch": "arm64",
    "os": "25.5.0",
    "memory_bytes": 38654705664,
    "bun": "1.3.14",
    "native": {
      "python": "3.12.14",
      "platform": "macOS-26.5.1-arm64-arm-64bit",
      "arch": "arm64",
      "mlx": "0.32.2",
      "numpy": "2.5.3",
      "tokenizers": "0.23.2",
      "device": "gpu",
      "dtype": "float16",
      "batch_size": 1,
      "compile": false,
      "cache_prompts": false,
      "max_len": 1024,
      "head_max_len": 256
    }
  },
  "methodology": {
    "planned_calls": 362,
    "initial_calls": 1,
    "warmups": 1,
    "unique_cases": 72,
    "cases_per_type": 24,
    "measured_repetitions": 3,
    "measured_calls": 216,
    "permutation_calls": 144,
    "concurrency": 1,
    "request_timeout_ms": 60000,
    "run_timeout_ms": 480000,
    "stop_rule": "Stop after three consecutive invalid/error responses; no retries. Stop immediately on request timeout and collect owned process.",
    "latency_boundary": "Python Agent.predict through complete evaluated response; excludes process IPC, Sys1 normalization/validation, model load and HTTP.",
    "throughput_boundary": "Measured loop wall time including process IPC and normalization/validation; excludes model load, report persistence and permutations.",
    "usage": "Upstream retained unpadded sequence tokens summed across rows, including special tokens and repeated state; exact full-input equality preflight excludes truncation. Output zero: classifier returns probabilities without generated answer text.",
    "normalization": "Preserve raw_response including original model, action fields, and noul confidence. Explicit normalized projection uses pinned checkpoint identity, drops action and noul confidence, preserves all probabilities and other SystemOne fields without modification.",
    "grading": "Sys1 frozen decisions-v3 gradeDecision and validateResponseForRequest. Choice exact label; noul threshold0.5 and ties incorrect; score unique probability argmax and ties incorrect; score MAE separately.",
    "quality_scope": "Exploratory direct-upstream evidence on72 authored synthetic cases across9families; frozen before execution. Not a production benchmark, calibration study or independent third-party evaluation. Repeats/permutations are not independent cases.",
    "invariance": "Six valid identical semantic choices across every permutation; does not imply correctness.",
    "confidence": "Laya choice/score entropy confidence; noul maximum probability. Different semantics from Sys1 Qwen readout; not calibrated by this benchmark.",
    "preflight": {
      "checked": 362,
      "max_len": 1024,
      "head_max_len": 256,
      "max_untruncated_tokens": 166,
      "max_option_tokens": 18,
      "all_exact": true
    }
  },
  "samples": [
    {
      "phase": "initial",
      "repetition": 0,
      "case_id": "warmup-form-01",
      "family": "warmup-forms",
      "type": "choice",
      "order": [
        "submit",
        "correct",
        "wait"
      ],
      "expected": {
        "type": "choice",
        "choice": "submit"
      },
      "elapsed_ms": 1814.2347920220345,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "submit",
            "probabilities": {
              "submit": 0.5301,
              "correct": 0.2957,
              "wait": 0.1742
            },
            "confidence": 0.0887
          }
        },
        "usage": {
          "input_tokens": 59,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0887,
            "action": {
              "act_probability": 1
            },
            "choice": "submit",
            "probabilities": {
              "submit": 0.5301,
              "correct": 0.2957,
              "wait": 0.1742
            }
          }
        },
        "usage": {
          "input_tokens": 59,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "submit",
        "score_absolute_error": null,
        "confidence": 0.0887,
        "tie": false
      }
    },
    {
      "phase": "warmup",
      "repetition": 0,
      "case_id": "warmup-form-02",
      "family": "warmup-forms",
      "type": "choice",
      "order": [
        "submit",
        "correct",
        "wait"
      ],
      "expected": {
        "type": "choice",
        "choice": "correct"
      },
      "elapsed_ms": 11.814749974291772,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "submit",
            "probabilities": {
              "submit": 0.4628,
              "correct": 0.3871,
              "wait": 0.1501
            },
            "confidence": 0.0819
          }
        },
        "usage": {
          "input_tokens": 54,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0819,
            "action": {
              "act_probability": 1
            },
            "choice": "submit",
            "probabilities": {
              "submit": 0.4628,
              "correct": 0.3871,
              "wait": 0.1501
            }
          }
        },
        "usage": {
          "input_tokens": 54,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "submit",
        "score_absolute_error": null,
        "confidence": 0.0819,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "image-layout-01",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "landscape",
        "balanced",
        "portrait"
      ],
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "elapsed_ms": 19.605750043410808,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "landscape": 0.3814,
              "balanced": 0.3311,
              "portrait": 0.2875
            },
            "confidence": 0.006
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.006,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "landscape": 0.3814,
              "balanced": 0.3311,
              "portrait": 0.2875
            }
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.006,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "image-layout-02",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "portrait",
        "landscape"
      ],
      "expected": {
        "type": "choice",
        "choice": "portrait"
      },
      "elapsed_ms": 8.78579099662602,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.3047,
              "portrait": 0.2637,
              "landscape": 0.4316
            },
            "confidence": 0.0203
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0203,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.3047,
              "portrait": 0.2637,
              "landscape": 0.4316
            }
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.0203,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "image-layout-03",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "landscape",
        "portrait",
        "balanced"
      ],
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "elapsed_ms": 8.476292015984654,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "portrait",
            "probabilities": {
              "landscape": 0.3427,
              "portrait": 0.3492,
              "balanced": 0.3081
            },
            "confidence": 0.0014
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0014,
            "action": {
              "act_probability": 1
            },
            "choice": "portrait",
            "probabilities": {
              "landscape": 0.3427,
              "portrait": 0.3492,
              "balanced": 0.3081
            }
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "portrait",
        "score_absolute_error": null,
        "confidence": 0.0014,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "image-layout-04",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "landscape",
        "portrait"
      ],
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "elapsed_ms": 8.705792017281055,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.2776,
              "landscape": 0.4783,
              "portrait": 0.2441
            },
            "confidence": 0.0418
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0418,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.2776,
              "landscape": 0.4783,
              "portrait": 0.2441
            }
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.0418,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "image-layout-05",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "portrait",
        "landscape",
        "balanced"
      ],
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "elapsed_ms": 8.340124972164631,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "portrait": 0.2672,
              "landscape": 0.4149,
              "balanced": 0.3179
            },
            "confidence": 0.0152
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0152,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "portrait": 0.2672,
              "landscape": 0.4149,
              "balanced": 0.3179
            }
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.0152,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "image-layout-06",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "landscape",
        "portrait"
      ],
      "expected": {
        "type": "choice",
        "choice": "portrait"
      },
      "elapsed_ms": 9.44654201157391,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.2731,
              "landscape": 0.4137,
              "portrait": 0.3132
            },
            "confidence": 0.014
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.014,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.2731,
              "landscape": 0.4137,
              "portrait": 0.3132
            }
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.014,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "image-layout-07",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "portrait",
        "landscape"
      ],
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "elapsed_ms": 17.61066698236391,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.2631,
              "portrait": 0.3522,
              "landscape": 0.3846
            },
            "confidence": 0.0112
          }
        },
        "usage": {
          "input_tokens": 136,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0112,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.2631,
              "portrait": 0.3522,
              "landscape": 0.3846
            }
          }
        },
        "usage": {
          "input_tokens": 136,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.0112,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "image-layout-08",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "portrait",
        "landscape",
        "balanced"
      ],
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "elapsed_ms": 9.17537504574284,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "portrait",
            "probabilities": {
              "portrait": 0.5268,
              "landscape": 0.2492,
              "balanced": 0.224
            },
            "confidence": 0.0724
          }
        },
        "usage": {
          "input_tokens": 122,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0724,
            "action": {
              "act_probability": 1
            },
            "choice": "portrait",
            "probabilities": {
              "portrait": 0.5268,
              "landscape": 0.2492,
              "balanced": 0.224
            }
          }
        },
        "usage": {
          "input_tokens": 122,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "portrait",
        "score_absolute_error": null,
        "confidence": 0.0724,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "notification-delivery-01",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "digest",
        "email"
      ],
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "elapsed_ms": 13.382500037550926,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "in_app": 0.2397,
              "digest": 0.2275,
              "email": 0.5328
            },
            "confidence": 0.0764
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0764,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "in_app": 0.2397,
              "digest": 0.2275,
              "email": 0.5328
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.0764,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "notification-delivery-02",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "digest",
        "email"
      ],
      "expected": {
        "type": "choice",
        "choice": "in_app"
      },
      "elapsed_ms": 9.29329200880602,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "in_app": 0.2571,
              "digest": 0.2312,
              "email": 0.5117
            },
            "confidence": 0.0619
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0619,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "in_app": 0.2571,
              "digest": 0.2312,
              "email": 0.5117
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.0619,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "notification-delivery-03",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "email",
        "digest",
        "in_app"
      ],
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "elapsed_ms": 8.40804196195677,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "email": 0.4489,
              "digest": 0.3116,
              "in_app": 0.2395
            },
            "confidence": 0.0304
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0304,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "email": 0.4489,
              "digest": 0.3116,
              "in_app": 0.2395
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.0304,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "notification-delivery-04",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "in_app",
        "email"
      ],
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "elapsed_ms": 8.368666982278228,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "digest": 0.1798,
              "in_app": 0.2749,
              "email": 0.5453
            },
            "confidence": 0.0951
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0951,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "digest": 0.1798,
              "in_app": 0.2749,
              "email": 0.5453
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.0951,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "notification-delivery-05",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "digest",
        "email"
      ],
      "expected": {
        "type": "choice",
        "choice": "in_app"
      },
      "elapsed_ms": 8.48129199584946,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "in_app": 0.2955,
              "digest": 0.2817,
              "email": 0.4228
            },
            "confidence": 0.0159
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0159,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "in_app": 0.2955,
              "digest": 0.2817,
              "email": 0.4228
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.0159,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "notification-delivery-06",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "email",
        "digest",
        "in_app"
      ],
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "elapsed_ms": 8.881166984792799,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "email": 0.5312,
              "digest": 0.226,
              "in_app": 0.2428
            },
            "confidence": 0.0753
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0753,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "email": 0.5312,
              "digest": 0.226,
              "in_app": 0.2428
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.0753,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "notification-delivery-07",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "in_app",
        "email"
      ],
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "elapsed_ms": 26.5361659694463,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "digest",
            "probabilities": {
              "digest": 0.4598,
              "in_app": 0.2653,
              "email": 0.2749
            },
            "confidence": 0.0313
          }
        },
        "usage": {
          "input_tokens": 154,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0313,
            "action": {
              "act_probability": 1
            },
            "choice": "digest",
            "probabilities": {
              "digest": 0.4598,
              "in_app": 0.2653,
              "email": 0.2749
            }
          }
        },
        "usage": {
          "input_tokens": 154,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "digest",
        "score_absolute_error": null,
        "confidence": 0.0313,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "notification-delivery-08",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "email",
        "in_app"
      ],
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "elapsed_ms": 9.275625052396208,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "digest": 0.2652,
              "email": 0.3729,
              "in_app": 0.3619
            },
            "confidence": 0.01
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.01,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "digest": 0.2652,
              "email": 0.3729,
              "in_app": 0.3619
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.01,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "release-note-group-01",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "fix",
        "breaking",
        "addition"
      ],
      "expected": {
        "type": "choice",
        "choice": "breaking"
      },
      "elapsed_ms": 15.324584033805877,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "fix": 0.3008,
              "breaking": 0.2342,
              "addition": 0.465
            },
            "confidence": 0.0375
          }
        },
        "usage": {
          "input_tokens": 148,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0375,
            "action": {
              "act_probability": 1
            },
            "choice": "addition",
            "probabilities": {
              "fix": 0.3008,
              "breaking": 0.2342,
              "addition": 0.465
            }
          }
        },
        "usage": {
          "input_tokens": 148,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.0375,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "release-note-group-02",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "breaking",
        "fix"
      ],
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "elapsed_ms": 8.577833010349423,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "addition": 0.128,
              "breaking": 0.1385,
              "fix": 0.7335
            },
            "confidence": 0.3044
          }
        },
        "usage": {
          "input_tokens": 140,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.3044,
            "action": {
              "act_probability": 1
            },
            "choice": "fix",
            "probabilities": {
              "addition": 0.128,
              "breaking": 0.1385,
              "fix": 0.7335
            }
          }
        },
        "usage": {
          "input_tokens": 140,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.3044,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "release-note-group-03",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "breaking",
        "fix"
      ],
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "elapsed_ms": 8.483875019010156,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "addition": 0.4816,
              "breaking": 0.162,
              "fix": 0.3564
            },
            "confidence": 0.0766
          }
        },
        "usage": {
          "input_tokens": 137,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0766,
            "action": {
              "act_probability": 1
            },
            "choice": "addition",
            "probabilities": {
              "addition": 0.4816,
              "breaking": 0.162,
              "fix": 0.3564
            }
          }
        },
        "usage": {
          "input_tokens": 137,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.0766,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "release-note-group-04",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "fix",
        "breaking",
        "addition"
      ],
      "expected": {
        "type": "choice",
        "choice": "breaking"
      },
      "elapsed_ms": 9.40854096552357,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "fix": 0.4896,
              "breaking": 0.2786,
              "addition": 0.2319
            },
            "confidence": 0.0492
          }
        },
        "usage": {
          "input_tokens": 145,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0492,
            "action": {
              "act_probability": 1
            },
            "choice": "fix",
            "probabilities": {
              "fix": 0.4896,
              "breaking": 0.2786,
              "addition": 0.2319
            }
          }
        },
        "usage": {
          "input_tokens": 145,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.0492,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "release-note-group-05",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "addition",
        "fix"
      ],
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "elapsed_ms": 9.669082995969802,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "breaking": 0.1838,
              "addition": 0.1464,
              "fix": 0.6698
            },
            "confidence": 0.2162
          }
        },
        "usage": {
          "input_tokens": 143,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.2162,
            "action": {
              "act_probability": 1
            },
            "choice": "fix",
            "probabilities": {
              "breaking": 0.1838,
              "addition": 0.1464,
              "fix": 0.6698
            }
          }
        },
        "usage": {
          "input_tokens": 143,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.2162,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "release-note-group-06",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "fix",
        "breaking"
      ],
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "elapsed_ms": 8.62966600107029,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "addition": 0.4043,
              "fix": 0.3457,
              "breaking": 0.25
            },
            "confidence": 0.017
          }
        },
        "usage": {
          "input_tokens": 148,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.017,
            "action": {
              "act_probability": 1
            },
            "choice": "addition",
            "probabilities": {
              "addition": 0.4043,
              "fix": 0.3457,
              "breaking": 0.25
            }
          }
        },
        "usage": {
          "input_tokens": 148,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.017,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "release-note-group-07",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "fix",
        "addition"
      ],
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "elapsed_ms": 8.872667036484927,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "breaking": 0.1442,
              "fix": 0.6103,
              "addition": 0.2455
            },
            "confidence": 0.1576
          }
        },
        "usage": {
          "input_tokens": 142,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.1576,
            "action": {
              "act_probability": 1
            },
            "choice": "fix",
            "probabilities": {
              "breaking": 0.1442,
              "fix": 0.6103,
              "addition": 0.2455
            }
          }
        },
        "usage": {
          "input_tokens": 142,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.1576,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "release-note-group-08",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "fix",
        "addition"
      ],
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "elapsed_ms": 8.474083966575563,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "breaking": 0.289,
              "fix": 0.4007,
              "addition": 0.3103
            },
            "confidence": 0.0094
          }
        },
        "usage": {
          "input_tokens": 138,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0094,
            "action": {
              "act_probability": 1
            },
            "choice": "fix",
            "probabilities": {
              "breaking": 0.289,
              "fix": 0.4007,
              "addition": 0.3103
            }
          }
        },
        "usage": {
          "input_tokens": 138,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.0094,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "demo-discount-01",
      "family": "demo-discount",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": true
      },
      "elapsed_ms": 22.4246249999851,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.6902
          }
        },
        "usage": {
          "input_tokens": 146,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.6902,
            "action": {
              "act_probability": 1
            },
            "noul": 0.6902
          }
        },
        "usage": {
          "input_tokens": 146,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.6902,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "demo-discount-02",
      "family": "demo-discount",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": false
      },
      "elapsed_ms": 8.822333009447902,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.2415
          }
        },
        "usage": {
          "input_tokens": 146,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.7585,
            "action": {
              "act_probability": 1
            },
            "noul": 0.2415
          }
        },
        "usage": {
          "input_tokens": 146,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": false,
        "score_absolute_error": null,
        "confidence": 0.7585,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "demo-discount-03",
      "family": "demo-discount",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": true
      },
      "elapsed_ms": 9.41087503451854,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.2825
          }
        },
        "usage": {
          "input_tokens": 146,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.7175,
            "action": {
              "act_probability": 1
            },
            "noul": 0.2825
          }
        },
        "usage": {
          "input_tokens": 146,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": false,
        "score_absolute_error": null,
        "confidence": 0.7175,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "demo-discount-04",
      "family": "demo-discount",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": false
      },
      "elapsed_ms": 9.768166986759752,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.3083
          }
        },
        "usage": {
          "input_tokens": 146,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.6917,
            "action": {
              "act_probability": 1
            },
            "noul": 0.3083
          }
        },
        "usage": {
          "input_tokens": 146,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": false,
        "score_absolute_error": null,
        "confidence": 0.6917,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "demo-discount-05",
      "family": "demo-discount",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": true
      },
      "elapsed_ms": 9.313374990597367,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.5631
          }
        },
        "usage": {
          "input_tokens": 148,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.5631,
            "action": {
              "act_probability": 1
            },
            "noul": 0.5631
          }
        },
        "usage": {
          "input_tokens": 148,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.5631,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "demo-discount-06",
      "family": "demo-discount",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": false
      },
      "elapsed_ms": 8.654417004436255,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.1742
          }
        },
        "usage": {
          "input_tokens": 146,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.8258,
            "action": {
              "act_probability": 1
            },
            "noul": 0.1742
          }
        },
        "usage": {
          "input_tokens": 146,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": false,
        "score_absolute_error": null,
        "confidence": 0.8258,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "demo-discount-07",
      "family": "demo-discount",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": true
      },
      "elapsed_ms": 8.715457981452346,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.2773
          }
        },
        "usage": {
          "input_tokens": 146,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.7227,
            "action": {
              "act_probability": 1
            },
            "noul": 0.2773
          }
        },
        "usage": {
          "input_tokens": 146,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": false,
        "score_absolute_error": null,
        "confidence": 0.7227,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "demo-discount-08",
      "family": "demo-discount",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": false
      },
      "elapsed_ms": 8.992208982817829,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.2383
          }
        },
        "usage": {
          "input_tokens": 146,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.7617,
            "action": {
              "act_probability": 1
            },
            "noul": 0.2383
          }
        },
        "usage": {
          "input_tokens": 146,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": false,
        "score_absolute_error": null,
        "confidence": 0.7617,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "overnight-counter-01",
      "family": "overnight-counter",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": true
      },
      "elapsed_ms": 9.351416025310755,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.6393
          }
        },
        "usage": {
          "input_tokens": 112,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.6393,
            "action": {
              "act_probability": 1
            },
            "noul": 0.6393
          }
        },
        "usage": {
          "input_tokens": 112,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.6393,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "overnight-counter-02",
      "family": "overnight-counter",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": false
      },
      "elapsed_ms": 8.364290988538414,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.5042
          }
        },
        "usage": {
          "input_tokens": 112,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.5042,
            "action": {
              "act_probability": 1
            },
            "noul": 0.5042
          }
        },
        "usage": {
          "input_tokens": 112,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.5042,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "overnight-counter-03",
      "family": "overnight-counter",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": true
      },
      "elapsed_ms": 8.456208975985646,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.3613
          }
        },
        "usage": {
          "input_tokens": 112,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.6387,
            "action": {
              "act_probability": 1
            },
            "noul": 0.3613
          }
        },
        "usage": {
          "input_tokens": 112,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": false,
        "score_absolute_error": null,
        "confidence": 0.6387,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "overnight-counter-04",
      "family": "overnight-counter",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": false
      },
      "elapsed_ms": 8.466875005979091,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.5841
          }
        },
        "usage": {
          "input_tokens": 112,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.5841,
            "action": {
              "act_probability": 1
            },
            "noul": 0.5841
          }
        },
        "usage": {
          "input_tokens": 112,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.5841,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "overnight-counter-05",
      "family": "overnight-counter",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": true
      },
      "elapsed_ms": 8.551625011023134,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.4752
          }
        },
        "usage": {
          "input_tokens": 112,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.5248,
            "action": {
              "act_probability": 1
            },
            "noul": 0.4752
          }
        },
        "usage": {
          "input_tokens": 112,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": false,
        "score_absolute_error": null,
        "confidence": 0.5247999999999999,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "overnight-counter-06",
      "family": "overnight-counter",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": false
      },
      "elapsed_ms": 8.243040996603668,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.5772
          }
        },
        "usage": {
          "input_tokens": 112,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.5772,
            "action": {
              "act_probability": 1
            },
            "noul": 0.5772
          }
        },
        "usage": {
          "input_tokens": 112,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.5772,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "overnight-counter-07",
      "family": "overnight-counter",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": true
      },
      "elapsed_ms": 8.461249992251396,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.5944
          }
        },
        "usage": {
          "input_tokens": 112,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.5944,
            "action": {
              "act_probability": 1
            },
            "noul": 0.5944
          }
        },
        "usage": {
          "input_tokens": 112,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.5944,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "overnight-counter-08",
      "family": "overnight-counter",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": false
      },
      "elapsed_ms": 8.508124970830977,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.3919
          }
        },
        "usage": {
          "input_tokens": 112,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.6081,
            "action": {
              "act_probability": 1
            },
            "noul": 0.3919
          }
        },
        "usage": {
          "input_tokens": 112,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": false,
        "score_absolute_error": null,
        "confidence": 0.6081,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "tag-filter-01",
      "family": "tag-filter",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": true
      },
      "elapsed_ms": 9.341957978904247,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.6709
          }
        },
        "usage": {
          "input_tokens": 126,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.6709,
            "action": {
              "act_probability": 1
            },
            "noul": 0.6709
          }
        },
        "usage": {
          "input_tokens": 126,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.6709,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "tag-filter-02",
      "family": "tag-filter",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": false
      },
      "elapsed_ms": 8.60800000373274,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.4716
          }
        },
        "usage": {
          "input_tokens": 121,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.5284,
            "action": {
              "act_probability": 1
            },
            "noul": 0.4716
          }
        },
        "usage": {
          "input_tokens": 121,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": false,
        "score_absolute_error": null,
        "confidence": 0.5284,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "tag-filter-03",
      "family": "tag-filter",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": true
      },
      "elapsed_ms": 9.21770796412602,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.7458
          }
        },
        "usage": {
          "input_tokens": 127,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.7458,
            "action": {
              "act_probability": 1
            },
            "noul": 0.7458
          }
        },
        "usage": {
          "input_tokens": 127,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.7458,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "tag-filter-04",
      "family": "tag-filter",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": false
      },
      "elapsed_ms": 8.663667016662657,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.7411
          }
        },
        "usage": {
          "input_tokens": 126,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.7411,
            "action": {
              "act_probability": 1
            },
            "noul": 0.7411
          }
        },
        "usage": {
          "input_tokens": 126,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.7411,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "tag-filter-05",
      "family": "tag-filter",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": true
      },
      "elapsed_ms": 8.91870801569894,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.4396
          }
        },
        "usage": {
          "input_tokens": 116,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.5604,
            "action": {
              "act_probability": 1
            },
            "noul": 0.4396
          }
        },
        "usage": {
          "input_tokens": 116,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": false,
        "score_absolute_error": null,
        "confidence": 0.5604,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "tag-filter-06",
      "family": "tag-filter",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": false
      },
      "elapsed_ms": 9.142583003267646,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.7311
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.7311,
            "action": {
              "act_probability": 1
            },
            "noul": 0.7311
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.7311,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "tag-filter-07",
      "family": "tag-filter",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": true
      },
      "elapsed_ms": 8.629334042780101,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.7006
          }
        },
        "usage": {
          "input_tokens": 127,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.7006,
            "action": {
              "act_probability": 1
            },
            "noul": 0.7006
          }
        },
        "usage": {
          "input_tokens": 127,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.7006,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "tag-filter-08",
      "family": "tag-filter",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": false
      },
      "elapsed_ms": 8.940958010498434,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.5555
          }
        },
        "usage": {
          "input_tokens": 134,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.5555,
            "action": {
              "act_probability": 1
            },
            "noul": 0.5555
          }
        },
        "usage": {
          "input_tokens": 134,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.5555,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "mini-game-points-01",
      "family": "mini-game-points",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 0
      },
      "elapsed_ms": 46.21466697426513,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 0.3268,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.815,
              "1": 0.098,
              "2": 0.0322,
              "3": 0.0548
            },
            "confidence": 0.5209
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.5209,
            "action": {
              "act_probability": 1
            },
            "score": 0.3268,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.815,
              "1": 0.098,
              "2": 0.0322,
              "3": 0.0548
            }
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": 0,
        "score_absolute_error": 0.3268,
        "confidence": 0.5209,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "mini-game-points-02",
      "family": "mini-game-points",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 1
      },
      "elapsed_ms": 9.119833004660904,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 0.419,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.7116,
              "1": 0.2054,
              "2": 0.0355,
              "3": 0.0475
            },
            "confidence": 0.4009
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.4009,
            "action": {
              "act_probability": 1
            },
            "score": 0.419,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.7116,
              "1": 0.2054,
              "2": 0.0355,
              "3": 0.0475
            }
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": 0,
        "score_absolute_error": 0.581,
        "confidence": 0.4009,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "mini-game-points-03",
      "family": "mini-game-points",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 2
      },
      "elapsed_ms": 8.685457985848188,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 0.7044,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.5313,
              "1": 0.3098,
              "2": 0.082,
              "3": 0.0768
            },
            "confidence": 0.2055
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.2055,
            "action": {
              "act_probability": 1
            },
            "score": 0.7044,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.5313,
              "1": 0.3098,
              "2": 0.082,
              "3": 0.0768
            }
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": 0,
        "score_absolute_error": 1.2955999999999999,
        "confidence": 0.2055,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "mini-game-points-04",
      "family": "mini-game-points",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 3
      },
      "elapsed_ms": 9.076207992620766,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 0.8369,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.362,
              "1": 0.5025,
              "2": 0.0721,
              "3": 0.0634
            },
            "confidence": 0.2223
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.2223,
            "action": {
              "act_probability": 1
            },
            "score": 0.8369,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.362,
              "1": 0.5025,
              "2": 0.0721,
              "3": 0.0634
            }
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": 1,
        "score_absolute_error": 2.1631,
        "confidence": 0.2223,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "mini-game-points-05",
      "family": "mini-game-points",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 0
      },
      "elapsed_ms": 8.76991602126509,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.298,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.3775,
              "1": 0.2041,
              "2": 0.1611,
              "3": 0.2572
            },
            "confidence": 0.0366
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.0366,
            "action": {
              "act_probability": 1
            },
            "score": 1.298,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.3775,
              "1": 0.2041,
              "2": 0.1611,
              "3": 0.2572
            }
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": 0,
        "score_absolute_error": 1.298,
        "confidence": 0.0366,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "mini-game-points-06",
      "family": "mini-game-points",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 1
      },
      "elapsed_ms": 8.771083026658744,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.4631,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.1876,
              "1": 0.3303,
              "2": 0.3135,
              "3": 0.1686
            },
            "confidence": 0.0308
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.0308,
            "action": {
              "act_probability": 1
            },
            "score": 1.4631,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.1876,
              "1": 0.3303,
              "2": 0.3135,
              "3": 0.1686
            }
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": 1,
        "score_absolute_error": 0.46310000000000007,
        "confidence": 0.0308,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "mini-game-points-07",
      "family": "mini-game-points",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 2
      },
      "elapsed_ms": 10.496750008314848,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 0.8853,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.5777,
              "1": 0.0993,
              "2": 0.183,
              "3": 0.14
            },
            "confidence": 0.1832
          }
        },
        "usage": {
          "input_tokens": 135,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.1832,
            "action": {
              "act_probability": 1
            },
            "score": 0.8853,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.5777,
              "1": 0.0993,
              "2": 0.183,
              "3": 0.14
            }
          }
        },
        "usage": {
          "input_tokens": 135,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": 0,
        "score_absolute_error": 1.1147,
        "confidence": 0.1832,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "mini-game-points-08",
      "family": "mini-game-points",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 3
      },
      "elapsed_ms": 8.80504201631993,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.7563,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.1714,
              "1": 0.1407,
              "2": 0.448,
              "3": 0.2399
            },
            "confidence": 0.0763
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.0763,
            "action": {
              "act_probability": 1
            },
            "score": 1.7563,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.1714,
              "1": 0.1407,
              "2": 0.448,
              "3": 0.2399
            }
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": 2,
        "score_absolute_error": 1.2437,
        "confidence": 0.0763,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "display-detail-01",
      "family": "display-detail",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 0
      },
      "elapsed_ms": 9.798791026696563,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.0425,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.3503,
              "1": 0.3572,
              "2": 0.1923,
              "3": 0.1002
            },
            "confidence": 0.0747
          }
        },
        "usage": {
          "input_tokens": 135,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.0747,
            "action": {
              "act_probability": 1
            },
            "score": 1.0425,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.3503,
              "1": 0.3572,
              "2": 0.1923,
              "3": 0.1002
            }
          }
        },
        "usage": {
          "input_tokens": 135,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": 1,
        "score_absolute_error": 1.0425,
        "confidence": 0.0747,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "display-detail-02",
      "family": "display-detail",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 1
      },
      "elapsed_ms": 9.061750024557114,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 0.9566,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.3638,
              "1": 0.3967,
              "2": 0.1588,
              "3": 0.0808
            },
            "confidence": 0.1127
          }
        },
        "usage": {
          "input_tokens": 135,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.1127,
            "action": {
              "act_probability": 1
            },
            "score": 0.9566,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.3638,
              "1": 0.3967,
              "2": 0.1588,
              "3": 0.0808
            }
          }
        },
        "usage": {
          "input_tokens": 135,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": 1,
        "score_absolute_error": 0.043399999999999994,
        "confidence": 0.1127,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "display-detail-03",
      "family": "display-detail",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 2
      },
      "elapsed_ms": 8.758375013712794,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.4239,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.223,
              "1": 0.3111,
              "2": 0.285,
              "3": 0.1809
            },
            "confidence": 0.0154
          }
        },
        "usage": {
          "input_tokens": 136,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.0154,
            "action": {
              "act_probability": 1
            },
            "score": 1.4239,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.223,
              "1": 0.3111,
              "2": 0.285,
              "3": 0.1809
            }
          }
        },
        "usage": {
          "input_tokens": 136,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": 1,
        "score_absolute_error": 0.5761000000000001,
        "confidence": 0.0154,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "display-detail-04",
      "family": "display-detail",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 3
      },
      "elapsed_ms": 8.910208009183407,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.5758,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.2245,
              "1": 0.2544,
              "2": 0.2419,
              "3": 0.2792
            },
            "confidence": 0.0023
          }
        },
        "usage": {
          "input_tokens": 135,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.0023,
            "action": {
              "act_probability": 1
            },
            "score": 1.5758,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.2245,
              "1": 0.2544,
              "2": 0.2419,
              "3": 0.2792
            }
          }
        },
        "usage": {
          "input_tokens": 135,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": 3,
        "score_absolute_error": 1.4242,
        "confidence": 0.0023,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "display-detail-05",
      "family": "display-detail",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 0
      },
      "elapsed_ms": 8.876208041328937,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.0254,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.3443,
              "1": 0.3817,
              "2": 0.1784,
              "3": 0.0956
            },
            "confidence": 0.0863
          }
        },
        "usage": {
          "input_tokens": 135,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.0863,
            "action": {
              "act_probability": 1
            },
            "score": 1.0254,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.3443,
              "1": 0.3817,
              "2": 0.1784,
              "3": 0.0956
            }
          }
        },
        "usage": {
          "input_tokens": 135,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": 1,
        "score_absolute_error": 1.0254,
        "confidence": 0.0863,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "display-detail-06",
      "family": "display-detail",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 1
      },
      "elapsed_ms": 8.951833995524794,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.1145,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.2822,
              "1": 0.3923,
              "2": 0.2541,
              "3": 0.0713
            },
            "confidence": 0.0907
          }
        },
        "usage": {
          "input_tokens": 135,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.0907,
            "action": {
              "act_probability": 1
            },
            "score": 1.1145,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.2822,
              "1": 0.3923,
              "2": 0.2541,
              "3": 0.0713
            }
          }
        },
        "usage": {
          "input_tokens": 135,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": 1,
        "score_absolute_error": 0.11450000000000005,
        "confidence": 0.0907,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "display-detail-07",
      "family": "display-detail",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 2
      },
      "elapsed_ms": 8.780125004705042,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.4711,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.2429,
              "1": 0.2743,
              "2": 0.2516,
              "3": 0.2312
            },
            "confidence": 0.0014
          }
        },
        "usage": {
          "input_tokens": 135,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.0014,
            "action": {
              "act_probability": 1
            },
            "score": 1.4711,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.2429,
              "1": 0.2743,
              "2": 0.2516,
              "3": 0.2312
            }
          }
        },
        "usage": {
          "input_tokens": 135,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": 1,
        "score_absolute_error": 0.5288999999999999,
        "confidence": 0.0014,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "display-detail-08",
      "family": "display-detail",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 3
      },
      "elapsed_ms": 8.887792006134987,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.3493,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.2514,
              "1": 0.3186,
              "2": 0.2591,
              "3": 0.1708
            },
            "confidence": 0.0166
          }
        },
        "usage": {
          "input_tokens": 145,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.0166,
            "action": {
              "act_probability": 1
            },
            "score": 1.3493,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.2514,
              "1": 0.3186,
              "2": 0.2591,
              "3": 0.1708
            }
          }
        },
        "usage": {
          "input_tokens": 145,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": 1,
        "score_absolute_error": 1.6507,
        "confidence": 0.0166,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "glossary-strength-01",
      "family": "glossary-strength",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 0
      },
      "elapsed_ms": 9.50841698795557,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.752,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.1063,
              "1": 0.2963,
              "2": 0.3367,
              "3": 0.2608
            },
            "confidence": 0.051
          }
        },
        "usage": {
          "input_tokens": 155,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.051,
            "action": {
              "act_probability": 1
            },
            "score": 1.752,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.1063,
              "1": 0.2963,
              "2": 0.3367,
              "3": 0.2608
            }
          }
        },
        "usage": {
          "input_tokens": 155,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": 2,
        "score_absolute_error": 1.752,
        "confidence": 0.051,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "glossary-strength-02",
      "family": "glossary-strength",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 1
      },
      "elapsed_ms": 13.927707972470671,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.8362,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.1039,
              "1": 0.2417,
              "2": 0.3686,
              "3": 0.2858
            },
            "confidence": 0.0591
          }
        },
        "usage": {
          "input_tokens": 161,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.0591,
            "action": {
              "act_probability": 1
            },
            "score": 1.8362,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.1039,
              "1": 0.2417,
              "2": 0.3686,
              "3": 0.2858
            }
          }
        },
        "usage": {
          "input_tokens": 161,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": 2,
        "score_absolute_error": 0.8362,
        "confidence": 0.0591,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "glossary-strength-03",
      "family": "glossary-strength",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 2
      },
      "elapsed_ms": 9.04354196973145,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.9079,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.0578,
              "1": 0.2638,
              "2": 0.3912,
              "3": 0.2872
            },
            "confidence": 0.1043
          }
        },
        "usage": {
          "input_tokens": 154,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.1043,
            "action": {
              "act_probability": 1
            },
            "score": 1.9079,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.0578,
              "1": 0.2638,
              "2": 0.3912,
              "3": 0.2872
            }
          }
        },
        "usage": {
          "input_tokens": 154,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": 2,
        "score_absolute_error": 0.09210000000000007,
        "confidence": 0.1043,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "glossary-strength-04",
      "family": "glossary-strength",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 3
      },
      "elapsed_ms": 9.580334008205682,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.965,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.052,
              "1": 0.2417,
              "2": 0.3957,
              "3": 0.3106
            },
            "confidence": 0.1149
          }
        },
        "usage": {
          "input_tokens": 163,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.1149,
            "action": {
              "act_probability": 1
            },
            "score": 1.965,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.052,
              "1": 0.2417,
              "2": 0.3957,
              "3": 0.3106
            }
          }
        },
        "usage": {
          "input_tokens": 163,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": 2,
        "score_absolute_error": 1.035,
        "confidence": 0.1149,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "glossary-strength-05",
      "family": "glossary-strength",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 0
      },
      "elapsed_ms": 9.723083989229053,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.9368,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.0727,
              "1": 0.2619,
              "2": 0.3211,
              "3": 0.3442
            },
            "confidence": 0.0814
          }
        },
        "usage": {
          "input_tokens": 164,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.0814,
            "action": {
              "act_probability": 1
            },
            "score": 1.9368,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.0727,
              "1": 0.2619,
              "2": 0.3211,
              "3": 0.3442
            }
          }
        },
        "usage": {
          "input_tokens": 164,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": 3,
        "score_absolute_error": 1.9368,
        "confidence": 0.0814,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "glossary-strength-06",
      "family": "glossary-strength",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 1
      },
      "elapsed_ms": 9.584582992829382,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.8498,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.093,
              "1": 0.2742,
              "2": 0.3228,
              "3": 0.31
            },
            "confidence": 0.0596
          }
        },
        "usage": {
          "input_tokens": 165,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.0596,
            "action": {
              "act_probability": 1
            },
            "score": 1.8498,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.093,
              "1": 0.2742,
              "2": 0.3228,
              "3": 0.31
            }
          }
        },
        "usage": {
          "input_tokens": 165,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": 2,
        "score_absolute_error": 0.8498000000000001,
        "confidence": 0.0596,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "glossary-strength-07",
      "family": "glossary-strength",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 2
      },
      "elapsed_ms": 9.920666983816773,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.8364,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.0922,
              "1": 0.2656,
              "2": 0.3556,
              "3": 0.2865
            },
            "confidence": 0.0639
          }
        },
        "usage": {
          "input_tokens": 165,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.0639,
            "action": {
              "act_probability": 1
            },
            "score": 1.8364,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.0922,
              "1": 0.2656,
              "2": 0.3556,
              "3": 0.2865
            }
          }
        },
        "usage": {
          "input_tokens": 165,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": 2,
        "score_absolute_error": 0.16359999999999997,
        "confidence": 0.0639,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 0,
      "case_id": "glossary-strength-08",
      "family": "glossary-strength",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 3
      },
      "elapsed_ms": 9.698249981738627,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 2.119,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.0191,
              "1": 0.1896,
              "2": 0.4445,
              "3": 0.3468
            },
            "confidence": 0.1932
          }
        },
        "usage": {
          "input_tokens": 166,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.1932,
            "action": {
              "act_probability": 1
            },
            "score": 2.119,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.0191,
              "1": 0.1896,
              "2": 0.4445,
              "3": 0.3468
            }
          }
        },
        "usage": {
          "input_tokens": 166,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": 2,
        "score_absolute_error": 0.8809999999999998,
        "confidence": 0.1932,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "image-layout-01",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "landscape",
        "balanced",
        "portrait"
      ],
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "elapsed_ms": 8.814082946628332,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "landscape": 0.3814,
              "balanced": 0.3311,
              "portrait": 0.2875
            },
            "confidence": 0.006
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.006,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "landscape": 0.3814,
              "balanced": 0.3311,
              "portrait": 0.2875
            }
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.006,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "image-layout-02",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "portrait",
        "landscape"
      ],
      "expected": {
        "type": "choice",
        "choice": "portrait"
      },
      "elapsed_ms": 8.840000024065375,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.3047,
              "portrait": 0.2637,
              "landscape": 0.4316
            },
            "confidence": 0.0203
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0203,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.3047,
              "portrait": 0.2637,
              "landscape": 0.4316
            }
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.0203,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "image-layout-03",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "landscape",
        "portrait",
        "balanced"
      ],
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "elapsed_ms": 8.88949999352917,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "portrait",
            "probabilities": {
              "landscape": 0.3427,
              "portrait": 0.3492,
              "balanced": 0.3081
            },
            "confidence": 0.0014
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0014,
            "action": {
              "act_probability": 1
            },
            "choice": "portrait",
            "probabilities": {
              "landscape": 0.3427,
              "portrait": 0.3492,
              "balanced": 0.3081
            }
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "portrait",
        "score_absolute_error": null,
        "confidence": 0.0014,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "image-layout-04",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "landscape",
        "portrait"
      ],
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "elapsed_ms": 8.897374966181815,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.2776,
              "landscape": 0.4783,
              "portrait": 0.2441
            },
            "confidence": 0.0418
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0418,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.2776,
              "landscape": 0.4783,
              "portrait": 0.2441
            }
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.0418,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "image-layout-05",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "portrait",
        "landscape",
        "balanced"
      ],
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "elapsed_ms": 9.084541001357138,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "portrait": 0.2672,
              "landscape": 0.4149,
              "balanced": 0.3179
            },
            "confidence": 0.0152
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0152,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "portrait": 0.2672,
              "landscape": 0.4149,
              "balanced": 0.3179
            }
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.0152,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "image-layout-06",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "landscape",
        "portrait"
      ],
      "expected": {
        "type": "choice",
        "choice": "portrait"
      },
      "elapsed_ms": 10.211583983618766,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.2731,
              "landscape": 0.4137,
              "portrait": 0.3132
            },
            "confidence": 0.014
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.014,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.2731,
              "landscape": 0.4137,
              "portrait": 0.3132
            }
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.014,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "image-layout-07",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "portrait",
        "landscape"
      ],
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "elapsed_ms": 9.4775419565849,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.2631,
              "portrait": 0.3522,
              "landscape": 0.3846
            },
            "confidence": 0.0112
          }
        },
        "usage": {
          "input_tokens": 136,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0112,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.2631,
              "portrait": 0.3522,
              "landscape": 0.3846
            }
          }
        },
        "usage": {
          "input_tokens": 136,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.0112,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "image-layout-08",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "portrait",
        "landscape",
        "balanced"
      ],
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "elapsed_ms": 10.100333951413631,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "portrait",
            "probabilities": {
              "portrait": 0.5268,
              "landscape": 0.2492,
              "balanced": 0.224
            },
            "confidence": 0.0724
          }
        },
        "usage": {
          "input_tokens": 122,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0724,
            "action": {
              "act_probability": 1
            },
            "choice": "portrait",
            "probabilities": {
              "portrait": 0.5268,
              "landscape": 0.2492,
              "balanced": 0.224
            }
          }
        },
        "usage": {
          "input_tokens": 122,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "portrait",
        "score_absolute_error": null,
        "confidence": 0.0724,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "notification-delivery-01",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "digest",
        "email"
      ],
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "elapsed_ms": 9.570499998517334,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "in_app": 0.2397,
              "digest": 0.2275,
              "email": 0.5328
            },
            "confidence": 0.0764
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0764,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "in_app": 0.2397,
              "digest": 0.2275,
              "email": 0.5328
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.0764,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "notification-delivery-02",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "digest",
        "email"
      ],
      "expected": {
        "type": "choice",
        "choice": "in_app"
      },
      "elapsed_ms": 9.126792021561414,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "in_app": 0.2571,
              "digest": 0.2312,
              "email": 0.5117
            },
            "confidence": 0.0619
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0619,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "in_app": 0.2571,
              "digest": 0.2312,
              "email": 0.5117
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.0619,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "notification-delivery-03",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "email",
        "digest",
        "in_app"
      ],
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "elapsed_ms": 9.465500013902783,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "email": 0.4489,
              "digest": 0.3116,
              "in_app": 0.2395
            },
            "confidence": 0.0304
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0304,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "email": 0.4489,
              "digest": 0.3116,
              "in_app": 0.2395
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.0304,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "notification-delivery-04",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "in_app",
        "email"
      ],
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "elapsed_ms": 9.549250011332333,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "digest": 0.1798,
              "in_app": 0.2749,
              "email": 0.5453
            },
            "confidence": 0.0951
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0951,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "digest": 0.1798,
              "in_app": 0.2749,
              "email": 0.5453
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.0951,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "notification-delivery-05",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "digest",
        "email"
      ],
      "expected": {
        "type": "choice",
        "choice": "in_app"
      },
      "elapsed_ms": 9.580499958246946,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "in_app": 0.2955,
              "digest": 0.2817,
              "email": 0.4228
            },
            "confidence": 0.0159
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0159,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "in_app": 0.2955,
              "digest": 0.2817,
              "email": 0.4228
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.0159,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "notification-delivery-06",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "email",
        "digest",
        "in_app"
      ],
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "elapsed_ms": 9.453625010792166,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "email": 0.5312,
              "digest": 0.226,
              "in_app": 0.2428
            },
            "confidence": 0.0753
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0753,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "email": 0.5312,
              "digest": 0.226,
              "in_app": 0.2428
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.0753,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "notification-delivery-07",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "in_app",
        "email"
      ],
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "elapsed_ms": 9.426958044059575,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "digest",
            "probabilities": {
              "digest": 0.4598,
              "in_app": 0.2653,
              "email": 0.2749
            },
            "confidence": 0.0313
          }
        },
        "usage": {
          "input_tokens": 154,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0313,
            "action": {
              "act_probability": 1
            },
            "choice": "digest",
            "probabilities": {
              "digest": 0.4598,
              "in_app": 0.2653,
              "email": 0.2749
            }
          }
        },
        "usage": {
          "input_tokens": 154,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "digest",
        "score_absolute_error": null,
        "confidence": 0.0313,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "notification-delivery-08",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "email",
        "in_app"
      ],
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "elapsed_ms": 9.827959001995623,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "digest": 0.2652,
              "email": 0.3729,
              "in_app": 0.3619
            },
            "confidence": 0.01
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.01,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "digest": 0.2652,
              "email": 0.3729,
              "in_app": 0.3619
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.01,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "release-note-group-01",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "fix",
        "breaking",
        "addition"
      ],
      "expected": {
        "type": "choice",
        "choice": "breaking"
      },
      "elapsed_ms": 9.846707980614156,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "fix": 0.3008,
              "breaking": 0.2342,
              "addition": 0.465
            },
            "confidence": 0.0375
          }
        },
        "usage": {
          "input_tokens": 148,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0375,
            "action": {
              "act_probability": 1
            },
            "choice": "addition",
            "probabilities": {
              "fix": 0.3008,
              "breaking": 0.2342,
              "addition": 0.465
            }
          }
        },
        "usage": {
          "input_tokens": 148,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.0375,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "release-note-group-02",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "breaking",
        "fix"
      ],
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "elapsed_ms": 9.363208955619484,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "addition": 0.128,
              "breaking": 0.1385,
              "fix": 0.7335
            },
            "confidence": 0.3044
          }
        },
        "usage": {
          "input_tokens": 140,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.3044,
            "action": {
              "act_probability": 1
            },
            "choice": "fix",
            "probabilities": {
              "addition": 0.128,
              "breaking": 0.1385,
              "fix": 0.7335
            }
          }
        },
        "usage": {
          "input_tokens": 140,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.3044,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "release-note-group-03",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "breaking",
        "fix"
      ],
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "elapsed_ms": 9.22616699244827,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "addition": 0.4816,
              "breaking": 0.162,
              "fix": 0.3564
            },
            "confidence": 0.0766
          }
        },
        "usage": {
          "input_tokens": 137,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0766,
            "action": {
              "act_probability": 1
            },
            "choice": "addition",
            "probabilities": {
              "addition": 0.4816,
              "breaking": 0.162,
              "fix": 0.3564
            }
          }
        },
        "usage": {
          "input_tokens": 137,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.0766,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "release-note-group-04",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "fix",
        "breaking",
        "addition"
      ],
      "expected": {
        "type": "choice",
        "choice": "breaking"
      },
      "elapsed_ms": 9.802750020753592,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "fix": 0.4896,
              "breaking": 0.2786,
              "addition": 0.2319
            },
            "confidence": 0.0492
          }
        },
        "usage": {
          "input_tokens": 145,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0492,
            "action": {
              "act_probability": 1
            },
            "choice": "fix",
            "probabilities": {
              "fix": 0.4896,
              "breaking": 0.2786,
              "addition": 0.2319
            }
          }
        },
        "usage": {
          "input_tokens": 145,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.0492,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "release-note-group-05",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "addition",
        "fix"
      ],
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "elapsed_ms": 9.41925001097843,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "breaking": 0.1838,
              "addition": 0.1464,
              "fix": 0.6698
            },
            "confidence": 0.2162
          }
        },
        "usage": {
          "input_tokens": 143,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.2162,
            "action": {
              "act_probability": 1
            },
            "choice": "fix",
            "probabilities": {
              "breaking": 0.1838,
              "addition": 0.1464,
              "fix": 0.6698
            }
          }
        },
        "usage": {
          "input_tokens": 143,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.2162,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "release-note-group-06",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "fix",
        "breaking"
      ],
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "elapsed_ms": 9.385542012751102,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "addition": 0.4043,
              "fix": 0.3457,
              "breaking": 0.25
            },
            "confidence": 0.017
          }
        },
        "usage": {
          "input_tokens": 148,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.017,
            "action": {
              "act_probability": 1
            },
            "choice": "addition",
            "probabilities": {
              "addition": 0.4043,
              "fix": 0.3457,
              "breaking": 0.25
            }
          }
        },
        "usage": {
          "input_tokens": 148,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.017,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "release-note-group-07",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "fix",
        "addition"
      ],
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "elapsed_ms": 9.85645898617804,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "breaking": 0.1442,
              "fix": 0.6103,
              "addition": 0.2455
            },
            "confidence": 0.1576
          }
        },
        "usage": {
          "input_tokens": 142,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.1576,
            "action": {
              "act_probability": 1
            },
            "choice": "fix",
            "probabilities": {
              "breaking": 0.1442,
              "fix": 0.6103,
              "addition": 0.2455
            }
          }
        },
        "usage": {
          "input_tokens": 142,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.1576,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "release-note-group-08",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "fix",
        "addition"
      ],
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "elapsed_ms": 9.719582973048091,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "breaking": 0.289,
              "fix": 0.4007,
              "addition": 0.3103
            },
            "confidence": 0.0094
          }
        },
        "usage": {
          "input_tokens": 138,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0094,
            "action": {
              "act_probability": 1
            },
            "choice": "fix",
            "probabilities": {
              "breaking": 0.289,
              "fix": 0.4007,
              "addition": 0.3103
            }
          }
        },
        "usage": {
          "input_tokens": 138,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.0094,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "demo-discount-01",
      "family": "demo-discount",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": true
      },
      "elapsed_ms": 9.214707999490201,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.6902
          }
        },
        "usage": {
          "input_tokens": 146,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.6902,
            "action": {
              "act_probability": 1
            },
            "noul": 0.6902
          }
        },
        "usage": {
          "input_tokens": 146,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.6902,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "demo-discount-02",
      "family": "demo-discount",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": false
      },
      "elapsed_ms": 9.459333028644323,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.2415
          }
        },
        "usage": {
          "input_tokens": 146,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.7585,
            "action": {
              "act_probability": 1
            },
            "noul": 0.2415
          }
        },
        "usage": {
          "input_tokens": 146,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": false,
        "score_absolute_error": null,
        "confidence": 0.7585,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "demo-discount-03",
      "family": "demo-discount",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": true
      },
      "elapsed_ms": 9.314791997894645,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.2825
          }
        },
        "usage": {
          "input_tokens": 146,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.7175,
            "action": {
              "act_probability": 1
            },
            "noul": 0.2825
          }
        },
        "usage": {
          "input_tokens": 146,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": false,
        "score_absolute_error": null,
        "confidence": 0.7175,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "demo-discount-04",
      "family": "demo-discount",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": false
      },
      "elapsed_ms": 10.042832989711314,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.3083
          }
        },
        "usage": {
          "input_tokens": 146,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.6917,
            "action": {
              "act_probability": 1
            },
            "noul": 0.3083
          }
        },
        "usage": {
          "input_tokens": 146,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": false,
        "score_absolute_error": null,
        "confidence": 0.6917,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "demo-discount-05",
      "family": "demo-discount",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": true
      },
      "elapsed_ms": 9.479209023993462,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.5631
          }
        },
        "usage": {
          "input_tokens": 148,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.5631,
            "action": {
              "act_probability": 1
            },
            "noul": 0.5631
          }
        },
        "usage": {
          "input_tokens": 148,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.5631,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "demo-discount-06",
      "family": "demo-discount",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": false
      },
      "elapsed_ms": 9.382791991811246,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.1742
          }
        },
        "usage": {
          "input_tokens": 146,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.8258,
            "action": {
              "act_probability": 1
            },
            "noul": 0.1742
          }
        },
        "usage": {
          "input_tokens": 146,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": false,
        "score_absolute_error": null,
        "confidence": 0.8258,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "demo-discount-07",
      "family": "demo-discount",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": true
      },
      "elapsed_ms": 9.396082954481244,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.2773
          }
        },
        "usage": {
          "input_tokens": 146,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.7227,
            "action": {
              "act_probability": 1
            },
            "noul": 0.2773
          }
        },
        "usage": {
          "input_tokens": 146,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": false,
        "score_absolute_error": null,
        "confidence": 0.7227,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "demo-discount-08",
      "family": "demo-discount",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": false
      },
      "elapsed_ms": 9.349333995487541,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.2383
          }
        },
        "usage": {
          "input_tokens": 146,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.7617,
            "action": {
              "act_probability": 1
            },
            "noul": 0.2383
          }
        },
        "usage": {
          "input_tokens": 146,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": false,
        "score_absolute_error": null,
        "confidence": 0.7617,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "overnight-counter-01",
      "family": "overnight-counter",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": true
      },
      "elapsed_ms": 9.328124986495823,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.6393
          }
        },
        "usage": {
          "input_tokens": 112,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.6393,
            "action": {
              "act_probability": 1
            },
            "noul": 0.6393
          }
        },
        "usage": {
          "input_tokens": 112,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.6393,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "overnight-counter-02",
      "family": "overnight-counter",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": false
      },
      "elapsed_ms": 9.167624986730516,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.5042
          }
        },
        "usage": {
          "input_tokens": 112,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.5042,
            "action": {
              "act_probability": 1
            },
            "noul": 0.5042
          }
        },
        "usage": {
          "input_tokens": 112,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.5042,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "overnight-counter-03",
      "family": "overnight-counter",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": true
      },
      "elapsed_ms": 9.325334045570344,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.3613
          }
        },
        "usage": {
          "input_tokens": 112,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.6387,
            "action": {
              "act_probability": 1
            },
            "noul": 0.3613
          }
        },
        "usage": {
          "input_tokens": 112,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": false,
        "score_absolute_error": null,
        "confidence": 0.6387,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "overnight-counter-04",
      "family": "overnight-counter",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": false
      },
      "elapsed_ms": 9.37012501526624,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.5841
          }
        },
        "usage": {
          "input_tokens": 112,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.5841,
            "action": {
              "act_probability": 1
            },
            "noul": 0.5841
          }
        },
        "usage": {
          "input_tokens": 112,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.5841,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "overnight-counter-05",
      "family": "overnight-counter",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": true
      },
      "elapsed_ms": 9.31854103691876,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.4752
          }
        },
        "usage": {
          "input_tokens": 112,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.5248,
            "action": {
              "act_probability": 1
            },
            "noul": 0.4752
          }
        },
        "usage": {
          "input_tokens": 112,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": false,
        "score_absolute_error": null,
        "confidence": 0.5247999999999999,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "overnight-counter-06",
      "family": "overnight-counter",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": false
      },
      "elapsed_ms": 9.65387502219528,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.5772
          }
        },
        "usage": {
          "input_tokens": 112,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.5772,
            "action": {
              "act_probability": 1
            },
            "noul": 0.5772
          }
        },
        "usage": {
          "input_tokens": 112,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.5772,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "overnight-counter-07",
      "family": "overnight-counter",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": true
      },
      "elapsed_ms": 9.273791045416147,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.5944
          }
        },
        "usage": {
          "input_tokens": 112,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.5944,
            "action": {
              "act_probability": 1
            },
            "noul": 0.5944
          }
        },
        "usage": {
          "input_tokens": 112,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.5944,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "overnight-counter-08",
      "family": "overnight-counter",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": false
      },
      "elapsed_ms": 9.153541992418468,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.3919
          }
        },
        "usage": {
          "input_tokens": 112,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.6081,
            "action": {
              "act_probability": 1
            },
            "noul": 0.3919
          }
        },
        "usage": {
          "input_tokens": 112,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": false,
        "score_absolute_error": null,
        "confidence": 0.6081,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "tag-filter-01",
      "family": "tag-filter",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": true
      },
      "elapsed_ms": 9.462540969252586,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.6709
          }
        },
        "usage": {
          "input_tokens": 126,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.6709,
            "action": {
              "act_probability": 1
            },
            "noul": 0.6709
          }
        },
        "usage": {
          "input_tokens": 126,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.6709,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "tag-filter-02",
      "family": "tag-filter",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": false
      },
      "elapsed_ms": 10.44916600221768,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.4716
          }
        },
        "usage": {
          "input_tokens": 121,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.5284,
            "action": {
              "act_probability": 1
            },
            "noul": 0.4716
          }
        },
        "usage": {
          "input_tokens": 121,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": false,
        "score_absolute_error": null,
        "confidence": 0.5284,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "tag-filter-03",
      "family": "tag-filter",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": true
      },
      "elapsed_ms": 9.485499991569668,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.7458
          }
        },
        "usage": {
          "input_tokens": 127,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.7458,
            "action": {
              "act_probability": 1
            },
            "noul": 0.7458
          }
        },
        "usage": {
          "input_tokens": 127,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.7458,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "tag-filter-04",
      "family": "tag-filter",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": false
      },
      "elapsed_ms": 9.623957972507924,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.7411
          }
        },
        "usage": {
          "input_tokens": 126,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.7411,
            "action": {
              "act_probability": 1
            },
            "noul": 0.7411
          }
        },
        "usage": {
          "input_tokens": 126,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.7411,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "tag-filter-05",
      "family": "tag-filter",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": true
      },
      "elapsed_ms": 9.533458040095866,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.4396
          }
        },
        "usage": {
          "input_tokens": 116,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.5604,
            "action": {
              "act_probability": 1
            },
            "noul": 0.4396
          }
        },
        "usage": {
          "input_tokens": 116,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": false,
        "score_absolute_error": null,
        "confidence": 0.5604,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "tag-filter-06",
      "family": "tag-filter",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": false
      },
      "elapsed_ms": 9.638165996875614,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.7311
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.7311,
            "action": {
              "act_probability": 1
            },
            "noul": 0.7311
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.7311,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "tag-filter-07",
      "family": "tag-filter",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": true
      },
      "elapsed_ms": 9.988208010327071,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.7006
          }
        },
        "usage": {
          "input_tokens": 127,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.7006,
            "action": {
              "act_probability": 1
            },
            "noul": 0.7006
          }
        },
        "usage": {
          "input_tokens": 127,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.7006,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "tag-filter-08",
      "family": "tag-filter",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": false
      },
      "elapsed_ms": 9.628125000745058,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.5555
          }
        },
        "usage": {
          "input_tokens": 134,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.5555,
            "action": {
              "act_probability": 1
            },
            "noul": 0.5555
          }
        },
        "usage": {
          "input_tokens": 134,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.5555,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "mini-game-points-01",
      "family": "mini-game-points",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 0
      },
      "elapsed_ms": 9.854917007032782,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 0.3268,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.815,
              "1": 0.098,
              "2": 0.0322,
              "3": 0.0548
            },
            "confidence": 0.5209
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.5209,
            "action": {
              "act_probability": 1
            },
            "score": 0.3268,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.815,
              "1": 0.098,
              "2": 0.0322,
              "3": 0.0548
            }
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": 0,
        "score_absolute_error": 0.3268,
        "confidence": 0.5209,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "mini-game-points-02",
      "family": "mini-game-points",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 1
      },
      "elapsed_ms": 9.673333028331399,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 0.419,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.7116,
              "1": 0.2054,
              "2": 0.0355,
              "3": 0.0475
            },
            "confidence": 0.4009
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.4009,
            "action": {
              "act_probability": 1
            },
            "score": 0.419,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.7116,
              "1": 0.2054,
              "2": 0.0355,
              "3": 0.0475
            }
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": 0,
        "score_absolute_error": 0.581,
        "confidence": 0.4009,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "mini-game-points-03",
      "family": "mini-game-points",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 2
      },
      "elapsed_ms": 9.714999992866069,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 0.7044,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.5313,
              "1": 0.3098,
              "2": 0.082,
              "3": 0.0768
            },
            "confidence": 0.2055
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.2055,
            "action": {
              "act_probability": 1
            },
            "score": 0.7044,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.5313,
              "1": 0.3098,
              "2": 0.082,
              "3": 0.0768
            }
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": 0,
        "score_absolute_error": 1.2955999999999999,
        "confidence": 0.2055,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "mini-game-points-04",
      "family": "mini-game-points",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 3
      },
      "elapsed_ms": 9.603084006812423,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 0.8369,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.362,
              "1": 0.5025,
              "2": 0.0721,
              "3": 0.0634
            },
            "confidence": 0.2223
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.2223,
            "action": {
              "act_probability": 1
            },
            "score": 0.8369,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.362,
              "1": 0.5025,
              "2": 0.0721,
              "3": 0.0634
            }
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": 1,
        "score_absolute_error": 2.1631,
        "confidence": 0.2223,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "mini-game-points-05",
      "family": "mini-game-points",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 0
      },
      "elapsed_ms": 10.017708991654217,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.298,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.3775,
              "1": 0.2041,
              "2": 0.1611,
              "3": 0.2572
            },
            "confidence": 0.0366
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.0366,
            "action": {
              "act_probability": 1
            },
            "score": 1.298,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.3775,
              "1": 0.2041,
              "2": 0.1611,
              "3": 0.2572
            }
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": 0,
        "score_absolute_error": 1.298,
        "confidence": 0.0366,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "mini-game-points-06",
      "family": "mini-game-points",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 1
      },
      "elapsed_ms": 9.65816603275016,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.4631,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.1876,
              "1": 0.3303,
              "2": 0.3135,
              "3": 0.1686
            },
            "confidence": 0.0308
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.0308,
            "action": {
              "act_probability": 1
            },
            "score": 1.4631,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.1876,
              "1": 0.3303,
              "2": 0.3135,
              "3": 0.1686
            }
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": 1,
        "score_absolute_error": 0.46310000000000007,
        "confidence": 0.0308,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "mini-game-points-07",
      "family": "mini-game-points",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 2
      },
      "elapsed_ms": 9.626833023503423,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 0.8853,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.5777,
              "1": 0.0993,
              "2": 0.183,
              "3": 0.14
            },
            "confidence": 0.1832
          }
        },
        "usage": {
          "input_tokens": 135,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.1832,
            "action": {
              "act_probability": 1
            },
            "score": 0.8853,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.5777,
              "1": 0.0993,
              "2": 0.183,
              "3": 0.14
            }
          }
        },
        "usage": {
          "input_tokens": 135,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": 0,
        "score_absolute_error": 1.1147,
        "confidence": 0.1832,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "mini-game-points-08",
      "family": "mini-game-points",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 3
      },
      "elapsed_ms": 9.751500037964433,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.7563,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.1714,
              "1": 0.1407,
              "2": 0.448,
              "3": 0.2399
            },
            "confidence": 0.0763
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.0763,
            "action": {
              "act_probability": 1
            },
            "score": 1.7563,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.1714,
              "1": 0.1407,
              "2": 0.448,
              "3": 0.2399
            }
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": 2,
        "score_absolute_error": 1.2437,
        "confidence": 0.0763,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "display-detail-01",
      "family": "display-detail",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 0
      },
      "elapsed_ms": 9.637290961109102,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.0425,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.3503,
              "1": 0.3572,
              "2": 0.1923,
              "3": 0.1002
            },
            "confidence": 0.0747
          }
        },
        "usage": {
          "input_tokens": 135,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.0747,
            "action": {
              "act_probability": 1
            },
            "score": 1.0425,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.3503,
              "1": 0.3572,
              "2": 0.1923,
              "3": 0.1002
            }
          }
        },
        "usage": {
          "input_tokens": 135,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": 1,
        "score_absolute_error": 1.0425,
        "confidence": 0.0747,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "display-detail-02",
      "family": "display-detail",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 1
      },
      "elapsed_ms": 9.737958025652915,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 0.9566,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.3638,
              "1": 0.3967,
              "2": 0.1588,
              "3": 0.0808
            },
            "confidence": 0.1127
          }
        },
        "usage": {
          "input_tokens": 135,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.1127,
            "action": {
              "act_probability": 1
            },
            "score": 0.9566,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.3638,
              "1": 0.3967,
              "2": 0.1588,
              "3": 0.0808
            }
          }
        },
        "usage": {
          "input_tokens": 135,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": 1,
        "score_absolute_error": 0.043399999999999994,
        "confidence": 0.1127,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "display-detail-03",
      "family": "display-detail",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 2
      },
      "elapsed_ms": 9.845957974903286,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.4239,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.223,
              "1": 0.3111,
              "2": 0.285,
              "3": 0.1809
            },
            "confidence": 0.0154
          }
        },
        "usage": {
          "input_tokens": 136,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.0154,
            "action": {
              "act_probability": 1
            },
            "score": 1.4239,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.223,
              "1": 0.3111,
              "2": 0.285,
              "3": 0.1809
            }
          }
        },
        "usage": {
          "input_tokens": 136,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": 1,
        "score_absolute_error": 0.5761000000000001,
        "confidence": 0.0154,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "display-detail-04",
      "family": "display-detail",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 3
      },
      "elapsed_ms": 9.535417018923908,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.5758,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.2245,
              "1": 0.2544,
              "2": 0.2419,
              "3": 0.2792
            },
            "confidence": 0.0023
          }
        },
        "usage": {
          "input_tokens": 135,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.0023,
            "action": {
              "act_probability": 1
            },
            "score": 1.5758,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.2245,
              "1": 0.2544,
              "2": 0.2419,
              "3": 0.2792
            }
          }
        },
        "usage": {
          "input_tokens": 135,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": 3,
        "score_absolute_error": 1.4242,
        "confidence": 0.0023,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "display-detail-05",
      "family": "display-detail",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 0
      },
      "elapsed_ms": 9.724417002871633,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.0254,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.3443,
              "1": 0.3817,
              "2": 0.1784,
              "3": 0.0956
            },
            "confidence": 0.0863
          }
        },
        "usage": {
          "input_tokens": 135,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.0863,
            "action": {
              "act_probability": 1
            },
            "score": 1.0254,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.3443,
              "1": 0.3817,
              "2": 0.1784,
              "3": 0.0956
            }
          }
        },
        "usage": {
          "input_tokens": 135,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": 1,
        "score_absolute_error": 1.0254,
        "confidence": 0.0863,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "display-detail-06",
      "family": "display-detail",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 1
      },
      "elapsed_ms": 10.009000019636005,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.1145,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.2822,
              "1": 0.3923,
              "2": 0.2541,
              "3": 0.0713
            },
            "confidence": 0.0907
          }
        },
        "usage": {
          "input_tokens": 135,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.0907,
            "action": {
              "act_probability": 1
            },
            "score": 1.1145,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.2822,
              "1": 0.3923,
              "2": 0.2541,
              "3": 0.0713
            }
          }
        },
        "usage": {
          "input_tokens": 135,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": 1,
        "score_absolute_error": 0.11450000000000005,
        "confidence": 0.0907,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "display-detail-07",
      "family": "display-detail",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 2
      },
      "elapsed_ms": 9.983083000406623,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.4711,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.2429,
              "1": 0.2743,
              "2": 0.2516,
              "3": 0.2312
            },
            "confidence": 0.0014
          }
        },
        "usage": {
          "input_tokens": 135,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.0014,
            "action": {
              "act_probability": 1
            },
            "score": 1.4711,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.2429,
              "1": 0.2743,
              "2": 0.2516,
              "3": 0.2312
            }
          }
        },
        "usage": {
          "input_tokens": 135,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": 1,
        "score_absolute_error": 0.5288999999999999,
        "confidence": 0.0014,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "display-detail-08",
      "family": "display-detail",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 3
      },
      "elapsed_ms": 10.558582958765328,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.3493,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.2514,
              "1": 0.3186,
              "2": 0.2591,
              "3": 0.1708
            },
            "confidence": 0.0166
          }
        },
        "usage": {
          "input_tokens": 145,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.0166,
            "action": {
              "act_probability": 1
            },
            "score": 1.3493,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.2514,
              "1": 0.3186,
              "2": 0.2591,
              "3": 0.1708
            }
          }
        },
        "usage": {
          "input_tokens": 145,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": 1,
        "score_absolute_error": 1.6507,
        "confidence": 0.0166,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "glossary-strength-01",
      "family": "glossary-strength",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 0
      },
      "elapsed_ms": 11.898124997969717,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.752,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.1063,
              "1": 0.2963,
              "2": 0.3367,
              "3": 0.2608
            },
            "confidence": 0.051
          }
        },
        "usage": {
          "input_tokens": 155,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.051,
            "action": {
              "act_probability": 1
            },
            "score": 1.752,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.1063,
              "1": 0.2963,
              "2": 0.3367,
              "3": 0.2608
            }
          }
        },
        "usage": {
          "input_tokens": 155,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": 2,
        "score_absolute_error": 1.752,
        "confidence": 0.051,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "glossary-strength-02",
      "family": "glossary-strength",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 1
      },
      "elapsed_ms": 11.244458029977977,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.8362,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.1039,
              "1": 0.2417,
              "2": 0.3686,
              "3": 0.2858
            },
            "confidence": 0.0591
          }
        },
        "usage": {
          "input_tokens": 161,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.0591,
            "action": {
              "act_probability": 1
            },
            "score": 1.8362,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.1039,
              "1": 0.2417,
              "2": 0.3686,
              "3": 0.2858
            }
          }
        },
        "usage": {
          "input_tokens": 161,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": 2,
        "score_absolute_error": 0.8362,
        "confidence": 0.0591,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "glossary-strength-03",
      "family": "glossary-strength",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 2
      },
      "elapsed_ms": 10.59983397135511,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.9079,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.0578,
              "1": 0.2638,
              "2": 0.3912,
              "3": 0.2872
            },
            "confidence": 0.1043
          }
        },
        "usage": {
          "input_tokens": 154,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.1043,
            "action": {
              "act_probability": 1
            },
            "score": 1.9079,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.0578,
              "1": 0.2638,
              "2": 0.3912,
              "3": 0.2872
            }
          }
        },
        "usage": {
          "input_tokens": 154,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": 2,
        "score_absolute_error": 0.09210000000000007,
        "confidence": 0.1043,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "glossary-strength-04",
      "family": "glossary-strength",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 3
      },
      "elapsed_ms": 11.204125010408461,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.965,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.052,
              "1": 0.2417,
              "2": 0.3957,
              "3": 0.3106
            },
            "confidence": 0.1149
          }
        },
        "usage": {
          "input_tokens": 163,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.1149,
            "action": {
              "act_probability": 1
            },
            "score": 1.965,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.052,
              "1": 0.2417,
              "2": 0.3957,
              "3": 0.3106
            }
          }
        },
        "usage": {
          "input_tokens": 163,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": 2,
        "score_absolute_error": 1.035,
        "confidence": 0.1149,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "glossary-strength-05",
      "family": "glossary-strength",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 0
      },
      "elapsed_ms": 10.61758300056681,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.9368,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.0727,
              "1": 0.2619,
              "2": 0.3211,
              "3": 0.3442
            },
            "confidence": 0.0814
          }
        },
        "usage": {
          "input_tokens": 164,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.0814,
            "action": {
              "act_probability": 1
            },
            "score": 1.9368,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.0727,
              "1": 0.2619,
              "2": 0.3211,
              "3": 0.3442
            }
          }
        },
        "usage": {
          "input_tokens": 164,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": 3,
        "score_absolute_error": 1.9368,
        "confidence": 0.0814,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "glossary-strength-06",
      "family": "glossary-strength",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 1
      },
      "elapsed_ms": 10.57774998480454,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.8498,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.093,
              "1": 0.2742,
              "2": 0.3228,
              "3": 0.31
            },
            "confidence": 0.0596
          }
        },
        "usage": {
          "input_tokens": 165,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.0596,
            "action": {
              "act_probability": 1
            },
            "score": 1.8498,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.093,
              "1": 0.2742,
              "2": 0.3228,
              "3": 0.31
            }
          }
        },
        "usage": {
          "input_tokens": 165,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": 2,
        "score_absolute_error": 0.8498000000000001,
        "confidence": 0.0596,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "glossary-strength-07",
      "family": "glossary-strength",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 2
      },
      "elapsed_ms": 11.018041986972094,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.8364,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.0922,
              "1": 0.2656,
              "2": 0.3556,
              "3": 0.2865
            },
            "confidence": 0.0639
          }
        },
        "usage": {
          "input_tokens": 165,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.0639,
            "action": {
              "act_probability": 1
            },
            "score": 1.8364,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.0922,
              "1": 0.2656,
              "2": 0.3556,
              "3": 0.2865
            }
          }
        },
        "usage": {
          "input_tokens": 165,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": 2,
        "score_absolute_error": 0.16359999999999997,
        "confidence": 0.0639,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 1,
      "case_id": "glossary-strength-08",
      "family": "glossary-strength",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 3
      },
      "elapsed_ms": 10.728334018494934,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 2.119,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.0191,
              "1": 0.1896,
              "2": 0.4445,
              "3": 0.3468
            },
            "confidence": 0.1932
          }
        },
        "usage": {
          "input_tokens": 166,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.1932,
            "action": {
              "act_probability": 1
            },
            "score": 2.119,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.0191,
              "1": 0.1896,
              "2": 0.4445,
              "3": 0.3468
            }
          }
        },
        "usage": {
          "input_tokens": 166,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": 2,
        "score_absolute_error": 0.8809999999999998,
        "confidence": 0.1932,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "image-layout-01",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "landscape",
        "balanced",
        "portrait"
      ],
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "elapsed_ms": 10.245584009680897,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "landscape": 0.3814,
              "balanced": 0.3311,
              "portrait": 0.2875
            },
            "confidence": 0.006
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.006,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "landscape": 0.3814,
              "balanced": 0.3311,
              "portrait": 0.2875
            }
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.006,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "image-layout-02",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "portrait",
        "landscape"
      ],
      "expected": {
        "type": "choice",
        "choice": "portrait"
      },
      "elapsed_ms": 10.346167022362351,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.3047,
              "portrait": 0.2637,
              "landscape": 0.4316
            },
            "confidence": 0.0203
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0203,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.3047,
              "portrait": 0.2637,
              "landscape": 0.4316
            }
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.0203,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "image-layout-03",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "landscape",
        "portrait",
        "balanced"
      ],
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "elapsed_ms": 10.300666966941208,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "portrait",
            "probabilities": {
              "landscape": 0.3427,
              "portrait": 0.3492,
              "balanced": 0.3081
            },
            "confidence": 0.0014
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0014,
            "action": {
              "act_probability": 1
            },
            "choice": "portrait",
            "probabilities": {
              "landscape": 0.3427,
              "portrait": 0.3492,
              "balanced": 0.3081
            }
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "portrait",
        "score_absolute_error": null,
        "confidence": 0.0014,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "image-layout-04",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "landscape",
        "portrait"
      ],
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "elapsed_ms": 11.351583001669496,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.2776,
              "landscape": 0.4783,
              "portrait": 0.2441
            },
            "confidence": 0.0418
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0418,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.2776,
              "landscape": 0.4783,
              "portrait": 0.2441
            }
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.0418,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "image-layout-05",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "portrait",
        "landscape",
        "balanced"
      ],
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "elapsed_ms": 10.564208030700684,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "portrait": 0.2672,
              "landscape": 0.4149,
              "balanced": 0.3179
            },
            "confidence": 0.0152
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0152,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "portrait": 0.2672,
              "landscape": 0.4149,
              "balanced": 0.3179
            }
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.0152,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "image-layout-06",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "landscape",
        "portrait"
      ],
      "expected": {
        "type": "choice",
        "choice": "portrait"
      },
      "elapsed_ms": 10.50525001483038,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.2731,
              "landscape": 0.4137,
              "portrait": 0.3132
            },
            "confidence": 0.014
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.014,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.2731,
              "landscape": 0.4137,
              "portrait": 0.3132
            }
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.014,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "image-layout-07",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "portrait",
        "landscape"
      ],
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "elapsed_ms": 10.82991698058322,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.2631,
              "portrait": 0.3522,
              "landscape": 0.3846
            },
            "confidence": 0.0112
          }
        },
        "usage": {
          "input_tokens": 136,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0112,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.2631,
              "portrait": 0.3522,
              "landscape": 0.3846
            }
          }
        },
        "usage": {
          "input_tokens": 136,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.0112,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "image-layout-08",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "portrait",
        "landscape",
        "balanced"
      ],
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "elapsed_ms": 10.90233400464058,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "portrait",
            "probabilities": {
              "portrait": 0.5268,
              "landscape": 0.2492,
              "balanced": 0.224
            },
            "confidence": 0.0724
          }
        },
        "usage": {
          "input_tokens": 122,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0724,
            "action": {
              "act_probability": 1
            },
            "choice": "portrait",
            "probabilities": {
              "portrait": 0.5268,
              "landscape": 0.2492,
              "balanced": 0.224
            }
          }
        },
        "usage": {
          "input_tokens": 122,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "portrait",
        "score_absolute_error": null,
        "confidence": 0.0724,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "notification-delivery-01",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "digest",
        "email"
      ],
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "elapsed_ms": 10.84470801288262,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "in_app": 0.2397,
              "digest": 0.2275,
              "email": 0.5328
            },
            "confidence": 0.0764
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0764,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "in_app": 0.2397,
              "digest": 0.2275,
              "email": 0.5328
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.0764,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "notification-delivery-02",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "digest",
        "email"
      ],
      "expected": {
        "type": "choice",
        "choice": "in_app"
      },
      "elapsed_ms": 11.191457975655794,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "in_app": 0.2571,
              "digest": 0.2312,
              "email": 0.5117
            },
            "confidence": 0.0619
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0619,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "in_app": 0.2571,
              "digest": 0.2312,
              "email": 0.5117
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.0619,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "notification-delivery-03",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "email",
        "digest",
        "in_app"
      ],
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "elapsed_ms": 11.335499992128462,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "email": 0.4489,
              "digest": 0.3116,
              "in_app": 0.2395
            },
            "confidence": 0.0304
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0304,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "email": 0.4489,
              "digest": 0.3116,
              "in_app": 0.2395
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.0304,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "notification-delivery-04",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "in_app",
        "email"
      ],
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "elapsed_ms": 11.309291992802173,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "digest": 0.1798,
              "in_app": 0.2749,
              "email": 0.5453
            },
            "confidence": 0.0951
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0951,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "digest": 0.1798,
              "in_app": 0.2749,
              "email": 0.5453
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.0951,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "notification-delivery-05",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "digest",
        "email"
      ],
      "expected": {
        "type": "choice",
        "choice": "in_app"
      },
      "elapsed_ms": 11.077042028773576,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "in_app": 0.2955,
              "digest": 0.2817,
              "email": 0.4228
            },
            "confidence": 0.0159
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0159,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "in_app": 0.2955,
              "digest": 0.2817,
              "email": 0.4228
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.0159,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "notification-delivery-06",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "email",
        "digest",
        "in_app"
      ],
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "elapsed_ms": 11.137499997857958,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "email": 0.5312,
              "digest": 0.226,
              "in_app": 0.2428
            },
            "confidence": 0.0753
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0753,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "email": 0.5312,
              "digest": 0.226,
              "in_app": 0.2428
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.0753,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "notification-delivery-07",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "in_app",
        "email"
      ],
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "elapsed_ms": 11.28937501925975,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "digest",
            "probabilities": {
              "digest": 0.4598,
              "in_app": 0.2653,
              "email": 0.2749
            },
            "confidence": 0.0313
          }
        },
        "usage": {
          "input_tokens": 154,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0313,
            "action": {
              "act_probability": 1
            },
            "choice": "digest",
            "probabilities": {
              "digest": 0.4598,
              "in_app": 0.2653,
              "email": 0.2749
            }
          }
        },
        "usage": {
          "input_tokens": 154,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "digest",
        "score_absolute_error": null,
        "confidence": 0.0313,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "notification-delivery-08",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "email",
        "in_app"
      ],
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "elapsed_ms": 11.358332994859666,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "digest": 0.2652,
              "email": 0.3729,
              "in_app": 0.3619
            },
            "confidence": 0.01
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.01,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "digest": 0.2652,
              "email": 0.3729,
              "in_app": 0.3619
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.01,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "release-note-group-01",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "fix",
        "breaking",
        "addition"
      ],
      "expected": {
        "type": "choice",
        "choice": "breaking"
      },
      "elapsed_ms": 11.315999960061163,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "fix": 0.3008,
              "breaking": 0.2342,
              "addition": 0.465
            },
            "confidence": 0.0375
          }
        },
        "usage": {
          "input_tokens": 148,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0375,
            "action": {
              "act_probability": 1
            },
            "choice": "addition",
            "probabilities": {
              "fix": 0.3008,
              "breaking": 0.2342,
              "addition": 0.465
            }
          }
        },
        "usage": {
          "input_tokens": 148,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.0375,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "release-note-group-02",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "breaking",
        "fix"
      ],
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "elapsed_ms": 11.31199998781085,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "addition": 0.128,
              "breaking": 0.1385,
              "fix": 0.7335
            },
            "confidence": 0.3044
          }
        },
        "usage": {
          "input_tokens": 140,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.3044,
            "action": {
              "act_probability": 1
            },
            "choice": "fix",
            "probabilities": {
              "addition": 0.128,
              "breaking": 0.1385,
              "fix": 0.7335
            }
          }
        },
        "usage": {
          "input_tokens": 140,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.3044,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "release-note-group-03",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "breaking",
        "fix"
      ],
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "elapsed_ms": 11.3535420387052,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "addition": 0.4816,
              "breaking": 0.162,
              "fix": 0.3564
            },
            "confidence": 0.0766
          }
        },
        "usage": {
          "input_tokens": 137,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0766,
            "action": {
              "act_probability": 1
            },
            "choice": "addition",
            "probabilities": {
              "addition": 0.4816,
              "breaking": 0.162,
              "fix": 0.3564
            }
          }
        },
        "usage": {
          "input_tokens": 137,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.0766,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "release-note-group-04",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "fix",
        "breaking",
        "addition"
      ],
      "expected": {
        "type": "choice",
        "choice": "breaking"
      },
      "elapsed_ms": 11.57833298202604,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "fix": 0.4896,
              "breaking": 0.2786,
              "addition": 0.2319
            },
            "confidence": 0.0492
          }
        },
        "usage": {
          "input_tokens": 145,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0492,
            "action": {
              "act_probability": 1
            },
            "choice": "fix",
            "probabilities": {
              "fix": 0.4896,
              "breaking": 0.2786,
              "addition": 0.2319
            }
          }
        },
        "usage": {
          "input_tokens": 145,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.0492,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "release-note-group-05",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "addition",
        "fix"
      ],
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "elapsed_ms": 11.378499970305711,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "breaking": 0.1838,
              "addition": 0.1464,
              "fix": 0.6698
            },
            "confidence": 0.2162
          }
        },
        "usage": {
          "input_tokens": 143,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.2162,
            "action": {
              "act_probability": 1
            },
            "choice": "fix",
            "probabilities": {
              "breaking": 0.1838,
              "addition": 0.1464,
              "fix": 0.6698
            }
          }
        },
        "usage": {
          "input_tokens": 143,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.2162,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "release-note-group-06",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "fix",
        "breaking"
      ],
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "elapsed_ms": 11.68020797194913,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "addition": 0.4043,
              "fix": 0.3457,
              "breaking": 0.25
            },
            "confidence": 0.017
          }
        },
        "usage": {
          "input_tokens": 148,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.017,
            "action": {
              "act_probability": 1
            },
            "choice": "addition",
            "probabilities": {
              "addition": 0.4043,
              "fix": 0.3457,
              "breaking": 0.25
            }
          }
        },
        "usage": {
          "input_tokens": 148,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.017,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "release-note-group-07",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "fix",
        "addition"
      ],
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "elapsed_ms": 11.594415991567075,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "breaking": 0.1442,
              "fix": 0.6103,
              "addition": 0.2455
            },
            "confidence": 0.1576
          }
        },
        "usage": {
          "input_tokens": 142,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.1576,
            "action": {
              "act_probability": 1
            },
            "choice": "fix",
            "probabilities": {
              "breaking": 0.1442,
              "fix": 0.6103,
              "addition": 0.2455
            }
          }
        },
        "usage": {
          "input_tokens": 142,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.1576,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "release-note-group-08",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "fix",
        "addition"
      ],
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "elapsed_ms": 11.629332962911576,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "breaking": 0.289,
              "fix": 0.4007,
              "addition": 0.3103
            },
            "confidence": 0.0094
          }
        },
        "usage": {
          "input_tokens": 138,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0094,
            "action": {
              "act_probability": 1
            },
            "choice": "fix",
            "probabilities": {
              "breaking": 0.289,
              "fix": 0.4007,
              "addition": 0.3103
            }
          }
        },
        "usage": {
          "input_tokens": 138,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.0094,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "demo-discount-01",
      "family": "demo-discount",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": true
      },
      "elapsed_ms": 12.087833951227367,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.6902
          }
        },
        "usage": {
          "input_tokens": 146,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.6902,
            "action": {
              "act_probability": 1
            },
            "noul": 0.6902
          }
        },
        "usage": {
          "input_tokens": 146,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.6902,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "demo-discount-02",
      "family": "demo-discount",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": false
      },
      "elapsed_ms": 11.503665999043733,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.2415
          }
        },
        "usage": {
          "input_tokens": 146,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.7585,
            "action": {
              "act_probability": 1
            },
            "noul": 0.2415
          }
        },
        "usage": {
          "input_tokens": 146,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": false,
        "score_absolute_error": null,
        "confidence": 0.7585,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "demo-discount-03",
      "family": "demo-discount",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": true
      },
      "elapsed_ms": 11.453750019427389,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.2825
          }
        },
        "usage": {
          "input_tokens": 146,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.7175,
            "action": {
              "act_probability": 1
            },
            "noul": 0.2825
          }
        },
        "usage": {
          "input_tokens": 146,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": false,
        "score_absolute_error": null,
        "confidence": 0.7175,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "demo-discount-04",
      "family": "demo-discount",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": false
      },
      "elapsed_ms": 11.744415969587862,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.3083
          }
        },
        "usage": {
          "input_tokens": 146,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.6917,
            "action": {
              "act_probability": 1
            },
            "noul": 0.3083
          }
        },
        "usage": {
          "input_tokens": 146,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": false,
        "score_absolute_error": null,
        "confidence": 0.6917,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "demo-discount-05",
      "family": "demo-discount",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": true
      },
      "elapsed_ms": 11.498040985316038,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.5631
          }
        },
        "usage": {
          "input_tokens": 148,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.5631,
            "action": {
              "act_probability": 1
            },
            "noul": 0.5631
          }
        },
        "usage": {
          "input_tokens": 148,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.5631,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "demo-discount-06",
      "family": "demo-discount",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": false
      },
      "elapsed_ms": 11.830582981929183,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.1742
          }
        },
        "usage": {
          "input_tokens": 146,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.8258,
            "action": {
              "act_probability": 1
            },
            "noul": 0.1742
          }
        },
        "usage": {
          "input_tokens": 146,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": false,
        "score_absolute_error": null,
        "confidence": 0.8258,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "demo-discount-07",
      "family": "demo-discount",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": true
      },
      "elapsed_ms": 11.89466705545783,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.2773
          }
        },
        "usage": {
          "input_tokens": 146,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.7227,
            "action": {
              "act_probability": 1
            },
            "noul": 0.2773
          }
        },
        "usage": {
          "input_tokens": 146,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": false,
        "score_absolute_error": null,
        "confidence": 0.7227,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "demo-discount-08",
      "family": "demo-discount",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": false
      },
      "elapsed_ms": 11.74045802326873,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.2383
          }
        },
        "usage": {
          "input_tokens": 146,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.7617,
            "action": {
              "act_probability": 1
            },
            "noul": 0.2383
          }
        },
        "usage": {
          "input_tokens": 146,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": false,
        "score_absolute_error": null,
        "confidence": 0.7617,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "overnight-counter-01",
      "family": "overnight-counter",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": true
      },
      "elapsed_ms": 11.419499991461635,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.6393
          }
        },
        "usage": {
          "input_tokens": 112,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.6393,
            "action": {
              "act_probability": 1
            },
            "noul": 0.6393
          }
        },
        "usage": {
          "input_tokens": 112,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.6393,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "overnight-counter-02",
      "family": "overnight-counter",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": false
      },
      "elapsed_ms": 11.517209000885487,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.5042
          }
        },
        "usage": {
          "input_tokens": 112,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.5042,
            "action": {
              "act_probability": 1
            },
            "noul": 0.5042
          }
        },
        "usage": {
          "input_tokens": 112,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.5042,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "overnight-counter-03",
      "family": "overnight-counter",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": true
      },
      "elapsed_ms": 11.301249964162707,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.3613
          }
        },
        "usage": {
          "input_tokens": 112,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.6387,
            "action": {
              "act_probability": 1
            },
            "noul": 0.3613
          }
        },
        "usage": {
          "input_tokens": 112,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": false,
        "score_absolute_error": null,
        "confidence": 0.6387,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "overnight-counter-04",
      "family": "overnight-counter",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": false
      },
      "elapsed_ms": 11.619500000961125,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.5841
          }
        },
        "usage": {
          "input_tokens": 112,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.5841,
            "action": {
              "act_probability": 1
            },
            "noul": 0.5841
          }
        },
        "usage": {
          "input_tokens": 112,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.5841,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "overnight-counter-05",
      "family": "overnight-counter",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": true
      },
      "elapsed_ms": 11.546249967068434,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.4752
          }
        },
        "usage": {
          "input_tokens": 112,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.5248,
            "action": {
              "act_probability": 1
            },
            "noul": 0.4752
          }
        },
        "usage": {
          "input_tokens": 112,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": false,
        "score_absolute_error": null,
        "confidence": 0.5247999999999999,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "overnight-counter-06",
      "family": "overnight-counter",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": false
      },
      "elapsed_ms": 11.417875008191913,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.5772
          }
        },
        "usage": {
          "input_tokens": 112,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.5772,
            "action": {
              "act_probability": 1
            },
            "noul": 0.5772
          }
        },
        "usage": {
          "input_tokens": 112,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.5772,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "overnight-counter-07",
      "family": "overnight-counter",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": true
      },
      "elapsed_ms": 12.072957993950695,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.5944
          }
        },
        "usage": {
          "input_tokens": 112,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.5944,
            "action": {
              "act_probability": 1
            },
            "noul": 0.5944
          }
        },
        "usage": {
          "input_tokens": 112,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.5944,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "overnight-counter-08",
      "family": "overnight-counter",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": false
      },
      "elapsed_ms": 11.779958964325488,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.3919
          }
        },
        "usage": {
          "input_tokens": 112,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.6081,
            "action": {
              "act_probability": 1
            },
            "noul": 0.3919
          }
        },
        "usage": {
          "input_tokens": 112,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": false,
        "score_absolute_error": null,
        "confidence": 0.6081,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "tag-filter-01",
      "family": "tag-filter",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": true
      },
      "elapsed_ms": 11.958874994888902,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.6709
          }
        },
        "usage": {
          "input_tokens": 126,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.6709,
            "action": {
              "act_probability": 1
            },
            "noul": 0.6709
          }
        },
        "usage": {
          "input_tokens": 126,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.6709,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "tag-filter-02",
      "family": "tag-filter",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": false
      },
      "elapsed_ms": 12.03183300094679,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.4716
          }
        },
        "usage": {
          "input_tokens": 121,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.5284,
            "action": {
              "act_probability": 1
            },
            "noul": 0.4716
          }
        },
        "usage": {
          "input_tokens": 121,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": false,
        "score_absolute_error": null,
        "confidence": 0.5284,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "tag-filter-03",
      "family": "tag-filter",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": true
      },
      "elapsed_ms": 12.073500023689121,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.7458
          }
        },
        "usage": {
          "input_tokens": 127,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.7458,
            "action": {
              "act_probability": 1
            },
            "noul": 0.7458
          }
        },
        "usage": {
          "input_tokens": 127,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.7458,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "tag-filter-04",
      "family": "tag-filter",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": false
      },
      "elapsed_ms": 12.492042034864426,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.7411
          }
        },
        "usage": {
          "input_tokens": 126,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.7411,
            "action": {
              "act_probability": 1
            },
            "noul": 0.7411
          }
        },
        "usage": {
          "input_tokens": 126,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.7411,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "tag-filter-05",
      "family": "tag-filter",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": true
      },
      "elapsed_ms": 14.040042005944997,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.4396
          }
        },
        "usage": {
          "input_tokens": 116,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.5604,
            "action": {
              "act_probability": 1
            },
            "noul": 0.4396
          }
        },
        "usage": {
          "input_tokens": 116,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": false,
        "score_absolute_error": null,
        "confidence": 0.5604,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "tag-filter-06",
      "family": "tag-filter",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": false
      },
      "elapsed_ms": 13.296916033141315,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.7311
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.7311,
            "action": {
              "act_probability": 1
            },
            "noul": 0.7311
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.7311,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "tag-filter-07",
      "family": "tag-filter",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": true
      },
      "elapsed_ms": 12.752500013448298,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.7006
          }
        },
        "usage": {
          "input_tokens": 127,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.7006,
            "action": {
              "act_probability": 1
            },
            "noul": 0.7006
          }
        },
        "usage": {
          "input_tokens": 127,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.7006,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "tag-filter-08",
      "family": "tag-filter",
      "type": "noul",
      "order": null,
      "expected": {
        "type": "noul",
        "noul": false
      },
      "elapsed_ms": 12.87616603076458,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "noul",
            "noul": 0.5555
          }
        },
        "usage": {
          "input_tokens": 134,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "noul",
            "confidence": 0.5555,
            "action": {
              "act_probability": 1
            },
            "noul": 0.5555
          }
        },
        "usage": {
          "input_tokens": 134,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": true,
        "score_absolute_error": null,
        "confidence": 0.5555,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "mini-game-points-01",
      "family": "mini-game-points",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 0
      },
      "elapsed_ms": 12.610457953996956,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 0.3268,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.815,
              "1": 0.098,
              "2": 0.0322,
              "3": 0.0548
            },
            "confidence": 0.5209
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.5209,
            "action": {
              "act_probability": 1
            },
            "score": 0.3268,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.815,
              "1": 0.098,
              "2": 0.0322,
              "3": 0.0548
            }
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": 0,
        "score_absolute_error": 0.3268,
        "confidence": 0.5209,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "mini-game-points-02",
      "family": "mini-game-points",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 1
      },
      "elapsed_ms": 12.538790993858129,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 0.419,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.7116,
              "1": 0.2054,
              "2": 0.0355,
              "3": 0.0475
            },
            "confidence": 0.4009
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.4009,
            "action": {
              "act_probability": 1
            },
            "score": 0.419,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.7116,
              "1": 0.2054,
              "2": 0.0355,
              "3": 0.0475
            }
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": 0,
        "score_absolute_error": 0.581,
        "confidence": 0.4009,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "mini-game-points-03",
      "family": "mini-game-points",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 2
      },
      "elapsed_ms": 13.226040988229215,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 0.7044,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.5313,
              "1": 0.3098,
              "2": 0.082,
              "3": 0.0768
            },
            "confidence": 0.2055
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.2055,
            "action": {
              "act_probability": 1
            },
            "score": 0.7044,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.5313,
              "1": 0.3098,
              "2": 0.082,
              "3": 0.0768
            }
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": 0,
        "score_absolute_error": 1.2955999999999999,
        "confidence": 0.2055,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "mini-game-points-04",
      "family": "mini-game-points",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 3
      },
      "elapsed_ms": 13.14737496431917,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 0.8369,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.362,
              "1": 0.5025,
              "2": 0.0721,
              "3": 0.0634
            },
            "confidence": 0.2223
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.2223,
            "action": {
              "act_probability": 1
            },
            "score": 0.8369,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.362,
              "1": 0.5025,
              "2": 0.0721,
              "3": 0.0634
            }
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": 1,
        "score_absolute_error": 2.1631,
        "confidence": 0.2223,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "mini-game-points-05",
      "family": "mini-game-points",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 0
      },
      "elapsed_ms": 13.410334009677172,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.298,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.3775,
              "1": 0.2041,
              "2": 0.1611,
              "3": 0.2572
            },
            "confidence": 0.0366
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.0366,
            "action": {
              "act_probability": 1
            },
            "score": 1.298,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.3775,
              "1": 0.2041,
              "2": 0.1611,
              "3": 0.2572
            }
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": 0,
        "score_absolute_error": 1.298,
        "confidence": 0.0366,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "mini-game-points-06",
      "family": "mini-game-points",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 1
      },
      "elapsed_ms": 13.640541990753263,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.4631,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.1876,
              "1": 0.3303,
              "2": 0.3135,
              "3": 0.1686
            },
            "confidence": 0.0308
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.0308,
            "action": {
              "act_probability": 1
            },
            "score": 1.4631,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.1876,
              "1": 0.3303,
              "2": 0.3135,
              "3": 0.1686
            }
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": 1,
        "score_absolute_error": 0.46310000000000007,
        "confidence": 0.0308,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "mini-game-points-07",
      "family": "mini-game-points",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 2
      },
      "elapsed_ms": 14.393249992281199,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 0.8853,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.5777,
              "1": 0.0993,
              "2": 0.183,
              "3": 0.14
            },
            "confidence": 0.1832
          }
        },
        "usage": {
          "input_tokens": 135,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.1832,
            "action": {
              "act_probability": 1
            },
            "score": 0.8853,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.5777,
              "1": 0.0993,
              "2": 0.183,
              "3": 0.14
            }
          }
        },
        "usage": {
          "input_tokens": 135,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": 0,
        "score_absolute_error": 1.1147,
        "confidence": 0.1832,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "mini-game-points-08",
      "family": "mini-game-points",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 3
      },
      "elapsed_ms": 13.682082993909717,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.7563,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.1714,
              "1": 0.1407,
              "2": 0.448,
              "3": 0.2399
            },
            "confidence": 0.0763
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.0763,
            "action": {
              "act_probability": 1
            },
            "score": 1.7563,
            "legend": {
              "0": "Final clamped points equal 0",
              "1": "Final clamped points equal 1",
              "2": "Final clamped points equal 2",
              "3": "Final clamped points equal 3"
            },
            "probabilities": {
              "0": 0.1714,
              "1": 0.1407,
              "2": 0.448,
              "3": 0.2399
            }
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": 2,
        "score_absolute_error": 1.2437,
        "confidence": 0.0763,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "display-detail-01",
      "family": "display-detail",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 0
      },
      "elapsed_ms": 13.676624977961183,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.0425,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.3503,
              "1": 0.3572,
              "2": 0.1923,
              "3": 0.1002
            },
            "confidence": 0.0747
          }
        },
        "usage": {
          "input_tokens": 135,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.0747,
            "action": {
              "act_probability": 1
            },
            "score": 1.0425,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.3503,
              "1": 0.3572,
              "2": 0.1923,
              "3": 0.1002
            }
          }
        },
        "usage": {
          "input_tokens": 135,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": 1,
        "score_absolute_error": 1.0425,
        "confidence": 0.0747,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "display-detail-02",
      "family": "display-detail",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 1
      },
      "elapsed_ms": 14.013790991157293,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 0.9566,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.3638,
              "1": 0.3967,
              "2": 0.1588,
              "3": 0.0808
            },
            "confidence": 0.1127
          }
        },
        "usage": {
          "input_tokens": 135,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.1127,
            "action": {
              "act_probability": 1
            },
            "score": 0.9566,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.3638,
              "1": 0.3967,
              "2": 0.1588,
              "3": 0.0808
            }
          }
        },
        "usage": {
          "input_tokens": 135,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": 1,
        "score_absolute_error": 0.043399999999999994,
        "confidence": 0.1127,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "display-detail-03",
      "family": "display-detail",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 2
      },
      "elapsed_ms": 13.825332978740335,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.4239,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.223,
              "1": 0.3111,
              "2": 0.285,
              "3": 0.1809
            },
            "confidence": 0.0154
          }
        },
        "usage": {
          "input_tokens": 136,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.0154,
            "action": {
              "act_probability": 1
            },
            "score": 1.4239,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.223,
              "1": 0.3111,
              "2": 0.285,
              "3": 0.1809
            }
          }
        },
        "usage": {
          "input_tokens": 136,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": 1,
        "score_absolute_error": 0.5761000000000001,
        "confidence": 0.0154,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "display-detail-04",
      "family": "display-detail",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 3
      },
      "elapsed_ms": 13.841499981936067,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.5758,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.2245,
              "1": 0.2544,
              "2": 0.2419,
              "3": 0.2792
            },
            "confidence": 0.0023
          }
        },
        "usage": {
          "input_tokens": 135,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.0023,
            "action": {
              "act_probability": 1
            },
            "score": 1.5758,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.2245,
              "1": 0.2544,
              "2": 0.2419,
              "3": 0.2792
            }
          }
        },
        "usage": {
          "input_tokens": 135,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": 3,
        "score_absolute_error": 1.4242,
        "confidence": 0.0023,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "display-detail-05",
      "family": "display-detail",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 0
      },
      "elapsed_ms": 14.055375009775162,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.0254,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.3443,
              "1": 0.3817,
              "2": 0.1784,
              "3": 0.0956
            },
            "confidence": 0.0863
          }
        },
        "usage": {
          "input_tokens": 135,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.0863,
            "action": {
              "act_probability": 1
            },
            "score": 1.0254,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.3443,
              "1": 0.3817,
              "2": 0.1784,
              "3": 0.0956
            }
          }
        },
        "usage": {
          "input_tokens": 135,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": 1,
        "score_absolute_error": 1.0254,
        "confidence": 0.0863,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "display-detail-06",
      "family": "display-detail",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 1
      },
      "elapsed_ms": 13.761667010840029,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.1145,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.2822,
              "1": 0.3923,
              "2": 0.2541,
              "3": 0.0713
            },
            "confidence": 0.0907
          }
        },
        "usage": {
          "input_tokens": 135,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.0907,
            "action": {
              "act_probability": 1
            },
            "score": 1.1145,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.2822,
              "1": 0.3923,
              "2": 0.2541,
              "3": 0.0713
            }
          }
        },
        "usage": {
          "input_tokens": 135,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": 1,
        "score_absolute_error": 0.11450000000000005,
        "confidence": 0.0907,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "display-detail-07",
      "family": "display-detail",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 2
      },
      "elapsed_ms": 14.660749991890043,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.4711,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.2429,
              "1": 0.2743,
              "2": 0.2516,
              "3": 0.2312
            },
            "confidence": 0.0014
          }
        },
        "usage": {
          "input_tokens": 135,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.0014,
            "action": {
              "act_probability": 1
            },
            "score": 1.4711,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.2429,
              "1": 0.2743,
              "2": 0.2516,
              "3": 0.2312
            }
          }
        },
        "usage": {
          "input_tokens": 135,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": 1,
        "score_absolute_error": 0.5288999999999999,
        "confidence": 0.0014,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "display-detail-08",
      "family": "display-detail",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 3
      },
      "elapsed_ms": 14.263041026424617,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.3493,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.2514,
              "1": 0.3186,
              "2": 0.2591,
              "3": 0.1708
            },
            "confidence": 0.0166
          }
        },
        "usage": {
          "input_tokens": 145,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.0166,
            "action": {
              "act_probability": 1
            },
            "score": 1.3493,
            "legend": {
              "0": "Final detail level 0",
              "1": "Final detail level 1",
              "2": "Final detail level 2",
              "3": "Final detail level 3"
            },
            "probabilities": {
              "0": 0.2514,
              "1": 0.3186,
              "2": 0.2591,
              "3": 0.1708
            }
          }
        },
        "usage": {
          "input_tokens": 145,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": 1,
        "score_absolute_error": 1.6507,
        "confidence": 0.0166,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "glossary-strength-01",
      "family": "glossary-strength",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 0
      },
      "elapsed_ms": 14.53491603024304,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.752,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.1063,
              "1": 0.2963,
              "2": 0.3367,
              "3": 0.2608
            },
            "confidence": 0.051
          }
        },
        "usage": {
          "input_tokens": 155,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.051,
            "action": {
              "act_probability": 1
            },
            "score": 1.752,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.1063,
              "1": 0.2963,
              "2": 0.3367,
              "3": 0.2608
            }
          }
        },
        "usage": {
          "input_tokens": 155,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": 2,
        "score_absolute_error": 1.752,
        "confidence": 0.051,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "glossary-strength-02",
      "family": "glossary-strength",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 1
      },
      "elapsed_ms": 14.900041976943612,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.8362,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.1039,
              "1": 0.2417,
              "2": 0.3686,
              "3": 0.2858
            },
            "confidence": 0.0591
          }
        },
        "usage": {
          "input_tokens": 161,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.0591,
            "action": {
              "act_probability": 1
            },
            "score": 1.8362,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.1039,
              "1": 0.2417,
              "2": 0.3686,
              "3": 0.2858
            }
          }
        },
        "usage": {
          "input_tokens": 161,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": 2,
        "score_absolute_error": 0.8362,
        "confidence": 0.0591,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "glossary-strength-03",
      "family": "glossary-strength",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 2
      },
      "elapsed_ms": 14.571124978829175,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.9079,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.0578,
              "1": 0.2638,
              "2": 0.3912,
              "3": 0.2872
            },
            "confidence": 0.1043
          }
        },
        "usage": {
          "input_tokens": 154,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.1043,
            "action": {
              "act_probability": 1
            },
            "score": 1.9079,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.0578,
              "1": 0.2638,
              "2": 0.3912,
              "3": 0.2872
            }
          }
        },
        "usage": {
          "input_tokens": 154,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": 2,
        "score_absolute_error": 0.09210000000000007,
        "confidence": 0.1043,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "glossary-strength-04",
      "family": "glossary-strength",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 3
      },
      "elapsed_ms": 16.19316596770659,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.965,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.052,
              "1": 0.2417,
              "2": 0.3957,
              "3": 0.3106
            },
            "confidence": 0.1149
          }
        },
        "usage": {
          "input_tokens": 163,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.1149,
            "action": {
              "act_probability": 1
            },
            "score": 1.965,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.052,
              "1": 0.2417,
              "2": 0.3957,
              "3": 0.3106
            }
          }
        },
        "usage": {
          "input_tokens": 163,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": 2,
        "score_absolute_error": 1.035,
        "confidence": 0.1149,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "glossary-strength-05",
      "family": "glossary-strength",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 0
      },
      "elapsed_ms": 15.40162501623854,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.9368,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.0727,
              "1": 0.2619,
              "2": 0.3211,
              "3": 0.3442
            },
            "confidence": 0.0814
          }
        },
        "usage": {
          "input_tokens": 164,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.0814,
            "action": {
              "act_probability": 1
            },
            "score": 1.9368,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.0727,
              "1": 0.2619,
              "2": 0.3211,
              "3": 0.3442
            }
          }
        },
        "usage": {
          "input_tokens": 164,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": 3,
        "score_absolute_error": 1.9368,
        "confidence": 0.0814,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "glossary-strength-06",
      "family": "glossary-strength",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 1
      },
      "elapsed_ms": 15.607583976816386,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.8498,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.093,
              "1": 0.2742,
              "2": 0.3228,
              "3": 0.31
            },
            "confidence": 0.0596
          }
        },
        "usage": {
          "input_tokens": 165,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.0596,
            "action": {
              "act_probability": 1
            },
            "score": 1.8498,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.093,
              "1": 0.2742,
              "2": 0.3228,
              "3": 0.31
            }
          }
        },
        "usage": {
          "input_tokens": 165,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": 2,
        "score_absolute_error": 0.8498000000000001,
        "confidence": 0.0596,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "glossary-strength-07",
      "family": "glossary-strength",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 2
      },
      "elapsed_ms": 16.181209008209407,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 1.8364,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.0922,
              "1": 0.2656,
              "2": 0.3556,
              "3": 0.2865
            },
            "confidence": 0.0639
          }
        },
        "usage": {
          "input_tokens": 165,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.0639,
            "action": {
              "act_probability": 1
            },
            "score": 1.8364,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.0922,
              "1": 0.2656,
              "2": 0.3556,
              "3": 0.2865
            }
          }
        },
        "usage": {
          "input_tokens": 165,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": 2,
        "score_absolute_error": 0.16359999999999997,
        "confidence": 0.0639,
        "tie": false
      }
    },
    {
      "phase": "measured",
      "repetition": 2,
      "case_id": "glossary-strength-08",
      "family": "glossary-strength",
      "type": "score",
      "order": null,
      "expected": {
        "type": "score",
        "level": 3
      },
      "elapsed_ms": 15.61062497785315,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "score",
            "score": 2.119,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.0191,
              "1": 0.1896,
              "2": 0.4445,
              "3": 0.3468
            },
            "confidence": 0.1932
          }
        },
        "usage": {
          "input_tokens": 166,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "score",
            "confidence": 0.1932,
            "action": {
              "act_probability": 1
            },
            "score": 2.119,
            "legend": {
              "0": "Standalone orbit is absent",
              "1": "Orbit present; definition phrase absent",
              "2": "Orbit and definition present; example phrase absent",
              "3": "Orbit, definition phrase and example phrase all present"
            },
            "probabilities": {
              "0": 0.0191,
              "1": 0.1896,
              "2": 0.4445,
              "3": 0.3468
            }
          }
        },
        "usage": {
          "input_tokens": 166,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": 2,
        "score_absolute_error": 0.8809999999999998,
        "confidence": 0.1932,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 0,
      "case_id": "image-layout-01",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "landscape",
        "balanced",
        "portrait"
      ],
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "elapsed_ms": 15.765333024319261,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "landscape": 0.3814,
              "balanced": 0.3311,
              "portrait": 0.2875
            },
            "confidence": 0.006
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.006,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "landscape": 0.3814,
              "balanced": 0.3311,
              "portrait": 0.2875
            }
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.006,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 1,
      "case_id": "image-layout-01",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "landscape",
        "portrait",
        "balanced"
      ],
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "elapsed_ms": 15.86483302526176,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "landscape": 0.3608,
              "portrait": 0.3153,
              "balanced": 0.3239
            },
            "confidence": 0.0016
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0016,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "landscape": 0.3608,
              "portrait": 0.3153,
              "balanced": 0.3239
            }
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.0016,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 2,
      "case_id": "image-layout-01",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "landscape",
        "portrait"
      ],
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "elapsed_ms": 15.350000001490116,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.2969,
              "landscape": 0.3817,
              "portrait": 0.3213
            },
            "confidence": 0.0051
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0051,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.2969,
              "landscape": 0.3817,
              "portrait": 0.3213
            }
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.0051,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 3,
      "case_id": "image-layout-01",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "portrait",
        "landscape"
      ],
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "elapsed_ms": 15.33824997022748,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.3051,
              "portrait": 0.2694,
              "landscape": 0.4255
            },
            "confidence": 0.0177
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0177,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.3051,
              "portrait": 0.2694,
              "landscape": 0.4255
            }
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.0177,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 4,
      "case_id": "image-layout-01",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "portrait",
        "landscape",
        "balanced"
      ],
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "elapsed_ms": 15.316875011194497,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "portrait": 0.284,
              "landscape": 0.3855,
              "balanced": 0.3306
            },
            "confidence": 0.0071
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0071,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "portrait": 0.284,
              "landscape": 0.3855,
              "balanced": 0.3306
            }
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.0071,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 5,
      "case_id": "image-layout-01",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "portrait",
        "balanced",
        "landscape"
      ],
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "elapsed_ms": 16.04600000428036,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "portrait": 0.2732,
              "balanced": 0.3267,
              "landscape": 0.4001
            },
            "confidence": 0.011
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.011,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "portrait": 0.2732,
              "balanced": 0.3267,
              "landscape": 0.4001
            }
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.011,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 0,
      "case_id": "image-layout-02",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "portrait",
        "landscape"
      ],
      "expected": {
        "type": "choice",
        "choice": "portrait"
      },
      "elapsed_ms": 15.713415981736034,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.3047,
              "portrait": 0.2637,
              "landscape": 0.4316
            },
            "confidence": 0.0203
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0203,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.3047,
              "portrait": 0.2637,
              "landscape": 0.4316
            }
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.0203,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 1,
      "case_id": "image-layout-02",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "landscape",
        "portrait"
      ],
      "expected": {
        "type": "choice",
        "choice": "portrait"
      },
      "elapsed_ms": 15.771041973493993,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.3084,
              "landscape": 0.3723,
              "portrait": 0.3192
            },
            "confidence": 0.0031
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0031,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.3084,
              "landscape": 0.3723,
              "portrait": 0.3192
            }
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.0031,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 2,
      "case_id": "image-layout-02",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "portrait",
        "balanced",
        "landscape"
      ],
      "expected": {
        "type": "choice",
        "choice": "portrait"
      },
      "elapsed_ms": 16.932709026150405,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "portrait": 0.2591,
              "balanced": 0.3267,
              "landscape": 0.4142
            },
            "confidence": 0.0165
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0165,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "portrait": 0.2591,
              "balanced": 0.3267,
              "landscape": 0.4142
            }
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.0165,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 3,
      "case_id": "image-layout-02",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "portrait",
        "landscape",
        "balanced"
      ],
      "expected": {
        "type": "choice",
        "choice": "portrait"
      },
      "elapsed_ms": 16.037499997764826,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "portrait": 0.263,
              "landscape": 0.4135,
              "balanced": 0.3235
            },
            "confidence": 0.0156
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0156,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "portrait": 0.263,
              "landscape": 0.4135,
              "balanced": 0.3235
            }
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.0156,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 4,
      "case_id": "image-layout-02",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "landscape",
        "balanced",
        "portrait"
      ],
      "expected": {
        "type": "choice",
        "choice": "portrait"
      },
      "elapsed_ms": 16.34420797927305,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "landscape": 0.374,
              "balanced": 0.3336,
              "portrait": 0.2924
            },
            "confidence": 0.0046
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0046,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "landscape": 0.374,
              "balanced": 0.3336,
              "portrait": 0.2924
            }
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.0046,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 5,
      "case_id": "image-layout-02",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "landscape",
        "portrait",
        "balanced"
      ],
      "expected": {
        "type": "choice",
        "choice": "portrait"
      },
      "elapsed_ms": 16.983167035505176,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "landscape": 0.3428,
              "portrait": 0.3242,
              "balanced": 0.3329
            },
            "confidence": 0.0002
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0002,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "landscape": 0.3428,
              "portrait": 0.3242,
              "balanced": 0.3329
            }
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.0002,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 0,
      "case_id": "image-layout-03",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "landscape",
        "portrait",
        "balanced"
      ],
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "elapsed_ms": 16.620040987618268,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "portrait",
            "probabilities": {
              "landscape": 0.3427,
              "portrait": 0.3492,
              "balanced": 0.3081
            },
            "confidence": 0.0014
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0014,
            "action": {
              "act_probability": 1
            },
            "choice": "portrait",
            "probabilities": {
              "landscape": 0.3427,
              "portrait": 0.3492,
              "balanced": 0.3081
            }
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "portrait",
        "score_absolute_error": null,
        "confidence": 0.0014,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 1,
      "case_id": "image-layout-03",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "landscape",
        "balanced",
        "portrait"
      ],
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "elapsed_ms": 16.886249999515712,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "landscape": 0.3774,
              "balanced": 0.3355,
              "portrait": 0.2871
            },
            "confidence": 0.0056
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0056,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "landscape": 0.3774,
              "balanced": 0.3355,
              "portrait": 0.2871
            }
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.0056,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 2,
      "case_id": "image-layout-03",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "portrait",
        "landscape",
        "balanced"
      ],
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "elapsed_ms": 17.031666007824242,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "portrait": 0.3278,
              "landscape": 0.3734,
              "balanced": 0.2988
            },
            "confidence": 0.0038
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0038,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "portrait": 0.3278,
              "landscape": 0.3734,
              "balanced": 0.2988
            }
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.0038,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 3,
      "case_id": "image-layout-03",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "portrait",
        "balanced",
        "landscape"
      ],
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "elapsed_ms": 16.90966699970886,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "portrait": 0.3088,
              "balanced": 0.3175,
              "landscape": 0.3737
            },
            "confidence": 0.0033
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0033,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "portrait": 0.3088,
              "balanced": 0.3175,
              "landscape": 0.3737
            }
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.0033,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 4,
      "case_id": "image-layout-03",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "landscape",
        "portrait"
      ],
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "elapsed_ms": 17.212250037118793,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.3169,
              "landscape": 0.3546,
              "portrait": 0.3285
            },
            "confidence": 0.001
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.001,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.3169,
              "landscape": 0.3546,
              "portrait": 0.3285
            }
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.001,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 5,
      "case_id": "image-layout-03",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "portrait",
        "landscape"
      ],
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "elapsed_ms": 17.68062502378598,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.3202,
              "portrait": 0.2857,
              "landscape": 0.3941
            },
            "confidence": 0.0083
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0083,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.3202,
              "portrait": 0.2857,
              "landscape": 0.3941
            }
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.0083,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 0,
      "case_id": "image-layout-04",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "landscape",
        "portrait"
      ],
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "elapsed_ms": 18.494041985832155,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.2776,
              "landscape": 0.4783,
              "portrait": 0.2441
            },
            "confidence": 0.0418
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0418,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.2776,
              "landscape": 0.4783,
              "portrait": 0.2441
            }
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.0418,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 1,
      "case_id": "image-layout-04",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "portrait",
        "landscape"
      ],
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "elapsed_ms": 17.42445898707956,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.2547,
              "portrait": 0.1608,
              "landscape": 0.5845
            },
            "confidence": 0.1297
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.1297,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.2547,
              "portrait": 0.1608,
              "landscape": 0.5845
            }
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.1297,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 2,
      "case_id": "image-layout-04",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "landscape",
        "balanced",
        "portrait"
      ],
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "elapsed_ms": 18.96987500367686,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "landscape": 0.4928,
              "balanced": 0.2905,
              "portrait": 0.2167
            },
            "confidence": 0.0541
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0541,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "landscape": 0.4928,
              "balanced": 0.2905,
              "portrait": 0.2167
            }
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.0541,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 3,
      "case_id": "image-layout-04",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "landscape",
        "portrait",
        "balanced"
      ],
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "elapsed_ms": 20.846291969064623,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "landscape": 0.475,
              "portrait": 0.2496,
              "balanced": 0.2754
            },
            "confidence": 0.0396
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0396,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "landscape": 0.475,
              "portrait": 0.2496,
              "balanced": 0.2754
            }
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.0396,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 4,
      "case_id": "image-layout-04",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "portrait",
        "balanced",
        "landscape"
      ],
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "elapsed_ms": 20.16654098406434,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "portrait": 0.1832,
              "balanced": 0.2712,
              "landscape": 0.5456
            },
            "confidence": 0.094
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.094,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "portrait": 0.1832,
              "balanced": 0.2712,
              "landscape": 0.5456
            }
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.094,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 5,
      "case_id": "image-layout-04",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "portrait",
        "landscape",
        "balanced"
      ],
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "elapsed_ms": 18.88058305485174,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "portrait": 0.1925,
              "landscape": 0.5234,
              "balanced": 0.2841
            },
            "confidence": 0.0774
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0774,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "portrait": 0.1925,
              "landscape": 0.5234,
              "balanced": 0.2841
            }
          }
        },
        "usage": {
          "input_tokens": 124,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.0774,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 0,
      "case_id": "image-layout-05",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "portrait",
        "landscape",
        "balanced"
      ],
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "elapsed_ms": 19.26279196050018,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "portrait": 0.2672,
              "landscape": 0.4149,
              "balanced": 0.3179
            },
            "confidence": 0.0152
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0152,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "portrait": 0.2672,
              "landscape": 0.4149,
              "balanced": 0.3179
            }
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.0152,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 1,
      "case_id": "image-layout-05",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "portrait",
        "balanced",
        "landscape"
      ],
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "elapsed_ms": 19.566583971027285,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "portrait": 0.2648,
              "balanced": 0.334,
              "landscape": 0.4012
            },
            "confidence": 0.0128
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0128,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "portrait": 0.2648,
              "balanced": 0.334,
              "landscape": 0.4012
            }
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.0128,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 2,
      "case_id": "image-layout-05",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "landscape",
        "portrait",
        "balanced"
      ],
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "elapsed_ms": 20.137666957452893,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "landscape": 0.3745,
              "portrait": 0.3118,
              "balanced": 0.3137
            },
            "confidence": 0.0034
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0034,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "landscape": 0.3745,
              "portrait": 0.3118,
              "balanced": 0.3137
            }
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.0034,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 3,
      "case_id": "image-layout-05",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "landscape",
        "balanced",
        "portrait"
      ],
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "elapsed_ms": 20.667542004957795,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "landscape": 0.3932,
              "balanced": 0.3384,
              "portrait": 0.2684
            },
            "confidence": 0.0108
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0108,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "landscape": 0.3932,
              "balanced": 0.3384,
              "portrait": 0.2684
            }
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.0108,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 4,
      "case_id": "image-layout-05",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "portrait",
        "landscape"
      ],
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "elapsed_ms": 20.05712501704693,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.3096,
              "portrait": 0.2583,
              "landscape": 0.4321
            },
            "confidence": 0.0213
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0213,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.3096,
              "portrait": 0.2583,
              "landscape": 0.4321
            }
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.0213,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 5,
      "case_id": "image-layout-05",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "landscape",
        "portrait"
      ],
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "elapsed_ms": 20.914833003189415,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.3055,
              "landscape": 0.3987,
              "portrait": 0.2957
            },
            "confidence": 0.0086
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0086,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.3055,
              "landscape": 0.3987,
              "portrait": 0.2957
            }
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.0086,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 0,
      "case_id": "image-layout-06",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "landscape",
        "portrait"
      ],
      "expected": {
        "type": "choice",
        "choice": "portrait"
      },
      "elapsed_ms": 21.519834001082927,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.2731,
              "landscape": 0.4137,
              "portrait": 0.3132
            },
            "confidence": 0.014
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.014,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.2731,
              "landscape": 0.4137,
              "portrait": 0.3132
            }
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.014,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 1,
      "case_id": "image-layout-06",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "portrait",
        "landscape"
      ],
      "expected": {
        "type": "choice",
        "choice": "portrait"
      },
      "elapsed_ms": 20.3206249861978,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.2795,
              "portrait": 0.2771,
              "landscape": 0.4434
            },
            "confidence": 0.0237
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0237,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.2795,
              "portrait": 0.2771,
              "landscape": 0.4434
            }
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.0237,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 2,
      "case_id": "image-layout-06",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "landscape",
        "balanced",
        "portrait"
      ],
      "expected": {
        "type": "choice",
        "choice": "portrait"
      },
      "elapsed_ms": 20.674250030424446,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "landscape": 0.419,
              "balanced": 0.3081,
              "portrait": 0.2729
            },
            "confidence": 0.0155
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0155,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "landscape": 0.419,
              "balanced": 0.3081,
              "portrait": 0.2729
            }
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.0155,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 3,
      "case_id": "image-layout-06",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "landscape",
        "portrait",
        "balanced"
      ],
      "expected": {
        "type": "choice",
        "choice": "portrait"
      },
      "elapsed_ms": 20.25479095755145,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "landscape": 0.3808,
              "portrait": 0.3259,
              "balanced": 0.2932
            },
            "confidence": 0.0053
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0053,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "landscape": 0.3808,
              "portrait": 0.3259,
              "balanced": 0.2932
            }
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.0053,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 4,
      "case_id": "image-layout-06",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "portrait",
        "balanced",
        "landscape"
      ],
      "expected": {
        "type": "choice",
        "choice": "portrait"
      },
      "elapsed_ms": 20.49666701350361,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "portrait": 0.2872,
              "balanced": 0.2904,
              "landscape": 0.4224
            },
            "confidence": 0.0157
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0157,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "portrait": 0.2872,
              "balanced": 0.2904,
              "landscape": 0.4224
            }
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.0157,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 5,
      "case_id": "image-layout-06",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "portrait",
        "landscape",
        "balanced"
      ],
      "expected": {
        "type": "choice",
        "choice": "portrait"
      },
      "elapsed_ms": 21.79237501695752,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "portrait": 0.2889,
              "landscape": 0.4294,
              "balanced": 0.2817
            },
            "confidence": 0.0182
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0182,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "portrait": 0.2889,
              "landscape": 0.4294,
              "balanced": 0.2817
            }
          }
        },
        "usage": {
          "input_tokens": 123,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.0182,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 0,
      "case_id": "image-layout-07",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "portrait",
        "landscape"
      ],
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "elapsed_ms": 21.968542016111314,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.2631,
              "portrait": 0.3522,
              "landscape": 0.3846
            },
            "confidence": 0.0112
          }
        },
        "usage": {
          "input_tokens": 136,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0112,
            "action": {
              "act_probability": 1
            },
            "choice": "landscape",
            "probabilities": {
              "balanced": 0.2631,
              "portrait": 0.3522,
              "landscape": 0.3846
            }
          }
        },
        "usage": {
          "input_tokens": 136,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "landscape",
        "score_absolute_error": null,
        "confidence": 0.0112,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 1,
      "case_id": "image-layout-07",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "landscape",
        "portrait"
      ],
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "elapsed_ms": 22.960708010941744,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "portrait",
            "probabilities": {
              "balanced": 0.2743,
              "landscape": 0.3481,
              "portrait": 0.3775
            },
            "confidence": 0.0079
          }
        },
        "usage": {
          "input_tokens": 136,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0079,
            "action": {
              "act_probability": 1
            },
            "choice": "portrait",
            "probabilities": {
              "balanced": 0.2743,
              "landscape": 0.3481,
              "portrait": 0.3775
            }
          }
        },
        "usage": {
          "input_tokens": 136,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "portrait",
        "score_absolute_error": null,
        "confidence": 0.0079,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 2,
      "case_id": "image-layout-07",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "portrait",
        "balanced",
        "landscape"
      ],
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "elapsed_ms": 22.537750017363578,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "portrait",
            "probabilities": {
              "portrait": 0.3964,
              "balanced": 0.2752,
              "landscape": 0.3283
            },
            "confidence": 0.0101
          }
        },
        "usage": {
          "input_tokens": 136,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0101,
            "action": {
              "act_probability": 1
            },
            "choice": "portrait",
            "probabilities": {
              "portrait": 0.3964,
              "balanced": 0.2752,
              "landscape": 0.3283
            }
          }
        },
        "usage": {
          "input_tokens": 136,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "portrait",
        "score_absolute_error": null,
        "confidence": 0.0101,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 3,
      "case_id": "image-layout-07",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "portrait",
        "landscape",
        "balanced"
      ],
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "elapsed_ms": 21.344458975363523,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "portrait",
            "probabilities": {
              "portrait": 0.395,
              "landscape": 0.3054,
              "balanced": 0.2996
            },
            "confidence": 0.0076
          }
        },
        "usage": {
          "input_tokens": 136,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0076,
            "action": {
              "act_probability": 1
            },
            "choice": "portrait",
            "probabilities": {
              "portrait": 0.395,
              "landscape": 0.3054,
              "balanced": 0.2996
            }
          }
        },
        "usage": {
          "input_tokens": 136,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "portrait",
        "score_absolute_error": null,
        "confidence": 0.0076,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 4,
      "case_id": "image-layout-07",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "landscape",
        "balanced",
        "portrait"
      ],
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "elapsed_ms": 22.203417029231787,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "portrait",
            "probabilities": {
              "landscape": 0.339,
              "balanced": 0.3018,
              "portrait": 0.3592
            },
            "confidence": 0.0023
          }
        },
        "usage": {
          "input_tokens": 136,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0023,
            "action": {
              "act_probability": 1
            },
            "choice": "portrait",
            "probabilities": {
              "landscape": 0.339,
              "balanced": 0.3018,
              "portrait": 0.3592
            }
          }
        },
        "usage": {
          "input_tokens": 136,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "portrait",
        "score_absolute_error": null,
        "confidence": 0.0023,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 5,
      "case_id": "image-layout-07",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "landscape",
        "portrait",
        "balanced"
      ],
      "expected": {
        "type": "choice",
        "choice": "balanced"
      },
      "elapsed_ms": 22.402332979254425,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "portrait",
            "probabilities": {
              "landscape": 0.2956,
              "portrait": 0.3722,
              "balanced": 0.3322
            },
            "confidence": 0.004
          }
        },
        "usage": {
          "input_tokens": 136,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.004,
            "action": {
              "act_probability": 1
            },
            "choice": "portrait",
            "probabilities": {
              "landscape": 0.2956,
              "portrait": 0.3722,
              "balanced": 0.3322
            }
          }
        },
        "usage": {
          "input_tokens": 136,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "portrait",
        "score_absolute_error": null,
        "confidence": 0.004,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 0,
      "case_id": "image-layout-08",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "portrait",
        "landscape",
        "balanced"
      ],
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "elapsed_ms": 21.912834024988115,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "portrait",
            "probabilities": {
              "portrait": 0.5268,
              "landscape": 0.2492,
              "balanced": 0.224
            },
            "confidence": 0.0724
          }
        },
        "usage": {
          "input_tokens": 122,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0724,
            "action": {
              "act_probability": 1
            },
            "choice": "portrait",
            "probabilities": {
              "portrait": 0.5268,
              "landscape": 0.2492,
              "balanced": 0.224
            }
          }
        },
        "usage": {
          "input_tokens": 122,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "portrait",
        "score_absolute_error": null,
        "confidence": 0.0724,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 1,
      "case_id": "image-layout-08",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "portrait",
        "balanced",
        "landscape"
      ],
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "elapsed_ms": 22.68066699616611,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "portrait",
            "probabilities": {
              "portrait": 0.5548,
              "balanced": 0.2149,
              "landscape": 0.2302
            },
            "confidence": 0.0939
          }
        },
        "usage": {
          "input_tokens": 122,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0939,
            "action": {
              "act_probability": 1
            },
            "choice": "portrait",
            "probabilities": {
              "portrait": 0.5548,
              "balanced": 0.2149,
              "landscape": 0.2302
            }
          }
        },
        "usage": {
          "input_tokens": 122,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "portrait",
        "score_absolute_error": null,
        "confidence": 0.0939,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 2,
      "case_id": "image-layout-08",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "landscape",
        "portrait",
        "balanced"
      ],
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "elapsed_ms": 22.239250014536083,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "portrait",
            "probabilities": {
              "landscape": 0.2136,
              "portrait": 0.5976,
              "balanced": 0.1888
            },
            "confidence": 0.1333
          }
        },
        "usage": {
          "input_tokens": 122,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.1333,
            "action": {
              "act_probability": 1
            },
            "choice": "portrait",
            "probabilities": {
              "landscape": 0.2136,
              "portrait": 0.5976,
              "balanced": 0.1888
            }
          }
        },
        "usage": {
          "input_tokens": 122,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "portrait",
        "score_absolute_error": null,
        "confidence": 0.1333,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 3,
      "case_id": "image-layout-08",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "landscape",
        "balanced",
        "portrait"
      ],
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "elapsed_ms": 21.822458948008716,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "portrait",
            "probabilities": {
              "landscape": 0.1743,
              "balanced": 0.1839,
              "portrait": 0.6418
            },
            "confidence": 0.1803
          }
        },
        "usage": {
          "input_tokens": 122,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.1803,
            "action": {
              "act_probability": 1
            },
            "choice": "portrait",
            "probabilities": {
              "landscape": 0.1743,
              "balanced": 0.1839,
              "portrait": 0.6418
            }
          }
        },
        "usage": {
          "input_tokens": 122,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "portrait",
        "score_absolute_error": null,
        "confidence": 0.1803,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 4,
      "case_id": "image-layout-08",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "portrait",
        "landscape"
      ],
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "elapsed_ms": 21.640708029735833,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "portrait",
            "probabilities": {
              "balanced": 0.1901,
              "portrait": 0.5824,
              "landscape": 0.2275
            },
            "confidence": 0.1195
          }
        },
        "usage": {
          "input_tokens": 122,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.1195,
            "action": {
              "act_probability": 1
            },
            "choice": "portrait",
            "probabilities": {
              "balanced": 0.1901,
              "portrait": 0.5824,
              "landscape": 0.2275
            }
          }
        },
        "usage": {
          "input_tokens": 122,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "portrait",
        "score_absolute_error": null,
        "confidence": 0.1195,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 5,
      "case_id": "image-layout-08",
      "family": "image-layout",
      "type": "choice",
      "order": [
        "balanced",
        "landscape",
        "portrait"
      ],
      "expected": {
        "type": "choice",
        "choice": "landscape"
      },
      "elapsed_ms": 21.71454200288281,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "portrait",
            "probabilities": {
              "balanced": 0.1622,
              "landscape": 0.2403,
              "portrait": 0.5976
            },
            "confidence": 0.1395
          }
        },
        "usage": {
          "input_tokens": 122,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.1395,
            "action": {
              "act_probability": 1
            },
            "choice": "portrait",
            "probabilities": {
              "balanced": 0.1622,
              "landscape": 0.2403,
              "portrait": 0.5976
            }
          }
        },
        "usage": {
          "input_tokens": 122,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "portrait",
        "score_absolute_error": null,
        "confidence": 0.1395,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 0,
      "case_id": "notification-delivery-01",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "digest",
        "email"
      ],
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "elapsed_ms": 22.653500025626272,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "in_app": 0.2397,
              "digest": 0.2275,
              "email": 0.5328
            },
            "confidence": 0.0764
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0764,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "in_app": 0.2397,
              "digest": 0.2275,
              "email": 0.5328
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.0764,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 1,
      "case_id": "notification-delivery-01",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "email",
        "digest"
      ],
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "elapsed_ms": 23.412041016854346,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "in_app": 0.279,
              "email": 0.4859,
              "digest": 0.2351
            },
            "confidence": 0.0468
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0468,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "in_app": 0.279,
              "email": 0.4859,
              "digest": 0.2351
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.0468,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 2,
      "case_id": "notification-delivery-01",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "in_app",
        "email"
      ],
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "elapsed_ms": 23.087042034603655,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "digest": 0.1653,
              "in_app": 0.2575,
              "email": 0.5773
            },
            "confidence": 0.1225
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.1225,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "digest": 0.1653,
              "in_app": 0.2575,
              "email": 0.5773
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.1225,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 3,
      "case_id": "notification-delivery-01",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "email",
        "in_app"
      ],
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "elapsed_ms": 23.0860419687815,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "digest": 0.2225,
              "email": 0.5065,
              "in_app": 0.2711
            },
            "confidence": 0.06
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.06,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "digest": 0.2225,
              "email": 0.5065,
              "in_app": 0.2711
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.06,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 4,
      "case_id": "notification-delivery-01",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "email",
        "in_app",
        "digest"
      ],
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "elapsed_ms": 22.926750010810792,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "email": 0.5605,
              "in_app": 0.205,
              "digest": 0.2346
            },
            "confidence": 0.0993
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0993,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "email": 0.5605,
              "in_app": 0.205,
              "digest": 0.2346
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.0993,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 5,
      "case_id": "notification-delivery-01",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "email",
        "digest",
        "in_app"
      ],
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "elapsed_ms": 23.158916970714927,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "email": 0.5909,
              "digest": 0.1924,
              "in_app": 0.2166
            },
            "confidence": 0.1267
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.1267,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "email": 0.5909,
              "digest": 0.1924,
              "in_app": 0.2166
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.1267,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 0,
      "case_id": "notification-delivery-02",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "digest",
        "email"
      ],
      "expected": {
        "type": "choice",
        "choice": "in_app"
      },
      "elapsed_ms": 22.884625010192394,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "in_app": 0.2571,
              "digest": 0.2312,
              "email": 0.5117
            },
            "confidence": 0.0619
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0619,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "in_app": 0.2571,
              "digest": 0.2312,
              "email": 0.5117
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.0619,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 1,
      "case_id": "notification-delivery-02",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "email",
        "digest"
      ],
      "expected": {
        "type": "choice",
        "choice": "in_app"
      },
      "elapsed_ms": 23.19674997124821,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "in_app": 0.2747,
              "email": 0.5161,
              "digest": 0.2092
            },
            "confidence": 0.0683
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0683,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "in_app": 0.2747,
              "email": 0.5161,
              "digest": 0.2092
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.0683,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 2,
      "case_id": "notification-delivery-02",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "in_app",
        "email"
      ],
      "expected": {
        "type": "choice",
        "choice": "in_app"
      },
      "elapsed_ms": 22.624125005677342,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "digest": 0.1758,
              "in_app": 0.2734,
              "email": 0.5509
            },
            "confidence": 0.1002
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.1002,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "digest": 0.1758,
              "in_app": 0.2734,
              "email": 0.5509
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.1002,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 3,
      "case_id": "notification-delivery-02",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "email",
        "in_app"
      ],
      "expected": {
        "type": "choice",
        "choice": "in_app"
      },
      "elapsed_ms": 22.350375016685575,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "digest": 0.2262,
              "email": 0.5018,
              "in_app": 0.272
            },
            "confidence": 0.0566
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0566,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "digest": 0.2262,
              "email": 0.5018,
              "in_app": 0.272
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.0566,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 4,
      "case_id": "notification-delivery-02",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "email",
        "in_app",
        "digest"
      ],
      "expected": {
        "type": "choice",
        "choice": "in_app"
      },
      "elapsed_ms": 22.248542052693665,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "email": 0.5914,
              "in_app": 0.2045,
              "digest": 0.2041
            },
            "confidence": 0.1265
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.1265,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "email": 0.5914,
              "in_app": 0.2045,
              "digest": 0.2041
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.1265,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 5,
      "case_id": "notification-delivery-02",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "email",
        "digest",
        "in_app"
      ],
      "expected": {
        "type": "choice",
        "choice": "in_app"
      },
      "elapsed_ms": 21.91658300580457,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "email": 0.6084,
              "digest": 0.1933,
              "in_app": 0.1984
            },
            "confidence": 0.1436
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.1436,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "email": 0.6084,
              "digest": 0.1933,
              "in_app": 0.1984
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.1436,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 0,
      "case_id": "notification-delivery-03",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "email",
        "digest",
        "in_app"
      ],
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "elapsed_ms": 21.678083983715624,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "email": 0.4489,
              "digest": 0.3116,
              "in_app": 0.2395
            },
            "confidence": 0.0304
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0304,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "email": 0.4489,
              "digest": 0.3116,
              "in_app": 0.2395
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.0304,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 1,
      "case_id": "notification-delivery-03",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "email",
        "in_app",
        "digest"
      ],
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "elapsed_ms": 21.8452499830164,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "digest",
            "probabilities": {
              "email": 0.358,
              "in_app": 0.1691,
              "digest": 0.4729
            },
            "confidence": 0.0694
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0694,
            "action": {
              "act_probability": 1
            },
            "choice": "digest",
            "probabilities": {
              "email": 0.358,
              "in_app": 0.1691,
              "digest": 0.4729
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "digest",
        "score_absolute_error": null,
        "confidence": 0.0694,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 2,
      "case_id": "notification-delivery-03",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "email",
        "in_app"
      ],
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "elapsed_ms": 22.358415997587144,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "digest": 0.295,
              "email": 0.4451,
              "in_app": 0.2599
            },
            "confidence": 0.0255
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0255,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "digest": 0.295,
              "email": 0.4451,
              "in_app": 0.2599
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.0255,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 3,
      "case_id": "notification-delivery-03",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "in_app",
        "email"
      ],
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "elapsed_ms": 22.779209015425295,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "digest": 0.2092,
              "in_app": 0.21,
              "email": 0.5808
            },
            "confidence": 0.1165
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.1165,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "digest": 0.2092,
              "in_app": 0.21,
              "email": 0.5808
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.1165,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 4,
      "case_id": "notification-delivery-03",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "email",
        "digest"
      ],
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "elapsed_ms": 21.53175004059449,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "digest",
            "probabilities": {
              "in_app": 0.2185,
              "email": 0.3682,
              "digest": 0.4134
            },
            "confidence": 0.0303
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0303,
            "action": {
              "act_probability": 1
            },
            "choice": "digest",
            "probabilities": {
              "in_app": 0.2185,
              "email": 0.3682,
              "digest": 0.4134
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "digest",
        "score_absolute_error": null,
        "confidence": 0.0303,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 5,
      "case_id": "notification-delivery-03",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "digest",
        "email"
      ],
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "elapsed_ms": 21.091458038426936,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "in_app": 0.177,
              "digest": 0.3684,
              "email": 0.4545
            },
            "confidence": 0.0599
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0599,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "in_app": 0.177,
              "digest": 0.3684,
              "email": 0.4545
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.0599,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 0,
      "case_id": "notification-delivery-04",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "in_app",
        "email"
      ],
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "elapsed_ms": 20.983166992664337,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "digest": 0.1798,
              "in_app": 0.2749,
              "email": 0.5453
            },
            "confidence": 0.0951
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0951,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "digest": 0.1798,
              "in_app": 0.2749,
              "email": 0.5453
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.0951,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 1,
      "case_id": "notification-delivery-04",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "email",
        "in_app"
      ],
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "elapsed_ms": 23.127208987716585,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "digest": 0.2278,
              "email": 0.4761,
              "in_app": 0.2961
            },
            "confidence": 0.0436
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0436,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "digest": 0.2278,
              "email": 0.4761,
              "in_app": 0.2961
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.0436,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 2,
      "case_id": "notification-delivery-04",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "digest",
        "email"
      ],
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "elapsed_ms": 21.914292010478675,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "in_app": 0.2593,
              "digest": 0.2585,
              "email": 0.4822
            },
            "confidence": 0.0429
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0429,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "in_app": 0.2593,
              "digest": 0.2585,
              "email": 0.4822
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.0429,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 3,
      "case_id": "notification-delivery-04",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "email",
        "digest"
      ],
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "elapsed_ms": 21.632292016875,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "in_app": 0.2895,
              "email": 0.4491,
              "digest": 0.2614
            },
            "confidence": 0.0269
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0269,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "in_app": 0.2895,
              "email": 0.4491,
              "digest": 0.2614
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.0269,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 4,
      "case_id": "notification-delivery-04",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "email",
        "digest",
        "in_app"
      ],
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "elapsed_ms": 22.184709028806537,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "email": 0.5365,
              "digest": 0.2178,
              "in_app": 0.2457
            },
            "confidence": 0.0798
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0798,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "email": 0.5365,
              "digest": 0.2178,
              "in_app": 0.2457
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.0798,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 5,
      "case_id": "notification-delivery-04",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "email",
        "in_app",
        "digest"
      ],
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "elapsed_ms": 22.375250002369285,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "email": 0.5237,
              "in_app": 0.2163,
              "digest": 0.26
            },
            "confidence": 0.0714
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0714,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "email": 0.5237,
              "in_app": 0.2163,
              "digest": 0.26
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.0714,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 0,
      "case_id": "notification-delivery-05",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "digest",
        "email"
      ],
      "expected": {
        "type": "choice",
        "choice": "in_app"
      },
      "elapsed_ms": 22.526165994349867,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "in_app": 0.2955,
              "digest": 0.2817,
              "email": 0.4228
            },
            "confidence": 0.0159
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0159,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "in_app": 0.2955,
              "digest": 0.2817,
              "email": 0.4228
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.0159,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 1,
      "case_id": "notification-delivery-05",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "email",
        "digest"
      ],
      "expected": {
        "type": "choice",
        "choice": "in_app"
      },
      "elapsed_ms": 20.949416037183255,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "in_app": 0.3477,
              "email": 0.3525,
              "digest": 0.2999
            },
            "confidence": 0.0023
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0023,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "in_app": 0.3477,
              "email": 0.3525,
              "digest": 0.2999
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.0023,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 2,
      "case_id": "notification-delivery-05",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "in_app",
        "email"
      ],
      "expected": {
        "type": "choice",
        "choice": "in_app"
      },
      "elapsed_ms": 20.18783299718052,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "digest": 0.2031,
              "in_app": 0.2949,
              "email": 0.502
            },
            "confidence": 0.0626
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0626,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "digest": 0.2031,
              "in_app": 0.2949,
              "email": 0.502
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.0626,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 3,
      "case_id": "notification-delivery-05",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "email",
        "in_app"
      ],
      "expected": {
        "type": "choice",
        "choice": "in_app"
      },
      "elapsed_ms": 20.229749963618815,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "digest": 0.2551,
              "email": 0.407,
              "in_app": 0.3379
            },
            "confidence": 0.016
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.016,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "digest": 0.2551,
              "email": 0.407,
              "in_app": 0.3379
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.016,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 4,
      "case_id": "notification-delivery-05",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "email",
        "in_app",
        "digest"
      ],
      "expected": {
        "type": "choice",
        "choice": "in_app"
      },
      "elapsed_ms": 19.43833299446851,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "email": 0.45,
              "in_app": 0.2637,
              "digest": 0.2863
            },
            "confidence": 0.027
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.027,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "email": 0.45,
              "in_app": 0.2637,
              "digest": 0.2863
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.027,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 5,
      "case_id": "notification-delivery-05",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "email",
        "digest",
        "in_app"
      ],
      "expected": {
        "type": "choice",
        "choice": "in_app"
      },
      "elapsed_ms": 19.820792018435895,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "email": 0.4447,
              "digest": 0.2571,
              "in_app": 0.2982
            },
            "confidence": 0.0257
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0257,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "email": 0.4447,
              "digest": 0.2571,
              "in_app": 0.2982
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.0257,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 0,
      "case_id": "notification-delivery-06",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "email",
        "digest",
        "in_app"
      ],
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "elapsed_ms": 19.73383396398276,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "email": 0.5312,
              "digest": 0.226,
              "in_app": 0.2428
            },
            "confidence": 0.0753
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0753,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "email": 0.5312,
              "digest": 0.226,
              "in_app": 0.2428
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.0753,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 1,
      "case_id": "notification-delivery-06",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "email",
        "in_app",
        "digest"
      ],
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "elapsed_ms": 19.485583005007356,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "email": 0.5409,
              "in_app": 0.2239,
              "digest": 0.2353
            },
            "confidence": 0.0826
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0826,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "email": 0.5409,
              "in_app": 0.2239,
              "digest": 0.2353
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.0826,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 2,
      "case_id": "notification-delivery-06",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "email",
        "in_app"
      ],
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "elapsed_ms": 20.094375009648502,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "digest": 0.2371,
              "email": 0.4617,
              "in_app": 0.3012
            },
            "confidence": 0.0356
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0356,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "digest": 0.2371,
              "email": 0.4617,
              "in_app": 0.3012
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.0356,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 3,
      "case_id": "notification-delivery-06",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "in_app",
        "email"
      ],
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "elapsed_ms": 19.695541996043175,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "digest": 0.1896,
              "in_app": 0.2861,
              "email": 0.5242
            },
            "confidence": 0.0789
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0789,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "digest": 0.1896,
              "in_app": 0.2861,
              "email": 0.5242
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.0789,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 4,
      "case_id": "notification-delivery-06",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "email",
        "digest"
      ],
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "elapsed_ms": 19.794291991274804,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "in_app": 0.299,
              "email": 0.4479,
              "digest": 0.2532
            },
            "confidence": 0.0274
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0274,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "in_app": 0.299,
              "email": 0.4479,
              "digest": 0.2532
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.0274,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 5,
      "case_id": "notification-delivery-06",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "digest",
        "email"
      ],
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "elapsed_ms": 19.91312496829778,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "in_app": 0.2691,
              "digest": 0.2678,
              "email": 0.4631
            },
            "confidence": 0.0328
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0328,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "in_app": 0.2691,
              "digest": 0.2678,
              "email": 0.4631
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.0328,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 0,
      "case_id": "notification-delivery-07",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "in_app",
        "email"
      ],
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "elapsed_ms": 22.28454197756946,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "digest",
            "probabilities": {
              "digest": 0.4598,
              "in_app": 0.2653,
              "email": 0.2749
            },
            "confidence": 0.0313
          }
        },
        "usage": {
          "input_tokens": 154,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0313,
            "action": {
              "act_probability": 1
            },
            "choice": "digest",
            "probabilities": {
              "digest": 0.4598,
              "in_app": 0.2653,
              "email": 0.2749
            }
          }
        },
        "usage": {
          "input_tokens": 154,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "digest",
        "score_absolute_error": null,
        "confidence": 0.0313,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 1,
      "case_id": "notification-delivery-07",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "email",
        "in_app"
      ],
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "elapsed_ms": 19.13195796078071,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "digest",
            "probabilities": {
              "digest": 0.4042,
              "email": 0.3109,
              "in_app": 0.2849
            },
            "confidence": 0.0105
          }
        },
        "usage": {
          "input_tokens": 154,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0105,
            "action": {
              "act_probability": 1
            },
            "choice": "digest",
            "probabilities": {
              "digest": 0.4042,
              "email": 0.3109,
              "in_app": 0.2849
            }
          }
        },
        "usage": {
          "input_tokens": 154,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "digest",
        "score_absolute_error": null,
        "confidence": 0.0105,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 2,
      "case_id": "notification-delivery-07",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "digest",
        "email"
      ],
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "elapsed_ms": 19.92079196497798,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "digest",
            "probabilities": {
              "in_app": 0.1815,
              "digest": 0.6096,
              "email": 0.209
            },
            "confidence": 0.1456
          }
        },
        "usage": {
          "input_tokens": 154,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.1456,
            "action": {
              "act_probability": 1
            },
            "choice": "digest",
            "probabilities": {
              "in_app": 0.1815,
              "digest": 0.6096,
              "email": 0.209
            }
          }
        },
        "usage": {
          "input_tokens": 154,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "digest",
        "score_absolute_error": null,
        "confidence": 0.1456,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 3,
      "case_id": "notification-delivery-07",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "email",
        "digest"
      ],
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "elapsed_ms": 19.815833016764373,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "digest",
            "probabilities": {
              "in_app": 0.1799,
              "email": 0.2253,
              "digest": 0.5949
            },
            "confidence": 0.1323
          }
        },
        "usage": {
          "input_tokens": 154,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.1323,
            "action": {
              "act_probability": 1
            },
            "choice": "digest",
            "probabilities": {
              "in_app": 0.1799,
              "email": 0.2253,
              "digest": 0.5949
            }
          }
        },
        "usage": {
          "input_tokens": 154,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "digest",
        "score_absolute_error": null,
        "confidence": 0.1323,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 4,
      "case_id": "notification-delivery-07",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "email",
        "digest",
        "in_app"
      ],
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "elapsed_ms": 19.677707983646542,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "digest",
            "probabilities": {
              "email": 0.2493,
              "digest": 0.4648,
              "in_app": 0.2859
            },
            "confidence": 0.0348
          }
        },
        "usage": {
          "input_tokens": 154,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0348,
            "action": {
              "act_probability": 1
            },
            "choice": "digest",
            "probabilities": {
              "email": 0.2493,
              "digest": 0.4648,
              "in_app": 0.2859
            }
          }
        },
        "usage": {
          "input_tokens": 154,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "digest",
        "score_absolute_error": null,
        "confidence": 0.0348,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 5,
      "case_id": "notification-delivery-07",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "email",
        "in_app",
        "digest"
      ],
      "expected": {
        "type": "choice",
        "choice": "email"
      },
      "elapsed_ms": 20.30683303019032,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "digest",
            "probabilities": {
              "email": 0.2191,
              "in_app": 0.2285,
              "digest": 0.5524
            },
            "confidence": 0.0917
          }
        },
        "usage": {
          "input_tokens": 154,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0917,
            "action": {
              "act_probability": 1
            },
            "choice": "digest",
            "probabilities": {
              "email": 0.2191,
              "in_app": 0.2285,
              "digest": 0.5524
            }
          }
        },
        "usage": {
          "input_tokens": 154,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "digest",
        "score_absolute_error": null,
        "confidence": 0.0917,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 0,
      "case_id": "notification-delivery-08",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "email",
        "in_app"
      ],
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "elapsed_ms": 19.415083981584758,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "digest": 0.2652,
              "email": 0.3729,
              "in_app": 0.3619
            },
            "confidence": 0.01
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.01,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "digest": 0.2652,
              "email": 0.3729,
              "in_app": 0.3619
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.01,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 1,
      "case_id": "notification-delivery-08",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "digest",
        "in_app",
        "email"
      ],
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "elapsed_ms": 20.433041965588927,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "digest": 0.2198,
              "in_app": 0.3039,
              "email": 0.4763
            },
            "confidence": 0.0458
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0458,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "digest": 0.2198,
              "in_app": 0.3039,
              "email": 0.4763
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.0458,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 2,
      "case_id": "notification-delivery-08",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "email",
        "digest",
        "in_app"
      ],
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "elapsed_ms": 18.478624988347292,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "email": 0.3776,
              "digest": 0.2883,
              "in_app": 0.3341
            },
            "confidence": 0.0055
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0055,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "email": 0.3776,
              "digest": 0.2883,
              "in_app": 0.3341
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.0055,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 3,
      "case_id": "notification-delivery-08",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "email",
        "in_app",
        "digest"
      ],
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "elapsed_ms": 18.636290973518044,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "email": 0.3997,
              "in_app": 0.2789,
              "digest": 0.3214
            },
            "confidence": 0.0101
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0101,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "email": 0.3997,
              "in_app": 0.2789,
              "digest": 0.3214
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.0101,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 4,
      "case_id": "notification-delivery-08",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "digest",
        "email"
      ],
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "elapsed_ms": 18.517374992370605,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "in_app": 0.2989,
              "digest": 0.2983,
              "email": 0.4027
            },
            "confidence": 0.0096
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0096,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "in_app": 0.2989,
              "digest": 0.2983,
              "email": 0.4027
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0.0096,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 5,
      "case_id": "notification-delivery-08",
      "family": "notification-delivery",
      "type": "choice",
      "order": [
        "in_app",
        "email",
        "digest"
      ],
      "expected": {
        "type": "choice",
        "choice": "digest"
      },
      "elapsed_ms": 18.696292012464255,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "email",
            "probabilities": {
              "in_app": 0.3336,
              "email": 0.3355,
              "digest": 0.3309
            },
            "confidence": 0
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0,
            "action": {
              "act_probability": 1
            },
            "choice": "email",
            "probabilities": {
              "in_app": 0.3336,
              "email": 0.3355,
              "digest": 0.3309
            }
          }
        },
        "usage": {
          "input_tokens": 139,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "email",
        "score_absolute_error": null,
        "confidence": 0,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 0,
      "case_id": "release-note-group-01",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "fix",
        "breaking",
        "addition"
      ],
      "expected": {
        "type": "choice",
        "choice": "breaking"
      },
      "elapsed_ms": 17.991250031627715,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "fix": 0.3008,
              "breaking": 0.2342,
              "addition": 0.465
            },
            "confidence": 0.0375
          }
        },
        "usage": {
          "input_tokens": 148,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0375,
            "action": {
              "act_probability": 1
            },
            "choice": "addition",
            "probabilities": {
              "fix": 0.3008,
              "breaking": 0.2342,
              "addition": 0.465
            }
          }
        },
        "usage": {
          "input_tokens": 148,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.0375,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 1,
      "case_id": "release-note-group-01",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "fix",
        "addition",
        "breaking"
      ],
      "expected": {
        "type": "choice",
        "choice": "breaking"
      },
      "elapsed_ms": 19.103166996501386,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "fix": 0.3084,
              "addition": 0.453,
              "breaking": 0.2387
            },
            "confidence": 0.032
          }
        },
        "usage": {
          "input_tokens": 148,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.032,
            "action": {
              "act_probability": 1
            },
            "choice": "addition",
            "probabilities": {
              "fix": 0.3084,
              "addition": 0.453,
              "breaking": 0.2387
            }
          }
        },
        "usage": {
          "input_tokens": 148,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.032,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 2,
      "case_id": "release-note-group-01",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "fix",
        "addition"
      ],
      "expected": {
        "type": "choice",
        "choice": "breaking"
      },
      "elapsed_ms": 17.621958977542818,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "breaking": 0.227,
              "fix": 0.3143,
              "addition": 0.4587
            },
            "confidence": 0.0371
          }
        },
        "usage": {
          "input_tokens": 148,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0371,
            "action": {
              "act_probability": 1
            },
            "choice": "addition",
            "probabilities": {
              "breaking": 0.227,
              "fix": 0.3143,
              "addition": 0.4587
            }
          }
        },
        "usage": {
          "input_tokens": 148,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.0371,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 3,
      "case_id": "release-note-group-01",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "addition",
        "fix"
      ],
      "expected": {
        "type": "choice",
        "choice": "breaking"
      },
      "elapsed_ms": 17.386624997016042,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "breaking": 0.2142,
              "addition": 0.4644,
              "fix": 0.3213
            },
            "confidence": 0.0433
          }
        },
        "usage": {
          "input_tokens": 148,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0433,
            "action": {
              "act_probability": 1
            },
            "choice": "addition",
            "probabilities": {
              "breaking": 0.2142,
              "addition": 0.4644,
              "fix": 0.3213
            }
          }
        },
        "usage": {
          "input_tokens": 148,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.0433,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 4,
      "case_id": "release-note-group-01",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "fix",
        "breaking"
      ],
      "expected": {
        "type": "choice",
        "choice": "breaking"
      },
      "elapsed_ms": 17.07608299329877,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "addition": 0.4098,
              "fix": 0.3227,
              "breaking": 0.2675
            },
            "confidence": 0.0139
          }
        },
        "usage": {
          "input_tokens": 148,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0139,
            "action": {
              "act_probability": 1
            },
            "choice": "addition",
            "probabilities": {
              "addition": 0.4098,
              "fix": 0.3227,
              "breaking": 0.2675
            }
          }
        },
        "usage": {
          "input_tokens": 148,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.0139,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 5,
      "case_id": "release-note-group-01",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "breaking",
        "fix"
      ],
      "expected": {
        "type": "choice",
        "choice": "breaking"
      },
      "elapsed_ms": 17.155124980490655,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "addition": 0.3999,
              "breaking": 0.2221,
              "fix": 0.3779
            },
            "confidence": 0.0275
          }
        },
        "usage": {
          "input_tokens": 148,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0275,
            "action": {
              "act_probability": 1
            },
            "choice": "addition",
            "probabilities": {
              "addition": 0.3999,
              "breaking": 0.2221,
              "fix": 0.3779
            }
          }
        },
        "usage": {
          "input_tokens": 148,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.0275,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 0,
      "case_id": "release-note-group-02",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "breaking",
        "fix"
      ],
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "elapsed_ms": 17.909665999468416,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "addition": 0.128,
              "breaking": 0.1385,
              "fix": 0.7335
            },
            "confidence": 0.3044
          }
        },
        "usage": {
          "input_tokens": 140,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.3044,
            "action": {
              "act_probability": 1
            },
            "choice": "fix",
            "probabilities": {
              "addition": 0.128,
              "breaking": 0.1385,
              "fix": 0.7335
            }
          }
        },
        "usage": {
          "input_tokens": 140,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.3044,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 1,
      "case_id": "release-note-group-02",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "fix",
        "breaking"
      ],
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "elapsed_ms": 19.357083016075194,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "addition": 0.1255,
              "fix": 0.7528,
              "breaking": 0.1217
            },
            "confidence": 0.335
          }
        },
        "usage": {
          "input_tokens": 140,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.335,
            "action": {
              "act_probability": 1
            },
            "choice": "fix",
            "probabilities": {
              "addition": 0.1255,
              "fix": 0.7528,
              "breaking": 0.1217
            }
          }
        },
        "usage": {
          "input_tokens": 140,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.335,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 2,
      "case_id": "release-note-group-02",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "addition",
        "fix"
      ],
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "elapsed_ms": 16.98337501147762,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "breaking": 0.1091,
              "addition": 0.1425,
              "fix": 0.7484
            },
            "confidence": 0.3298
          }
        },
        "usage": {
          "input_tokens": 140,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.3298,
            "action": {
              "act_probability": 1
            },
            "choice": "fix",
            "probabilities": {
              "breaking": 0.1091,
              "addition": 0.1425,
              "fix": 0.7484
            }
          }
        },
        "usage": {
          "input_tokens": 140,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.3298,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 3,
      "case_id": "release-note-group-02",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "fix",
        "addition"
      ],
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "elapsed_ms": 16.71883399831131,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "breaking": 0.1082,
              "fix": 0.7697,
              "addition": 0.1221
            },
            "confidence": 0.3639
          }
        },
        "usage": {
          "input_tokens": 140,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.3639,
            "action": {
              "act_probability": 1
            },
            "choice": "fix",
            "probabilities": {
              "breaking": 0.1082,
              "fix": 0.7697,
              "addition": 0.1221
            }
          }
        },
        "usage": {
          "input_tokens": 140,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.3639,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 4,
      "case_id": "release-note-group-02",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "fix",
        "addition",
        "breaking"
      ],
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "elapsed_ms": 16.950457997154444,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "fix": 0.78,
              "addition": 0.1099,
              "breaking": 0.1101
            },
            "confidence": 0.3816
          }
        },
        "usage": {
          "input_tokens": 140,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.3816,
            "action": {
              "act_probability": 1
            },
            "choice": "fix",
            "probabilities": {
              "fix": 0.78,
              "addition": 0.1099,
              "breaking": 0.1101
            }
          }
        },
        "usage": {
          "input_tokens": 140,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.3816,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 5,
      "case_id": "release-note-group-02",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "fix",
        "breaking",
        "addition"
      ],
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "elapsed_ms": 17.41995895281434,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "fix": 0.7703,
              "breaking": 0.1059,
              "addition": 0.1238
            },
            "confidence": 0.3651
          }
        },
        "usage": {
          "input_tokens": 140,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.3651,
            "action": {
              "act_probability": 1
            },
            "choice": "fix",
            "probabilities": {
              "fix": 0.7703,
              "breaking": 0.1059,
              "addition": 0.1238
            }
          }
        },
        "usage": {
          "input_tokens": 140,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.3651,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 0,
      "case_id": "release-note-group-03",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "breaking",
        "fix"
      ],
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "elapsed_ms": 16.411625023465604,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "addition": 0.4816,
              "breaking": 0.162,
              "fix": 0.3564
            },
            "confidence": 0.0766
          }
        },
        "usage": {
          "input_tokens": 137,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0766,
            "action": {
              "act_probability": 1
            },
            "choice": "addition",
            "probabilities": {
              "addition": 0.4816,
              "breaking": 0.162,
              "fix": 0.3564
            }
          }
        },
        "usage": {
          "input_tokens": 137,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.0766,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 1,
      "case_id": "release-note-group-03",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "fix",
        "breaking"
      ],
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "elapsed_ms": 16.596708039287478,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "addition": 0.5126,
              "fix": 0.3216,
              "breaking": 0.1657
            },
            "confidence": 0.085
          }
        },
        "usage": {
          "input_tokens": 137,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.085,
            "action": {
              "act_probability": 1
            },
            "choice": "addition",
            "probabilities": {
              "addition": 0.5126,
              "fix": 0.3216,
              "breaking": 0.1657
            }
          }
        },
        "usage": {
          "input_tokens": 137,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.085,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 2,
      "case_id": "release-note-group-03",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "addition",
        "fix"
      ],
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "elapsed_ms": 16.52762497542426,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "breaking": 0.1469,
              "addition": 0.4882,
              "fix": 0.3649
            },
            "confidence": 0.0901
          }
        },
        "usage": {
          "input_tokens": 137,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0901,
            "action": {
              "act_probability": 1
            },
            "choice": "addition",
            "probabilities": {
              "breaking": 0.1469,
              "addition": 0.4882,
              "fix": 0.3649
            }
          }
        },
        "usage": {
          "input_tokens": 137,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.0901,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 3,
      "case_id": "release-note-group-03",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "fix",
        "addition"
      ],
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "elapsed_ms": 16.57608401728794,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "breaking": 0.1735,
              "fix": 0.4041,
              "addition": 0.4224
            },
            "confidence": 0.0588
          }
        },
        "usage": {
          "input_tokens": 137,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0588,
            "action": {
              "act_probability": 1
            },
            "choice": "addition",
            "probabilities": {
              "breaking": 0.1735,
              "fix": 0.4041,
              "addition": 0.4224
            }
          }
        },
        "usage": {
          "input_tokens": 137,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.0588,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 4,
      "case_id": "release-note-group-03",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "fix",
        "addition",
        "breaking"
      ],
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "elapsed_ms": 16.44433301407844,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "fix": 0.4143,
              "addition": 0.4609,
              "breaking": 0.1248
            },
            "confidence": 0.1064
          }
        },
        "usage": {
          "input_tokens": 137,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.1064,
            "action": {
              "act_probability": 1
            },
            "choice": "addition",
            "probabilities": {
              "fix": 0.4143,
              "addition": 0.4609,
              "breaking": 0.1248
            }
          }
        },
        "usage": {
          "input_tokens": 137,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.1064,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 5,
      "case_id": "release-note-group-03",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "fix",
        "breaking",
        "addition"
      ],
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "elapsed_ms": 16.412500001024455,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "fix": 0.4132,
              "breaking": 0.1623,
              "addition": 0.4245
            },
            "confidence": 0.0679
          }
        },
        "usage": {
          "input_tokens": 137,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0679,
            "action": {
              "act_probability": 1
            },
            "choice": "addition",
            "probabilities": {
              "fix": 0.4132,
              "breaking": 0.1623,
              "addition": 0.4245
            }
          }
        },
        "usage": {
          "input_tokens": 137,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.0679,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 0,
      "case_id": "release-note-group-04",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "fix",
        "breaking",
        "addition"
      ],
      "expected": {
        "type": "choice",
        "choice": "breaking"
      },
      "elapsed_ms": 16.32908300962299,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "fix": 0.4896,
              "breaking": 0.2786,
              "addition": 0.2319
            },
            "confidence": 0.0492
          }
        },
        "usage": {
          "input_tokens": 145,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0492,
            "action": {
              "act_probability": 1
            },
            "choice": "fix",
            "probabilities": {
              "fix": 0.4896,
              "breaking": 0.2786,
              "addition": 0.2319
            }
          }
        },
        "usage": {
          "input_tokens": 145,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.0492,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 1,
      "case_id": "release-note-group-04",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "fix",
        "addition",
        "breaking"
      ],
      "expected": {
        "type": "choice",
        "choice": "breaking"
      },
      "elapsed_ms": 16.59491698956117,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "fix": 0.554,
              "addition": 0.2188,
              "breaking": 0.2272
            },
            "confidence": 0.0931
          }
        },
        "usage": {
          "input_tokens": 145,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0931,
            "action": {
              "act_probability": 1
            },
            "choice": "fix",
            "probabilities": {
              "fix": 0.554,
              "addition": 0.2188,
              "breaking": 0.2272
            }
          }
        },
        "usage": {
          "input_tokens": 145,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.0931,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 2,
      "case_id": "release-note-group-04",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "fix",
        "addition"
      ],
      "expected": {
        "type": "choice",
        "choice": "breaking"
      },
      "elapsed_ms": 16.10279199667275,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "breaking": 0.2635,
              "fix": 0.5449,
              "addition": 0.1916
            },
            "confidence": 0.0908
          }
        },
        "usage": {
          "input_tokens": 145,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0908,
            "action": {
              "act_probability": 1
            },
            "choice": "fix",
            "probabilities": {
              "breaking": 0.2635,
              "fix": 0.5449,
              "addition": 0.1916
            }
          }
        },
        "usage": {
          "input_tokens": 145,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.0908,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 3,
      "case_id": "release-note-group-04",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "addition",
        "fix"
      ],
      "expected": {
        "type": "choice",
        "choice": "breaking"
      },
      "elapsed_ms": 16.60837500821799,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "breaking": 0.2772,
              "addition": 0.1597,
              "fix": 0.563
            },
            "confidence": 0.1152
          }
        },
        "usage": {
          "input_tokens": 145,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.1152,
            "action": {
              "act_probability": 1
            },
            "choice": "fix",
            "probabilities": {
              "breaking": 0.2772,
              "addition": 0.1597,
              "fix": 0.563
            }
          }
        },
        "usage": {
          "input_tokens": 145,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.1152,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 4,
      "case_id": "release-note-group-04",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "fix",
        "breaking"
      ],
      "expected": {
        "type": "choice",
        "choice": "breaking"
      },
      "elapsed_ms": 15.96987497759983,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "addition": 0.2016,
              "fix": 0.5252,
              "breaking": 0.2732
            },
            "confidence": 0.0756
          }
        },
        "usage": {
          "input_tokens": 145,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0756,
            "action": {
              "act_probability": 1
            },
            "choice": "fix",
            "probabilities": {
              "addition": 0.2016,
              "fix": 0.5252,
              "breaking": 0.2732
            }
          }
        },
        "usage": {
          "input_tokens": 145,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.0756,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 5,
      "case_id": "release-note-group-04",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "breaking",
        "fix"
      ],
      "expected": {
        "type": "choice",
        "choice": "breaking"
      },
      "elapsed_ms": 15.802500012796372,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "addition": 0.1708,
              "breaking": 0.272,
              "fix": 0.5571
            },
            "confidence": 0.1062
          }
        },
        "usage": {
          "input_tokens": 145,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.1062,
            "action": {
              "act_probability": 1
            },
            "choice": "fix",
            "probabilities": {
              "addition": 0.1708,
              "breaking": 0.272,
              "fix": 0.5571
            }
          }
        },
        "usage": {
          "input_tokens": 145,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.1062,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 0,
      "case_id": "release-note-group-05",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "addition",
        "fix"
      ],
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "elapsed_ms": 15.666958002839237,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "breaking": 0.1838,
              "addition": 0.1464,
              "fix": 0.6698
            },
            "confidence": 0.2162
          }
        },
        "usage": {
          "input_tokens": 143,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.2162,
            "action": {
              "act_probability": 1
            },
            "choice": "fix",
            "probabilities": {
              "breaking": 0.1838,
              "addition": 0.1464,
              "fix": 0.6698
            }
          }
        },
        "usage": {
          "input_tokens": 143,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.2162,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 1,
      "case_id": "release-note-group-05",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "fix",
        "addition"
      ],
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "elapsed_ms": 15.766249969601631,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "breaking": 0.1413,
              "fix": 0.7238,
              "addition": 0.1349
            },
            "confidence": 0.2893
          }
        },
        "usage": {
          "input_tokens": 143,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.2893,
            "action": {
              "act_probability": 1
            },
            "choice": "fix",
            "probabilities": {
              "breaking": 0.1413,
              "fix": 0.7238,
              "addition": 0.1349
            }
          }
        },
        "usage": {
          "input_tokens": 143,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.2893,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 2,
      "case_id": "release-note-group-05",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "breaking",
        "fix"
      ],
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "elapsed_ms": 15.253457997459918,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "addition": 0.1252,
              "breaking": 0.182,
              "fix": 0.6929
            },
            "confidence": 0.2496
          }
        },
        "usage": {
          "input_tokens": 143,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.2496,
            "action": {
              "act_probability": 1
            },
            "choice": "fix",
            "probabilities": {
              "addition": 0.1252,
              "breaking": 0.182,
              "fix": 0.6929
            }
          }
        },
        "usage": {
          "input_tokens": 143,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.2496,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 3,
      "case_id": "release-note-group-05",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "fix",
        "breaking"
      ],
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "elapsed_ms": 15.439041948411614,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "addition": 0.1142,
              "fix": 0.7191,
              "breaking": 0.1667
            },
            "confidence": 0.2868
          }
        },
        "usage": {
          "input_tokens": 143,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.2868,
            "action": {
              "act_probability": 1
            },
            "choice": "fix",
            "probabilities": {
              "addition": 0.1142,
              "fix": 0.7191,
              "breaking": 0.1667
            }
          }
        },
        "usage": {
          "input_tokens": 143,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.2868,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 4,
      "case_id": "release-note-group-05",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "fix",
        "breaking",
        "addition"
      ],
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "elapsed_ms": 15.514458005782217,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "fix": 0.7251,
              "breaking": 0.1185,
              "addition": 0.1564
            },
            "confidence": 0.2937
          }
        },
        "usage": {
          "input_tokens": 143,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.2937,
            "action": {
              "act_probability": 1
            },
            "choice": "fix",
            "probabilities": {
              "fix": 0.7251,
              "breaking": 0.1185,
              "addition": 0.1564
            }
          }
        },
        "usage": {
          "input_tokens": 143,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.2937,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 5,
      "case_id": "release-note-group-05",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "fix",
        "addition",
        "breaking"
      ],
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "elapsed_ms": 15.330458001699299,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "fix": 0.7266,
              "addition": 0.1187,
              "breaking": 0.1547
            },
            "confidence": 0.2957
          }
        },
        "usage": {
          "input_tokens": 143,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.2957,
            "action": {
              "act_probability": 1
            },
            "choice": "fix",
            "probabilities": {
              "fix": 0.7266,
              "addition": 0.1187,
              "breaking": 0.1547
            }
          }
        },
        "usage": {
          "input_tokens": 143,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.2957,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 0,
      "case_id": "release-note-group-06",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "fix",
        "breaking"
      ],
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "elapsed_ms": 15.053124981932342,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "addition": 0.4043,
              "fix": 0.3457,
              "breaking": 0.25
            },
            "confidence": 0.017
          }
        },
        "usage": {
          "input_tokens": 148,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.017,
            "action": {
              "act_probability": 1
            },
            "choice": "addition",
            "probabilities": {
              "addition": 0.4043,
              "fix": 0.3457,
              "breaking": 0.25
            }
          }
        },
        "usage": {
          "input_tokens": 148,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.017,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 1,
      "case_id": "release-note-group-06",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "breaking",
        "fix"
      ],
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "elapsed_ms": 15.173166990280151,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "addition": 0.4081,
              "breaking": 0.2295,
              "fix": 0.3624
            },
            "confidence": 0.0248
          }
        },
        "usage": {
          "input_tokens": 148,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0248,
            "action": {
              "act_probability": 1
            },
            "choice": "addition",
            "probabilities": {
              "addition": 0.4081,
              "breaking": 0.2295,
              "fix": 0.3624
            }
          }
        },
        "usage": {
          "input_tokens": 148,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.0248,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 2,
      "case_id": "release-note-group-06",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "fix",
        "addition",
        "breaking"
      ],
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "elapsed_ms": 15.01899998402223,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "fix": 0.3502,
              "addition": 0.4195,
              "breaking": 0.2303
            },
            "confidence": 0.026
          }
        },
        "usage": {
          "input_tokens": 148,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.026,
            "action": {
              "act_probability": 1
            },
            "choice": "addition",
            "probabilities": {
              "fix": 0.3502,
              "addition": 0.4195,
              "breaking": 0.2303
            }
          }
        },
        "usage": {
          "input_tokens": 148,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.026,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 3,
      "case_id": "release-note-group-06",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "fix",
        "breaking",
        "addition"
      ],
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "elapsed_ms": 15.104458027053624,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "fix": 0.3528,
              "breaking": 0.2098,
              "addition": 0.4374
            },
            "confidence": 0.038
          }
        },
        "usage": {
          "input_tokens": 148,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.038,
            "action": {
              "act_probability": 1
            },
            "choice": "addition",
            "probabilities": {
              "fix": 0.3528,
              "breaking": 0.2098,
              "addition": 0.4374
            }
          }
        },
        "usage": {
          "input_tokens": 148,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.038,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 4,
      "case_id": "release-note-group-06",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "addition",
        "fix"
      ],
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "elapsed_ms": 14.923084003385156,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "breaking": 0.2153,
              "addition": 0.4112,
              "fix": 0.3735
            },
            "confidence": 0.0316
          }
        },
        "usage": {
          "input_tokens": 148,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0316,
            "action": {
              "act_probability": 1
            },
            "choice": "addition",
            "probabilities": {
              "breaking": 0.2153,
              "addition": 0.4112,
              "fix": 0.3735
            }
          }
        },
        "usage": {
          "input_tokens": 148,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.0316,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 5,
      "case_id": "release-note-group-06",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "fix",
        "addition"
      ],
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "elapsed_ms": 15.250166004989296,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "breaking": 0.2219,
              "fix": 0.3831,
              "addition": 0.395
            },
            "confidence": 0.0274
          }
        },
        "usage": {
          "input_tokens": 148,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0274,
            "action": {
              "act_probability": 1
            },
            "choice": "addition",
            "probabilities": {
              "breaking": 0.2219,
              "fix": 0.3831,
              "addition": 0.395
            }
          }
        },
        "usage": {
          "input_tokens": 148,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.0274,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 0,
      "case_id": "release-note-group-07",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "fix",
        "addition"
      ],
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "elapsed_ms": 15.32445801422,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "breaking": 0.1442,
              "fix": 0.6103,
              "addition": 0.2455
            },
            "confidence": 0.1576
          }
        },
        "usage": {
          "input_tokens": 142,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.1576,
            "action": {
              "act_probability": 1
            },
            "choice": "fix",
            "probabilities": {
              "breaking": 0.1442,
              "fix": 0.6103,
              "addition": 0.2455
            }
          }
        },
        "usage": {
          "input_tokens": 142,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.1576,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 1,
      "case_id": "release-note-group-07",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "addition",
        "fix"
      ],
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "elapsed_ms": 15.824500005692244,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "breaking": 0.1333,
              "addition": 0.2337,
              "fix": 0.633
            },
            "confidence": 0.1828
          }
        },
        "usage": {
          "input_tokens": 142,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.1828,
            "action": {
              "act_probability": 1
            },
            "choice": "fix",
            "probabilities": {
              "breaking": 0.1333,
              "addition": 0.2337,
              "fix": 0.633
            }
          }
        },
        "usage": {
          "input_tokens": 142,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.1828,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 2,
      "case_id": "release-note-group-07",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "fix",
        "breaking",
        "addition"
      ],
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "elapsed_ms": 14.974792022258043,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "fix": 0.6411,
              "breaking": 0.1421,
              "addition": 0.2168
            },
            "confidence": 0.1865
          }
        },
        "usage": {
          "input_tokens": 142,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.1865,
            "action": {
              "act_probability": 1
            },
            "choice": "fix",
            "probabilities": {
              "fix": 0.6411,
              "breaking": 0.1421,
              "addition": 0.2168
            }
          }
        },
        "usage": {
          "input_tokens": 142,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.1865,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 3,
      "case_id": "release-note-group-07",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "fix",
        "addition",
        "breaking"
      ],
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "elapsed_ms": 14.896708016749471,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "fix": 0.6519,
              "addition": 0.223,
              "breaking": 0.1251
            },
            "confidence": 0.2048
          }
        },
        "usage": {
          "input_tokens": 142,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.2048,
            "action": {
              "act_probability": 1
            },
            "choice": "fix",
            "probabilities": {
              "fix": 0.6519,
              "addition": 0.223,
              "breaking": 0.1251
            }
          }
        },
        "usage": {
          "input_tokens": 142,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.2048,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 4,
      "case_id": "release-note-group-07",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "breaking",
        "fix"
      ],
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "elapsed_ms": 15.056542004458606,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "addition": 0.2278,
              "breaking": 0.1523,
              "fix": 0.6199
            },
            "confidence": 0.1625
          }
        },
        "usage": {
          "input_tokens": 142,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.1625,
            "action": {
              "act_probability": 1
            },
            "choice": "fix",
            "probabilities": {
              "addition": 0.2278,
              "breaking": 0.1523,
              "fix": 0.6199
            }
          }
        },
        "usage": {
          "input_tokens": 142,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.1625,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 5,
      "case_id": "release-note-group-07",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "fix",
        "breaking"
      ],
      "expected": {
        "type": "choice",
        "choice": "fix"
      },
      "elapsed_ms": 14.64633399154991,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "addition": 0.2549,
              "fix": 0.5942,
              "breaking": 0.1508
            },
            "confidence": 0.1416
          }
        },
        "usage": {
          "input_tokens": 142,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.1416,
            "action": {
              "act_probability": 1
            },
            "choice": "fix",
            "probabilities": {
              "addition": 0.2549,
              "fix": 0.5942,
              "breaking": 0.1508
            }
          }
        },
        "usage": {
          "input_tokens": 142,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.1416,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 0,
      "case_id": "release-note-group-08",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "fix",
        "addition"
      ],
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "elapsed_ms": 14.499708020593971,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "breaking": 0.289,
              "fix": 0.4007,
              "addition": 0.3103
            },
            "confidence": 0.0094
          }
        },
        "usage": {
          "input_tokens": 138,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0094,
            "action": {
              "act_probability": 1
            },
            "choice": "fix",
            "probabilities": {
              "breaking": 0.289,
              "fix": 0.4007,
              "addition": 0.3103
            }
          }
        },
        "usage": {
          "input_tokens": 138,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.0094,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 1,
      "case_id": "release-note-group-08",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "breaking",
        "addition",
        "fix"
      ],
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "elapsed_ms": 15.337749966420233,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "breaking": 0.3263,
              "addition": 0.3209,
              "fix": 0.3529
            },
            "confidence": 0.0008
          }
        },
        "usage": {
          "input_tokens": 138,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0008,
            "action": {
              "act_probability": 1
            },
            "choice": "fix",
            "probabilities": {
              "breaking": 0.3263,
              "addition": 0.3209,
              "fix": 0.3529
            }
          }
        },
        "usage": {
          "input_tokens": 138,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.0008,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 2,
      "case_id": "release-note-group-08",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "fix",
        "breaking",
        "addition"
      ],
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "elapsed_ms": 14.622624963521957,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "fix": 0.3705,
              "breaking": 0.3053,
              "addition": 0.3242
            },
            "confidence": 0.003
          }
        },
        "usage": {
          "input_tokens": 138,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.003,
            "action": {
              "act_probability": 1
            },
            "choice": "fix",
            "probabilities": {
              "fix": 0.3705,
              "breaking": 0.3053,
              "addition": 0.3242
            }
          }
        },
        "usage": {
          "input_tokens": 138,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.003,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 3,
      "case_id": "release-note-group-08",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "fix",
        "addition",
        "breaking"
      ],
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "elapsed_ms": 14.71933297580108,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "fix": 0.3692,
              "addition": 0.3449,
              "breaking": 0.2859
            },
            "confidence": 0.0051
          }
        },
        "usage": {
          "input_tokens": 138,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0051,
            "action": {
              "act_probability": 1
            },
            "choice": "fix",
            "probabilities": {
              "fix": 0.3692,
              "addition": 0.3449,
              "breaking": 0.2859
            }
          }
        },
        "usage": {
          "input_tokens": 138,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.0051,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 4,
      "case_id": "release-note-group-08",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "breaking",
        "fix"
      ],
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "elapsed_ms": 14.858125010505319,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "addition",
            "probabilities": {
              "addition": 0.3663,
              "breaking": 0.2697,
              "fix": 0.3639
            },
            "confidence": 0.0086
          }
        },
        "usage": {
          "input_tokens": 138,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0086,
            "action": {
              "act_probability": 1
            },
            "choice": "addition",
            "probabilities": {
              "addition": 0.3663,
              "breaking": 0.2697,
              "fix": 0.3639
            }
          }
        },
        "usage": {
          "input_tokens": 138,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": true,
        "predicted": "addition",
        "score_absolute_error": null,
        "confidence": 0.0086,
        "tie": false
      }
    },
    {
      "phase": "permutation",
      "repetition": 5,
      "case_id": "release-note-group-08",
      "family": "release-note-group",
      "type": "choice",
      "order": [
        "addition",
        "fix",
        "breaking"
      ],
      "expected": {
        "type": "choice",
        "choice": "addition"
      },
      "elapsed_ms": 14.611082966439426,
      "http_status": null,
      "valid": true,
      "error": null,
      "response": {
        "model": "aac6fef/laya-typed-decisions-mlx@28416e78cb26a239a4eabaa2e084904ec5e6cacb",
        "answers": {
          "decision": {
            "type": "choice",
            "choice": "fix",
            "probabilities": {
              "addition": 0.3283,
              "fix": 0.3602,
              "breaking": 0.3115
            },
            "confidence": 0.0017
          }
        },
        "usage": {
          "input_tokens": 138,
          "output_tokens": 0
        }
      },
      "raw_response": {
        "model": "laya-rl-agent",
        "answers": {
          "decision": {
            "type": "choice",
            "confidence": 0.0017,
            "action": {
              "act_probability": 1
            },
            "choice": "fix",
            "probabilities": {
              "addition": 0.3283,
              "fix": 0.3602,
              "breaking": 0.3115
            }
          }
        },
        "usage": {
          "input_tokens": 138,
          "output_tokens": 0
        }
      },
      "worker_error": null,
      "grade": {
        "correct": false,
        "predicted": "fix",
        "score_absolute_error": null,
        "confidence": 0.0017,
        "tie": false
      }
    }
  ],
  "summary": {
    "attempted_calls": 362,
    "skipped_calls": 0,
    "first_pass_planned": 72,
    "first_pass_observed": 72,
    "first_pass_skipped": 0,
    "first_pass_correct": 30,
    "by_type": {
      "choice": {
        "planned": 24,
        "observed": 24,
        "skipped": 0,
        "correct": 9,
        "valid": 24,
        "ties": 0
      },
      "noul": {
        "planned": 24,
        "observed": 24,
        "skipped": 0,
        "correct": 13,
        "valid": 24,
        "ties": 0
      },
      "score": {
        "planned": 24,
        "observed": 24,
        "skipped": 0,
        "correct": 8,
        "valid": 24,
        "ties": 0
      }
    },
    "by_family": {
      "image-layout": {
        "observed": 8,
        "correct": 2
      },
      "notification-delivery": {
        "observed": 8,
        "correct": 2
      },
      "release-note-group": {
        "observed": 8,
        "correct": 5
      },
      "demo-discount": {
        "observed": 8,
        "correct": 6
      },
      "overnight-counter": {
        "observed": 8,
        "correct": 3
      },
      "tag-filter": {
        "observed": 8,
        "correct": 4
      },
      "mini-game-points": {
        "observed": 8,
        "correct": 3
      },
      "display-detail": {
        "observed": 8,
        "correct": 3
      },
      "glossary-strength": {
        "observed": 8,
        "correct": 2
      }
    },
    "measured_attempts": 216,
    "measured_valid": 216,
    "measured_failures": 0,
    "measured_correct": 90,
    "p50_ms": 9.737958025652915,
    "p95_ms": 15.324584033805877,
    "p50_valid_ms": 9.737958025652915,
    "p95_valid_ms": 15.324584033805877,
    "measured_elapsed_ms": 2410.049458,
    "valid_decisions_per_second": 89.62471673890437,
    "first_score_mae": 0.9349250000000002,
    "score_mae_observed": 24,
    "permutations": {
      "attempts": 144,
      "valid": 144,
      "correct": 57,
      "ties": 0,
      "complete_case_groups": 24,
      "invariant_case_groups": 20,
      "by_correct_option_position": [
        {
          "position": 0,
          "observed": 48,
          "valid": 48,
          "correct": 19
        },
        {
          "position": 1,
          "observed": 48,
          "valid": 48,
          "correct": 18
        },
        {
          "position": 2,
          "observed": 48,
          "valid": 48,
          "correct": 20
        }
      ]
    },
    "measured_usage": {
      "known_calls": 216,
      "unknown_calls": 0,
      "input_tokens": 29157,
      "output_tokens": 0
    },
    "all_usage": {
      "known_calls": 362,
      "unknown_calls": 0,
      "input_tokens": 48860,
      "output_tokens": 0
    },
    "estimated_measured_input_cost_usd": null,
    "estimated_all_input_cost_usd": null
  },
  "cleanup": {
    "event": "disposed",
    "active_memory_bytes": 22,
    "cache_memory_bytes": 0
  },
  "model_load_ms": 78.25054199201986,
  "worker_exit": {
    "code": null,
    "signal": "SIGTERM"
  },
  "worker_stderr_bytes": 0
}