{ "version": 1, "benchmark": "forms-v1-local-adapter", "status": "complete", "started_at": "2026-09-19T17:25:50.876Z", "finished_at": "2026-09-19T17:27:08.707Z", "requested_model_ids": [ "cua-s1-forms", "needle3", "qwen3-0.6b", "qwen3-1.7b" ], "source": { "commit": "83ca2991a44c84183df116d33877f9ee74a97116", "relevant_worktree_modified": false, "harness_sha256": "79c0ff8ed8c65cc0d4ff4d6af34d9209e0fd93c2d72560a9b9600a1aeac2446c", "fixture_sha256": "8e3073ac4ad1fb460fe574084b40631577ece6a49e190fca2d89659cee15604c", "registry_source_sha256": "51afe7d4981c75028cd0ea78821755c458eff95d92b017e7174ede1b20b3fe13", "runtime_source_sha256": "02841e61a07ecd12bcec96829757a17300cffaa53414e807485882fffbaad7e3", "lockfile_sha256": "2641927e4207467ad05df6fb30afa5384d68385b8c6271576ff7bf544c9fb178" }, "environment": { "platform": "darwin", "arch": "arm64", "os_release": "25.5.0", "bun": "1.3.14", "cpu_model": "Apple M5 Max", "logical_cpus": 18, "system_memory_bytes": 38654705664, "native_backend": "metal", "gpu_offloading": true, "native_probe": { "ok": true, "elapsed_ms": 331, "failure_code": null }, "native_backend_evidence": "Separate capability probe outside measured calls; the adapter does not expose a per-inference GPU measurement.", "memory_measurement": "System capacity only; no peak process-group RAM or GPU-memory measurement." }, "methodology": { "fixture": "forms-v1", "unique_cases": 20, "fresh_runner_samples": 3, "warmups": 2, "repetitions": 5, "repeated_calls_per_model": 100, "concurrency": 1, "context_tokens": 2048, "request_timeout_ms": 60000, "run_timeout_ms": 900000, "stop_rule": "Stop a model after three consecutive invalid/error responses; record actual and skipped calls. Incorrect but valid answers do not trigger this rule.", "latency_boundary": "LocalRunner.decide wall time, including its manifest lookup, adaptation, model work and worker IPC; excludes response validation, HTTP, downloads and hash verification.", "first_call_boundary": "Fresh LocalRunner, not a cold machine: verification has read weights into OS file caches; parent runtime/JIT and OS caches are not reset.", "percentile_method": "nearest rank, ceil(p*n)-1 in sorted elapsed times", "quality_scope": "Twenty authored synthetic form-action cases. This small public sanity check does not establish general model quality.", "raw_answers": "Only the fixed public synthetic fixture is accepted. No private inputs, config, credentials or hosted calls are read." }, "models": [ { "id": "cua-s1-forms", "kind": "scorer", "adapter": "option-scorer", "weight_sha256": "f5077f0c9baf6b5fc10f21512e1aa15207a395598416a6ffdd95f0d3dd5ab8df", "weight_bytes": 2840436, "engine_sha256": null, "engine_bytes": null, "usage_semantics": "UTF-8 byte option scoring without generation; protocol token counters are 0 and token throughput is not applicable.", "repeated_state": "Same runner and resident model after two warmups.", "status": "complete", "planned_calls": 105, "skipped_calls": 0, "samples": [ { "phase": "fresh_runner", "repetition": 0, "case_id": "form-01", "expected": "submit", "elapsed_ms": 79.86441599999989, "schema_valid": true, "correct": true, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.581, "correct": 0.352, "wait": 0.067 }, "confidence": 0.371 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.371 } } }, { "phase": "fresh_runner", "repetition": 1, "case_id": "form-01", "expected": "submit", "elapsed_ms": 66.30604199999993, "schema_valid": true, "correct": true, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.581, "correct": 0.352, "wait": 0.067 }, "confidence": 0.371 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.371 } } }, { "phase": "fresh_runner", "repetition": 2, "case_id": "form-01", "expected": "submit", "elapsed_ms": 61.58375000000001, "schema_valid": true, "correct": true, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.581, "correct": 0.352, "wait": 0.067 }, "confidence": 0.371 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.371 } } }, { "phase": "warmup", "repetition": 0, "case_id": "form-01", "expected": "submit", "elapsed_ms": 63.54825000000005, "schema_valid": true, "correct": true, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.581, "correct": 0.352, "wait": 0.067 }, "confidence": 0.371 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.371 } } }, { "phase": "warmup", "repetition": 1, "case_id": "form-02", "expected": "correct", "elapsed_ms": 49.73562500000003, "schema_valid": true, "correct": true, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.04, "correct": 0.86, "wait": 0.1 }, "confidence": 0.79 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.79 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-01", "expected": "submit", "elapsed_ms": 62.321290999999974, "schema_valid": true, "correct": true, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.581, "correct": 0.352, "wait": 0.067 }, "confidence": 0.371 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.371 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-02", "expected": "correct", "elapsed_ms": 49.47929199999999, "schema_valid": true, "correct": true, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.04, "correct": 0.86, "wait": 0.1 }, "confidence": 0.79 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.79 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-03", "expected": "wait", "elapsed_ms": 51.46137499999986, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.134, "correct": 0.865, "wait": 0 }, "confidence": 0.798 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.798 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-04", "expected": "submit", "elapsed_ms": 55.711333000000195, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "wait", "probabilities": { "submit": 0.03, "correct": 0.082, "wait": 0.888 }, "confidence": 0.831 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.831 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-05", "expected": "correct", "elapsed_ms": 50.4290840000001, "schema_valid": true, "correct": true, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.094, "correct": 0.777, "wait": 0.129 }, "confidence": 0.665 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.665 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-06", "expected": "wait", "elapsed_ms": 55.53233299999988, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.73, "correct": 0.269, "wait": 0.002 }, "confidence": 0.594 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.594 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-07", "expected": "submit", "elapsed_ms": 57.75712500000009, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.476, "correct": 0.524, "wait": 0 }, "confidence": 0.286 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.286 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-08", "expected": "correct", "elapsed_ms": 50.090417000000116, "schema_valid": true, "correct": true, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.153, "correct": 0.683, "wait": 0.164 }, "confidence": 0.524 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.524 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-09", "expected": "wait", "elapsed_ms": 53.584249999999884, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.284, "correct": 0.709, "wait": 0.007 }, "confidence": 0.563 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.563 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-10", "expected": "submit", "elapsed_ms": 54.44337499999983, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.182, "correct": 0.665, "wait": 0.153 }, "confidence": 0.498 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.498 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-11", "expected": "correct", "elapsed_ms": 50.9787080000001, "schema_valid": true, "correct": true, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.101, "correct": 0.893, "wait": 0.006 }, "confidence": 0.839 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.839 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-12", "expected": "wait", "elapsed_ms": 54.80820799999992, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.272, "correct": 0.721, "wait": 0.007 }, "confidence": 0.581 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.581 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-13", "expected": "submit", "elapsed_ms": 54.152250000000095, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.246, "correct": 0.726, "wait": 0.029 }, "confidence": 0.589 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.589 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-14", "expected": "correct", "elapsed_ms": 51.31845800000019, "schema_valid": true, "correct": true, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.114, "correct": 0.845, "wait": 0.041 }, "confidence": 0.768 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.768 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-15", "expected": "wait", "elapsed_ms": 54.94541700000036, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.177, "correct": 0.822, "wait": 0 }, "confidence": 0.733 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.733 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-16", "expected": "submit", "elapsed_ms": 58.8559580000001, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.454, "correct": 0.542, "wait": 0.003 }, "confidence": 0.314 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.314 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-17", "expected": "correct", "elapsed_ms": 60.98229100000026, "schema_valid": true, "correct": true, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.205, "correct": 0.794, "wait": 0.001 }, "confidence": 0.691 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.691 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-18", "expected": "wait", "elapsed_ms": 60.50716700000021, "schema_valid": true, "correct": true, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "wait", "probabilities": { "submit": 0.042, "correct": 0.472, "wait": 0.486 }, "confidence": 0.229 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.229 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-19", "expected": "submit", "elapsed_ms": 58.14566599999989, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.082, "correct": 0.914, "wait": 0.004 }, "confidence": 0.871 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.871 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-20", "expected": "correct", "elapsed_ms": 49.57237499999974, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.937, "correct": 0.063, "wait": 0 }, "confidence": 0.905 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.905 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-01", "expected": "submit", "elapsed_ms": 59.50358300000016, "schema_valid": true, "correct": true, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.581, "correct": 0.352, "wait": 0.067 }, "confidence": 0.371 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.371 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-02", "expected": "correct", "elapsed_ms": 50.69770800000015, "schema_valid": true, "correct": true, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.04, "correct": 0.86, "wait": 0.1 }, "confidence": 0.79 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.79 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-03", "expected": "wait", "elapsed_ms": 51.69275000000016, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.134, "correct": 0.865, "wait": 0 }, "confidence": 0.798 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.798 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-04", "expected": "submit", "elapsed_ms": 57.860833000000184, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "wait", "probabilities": { "submit": 0.03, "correct": 0.082, "wait": 0.888 }, "confidence": 0.831 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.831 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-05", "expected": "correct", "elapsed_ms": 51.936709000000064, "schema_valid": true, "correct": true, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.094, "correct": 0.777, "wait": 0.129 }, "confidence": 0.665 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.665 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-06", "expected": "wait", "elapsed_ms": 53.545500000000175, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.73, "correct": 0.269, "wait": 0.002 }, "confidence": 0.594 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.594 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-07", "expected": "submit", "elapsed_ms": 58.423125000000255, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.476, "correct": 0.524, "wait": 0 }, "confidence": 0.286 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.286 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-08", "expected": "correct", "elapsed_ms": 49.04679199999964, "schema_valid": true, "correct": true, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.153, "correct": 0.683, "wait": 0.164 }, "confidence": 0.524 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.524 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-09", "expected": "wait", "elapsed_ms": 52.31387500000028, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.284, "correct": 0.709, "wait": 0.007 }, "confidence": 0.563 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.563 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-10", "expected": "submit", "elapsed_ms": 55.95283300000028, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.182, "correct": 0.665, "wait": 0.153 }, "confidence": 0.498 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.498 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-11", "expected": "correct", "elapsed_ms": 49.8567909999997, "schema_valid": true, "correct": true, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.101, "correct": 0.893, "wait": 0.006 }, "confidence": 0.839 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.839 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-12", "expected": "wait", "elapsed_ms": 52.45979200000011, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.272, "correct": 0.721, "wait": 0.007 }, "confidence": 0.581 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.581 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-13", "expected": "submit", "elapsed_ms": 53.76145799999995, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.246, "correct": 0.726, "wait": 0.029 }, "confidence": 0.589 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.589 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-14", "expected": "correct", "elapsed_ms": 54.02229200000011, "schema_valid": true, "correct": true, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.114, "correct": 0.845, "wait": 0.041 }, "confidence": 0.768 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.768 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-15", "expected": "wait", "elapsed_ms": 56.80979100000013, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.177, "correct": 0.822, "wait": 0 }, "confidence": 0.733 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.733 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-16", "expected": "submit", "elapsed_ms": 56.675666999999976, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.454, "correct": 0.542, "wait": 0.003 }, "confidence": 0.314 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.314 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-17", "expected": "correct", "elapsed_ms": 56.59495800000013, "schema_valid": true, "correct": true, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.205, "correct": 0.794, "wait": 0.001 }, "confidence": 0.691 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.691 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-18", "expected": "wait", "elapsed_ms": 62.01058400000011, "schema_valid": true, "correct": true, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "wait", "probabilities": { "submit": 0.042, "correct": 0.472, "wait": 0.486 }, "confidence": 0.229 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.229 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-19", "expected": "submit", "elapsed_ms": 58.557624999999916, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.082, "correct": 0.914, "wait": 0.004 }, "confidence": 0.871 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.871 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-20", "expected": "correct", "elapsed_ms": 49.35558400000036, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.937, "correct": 0.063, "wait": 0 }, "confidence": 0.905 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.905 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-01", "expected": "submit", "elapsed_ms": 61.41850000000022, "schema_valid": true, "correct": true, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.581, "correct": 0.352, "wait": 0.067 }, "confidence": 0.371 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.371 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-02", "expected": "correct", "elapsed_ms": 51.52195799999981, "schema_valid": true, "correct": true, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.04, "correct": 0.86, "wait": 0.1 }, "confidence": 0.79 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.79 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-03", "expected": "wait", "elapsed_ms": 53.95470800000021, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.134, "correct": 0.865, "wait": 0 }, "confidence": 0.798 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.798 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-04", "expected": "submit", "elapsed_ms": 58.14183299999968, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "wait", "probabilities": { "submit": 0.03, "correct": 0.082, "wait": 0.888 }, "confidence": 0.831 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.831 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-05", "expected": "correct", "elapsed_ms": 52.89300000000003, "schema_valid": true, "correct": true, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.094, "correct": 0.777, "wait": 0.129 }, "confidence": 0.665 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.665 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-06", "expected": "wait", "elapsed_ms": 54.58766700000024, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.73, "correct": 0.269, "wait": 0.002 }, "confidence": 0.594 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.594 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-07", "expected": "submit", "elapsed_ms": 56.880709000000024, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.476, "correct": 0.524, "wait": 0 }, "confidence": 0.286 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.286 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-08", "expected": "correct", "elapsed_ms": 48.72333299999991, "schema_valid": true, "correct": true, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.153, "correct": 0.683, "wait": 0.164 }, "confidence": 0.524 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.524 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-09", "expected": "wait", "elapsed_ms": 53.25725000000057, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.284, "correct": 0.709, "wait": 0.007 }, "confidence": 0.563 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.563 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-10", "expected": "submit", "elapsed_ms": 55.71520800000053, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.182, "correct": 0.665, "wait": 0.153 }, "confidence": 0.498 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.498 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-11", "expected": "correct", "elapsed_ms": 48.75933299999997, "schema_valid": true, "correct": true, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.101, "correct": 0.893, "wait": 0.006 }, "confidence": 0.839 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.839 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-12", "expected": "wait", "elapsed_ms": 54.20012499999939, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.272, "correct": 0.721, "wait": 0.007 }, "confidence": 0.581 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.581 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-13", "expected": "submit", "elapsed_ms": 53.9037500000004, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.246, "correct": 0.726, "wait": 0.029 }, "confidence": 0.589 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.589 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-14", "expected": "correct", "elapsed_ms": 54.38587499999994, "schema_valid": true, "correct": true, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.114, "correct": 0.845, "wait": 0.041 }, "confidence": 0.768 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.768 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-15", "expected": "wait", "elapsed_ms": 57.27379199999996, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.177, "correct": 0.822, "wait": 0 }, "confidence": 0.733 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.733 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-16", "expected": "submit", "elapsed_ms": 57.41675000000032, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.454, "correct": 0.542, "wait": 0.003 }, "confidence": 0.314 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.314 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-17", "expected": "correct", "elapsed_ms": 54.50379099999918, "schema_valid": true, "correct": true, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.205, "correct": 0.794, "wait": 0.001 }, "confidence": 0.691 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.691 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-18", "expected": "wait", "elapsed_ms": 58.05562500000087, "schema_valid": true, "correct": true, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "wait", "probabilities": { "submit": 0.042, "correct": 0.472, "wait": 0.486 }, "confidence": 0.229 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.229 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-19", "expected": "submit", "elapsed_ms": 57.61591700000008, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.082, "correct": 0.914, "wait": 0.004 }, "confidence": 0.871 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.871 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-20", "expected": "correct", "elapsed_ms": 49.89866699999948, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.937, "correct": 0.063, "wait": 0 }, "confidence": 0.905 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.905 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-01", "expected": "submit", "elapsed_ms": 60.33495800000037, "schema_valid": true, "correct": true, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.581, "correct": 0.352, "wait": 0.067 }, "confidence": 0.371 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.371 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-02", "expected": "correct", "elapsed_ms": 50.69354199999998, "schema_valid": true, "correct": true, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.04, "correct": 0.86, "wait": 0.1 }, "confidence": 0.79 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.79 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-03", "expected": "wait", "elapsed_ms": 52.5644999999995, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.134, "correct": 0.865, "wait": 0 }, "confidence": 0.798 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.798 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-04", "expected": "submit", "elapsed_ms": 58.229583999999704, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "wait", "probabilities": { "submit": 0.03, "correct": 0.082, "wait": 0.888 }, "confidence": 0.831 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.831 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-05", "expected": "correct", "elapsed_ms": 51.82666699999936, "schema_valid": true, "correct": true, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.094, "correct": 0.777, "wait": 0.129 }, "confidence": 0.665 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.665 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-06", "expected": "wait", "elapsed_ms": 54.262541000000056, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.73, "correct": 0.269, "wait": 0.002 }, "confidence": 0.594 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.594 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-07", "expected": "submit", "elapsed_ms": 56.98358300000018, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.476, "correct": 0.524, "wait": 0 }, "confidence": 0.286 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.286 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-08", "expected": "correct", "elapsed_ms": 49.3949590000002, "schema_valid": true, "correct": true, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.153, "correct": 0.683, "wait": 0.164 }, "confidence": 0.524 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.524 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-09", "expected": "wait", "elapsed_ms": 54.47554200000013, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.284, "correct": 0.709, "wait": 0.007 }, "confidence": 0.563 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.563 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-10", "expected": "submit", "elapsed_ms": 57.10862499999985, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.182, "correct": 0.665, "wait": 0.153 }, "confidence": 0.498 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.498 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-11", "expected": "correct", "elapsed_ms": 50.40520800000013, "schema_valid": true, "correct": true, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.101, "correct": 0.893, "wait": 0.006 }, "confidence": 0.839 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.839 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-12", "expected": "wait", "elapsed_ms": 55.96283400000084, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.272, "correct": 0.721, "wait": 0.007 }, "confidence": 0.581 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.581 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-13", "expected": "submit", "elapsed_ms": 55.7524999999996, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.246, "correct": 0.726, "wait": 0.029 }, "confidence": 0.589 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.589 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-14", "expected": "correct", "elapsed_ms": 54.4048329999996, "schema_valid": true, "correct": true, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.114, "correct": 0.845, "wait": 0.041 }, "confidence": 0.768 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.768 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-15", "expected": "wait", "elapsed_ms": 57.54545899999994, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.177, "correct": 0.822, "wait": 0 }, "confidence": 0.733 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.733 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-16", "expected": "submit", "elapsed_ms": 57.64041599999928, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.454, "correct": 0.542, "wait": 0.003 }, "confidence": 0.314 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.314 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-17", "expected": "correct", "elapsed_ms": 56.87383399999999, "schema_valid": true, "correct": true, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.205, "correct": 0.794, "wait": 0.001 }, "confidence": 0.691 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.691 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-18", "expected": "wait", "elapsed_ms": 60.07866699999977, "schema_valid": true, "correct": true, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "wait", "probabilities": { "submit": 0.042, "correct": 0.472, "wait": 0.486 }, "confidence": 0.229 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.229 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-19", "expected": "submit", "elapsed_ms": 56.95562500000051, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.082, "correct": 0.914, "wait": 0.004 }, "confidence": 0.871 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.871 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-20", "expected": "correct", "elapsed_ms": 49.066124999999374, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.937, "correct": 0.063, "wait": 0 }, "confidence": 0.905 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.905 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-01", "expected": "submit", "elapsed_ms": 61.927999999999884, "schema_valid": true, "correct": true, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.581, "correct": 0.352, "wait": 0.067 }, "confidence": 0.371 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.371 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-02", "expected": "correct", "elapsed_ms": 51.54833400000007, "schema_valid": true, "correct": true, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.04, "correct": 0.86, "wait": 0.1 }, "confidence": 0.79 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.79 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-03", "expected": "wait", "elapsed_ms": 54.76237500000025, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.134, "correct": 0.865, "wait": 0 }, "confidence": 0.798 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.798 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-04", "expected": "submit", "elapsed_ms": 59.949125000000095, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "wait", "probabilities": { "submit": 0.03, "correct": 0.082, "wait": 0.888 }, "confidence": 0.831 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.831 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-05", "expected": "correct", "elapsed_ms": 53.02070800000001, "schema_valid": true, "correct": true, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.094, "correct": 0.777, "wait": 0.129 }, "confidence": 0.665 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.665 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-06", "expected": "wait", "elapsed_ms": 55.67108299999927, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.73, "correct": 0.269, "wait": 0.002 }, "confidence": 0.594 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.594 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-07", "expected": "submit", "elapsed_ms": 57.5689169999996, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.476, "correct": 0.524, "wait": 0 }, "confidence": 0.286 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.286 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-08", "expected": "correct", "elapsed_ms": 51.00245900000027, "schema_valid": true, "correct": true, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.153, "correct": 0.683, "wait": 0.164 }, "confidence": 0.524 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.524 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-09", "expected": "wait", "elapsed_ms": 53.78179100000034, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.284, "correct": 0.709, "wait": 0.007 }, "confidence": 0.563 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.563 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-10", "expected": "submit", "elapsed_ms": 57.371791999999914, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.182, "correct": 0.665, "wait": 0.153 }, "confidence": 0.498 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.498 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-11", "expected": "correct", "elapsed_ms": 51.62029200000052, "schema_valid": true, "correct": true, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.101, "correct": 0.893, "wait": 0.006 }, "confidence": 0.839 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.839 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-12", "expected": "wait", "elapsed_ms": 56.09216699999979, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.272, "correct": 0.721, "wait": 0.007 }, "confidence": 0.581 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.581 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-13", "expected": "submit", "elapsed_ms": 55.01445799999965, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.246, "correct": 0.726, "wait": 0.029 }, "confidence": 0.589 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.589 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-14", "expected": "correct", "elapsed_ms": 54.38420799999949, "schema_valid": true, "correct": true, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.114, "correct": 0.845, "wait": 0.041 }, "confidence": 0.768 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.768 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-15", "expected": "wait", "elapsed_ms": 58.491416999999274, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.177, "correct": 0.822, "wait": 0 }, "confidence": 0.733 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.733 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-16", "expected": "submit", "elapsed_ms": 58.561499999999796, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.454, "correct": 0.542, "wait": 0.003 }, "confidence": 0.314 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.314 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-17", "expected": "correct", "elapsed_ms": 57.618499999999585, "schema_valid": true, "correct": true, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.205, "correct": 0.794, "wait": 0.001 }, "confidence": 0.691 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.691 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-18", "expected": "wait", "elapsed_ms": 60.736000000000786, "schema_valid": true, "correct": true, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "wait", "probabilities": { "submit": 0.042, "correct": 0.472, "wait": 0.486 }, "confidence": 0.229 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.229 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-19", "expected": "submit", "elapsed_ms": 58.81854199999998, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.082, "correct": 0.914, "wait": 0.004 }, "confidence": 0.871 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.871 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-20", "expected": "correct", "elapsed_ms": 49.41487500000039, "schema_valid": true, "correct": false, "error": null, "response": { "model": "cua-s1-forms", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.937, "correct": 0.063, "wait": 0 }, "confidence": 0.905 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.905 } } } ], "summary": { "fresh_runner_ms": [ 79.86441599999989, 66.30604199999993, 61.58375000000001 ], "repeated_attempts": 100, "repeated_valid": 100, "repeated_failures": 0, "repeated_correct": 40, "unique_first_pass_cases": 20, "unique_first_pass_correct": 8, "accuracy_denominator": "unique_first_pass_cases is the observed denominator; a complete pass contains 20 authored cases. Failures count as incorrect. Repeats are not independent examples.", "p50_attempt_ms": 54.76237500000025, "p95_attempt_ms": 60.736000000000786, "p50_valid_ms": 54.76237500000025, "p95_valid_ms": 60.736000000000786, "repeated_total_elapsed_ms": 5508.096958, "attempted_decisions_per_second": 18.155090725982824, "valid_decisions_per_second": 18.155090725982824, "repeated_reported_input_tokens": 0, "repeated_reported_output_tokens": 0 } }, { "id": "needle3", "kind": "needle", "adapter": "needle-extract", "weight_sha256": "c9d915eca282ed42d1a09b143b592adb4cc6744ffe2d294adf5cfc5548170c38", "weight_bytes": 35335380, "engine_sha256": "bb3921bb87c6492da44b93326211d461cc2f078c5e55d4d90206e3ff69e924b9", "engine_bytes": 791384, "usage_semantics": "Native token usage is not exposed by this adapter; protocol 0/0 counters do not mean zero token work.", "repeated_state": "New engine process for every request; no resident-model warm result.", "status": "complete", "planned_calls": 105, "skipped_calls": 0, "samples": [ { "phase": "fresh_runner", "repetition": 0, "case_id": "form-01", "expected": "submit", "elapsed_ms": 303.46408299999985, "schema_valid": true, "correct": true, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "fresh_runner", "repetition": 1, "case_id": "form-01", "expected": "submit", "elapsed_ms": 136.74175000000014, "schema_valid": true, "correct": true, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "fresh_runner", "repetition": 2, "case_id": "form-01", "expected": "submit", "elapsed_ms": 143.05608399999983, "schema_valid": true, "correct": true, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "warmup", "repetition": 0, "case_id": "form-01", "expected": "submit", "elapsed_ms": 152.51675000000068, "schema_valid": true, "correct": true, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "warmup", "repetition": 1, "case_id": "form-02", "expected": "correct", "elapsed_ms": 146.4102499999999, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-01", "expected": "submit", "elapsed_ms": 161.53154199999972, "schema_valid": true, "correct": true, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-02", "expected": "correct", "elapsed_ms": 147.61375000000044, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-03", "expected": "wait", "elapsed_ms": 140.70908299999974, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.97, "correct": 0.015, "wait": 0.015 }, "confidence": 0.97 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.97 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-04", "expected": "submit", "elapsed_ms": 194.68904199999997, "schema_valid": true, "correct": true, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.742, "correct": 0.129, "wait": 0.129 }, "confidence": 0.742 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.742 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-05", "expected": "correct", "elapsed_ms": 150.10120800000004, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-06", "expected": "wait", "elapsed_ms": 161.05466700000034, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-07", "expected": "submit", "elapsed_ms": 146.50729200000023, "schema_valid": true, "correct": true, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.974, "correct": 0.013, "wait": 0.013 }, "confidence": 0.974 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.974 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-08", "expected": "correct", "elapsed_ms": 147.8217080000013, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-09", "expected": "wait", "elapsed_ms": 137.7893750000003, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-10", "expected": "submit", "elapsed_ms": 165.19216700000106, "schema_valid": true, "correct": true, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.883, "correct": 0.058, "wait": 0.058 }, "confidence": 0.883 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.883 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-11", "expected": "correct", "elapsed_ms": 159.8021659999995, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.976, "correct": 0.012, "wait": 0.012 }, "confidence": 0.976 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.976 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-12", "expected": "wait", "elapsed_ms": 158.39837500000067, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.994, "correct": 0.003, "wait": 0.003 }, "confidence": 0.994 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.994 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-13", "expected": "submit", "elapsed_ms": 147.76616699999977, "schema_valid": true, "correct": true, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-14", "expected": "correct", "elapsed_ms": 147.32754199999908, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-15", "expected": "wait", "elapsed_ms": 146.75695900000028, "schema_valid": true, "correct": true, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "wait", "probabilities": { "submit": 0.005, "correct": 0.005, "wait": 0.99 }, "confidence": 0.99 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.99 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-16", "expected": "submit", "elapsed_ms": 149.9496250000011, "schema_valid": true, "correct": true, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-17", "expected": "correct", "elapsed_ms": 171.2703330000004, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.991, "correct": 0.005, "wait": 0.005 }, "confidence": 0.991 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.991 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-18", "expected": "wait", "elapsed_ms": 146.83495799999946, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.976, "correct": 0.012, "wait": 0.012 }, "confidence": 0.976 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.976 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-19", "expected": "submit", "elapsed_ms": 153.43091600000116, "schema_valid": true, "correct": true, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.979, "correct": 0.011, "wait": 0.011 }, "confidence": 0.979 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.979 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-20", "expected": "correct", "elapsed_ms": 135.40300000000025, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-01", "expected": "submit", "elapsed_ms": 156.89920800000073, "schema_valid": true, "correct": true, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-02", "expected": "correct", "elapsed_ms": 143.55937500000073, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-03", "expected": "wait", "elapsed_ms": 139.34462499999972, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.97, "correct": 0.015, "wait": 0.015 }, "confidence": 0.97 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.97 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-04", "expected": "submit", "elapsed_ms": 185.83404100000007, "schema_valid": true, "correct": true, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.742, "correct": 0.129, "wait": 0.129 }, "confidence": 0.742 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.742 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-05", "expected": "correct", "elapsed_ms": 142.13466700000026, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-06", "expected": "wait", "elapsed_ms": 147.4657499999994, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-07", "expected": "submit", "elapsed_ms": 142.30625000000146, "schema_valid": true, "correct": true, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.974, "correct": 0.013, "wait": 0.013 }, "confidence": 0.974 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.974 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-08", "expected": "correct", "elapsed_ms": 139.90599999999904, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-09", "expected": "wait", "elapsed_ms": 129.6704579999987, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-10", "expected": "submit", "elapsed_ms": 147.36333299999933, "schema_valid": true, "correct": true, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.883, "correct": 0.058, "wait": 0.058 }, "confidence": 0.883 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.883 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-11", "expected": "correct", "elapsed_ms": 145.0665829999998, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.976, "correct": 0.012, "wait": 0.012 }, "confidence": 0.976 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.976 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-12", "expected": "wait", "elapsed_ms": 142.5905419999999, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.994, "correct": 0.003, "wait": 0.003 }, "confidence": 0.994 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.994 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-13", "expected": "submit", "elapsed_ms": 135.520833999999, "schema_valid": true, "correct": true, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-14", "expected": "correct", "elapsed_ms": 134.67633399999977, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-15", "expected": "wait", "elapsed_ms": 132.78795900000114, "schema_valid": true, "correct": true, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "wait", "probabilities": { "submit": 0.005, "correct": 0.005, "wait": 0.99 }, "confidence": 0.99 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.99 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-16", "expected": "submit", "elapsed_ms": 146.63416600000164, "schema_valid": true, "correct": true, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-17", "expected": "correct", "elapsed_ms": 175.7713330000006, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.991, "correct": 0.005, "wait": 0.005 }, "confidence": 0.991 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.991 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-18", "expected": "wait", "elapsed_ms": 148.59108299999934, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.976, "correct": 0.012, "wait": 0.012 }, "confidence": 0.976 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.976 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-19", "expected": "submit", "elapsed_ms": 155.86966699999903, "schema_valid": true, "correct": true, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.979, "correct": 0.011, "wait": 0.011 }, "confidence": 0.979 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.979 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-20", "expected": "correct", "elapsed_ms": 128.53508300000067, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-01", "expected": "submit", "elapsed_ms": 145.30041700000038, "schema_valid": true, "correct": true, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-02", "expected": "correct", "elapsed_ms": 133.3613750000004, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-03", "expected": "wait", "elapsed_ms": 125.85054199999831, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.97, "correct": 0.015, "wait": 0.015 }, "confidence": 0.97 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.97 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-04", "expected": "submit", "elapsed_ms": 172.60812500000065, "schema_valid": true, "correct": true, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.742, "correct": 0.129, "wait": 0.129 }, "confidence": 0.742 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.742 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-05", "expected": "correct", "elapsed_ms": 130.38275000000067, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-06", "expected": "wait", "elapsed_ms": 134.39866699999948, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-07", "expected": "submit", "elapsed_ms": 133.02670899999976, "schema_valid": true, "correct": true, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.974, "correct": 0.013, "wait": 0.013 }, "confidence": 0.974 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.974 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-08", "expected": "correct", "elapsed_ms": 141.4299579999988, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-09", "expected": "wait", "elapsed_ms": 136.66283400000066, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-10", "expected": "submit", "elapsed_ms": 161.3417499999996, "schema_valid": true, "correct": true, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.883, "correct": 0.058, "wait": 0.058 }, "confidence": 0.883 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.883 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-11", "expected": "correct", "elapsed_ms": 149.9434999999994, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.976, "correct": 0.012, "wait": 0.012 }, "confidence": 0.976 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.976 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-12", "expected": "wait", "elapsed_ms": 154.83795800000007, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.994, "correct": 0.003, "wait": 0.003 }, "confidence": 0.994 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.994 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-13", "expected": "submit", "elapsed_ms": 143.9307919999992, "schema_valid": true, "correct": true, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-14", "expected": "correct", "elapsed_ms": 141.36004199999843, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-15", "expected": "wait", "elapsed_ms": 144.8221250000006, "schema_valid": true, "correct": true, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "wait", "probabilities": { "submit": 0.005, "correct": 0.005, "wait": 0.99 }, "confidence": 0.99 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.99 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-16", "expected": "submit", "elapsed_ms": 139.70391700000073, "schema_valid": true, "correct": true, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-17", "expected": "correct", "elapsed_ms": 158.8412499999995, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.991, "correct": 0.005, "wait": 0.005 }, "confidence": 0.991 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.991 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-18", "expected": "wait", "elapsed_ms": 133.87066699999923, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.976, "correct": 0.012, "wait": 0.012 }, "confidence": 0.976 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.976 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-19", "expected": "submit", "elapsed_ms": 137.84162500000093, "schema_valid": true, "correct": true, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.979, "correct": 0.011, "wait": 0.011 }, "confidence": 0.979 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.979 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-20", "expected": "correct", "elapsed_ms": 122.1942909999998, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-01", "expected": "submit", "elapsed_ms": 148.0635000000002, "schema_valid": true, "correct": true, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-02", "expected": "correct", "elapsed_ms": 137.90466699999888, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-03", "expected": "wait", "elapsed_ms": 133.41366599999674, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.97, "correct": 0.015, "wait": 0.015 }, "confidence": 0.97 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.97 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-04", "expected": "submit", "elapsed_ms": 169.66108299999905, "schema_valid": true, "correct": true, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.742, "correct": 0.129, "wait": 0.129 }, "confidence": 0.742 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.742 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-05", "expected": "correct", "elapsed_ms": 127.45133300000089, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-06", "expected": "wait", "elapsed_ms": 140.82220900000175, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-07", "expected": "submit", "elapsed_ms": 138.44558300000062, "schema_valid": true, "correct": true, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.974, "correct": 0.013, "wait": 0.013 }, "confidence": 0.974 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.974 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-08", "expected": "correct", "elapsed_ms": 141.84449999999924, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-09", "expected": "wait", "elapsed_ms": 139.07554100000198, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-10", "expected": "submit", "elapsed_ms": 163.53991700000188, "schema_valid": true, "correct": true, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.883, "correct": 0.058, "wait": 0.058 }, "confidence": 0.883 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.883 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-11", "expected": "correct", "elapsed_ms": 151.35408400000233, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.976, "correct": 0.012, "wait": 0.012 }, "confidence": 0.976 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.976 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-12", "expected": "wait", "elapsed_ms": 157.22120800000266, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.994, "correct": 0.003, "wait": 0.003 }, "confidence": 0.994 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.994 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-13", "expected": "submit", "elapsed_ms": 142.64391700000124, "schema_valid": true, "correct": true, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-14", "expected": "correct", "elapsed_ms": 142.07337499999994, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-15", "expected": "wait", "elapsed_ms": 145.58150000000023, "schema_valid": true, "correct": true, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "wait", "probabilities": { "submit": 0.005, "correct": 0.005, "wait": 0.99 }, "confidence": 0.99 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.99 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-16", "expected": "submit", "elapsed_ms": 137.8604579999992, "schema_valid": true, "correct": true, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-17", "expected": "correct", "elapsed_ms": 152.77683400000024, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.991, "correct": 0.005, "wait": 0.005 }, "confidence": 0.991 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.991 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-18", "expected": "wait", "elapsed_ms": 130.30816699999923, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.976, "correct": 0.012, "wait": 0.012 }, "confidence": 0.976 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.976 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-19", "expected": "submit", "elapsed_ms": 135.89716699999917, "schema_valid": true, "correct": true, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.979, "correct": 0.011, "wait": 0.011 }, "confidence": 0.979 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.979 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-20", "expected": "correct", "elapsed_ms": 129.10129199999938, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-01", "expected": "submit", "elapsed_ms": 151.40512500000114, "schema_valid": true, "correct": true, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-02", "expected": "correct", "elapsed_ms": 156.4778750000005, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-03", "expected": "wait", "elapsed_ms": 143.91295799999716, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.97, "correct": 0.015, "wait": 0.015 }, "confidence": 0.97 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.97 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-04", "expected": "submit", "elapsed_ms": 192.89862500000163, "schema_valid": true, "correct": true, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.742, "correct": 0.129, "wait": 0.129 }, "confidence": 0.742 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.742 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-05", "expected": "correct", "elapsed_ms": 143.53174999999828, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-06", "expected": "wait", "elapsed_ms": 137.9652500000011, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-07", "expected": "submit", "elapsed_ms": 141.37162499999977, "schema_valid": true, "correct": true, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.974, "correct": 0.013, "wait": 0.013 }, "confidence": 0.974 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.974 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-08", "expected": "correct", "elapsed_ms": 137.9830830000028, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-09", "expected": "wait", "elapsed_ms": 133.04245900000024, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-10", "expected": "submit", "elapsed_ms": 143.97916700000133, "schema_valid": true, "correct": true, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.883, "correct": 0.058, "wait": 0.058 }, "confidence": 0.883 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.883 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-11", "expected": "correct", "elapsed_ms": 144.93604100000084, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.976, "correct": 0.012, "wait": 0.012 }, "confidence": 0.976 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.976 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-12", "expected": "wait", "elapsed_ms": 144.90983399999823, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.994, "correct": 0.003, "wait": 0.003 }, "confidence": 0.994 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.994 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-13", "expected": "submit", "elapsed_ms": 139.24770799999897, "schema_valid": true, "correct": true, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-14", "expected": "correct", "elapsed_ms": 133.66508299999987, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-15", "expected": "wait", "elapsed_ms": 137.7534160000032, "schema_valid": true, "correct": true, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "wait", "probabilities": { "submit": 0.005, "correct": 0.005, "wait": 0.99 }, "confidence": 0.99 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.99 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-16", "expected": "submit", "elapsed_ms": 132.07029200000034, "schema_valid": true, "correct": true, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-17", "expected": "correct", "elapsed_ms": 149.73291699999754, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.991, "correct": 0.005, "wait": 0.005 }, "confidence": 0.991 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.991 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-18", "expected": "wait", "elapsed_ms": 123.34441700000025, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.976, "correct": 0.012, "wait": 0.012 }, "confidence": 0.976 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.976 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-19", "expected": "submit", "elapsed_ms": 132.27512500000012, "schema_valid": true, "correct": true, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.979, "correct": 0.011, "wait": 0.011 }, "confidence": 0.979 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.979 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-20", "expected": "correct", "elapsed_ms": 126.7052909999984, "schema_valid": true, "correct": false, "error": null, "response": { "model": "needle3", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 0, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } } ], "summary": { "fresh_runner_ms": [ 303.46408299999985, 136.74175000000014, 143.05608399999983 ], "repeated_attempts": 100, "repeated_valid": 100, "repeated_failures": 0, "repeated_correct": 40, "unique_first_pass_cases": 20, "unique_first_pass_correct": 8, "accuracy_denominator": "unique_first_pass_cases is the observed denominator; a complete pass contains 20 authored cases. Failures count as incorrect. Repeats are not independent examples.", "p50_attempt_ms": 143.53174999999828, "p95_attempt_ms": 171.2703330000004, "p50_valid_ms": 143.53174999999828, "p95_valid_ms": 171.2703330000004, "repeated_total_elapsed_ms": 14555.8825, "attempted_decisions_per_second": 6.870074693169583, "valid_decisions_per_second": 6.870074693169583, "repeated_reported_input_tokens": 0, "repeated_reported_output_tokens": 0 } }, { "id": "qwen3-0.6b", "kind": "gguf", "adapter": "generic-gguf", "weight_sha256": "33bcc57074ec7b6eada5a90651ee546ec0c2b271002c22baf9f1b2dd1e8f75cb", "weight_bytes": 382156480, "engine_sha256": null, "engine_bytes": null, "usage_semantics": "Actual fully wrapped prompt tokens, separately evaluated per question; output_tokens=0 means no generated text returned, not no computation.", "repeated_state": "Same runner and resident model after two warmups.", "status": "complete", "planned_calls": 105, "skipped_calls": 0, "samples": [ { "phase": "fresh_runner", "repetition": 0, "case_id": "form-01", "expected": "submit", "elapsed_ms": 1189.5801250000004, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 133, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "fresh_runner", "repetition": 1, "case_id": "form-01", "expected": "submit", "elapsed_ms": 1093.4120000000003, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 133, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "fresh_runner", "repetition": 2, "case_id": "form-01", "expected": "submit", "elapsed_ms": 1099.1100000000006, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 133, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "warmup", "repetition": 0, "case_id": "form-01", "expected": "submit", "elapsed_ms": 1055.0323749999989, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 133, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "warmup", "repetition": 1, "case_id": "form-02", "expected": "correct", "elapsed_ms": 249.44404199999917, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.904, "correct": 0.095, "wait": 0.001 }, "confidence": 0.856 } }, "usage": { "input_tokens": 128, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.993, "concentration": 0.856 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-01", "expected": "submit", "elapsed_ms": 202.82729200000176, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 133, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-02", "expected": "correct", "elapsed_ms": 200.408875000001, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.904, "correct": 0.095, "wait": 0.001 }, "confidence": 0.856 } }, "usage": { "input_tokens": 128, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.993, "concentration": 0.856 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-03", "expected": "wait", "elapsed_ms": 207.3091659999991, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 131, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-04", "expected": "submit", "elapsed_ms": 204.9934579999972, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 128, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-05", "expected": "correct", "elapsed_ms": 186.8468329999996, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.995, "correct": 0.002, "wait": 0.002 }, "confidence": 0.993 } }, "usage": { "input_tokens": 128, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.996, "concentration": 0.993 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-06", "expected": "wait", "elapsed_ms": 195.3171250000014, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.982, "correct": 0.002, "wait": 0.016 }, "confidence": 0.973 } }, "usage": { "input_tokens": 133, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.996, "concentration": 0.973 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-07", "expected": "submit", "elapsed_ms": 200.6309590000019, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 131, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-08", "expected": "correct", "elapsed_ms": 184.09216700000252, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.937, "correct": 0.063, "wait": 0 }, "confidence": 0.905 } }, "usage": { "input_tokens": 126, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.998, "concentration": 0.905 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-09", "expected": "wait", "elapsed_ms": 202.18216699999903, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.967, "correct": 0.003, "wait": 0.031 }, "confidence": 0.95 } }, "usage": { "input_tokens": 129, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.987, "concentration": 0.95 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-10", "expected": "submit", "elapsed_ms": 209.48008299999856, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 130, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-11", "expected": "correct", "elapsed_ms": 188.62758299999768, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 127, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.996, "concentration": 1 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-12", "expected": "wait", "elapsed_ms": 195.51075000000128, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.941, "correct": 0.002, "wait": 0.057 }, "confidence": 0.911 } }, "usage": { "input_tokens": 130, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.976, "concentration": 0.911 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-13", "expected": "submit", "elapsed_ms": 200.66520799999853, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 0.999 } }, "usage": { "input_tokens": 128, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.999, "concentration": 0.999 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-14", "expected": "correct", "elapsed_ms": 200.53670799999963, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.996, "correct": 0.003, "wait": 0.001 }, "confidence": 0.994 } }, "usage": { "input_tokens": 129, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.994 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-15", "expected": "wait", "elapsed_ms": 210.22729199999958, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "wait", "probabilities": { "submit": 0.427, "correct": 0, "wait": 0.572 }, "confidence": 0.358 } }, "usage": { "input_tokens": 130, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.996, "concentration": 0.358 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-16", "expected": "submit", "elapsed_ms": 199.39854200000264, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 132, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-17", "expected": "correct", "elapsed_ms": 187.5190419999999, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.644, "correct": 0.353, "wait": 0.002 }, "confidence": 0.466 } }, "usage": { "input_tokens": 131, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.994, "concentration": 0.466 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-18", "expected": "wait", "elapsed_ms": 207.82849999999962, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.994, "correct": 0.001, "wait": 0.004 }, "confidence": 0.992 } }, "usage": { "input_tokens": 133, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.998, "concentration": 0.992 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-19", "expected": "submit", "elapsed_ms": 209.04070800000045, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 131, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-20", "expected": "correct", "elapsed_ms": 192.45829100000265, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.999, "correct": 0.001, "wait": 0 }, "confidence": 0.999 } }, "usage": { "input_tokens": 124, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.998, "concentration": 0.999 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-01", "expected": "submit", "elapsed_ms": 202.7760419999977, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 133, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-02", "expected": "correct", "elapsed_ms": 201.0690829999985, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.904, "correct": 0.095, "wait": 0.001 }, "confidence": 0.856 } }, "usage": { "input_tokens": 128, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.993, "concentration": 0.856 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-03", "expected": "wait", "elapsed_ms": 199.91129200000069, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 131, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-04", "expected": "submit", "elapsed_ms": 225.14733299999716, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 128, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-05", "expected": "correct", "elapsed_ms": 217.58537499999875, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.995, "correct": 0.002, "wait": 0.002 }, "confidence": 0.993 } }, "usage": { "input_tokens": 128, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.996, "concentration": 0.993 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-06", "expected": "wait", "elapsed_ms": 212.74120799999946, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.982, "correct": 0.002, "wait": 0.016 }, "confidence": 0.973 } }, "usage": { "input_tokens": 133, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.996, "concentration": 0.973 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-07", "expected": "submit", "elapsed_ms": 224.17212500000096, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 131, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-08", "expected": "correct", "elapsed_ms": 223.4112090000017, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.937, "correct": 0.063, "wait": 0 }, "confidence": 0.905 } }, "usage": { "input_tokens": 126, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.998, "concentration": 0.905 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-09", "expected": "wait", "elapsed_ms": 207.7339590000047, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.967, "correct": 0.003, "wait": 0.031 }, "confidence": 0.95 } }, "usage": { "input_tokens": 129, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.987, "concentration": 0.95 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-10", "expected": "submit", "elapsed_ms": 209.08987499999785, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 130, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-11", "expected": "correct", "elapsed_ms": 209.0039159999942, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 127, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.996, "concentration": 1 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-12", "expected": "wait", "elapsed_ms": 202.6276249999937, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.941, "correct": 0.002, "wait": 0.057 }, "confidence": 0.911 } }, "usage": { "input_tokens": 130, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.976, "concentration": 0.911 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-13", "expected": "submit", "elapsed_ms": 206.34154199999466, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 0.999 } }, "usage": { "input_tokens": 128, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.999, "concentration": 0.999 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-14", "expected": "correct", "elapsed_ms": 210.828042000001, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.996, "correct": 0.003, "wait": 0.001 }, "confidence": 0.994 } }, "usage": { "input_tokens": 129, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.994 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-15", "expected": "wait", "elapsed_ms": 204.59937499999796, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "wait", "probabilities": { "submit": 0.427, "correct": 0, "wait": 0.572 }, "confidence": 0.358 } }, "usage": { "input_tokens": 130, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.996, "concentration": 0.358 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-16", "expected": "submit", "elapsed_ms": 223.79420899999968, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 132, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-17", "expected": "correct", "elapsed_ms": 201.26295899999968, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.644, "correct": 0.353, "wait": 0.002 }, "confidence": 0.466 } }, "usage": { "input_tokens": 131, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.994, "concentration": 0.466 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-18", "expected": "wait", "elapsed_ms": 202.74370900000213, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.994, "correct": 0.001, "wait": 0.004 }, "confidence": 0.992 } }, "usage": { "input_tokens": 133, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.998, "concentration": 0.992 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-19", "expected": "submit", "elapsed_ms": 211.91654199999903, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 131, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-20", "expected": "correct", "elapsed_ms": 191.84399999999732, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.999, "correct": 0.001, "wait": 0 }, "confidence": 0.999 } }, "usage": { "input_tokens": 124, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.998, "concentration": 0.999 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-01", "expected": "submit", "elapsed_ms": 194.52412500000355, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 133, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-02", "expected": "correct", "elapsed_ms": 202.1772920000003, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.904, "correct": 0.095, "wait": 0.001 }, "confidence": 0.856 } }, "usage": { "input_tokens": 128, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.993, "concentration": 0.856 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-03", "expected": "wait", "elapsed_ms": 218.1195000000007, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 131, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-04", "expected": "submit", "elapsed_ms": 188.02587499999936, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 128, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-05", "expected": "correct", "elapsed_ms": 202.84458299999824, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.995, "correct": 0.002, "wait": 0.002 }, "confidence": 0.993 } }, "usage": { "input_tokens": 128, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.996, "concentration": 0.993 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-06", "expected": "wait", "elapsed_ms": 205.95354200000293, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.982, "correct": 0.002, "wait": 0.016 }, "confidence": 0.973 } }, "usage": { "input_tokens": 133, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.996, "concentration": 0.973 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-07", "expected": "submit", "elapsed_ms": 197.75829199999862, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 131, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-08", "expected": "correct", "elapsed_ms": 192.03312499999447, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.937, "correct": 0.063, "wait": 0 }, "confidence": 0.905 } }, "usage": { "input_tokens": 126, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.998, "concentration": 0.905 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-09", "expected": "wait", "elapsed_ms": 214.82695799999783, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.967, "correct": 0.003, "wait": 0.031 }, "confidence": 0.95 } }, "usage": { "input_tokens": 129, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.987, "concentration": 0.95 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-10", "expected": "submit", "elapsed_ms": 206.6151669999963, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 130, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-11", "expected": "correct", "elapsed_ms": 218.58483299999352, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 127, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.996, "concentration": 1 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-12", "expected": "wait", "elapsed_ms": 213.62429200000042, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.941, "correct": 0.002, "wait": 0.057 }, "confidence": 0.911 } }, "usage": { "input_tokens": 130, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.976, "concentration": 0.911 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-13", "expected": "submit", "elapsed_ms": 215.92933300000004, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 0.999 } }, "usage": { "input_tokens": 128, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.999, "concentration": 0.999 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-14", "expected": "correct", "elapsed_ms": 227.84008300000278, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.996, "correct": 0.003, "wait": 0.001 }, "confidence": 0.994 } }, "usage": { "input_tokens": 129, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.994 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-15", "expected": "wait", "elapsed_ms": 221.41204100000323, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "wait", "probabilities": { "submit": 0.427, "correct": 0, "wait": 0.572 }, "confidence": 0.358 } }, "usage": { "input_tokens": 130, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.996, "concentration": 0.358 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-16", "expected": "submit", "elapsed_ms": 205.62733299999672, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 132, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-17", "expected": "correct", "elapsed_ms": 220.47970799999894, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.644, "correct": 0.353, "wait": 0.002 }, "confidence": 0.466 } }, "usage": { "input_tokens": 131, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.994, "concentration": 0.466 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-18", "expected": "wait", "elapsed_ms": 216.1102499999979, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.994, "correct": 0.001, "wait": 0.004 }, "confidence": 0.992 } }, "usage": { "input_tokens": 133, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.998, "concentration": 0.992 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-19", "expected": "submit", "elapsed_ms": 192.62454199999775, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 131, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-20", "expected": "correct", "elapsed_ms": 203.2976669999989, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.999, "correct": 0.001, "wait": 0 }, "confidence": 0.999 } }, "usage": { "input_tokens": 124, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.998, "concentration": 0.999 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-01", "expected": "submit", "elapsed_ms": 202.07966599999781, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 133, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-02", "expected": "correct", "elapsed_ms": 181.79737499999464, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.904, "correct": 0.095, "wait": 0.001 }, "confidence": 0.856 } }, "usage": { "input_tokens": 128, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.993, "concentration": 0.856 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-03", "expected": "wait", "elapsed_ms": 207.05912499999977, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 131, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-04", "expected": "submit", "elapsed_ms": 217.03258300000016, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 128, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-05", "expected": "correct", "elapsed_ms": 218.843958999998, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.995, "correct": 0.002, "wait": 0.002 }, "confidence": 0.993 } }, "usage": { "input_tokens": 128, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.996, "concentration": 0.993 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-06", "expected": "wait", "elapsed_ms": 223.65575000000536, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.982, "correct": 0.002, "wait": 0.016 }, "confidence": 0.973 } }, "usage": { "input_tokens": 133, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.996, "concentration": 0.973 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-07", "expected": "submit", "elapsed_ms": 198.651291999995, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 131, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-08", "expected": "correct", "elapsed_ms": 191.12245899999834, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.937, "correct": 0.063, "wait": 0 }, "confidence": 0.905 } }, "usage": { "input_tokens": 126, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.998, "concentration": 0.905 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-09", "expected": "wait", "elapsed_ms": 213.64050000000134, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.967, "correct": 0.003, "wait": 0.031 }, "confidence": 0.95 } }, "usage": { "input_tokens": 129, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.987, "concentration": 0.95 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-10", "expected": "submit", "elapsed_ms": 213.70987500000047, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 130, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-11", "expected": "correct", "elapsed_ms": 190.8886249999996, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 127, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.996, "concentration": 1 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-12", "expected": "wait", "elapsed_ms": 209.88254099999904, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.941, "correct": 0.002, "wait": 0.057 }, "confidence": 0.911 } }, "usage": { "input_tokens": 130, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.976, "concentration": 0.911 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-13", "expected": "submit", "elapsed_ms": 216.7559160000019, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 0.999 } }, "usage": { "input_tokens": 128, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.999, "concentration": 0.999 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-14", "expected": "correct", "elapsed_ms": 206.86699999999837, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.996, "correct": 0.003, "wait": 0.001 }, "confidence": 0.994 } }, "usage": { "input_tokens": 129, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.994 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-15", "expected": "wait", "elapsed_ms": 212.44654199999786, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "wait", "probabilities": { "submit": 0.427, "correct": 0, "wait": 0.572 }, "confidence": 0.358 } }, "usage": { "input_tokens": 130, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.996, "concentration": 0.358 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-16", "expected": "submit", "elapsed_ms": 224.32954100000643, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 132, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-17", "expected": "correct", "elapsed_ms": 215.81433300000208, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.644, "correct": 0.353, "wait": 0.002 }, "confidence": 0.466 } }, "usage": { "input_tokens": 131, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.994, "concentration": 0.466 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-18", "expected": "wait", "elapsed_ms": 219.34545900000376, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.994, "correct": 0.001, "wait": 0.004 }, "confidence": 0.992 } }, "usage": { "input_tokens": 133, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.998, "concentration": 0.992 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-19", "expected": "submit", "elapsed_ms": 237.35404199999903, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 131, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-20", "expected": "correct", "elapsed_ms": 231.54550000000017, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.999, "correct": 0.001, "wait": 0 }, "confidence": 0.999 } }, "usage": { "input_tokens": 124, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.998, "concentration": 0.999 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-01", "expected": "submit", "elapsed_ms": 234.43445899999642, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 133, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-02", "expected": "correct", "elapsed_ms": 201.07779100000334, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.904, "correct": 0.095, "wait": 0.001 }, "confidence": 0.856 } }, "usage": { "input_tokens": 128, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.993, "concentration": 0.856 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-03", "expected": "wait", "elapsed_ms": 195.69054100000358, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 131, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-04", "expected": "submit", "elapsed_ms": 207.7746249999982, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 128, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-05", "expected": "correct", "elapsed_ms": 210.33508299999812, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.995, "correct": 0.002, "wait": 0.002 }, "confidence": 0.993 } }, "usage": { "input_tokens": 128, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.996, "concentration": 0.993 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-06", "expected": "wait", "elapsed_ms": 201.29550000000017, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.982, "correct": 0.002, "wait": 0.016 }, "confidence": 0.973 } }, "usage": { "input_tokens": 133, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.996, "concentration": 0.973 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-07", "expected": "submit", "elapsed_ms": 201.31079199999658, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 131, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-08", "expected": "correct", "elapsed_ms": 194.52345899999636, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.937, "correct": 0.063, "wait": 0 }, "confidence": 0.905 } }, "usage": { "input_tokens": 126, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.998, "concentration": 0.905 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-09", "expected": "wait", "elapsed_ms": 191.24362499999552, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.967, "correct": 0.003, "wait": 0.031 }, "confidence": 0.95 } }, "usage": { "input_tokens": 129, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.987, "concentration": 0.95 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-10", "expected": "submit", "elapsed_ms": 209.73641700000007, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 130, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-11", "expected": "correct", "elapsed_ms": 191.90374999999767, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 127, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.996, "concentration": 1 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-12", "expected": "wait", "elapsed_ms": 188.56958300000406, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.941, "correct": 0.002, "wait": 0.057 }, "confidence": 0.911 } }, "usage": { "input_tokens": 130, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.976, "concentration": 0.911 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-13", "expected": "submit", "elapsed_ms": 202.3744579999984, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 0.999 } }, "usage": { "input_tokens": 128, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.999, "concentration": 0.999 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-14", "expected": "correct", "elapsed_ms": 207.15529100000276, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.996, "correct": 0.003, "wait": 0.001 }, "confidence": 0.994 } }, "usage": { "input_tokens": 129, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.994 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-15", "expected": "wait", "elapsed_ms": 197.36462500000198, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "wait", "probabilities": { "submit": 0.427, "correct": 0, "wait": 0.572 }, "confidence": 0.358 } }, "usage": { "input_tokens": 130, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.996, "concentration": 0.358 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-16", "expected": "submit", "elapsed_ms": 219.02304200000071, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 132, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-17", "expected": "correct", "elapsed_ms": 213.06179200000042, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.644, "correct": 0.353, "wait": 0.002 }, "confidence": 0.466 } }, "usage": { "input_tokens": 131, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.994, "concentration": 0.466 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-18", "expected": "wait", "elapsed_ms": 221.0951669999995, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.994, "correct": 0.001, "wait": 0.004 }, "confidence": 0.992 } }, "usage": { "input_tokens": 133, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.998, "concentration": 0.992 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-19", "expected": "submit", "elapsed_ms": 224.28370899999572, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 131, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-20", "expected": "correct", "elapsed_ms": 233.33558400000038, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-0.6b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.999, "correct": 0.001, "wait": 0 }, "confidence": 0.999 } }, "usage": { "input_tokens": 124, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 0.998, "concentration": 0.999 } } } ], "summary": { "fresh_runner_ms": [ 1189.5801250000004, 1093.4120000000003, 1099.1100000000006 ], "repeated_attempts": 100, "repeated_valid": 100, "repeated_failures": 0, "repeated_correct": 40, "unique_first_pass_cases": 20, "unique_first_pass_correct": 8, "accuracy_denominator": "unique_first_pass_cases is the observed denominator; a complete pass contains 20 authored cases. Failures count as incorrect. Repeats are not independent examples.", "p50_attempt_ms": 206.6151669999963, "p95_attempt_ms": 225.14733299999716, "p50_valid_ms": 206.6151669999963, "p95_valid_ms": 225.14733299999716, "repeated_total_elapsed_ms": 20723.994082999998, "attempted_decisions_per_second": 4.8253246743604565, "valid_decisions_per_second": 4.8253246743604565, "repeated_reported_input_tokens": 12960, "repeated_reported_output_tokens": 0 } }, { "id": "qwen3-1.7b", "kind": "gguf", "adapter": "generic-gguf", "weight_sha256": "b139949c5bd74937ad8ed8c8cf3d9ffb1e99c866c823204dc42c0d91fa181897", "weight_bytes": 1107409472, "engine_sha256": null, "engine_bytes": null, "usage_semantics": "Actual fully wrapped prompt tokens, separately evaluated per question; output_tokens=0 means no generated text returned, not no computation.", "repeated_state": "Same runner and resident model after two warmups.", "status": "complete", "planned_calls": 105, "skipped_calls": 0, "samples": [ { "phase": "fresh_runner", "repetition": 0, "case_id": "form-01", "expected": "submit", "elapsed_ms": 1625.4192079999993, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 133, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "fresh_runner", "repetition": 1, "case_id": "form-01", "expected": "submit", "elapsed_ms": 1145.854333000003, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 133, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "fresh_runner", "repetition": 2, "case_id": "form-01", "expected": "submit", "elapsed_ms": 1109.9770830000052, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 133, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "warmup", "repetition": 0, "case_id": "form-01", "expected": "submit", "elapsed_ms": 1193.4566669999986, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 133, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "warmup", "repetition": 1, "case_id": "form-02", "expected": "correct", "elapsed_ms": 323.6790000000037, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0, "correct": 1, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 128, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-01", "expected": "submit", "elapsed_ms": 230.08662499999627, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 133, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-02", "expected": "correct", "elapsed_ms": 230.7075829999958, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0, "correct": 1, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 128, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-03", "expected": "wait", "elapsed_ms": 251.64741700000013, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.009, "correct": 0.581, "wait": 0.41 }, "confidence": 0.371 } }, "usage": { "input_tokens": 131, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.371 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-04", "expected": "submit", "elapsed_ms": 248.72341599999345, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 128, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-05", "expected": "correct", "elapsed_ms": 250.55249999999796, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0, "correct": 1, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 128, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-06", "expected": "wait", "elapsed_ms": 249.47166600000492, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "wait", "probabilities": { "submit": 0, "correct": 0, "wait": 1 }, "confidence": 1 } }, "usage": { "input_tokens": 133, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-07", "expected": "submit", "elapsed_ms": 245.72958300000028, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 131, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-08", "expected": "correct", "elapsed_ms": 261.0907089999964, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0, "correct": 1, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 126, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-09", "expected": "wait", "elapsed_ms": 269.1059169999935, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "wait", "probabilities": { "submit": 0, "correct": 0, "wait": 1 }, "confidence": 1 } }, "usage": { "input_tokens": 129, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-10", "expected": "submit", "elapsed_ms": 234.85191599999962, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 130, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-11", "expected": "correct", "elapsed_ms": 265.7122079999972, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0, "correct": 1, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 127, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-12", "expected": "wait", "elapsed_ms": 279.2445829999997, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "wait", "probabilities": { "submit": 0, "correct": 0, "wait": 1 }, "confidence": 1 } }, "usage": { "input_tokens": 130, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-13", "expected": "submit", "elapsed_ms": 270.64137499999924, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.554, "correct": 0.446, "wait": 0 }, "confidence": 0.33 } }, "usage": { "input_tokens": 128, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.33 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-14", "expected": "correct", "elapsed_ms": 287.49995800000033, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.998, "correct": 0.002, "wait": 0 }, "confidence": 0.997 } }, "usage": { "input_tokens": 129, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.997 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-15", "expected": "wait", "elapsed_ms": 267.33358299999963, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "wait", "probabilities": { "submit": 0, "correct": 0, "wait": 1 }, "confidence": 1 } }, "usage": { "input_tokens": 130, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-16", "expected": "submit", "elapsed_ms": 259.5428749999992, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.998, "correct": 0.002, "wait": 0 }, "confidence": 0.998 } }, "usage": { "input_tokens": 132, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.998 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-17", "expected": "correct", "elapsed_ms": 265.87054199999693, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0, "correct": 1, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 131, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-18", "expected": "wait", "elapsed_ms": 271.4145420000059, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "wait", "probabilities": { "submit": 0, "correct": 0, "wait": 1 }, "confidence": 1 } }, "usage": { "input_tokens": 133, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-19", "expected": "submit", "elapsed_ms": 261.0371659999946, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 131, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 0, "case_id": "form-20", "expected": "correct", "elapsed_ms": 257.6913330000025, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0, "correct": 1, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 124, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-01", "expected": "submit", "elapsed_ms": 273.3106670000052, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 133, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-02", "expected": "correct", "elapsed_ms": 246.74216599999636, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0, "correct": 1, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 128, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-03", "expected": "wait", "elapsed_ms": 264.2469590000037, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.009, "correct": 0.581, "wait": 0.41 }, "confidence": 0.371 } }, "usage": { "input_tokens": 131, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.371 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-04", "expected": "submit", "elapsed_ms": 262.3866249999992, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 128, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-05", "expected": "correct", "elapsed_ms": 267.6146660000013, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0, "correct": 1, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 128, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-06", "expected": "wait", "elapsed_ms": 262.6475419999988, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "wait", "probabilities": { "submit": 0, "correct": 0, "wait": 1 }, "confidence": 1 } }, "usage": { "input_tokens": 133, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-07", "expected": "submit", "elapsed_ms": 261.17375000000175, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 131, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-08", "expected": "correct", "elapsed_ms": 248.68262500000128, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0, "correct": 1, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 126, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-09", "expected": "wait", "elapsed_ms": 259.5629590000026, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "wait", "probabilities": { "submit": 0, "correct": 0, "wait": 1 }, "confidence": 1 } }, "usage": { "input_tokens": 129, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-10", "expected": "submit", "elapsed_ms": 261.91212500000256, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 130, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-11", "expected": "correct", "elapsed_ms": 236.3049170000013, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0, "correct": 1, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 127, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-12", "expected": "wait", "elapsed_ms": 263.1583329999994, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "wait", "probabilities": { "submit": 0, "correct": 0, "wait": 1 }, "confidence": 1 } }, "usage": { "input_tokens": 130, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-13", "expected": "submit", "elapsed_ms": 247.38195899999846, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.554, "correct": 0.446, "wait": 0 }, "confidence": 0.33 } }, "usage": { "input_tokens": 128, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.33 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-14", "expected": "correct", "elapsed_ms": 241.3242919999975, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.998, "correct": 0.002, "wait": 0 }, "confidence": 0.997 } }, "usage": { "input_tokens": 129, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.997 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-15", "expected": "wait", "elapsed_ms": 258.833458000001, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "wait", "probabilities": { "submit": 0, "correct": 0, "wait": 1 }, "confidence": 1 } }, "usage": { "input_tokens": 130, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-16", "expected": "submit", "elapsed_ms": 258.3274999999994, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.998, "correct": 0.002, "wait": 0 }, "confidence": 0.998 } }, "usage": { "input_tokens": 132, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.998 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-17", "expected": "correct", "elapsed_ms": 267.4904170000009, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0, "correct": 1, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 131, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-18", "expected": "wait", "elapsed_ms": 261.5512080000044, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "wait", "probabilities": { "submit": 0, "correct": 0, "wait": 1 }, "confidence": 1 } }, "usage": { "input_tokens": 133, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-19", "expected": "submit", "elapsed_ms": 265.71795900000143, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 131, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 1, "case_id": "form-20", "expected": "correct", "elapsed_ms": 239.67441600000166, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0, "correct": 1, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 124, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-01", "expected": "submit", "elapsed_ms": 247.05941700000403, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 133, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-02", "expected": "correct", "elapsed_ms": 249.8391249999986, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0, "correct": 1, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 128, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-03", "expected": "wait", "elapsed_ms": 261.40029199999844, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.009, "correct": 0.581, "wait": 0.41 }, "confidence": 0.371 } }, "usage": { "input_tokens": 131, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.371 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-04", "expected": "submit", "elapsed_ms": 252.57033299999603, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 128, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-05", "expected": "correct", "elapsed_ms": 253.278916999996, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0, "correct": 1, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 128, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-06", "expected": "wait", "elapsed_ms": 248.64958299999853, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "wait", "probabilities": { "submit": 0, "correct": 0, "wait": 1 }, "confidence": 1 } }, "usage": { "input_tokens": 133, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-07", "expected": "submit", "elapsed_ms": 260.4862920000087, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 131, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-08", "expected": "correct", "elapsed_ms": 241.9515420000098, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0, "correct": 1, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 126, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-09", "expected": "wait", "elapsed_ms": 259.69879200001014, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "wait", "probabilities": { "submit": 0, "correct": 0, "wait": 1 }, "confidence": 1 } }, "usage": { "input_tokens": 129, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-10", "expected": "submit", "elapsed_ms": 245.83950000000186, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 130, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-11", "expected": "correct", "elapsed_ms": 247.70920899999328, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0, "correct": 1, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 127, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-12", "expected": "wait", "elapsed_ms": 242.37633300000743, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "wait", "probabilities": { "submit": 0, "correct": 0, "wait": 1 }, "confidence": 1 } }, "usage": { "input_tokens": 130, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-13", "expected": "submit", "elapsed_ms": 252.92225000000326, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.554, "correct": 0.446, "wait": 0 }, "confidence": 0.33 } }, "usage": { "input_tokens": 128, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.33 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-14", "expected": "correct", "elapsed_ms": 261.27224999999453, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.998, "correct": 0.002, "wait": 0 }, "confidence": 0.997 } }, "usage": { "input_tokens": 129, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.997 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-15", "expected": "wait", "elapsed_ms": 264.23174999999173, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "wait", "probabilities": { "submit": 0, "correct": 0, "wait": 1 }, "confidence": 1 } }, "usage": { "input_tokens": 130, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-16", "expected": "submit", "elapsed_ms": 262.6280829999887, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.998, "correct": 0.002, "wait": 0 }, "confidence": 0.998 } }, "usage": { "input_tokens": 132, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.998 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-17", "expected": "correct", "elapsed_ms": 268.02379200000723, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0, "correct": 1, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 131, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-18", "expected": "wait", "elapsed_ms": 250.77991700000712, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "wait", "probabilities": { "submit": 0, "correct": 0, "wait": 1 }, "confidence": 1 } }, "usage": { "input_tokens": 133, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-19", "expected": "submit", "elapsed_ms": 268.72491599999194, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 131, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 2, "case_id": "form-20", "expected": "correct", "elapsed_ms": 257.181917000009, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0, "correct": 1, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 124, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-01", "expected": "submit", "elapsed_ms": 255.1618749999907, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 133, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-02", "expected": "correct", "elapsed_ms": 260.2511250000098, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0, "correct": 1, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 128, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-03", "expected": "wait", "elapsed_ms": 262.38579200000095, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.009, "correct": 0.581, "wait": 0.41 }, "confidence": 0.371 } }, "usage": { "input_tokens": 131, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.371 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-04", "expected": "submit", "elapsed_ms": 233.35662500000035, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 128, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-05", "expected": "correct", "elapsed_ms": 252.60720899999433, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0, "correct": 1, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 128, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-06", "expected": "wait", "elapsed_ms": 255.66633300000103, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "wait", "probabilities": { "submit": 0, "correct": 0, "wait": 1 }, "confidence": 1 } }, "usage": { "input_tokens": 133, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-07", "expected": "submit", "elapsed_ms": 258.275999999998, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 131, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-08", "expected": "correct", "elapsed_ms": 248.60670799999207, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0, "correct": 1, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 126, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-09", "expected": "wait", "elapsed_ms": 265.7242499999993, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "wait", "probabilities": { "submit": 0, "correct": 0, "wait": 1 }, "confidence": 1 } }, "usage": { "input_tokens": 129, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-10", "expected": "submit", "elapsed_ms": 234.81874999999127, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 130, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-11", "expected": "correct", "elapsed_ms": 230.40312500001164, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0, "correct": 1, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 127, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-12", "expected": "wait", "elapsed_ms": 248.32887499999197, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "wait", "probabilities": { "submit": 0, "correct": 0, "wait": 1 }, "confidence": 1 } }, "usage": { "input_tokens": 130, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-13", "expected": "submit", "elapsed_ms": 218.69054100000358, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.554, "correct": 0.446, "wait": 0 }, "confidence": 0.33 } }, "usage": { "input_tokens": 128, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.33 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-14", "expected": "correct", "elapsed_ms": 250.45429100000183, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.998, "correct": 0.002, "wait": 0 }, "confidence": 0.997 } }, "usage": { "input_tokens": 129, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.997 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-15", "expected": "wait", "elapsed_ms": 265.9974999999977, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "wait", "probabilities": { "submit": 0, "correct": 0, "wait": 1 }, "confidence": 1 } }, "usage": { "input_tokens": 130, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-16", "expected": "submit", "elapsed_ms": 265.59662499999104, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.998, "correct": 0.002, "wait": 0 }, "confidence": 0.998 } }, "usage": { "input_tokens": 132, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.998 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-17", "expected": "correct", "elapsed_ms": 261.84154199999466, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0, "correct": 1, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 131, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-18", "expected": "wait", "elapsed_ms": 272.162124999988, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "wait", "probabilities": { "submit": 0, "correct": 0, "wait": 1 }, "confidence": 1 } }, "usage": { "input_tokens": 133, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-19", "expected": "submit", "elapsed_ms": 262.2152500000084, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 131, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 3, "case_id": "form-20", "expected": "correct", "elapsed_ms": 265.59741600000416, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0, "correct": 1, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 124, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-01", "expected": "submit", "elapsed_ms": 264.97562499999185, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 133, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-02", "expected": "correct", "elapsed_ms": 244.8200829999987, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0, "correct": 1, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 128, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-03", "expected": "wait", "elapsed_ms": 260.88187499999185, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0.009, "correct": 0.581, "wait": 0.41 }, "confidence": 0.371 } }, "usage": { "input_tokens": 131, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.371 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-04", "expected": "submit", "elapsed_ms": 273.6992500000051, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 128, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-05", "expected": "correct", "elapsed_ms": 229.0596249999944, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0, "correct": 1, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 128, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-06", "expected": "wait", "elapsed_ms": 259.65245900000446, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "wait", "probabilities": { "submit": 0, "correct": 0, "wait": 1 }, "confidence": 1 } }, "usage": { "input_tokens": 133, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-07", "expected": "submit", "elapsed_ms": 265.1915000000008, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 131, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-08", "expected": "correct", "elapsed_ms": 255.85762499998964, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0, "correct": 1, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 126, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-09", "expected": "wait", "elapsed_ms": 247.96516699998756, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "wait", "probabilities": { "submit": 0, "correct": 0, "wait": 1 }, "confidence": 1 } }, "usage": { "input_tokens": 129, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-10", "expected": "submit", "elapsed_ms": 253.98520799999824, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 130, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-11", "expected": "correct", "elapsed_ms": 239.828416999997, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0, "correct": 1, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 127, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-12", "expected": "wait", "elapsed_ms": 249.02537499999744, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "wait", "probabilities": { "submit": 0, "correct": 0, "wait": 1 }, "confidence": 1 } }, "usage": { "input_tokens": 130, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-13", "expected": "submit", "elapsed_ms": 246.6310419999936, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.554, "correct": 0.446, "wait": 0 }, "confidence": 0.33 } }, "usage": { "input_tokens": 128, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.33 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-14", "expected": "correct", "elapsed_ms": 236.21816699999908, "schema_valid": true, "correct": false, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.998, "correct": 0.002, "wait": 0 }, "confidence": 0.997 } }, "usage": { "input_tokens": 129, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.997 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-15", "expected": "wait", "elapsed_ms": 258.99041700000816, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "wait", "probabilities": { "submit": 0, "correct": 0, "wait": 1 }, "confidence": 1 } }, "usage": { "input_tokens": 130, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-16", "expected": "submit", "elapsed_ms": 258.85516700000153, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 0.998, "correct": 0.002, "wait": 0 }, "confidence": 0.998 } }, "usage": { "input_tokens": 132, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 0.998 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-17", "expected": "correct", "elapsed_ms": 251.54145799999242, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0, "correct": 1, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 131, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-18", "expected": "wait", "elapsed_ms": 259.87358400000085, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "wait", "probabilities": { "submit": 0, "correct": 0, "wait": 1 }, "confidence": 1 } }, "usage": { "input_tokens": 133, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-19", "expected": "submit", "elapsed_ms": 258.7069999999949, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "submit", "probabilities": { "submit": 1, "correct": 0, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 131, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } }, { "phase": "repeated", "repetition": 4, "case_id": "form-20", "expected": "correct", "elapsed_ms": 239.62979199999245, "schema_valid": true, "correct": true, "error": null, "response": { "model": "qwen3-1.7b", "answers": { "action": { "type": "choice", "choice": "correct", "probabilities": { "submit": 0, "correct": 1, "wait": 0 }, "confidence": 1 } }, "usage": { "input_tokens": 124, "output_tokens": 0 } }, "diagnostics": { "action": { "coverage": 1, "concentration": 1 } } } ], "summary": { "fresh_runner_ms": [ 1625.4192079999993, 1145.854333000003, 1109.9770830000052 ], "repeated_attempts": 100, "repeated_valid": 100, "repeated_failures": 0, "repeated_correct": 90, "unique_first_pass_cases": 20, "unique_first_pass_correct": 18, "accuracy_denominator": "unique_first_pass_cases is the observed denominator; a complete pass contains 20 authored cases. Failures count as incorrect. Repeats are not independent examples.", "p50_attempt_ms": 258.3274999999994, "p95_attempt_ms": 271.4145420000059, "p50_valid_ms": 258.3274999999994, "p95_valid_ms": 271.4145420000059, "repeated_total_elapsed_ms": 25539.607459000006, "attempted_decisions_per_second": 3.915486961204668, "valid_decisions_per_second": 3.915486961204668, "repeated_reported_input_tokens": 12960, "repeated_reported_output_tokens": 0 } } ] }