| { |
| "n": 200, |
| "images": 199, |
| "accuracy_all": 0.63, |
| "accuracy": { |
| "noul": null, |
| "choice": 0.63 |
| }, |
| "nll": 0.9023860797262202, |
| "brier": 0.48241933227960376, |
| "ece": 0.05323468044400214, |
| "ece_bins": 15, |
| "reliability": [ |
| { |
| "bin": [ |
| 0.2, |
| 0.3 |
| ], |
| "n": 8, |
| "confidence": 0.28236541524529457, |
| "accuracy": 0.25 |
| }, |
| { |
| "bin": [ |
| 0.3, |
| 0.4 |
| ], |
| "n": 41, |
| "confidence": 0.3474449958743119, |
| "accuracy": 0.36585365853658536 |
| }, |
| { |
| "bin": [ |
| 0.4, |
| 0.5 |
| ], |
| "n": 31, |
| "confidence": 0.44290193915367126, |
| "accuracy": 0.5161290322580645 |
| }, |
| { |
| "bin": [ |
| 0.5, |
| 0.6 |
| ], |
| "n": 20, |
| "confidence": 0.5382369369268417, |
| "accuracy": 0.6 |
| }, |
| { |
| "bin": [ |
| 0.6, |
| 0.7 |
| ], |
| "n": 29, |
| "confidence": 0.6475593776538454, |
| "accuracy": 0.6206896551724138 |
| }, |
| { |
| "bin": [ |
| 0.7, |
| 0.8 |
| ], |
| "n": 18, |
| "confidence": 0.7407801515526242, |
| "accuracy": 0.7777777777777778 |
| }, |
| { |
| "bin": [ |
| 0.8, |
| 0.9 |
| ], |
| "n": 17, |
| "confidence": 0.8483305994202109, |
| "accuracy": 0.9411764705882353 |
| }, |
| { |
| "bin": [ |
| 0.9, |
| 1.0 |
| ], |
| "n": 36, |
| "confidence": 0.9695057719945908, |
| "accuracy": 0.9166666666666666 |
| } |
| ], |
| "thresholds": { |
| "0.6": { |
| "coverage": 0.5, |
| "accuracy": 0.81 |
| }, |
| "0.7": { |
| "coverage": 0.355, |
| "accuracy": 0.8873239436619719 |
| }, |
| "0.8": { |
| "coverage": 0.265, |
| "accuracy": 0.9245283018867925 |
| }, |
| "0.9": { |
| "coverage": 0.18, |
| "accuracy": 0.9166666666666666 |
| } |
| }, |
| "by_source": { |
| "aokvqa": { |
| "n": 200, |
| "images": 199, |
| "accuracy": 0.63, |
| "nll": 0.9023860797262202, |
| "ece": 0.05323468044400214 |
| } |
| }, |
| "by_kind": { |
| "aokvqa": { |
| "n": 200, |
| "images": 199, |
| "accuracy": 0.63, |
| "nll": 0.9023860797262202, |
| "ece": 0.05323468044400214 |
| } |
| }, |
| "by_axis": { |
| "-": { |
| "n": 200, |
| "images": 199, |
| "accuracy": 0.63, |
| "nll": 0.9023860797262202, |
| "ece": 0.05323468044400214 |
| } |
| }, |
| "by_options": { |
| "4": { |
| "n": 200, |
| "images": 199, |
| "accuracy": 0.63, |
| "nll": 0.9023860797262202, |
| "ece": 0.05323468044400214 |
| } |
| }, |
| "ms_per_question": { |
| "p50": 289.0499159984756, |
| "p95": 301.4491552057734, |
| "overall": 289.6095040804357 |
| }, |
| "request_latency_ms": { |
| "p50": 289.0499159984756, |
| "p95": 301.4491552057734, |
| "requests": 200, |
| "warmup_excluded": 3 |
| }, |
| "questions_per_request": { |
| "mean": 1.0, |
| "p50": 1.0, |
| "max": 1 |
| }, |
| "latency_ms": { |
| "p50": 289.0499159984756, |
| "p95": 301.4491552057734, |
| "requests": 200, |
| "warmup_excluded": 3 |
| }, |
| "control": "none", |
| "model": "devision-v0.3", |
| "reading": { |
| "mode": "none", |
| "routed_questions": 0, |
| "read_questions": 0 |
| }, |
| "temperature": { |
| "choice": 2.3641862869262695, |
| "noul": 1.0895380973815918, |
| "by_options": { |
| "choice:2": 3.563119649887085, |
| "choice:3-5": 2.1145524978637695, |
| "noul:2": 1.0895380973815918 |
| } |
| }, |
| "grouping": { |
| "max_questions_per_request": 1, |
| "by": "none (one question per request)", |
| "max_questions_per_call": "one request per call" |
| }, |
| "environment": { |
| "threads": 4, |
| "machine": "x86_64", |
| "system": "Linux" |
| }, |
| "run": { |
| "checkpoint": "runs/r21-soft/train", |
| "data": "runs/v04-cpu/latency200.jsonl", |
| "data_sha256": "e0f4d5b0f82ff59ba91d141f49833ad9a097ac36842b98c32e2c4a834d0b9691", |
| "role": "monitoring", |
| "device": "cpu", |
| "temperature": "fitted", |
| "control": "none", |
| "seed": 0, |
| "code": "a6b4e8a+dirty", |
| "reading": { |
| "mode": "none", |
| "routed_questions": 0, |
| "read_questions": 0 |
| }, |
| "code_identity": { |
| "modules": { |
| "devision": { |
| "path": "/home/bruce/proj/devision/src/devision/__init__.py", |
| "sha256": "aa00e0d329f33444a51bc4cab7691911b6b226664fdc2fbd438103de8d0c27a1" |
| }, |
| "devision.image": { |
| "path": "/home/bruce/proj/devision/src/devision/image.py", |
| "sha256": "a139a63f985b8a210de4b9a5327dd56cc078a986cb36ebcc7fd3c4aba4bef312" |
| }, |
| "devision.infer": { |
| "path": "/home/bruce/proj/devision/src/devision/infer.py", |
| "sha256": "d7e3e5f8bc54278cbdba99f77da567344cf40e9748c6b1a55ecb48e48d593893" |
| }, |
| "devision.model": { |
| "path": "/home/bruce/proj/devision/src/devision/model.py", |
| "sha256": "079e9c9c60545242146a99ec66ba09a5cd1435545967b37b2649e071b3245209" |
| }, |
| "devision.reading": { |
| "path": "/home/bruce/proj/devision/src/devision/reading.py", |
| "sha256": "dc2f3cc2151ec4761dcfc404b28d609a932aac70ff0ff02b0dd6e170b6c1f8ba" |
| }, |
| "devision.train": { |
| "path": "/home/bruce/proj/devision/src/devision/train/__init__.py", |
| "sha256": "16c32ba2bae66029fb0afd4946827c475729aa45c7dde335864d002c564df289" |
| }, |
| "devision.train.cli": { |
| "path": "/home/bruce/proj/devision/src/devision/train/cli.py", |
| "sha256": "aab8811a63c89726fd78dd7ea88ab930c6d115b529a011173ccc8637f081d342" |
| }, |
| "devision.train.evaluate": { |
| "path": "/home/bruce/proj/devision/src/devision/train/evaluate.py", |
| "sha256": "4d5df39b37b0854ead1b8c7d0a04933e24ab9b05e0e341f0bdbfbc6911af2d90" |
| }, |
| "devision.train.samples": { |
| "path": "/home/bruce/proj/devision/src/devision/train/samples.py", |
| "sha256": "5a4d5a75ec31380f5470a22c0ba4e370e170ba0499e5d4a77ed573450f9c3e31" |
| } |
| }, |
| "version": "0.3.0.dev1", |
| "python": "3.12.3" |
| } |
| } |
| } |
