70 lines
13 KiB
JSON
70 lines
13 KiB
JSON
{"type":"config","name":"latency-p95-chat-completions","metricName":"p95_ms","metricUnit":"ms","bestDirection":"lower"}
|
|
{"run":1,"commit":"ceed0e5","metric":941.252,"metrics":{"p50_ms":908.457,"mean_ms":917.123,"completion_tps_mean":52.427,"success_count":7},"status":"keep","description":"baseline","timestamp":1774557130,"segment":0}
|
|
{"run":2,"commit":"5269f43","metric":1032.315,"metrics":{"p50_ms":1020.402,"mean_ms":1011.466,"completion_tps_mean":47.671,"success_count":7},"status":"discard","description":"use stack buffer for token detokenization","timestamp":1774557493,"segment":0}
|
|
{"run":3,"commit":"994e607","metric":1004.435,"metrics":{"p50_ms":998.919,"mean_ms":990.771,"completion_tps_mean":48.528,"success_count":7},"status":"discard","description":"slim turbo4b runtime profile","timestamp":1774557635,"segment":0}
|
|
{"run":4,"commit":"db92212","metric":3027.145,"metrics":{"p50_ms":974.937,"mean_ms":1467.026,"completion_tps_mean":41.360,"success_count":7},"status":"discard","description":"switch to qwen-style chat template","timestamp":1774558217,"segment":0}
|
|
{"run":5,"commit":"caf4654","metric":965.663,"metrics":{"p50_ms":877.030,"mean_ms":897.251,"completion_tps_mean":53.686,"success_count":7},"status":"discard","description":"stop generation on default turn boundaries","timestamp":1774558444,"segment":0}
|
|
{"run":6,"commit":"e8402d2","metric":925.587,"metrics":{"p50_ms":893.913,"mean_ms":899.895,"completion_tps_mean":53.618,"success_count":7},"status":"keep","description":"disable llama perf counters in hot path","timestamp":1774558642,"segment":0}
|
|
{"type":"config","name":"dynamic-chat-templates","metricName":"template_score","metricUnit":"%","bestDirection":"higher"}
|
|
{"run":7,"commit":"89ef220","metric":20.0,"metrics":{"passed_cases":1,"failed_cases":4,"total_cases":5},"status":"keep","description":"baseline","timestamp":1774559683,"segment":1}
|
|
{"run":8,"commit":"ceb83f2","metric":100.0,"metrics":{"passed_cases":5,"failed_cases":0,"total_cases":5},"status":"keep","description":"add family-based chat templates","timestamp":1774559814,"segment":1}
|
|
{"type":"config","name":"dynamic-chat-templates-hardening","metricName":"template_score","metricUnit":"%","bestDirection":"higher"}
|
|
{"run":9,"commit":"5f3ed5e","metric":71.429,"metrics":{"passed_cases":5,"failed_cases":2,"total_cases":7},"status":"keep","description":"baseline","timestamp":1774560085,"segment":2}
|
|
{"run":10,"commit":"fd6d35e","metric":100.0,"metrics":{"passed_cases":7,"failed_cases":0,"total_cases":7},"status":"keep","description":"harden llama chat template rendering","timestamp":1774560229,"segment":2}
|
|
{"type":"config","name":"dynamic-chat-templates-aliases","metricName":"template_score","metricUnit":"%","bestDirection":"higher"}
|
|
{"run":11,"commit":"2efe601","metric":88.889,"metrics":{"passed_cases":8,"failed_cases":1,"total_cases":9},"status":"keep","description":"baseline","timestamp":1774560452,"segment":3}
|
|
{"run":12,"commit":"140c9b2","metric":100.0,"metrics":{"passed_cases":9,"failed_cases":0,"total_cases":9},"status":"keep","description":"detect template family from aliases","timestamp":1774560533,"segment":3}
|
|
{"type":"config","name":"dynamic-chat-templates-variants","metricName":"template_score","metricUnit":"%","bestDirection":"higher"}
|
|
{"run":13,"commit":"ffab766","metric":90.909,"metrics":{"passed_cases":10,"failed_cases":1,"total_cases":11},"status":"keep","description":"baseline","timestamp":1774560738,"segment":4}
|
|
{"run":14,"commit":"f200733","metric":100.0,"metrics":{"passed_cases":11,"failed_cases":0,"total_cases":11},"status":"keep","description":"detect moonshot as kimi family","timestamp":1774560829,"segment":4}
|
|
{"type":"config","name":"localllm-cli-control-surface","metricName":"cli_surface_score","metricUnit":"%","bestDirection":"higher"}
|
|
{"run":15,"commit":"fd773a9","metric":18.182,"metrics":{"passed_cases":2,"failed_cases":9,"total_cases":11},"status":"keep","description":"baseline","timestamp":1774562027,"segment":5}
|
|
{"run":16,"commit":"70cc59a","metric":100.0,"metrics":{"passed_cases":11,"failed_cases":0,"total_cases":11},"status":"keep","description":"expand cli command surface","timestamp":1774562138,"segment":5}
|
|
{"type":"config","name":"localllm-cli-semantic-contracts","metricName":"cli_surface_score","metricUnit":"%","bestDirection":"higher"}
|
|
{"run":17,"commit":"53a7dbb","metric":68.75,"metrics":{"passed_cases":11,"failed_cases":5,"total_cases":16},"status":"keep","description":"baseline","timestamp":1774562358,"segment":6}
|
|
{"run":18,"commit":"66b30de","metric":100.0,"metrics":{"passed_cases":16,"failed_cases":0,"total_cases":16},"status":"keep","description":"enrich cli semantic summaries","timestamp":1774562434,"segment":6}
|
|
{"type":"config","name":"localllm-cli-execution-plans","metricName":"cli_surface_score","metricUnit":"%","bestDirection":"higher"}
|
|
{"run":19,"commit":"9154a7b","metric":69.565,"metrics":{"passed_cases":16,"failed_cases":7,"total_cases":23},"status":"keep","description":"baseline","timestamp":1774562688,"segment":7}
|
|
{"run":20,"commit":"bc584be","metric":100.0,"metrics":{"passed_cases":23,"failed_cases":0,"total_cases":23},"status":"keep","description":"add cli execution plans","timestamp":1774562759,"segment":7}
|
|
{"type":"config","name":"localllm-cli-execution-contracts","metricName":"cli_surface_score","metricUnit":"%","bestDirection":"higher"}
|
|
{"run":21,"commit":"3d1460b","metric":76.667,"metrics":{"passed_cases":23,"failed_cases":7,"total_cases":30},"status":"keep","description":"baseline","timestamp":1774563087,"segment":8}
|
|
{"run":22,"commit":"38acb52","metric":100.0,"metrics":{"passed_cases":30,"failed_cases":0,"total_cases":30},"status":"keep","description":"add cli execution contracts","timestamp":1774563260,"segment":8}
|
|
{"type":"config","name":"localllm-cli-execution-stubs","metricName":"cli_surface_score","metricUnit":"%","bestDirection":"higher"}
|
|
{"run":23,"commit":"7030290","metric":83.333,"metrics":{"passed_cases":30,"failed_cases":6,"total_cases":36},"status":"keep","description":"baseline","timestamp":1774563600,"segment":9}
|
|
{"run":24,"commit":"50d435a","metric":100.0,"metrics":{"passed_cases":36,"failed_cases":0,"total_cases":36},"status":"keep","description":"add cli execution stubs","timestamp":1774563741,"segment":9}
|
|
{"type":"config","name":"localllm-cli-execution-previews","metricName":"cli_surface_score","metricUnit":"%","bestDirection":"higher"}
|
|
{"run":25,"commit":"cd480c4","metric":85.714,"metrics":{"passed_cases":36,"failed_cases":6,"total_cases":42},"status":"keep","description":"baseline","timestamp":1774564038,"segment":10}
|
|
{"run":26,"commit":"c8ef7cf","metric":100.0,"metrics":{"passed_cases":42,"failed_cases":0,"total_cases":42},"status":"keep","description":"add cli execution previews","timestamp":1774564103,"segment":10}
|
|
{"type":"config","name":"localllm-cli-dry-run-output","metricName":"cli_surface_score","metricUnit":"%","bestDirection":"higher"}
|
|
{"run":27,"commit":"7738acf","metric":87.5,"metrics":{"passed_cases":42,"failed_cases":6,"total_cases":48},"status":"keep","description":"baseline","timestamp":1774564340,"segment":11}
|
|
{"run":28,"commit":"99e5041","metric":100.0,"metrics":{"passed_cases":48,"failed_cases":0,"total_cases":48},"status":"keep","description":"render cli dry-run previews","timestamp":1774564465,"segment":11}
|
|
{"type":"config","name":"localllm-cli-backend-invocations","metricName":"cli_surface_score","metricUnit":"%","bestDirection":"higher"}
|
|
{"run":29,"commit":"1ff5ba3","metric":94.118,"metrics":{"passed_cases":48,"failed_cases":3,"total_cases":51},"status":"keep","description":"baseline","timestamp":1774564791,"segment":12}
|
|
{"run":30,"commit":"bf40b2f","metric":100.0,"metrics":{"passed_cases":51,"failed_cases":0,"total_cases":51},"status":"keep","description":"add cli backend invocation specs","timestamp":1774564898,"segment":12}
|
|
{"type":"config","name":"localllm-cli-backend-requests","metricName":"cli_surface_score","metricUnit":"%","bestDirection":"higher"}
|
|
{"run":31,"commit":"f970c05","metric":94.444,"metrics":{"passed_cases":51,"failed_cases":3,"total_cases":54},"status":"keep","description":"baseline","timestamp":1774565247,"segment":13}
|
|
{"run":32,"commit":"5de35c5","metric":100.0,"metrics":{"passed_cases":54,"failed_cases":0,"total_cases":54},"status":"keep","description":"add cli backend request specs","timestamp":1774565368,"segment":13}
|
|
{"type":"config","name":"localllm-cli-backend-responses","metricName":"cli_surface_score","metricUnit":"%","bestDirection":"higher"}
|
|
{"run":33,"commit":"8cf03e3","metric":94.737,"metrics":{"passed_cases":54,"failed_cases":3,"total_cases":57},"status":"keep","description":"baseline","timestamp":1774566069,"segment":14}
|
|
{"run":34,"commit":"0506a4c","metric":100.0,"metrics":{"passed_cases":57,"failed_cases":0,"total_cases":57},"status":"keep","description":"add cli backend response previews","timestamp":1774566132,"segment":14}
|
|
{"type":"config","name":"localllm-cli-live-output","metricName":"cli_surface_score","metricUnit":"%","bestDirection":"higher"}
|
|
{"run":35,"commit":"e1ada2c","metric":90.476,"metrics":{"passed_cases":57,"failed_cases":6,"total_cases":63},"status":"keep","description":"baseline","timestamp":1774566405,"segment":15}
|
|
{"run":36,"commit":"c54107d","metric":100.0,"metrics":{"passed_cases":63,"failed_cases":0,"total_cases":63},"status":"keep","description":"render cli live outputs","timestamp":1774566456,"segment":15}
|
|
{"type":"config","name":"localllm-cli-live-execution","metricName":"cli_surface_score","metricUnit":"%","bestDirection":"higher"}
|
|
{"run":37,"commit":"388733a","metric":95.455,"metrics":{"passed_cases":63,"failed_cases":3,"total_cases":66},"status":"keep","description":"baseline","timestamp":1774566763,"segment":16}
|
|
{"run":38,"commit":"6920e2a","metric":100.0,"metrics":{"passed_cases":66,"failed_cases":0,"total_cases":66},"status":"keep","description":"add cli live execution pipeline","timestamp":1774566821,"segment":16}
|
|
{"type":"config","name":"localllm-cli-ureq-transport","metricName":"cli_surface_score","metricUnit":"%","bestDirection":"higher"}
|
|
{"run":39,"commit":"9860d64","metric":100.0,"metrics":{"passed_cases":69,"failed_cases":0,"total_cases":69},"status":"keep","description":"add ureq http transport and execute_live_command","timestamp":1774568181,"segment":17}
|
|
{"type":"config","name":"tauri-runtime-manager-unification","metricName":"runtime_surface_score","metricUnit":"%","bestDirection":"higher"}
|
|
{"run":40,"commit":"af2c8d3","metric":100.0,"metrics":{"unit_passed":1,"build_passed":1,"cargo_check_passed":1,"unified_control_plane":1,"design_refactor_done":1},"status":"keep","description":"unify runtime manager config/start flow and refactor runtime UI","timestamp":1774578913,"segment":18}
|
|
{"run":41,"commit":"af2c8d3","metric":100.0,"metrics":{"unit_passed":1,"build_passed":1,"cargo_check_passed":1,"health_probe_passed":1},"status":"keep","description":"replace tcp-only readiness with /health readiness probe","timestamp":1774579405,"segment":18}
|
|
{"run":42,"commit":"af2c8d3","metric":100.0,"metrics":{"unit_passed":1,"build_passed":1,"cargo_check_passed":1,"runtime_smoke_passed":1},"status":"keep","description":"add runtime UI smoke coverage and preserve saved model on empty rescans","timestamp":1774579810,"segment":18}
|
|
{"type":"config","name":"tauri-runtime-lifecycle-acceptance","metricName":"runtime_lifecycle_score","metricUnit":"%","bestDirection":"higher"}
|
|
{"run":43,"commit":"af2c8d3","metric":100.0,"metrics":{"unit_passed":1,"build_passed":1,"cargo_check_passed":1,"local_mode_cta_passed":1},"status":"keep","description":"align the Tauri lifecycle harness and keep the local CTA visible during mode switches","timestamp":1774585583,"segment":19}
|
|
{"run":44,"commit":"af2c8d3","metric":100.0,"metrics":{"unit_passed":1,"build_passed":1,"cargo_check_passed":1,"local_mode_cta_passed":1,"saved_startup_flow_passed":1,"local_onboarding_flow_passed":1},"status":"keep","description":"cover saved local startup transitions and local onboarding from browse to ready","timestamp":1774586404,"segment":19}
|
|
{"run":45,"commit":"af2c8d3","metric":100.0,"metrics":{"unit_passed":1,"build_passed":1,"cargo_check_passed":1,"local_mode_cta_passed":1,"saved_startup_flow_passed":1,"local_onboarding_flow_passed":1,"health_revalidation_passed":1},"status":"keep","description":"revalidate managed local health after startup and surface degraded state","timestamp":1774591917,"segment":19}
|
|
{"type":"config","name":"localllm-batch-assert","segment":20,"metricName":"prefill_batch_assert_score","metricUnit":"%","bestDirection":"higher"}
|
|
{"run":46,"commit":"41559be","metric":100.0,"metrics":{"prefill_plan_passed":1,"cargo_check_passed":1,"server_build_passed":1,"live_completion_passed":1},"status":"keep","description":"chunk prompt prefill explicitly and fix live batch metadata","timestamp":1774624080,"segment":20}
|
|
{"type":"config","name":"tauri-chat-active-history","segment":21,"metricName":"chat_history_score","metricUnit":"%","bestDirection":"higher"}
|
|
{"run":47,"commit":"41559be","metric":100.0,"metrics":{"app_send_flow_passed":1,"chat_switch_abort_guard_passed":1,"frontend_tests_passed":1,"frontend_build_passed":1},"status":"keep","description":"preserve in-flight generation when chatId switches to the newly created chat","timestamp":1774742800,"segment":21}
|