commit f83e8b83f6bca2637aae54bc3d15c5d1b9326432 Author: Evandro Fonseca Junior Date: Thu Mar 26 17:28:25 2026 -0300 chore: initialize autoresearch baseline diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c850cec --- /dev/null +++ b/.gitignore @@ -0,0 +1,30 @@ +# Rust build artifacts +target/ + +# llama.cpp build outputs (inside vendor/) +vendor/llama.cpp/build-dev/ +vendor/llama.cpp/build-prod/ + +# Model files (large binaries – store outside git) +models/*.gguf +models/*.bin +models/*.safetensors + +# Python +__pycache__/ +*.py[cod] +registry/__pycache__/ + +# Env files +.env +.env.* + +# OS +.DS_Store +Thumbs.db +nul + +# Editor +.vscode/settings.json +.idea/ +*.swp diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..994b807 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,1446 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anyhow" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" + +[[package]] +name = "api-contracts" +version = "0.1.0" +dependencies = [ + "chrono", + "serde", + "serde_json", + "uuid", +] + +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "axum" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b52af3cb4058c895d37317bb27508dccc8e5f2d39454016b297bf4a400597b8" +dependencies = [ + "axum-core", + "axum-macros", + "bytes", + "form_urlencoded", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "serde_core", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-macros" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "604fde5e028fea851ce1d8570bbdc034bec850d157f7569d10f347d06808c05c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "bitflags" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" + +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + +[[package]] +name = "bytes" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" + +[[package]] +name = "cc" +version = "1.2.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a0dd1ca384932ff3641c8718a02769f1698e7563dc6974ffd03346116310423" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chrono" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", + "wasip3", +] + +[[package]] +name = "h2" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "http" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "pin-utils", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "bytes", + "http", + "http-body", + "hyper", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "indexmap" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +dependencies = [ + "equivalent", + "hashbrown 0.16.1", + "serde", + "serde_core", +] + +[[package]] +name = "inference-core" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "futures", + "llama-sys", + "serde", + "serde_json", + "thiserror", + "tokio", + "tokio-stream", + "tracing", + "uuid", +] + +[[package]] +name = "inference-server" +version = "0.1.0" +dependencies = [ + "anyhow", + "api-contracts", + "async-stream", + "axum", + "bytes", + "futures", + "hyper", + "inference-core", + "metrics", + "serde", + "serde_json", + "thiserror", + "tokio", + "tokio-stream", + "tower", + "tower-http", + "tracing", + "tracing-subscriber", + "uuid", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + +[[package]] +name = "libc" +version = "0.2.183" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b646652bf6661599e1da8901b3b9522896f01e736bad5f723fe7a3a27f899d" + +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "llama-sys" +version = "0.1.0" +dependencies = [ + "anyhow", + "libloading", + "once_cell", + "tracing", +] + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + +[[package]] +name = "matchit" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" + +[[package]] +name = "memchr" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" + +[[package]] +name = "metrics" +version = "0.1.0" +dependencies = [ + "chrono", + "serde", + "serde_json", + "tokio", + "tracing", +] + +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "mio" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +dependencies = [ + "libc", + "wasi", + "windows-sys", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "semver" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "socket2" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "tokio" +version = "1.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys", +] + +[[package]] +name = "tokio-macros" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-stream" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +dependencies = [ + "bitflags", + "bytes", + "http", + "http-body", + "pin-project-lite", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "log", + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-serde" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1" +dependencies = [ + "serde", + "tracing-core", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "matchers", + "nu-ansi-term", + "once_cell", + "regex-automata", + "serde", + "serde_json", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", + "tracing-serde", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "uuid" +version = "1.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a68d3c8f01c0cfa54a75291d83601161799e4a89a39e0929f4b0354d88757a37" +dependencies = [ + "getrandom", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.2+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..39d1c77 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,61 @@ +[workspace] +resolver = "2" +members = [ + "crates/llama-sys", + "crates/inference-core", + "crates/inference-server", + "crates/api-contracts", + "crates/metrics", +] + +[workspace.package] +version = "0.1.0" +edition = "2021" +authors = [] +license = "MIT" + +[workspace.dependencies] +# Async runtime +tokio = { version = "1", features = ["full"] } +axum = { version = "0.8", features = ["macros"] } +hyper = { version = "1", features = ["full"] } +tower = "0.5" +tower-http = { version = "0.6", features = ["cors", "trace"] } + +# Serialization +serde = { version = "1", features = ["derive"] } +serde_json = "1" + +# Error handling +anyhow = "1" +thiserror = "2" + +# Logging / Tracing +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] } + +# FFI / dynamic loading +libloading = "0.8" +once_cell = "1" + +# Utilities +uuid = { version = "1", features = ["v4"] } +chrono = { version = "0.4", features = ["serde"] } +futures = "0.3" +tokio-stream = "0.1" +bytes = "1" + +[profile.dev] +opt-level = 0 +debug = true + +[profile.release] +opt-level = 3 +lto = "thin" +codegen-units = 1 +strip = "symbols" + +[profile.release-debug] +inherits = "release" +debug = true +strip = "none" diff --git a/README.MD b/README.MD new file mode 100644 index 0000000..7d9f7e2 --- /dev/null +++ b/README.MD @@ -0,0 +1,243 @@ +# ForgeLM + +High-performance local inference runtime for small and mid-sized language models, designed for efficient execution on commodity hardware. + +## Overview + +ForgeLM is a local-first inference runtime built for developers who need a clean, controllable, and efficient backend for running quantized language models on consumer GPUs and constrained systems. + +The project is centered on a practical engineering goal: extracting high throughput and low latency from compact and mid-sized models without turning the runtime into an opaque monolith or an overextended product shell. + +The initial focus is on text-only inference for models in the range of Qwen 4B / 9B class deployments, with special attention to NVIDIA GPUs such as the RTX 3070 and to fallback paths for weaker hardware. + +ForgeLM is not intended to be a desktop UI product. It is intended to be a modular inference backend that can be embedded, extended, benchmarked, and reasoned about as infrastructure. + +--- + +## Design Goals + +ForgeLM is built around a small number of non-negotiable goals: + +- **High throughput** + - Prioritize sustained token generation performance for small and mid-sized models. + +- **Low latency** + - Minimize time-to-first-token and reduce generation overhead in the hot path. + +- **Efficient memory usage** + - Make disciplined use of VRAM and RAM through careful model loading, batching strategy, and KV-cache policies. + +- **Predictable architecture** + - Keep the inference path understandable, inspectable, and modular. + +- **Hardware realism** + - Optimize for hardware that developers actually own, including 8 GB consumer GPUs and CPU fallback scenarios. + +- **Separation of concerns** + - Keep the inference core, API layer, model registry, and tooling clearly separated. + +- **Open integration surface** + - Provide a clean local API compatible with common inference client patterns. + +--- + +## Project Scope + +ForgeLM focuses on the runtime layer of local language model serving. + +### In scope + +- Local text inference +- GGUF-based model execution +- High-performance inference backends +- OpenAI-compatible HTTP API +- Streaming responses via SSE +- Runtime tuning for throughput and memory efficiency +- Model registry and download workflow +- Benchmarking and observability + +### Out of scope for the initial versions + +- Full desktop product UX +- Multimodal inference +- Enterprise orchestration features +- Training and fine-tuning workflows +- Plugin-heavy product abstractions +- Broad feature surface at the expense of runtime efficiency + +--- + +## Core Principles + +### 1. The hot path must remain small + +The token generation path is where performance is won or lost. ForgeLM keeps this path narrow: + +- model execution in a native backend +- Rust in the serving and orchestration layer +- no Python in the inference hot path +- controlled allocation and bounded runtime overhead + +### 2. Performance comes before feature breadth + +A runtime that supports everything but performs poorly is not useful for the target use case. ForgeLM deliberately starts with a constrained feature set in order to produce measurable runtime efficiency first. + +### 3. Public architecture should remain understandable + +The repository is intended to be inspectable by engineers. The design should be explainable without product-language shortcuts, hidden runtime behavior, or unnecessary framework complexity. + +### 4. Consumer hardware is a first-class target + +Many local inference projects are evaluated on unrealistic hardware assumptions. ForgeLM explicitly targets practical systems, including 8 GB GPUs and weaker fallback environments. + +--- + +## Technical Direction + +ForgeLM is structured as a layered system. + +### Inference Core + +The inference core is built around a native execution backend for quantized GGUF models. The initial backend strategy is centered on `llama.cpp`-class execution, selected for its strong hardware coverage, mature quantized model support, and practical performance characteristics. + +Responsibilities: + +- model loading +- tokenizer access +- context management +- decode loop execution +- GPU offloading control +- sampling configuration +- KV-cache handling + +### Serving Layer + +The HTTP serving layer is implemented in Rust and is responsible for: + +- request validation +- session orchestration +- streaming token delivery +- concurrency control +- backpressure +- metrics and health endpoints + +This layer exists to expose the runtime cleanly without polluting the core execution path. + +### Model Registry + +Model search, metadata resolution, and download management are intentionally isolated from the hot path. Registry operations may use a separate process and different tooling, but they do not participate in token generation. + +Responsibilities: + +- model discovery +- metadata normalization +- artifact download +- integrity verification +- local registry indexing + +### Tooling and Benchmarks + +Performance claims without reproducible measurement are not useful. Benchmarking and observability are part of the project rather than afterthoughts. + +Responsibilities: + +- throughput measurement +- latency measurement +- memory usage tracking +- benchmark replay +- runtime profiling + +--- + +## Initial Optimization Target + +The first optimization target is the practical deployment envelope of small and mid-sized text models on commodity NVIDIA hardware. + +That means prioritizing: + +- compact and mid-sized Qwen-class models +- strong throughput on RTX 3070-class systems +- viable degraded execution on weaker hardware +- text-only serving before broader capability expansion + +In practical terms, the project is designed to explore the trade space between: + +- model size +- quantization level +- context length +- GPU offloading depth +- batching strategy +- latency +- aggregate throughput +- output quality + +--- + +## Architectural Priorities + +The implementation roadmap is shaped by the following priorities: + +### Text-only first + +Text inference is the fastest path to a stable and high-performance serving core. Additional modalities, if ever added, must not distort the original runtime goals. + +### Single-model efficiency before multi-model orchestration + +Before adding model multiplexing or dynamic model switching, the runtime must demonstrate excellent single-model performance and stable serving behavior. + +### Controlled concurrency before maximum concurrency + +Concurrency is useful only when it improves real system throughput without collapsing latency or wasting memory. The project favors disciplined scheduling over superficial parallelism. + +### Measured optimization over speculative abstraction + +Every substantial optimization should be benchmarked. Every abstraction in the serving path should justify its cost. + +--- + +## Proposed Stack + +The current technical direction is: + +### Native backend +- C/C++ +- `llama.cpp`-class runtime backend +- CUDA path for NVIDIA acceleration + +### Serving layer +- Rust +- Tokio +- Axum +- Hyper +- FFI bindings generated with `bindgen` + +### Registry and model management +- Python +- isolated background process or service +- Hugging Face integration for model discovery and retrieval + +### UI +- minimal local web interface for runtime validation +- not a product-grade desktop shell + +--- + +## Repository Structure + +```text +forgelm/ +├── Cargo.toml +├── crates/ +│ ├── backend-sys/ +│ ├── inference-core/ +│ ├── inference-server/ +│ ├── api-contracts/ +│ └── metrics/ +├── registry/ +│ ├── service.py +│ └── requirements.txt +├── ui/ +│ └── index.html +├── vendor/ +│ └── backend/ +└── models/ \ No newline at end of file diff --git a/autoresearch.md b/autoresearch.md new file mode 100644 index 0000000..6692111 --- /dev/null +++ b/autoresearch.md @@ -0,0 +1,45 @@ +# Autoresearch: latency-p95-chat-completions + +## Objective +Optimize end-to-end p95 latency for local OpenAI-compatible `POST /v1/chat/completions` +requests against the Rust inference server, using the local Qwen GGUF model already +present in `models/`. The benchmark targets a short coding-oriented request that is +representative of editor integrations such as VSCode, OpenCode, Cline, and Roo. + +## Metrics +- **Primary**: p95_ms (milliseconds, lower is better) +- **Secondary**: p50_ms, mean_ms, completion_tps_mean, success_count + +## How to Run +`./autoresearch.sh` - builds the release server, starts it locally, runs a fixed HTTP benchmark, and outputs `METRIC name=number` lines. + +## Files in Scope +- `crates/inference-server/src/compat.rs` - request shaping, prompt preparation, auth/CORS/config, streaming mode decisions +- `crates/inference-server/src/routes.rs` - HTTP handlers, blocking/streaming response path, usage/reporting +- `crates/inference-server/src/main.rs` - runtime/server boot configuration +- `crates/inference-core/src/engine.rs` - engine load/warmup/session creation path +- `crates/inference-core/src/session.rs` - prompt prefill and token generation hot path +- `crates/inference-core/src/profile.rs` - runtime defaults that affect latency +- `crates/inference-core/src/sampler.rs` - per-request generation config +- `crates/metrics/src/*` - measurement plumbing if needed +- `experiments/bench_chat_latency.py` - benchmark harness for the autoresearch loop +- `autoresearch.sh` - loop benchmark entrypoint and fast preflight + +## Off Limits +- `models/` model binaries +- `vendor/` third-party sources and prebuilt llama binaries +- `registry/` Python registry service +- `ui/` frontend files +- broad API-surface feature additions unrelated to latency unless they directly reduce p95 + +## Constraints +- Keep the benchmark reproducible and deterministic enough for comparison. +- No new external dependencies. +- Preserve OpenAI-compatible behavior already implemented. +- Tests must pass for keeps. +- Prefer changes that improve p95 without making code materially harder to maintain. +- Aggressive refactors are allowed if they stay within the in-scope server/runtime path. + +## What's Been Tried +- Baseline session initialized; no latency experiments logged yet. +- Existing work before this autoresearch session already improved token detokenization and broadened OpenAI-compatible request/response handling, but this loop focuses specifically on p95 latency. diff --git a/autoresearch.sh b/autoresearch.sh new file mode 100644 index 0000000..45cd03f --- /dev/null +++ b/autoresearch.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$SCRIPT_DIR" +PORT="${LOCALLLM_BENCH_PORT:-18080}" +HOST="127.0.0.1:${PORT}" +MODEL_PATH="${LOCALLLM_MODEL:-$ROOT/models/Qwen3.5-4B-Abliterated-Claude-4.6-Opus-Reasoning-Distilled.i1-Q6_K.gguf}" +SERVER_BIN="$ROOT/target/release/localllm.exe" +LOG_DIR="$ROOT/logs" +SERVER_LOG="$LOG_DIR/autoresearch-server.log" + +mkdir -p "$LOG_DIR" "$ROOT/experiments" + +if [[ ! -f "$MODEL_PATH" ]]; then + echo "[ERROR] Model file not found: $MODEL_PATH" >&2 + exit 1 +fi + +cleanup() { + if [[ -n "${SERVER_PID:-}" ]]; then + kill "$SERVER_PID" >/dev/null 2>&1 || true + wait "$SERVER_PID" 2>/dev/null || true + fi +} +trap cleanup EXIT + +echo "[INFO] Fast preflight (cargo check)..." >&2 +cargo check -p inference-server --quiet + +echo "[INFO] Building release server..." >&2 +cargo build --release -p inference-server --bin localllm --quiet + +echo "[INFO] Starting server on $HOST ..." >&2 +LOCALLLM_ADDR="$HOST" \ +LOCALLLM_MODEL="$MODEL_PATH" \ +"$SERVER_BIN" >"$SERVER_LOG" 2>&1 & +SERVER_PID=$! + +python "$ROOT/experiments/bench_chat_latency.py" \ + --base-url "http://$HOST" \ + --server-log "$SERVER_LOG" diff --git a/crates/api-contracts/Cargo.toml b/crates/api-contracts/Cargo.toml new file mode 100644 index 0000000..25abfb8 --- /dev/null +++ b/crates/api-contracts/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "api-contracts" +version.workspace = true +edition.workspace = true + +[dependencies] +serde.workspace = true +serde_json.workspace = true +uuid.workspace = true +chrono.workspace = true diff --git a/crates/api-contracts/src/chat.rs b/crates/api-contracts/src/chat.rs new file mode 100644 index 0000000..ebcf3d1 --- /dev/null +++ b/crates/api-contracts/src/chat.rs @@ -0,0 +1,361 @@ +use serde::{Deserialize, Deserializer, Serialize}; +use serde_json::Value; + +#[derive(Debug, Clone, Deserialize)] +pub struct ChatCompletionRequest { + pub model: String, + pub messages: Vec, + #[serde(default = "default_max_tokens", alias = "max_completion_tokens")] + pub max_tokens: u32, + #[serde(default = "default_temperature")] + pub temperature: f32, + #[serde(default = "default_top_p")] + pub top_p: f32, + #[serde(default)] + pub stream: bool, + #[serde(default, deserialize_with = "deserialize_stop_sequences")] + pub stop: Vec, + #[serde(default = "default_n")] + pub n: u32, + #[serde(default)] + pub seed: Option, + #[serde(default)] + pub user: Option, + #[serde(default)] + pub response_format: Option, + #[serde(default)] + pub tools: Vec, + #[serde(default)] + pub tool_choice: Option, + #[serde(default)] + pub parallel_tool_calls: Option, + #[serde(default)] + pub stream_options: Option, +} + +fn default_max_tokens() -> u32 { + 1024 +} + +fn default_temperature() -> f32 { + 0.15 +} + +fn default_top_p() -> f32 { + 0.95 +} + +fn default_n() -> u32 { + 1 +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ChatMessage { + pub role: Role, + #[serde( + default, + deserialize_with = "deserialize_message_content", + skip_serializing_if = "Option::is_none" + )] + pub content: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tool_calls: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tool_call_id: Option, +} + +impl ChatMessage { + pub fn text(&self) -> &str { + self.content.as_deref().unwrap_or("") + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum Role { + System, + Developer, + User, + Assistant, + Tool, +} + +#[derive(Debug, Clone, Serialize)] +pub struct ChatCompletionResponse { + pub id: String, + pub object: &'static str, + pub created: i64, + pub model: String, + pub choices: Vec, + pub usage: Usage, +} + +#[derive(Debug, Clone, Serialize)] +pub struct Choice { + pub index: u32, + pub message: ChatMessage, + pub finish_reason: FinishReason, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum FinishReason { + Stop, + Length, + ToolCalls, +} + +#[derive(Debug, Clone, Serialize)] +pub struct ChatCompletionChunk { + pub id: String, + pub object: &'static str, + pub created: i64, + pub model: String, + pub choices: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub usage: Option, +} + +#[derive(Debug, Clone, Serialize)] +pub struct ChunkChoice { + pub index: u32, + pub delta: Delta, + pub finish_reason: Option, +} + +#[derive(Debug, Clone, Serialize)] +pub struct Delta { + #[serde(skip_serializing_if = "Option::is_none")] + pub role: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub content: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub tool_calls: Option>, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ToolDefinition { + #[serde(rename = "type")] + pub tool_type: ToolType, + pub function: FunctionDefinition, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum ToolType { + Function, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FunctionDefinition { + pub name: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option, + #[serde(default)] + pub parameters: Value, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ToolCall { + pub id: String, + #[serde(rename = "type")] + pub tool_type: ToolType, + pub function: ToolFunctionCall, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ToolFunctionCall { + pub name: String, + pub arguments: String, +} + +#[derive(Debug, Clone, Serialize)] +pub struct DeltaToolCall { + pub index: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub id: Option, + #[serde(rename = "type", skip_serializing_if = "Option::is_none")] + pub tool_type: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub function: Option, +} + +#[derive(Debug, Clone, Serialize)] +pub struct DeltaToolFunctionCall { + #[serde(skip_serializing_if = "Option::is_none")] + pub name: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub arguments: Option, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(untagged)] +pub enum ToolChoice { + Mode(String), + Specific(ToolChoiceObject), +} + +#[derive(Debug, Clone, Deserialize)] +pub struct ToolChoiceObject { + #[serde(rename = "type")] + pub choice_type: ToolType, + pub function: ToolChoiceFunction, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct ToolChoiceFunction { + pub name: String, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct ResponseFormat { + #[serde(rename = "type")] + pub format_type: String, + #[serde(default)] + pub json_schema: Option, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct StreamOptions { + #[serde(default)] + pub include_usage: bool, +} + +#[derive(Debug, Clone, Serialize, Default)] +pub struct Usage { + pub prompt_tokens: u32, + pub completion_tokens: u32, + pub total_tokens: u32, + pub tokens_per_second: f64, +} + +fn deserialize_stop_sequences<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let value = Option::::deserialize(deserializer)?; + let Some(value) = value else { + return Ok(Vec::new()); + }; + + match value { + Value::Null => Ok(Vec::new()), + Value::String(s) => Ok(vec![s]), + Value::Array(items) => items + .into_iter() + .map(|item| match item { + Value::String(s) => Ok(s), + _ => Err(serde::de::Error::custom("stop items must be strings")), + }) + .collect(), + _ => Err(serde::de::Error::custom( + "stop must be a string, array of strings, or null", + )), + } +} + +fn deserialize_message_content<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let value = Option::::deserialize(deserializer)?; + let Some(value) = value else { + return Ok(None); + }; + + match value { + Value::Null => Ok(None), + Value::String(s) => Ok(Some(s)), + Value::Array(items) => { + let mut text_parts = Vec::new(); + + for item in items { + match item { + Value::String(s) => text_parts.push(s), + Value::Object(map) => { + let item_type = map.get("type").and_then(Value::as_str).unwrap_or("text"); + if matches!(item_type, "text" | "input_text" | "output_text") { + if let Some(text) = map.get("text").and_then(Value::as_str) { + text_parts.push(text.to_owned()); + } else if let Some(text) = map.get("content").and_then(Value::as_str) { + text_parts.push(text.to_owned()); + } + } + } + _ => { + return Err(serde::de::Error::custom( + "message content array items must be strings or text parts", + )); + } + } + } + + if text_parts.is_empty() { + Ok(None) + } else { + Ok(Some(text_parts.join("\n"))) + } + } + _ => Err(serde::de::Error::custom( + "message content must be a string, array, or null", + )), + } +} + +#[cfg(test)] +mod tests { + use super::{ChatCompletionRequest, Role, ToolChoice}; + + #[test] + fn deserializes_openai_style_message_parts_and_aliases() { + let request: ChatCompletionRequest = serde_json::from_str( + r#"{ + "model": "alias-model", + "messages": [ + { + "role": "developer", + "content": [ + { "type": "text", "text": "Write code" }, + { "type": "input_text", "text": "Use Rust" } + ] + }, + { + "role": "user", + "content": "Build a parser" + } + ], + "max_completion_tokens": 77, + "stop": "", + "tool_choice": "auto" + }"#, + ) + .unwrap(); + + assert_eq!(request.max_tokens, 77); + assert_eq!(request.stop, vec![""]); + assert!(matches!(request.messages[0].role, Role::Developer)); + assert_eq!(request.messages[0].text(), "Write code\nUse Rust"); + assert!(matches!(request.tool_choice, Some(ToolChoice::Mode(ref mode)) if mode == "auto")); + } + + #[test] + fn deserializes_stop_arrays_and_null_content() { + let request: ChatCompletionRequest = serde_json::from_str( + r#"{ + "model": "alias-model", + "messages": [ + { "role": "assistant", "content": null }, + { "role": "tool", "tool_call_id": "call_1", "content": "done" } + ], + "stop": ["END", "STOP"] + }"#, + ) + .unwrap(); + + assert_eq!(request.stop, vec!["END", "STOP"]); + assert_eq!(request.messages[0].content, None); + assert_eq!(request.messages[1].tool_call_id.as_deref(), Some("call_1")); + } +} diff --git a/crates/api-contracts/src/error.rs b/crates/api-contracts/src/error.rs new file mode 100644 index 0000000..2fdf087 --- /dev/null +++ b/crates/api-contracts/src/error.rs @@ -0,0 +1,38 @@ +use serde::Serialize; + +#[derive(Debug, Clone, Serialize)] +pub struct ErrorResponse { + pub error: ApiError, +} + +#[derive(Debug, Clone, Serialize)] +pub struct ApiError { + pub message: String, + #[serde(rename = "type")] + pub error_type: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub param: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub code: Option, +} + +impl ApiError { + pub fn new(message: impl Into, error_type: impl Into) -> Self { + Self { + message: message.into(), + error_type: error_type.into(), + param: None, + code: None, + } + } + + pub fn with_code(mut self, code: impl Into) -> Self { + self.code = Some(code.into()); + self + } + + pub fn with_param(mut self, param: impl Into) -> Self { + self.param = Some(param.into()); + self + } +} diff --git a/crates/api-contracts/src/health.rs b/crates/api-contracts/src/health.rs new file mode 100644 index 0000000..af59295 --- /dev/null +++ b/crates/api-contracts/src/health.rs @@ -0,0 +1,9 @@ +use serde::Serialize; + +#[derive(Debug, Clone, Serialize)] +pub struct HealthResponse { + pub status: &'static str, + pub model_loaded: bool, + pub model_id: Option, + pub uptime_secs: u64, +} diff --git a/crates/api-contracts/src/lib.rs b/crates/api-contracts/src/lib.rs new file mode 100644 index 0000000..c674e9c --- /dev/null +++ b/crates/api-contracts/src/lib.rs @@ -0,0 +1,6 @@ +// api-contracts: shared request/response types (OpenAI-compatible) + +pub mod chat; +pub mod error; +pub mod health; +pub mod models; diff --git a/crates/api-contracts/src/models.rs b/crates/api-contracts/src/models.rs new file mode 100644 index 0000000..b7f333c --- /dev/null +++ b/crates/api-contracts/src/models.rs @@ -0,0 +1,21 @@ +use serde::Serialize; +use serde_json::Value; + +#[derive(Debug, Clone, Serialize)] +pub struct ModelList { + pub object: &'static str, + pub data: Vec, +} + +#[derive(Debug, Clone, Serialize)] +pub struct ModelInfo { + pub id: String, + pub object: &'static str, + pub created: i64, + pub owned_by: &'static str, + pub root: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub parent: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub permission: Vec, +} diff --git a/crates/inference-core/Cargo.toml b/crates/inference-core/Cargo.toml new file mode 100644 index 0000000..1d089ea --- /dev/null +++ b/crates/inference-core/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "inference-core" +version.workspace = true +edition.workspace = true + +[dependencies] +llama-sys = { path = "../llama-sys" } +anyhow.workspace = true +thiserror.workspace = true +tracing.workspace = true +serde.workspace = true +serde_json.workspace = true +tokio = { workspace = true } +futures.workspace = true +tokio-stream.workspace = true +uuid.workspace = true +chrono.workspace = true diff --git a/crates/inference-core/src/engine.rs b/crates/inference-core/src/engine.rs new file mode 100644 index 0000000..8aba11e --- /dev/null +++ b/crates/inference-core/src/engine.rs @@ -0,0 +1,305 @@ +//! InferenceEngine: loads a GGUF model via llama-sys FFI and manages the +//! lifetime of `llama_model` + `llama_context`. +//! +//! Design decisions: +//! - Single model, single context per engine instance (Phase 2 design). +//! - `LlamaLib` is loaded once and stored in an `Arc` so future session spawning +//! can share it without re-loading the dynamic library. +//! - All raw pointers are wrapped in RAII guards that call the matching +//! `llama_free` / `llama_model_free` on drop. +//! - The engine is `Send` but NOT `Sync` – callers must coordinate access +//! (the scheduler in a later phase will own this exclusively). + +use std::ffi::CString; +use std::path::PathBuf; +use std::sync::Arc; + +use anyhow::{bail, Context, Result}; +use tracing::{info, warn}; + +use llama_sys::{ + llama_context, llama_model, LlamaLib, LLAMA_FLASH_ATTN_TYPE_DISABLED, + LLAMA_FLASH_ATTN_TYPE_ENABLED, +}; + +use crate::profile::RuntimeProfile; +use crate::sampler::SamplerConfig; +use crate::session::GenerationSession; + +// ─── Configuration ───────────────────────────────────────────────────────── + +/// Configuration for the inference engine. +#[derive(Debug, Clone)] +pub struct EngineConfig { + /// Path to the GGUF model file. + pub model_path: PathBuf, + /// Runtime performance profile (used to fill in defaults). + pub profile: RuntimeProfile, + /// How many GPU layers to offload (999 = all). + pub n_gpu_layers: i32, + /// Context window size in tokens. + pub n_ctx: u32, + /// Physical batch size (prompt processing). + pub n_batch: u32, + /// Micro-batch size (generation step). + pub n_ubatch: u32, + /// Number of CPU threads for generation. + pub n_threads: i32, + /// Number of CPU threads for batch/prefill. + pub n_threads_batch: i32, + /// Enable flash attention. + pub flash_attn: bool, + /// Use mmap for model loading (false = load fully into RAM/VRAM). + pub use_mmap: bool, + /// RNG seed for sampling (0 = random). + pub seed: u32, +} + +impl Default for EngineConfig { + fn default() -> Self { + let prof = RuntimeProfile::Turbo4B; + let d = prof.defaults(); + Self { + model_path: PathBuf::new(), + profile: prof, + n_gpu_layers: d.n_gpu_layers, + n_ctx: d.n_ctx, + n_batch: d.n_batch, + n_ubatch: d.n_ubatch, + n_threads: d.n_threads, + n_threads_batch: d.n_threads, + flash_attn: d.flash_attn, + use_mmap: false, // --no-mmap recommended for CUDA + seed: 0, + } + } +} + +impl EngineConfig { + /// Build from a profile, applying its defaults. + pub fn from_profile(profile: RuntimeProfile, model_path: PathBuf) -> Self { + let d = profile.defaults(); + Self { + model_path, + profile, + n_gpu_layers: d.n_gpu_layers, + n_ctx: d.n_ctx, + n_batch: d.n_batch, + n_ubatch: d.n_ubatch, + n_threads: d.n_threads, + n_threads_batch: d.n_threads, + flash_attn: d.flash_attn, + use_mmap: false, + seed: 0, + } + } +} + +// ─── RAII wrappers ───────────────────────────────────────────────────────── + +/// Owns a `*mut llama_model` and frees it on drop. +struct OwnedModel { + ptr: *mut llama_model, + lib: Arc, +} + +impl Drop for OwnedModel { + fn drop(&mut self) { + if !self.ptr.is_null() { + unsafe { (self.lib.model_free)(self.ptr) }; + } + } +} + +// SAFETY: llama_model is reference-counted internally; we are the only owner. +unsafe impl Send for OwnedModel {} + +/// Owns a `*mut llama_context` and frees it on drop. +pub(crate) struct OwnedContext { + pub ptr: *mut llama_context, + pub lib: Arc, +} + +impl Drop for OwnedContext { + fn drop(&mut self) { + if !self.ptr.is_null() { + unsafe { (self.lib.free)(self.ptr) }; + } + } +} + +// SAFETY: we enforce single-threaded access through InferenceEngine. +unsafe impl Send for OwnedContext {} + +// ─── Engine ──────────────────────────────────────────────────────────────── + +/// Core inference engine. +/// +/// - Owns a loaded model and its associated context. +/// - Provides `new_session()` to start a streaming generation. +/// - NOT `Sync` – the scheduler must ensure single-threaded use. +pub struct InferenceEngine { + pub config: EngineConfig, + pub(crate) lib: Arc, + _model: OwnedModel, + pub(crate) ctx: OwnedContext, +} + +impl InferenceEngine { + /// Load a GGUF model and initialise a llama_context. + /// + /// Blocks until the model is fully loaded. + pub fn load(config: EngineConfig) -> Result { + if config.model_path.as_os_str().is_empty() { + bail!("EngineConfig.model_path is empty"); + } + if !config.model_path.exists() { + bail!("Model file not found: {}", config.model_path.display()); + } + + info!( + model = %config.model_path.display(), + profile = ?config.profile, + n_gpu_layers = config.n_gpu_layers, + n_ctx = config.n_ctx, + n_batch = config.n_batch, + n_ubatch = config.n_ubatch, + flash_attn = config.flash_attn, + use_mmap = config.use_mmap, + "Loading model via llama-sys" + ); + + // 1. Load dynamic library + let lib = Arc::new(LlamaLib::load_auto().context("Failed to load llama.dll")?); + + // 2. Backend init + unsafe { + (lib.backend_load_all)(); + (lib.backend_init)(); + } + info!("llama backend initialised"); + + // 3. Model params + let mut mparams = unsafe { (lib.model_default_params)() }; + mparams.n_gpu_layers = config.n_gpu_layers; + mparams.use_mmap = config.use_mmap; + + // 4. Load model from file + let model_path_c = CString::new(config.model_path.to_str().context("Non-UTF8 model path")?) + .context("CString::new failed")?; + + let model_ptr = unsafe { (lib.model_load_from_file)(model_path_c.as_ptr(), mparams) }; + if model_ptr.is_null() { + bail!("llama_model_load_from_file returned NULL – check model path and VRAM"); + } + + // Log model info + { + let mut desc = vec![0u8; 256]; + unsafe { + (lib.model_desc)(model_ptr, desc.as_mut_ptr() as *mut _, desc.len()); + } + let desc_str = std::ffi::CStr::from_bytes_until_nul(&desc) + .map(|s| s.to_string_lossy().into_owned()) + .unwrap_or_else(|_| "".into()); + let n_params = unsafe { (lib.model_n_params)(model_ptr) }; + let is_hybrid = unsafe { (lib.model_is_hybrid)(model_ptr) }; + let is_recurrent = unsafe { (lib.model_is_recurrent)(model_ptr) }; + info!( + desc = %desc_str, + params_b = n_params as f64 / 1e9, + is_hybrid, + is_recurrent, + "Model loaded" + ); + } + + let model = OwnedModel { + ptr: model_ptr, + lib: Arc::clone(&lib), + }; + + // 5. Context params + let mut cparams = unsafe { (lib.context_default_params)() }; + cparams.n_ctx = config.n_ctx; + cparams.n_batch = config.n_batch; + cparams.n_ubatch = config.n_ubatch; + cparams.n_threads = config.n_threads; + cparams.n_threads_batch = config.n_threads_batch; + cparams.flash_attn_type = if config.flash_attn { + LLAMA_FLASH_ATTN_TYPE_ENABLED + } else { + LLAMA_FLASH_ATTN_TYPE_DISABLED + }; + cparams.offload_kqv = true; + cparams.no_perf = false; + + // 6. Init context + let ctx_ptr = unsafe { (lib.init_from_model)(model_ptr, cparams) }; + if ctx_ptr.is_null() { + bail!("llama_init_from_model returned NULL – insufficient VRAM or bad params"); + } + + let actual_ctx = unsafe { (lib.n_ctx)(ctx_ptr) }; + info!(n_ctx = actual_ctx, "Context initialised"); + + let ctx = OwnedContext { + ptr: ctx_ptr, + lib: Arc::clone(&lib), + }; + + Ok(Self { + config, + lib, + _model: model, + ctx, + }) + } + + /// Warm up: run a single dummy decode to pre-allocate CUDA buffers. + /// + /// Call once after `load()` before accepting real requests. + pub fn warmup(&mut self) -> Result<()> { + info!("Running engine warmup (dummy decode)…"); + + unsafe { + // Mark context as warmup so llama.cpp skips perf counters + if let Ok(f) = std::panic::catch_unwind(|| {}) { + let _ = f; + } + // set_warmup may not exist in all builds; call decode with 1 token + let dummy_token: llama_sys::llama_token = 1; // BOS-ish token + let mut tokens = vec![dummy_token]; + let batch = (self.lib.batch_get_one)(tokens.as_mut_ptr(), 1); + let ret = (self.lib.decode)(self.ctx.ptr, batch); + if ret != 0 { + warn!(ret, "Warmup decode returned non-zero (non-fatal)"); + } + } + + info!("Warmup complete"); + Ok(()) + } + + /// Create a new generation session for a single request. + /// + /// `prompt` is the fully-formatted string (system + user turns already joined). + /// The engine must not be accessed concurrently while the session is active. + pub fn new_session( + &mut self, + prompt: &str, + sampler_cfg: SamplerConfig, + ) -> Result { + let vocab = unsafe { (self.lib.model_get_vocab)(self._model.ptr) }; + GenerationSession::with_vocab( + Arc::clone(&self.lib), + self.ctx.ptr, + vocab, + prompt, + sampler_cfg, + ) + } +} + +// SAFETY: we are the sole owner of the pointers; no concurrent access. +unsafe impl Send for InferenceEngine {} diff --git a/crates/inference-core/src/lib.rs b/crates/inference-core/src/lib.rs new file mode 100644 index 0000000..2acbdfa --- /dev/null +++ b/crates/inference-core/src/lib.rs @@ -0,0 +1,11 @@ +// inference-core: engine abstraction over llama-sys + +pub mod engine; +pub mod profile; +pub mod sampler; +pub mod session; + +pub use engine::{EngineConfig, InferenceEngine}; +pub use profile::RuntimeProfile; +pub use sampler::SamplerConfig; +pub use session::GenerationSession; diff --git a/crates/inference-core/src/profile.rs b/crates/inference-core/src/profile.rs new file mode 100644 index 0000000..78d6309 --- /dev/null +++ b/crates/inference-core/src/profile.rs @@ -0,0 +1,59 @@ +/// Predefined runtime profiles matching the plan targets. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RuntimeProfile { + /// Qwen3.5-4B – maximum TPS, text-only, flash attention, moderate context. + Turbo4B, + /// Qwen3.5-9B – balanced quality/speed, tuned KV cache. + Balanced9B, + /// CPU-only fallback for machines without a GPU. + CpuFallback, +} + +impl RuntimeProfile { + /// Returns sensible defaults for each profile. + pub fn defaults(&self) -> ProfileDefaults { + match self { + Self::Turbo4B => ProfileDefaults { + n_gpu_layers: 999, + n_ctx: 4096, + n_batch: 512, + n_ubatch: 128, + flash_attn: true, + n_threads: 8, + }, + Self::Balanced9B => ProfileDefaults { + n_gpu_layers: 999, + n_ctx: 2048, + n_batch: 256, + n_ubatch: 64, + flash_attn: true, + n_threads: 8, + }, + Self::CpuFallback => ProfileDefaults { + n_gpu_layers: 0, + n_ctx: 2048, + n_batch: 128, + n_ubatch: 32, + flash_attn: false, + n_threads: num_cpus(), + }, + } + } +} + +fn num_cpus() -> i32 { + // Simple fallback; replace with `num_cpus` crate if needed. + std::thread::available_parallelism() + .map(|n| n.get() as i32) + .unwrap_or(4) +} + +#[derive(Debug, Clone)] +pub struct ProfileDefaults { + pub n_gpu_layers: i32, + pub n_ctx: u32, + pub n_batch: u32, + pub n_ubatch: u32, + pub flash_attn: bool, + pub n_threads: i32, +} diff --git a/crates/inference-core/src/sampler.rs b/crates/inference-core/src/sampler.rs new file mode 100644 index 0000000..24cbeff --- /dev/null +++ b/crates/inference-core/src/sampler.rs @@ -0,0 +1,37 @@ +/// Sampling hyper-parameters for a single generation request. +#[derive(Debug, Clone)] +pub struct SamplerConfig { + pub temperature: f32, + pub top_p: f32, + pub top_k: i32, + pub repeat_penalty: f32, + pub max_tokens: u32, + pub stop_sequences: Vec, + pub seed: u32, +} + +impl Default for SamplerConfig { + fn default() -> Self { + Self { + temperature: 0.7, + top_p: 0.9, + top_k: 40, + repeat_penalty: 1.1, + max_tokens: 512, + stop_sequences: vec![], + seed: 0, + } + } +} + +impl SamplerConfig { + pub fn from_request(temp: f32, top_p: f32, max_tokens: u32, stop: Vec) -> Self { + Self { + temperature: temp, + top_p, + max_tokens, + stop_sequences: stop, + ..Default::default() + } + } +} diff --git a/crates/inference-core/src/session.rs b/crates/inference-core/src/session.rs new file mode 100644 index 0000000..a0defd6 --- /dev/null +++ b/crates/inference-core/src/session.rs @@ -0,0 +1,376 @@ +//! GenerationSession: single-request token-by-token generation loop. +//! +//! Flow: +//! 1. Tokenize the prompt (including system + user messages). +//! 2. Prefill: feed all prompt tokens through llama_decode() in one batch. +//! 3. Generation loop: sample → decode → yield token text → check stop. +//! +//! The session borrows raw pointers from InferenceEngine and must not outlive it. +//! In Phase 7+ this will be refactored to use a shared context pool. + +use std::ffi::{CStr, CString}; +use std::sync::Arc; +use std::time::Instant; + +use anyhow::{bail, Context, Result}; +use tracing::{debug, info}; + +use llama_sys::{llama_context, llama_token, llama_vocab, LlamaLib}; + +use crate::sampler::SamplerConfig; + +// ─── GenerationSession ───────────────────────────────────────────────────── + +pub struct GenerationSession { + lib: Arc, + ctx: *mut llama_context, + vocab: *const llama_vocab, + sampler: *mut llama_sys::llama_sampler, + stop_sequences: Vec, + + // generation state + n_decoded: i32, // number of tokens decoded so far (prefill + gen) + prompt_tokens: u32, + pub max_tokens: u32, + pub tokens_generated: u32, + pub finished: bool, + + // timing + pub prefill_ms: Option, + gen_start: Option, + pub tokens_per_sec: f64, +} + +// SAFETY: pointers come from InferenceEngine which is Send. +unsafe impl Send for GenerationSession {} + +impl GenerationSession { + /// Tokenize prompt and run prefill decode. Returns a session ready to generate. + pub fn new( + _lib: Arc, + _ctx: *mut llama_context, + _sampler_cfg: SamplerConfig, + ) -> Result { + // Get vocab from context's model (via a workaround: we pass vocab separately) + // For now we'll get it indirectly — see Engine::new_session which passes vocab. + // This constructor signature will be refined; use the vocab variant below. + bail!("Use GenerationSession::with_vocab() instead") + } + + /// Full constructor called by InferenceEngine::new_session(). + pub fn with_vocab( + lib: Arc, + ctx: *mut llama_context, + vocab: *const llama_vocab, + prompt: &str, + sampler_cfg: SamplerConfig, + ) -> Result { + // ── 1. Tokenize prompt ───────────────────────────────────────────── + let prompt_c = CString::new(prompt).context("Invalid UTF-8 in prompt")?; + + // First call: get token count + let ret = unsafe { + (lib.tokenize)( + vocab, + prompt_c.as_ptr(), + prompt.len() as i32, + std::ptr::null_mut(), + 0, + true, // add_special (BOS) + true, // parse_special + ) + }; + + let n_tokens_needed = if ret < 0 { -ret } else { ret }; + + if n_tokens_needed == 0 { + bail!("llama_tokenize failed to return token count"); + } + + let mut tokens: Vec = vec![0; n_tokens_needed as usize]; + let n_tokens = unsafe { + (lib.tokenize)( + vocab, + prompt_c.as_ptr(), + prompt.len() as i32, + tokens.as_mut_ptr(), + tokens.len() as i32, + true, + true, + ) + }; + + if n_tokens < 0 { + bail!( + "llama_tokenize (second call) failed (returned {})", + n_tokens + ); + } + tokens.truncate(n_tokens as usize); + + let n_prompt_tokens = tokens.len() as i32; + debug!(n_tokens = n_prompt_tokens, "Prompt tokenised"); + + // ── 2. Build sampler chain ───────────────────────────────────────── + let sc_params = unsafe { (lib.sampler_chain_default_params)() }; + let sampler_chain = unsafe { (lib.sampler_chain_init)(sc_params) }; + if sampler_chain.is_null() { + bail!("sampler_chain_init returned NULL"); + } + + unsafe { + if sampler_cfg.temperature <= 0.0 || sampler_cfg.temperature < 1e-6 { + // Greedy + let s = (lib.sampler_init_greedy)(); + (lib.sampler_chain_add)(sampler_chain, s); + } else { + // top-k → temp → top-p → dist (for randomness) + let s_topk = (lib.sampler_init_top_k)(sampler_cfg.top_k); + (lib.sampler_chain_add)(sampler_chain, s_topk); + + let s_temp = (lib.sampler_init_temp)(sampler_cfg.temperature); + (lib.sampler_chain_add)(sampler_chain, s_temp); + + let s_topp = (lib.sampler_init_top_p)(sampler_cfg.top_p, 1); + (lib.sampler_chain_add)(sampler_chain, s_topp); + + let s_dist = (lib.sampler_init_dist)(sampler_cfg.seed); + (lib.sampler_chain_add)(sampler_chain, s_dist); + } + } + + // ── 3. Prefill decode ────────────────────────────────────────────── + let prefill_start = Instant::now(); + + let ret = unsafe { + let batch = (lib.batch_get_one)(tokens.as_mut_ptr(), n_tokens); + (lib.decode)(ctx, batch) + }; + + if ret != 0 { + unsafe { (lib.sampler_free)(sampler_chain) }; + bail!( + "llama_decode (prefill) returned {} — context may be full", + ret + ); + } + + let prefill_ms = prefill_start.elapsed().as_secs_f64() * 1000.0; + let prefill_tps = n_prompt_tokens as f64 / (prefill_ms / 1000.0); + info!( + "Prefill finished in {:.2} ms ({:.2} t/s)", + prefill_ms, prefill_tps + ); + + // create session state + Ok(Self { + lib, + ctx, + vocab, + sampler: sampler_chain, // use sampler_chain here + stop_sequences: sampler_cfg.stop_sequences.clone(), + n_decoded: n_prompt_tokens, + prompt_tokens: n_prompt_tokens as u32, + max_tokens: sampler_cfg.max_tokens, + tokens_generated: 0, + finished: false, + prefill_ms: Some(prefill_ms), + gen_start: None, + tokens_per_sec: 0.0, + }) + } + + /// Sample and decode the next token. Returns `None` when generation ends. + /// + /// Each call advances the context by one token. + pub fn next_token(&mut self) -> Option { + if self.finished { + return None; + } + if self.tokens_generated >= self.max_tokens { + self.finished = true; + self.finalize_timing(); + return None; + } + + // Start timing on first generated token + if self.gen_start.is_none() { + self.gen_start = Some(Instant::now()); + } + + unsafe { + // Sample the next token + let new_token = (self.lib.sampler_sample)(self.sampler, self.ctx, -1); + + // Check end-of-generation + if (self.lib.vocab_is_eog)(self.vocab, new_token) { + self.finished = true; + self.finalize_timing(); + return None; + } + + // Decode (advance context with the new token) + let mut token_arr = [new_token]; + let batch = (self.lib.batch_get_one)(token_arr.as_mut_ptr(), 1); + let ret = (self.lib.decode)(self.ctx, batch); + if ret != 0 { + debug!(ret, "llama_decode (gen step) error — stopping"); + self.finished = true; + self.finalize_timing(); + return None; + } + + self.n_decoded += 1; + self.tokens_generated += 1; + + // Detokenize + let token_text = self.detokenize_token(new_token); + + // Check stop sequences + // (simple check: if generated text so far ends with a stop seq) + // For streaming we check per-token — good enough for Phase 2. + for stop in &self.stop_sequences { + if token_text.trim_end() == stop.as_str() { + self.finished = true; + self.finalize_timing(); + return None; + } + } + + Some(token_text) + } + } + + fn finalize_timing(&mut self) { + if let Some(start) = self.gen_start.take() { + let elapsed = start.elapsed().as_secs_f64(); + if elapsed > 0.0 { + self.tokens_per_sec = self.tokens_generated as f64 / elapsed; + } + } + info!( + tokens_generated = self.tokens_generated, + tps = format!("{:.1}", self.tokens_per_sec), + prefill_ms = self + .prefill_ms + .map(|v| format!("{:.1}", v)) + .unwrap_or_default(), + "Generation complete" + ); + } + + pub fn tokens_generated(&self) -> u32 { + self.tokens_generated + } + + pub fn prompt_tokens(&self) -> u32 { + self.prompt_tokens + } + + pub fn tps(&self) -> f64 { + self.tokens_per_sec + } + + pub fn prefill_ms(&self) -> Option { + self.prefill_ms + } + + fn detokenize_token(&self, token: llama_token) -> String { + if let Some(token_to_piece) = self.lib.token_to_piece { + return decode_token_piece_with(|buf, len| unsafe { + token_to_piece(self.vocab, token, buf, len, 0, true) + }); + } + + let text_ptr = unsafe { (self.lib.vocab_get_text)(self.vocab, token) }; + if text_ptr.is_null() { + String::new() + } else { + unsafe { CStr::from_ptr(text_ptr) } + .to_string_lossy() + .into_owned() + } + } +} + +fn decode_token_piece_with(mut call: F) -> String +where + F: FnMut(*mut std::ffi::c_char, i32) -> i32, +{ + let mut capacity = 64usize; + + loop { + let mut buffer = vec![0u8; capacity]; + let written = call( + buffer.as_mut_ptr() as *mut std::ffi::c_char, + capacity as i32, + ); + + if written < 0 { + capacity = (-written) as usize; + continue; + } + + let written = written as usize; + return String::from_utf8_lossy(&buffer[..written]).into_owned(); + } +} + +#[cfg(test)] +mod tests { + use super::decode_token_piece_with; + + #[test] + fn retries_when_piece_buffer_is_too_small() { + let mut calls = 0; + + let decoded = decode_token_piece_with(|buf, len| { + calls += 1; + + if len < 17 { + return -17; + } + + let bytes = b" Quantum computer"; + unsafe { + std::ptr::copy_nonoverlapping(bytes.as_ptr(), buf as *mut u8, bytes.len()); + } + bytes.len() as i32 + }); + + assert_eq!(decoded, " Quantum computer"); + assert_eq!(calls, 1); + } + + #[test] + fn decodes_piece_after_resizing_buffer() { + let mut calls = 0; + + let decoded = decode_token_piece_with(|buf, len| { + calls += 1; + + if calls == 1 { + return -18; + } + + assert!(len >= 18); + let bytes = b" Quantum computer"; + unsafe { + std::ptr::copy_nonoverlapping(bytes.as_ptr(), buf as *mut u8, bytes.len()); + } + bytes.len() as i32 + }); + + assert_eq!(decoded, " Quantum computer"); + assert_eq!(calls, 2); + } +} + +impl Drop for GenerationSession { + fn drop(&mut self) { + if !self.sampler.is_null() { + unsafe { (self.lib.sampler_free)(self.sampler) }; + self.sampler = std::ptr::null_mut(); + } + } +} diff --git a/crates/inference-server/Cargo.toml b/crates/inference-server/Cargo.toml new file mode 100644 index 0000000..4638fd8 --- /dev/null +++ b/crates/inference-server/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "inference-server" +version.workspace = true +edition.workspace = true +build = "build.rs" + +[[bin]] +name = "localllm" +path = "src/main.rs" + +[dependencies] +inference-core = { path = "../inference-core" } +api-contracts = { path = "../api-contracts" } +metrics = { path = "../metrics" } +tokio.workspace = true +axum.workspace = true +hyper.workspace = true +tower.workspace = true +tower-http.workspace = true +serde.workspace = true +serde_json.workspace = true +anyhow.workspace = true +thiserror.workspace = true +tracing.workspace = true +tracing-subscriber.workspace = true +futures.workspace = true +tokio-stream.workspace = true +bytes.workspace = true +uuid.workspace = true +async-stream = "0.3" diff --git a/crates/inference-server/build.rs b/crates/inference-server/build.rs new file mode 100644 index 0000000..b45a82a --- /dev/null +++ b/crates/inference-server/build.rs @@ -0,0 +1,38 @@ +fn main() { + let out_dir = std::env::var("OUT_DIR").unwrap(); + let target_dir = std::path::Path::new(&out_dir) + .ancestors() + .nth(3) // Volta 3 níveis de `target/debug/build/inference-server-XYZ/out` para `target/debug/` ou `target/release/` + .unwrap(); + + let vendor_dir = std::path::PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").unwrap()) + .parent() + .unwrap() + .parent() + .unwrap() + .join("vendor") + .join("llama-bin"); + + if vendor_dir.exists() { + println!( + "cargo:warning=Copying DLLs from {:?} to {:?}", + vendor_dir, target_dir + ); + + // Vamos copiar todos os arquivos .dll + if let Ok(entries) = std::fs::read_dir(&vendor_dir) { + for entry in entries.flatten() { + let path = entry.path(); + if path.is_file() && path.extension().and_then(|s| s.to_str()) == Some("dll") { + let file_name = path.file_name().unwrap(); + let dest = target_dir.join(file_name); + let _ = std::fs::copy(&path, &dest); + } + } + } + } + + // Rebuildar apenas se houver mudanças nessa build file ou nos bins da vendor + println!("cargo:rerun-if-changed=build.rs"); + println!("cargo:rerun-if-changed=../../vendor/llama-bin"); +} diff --git a/crates/inference-server/src/compat.rs b/crates/inference-server/src/compat.rs new file mode 100644 index 0000000..0adbc6d --- /dev/null +++ b/crates/inference-server/src/compat.rs @@ -0,0 +1,721 @@ +use std::collections::HashMap; + +use anyhow::{bail, Result}; +use api_contracts::{ + chat::{ + ChatCompletionRequest, DeltaToolCall, DeltaToolFunctionCall, FinishReason, ResponseFormat, + Role, ToolCall, ToolChoice, ToolDefinition, ToolFunctionCall, ToolType, + }, + error::ApiError, + models::ModelInfo, +}; +use axum::http::{HeaderMap, HeaderName, HeaderValue, StatusCode}; +use serde_json::{json, Value}; +use tower_http::cors::{AllowHeaders, AllowMethods, AllowOrigin, Any, CorsLayer}; + +const DEFAULT_SYSTEM_PROMPT: &str = concat!( + "You are a precise coding assistant running locally. ", + "Prioritize correct code, minimal speculation, explicit assumptions, and actionable fixes. ", + "When editing or generating code, prefer complete snippets, preserve requested APIs, and explain risks briefly." +); + +#[derive(Debug, Clone)] +pub struct ServerConfig { + pub api_key: Option, + pub auth_header: String, + pub auth_scheme: Option, + pub cors_origins: Vec, + pub default_system_prompt: String, + pub enable_tool_calls: bool, + pub model_catalog: ModelCatalog, +} + +impl ServerConfig { + pub fn from_env(model_name: String) -> Self { + let aliases = env_csv("LOCALLLM_MODEL_ALIASES"); + Self { + api_key: std::env::var("LOCALLLM_API_KEY") + .ok() + .filter(|v| !v.is_empty()), + auth_header: std::env::var("LOCALLLM_AUTH_HEADER") + .ok() + .filter(|v| !v.trim().is_empty()) + .unwrap_or_else(|| "authorization".to_owned()) + .to_ascii_lowercase(), + auth_scheme: std::env::var("LOCALLLM_AUTH_SCHEME") + .ok() + .map(|v| v.trim().to_owned()) + .filter(|v| !v.is_empty()), + cors_origins: { + let origins = env_csv("LOCALLLM_CORS_ORIGINS"); + if origins.is_empty() { + vec!["*".to_owned()] + } else { + origins + } + }, + default_system_prompt: std::env::var("LOCALLLM_SYSTEM_PROMPT") + .ok() + .filter(|v| !v.trim().is_empty()) + .unwrap_or_else(|| DEFAULT_SYSTEM_PROMPT.to_owned()), + enable_tool_calls: env_bool("LOCALLLM_ENABLE_TOOL_CALLS", true), + model_catalog: ModelCatalog::new(model_name, aliases), + } + } + + pub fn cors_layer(&self) -> CorsLayer { + let methods = AllowMethods::any(); + let headers = AllowHeaders::any(); + + if self.cors_origins.iter().any(|origin| origin == "*") { + return CorsLayer::new() + .allow_origin(Any) + .allow_methods(methods) + .allow_headers(headers); + } + + let origins: Vec = self + .cors_origins + .iter() + .filter_map(|origin| HeaderValue::from_str(origin).ok()) + .collect(); + + CorsLayer::new() + .allow_origin(AllowOrigin::list(origins)) + .allow_methods(methods) + .allow_headers(headers) + } + + pub fn resolve_model(&self, requested: &str) -> Option<&ModelDescriptor> { + self.model_catalog.resolve(requested) + } + + pub fn model_list(&self, created: i64) -> Vec { + self.model_catalog.to_model_list(created) + } +} + +#[derive(Debug, Clone)] +pub struct ModelCatalog { + models: Vec, + aliases: HashMap, +} + +impl ModelCatalog { + pub fn new(canonical_id: String, aliases: Vec) -> Self { + let mut models = vec![ModelDescriptor { + id: canonical_id.clone(), + root: canonical_id.clone(), + parent: None, + }]; + + let mut seen = std::collections::BTreeSet::new(); + seen.insert(canonical_id.to_ascii_lowercase()); + seen.insert("local-model".to_owned()); + seen.insert("default".to_owned()); + + for alias in [ + vec!["local-model".to_owned(), "default".to_owned()], + aliases, + ] + .concat() + { + let trimmed = alias.trim(); + if trimmed.is_empty() { + continue; + } + let lowered = trimmed.to_ascii_lowercase(); + if !seen.insert(lowered) { + continue; + } + models.push(ModelDescriptor { + id: trimmed.to_owned(), + root: canonical_id.clone(), + parent: Some(canonical_id.clone()), + }); + } + + let aliases = models + .iter() + .enumerate() + .map(|(idx, model)| (model.id.to_ascii_lowercase(), idx)) + .collect(); + + Self { models, aliases } + } + + pub fn resolve(&self, requested: &str) -> Option<&ModelDescriptor> { + self.aliases + .get(&requested.to_ascii_lowercase()) + .and_then(|idx| self.models.get(*idx)) + } + pub fn to_model_list(&self, created: i64) -> Vec { + self.models + .iter() + .map(|model| ModelInfo { + id: model.id.clone(), + object: "model", + created, + owned_by: "local", + root: model.root.clone(), + parent: model.parent.clone(), + permission: Vec::new(), + }) + .collect() + } +} + +#[derive(Debug, Clone)] +pub struct ModelDescriptor { + pub id: String, + pub root: String, + pub parent: Option, +} + +#[derive(Debug, Clone)] +pub struct PreparedPrompt { + pub prompt: String, + pub streaming_mode: StreamingMode, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum StreamingMode { + LiveTokens, + Buffered, +} + +#[derive(Debug, Clone)] +pub struct FinalizedAssistant { + pub content: Option, + pub tool_calls: Option>, + pub finish_reason: FinishReason, +} + +pub fn authorize_request( + headers: &HeaderMap, + config: &ServerConfig, +) -> Result<(), (StatusCode, ApiError)> { + let Some(expected_key) = config.api_key.as_deref() else { + return Ok(()); + }; + + let configured_header = HeaderName::from_bytes(config.auth_header.as_bytes()) + .ok() + .and_then(|name| headers.get(name)) + .and_then(|value| value.to_str().ok()) + .map(str::trim) + .filter(|value| !value.is_empty()); + + let fallback_bearer = headers + .get(axum::http::header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()) + .map(str::trim) + .filter(|value| !value.is_empty()); + + let fallback_api_key = headers + .get("x-api-key") + .and_then(|value| value.to_str().ok()) + .map(str::trim) + .filter(|value| !value.is_empty()); + + let provided = configured_header.or(fallback_bearer).or(fallback_api_key); + let Some(provided) = provided else { + return Err(( + StatusCode::UNAUTHORIZED, + ApiError::new("Missing API key", "invalid_request_error").with_code("missing_api_key"), + )); + }; + + let normalized = match config.auth_scheme.as_deref() { + Some(scheme) => provided + .strip_prefix(scheme) + .map(str::trim) + .unwrap_or(provided), + None => provided, + }; + + if normalized == expected_key || provided == expected_key { + Ok(()) + } else { + Err(( + StatusCode::UNAUTHORIZED, + ApiError::new("Invalid API key", "invalid_request_error").with_code("invalid_api_key"), + )) + } +} + +pub fn validate_request( + req: &ChatCompletionRequest, + config: &ServerConfig, +) -> Result<(), (StatusCode, ApiError)> { + if config.resolve_model(&req.model).is_none() { + return Err(( + StatusCode::NOT_FOUND, + ApiError::new( + format!("The model '{}' does not exist", req.model), + "invalid_request_error", + ) + .with_code("model_not_found") + .with_param("model"), + )); + } + + if req.messages.is_empty() { + return Err(( + StatusCode::BAD_REQUEST, + ApiError::new("messages must not be empty", "invalid_request_error") + .with_code("empty_messages") + .with_param("messages"), + )); + } + + if req.n != 1 { + return Err(( + StatusCode::BAD_REQUEST, + ApiError::new("Only n=1 is currently supported", "invalid_request_error") + .with_code("unsupported_value") + .with_param("n"), + )); + } + + if !req.tools.is_empty() && !config.enable_tool_calls { + return Err(( + StatusCode::BAD_REQUEST, + ApiError::new( + "Tool calling is disabled on this server", + "invalid_request_error", + ) + .with_code("tool_calls_disabled") + .with_param("tools"), + )); + } + + Ok(()) +} + +pub fn prepare_prompt(req: &ChatCompletionRequest, config: &ServerConfig) -> PreparedPrompt { + let mut prompt = String::with_capacity(1024); + let has_system = req + .messages + .iter() + .any(|message| matches!(message.role, Role::System | Role::Developer)); + + if !has_system { + prompt.push_str("System: "); + prompt.push_str(&config.default_system_prompt); + prompt.push('\n'); + } + + if let Some(format) = &req.response_format { + append_response_format_instruction(&mut prompt, format); + } + + if !req.tools.is_empty() { + append_tool_instructions(&mut prompt, &req.tools, req.tool_choice.as_ref()); + } + + for msg in &req.messages { + let role = match msg.role { + Role::System => "System", + Role::Developer => "Developer", + Role::User => "User", + Role::Assistant => "Assistant", + Role::Tool => "Tool", + }; + prompt.push_str(role); + if let Some(name) = &msg.name { + prompt.push_str("("); + prompt.push_str(name); + prompt.push(')'); + } + prompt.push_str(": "); + + if let Some(tool_calls) = &msg.tool_calls { + let tool_json = serde_json::to_string(tool_calls).unwrap_or_else(|_| "[]".to_owned()); + prompt.push_str(&tool_json); + } else { + prompt.push_str(msg.text()); + } + + if let Some(tool_call_id) = &msg.tool_call_id { + prompt.push_str(" [tool_call_id="); + prompt.push_str(tool_call_id); + prompt.push(']'); + } + + prompt.push('\n'); + } + + prompt.push_str("Assistant:"); + + PreparedPrompt { + prompt, + streaming_mode: if req.tools.is_empty() { + StreamingMode::LiveTokens + } else { + StreamingMode::Buffered + }, + } +} + +pub fn finalize_assistant_message( + raw_output: String, + req: &ChatCompletionRequest, +) -> FinalizedAssistant { + if !req.tools.is_empty() { + if let Some(tool_calls) = parse_tool_calls(&raw_output) { + return FinalizedAssistant { + content: None, + tool_calls: Some(tool_calls), + finish_reason: FinishReason::ToolCalls, + }; + } + } + + FinalizedAssistant { + content: Some(raw_output.trim().to_owned()), + tool_calls: None, + finish_reason: FinishReason::Stop, + } +} + +pub fn tool_calls_to_delta(tool_calls: &[ToolCall]) -> Vec { + tool_calls + .iter() + .enumerate() + .map(|(index, tool_call)| DeltaToolCall { + index: index as u32, + id: Some(tool_call.id.clone()), + tool_type: Some(tool_call.tool_type.clone()), + function: Some(DeltaToolFunctionCall { + name: Some(tool_call.function.name.clone()), + arguments: Some(tool_call.function.arguments.clone()), + }), + }) + .collect() +} + +fn append_response_format_instruction(prompt: &mut String, format: &ResponseFormat) { + match format.format_type.as_str() { + "json_object" => { + prompt.push_str("System: Return valid JSON only. Do not wrap it in markdown fences.\n"); + } + "json_schema" => { + prompt.push_str("System: Return valid JSON only and follow this schema exactly: "); + if let Some(schema) = &format.json_schema { + prompt.push_str(&schema.to_string()); + } + prompt.push('\n'); + } + _ => {} + } +} + +fn append_tool_instructions( + prompt: &mut String, + tools: &[ToolDefinition], + tool_choice: Option<&ToolChoice>, +) { + let tool_summaries: Vec = tools + .iter() + .map(|tool| { + json!({ + "name": tool.function.name, + "description": tool.function.description, + "parameters": tool.function.parameters, + }) + }) + .collect(); + + prompt.push_str("System: Available tools: "); + prompt.push_str(&serde_json::to_string(&tool_summaries).unwrap_or_else(|_| "[]".to_owned())); + prompt.push('\n'); + prompt + .push_str("System: If you need a tool, respond with JSON only using one of these shapes: "); + prompt.push_str(r#"{"tool_calls":[{"name":"tool_name","arguments":{}}]}"#); + prompt.push_str(" or "); + prompt.push_str(r#"{"name":"tool_name","arguments":{}}"#); + prompt.push_str(". If no tool is needed, answer normally.\n"); + + match tool_choice { + Some(ToolChoice::Mode(mode)) if mode == "required" => { + prompt.push_str("System: You must call a tool before replying.\n"); + } + Some(ToolChoice::Mode(mode)) if mode == "none" => { + prompt.push_str("System: Do not call tools in this response.\n"); + } + Some(ToolChoice::Specific(choice)) => { + prompt.push_str("System: You must call the tool named "); + prompt.push_str(&choice.function.name); + prompt.push_str(".\n"); + } + _ => {} + } +} + +fn parse_tool_calls(raw_output: &str) -> Option> { + let trimmed = raw_output.trim(); + if trimmed.is_empty() { + return None; + } + + let value: Value = serde_json::from_str(trimmed).ok()?; + match value { + Value::Object(map) => { + if let Some(tool_calls) = map.get("tool_calls").and_then(Value::as_array) { + let parsed: Vec = tool_calls + .iter() + .enumerate() + .filter_map(|(index, item)| parse_tool_call(item, index)) + .collect(); + if parsed.is_empty() { + None + } else { + Some(parsed) + } + } else { + parse_tool_call(&Value::Object(map), 0).map(|tool_call| vec![tool_call]) + } + } + _ => None, + } +} + +fn parse_tool_call(value: &Value, index: usize) -> Option { + let object = value.as_object()?; + let function = object.get("function").unwrap_or(value); + let function_obj = function.as_object()?; + let name = function_obj.get("name")?.as_str()?.trim(); + if name.is_empty() { + return None; + } + + let arguments = function_obj + .get("arguments") + .cloned() + .unwrap_or_else(|| Value::Object(Default::default())); + let arguments = match arguments { + Value::String(s) => s, + other => serde_json::to_string(&other).ok()?, + }; + + Some(ToolCall { + id: format!("call_{}", index + 1), + tool_type: ToolType::Function, + function: ToolFunctionCall { + name: name.to_owned(), + arguments, + }, + }) +} + +pub fn build_sampler(req: &ChatCompletionRequest) -> inference_core::SamplerConfig { + let mut sampler = inference_core::SamplerConfig::default(); + sampler.temperature = req.temperature.clamp(0.0, 2.0); + sampler.top_p = req.top_p.clamp(0.0, 1.0); + sampler.max_tokens = req.max_tokens.max(1); + sampler.stop_sequences = req.stop.clone(); + sampler.repeat_penalty = if req.tools.is_empty() { 1.05 } else { 1.0 }; + sampler.seed = req.seed.unwrap_or(0); + sampler +} + +fn env_csv(key: &str) -> Vec { + std::env::var(key) + .ok() + .map(|value| { + value + .split(',') + .map(str::trim) + .filter(|part| !part.is_empty()) + .map(ToOwned::to_owned) + .collect() + }) + .unwrap_or_default() +} + +fn env_bool(key: &str, default: bool) -> bool { + std::env::var(key) + .ok() + .and_then(|value| match value.trim().to_ascii_lowercase().as_str() { + "1" | "true" | "yes" | "on" => Some(true), + "0" | "false" | "no" | "off" => Some(false), + _ => None, + }) + .unwrap_or(default) +} + +pub fn ensure_supported_mode(req: &ChatCompletionRequest) -> Result<()> { + if matches!(req.tool_choice, Some(ToolChoice::Mode(ref mode)) if mode == "none") { + return Ok(()); + } + + if let Some(format) = &req.response_format { + if !matches!( + format.format_type.as_str(), + "json_object" | "json_schema" | "text" + ) { + bail!("unsupported response_format type: {}", format.format_type); + } + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::{ + authorize_request, finalize_assistant_message, prepare_prompt, validate_request, + FinalizedAssistant, ModelCatalog, ServerConfig, StreamingMode, + }; + use api_contracts::chat::ChatCompletionRequest; + use axum::http::{HeaderMap, HeaderValue, StatusCode}; + + fn base_config() -> ServerConfig { + ServerConfig { + api_key: None, + auth_header: "authorization".to_owned(), + auth_scheme: Some("Bearer".to_owned()), + cors_origins: vec!["*".to_owned()], + default_system_prompt: "Default coding system prompt".to_owned(), + enable_tool_calls: true, + model_catalog: ModelCatalog::new( + "real-model".to_owned(), + vec!["alias-model".to_owned()], + ), + } + } + + fn parse_request(json: &str) -> ChatCompletionRequest { + serde_json::from_str(json).unwrap() + } + + #[test] + fn resolves_model_aliases_case_insensitively() { + let catalog = ModelCatalog::new("real-model".to_owned(), vec!["Alias-Model".to_owned()]); + + assert_eq!(catalog.resolve("alias-model").unwrap().root, "real-model"); + assert_eq!(catalog.resolve("REAL-MODEL").unwrap().id, "real-model"); + assert!(catalog.resolve("missing").is_none()); + } + + #[test] + fn validates_model_and_n() { + let config = base_config(); + let request = parse_request( + r#"{ + "model": "missing-model", + "messages": [{"role":"user","content":"hi"}] + }"#, + ); + + let err = validate_request(&request, &config).unwrap_err(); + assert_eq!(err.0, StatusCode::NOT_FOUND); + + let request = parse_request( + r#"{ + "model": "alias-model", + "n": 2, + "messages": [{"role":"user","content":"hi"}] + }"#, + ); + let err = validate_request(&request, &config).unwrap_err(); + assert_eq!(err.0, StatusCode::BAD_REQUEST); + } + + #[test] + fn authorizes_bearer_and_x_api_key_headers() { + let mut config = base_config(); + config.api_key = Some("secret".to_owned()); + + let mut headers = HeaderMap::new(); + headers.insert("authorization", HeaderValue::from_static("Bearer secret")); + assert!(authorize_request(&headers, &config).is_ok()); + + let mut headers = HeaderMap::new(); + headers.insert("x-api-key", HeaderValue::from_static("secret")); + assert!(authorize_request(&headers, &config).is_ok()); + } + + #[test] + fn adds_default_system_prompt_and_tool_instructions() { + let config = base_config(); + let request = parse_request( + r#"{ + "model": "alias-model", + "messages": [{"role":"user","content":"List files"}], + "tools": [{ + "type": "function", + "function": { + "name": "list_files", + "description": "List project files", + "parameters": {"type": "object"} + } + }], + "tool_choice": "required" + }"#, + ); + + let prepared = prepare_prompt(&request, &config); + assert_eq!(prepared.streaming_mode, StreamingMode::Buffered); + assert!(prepared.prompt.contains("Default coding system prompt")); + assert!(prepared.prompt.contains("list_files")); + assert!(prepared.prompt.contains("must call a tool")); + } + + #[test] + fn extracts_tool_calls_from_model_output() { + let request = parse_request( + r#"{ + "model": "alias-model", + "messages": [{"role":"user","content":"List files"}], + "tools": [{ + "type": "function", + "function": { + "name": "list_files", + "parameters": {"type": "object"} + } + }] + }"#, + ); + + let finalized = finalize_assistant_message( + r#"{"tool_calls":[{"name":"list_files","arguments":{"path":"src"}}]}"#.to_owned(), + &request, + ); + + assert!(matches!( + finalized.finish_reason, + api_contracts::chat::FinishReason::ToolCalls + )); + assert_eq!(finalized.content, None); + let tool_calls = finalized.tool_calls.unwrap(); + assert_eq!(tool_calls[0].function.name, "list_files"); + assert_eq!(tool_calls[0].function.arguments, "{\"path\":\"src\"}"); + } + + #[test] + fn preserves_plain_text_when_no_tool_call_is_present() { + let request = parse_request( + r#"{ + "model": "alias-model", + "messages": [{"role":"user","content":"Explain borrow checker"}] + }"#, + ); + + let finalized = + finalize_assistant_message(" Use references and lifetimes. ".to_owned(), &request); + assert!(matches!( + finalized, + FinalizedAssistant { + content: Some(_), + tool_calls: None, + .. + } + )); + assert_eq!( + finalized.content.as_deref(), + Some("Use references and lifetimes.") + ); + } +} diff --git a/crates/inference-server/src/main.rs b/crates/inference-server/src/main.rs new file mode 100644 index 0000000..8b56ba8 --- /dev/null +++ b/crates/inference-server/src/main.rs @@ -0,0 +1,101 @@ +use std::net::SocketAddr; +use std::path::PathBuf; +use std::sync::Arc; + +use anyhow::Result; +use tokio::sync::Mutex; +use tracing::info; +use tracing_subscriber::{fmt, EnvFilter}; + +use inference_core::{EngineConfig, InferenceEngine, RuntimeProfile}; +use metrics::counters::PerfCounters; + +mod compat; +mod routes; +use compat::ServerConfig; +use routes::{build_router, AppState}; + +#[tokio::main] +async fn main() -> Result<()> { + // ── Tracing ────────────────────────────────────────────────────────── + fmt() + .with_env_filter( + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")), + ) + .with_target(false) + .init(); + + // ── Engine config (reads env vars; sane defaults for dev) ──────────── + let model_path = std::env::var("LOCALLLM_MODEL") + .map(PathBuf::from) + .unwrap_or_default(); + + let profile = match std::env::var("LOCALLLM_PROFILE").as_deref() { + Ok("balanced_9b") => RuntimeProfile::Balanced9B, + Ok("cpu_fallback") => RuntimeProfile::CpuFallback, + _ => RuntimeProfile::Turbo4B, + }; + + let defaults = profile.defaults(); + let config = EngineConfig { + model_path: model_path.clone(), + profile, + n_gpu_layers: env_i32("LOCALLLM_GPU_LAYERS", defaults.n_gpu_layers), + n_ctx: env_u32("LOCALLLM_CTX", defaults.n_ctx), + n_batch: env_u32("LOCALLLM_BATCH", defaults.n_batch), + n_ubatch: env_u32("LOCALLLM_UBATCH", defaults.n_ubatch), + n_threads: env_i32("LOCALLLM_THREADS", defaults.n_threads), + n_threads_batch: env_i32("LOCALLLM_THREADS_BATCH", defaults.n_threads), + flash_attn: defaults.flash_attn, + use_mmap: false, + seed: 0, + }; + + // ── Model name for API responses ───────────────────────────────────── + let model_name = model_path + .file_stem() + .map(|s| s.to_string_lossy().into_owned()) + .unwrap_or_else(|| "local-model".into()); + let server_config = Arc::new(ServerConfig::from_env(model_name.clone())); + + // ── Load engine ────────────────────────────────────────────────────── + let mut engine = InferenceEngine::load(config)?; + engine.warmup()?; + + let state = AppState { + engine: Arc::new(Mutex::new(engine)), + counters: PerfCounters::new(), + model_name: Arc::new(model_name), + server_config, + start_time: std::time::Instant::now(), + }; + + // ── Server ─────────────────────────────────────────────────────────── + let addr: SocketAddr = std::env::var("LOCALLLM_ADDR") + .unwrap_or_else(|_| "0.0.0.0:8080".into()) + .parse()?; + + let router = build_router(state); + let listener = tokio::net::TcpListener::bind(addr).await?; + + info!(addr = %addr, "LocalLLM server listening"); + axum::serve(listener, router).await?; + + Ok(()) +} + +// ── Env helpers ────────────────────────────────────────────────────────── + +fn env_i32(key: &str, default: i32) -> i32 { + std::env::var(key) + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(default) +} + +fn env_u32(key: &str, default: u32) -> u32 { + std::env::var(key) + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(default) +} diff --git a/crates/inference-server/src/routes.rs b/crates/inference-server/src/routes.rs new file mode 100644 index 0000000..cc82797 --- /dev/null +++ b/crates/inference-server/src/routes.rs @@ -0,0 +1,420 @@ +use axum::{ + extract::{Json, State}, + http::{HeaderMap, StatusCode}, + response::{IntoResponse, Response, Sse}, + routing::{get, post}, + Router, +}; +use std::{sync::Arc, time::SystemTime}; +use tokio::sync::Mutex; + +use api_contracts::{ + chat::{ + ChatCompletionChunk, ChatCompletionRequest, ChatCompletionResponse, ChatMessage, Choice, + ChunkChoice, Delta, FinishReason, Role, Usage, + }, + error::{ApiError, ErrorResponse}, + health::HealthResponse, + models::ModelList, +}; +use inference_core::InferenceEngine; +use metrics::counters::PerfCounters; + +use crate::compat::{ + authorize_request, build_sampler, ensure_supported_mode, finalize_assistant_message, + prepare_prompt, tool_calls_to_delta, validate_request, FinalizedAssistant, ServerConfig, + StreamingMode, +}; + +#[derive(Clone)] +pub struct AppState { + pub engine: Arc>, + pub counters: Arc, + pub model_name: Arc, + pub server_config: Arc, + pub start_time: std::time::Instant, +} + +pub fn build_router(state: AppState) -> Router { + let cors = state.server_config.cors_layer(); + Router::new() + .route("/health", get(health_handler)) + .route("/v1/models", get(models_handler)) + .route("/v1/chat/completions", post(chat_completions_handler)) + .layer(cors) + .with_state(state) +} + +fn unix_now() -> i64 { + SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap_or_default() + .as_secs() as i64 +} + +fn error_response(status: StatusCode, error: ApiError) -> Response { + (status, Json(ErrorResponse { error })).into_response() +} + +async fn health_handler(State(state): State) -> impl IntoResponse { + Json(HealthResponse { + status: "ok", + model_loaded: true, + model_id: Some(state.model_name.as_ref().clone()), + uptime_secs: state.start_time.elapsed().as_secs(), + }) +} + +async fn models_handler(State(state): State, headers: HeaderMap) -> Response { + if let Err((status, error)) = authorize_request(&headers, &state.server_config) { + return error_response(status, error); + } + + Json(ModelList { + object: "list", + data: state.server_config.model_list(unix_now()), + }) + .into_response() +} + +async fn chat_completions_handler( + State(state): State, + headers: HeaderMap, + Json(req): Json, +) -> Response { + if let Err((status, error)) = authorize_request(&headers, &state.server_config) { + return error_response(status, error); + } + + if let Err((status, error)) = validate_request(&req, &state.server_config) { + return error_response(status, error); + } + + if let Err(err) = ensure_supported_mode(&req) { + return error_response( + StatusCode::BAD_REQUEST, + ApiError::new(err.to_string(), "invalid_request_error").with_code("unsupported_mode"), + ); + } + + let sampler = build_sampler(&req); + let prepared = prepare_prompt(&req, &state.server_config); + + if req.stream { + stream_response( + state, + req, + sampler, + prepared.prompt, + prepared.streaming_mode, + ) + .await + } else { + blocking_response(state, req, sampler, prepared.prompt).await + } +} + +async fn stream_response( + state: AppState, + req: ChatCompletionRequest, + sampler: inference_core::SamplerConfig, + prompt: String, + streaming_mode: StreamingMode, +) -> Response { + use axum::response::sse::{Event, KeepAlive}; + + let id = uuid::Uuid::new_v4().to_string(); + let created = unix_now(); + let resolved_model = state + .server_config + .resolve_model(&req.model) + .map(|model| model.root.clone()) + .unwrap_or_else(|| state.model_name.as_ref().clone()); + + let engine_arc = Arc::clone(&state.engine); + let counters_arc = Arc::clone(&state.counters); + + let stream = async_stream::stream! { + let first_chunk = ChatCompletionChunk { + id: id.clone(), + object: "chat.completion.chunk", + created, + model: resolved_model.clone(), + choices: vec![ChunkChoice { + index: 0, + delta: Delta { role: Some(Role::Assistant), content: None, tool_calls: None }, + finish_reason: None, + }], + usage: None, + }; + yield Ok::<_, std::convert::Infallible>(Event::default().data(serde_json::to_string(&first_chunk).unwrap())); + + match streaming_mode { + StreamingMode::LiveTokens => { + let (tx, mut rx) = tokio::sync::mpsc::channel::(256); + let prompt = prompt.clone(); + + tokio::task::spawn_blocking(move || { + let rt = tokio::runtime::Handle::current(); + let mut engine = rt.block_on(engine_arc.lock()); + let mut record = metrics::recorder::GenerationRecord::new(0); + + let mut session = match engine.new_session(&prompt, sampler) { + Ok(session) => session, + Err(err) => { + tracing::error!(err = %err, "Failed to create session"); + let _ = rt.block_on(tx.send(StreamItem::Error)); + return; + } + }; + + while let Some(token) = session.next_token() { + record.tick(); + counters_arc.inc_tokens(1); + let _ = rt.block_on(tx.send(StreamItem::Token(token))); + } + + let usage = Usage { + prompt_tokens: session.prompt_tokens(), + completion_tokens: record.completion_tokens, + total_tokens: session.prompt_tokens() + record.completion_tokens, + tokens_per_second: record.tokens_per_second(), + }; + counters_arc.inc_completed(); + let _ = rt.block_on(tx.send(StreamItem::Done { + finish_reason: FinishReason::Stop, + usage, + })); + }); + + while let Some(item) = rx.recv().await { + match item { + StreamItem::Token(token) => { + let chunk = ChatCompletionChunk { + id: id.clone(), + object: "chat.completion.chunk", + created, + model: resolved_model.clone(), + choices: vec![ChunkChoice { + index: 0, + delta: Delta { role: None, content: Some(token), tool_calls: None }, + finish_reason: None, + }], + usage: None, + }; + yield Ok(Event::default().data(serde_json::to_string(&chunk).unwrap())); + } + StreamItem::Done { finish_reason, usage } => { + let final_chunk = ChatCompletionChunk { + id: id.clone(), + object: "chat.completion.chunk", + created, + model: resolved_model.clone(), + choices: vec![ChunkChoice { + index: 0, + delta: Delta { role: None, content: None, tool_calls: None }, + finish_reason: Some(finish_reason), + }], + usage: req.stream_options.as_ref().filter(|options| options.include_usage).map(|_| usage), + }; + yield Ok(Event::default().data(serde_json::to_string(&final_chunk).unwrap())); + yield Ok(Event::default().data("[DONE]")); + break; + } + StreamItem::Error => { + let chunk = ChatCompletionChunk { + id: id.clone(), + object: "chat.completion.chunk", + created, + model: resolved_model.clone(), + choices: vec![ChunkChoice { + index: 0, + delta: Delta { role: None, content: Some("internal error".to_owned()), tool_calls: None }, + finish_reason: Some(FinishReason::Stop), + }], + usage: None, + }; + yield Ok(Event::default().data(serde_json::to_string(&chunk).unwrap())); + yield Ok(Event::default().data("[DONE]")); + break; + } + } + } + } + StreamingMode::Buffered => { + match generate_full_completion(engine_arc, counters_arc, prompt, sampler, req.clone()).await { + Ok(result) => { + if let Some(tool_calls) = result.assistant.tool_calls { + let chunk = ChatCompletionChunk { + id: id.clone(), + object: "chat.completion.chunk", + created, + model: resolved_model.clone(), + choices: vec![ChunkChoice { + index: 0, + delta: Delta { + role: None, + content: None, + tool_calls: Some(tool_calls_to_delta(&tool_calls)), + }, + finish_reason: None, + }], + usage: None, + }; + yield Ok(Event::default().data(serde_json::to_string(&chunk).unwrap())); + } else if let Some(content) = result.assistant.content { + let chunk = ChatCompletionChunk { + id: id.clone(), + object: "chat.completion.chunk", + created, + model: resolved_model.clone(), + choices: vec![ChunkChoice { + index: 0, + delta: Delta { role: None, content: Some(content), tool_calls: None }, + finish_reason: None, + }], + usage: None, + }; + yield Ok(Event::default().data(serde_json::to_string(&chunk).unwrap())); + } + + let final_chunk = ChatCompletionChunk { + id: id.clone(), + object: "chat.completion.chunk", + created, + model: resolved_model.clone(), + choices: vec![ChunkChoice { + index: 0, + delta: Delta { role: None, content: None, tool_calls: None }, + finish_reason: Some(result.assistant.finish_reason), + }], + usage: req.stream_options.as_ref().filter(|options| options.include_usage).map(|_| result.usage), + }; + yield Ok(Event::default().data(serde_json::to_string(&final_chunk).unwrap())); + yield Ok(Event::default().data("[DONE]")); + } + Err(response) => { + yield Ok(Event::default().data("[DONE]")); + let _ = response; + } + } + } + } + }; + + Sse::new(stream) + .keep_alive(KeepAlive::default()) + .into_response() +} + +async fn blocking_response( + state: AppState, + req: ChatCompletionRequest, + sampler: inference_core::SamplerConfig, + prompt: String, +) -> Response { + let id = uuid::Uuid::new_v4().to_string(); + let model = state + .server_config + .resolve_model(&req.model) + .map(|descriptor| descriptor.root.clone()) + .unwrap_or_else(|| state.model_name.as_ref().clone()); + + match generate_full_completion( + Arc::clone(&state.engine), + Arc::clone(&state.counters), + prompt, + sampler, + req, + ) + .await + { + Ok(result) => Json(ChatCompletionResponse { + id, + object: "chat.completion", + created: unix_now(), + model, + choices: vec![Choice { + index: 0, + message: ChatMessage { + role: Role::Assistant, + content: result.assistant.content, + name: None, + tool_calls: result.assistant.tool_calls, + tool_call_id: None, + }, + finish_reason: result.assistant.finish_reason, + }], + usage: result.usage, + }) + .into_response(), + Err(response) => response, + } +} + +async fn generate_full_completion( + engine_arc: Arc>, + counters_arc: Arc, + prompt: String, + sampler: inference_core::SamplerConfig, + req: ChatCompletionRequest, +) -> Result { + let result = tokio::task::spawn_blocking(move || { + let rt = tokio::runtime::Handle::current(); + let mut engine = rt.block_on(engine_arc.lock()); + let mut record = metrics::recorder::GenerationRecord::new(0); + let mut content = String::new(); + + let mut session = engine.new_session(&prompt, sampler)?; + + while let Some(tok) = session.next_token() { + record.tick(); + content.push_str(&tok); + } + + counters_arc.inc_tokens(record.completion_tokens as u64); + counters_arc.inc_completed(); + + let usage = Usage { + prompt_tokens: session.prompt_tokens(), + completion_tokens: record.completion_tokens, + total_tokens: session.prompt_tokens() + record.completion_tokens, + tokens_per_second: record.tokens_per_second(), + }; + let assistant = finalize_assistant_message(content, &req); + Ok::<_, anyhow::Error>((assistant, usage)) + }) + .await; + + match result { + Ok(Ok((assistant, usage))) => Ok(GenerationResult { assistant, usage }), + Ok(Err(err)) => { + tracing::error!(err = %err, "Generation failed"); + Err(error_response( + StatusCode::INTERNAL_SERVER_ERROR, + ApiError::new("Generation failed", "server_error").with_code("generation_failed"), + )) + } + Err(err) => { + tracing::error!(err = %err, "Spawn blocking panicked"); + Err(error_response( + StatusCode::INTERNAL_SERVER_ERROR, + ApiError::new("Internal server error", "server_error").with_code("internal_error"), + )) + } + } +} + +struct GenerationResult { + assistant: FinalizedAssistant, + usage: Usage, +} + +enum StreamItem { + Token(String), + Done { + finish_reason: FinishReason, + usage: Usage, + }, + Error, +} diff --git a/crates/llama-sys/Cargo.toml b/crates/llama-sys/Cargo.toml new file mode 100644 index 0000000..b0cc774 --- /dev/null +++ b/crates/llama-sys/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "llama-sys" +version.workspace = true +edition.workspace = true + +[dependencies] +anyhow.workspace = true +libloading.workspace = true +once_cell.workspace = true +tracing.workspace = true + +[build-dependencies] +# bindgen e cmake ficam para quando compilarmos do source (Phase 3+) +# bindgen = "0.72" +# cmake = "0.1" + +[features] +default = ["cuda"] +cuda = [] +metal = [] +cpu-only = [] diff --git a/crates/llama-sys/build.rs b/crates/llama-sys/build.rs new file mode 100644 index 0000000..07268a6 --- /dev/null +++ b/crates/llama-sys/build.rs @@ -0,0 +1,14 @@ +/// Build script for llama-sys. +/// +/// Phase 2: Dynamic loading via libloading — no .lib needed. +/// The real llama.dll path is communicated to the crate via the +/// LLAMA_DLL_PATH env var or a sensible default. +/// +/// Phase 3+ will optionally compile llama.cpp from source. + +fn main() { + // Nothing to link here — we use libloading at runtime. + // Rerun if the env var changes. + println!("cargo:rerun-if-env-changed=LLAMA_DLL_PATH"); + println!("cargo:rerun-if-changed=build.rs"); +} diff --git a/crates/llama-sys/src/lib.rs b/crates/llama-sys/src/lib.rs new file mode 100644 index 0000000..85dd255 --- /dev/null +++ b/crates/llama-sys/src/lib.rs @@ -0,0 +1,552 @@ +//! llama-sys: Dynamic FFI bindings to llama.cpp via libloading. +//! +//! Loads `llama.dll` (Windows) / `libllama.so` (Linux) / `libllama.dylib` (macOS) +//! at runtime from the path specified by `LLAMA_DLL_PATH` env var, or from a +//! set of well-known locations. +//! +//! This approach requires no `.lib` import library and no source compilation. + +#![allow(non_upper_case_globals)] +#![allow(non_camel_case_types)] +#![allow(non_snake_case)] +#![allow(dead_code)] +#![allow(clippy::all)] + +use std::ffi::{c_char, c_void}; +use std::path::{Path, PathBuf}; + +pub use libloading::Library; + +// ─── Primitive type aliases ──────────────────────────────────────────────── + +pub type llama_pos = i32; +pub type llama_token = i32; +pub type llama_seq_id = i32; + +// ─── Opaque struct handles ───────────────────────────────────────────────── + +#[repr(C)] +pub struct llama_model { + _private: [u8; 0], +} + +#[repr(C)] +pub struct llama_context { + _private: [u8; 0], +} + +#[repr(C)] +pub struct llama_vocab { + _private: [u8; 0], +} + +#[repr(C)] +pub struct llama_sampler { + _private: [u8; 0], +} + +// ─── Model params ────────────────────────────────────────────────────────── + +#[repr(C)] +pub struct llama_model_params { + pub devices: *mut c_void, + pub tensor_buft_overrides: *const c_void, + pub n_gpu_layers: i32, + pub split_mode: i32, + pub main_gpu: i32, + pub tensor_split: *const f32, + pub progress_callback: Option bool>, + pub progress_callback_user_data: *mut c_void, + pub kv_overrides: *const c_void, + pub vocab_only: bool, + pub use_mmap: bool, + pub use_direct_io: bool, + pub use_mlock: bool, + pub check_tensors: bool, + pub use_extra_bufts: bool, + pub no_host: bool, + pub no_alloc: bool, +} + +// ─── Context params ──────────────────────────────────────────────────────── + +#[repr(C)] +pub struct llama_context_params { + pub n_ctx: u32, + pub n_batch: u32, + pub n_ubatch: u32, + pub n_seq_max: u32, + pub n_threads: i32, + pub n_threads_batch: i32, + pub rope_scaling_type: i32, + pub pooling_type: i32, + pub attention_type: i32, + pub flash_attn_type: i32, + pub rope_freq_base: f32, + pub rope_freq_scale: f32, + pub yarn_ext_factor: f32, + pub yarn_attn_factor: f32, + pub yarn_beta_fast: f32, + pub yarn_beta_slow: f32, + pub yarn_orig_ctx: u32, + pub defrag_thold: f32, + pub cb_eval: Option bool>, + pub cb_eval_user_data: *mut c_void, + pub type_k: i32, + pub type_v: i32, + pub abort_callback: Option bool>, + pub abort_callback_data: *mut c_void, + pub embeddings: bool, + pub offload_kqv: bool, + pub no_perf: bool, + pub op_offload: bool, + pub swa_full: bool, + pub kv_unified: bool, + pub samplers: *mut c_void, + pub n_samplers: usize, +} + +// ─── Batch ───────────────────────────────────────────────────────────────── + +#[repr(C)] +pub struct llama_batch { + pub n_tokens: i32, + pub token: *mut llama_token, + pub embd: *mut f32, + pub pos: *mut llama_pos, + pub n_seq_id: *mut i32, + pub seq_id: *mut *mut llama_seq_id, + pub logits: *mut i8, +} + +// ─── Sampler chain params ────────────────────────────────────────────────── + +#[repr(C)] +pub struct llama_sampler_chain_params { + pub no_perf: bool, +} + +// ─── Flash attn type ─────────────────────────────────────────────────────── + +pub const LLAMA_FLASH_ATTN_TYPE_AUTO: i32 = -1; +pub const LLAMA_FLASH_ATTN_TYPE_DISABLED: i32 = 0; +pub const LLAMA_FLASH_ATTN_TYPE_ENABLED: i32 = 1; + +// ─── Dynamic loader ──────────────────────────────────────────────────────── + +/// Container for all resolved function pointers from llama.dll. +pub struct LlamaLib { + // Keep the library alive as long as this struct is alive. + _lib: Library, + + // ── Lifecycle ────────────────────────────────────────────────────────── + pub backend_init: unsafe extern "C" fn(), + pub backend_free: unsafe extern "C" fn(), + pub backend_load_all: unsafe extern "C" fn(), + + // ── Defaults ────────────────────────────────────────────────────────── + pub model_default_params: unsafe extern "C" fn() -> llama_model_params, + pub context_default_params: unsafe extern "C" fn() -> llama_context_params, + pub sampler_chain_default_params: unsafe extern "C" fn() -> llama_sampler_chain_params, + + // ── Model ───────────────────────────────────────────────────────────── + pub model_load_from_file: + unsafe extern "C" fn(*const c_char, llama_model_params) -> *mut llama_model, + pub model_free: unsafe extern "C" fn(*mut llama_model), + pub model_get_vocab: unsafe extern "C" fn(*const llama_model) -> *const llama_vocab, + pub model_n_ctx_train: unsafe extern "C" fn(*const llama_model) -> i32, + pub model_is_recurrent: unsafe extern "C" fn(*const llama_model) -> bool, + pub model_is_hybrid: unsafe extern "C" fn(*const llama_model) -> bool, + pub model_desc: unsafe extern "C" fn(*const llama_model, *mut c_char, usize) -> i32, + pub model_n_params: unsafe extern "C" fn(*const llama_model) -> u64, + pub model_size: unsafe extern "C" fn(*const llama_model) -> u64, + + // ── Context ─────────────────────────────────────────────────────────── + pub init_from_model: + unsafe extern "C" fn(*mut llama_model, llama_context_params) -> *mut llama_context, + pub free: unsafe extern "C" fn(*mut llama_context), + pub n_ctx: unsafe extern "C" fn(*const llama_context) -> u32, + + // ── Batch ───────────────────────────────────────────────────────────── + pub batch_get_one: unsafe extern "C" fn(*mut llama_token, i32) -> llama_batch, + pub batch_init: unsafe extern "C" fn(i32, i32, i32) -> llama_batch, + pub batch_free: unsafe extern "C" fn(llama_batch), + + // ── Decode ──────────────────────────────────────────────────────────── + pub decode: unsafe extern "C" fn(*mut llama_context, llama_batch) -> i32, + pub set_n_threads: unsafe extern "C" fn(*mut llama_context, i32, i32), + pub set_warmup: unsafe extern "C" fn(*mut llama_context, bool), + pub synchronize: unsafe extern "C" fn(*mut llama_context), + + // ── Logits ──────────────────────────────────────────────────────────── + pub get_logits_ith: unsafe extern "C" fn(*mut llama_context, i32) -> *mut f32, + + // ── Vocab / Tokenizer ───────────────────────────────────────────────── + pub vocab_n_tokens: unsafe extern "C" fn(*const llama_vocab) -> i32, + pub vocab_is_eog: unsafe extern "C" fn(*const llama_vocab, llama_token) -> bool, + pub vocab_get_text: unsafe extern "C" fn(*const llama_vocab, llama_token) -> *const c_char, + pub token_to_piece: Option< + unsafe extern "C" fn(*const llama_vocab, llama_token, *mut c_char, i32, i32, bool) -> i32, + >, + pub tokenize: unsafe extern "C" fn( + *const llama_vocab, + *const c_char, + i32, + *mut llama_token, + i32, + bool, + bool, + ) -> i32, + pub token_bos: unsafe extern "C" fn(*const llama_vocab) -> llama_token, + pub token_eos: unsafe extern "C" fn(*const llama_vocab) -> llama_token, + + // ── Sampler ─────────────────────────────────────────────────────────── + pub sampler_chain_init: unsafe extern "C" fn(llama_sampler_chain_params) -> *mut llama_sampler, + pub sampler_chain_add: unsafe extern "C" fn(*mut llama_sampler, *mut llama_sampler), + pub sampler_free: unsafe extern "C" fn(*mut llama_sampler), + pub sampler_sample: + unsafe extern "C" fn(*mut llama_sampler, *mut llama_context, i32) -> llama_token, + pub sampler_init_greedy: unsafe extern "C" fn() -> *mut llama_sampler, + pub sampler_init_temp: unsafe extern "C" fn(f32) -> *mut llama_sampler, + pub sampler_init_top_p: unsafe extern "C" fn(f32, usize) -> *mut llama_sampler, + pub sampler_init_top_k: unsafe extern "C" fn(i32) -> *mut llama_sampler, + pub sampler_init_min_p: unsafe extern "C" fn(f32, usize) -> *mut llama_sampler, + pub sampler_init_dist: unsafe extern "C" fn(u32) -> *mut llama_sampler, + + // ── Misc ────────────────────────────────────────────────────────────── + pub supports_gpu_offload: unsafe extern "C" fn() -> bool, + pub time_us: unsafe extern "C" fn() -> i64, +} + +impl LlamaLib { + /// Load the llama shared library from the given path. + pub fn load(path: &Path) -> anyhow::Result { + use anyhow::Context; + + let lib = unsafe { Library::new(path) } + .with_context(|| format!("Failed to load llama library from {:?}", path))?; + + macro_rules! sym { + ($name:literal) => {{ + *unsafe { lib.get::(concat!($name, "\0").as_bytes()) } + .with_context(|| format!("Missing symbol: {}", $name))? + }}; + } + + // We need transmutes to specific fn types — use a helper macro: + macro_rules! load { + ($field:ident, $ty:ty, $name:literal) => {{ + let raw: libloading::Symbol = unsafe { + lib.get(concat!($name, "\0").as_bytes()) + .with_context(|| format!("Missing symbol: {}", $name))? + }; + unsafe { std::mem::transmute::<_, $ty>(*raw) } + }}; + } + + Ok(Self { + backend_init: load!(backend_init, unsafe extern "C" fn(), "llama_backend_init"), + backend_free: load!(backend_free, unsafe extern "C" fn(), "llama_backend_free"), + backend_load_all: { + let sym_name = b"ggml_backend_load_all\0"; + if let Ok(raw) = unsafe { lib.get::(sym_name) } { + unsafe { std::mem::transmute(*raw) } + } else { + let ggml_path = path.with_file_name("ggml.dll"); + if let Ok(ggml_lib) = unsafe { libloading::Library::new(&ggml_path) } { + // Faz a extração do raw *antes* de dar leak no Box, desfazendo ownership borrow issue do Rust + let raw_ptr = if let Ok(raw) = + unsafe { ggml_lib.get::(sym_name) } + { + Some(unsafe { std::mem::transmute::<_, unsafe extern "C" fn()>(*raw) }) + } else { + None + }; + + // Faz leak no handle do módulo pra manter vivo + std::mem::forget(ggml_lib); + + raw_ptr.unwrap_or_else(|| { + panic!("Missing symbol ggml_backend_load_all in both llama.dll and ggml.dll") + }) + } else { + anyhow::bail!("Missing symbol ggml_backend_load_all in llama.dll and could not load ggml.dll"); + } + } + }, + + model_default_params: load!( + model_default_params, + unsafe extern "C" fn() -> llama_model_params, + "llama_model_default_params" + ), + context_default_params: load!( + context_default_params, + unsafe extern "C" fn() -> llama_context_params, + "llama_context_default_params" + ), + sampler_chain_default_params: load!( + sampler_chain_default_params, + unsafe extern "C" fn() -> llama_sampler_chain_params, + "llama_sampler_chain_default_params" + ), + + model_load_from_file: load!( + model_load_from_file, + unsafe extern "C" fn(*const c_char, llama_model_params) -> *mut llama_model, + "llama_model_load_from_file" + ), + model_free: load!( + model_free, + unsafe extern "C" fn(*mut llama_model), + "llama_model_free" + ), + model_get_vocab: load!( + model_get_vocab, + unsafe extern "C" fn(*const llama_model) -> *const llama_vocab, + "llama_model_get_vocab" + ), + model_n_ctx_train: load!( + model_n_ctx_train, + unsafe extern "C" fn(*const llama_model) -> i32, + "llama_model_n_ctx_train" + ), + model_is_recurrent: load!( + model_is_recurrent, + unsafe extern "C" fn(*const llama_model) -> bool, + "llama_model_is_recurrent" + ), + model_is_hybrid: load!( + model_is_hybrid, + unsafe extern "C" fn(*const llama_model) -> bool, + "llama_model_is_hybrid" + ), + model_desc: load!( + model_desc, + unsafe extern "C" fn(*const llama_model, *mut c_char, usize) -> i32, + "llama_model_desc" + ), + model_n_params: load!( + model_n_params, + unsafe extern "C" fn(*const llama_model) -> u64, + "llama_model_n_params" + ), + model_size: load!( + model_size, + unsafe extern "C" fn(*const llama_model) -> u64, + "llama_model_size" + ), + + init_from_model: load!( + init_from_model, + unsafe extern "C" fn(*mut llama_model, llama_context_params) -> *mut llama_context, + "llama_init_from_model" + ), + free: load!(free, unsafe extern "C" fn(*mut llama_context), "llama_free"), + n_ctx: load!( + n_ctx, + unsafe extern "C" fn(*const llama_context) -> u32, + "llama_n_ctx" + ), + + batch_get_one: load!( + batch_get_one, + unsafe extern "C" fn(*mut llama_token, i32) -> llama_batch, + "llama_batch_get_one" + ), + batch_init: load!( + batch_init, + unsafe extern "C" fn(i32, i32, i32) -> llama_batch, + "llama_batch_init" + ), + batch_free: load!( + batch_free, + unsafe extern "C" fn(llama_batch), + "llama_batch_free" + ), + + decode: load!( + decode, + unsafe extern "C" fn(*mut llama_context, llama_batch) -> i32, + "llama_decode" + ), + set_n_threads: load!( + set_n_threads, + unsafe extern "C" fn(*mut llama_context, i32, i32), + "llama_set_n_threads" + ), + set_warmup: load!( + set_warmup, + unsafe extern "C" fn(*mut llama_context, bool), + "llama_set_warmup" + ), + synchronize: load!( + synchronize, + unsafe extern "C" fn(*mut llama_context), + "llama_synchronize" + ), + + get_logits_ith: load!( + get_logits_ith, + unsafe extern "C" fn(*mut llama_context, i32) -> *mut f32, + "llama_get_logits_ith" + ), + + vocab_n_tokens: load!( + vocab_n_tokens, + unsafe extern "C" fn(*const llama_vocab) -> i32, + "llama_vocab_n_tokens" + ), + vocab_is_eog: load!( + vocab_is_eog, + unsafe extern "C" fn(*const llama_vocab, llama_token) -> bool, + "llama_vocab_is_eog" + ), + vocab_get_text: load!( + vocab_get_text, + unsafe extern "C" fn(*const llama_vocab, llama_token) -> *const c_char, + "llama_vocab_get_text" + ), + token_to_piece: unsafe { + lib.get:: i32>(b"llama_token_to_piece\0") + .ok() + .map(|raw| *raw) + }, + tokenize: load!( + tokenize, + unsafe extern "C" fn( + *const llama_vocab, + *const c_char, + i32, + *mut llama_token, + i32, + bool, + bool, + ) -> i32, + "llama_tokenize" + ), + token_bos: load!( + token_bos, + unsafe extern "C" fn(*const llama_vocab) -> llama_token, + "llama_vocab_bos" + ), + token_eos: load!( + token_eos, + unsafe extern "C" fn(*const llama_vocab) -> llama_token, + "llama_vocab_eos" + ), + + sampler_chain_init: load!( + sampler_chain_init, + unsafe extern "C" fn(llama_sampler_chain_params) -> *mut llama_sampler, + "llama_sampler_chain_init" + ), + sampler_chain_add: load!( + sampler_chain_add, + unsafe extern "C" fn(*mut llama_sampler, *mut llama_sampler), + "llama_sampler_chain_add" + ), + sampler_free: load!( + sampler_free, + unsafe extern "C" fn(*mut llama_sampler), + "llama_sampler_free" + ), + sampler_sample: load!( + sampler_sample, + unsafe extern "C" fn(*mut llama_sampler, *mut llama_context, i32) -> llama_token, + "llama_sampler_sample" + ), + sampler_init_greedy: load!( + sampler_init_greedy, + unsafe extern "C" fn() -> *mut llama_sampler, + "llama_sampler_init_greedy" + ), + sampler_init_temp: load!( + sampler_init_temp, + unsafe extern "C" fn(f32) -> *mut llama_sampler, + "llama_sampler_init_temp" + ), + sampler_init_top_p: load!( + sampler_init_top_p, + unsafe extern "C" fn(f32, usize) -> *mut llama_sampler, + "llama_sampler_init_top_p" + ), + sampler_init_top_k: load!( + sampler_init_top_k, + unsafe extern "C" fn(i32) -> *mut llama_sampler, + "llama_sampler_init_top_k" + ), + sampler_init_min_p: load!( + sampler_init_min_p, + unsafe extern "C" fn(f32, usize) -> *mut llama_sampler, + "llama_sampler_init_min_p" + ), + sampler_init_dist: load!( + sampler_init_dist, + unsafe extern "C" fn(u32) -> *mut llama_sampler, + "llama_sampler_init_dist" + ), + + supports_gpu_offload: load!( + supports_gpu_offload, + unsafe extern "C" fn() -> bool, + "llama_supports_gpu_offload" + ), + time_us: load!(time_us, unsafe extern "C" fn() -> i64, "llama_time_us"), + + _lib: lib, + }) + } + + /// Resolve the DLL path from env var or known locations. + pub fn find_dll() -> anyhow::Result { + // 1. Explicit env override + if let Ok(p) = std::env::var("LLAMA_DLL_PATH") { + let path = PathBuf::from(p); + if path.exists() { + return Ok(path); + } + } + + // 2. Load directly relying on the Windows Library loader searching the .exe dir + // since our build.rs now copies all DLLs to target/{debug,release}/ + #[cfg(target_os = "windows")] + { + return Ok(PathBuf::from("llama.dll")); + } + + // 3. Unix-like: rely on LD_LIBRARY_PATH / rpath + #[cfg(not(target_os = "windows"))] + { + for name in &["libllama.so", "libllama.dylib"] { + if let Ok(lib) = Library::new(name) { + drop(lib); + return Ok(PathBuf::from(name)); + } + } + } + + anyhow::bail!( + "Cannot find llama shared library. Set LLAMA_DLL_PATH to the full path of llama.dll" + ) + } + + /// Convenience: find + load in one call. + pub fn load_auto() -> anyhow::Result { + let path = Self::find_dll()?; + tracing::info!(path = %path.display(), "Loading llama library"); + Self::load(&path) + } +} + +// SAFETY: LlamaLib only holds a Library and function pointers. +// All calls must be externally synchronized (enforced by InferenceEngine). +unsafe impl Send for LlamaLib {} +unsafe impl Sync for LlamaLib {} diff --git a/crates/metrics/Cargo.toml b/crates/metrics/Cargo.toml new file mode 100644 index 0000000..8f7414f --- /dev/null +++ b/crates/metrics/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "metrics" +version.workspace = true +edition.workspace = true + +[dependencies] +serde.workspace = true +serde_json.workspace = true +tracing.workspace = true +chrono.workspace = true +tokio.workspace = true diff --git a/crates/metrics/src/counters.rs b/crates/metrics/src/counters.rs new file mode 100644 index 0000000..69b5eed --- /dev/null +++ b/crates/metrics/src/counters.rs @@ -0,0 +1,46 @@ +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; + +/// Global atomic performance counters. +#[derive(Debug, Default)] +pub struct PerfCounters { + pub tokens_generated: AtomicU64, + pub requests_completed: AtomicU64, + pub requests_failed: AtomicU64, + pub total_prompt_tokens: AtomicU64, +} + +impl PerfCounters { + pub fn new() -> Arc { + Arc::new(Self::default()) + } + + pub fn inc_tokens(&self, n: u64) { + self.tokens_generated.fetch_add(n, Ordering::Relaxed); + } + + pub fn inc_completed(&self) { + self.requests_completed.fetch_add(1, Ordering::Relaxed); + } + + pub fn inc_failed(&self) { + self.requests_failed.fetch_add(1, Ordering::Relaxed); + } + + pub fn snapshot(&self) -> CountersSnapshot { + CountersSnapshot { + tokens_generated: self.tokens_generated.load(Ordering::Relaxed), + requests_completed: self.requests_completed.load(Ordering::Relaxed), + requests_failed: self.requests_failed.load(Ordering::Relaxed), + total_prompt_tokens: self.total_prompt_tokens.load(Ordering::Relaxed), + } + } +} + +#[derive(Debug, Clone, serde::Serialize)] +pub struct CountersSnapshot { + pub tokens_generated: u64, + pub requests_completed: u64, + pub requests_failed: u64, + pub total_prompt_tokens: u64, +} diff --git a/crates/metrics/src/lib.rs b/crates/metrics/src/lib.rs new file mode 100644 index 0000000..a3cc23e --- /dev/null +++ b/crates/metrics/src/lib.rs @@ -0,0 +1,4 @@ +// metrics: performance counters and structured telemetry + +pub mod counters; +pub mod recorder; diff --git a/crates/metrics/src/recorder.rs b/crates/metrics/src/recorder.rs new file mode 100644 index 0000000..991db2e --- /dev/null +++ b/crates/metrics/src/recorder.rs @@ -0,0 +1,34 @@ +use std::time::Instant; + +/// Records a single generation run and computes TPS. +#[derive(Debug)] +pub struct GenerationRecord { + pub start: Instant, + pub prompt_tokens: u32, + pub completion_tokens: u32, +} + +impl GenerationRecord { + pub fn new(prompt_tokens: u32) -> Self { + Self { + start: Instant::now(), + prompt_tokens, + completion_tokens: 0, + } + } + + pub fn tick(&mut self) { + self.completion_tokens += 1; + } + + pub fn elapsed_secs(&self) -> f64 { + self.start.elapsed().as_secs_f64() + } + + pub fn tokens_per_second(&self) -> f64 { + if self.completion_tokens == 0 { + return 0.0; + } + self.completion_tokens as f64 / self.elapsed_secs() + } +} diff --git a/experiments/bench_chat_latency.py b/experiments/bench_chat_latency.py new file mode 100644 index 0000000..ceff1ce --- /dev/null +++ b/experiments/bench_chat_latency.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +import argparse +import json +import math +import statistics +import sys +import time +import urllib.error +import urllib.request + + +PAYLOAD = { + "model": "local-model", + "stream": False, + "temperature": 0.0, + "top_p": 1.0, + "max_completion_tokens": 48, + "messages": [ + { + "role": "user", + "content": "Write a compact Rust function that parses a comma-separated list of u32 values and returns a Vec while ignoring empty segments.", + } + ], +} + + +def percentile(sorted_values, pct): + if not sorted_values: + return 0.0 + if len(sorted_values) == 1: + return float(sorted_values[0]) + rank = (len(sorted_values) - 1) * pct + low = math.floor(rank) + high = math.ceil(rank) + if low == high: + return float(sorted_values[low]) + weight = rank - low + return float(sorted_values[low] * (1.0 - weight) + sorted_values[high] * weight) + + +def wait_for_health(base_url, timeout_s, server_log): + deadline = time.time() + timeout_s + last_error = "server did not become healthy" + while time.time() < deadline: + try: + with urllib.request.urlopen(base_url + "/health", timeout=2) as response: + if response.status == 200: + return + except Exception as exc: # noqa: BLE001 + last_error = str(exc) + time.sleep(0.5) + + print("[ERROR] health check failed: {}".format(last_error), file=sys.stderr) + if server_log: + try: + with open(server_log, "r", encoding="utf-8", errors="replace") as fh: + lines = fh.readlines()[-40:] + print("[ERROR] tail of server log:", file=sys.stderr) + for line in lines: + print(line.rstrip(), file=sys.stderr) + except OSError as exc: # noqa: BLE001 + print("[ERROR] failed to read server log: {}".format(exc), file=sys.stderr) + sys.exit(1) + + +def run_once(base_url): + body = json.dumps(PAYLOAD).encode("utf-8") + req = urllib.request.Request( + base_url + "/v1/chat/completions", + data=body, + headers={"content-type": "application/json"}, + method="POST", + ) + started = time.perf_counter() + with urllib.request.urlopen(req, timeout=180) as response: + payload = json.loads(response.read().decode("utf-8")) + elapsed_ms = (time.perf_counter() - started) * 1000.0 + usage = payload.get("usage") or {} + return elapsed_ms, float(usage.get("tokens_per_second") or 0.0) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--base-url", required=True) + parser.add_argument("--server-log", default="") + parser.add_argument("--warmups", type=int, default=1) + parser.add_argument("--samples", type=int, default=7) + parser.add_argument("--health-timeout", type=float, default=120.0) + args = parser.parse_args() + + wait_for_health(args.base_url, args.health_timeout, args.server_log) + + for _ in range(args.warmups): + run_once(args.base_url) + + latencies = [] + tps_values = [] + for _ in range(args.samples): + latency_ms, tps = run_once(args.base_url) + latencies.append(latency_ms) + tps_values.append(tps) + + latencies_sorted = sorted(latencies) + p50_ms = percentile(latencies_sorted, 0.50) + p95_ms = percentile(latencies_sorted, 0.95) + mean_ms = statistics.fmean(latencies) + completion_tps_mean = statistics.fmean(tps_values) if tps_values else 0.0 + + print("METRIC p95_ms={:.3f}".format(p95_ms)) + print("METRIC p50_ms={:.3f}".format(p50_ms)) + print("METRIC mean_ms={:.3f}".format(mean_ms)) + print("METRIC completion_tps_mean={:.3f}".format(completion_tps_mean)) + print("METRIC success_count={}".format(len(latencies))) + + +if __name__ == "__main__": + main() diff --git a/experiments/worklog.md b/experiments/worklog.md new file mode 100644 index 0000000..0f3a865 --- /dev/null +++ b/experiments/worklog.md @@ -0,0 +1,13 @@ +# Autoresearch Worklog: latency-p95-chat-completions + +## Session Summary +- Objective: reduce end-to-end p95 latency for local `POST /v1/chat/completions` +- Primary metric: `p95_ms` (lower is better) +- Secondary metrics: `p50_ms`, `mean_ms`, `completion_tps_mean`, `success_count` +- Benchmark harness: `./autoresearch.sh` + +## Key Insights +- No experiment runs logged yet. + +## Next Ideas +- Measure baseline first, then inspect the hottest latency contributors in prompt preparation, request handling, and session creation. diff --git a/logs/logs_00001.txt b/logs/logs_00001.txt new file mode 100644 index 0000000..b0a2d95 --- /dev/null +++ b/logs/logs_00001.txt @@ -0,0 +1,4172 @@ +PS D:\Downloads\ABDM\Compressed\llama-b8533-bin-win-cuda-12.4-x64> +PS D:\Downloads\ABDM\Compressed\llama-b8533-bin-win-cuda-12.4-x64> & "D:\Downloads\ABDM\Compressed\llama-b8533-bin-win-cuda-12.4-x64\llama-cli.exe" ` +>> -m "D:\Documents\Github\ai-backend-studio\models\Qwen3.5-4B-Abliterated-Claude-4.6-Opus-Reasoning-Distilled.i1-Q6_K.gguf" ` +>> -p "Olá! Me diga quem você é em uma frase." ` +>> -n 80 ` +>> --n-gpu-layers 999 ` +>> --flash-attn on ` +>> --ctx-size 4096 ` +>> --no-display-prompt ` +>> -v +load_backend: loaded RPC backend from D:\Downloads\ABDM\Compressed\llama-b8533-bin-win-cuda-12.4-x64\ggml-rpc.dll +load_backend: loaded CPU backend from D:\Downloads\ABDM\Compressed\llama-b8533-bin-win-cuda-12.4-x64\ggml-cpu-haswell.dll +build: 8533 (0fac87b15) with Clang 19.1.5 for Windows x86_64 + +Loading model... |srv load_model: loading model 'D:\Documents\Github\ai-backend-studio\models\Qwen3.5-4B-Abliterated-Claude-4.6-Opus-Reasoning-Distilled.i1-Q6_K.gguf' +common_init_result: fitting params to device memory, for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on +llama_params_fit_impl: getting device memory data for initial parameters: +llama_model_loader: loaded meta data with 53 key-value pairs and 426 tensors from D:\Documents\Github\ai-backend-studio\models\Qwen3.5-4B-Abliterated-Claude-4.6-Opus-Reasoning-Distilled.i1-Q6_K.gguf (version GGUF V3 (latest)) +llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output. +llama_model_loader: - kv 0: general.architecture str = qwen35 +llama_model_loader: - kv 1: general.type str = model +llama_model_loader: - kv 2: general.name str = Qwen3.5-4B-Abliterated-Claude-4.6-Opu... +llama_model_loader: - kv 3: general.version str = 4.6 +llama_model_loader: - kv 4: general.finetune str = Abliterated-Claude-Opus-Reasoning-Dis... +llama_model_loader: - kv 5: general.basename str = Qwen3.5 +llama_model_loader: - kv 6: general.size_label str = 4B +llama_model_loader: - kv 7: general.license str = other +llama_model_loader: - kv 8: general.base_model.count u32 = 1 +llama_model_loader: - kv 9: general.base_model.0.name str = Qwen3.5 4B Claude 4.6 Opus Reasoning ... +llama_model_loader: - kv 10: general.base_model.0.version str = 4.6 +llama_model_loader: - kv 11: general.base_model.0.organization str = Jackrong +llama_model_loader: - kv 12: general.base_model.0.repo_url str = https://huggingface.co/Jackrong/Qwen3... +llama_model_loader: - kv 13: general.tags arr[str,5] = ["abliterated", "de-censored", "reaso... +llama_model_loader: - kv 14: qwen35.block_count u32 = 32 +llama_model_loader: - kv 15: qwen35.context_length u32 = 262144 +llama_model_loader: - kv 16: qwen35.embedding_length u32 = 2560 +llama_model_loader: - kv 17: qwen35.feed_forward_length u32 = 9216 +llama_model_loader: - kv 18: qwen35.attention.head_count u32 = 16 +llama_model_loader: - kv 19: qwen35.attention.head_count_kv u32 = 4 +llama_model_loader: - kv 20: qwen35.rope.dimension_sections arr[i32,4] = [11, 11, 10, 0] +llama_model_loader: - kv 21: qwen35.rope.freq_base f32 = 10000000.000000 +llama_model_loader: - kv 22: qwen35.attention.layer_norm_rms_epsilon f32 = 0.000001 +llama_model_loader: - kv 23: qwen35.attention.key_length u32 = 256 +llama_model_loader: - kv 24: qwen35.attention.value_length u32 = 256 +llama_model_loader: - kv 25: qwen35.ssm.conv_kernel u32 = 4 +llama_model_loader: - kv 26: qwen35.ssm.state_size u32 = 128 +llama_model_loader: - kv 27: qwen35.ssm.group_count u32 = 16 +llama_model_loader: - kv 28: qwen35.ssm.time_step_rank u32 = 32 +llama_model_loader: - kv 29: qwen35.ssm.inner_size u32 = 4096 +llama_model_loader: - kv 30: qwen35.full_attention_interval u32 = 4 +llama_model_loader: - kv 31: qwen35.rope.dimension_count u32 = 64 +llama_model_loader: - kv 32: tokenizer.ggml.model str = gpt2 +llama_model_loader: - kv 33: tokenizer.ggml.pre str = qwen35 +llama_model_loader: - kv 34: tokenizer.ggml.tokens arr[str,248320] = ["!", "\"", "#", "$", "%", "&", "'", ... +llama_model_loader: - kv 35: tokenizer.ggml.token_type arr[i32,248320] = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... -llama_model_loader: - kv 36: tokenizer.ggml.merges arr[str,247587] = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",... +llama_model_loader: - kv 37: tokenizer.ggml.eos_token_id u32 = 248046 +llama_model_loader: - kv 38: tokenizer.ggml.padding_token_id u32 = 248055 +llama_model_loader: - kv 39: tokenizer.chat_template str = {%- if tools %}\n {{- '<|im_start|>... +llama_model_loader: - kv 40: general.quantization_version u32 = 2 +llama_model_loader: - kv 41: general.file_type u32 = 18 +llama_model_loader: - kv 42: general.url str = https://huggingface.co/mradermacher/Q... +llama_model_loader: - kv 43: mradermacher.quantize_version str = 2 +llama_model_loader: - kv 44: mradermacher.quantized_by str = mradermacher +llama_model_loader: - kv 45: mradermacher.quantized_at str = 2026-03-10T09:59:38+01:00 +llama_model_loader: - kv 46: mradermacher.quantized_on str = nico1 +llama_model_loader: - kv 47: general.source.url str = https://huggingface.co/Abhiray/Qwen3.... +llama_model_loader: - kv 48: mradermacher.convert_type str = hf +llama_model_loader: - kv 49: quantize.imatrix.file str = Qwen3.5-4B-Abliterated-Claude-4.6-Opu... +llama_model_loader: - kv 50: quantize.imatrix.dataset str = imatrix-training-full-3 +llama_model_loader: - kv 51: quantize.imatrix.entries_count u32 = 248 +llama_model_loader: - kv 52: quantize.imatrix.chunks_count u32 = 319 +llama_model_loader: - type f32: 177 tensors +llama_model_loader: - type q6_K: 249 tensors +print_info: file format = GGUF V3 (latest) +print_info: file type = Q6_K +print_info: file size = 3.22 GiB (6.57 BPW) \init_tokenizer: initializing tokenizer for type 2 +load: 0 unused tokens +load: control token: 248062 '<|fim_suffix|>' is not marked as EOG +load: control token: 248072 '' is not marked as EOG +load: control token: 248050 '<|box_end|>' is not marked as EOG +load: control token: 248048 '<|object_ref_end|>' is not marked as EOG +load: control token: 248055 '<|vision_pad|>' is not marked as EOG +load: control token: 248060 '<|fim_prefix|>' is not marked as EOG +load: control token: 248049 '<|box_start|>' is not marked as EOG +load: control token: 248073 '' is not marked as EOG +load: control token: 248052 '<|quad_end|>' is not marked as EOG +load: control token: 248054 '<|vision_end|>' is not marked as EOG +load: control token: 248075 '' is not marked as EOG +load: control token: 248056 '<|image_pad|>' is not marked as EOG +load: control token: 248071 '<|audio_end|>' is not marked as EOG +load: control token: 248045 '<|im_start|>' is not marked as EOG +load: control token: 248047 '<|object_ref_start|>' is not marked as EOG +load: control token: 248051 '<|quad_start|>' is not marked as EOG +load: control token: 248053 '<|vision_start|>' is not marked as EOG +load: control token: 248057 '<|video_pad|>' is not marked as EOG +load: control token: 248061 '<|fim_middle|>' is not marked as EOG +load: control token: 248070 '<|audio_start|>' is not marked as EOG +load: control token: 248074 '' is not marked as EOG +load: control token: 248076 '<|audio_pad|>' is not marked as EOG |load: printing all EOG tokens: +load: - 248044 ('<|endoftext|>') +load: - 248046 ('<|im_end|>') +load: - 248063 ('<|fim_pad|>') +load: - 248064 ('<|repo_name|>') +load: - 248065 ('<|file_sep|>') +load: special tokens cache size = 33 +load: token to piece cache size = 1.7581 MB +print_info: arch = qwen35 +print_info: vocab_only = 0 +print_info: no_alloc = 1 +print_info: n_ctx_train = 262144 +print_info: n_embd = 2560 +print_info: n_embd_inp = 2560 +print_info: n_layer = 32 +print_info: n_head = 16 +print_info: n_head_kv = 4 +print_info: n_rot = 64 +print_info: n_swa = 0 +print_info: is_swa_any = 0 +print_info: n_embd_head_k = 256 +print_info: n_embd_head_v = 256 +print_info: n_gqa = 4 +print_info: n_embd_k_gqa = 1024 +print_info: n_embd_v_gqa = 1024 +print_info: f_norm_eps = 0.0e+00 +print_info: f_norm_rms_eps = 1.0e-06 +print_info: f_clamp_kqv = 0.0e+00 +print_info: f_max_alibi_bias = 0.0e+00 +print_info: f_logit_scale = 0.0e+00 +print_info: f_attn_scale = 0.0e+00 +print_info: n_ff = 9216 +print_info: n_expert = 0 +print_info: n_expert_used = 0 +print_info: n_expert_groups = 0 +print_info: n_group_used = 0 +print_info: causal attn = 1 +print_info: pooling type = -1 +print_info: rope type = 40 +print_info: rope scaling = linear +print_info: freq_base_train = 10000000.0 +print_info: freq_scale_train = 1 +print_info: n_ctx_orig_yarn = 262144 +print_info: rope_yarn_log_mul = 0.0000 +print_info: rope_finetuned = unknown +print_info: mrope sections = [11, 11, 10, 0] +print_info: ssm_d_conv = 4 +print_info: ssm_d_inner = 4096 +print_info: ssm_d_state = 128 +print_info: ssm_dt_rank = 32 +print_info: ssm_n_group = 16 +print_info: ssm_dt_b_c_rms = 0 +print_info: model type = 4B +print_info: model params = 4.21 B +print_info: general.name = Qwen3.5-4B-Abliterated-Claude-4.6-Opus-Reasoning-Distilled +print_info: vocab type = BPE +print_info: n_vocab = 248320 +print_info: n_merges = 247587 +print_info: BOS token = 11 ',' +print_info: EOS token = 248046 '<|im_end|>' +print_info: EOT token = 248046 '<|im_end|>' +print_info: PAD token = 248055 '<|vision_pad|>' +print_info: LF token = 198 'Ċ' +print_info: FIM PRE token = 248060 '<|fim_prefix|>' +print_info: FIM SUF token = 248062 '<|fim_suffix|>' +print_info: FIM MID token = 248061 '<|fim_middle|>' +print_info: FIM PAD token = 248063 '<|fim_pad|>' +print_info: FIM REP token = 248064 '<|repo_name|>' +print_info: FIM SEP token = 248065 '<|file_sep|>' +print_info: EOG token = 248044 '<|endoftext|>' +print_info: EOG token = 248046 '<|im_end|>' +print_info: EOG token = 248063 '<|fim_pad|>' +print_info: EOG token = 248064 '<|repo_name|>' +print_info: EOG token = 248065 '<|file_sep|>' +print_info: max token length = 256 +load_tensors: loading model tensors, this can take a while... (mmap = false, direct_io = false) +load_tensors: layer 0 assigned to device CPU, is_swa = 0 +load_tensors: layer 1 assigned to device CPU, is_swa = 0 +load_tensors: layer 2 assigned to device CPU, is_swa = 0 +load_tensors: layer 3 assigned to device CPU, is_swa = 0 +load_tensors: layer 4 assigned to device CPU, is_swa = 0 +load_tensors: layer 5 assigned to device CPU, is_swa = 0 +load_tensors: layer 6 assigned to device CPU, is_swa = 0 +load_tensors: layer 7 assigned to device CPU, is_swa = 0 +load_tensors: layer 8 assigned to device CPU, is_swa = 0 +load_tensors: layer 9 assigned to device CPU, is_swa = 0 +load_tensors: layer 10 assigned to device CPU, is_swa = 0 +load_tensors: layer 11 assigned to device CPU, is_swa = 0 +load_tensors: layer 12 assigned to device CPU, is_swa = 0 +load_tensors: layer 13 assigned to device CPU, is_swa = 0 +load_tensors: layer 14 assigned to device CPU, is_swa = 0 +load_tensors: layer 15 assigned to device CPU, is_swa = 0 +load_tensors: layer 16 assigned to device CPU, is_swa = 0 +load_tensors: layer 17 assigned to device CPU, is_swa = 0 +load_tensors: layer 18 assigned to device CPU, is_swa = 0 +load_tensors: layer 19 assigned to device CPU, is_swa = 0 +load_tensors: layer 20 assigned to device CPU, is_swa = 0 +load_tensors: layer 21 assigned to device CPU, is_swa = 0 +load_tensors: layer 22 assigned to device CPU, is_swa = 0 +load_tensors: layer 23 assigned to device CPU, is_swa = 0 +load_tensors: layer 24 assigned to device CPU, is_swa = 0 +load_tensors: layer 25 assigned to device CPU, is_swa = 0 +load_tensors: layer 26 assigned to device CPU, is_swa = 0 +load_tensors: layer 27 assigned to device CPU, is_swa = 0 +load_tensors: layer 28 assigned to device CPU, is_swa = 0 +load_tensors: layer 29 assigned to device CPU, is_swa = 0 +load_tensors: layer 30 assigned to device CPU, is_swa = 0 +load_tensors: layer 31 assigned to device CPU, is_swa = 0 +load_tensors: layer 32 assigned to device CPU, is_swa = 0 +create_tensor: loading tensor token_embd.weight +create_tensor: loading tensor output_norm.weight +create_tensor: loading tensor blk.0.attn_norm.weight +create_tensor: loading tensor blk.0.post_attention_norm.weight +create_tensor: loading tensor blk.0.attn_qkv.weight +create_tensor: loading tensor blk.0.attn_gate.weight +create_tensor: loading tensor blk.0.ssm_conv1d.weight +create_tensor: loading tensor blk.0.ssm_dt.bias +create_tensor: loading tensor blk.0.ssm_a +create_tensor: loading tensor blk.0.ssm_beta.weight +create_tensor: loading tensor blk.0.ssm_alpha.weight +create_tensor: loading tensor blk.0.ssm_norm.weight +create_tensor: loading tensor blk.0.ssm_out.weight +create_tensor: loading tensor blk.0.ffn_gate.weight +create_tensor: loading tensor blk.0.ffn_down.weight +create_tensor: loading tensor blk.0.ffn_up.weight +create_tensor: loading tensor blk.1.attn_norm.weight +create_tensor: loading tensor blk.1.post_attention_norm.weight +create_tensor: loading tensor blk.1.attn_qkv.weight +create_tensor: loading tensor blk.1.attn_gate.weight +create_tensor: loading tensor blk.1.ssm_conv1d.weight +create_tensor: loading tensor blk.1.ssm_dt.bias +create_tensor: loading tensor blk.1.ssm_a +create_tensor: loading tensor blk.1.ssm_beta.weight +create_tensor: loading tensor blk.1.ssm_alpha.weight +create_tensor: loading tensor blk.1.ssm_norm.weight +create_tensor: loading tensor blk.1.ssm_out.weight +create_tensor: loading tensor blk.1.ffn_gate.weight +create_tensor: loading tensor blk.1.ffn_down.weight +create_tensor: loading tensor blk.1.ffn_up.weight +create_tensor: loading tensor blk.2.attn_norm.weight +create_tensor: loading tensor blk.2.post_attention_norm.weight +create_tensor: loading tensor blk.2.attn_qkv.weight +create_tensor: loading tensor blk.2.attn_gate.weight +create_tensor: loading tensor blk.2.ssm_conv1d.weight +create_tensor: loading tensor blk.2.ssm_dt.bias +create_tensor: loading tensor blk.2.ssm_a +create_tensor: loading tensor blk.2.ssm_beta.weight +create_tensor: loading tensor blk.2.ssm_alpha.weight +create_tensor: loading tensor blk.2.ssm_norm.weight +create_tensor: loading tensor blk.2.ssm_out.weight +create_tensor: loading tensor blk.2.ffn_gate.weight +create_tensor: loading tensor blk.2.ffn_down.weight +create_tensor: loading tensor blk.2.ffn_up.weight +create_tensor: loading tensor blk.3.attn_norm.weight +create_tensor: loading tensor blk.3.post_attention_norm.weight +create_tensor: loading tensor blk.3.attn_q.weight +create_tensor: loading tensor blk.3.attn_k.weight +create_tensor: loading tensor blk.3.attn_v.weight +create_tensor: loading tensor blk.3.attn_output.weight +create_tensor: loading tensor blk.3.attn_q_norm.weight +create_tensor: loading tensor blk.3.attn_k_norm.weight +create_tensor: loading tensor blk.3.ffn_gate.weight +create_tensor: loading tensor blk.3.ffn_down.weight +create_tensor: loading tensor blk.3.ffn_up.weight +create_tensor: loading tensor blk.4.attn_norm.weight +create_tensor: loading tensor blk.4.post_attention_norm.weight +create_tensor: loading tensor blk.4.attn_qkv.weight +create_tensor: loading tensor blk.4.attn_gate.weight +create_tensor: loading tensor blk.4.ssm_conv1d.weight +create_tensor: loading tensor blk.4.ssm_dt.bias +create_tensor: loading tensor blk.4.ssm_a +create_tensor: loading tensor blk.4.ssm_beta.weight +create_tensor: loading tensor blk.4.ssm_alpha.weight +create_tensor: loading tensor blk.4.ssm_norm.weight +create_tensor: loading tensor blk.4.ssm_out.weight +create_tensor: loading tensor blk.4.ffn_gate.weight +create_tensor: loading tensor blk.4.ffn_down.weight +create_tensor: loading tensor blk.4.ffn_up.weight +create_tensor: loading tensor blk.5.attn_norm.weight +create_tensor: loading tensor blk.5.post_attention_norm.weight +create_tensor: loading tensor blk.5.attn_qkv.weight +create_tensor: loading tensor blk.5.attn_gate.weight +create_tensor: loading tensor blk.5.ssm_conv1d.weight +create_tensor: loading tensor blk.5.ssm_dt.bias +create_tensor: loading tensor blk.5.ssm_a +create_tensor: loading tensor blk.5.ssm_beta.weight +create_tensor: loading tensor blk.5.ssm_alpha.weight +create_tensor: loading tensor blk.5.ssm_norm.weight +create_tensor: loading tensor blk.5.ssm_out.weight +create_tensor: loading tensor blk.5.ffn_gate.weight +create_tensor: loading tensor blk.5.ffn_down.weight +create_tensor: loading tensor blk.5.ffn_up.weight +create_tensor: loading tensor blk.6.attn_norm.weight +create_tensor: loading tensor blk.6.post_attention_norm.weight +create_tensor: loading tensor blk.6.attn_qkv.weight +create_tensor: loading tensor blk.6.attn_gate.weight +create_tensor: loading tensor blk.6.ssm_conv1d.weight +create_tensor: loading tensor blk.6.ssm_dt.bias +create_tensor: loading tensor blk.6.ssm_a +create_tensor: loading tensor blk.6.ssm_beta.weight +create_tensor: loading tensor blk.6.ssm_alpha.weight +create_tensor: loading tensor blk.6.ssm_norm.weight +create_tensor: loading tensor blk.6.ssm_out.weight +create_tensor: loading tensor blk.6.ffn_gate.weight +create_tensor: loading tensor blk.6.ffn_down.weight +create_tensor: loading tensor blk.6.ffn_up.weight +create_tensor: loading tensor blk.7.attn_norm.weight +create_tensor: loading tensor blk.7.post_attention_norm.weight +create_tensor: loading tensor blk.7.attn_q.weight +create_tensor: loading tensor blk.7.attn_k.weight +create_tensor: loading tensor blk.7.attn_v.weight +create_tensor: loading tensor blk.7.attn_output.weight +create_tensor: loading tensor blk.7.attn_q_norm.weight +create_tensor: loading tensor blk.7.attn_k_norm.weight +create_tensor: loading tensor blk.7.ffn_gate.weight +create_tensor: loading tensor blk.7.ffn_down.weight +create_tensor: loading tensor blk.7.ffn_up.weight +create_tensor: loading tensor blk.8.attn_norm.weight +create_tensor: loading tensor blk.8.post_attention_norm.weight +create_tensor: loading tensor blk.8.attn_qkv.weight +create_tensor: loading tensor blk.8.attn_gate.weight +create_tensor: loading tensor blk.8.ssm_conv1d.weight +create_tensor: loading tensor blk.8.ssm_dt.bias +create_tensor: loading tensor blk.8.ssm_a +create_tensor: loading tensor blk.8.ssm_beta.weight +create_tensor: loading tensor blk.8.ssm_alpha.weight +create_tensor: loading tensor blk.8.ssm_norm.weight +create_tensor: loading tensor blk.8.ssm_out.weight +create_tensor: loading tensor blk.8.ffn_gate.weight +create_tensor: loading tensor blk.8.ffn_down.weight +create_tensor: loading tensor blk.8.ffn_up.weight +create_tensor: loading tensor blk.9.attn_norm.weight +create_tensor: loading tensor blk.9.post_attention_norm.weight +create_tensor: loading tensor blk.9.attn_qkv.weight +create_tensor: loading tensor blk.9.attn_gate.weight +create_tensor: loading tensor blk.9.ssm_conv1d.weight +create_tensor: loading tensor blk.9.ssm_dt.bias +create_tensor: loading tensor blk.9.ssm_a +create_tensor: loading tensor blk.9.ssm_beta.weight +create_tensor: loading tensor blk.9.ssm_alpha.weight +create_tensor: loading tensor blk.9.ssm_norm.weight +create_tensor: loading tensor blk.9.ssm_out.weight +create_tensor: loading tensor blk.9.ffn_gate.weight +create_tensor: loading tensor blk.9.ffn_down.weight +create_tensor: loading tensor blk.9.ffn_up.weight +create_tensor: loading tensor blk.10.attn_norm.weight +create_tensor: loading tensor blk.10.post_attention_norm.weight +create_tensor: loading tensor blk.10.attn_qkv.weight +create_tensor: loading tensor blk.10.attn_gate.weight +create_tensor: loading tensor blk.10.ssm_conv1d.weight +create_tensor: loading tensor blk.10.ssm_dt.bias +create_tensor: loading tensor blk.10.ssm_a +create_tensor: loading tensor blk.10.ssm_beta.weight +create_tensor: loading tensor blk.10.ssm_alpha.weight +create_tensor: loading tensor blk.10.ssm_norm.weight +create_tensor: loading tensor blk.10.ssm_out.weight +create_tensor: loading tensor blk.10.ffn_gate.weight +create_tensor: loading tensor blk.10.ffn_down.weight +create_tensor: loading tensor blk.10.ffn_up.weight +create_tensor: loading tensor blk.11.attn_norm.weight +create_tensor: loading tensor blk.11.post_attention_norm.weight +create_tensor: loading tensor blk.11.attn_q.weight +create_tensor: loading tensor blk.11.attn_k.weight +create_tensor: loading tensor blk.11.attn_v.weight +create_tensor: loading tensor blk.11.attn_output.weight +create_tensor: loading tensor blk.11.attn_q_norm.weight +create_tensor: loading tensor blk.11.attn_k_norm.weight +create_tensor: loading tensor blk.11.ffn_gate.weight +create_tensor: loading tensor blk.11.ffn_down.weight +create_tensor: loading tensor blk.11.ffn_up.weight +create_tensor: loading tensor blk.12.attn_norm.weight +create_tensor: loading tensor blk.12.post_attention_norm.weight +create_tensor: loading tensor blk.12.attn_qkv.weight +create_tensor: loading tensor blk.12.attn_gate.weight +create_tensor: loading tensor blk.12.ssm_conv1d.weight +create_tensor: loading tensor blk.12.ssm_dt.bias +create_tensor: loading tensor blk.12.ssm_a +create_tensor: loading tensor blk.12.ssm_beta.weight +create_tensor: loading tensor blk.12.ssm_alpha.weight +create_tensor: loading tensor blk.12.ssm_norm.weight +create_tensor: loading tensor blk.12.ssm_out.weight +create_tensor: loading tensor blk.12.ffn_gate.weight +create_tensor: loading tensor blk.12.ffn_down.weight +create_tensor: loading tensor blk.12.ffn_up.weight +create_tensor: loading tensor blk.13.attn_norm.weight +create_tensor: loading tensor blk.13.post_attention_norm.weight +create_tensor: loading tensor blk.13.attn_qkv.weight +create_tensor: loading tensor blk.13.attn_gate.weight +create_tensor: loading tensor blk.13.ssm_conv1d.weight +create_tensor: loading tensor blk.13.ssm_dt.bias +create_tensor: loading tensor blk.13.ssm_a +create_tensor: loading tensor blk.13.ssm_beta.weight +create_tensor: loading tensor blk.13.ssm_alpha.weight +create_tensor: loading tensor blk.13.ssm_norm.weight +create_tensor: loading tensor blk.13.ssm_out.weight +create_tensor: loading tensor blk.13.ffn_gate.weight +create_tensor: loading tensor blk.13.ffn_down.weight +create_tensor: loading tensor blk.13.ffn_up.weight +create_tensor: loading tensor blk.14.attn_norm.weight +create_tensor: loading tensor blk.14.post_attention_norm.weight +create_tensor: loading tensor blk.14.attn_qkv.weight +create_tensor: loading tensor blk.14.attn_gate.weight +create_tensor: loading tensor blk.14.ssm_conv1d.weight +create_tensor: loading tensor blk.14.ssm_dt.bias +create_tensor: loading tensor blk.14.ssm_a +create_tensor: loading tensor blk.14.ssm_beta.weight +create_tensor: loading tensor blk.14.ssm_alpha.weight +create_tensor: loading tensor blk.14.ssm_norm.weight +create_tensor: loading tensor blk.14.ssm_out.weight +create_tensor: loading tensor blk.14.ffn_gate.weight +create_tensor: loading tensor blk.14.ffn_down.weight +create_tensor: loading tensor blk.14.ffn_up.weight +create_tensor: loading tensor blk.15.attn_norm.weight +create_tensor: loading tensor blk.15.post_attention_norm.weight +create_tensor: loading tensor blk.15.attn_q.weight +create_tensor: loading tensor blk.15.attn_k.weight +create_tensor: loading tensor blk.15.attn_v.weight +create_tensor: loading tensor blk.15.attn_output.weight +create_tensor: loading tensor blk.15.attn_q_norm.weight +create_tensor: loading tensor blk.15.attn_k_norm.weight +create_tensor: loading tensor blk.15.ffn_gate.weight +create_tensor: loading tensor blk.15.ffn_down.weight +create_tensor: loading tensor blk.15.ffn_up.weight +create_tensor: loading tensor blk.16.attn_norm.weight +create_tensor: loading tensor blk.16.post_attention_norm.weight +create_tensor: loading tensor blk.16.attn_qkv.weight +create_tensor: loading tensor blk.16.attn_gate.weight +create_tensor: loading tensor blk.16.ssm_conv1d.weight +create_tensor: loading tensor blk.16.ssm_dt.bias +create_tensor: loading tensor blk.16.ssm_a +create_tensor: loading tensor blk.16.ssm_beta.weight +create_tensor: loading tensor blk.16.ssm_alpha.weight +create_tensor: loading tensor blk.16.ssm_norm.weight +create_tensor: loading tensor blk.16.ssm_out.weight +create_tensor: loading tensor blk.16.ffn_gate.weight +create_tensor: loading tensor blk.16.ffn_down.weight +create_tensor: loading tensor blk.16.ffn_up.weight +create_tensor: loading tensor blk.17.attn_norm.weight +create_tensor: loading tensor blk.17.post_attention_norm.weight +create_tensor: loading tensor blk.17.attn_qkv.weight +create_tensor: loading tensor blk.17.attn_gate.weight +create_tensor: loading tensor blk.17.ssm_conv1d.weight +create_tensor: loading tensor blk.17.ssm_dt.bias +create_tensor: loading tensor blk.17.ssm_a +create_tensor: loading tensor blk.17.ssm_beta.weight +create_tensor: loading tensor blk.17.ssm_alpha.weight +create_tensor: loading tensor blk.17.ssm_norm.weight +create_tensor: loading tensor blk.17.ssm_out.weight +create_tensor: loading tensor blk.17.ffn_gate.weight +create_tensor: loading tensor blk.17.ffn_down.weight +create_tensor: loading tensor blk.17.ffn_up.weight +create_tensor: loading tensor blk.18.attn_norm.weight +create_tensor: loading tensor blk.18.post_attention_norm.weight +create_tensor: loading tensor blk.18.attn_qkv.weight +create_tensor: loading tensor blk.18.attn_gate.weight +create_tensor: loading tensor blk.18.ssm_conv1d.weight +create_tensor: loading tensor blk.18.ssm_dt.bias +create_tensor: loading tensor blk.18.ssm_a +create_tensor: loading tensor blk.18.ssm_beta.weight +create_tensor: loading tensor blk.18.ssm_alpha.weight +create_tensor: loading tensor blk.18.ssm_norm.weight +create_tensor: loading tensor blk.18.ssm_out.weight +create_tensor: loading tensor blk.18.ffn_gate.weight +create_tensor: loading tensor blk.18.ffn_down.weight +create_tensor: loading tensor blk.18.ffn_up.weight +create_tensor: loading tensor blk.19.attn_norm.weight +create_tensor: loading tensor blk.19.post_attention_norm.weight +create_tensor: loading tensor blk.19.attn_q.weight +create_tensor: loading tensor blk.19.attn_k.weight +create_tensor: loading tensor blk.19.attn_v.weight +create_tensor: loading tensor blk.19.attn_output.weight +create_tensor: loading tensor blk.19.attn_q_norm.weight +create_tensor: loading tensor blk.19.attn_k_norm.weight +create_tensor: loading tensor blk.19.ffn_gate.weight +create_tensor: loading tensor blk.19.ffn_down.weight +create_tensor: loading tensor blk.19.ffn_up.weight +create_tensor: loading tensor blk.20.attn_norm.weight +create_tensor: loading tensor blk.20.post_attention_norm.weight +create_tensor: loading tensor blk.20.attn_qkv.weight +create_tensor: loading tensor blk.20.attn_gate.weight +create_tensor: loading tensor blk.20.ssm_conv1d.weight +create_tensor: loading tensor blk.20.ssm_dt.bias +create_tensor: loading tensor blk.20.ssm_a +create_tensor: loading tensor blk.20.ssm_beta.weight +create_tensor: loading tensor blk.20.ssm_alpha.weight +create_tensor: loading tensor blk.20.ssm_norm.weight +create_tensor: loading tensor blk.20.ssm_out.weight +create_tensor: loading tensor blk.20.ffn_gate.weight +create_tensor: loading tensor blk.20.ffn_down.weight +create_tensor: loading tensor blk.20.ffn_up.weight +create_tensor: loading tensor blk.21.attn_norm.weight +create_tensor: loading tensor blk.21.post_attention_norm.weight +create_tensor: loading tensor blk.21.attn_qkv.weight +create_tensor: loading tensor blk.21.attn_gate.weight +create_tensor: loading tensor blk.21.ssm_conv1d.weight +create_tensor: loading tensor blk.21.ssm_dt.bias +create_tensor: loading tensor blk.21.ssm_a +create_tensor: loading tensor blk.21.ssm_beta.weight +create_tensor: loading tensor blk.21.ssm_alpha.weight +create_tensor: loading tensor blk.21.ssm_norm.weight +create_tensor: loading tensor blk.21.ssm_out.weight +create_tensor: loading tensor blk.21.ffn_gate.weight +create_tensor: loading tensor blk.21.ffn_down.weight +create_tensor: loading tensor blk.21.ffn_up.weight +create_tensor: loading tensor blk.22.attn_norm.weight +create_tensor: loading tensor blk.22.post_attention_norm.weight +create_tensor: loading tensor blk.22.attn_qkv.weight +create_tensor: loading tensor blk.22.attn_gate.weight +create_tensor: loading tensor blk.22.ssm_conv1d.weight +create_tensor: loading tensor blk.22.ssm_dt.bias +create_tensor: loading tensor blk.22.ssm_a +create_tensor: loading tensor blk.22.ssm_beta.weight +create_tensor: loading tensor blk.22.ssm_alpha.weight +create_tensor: loading tensor blk.22.ssm_norm.weight +create_tensor: loading tensor blk.22.ssm_out.weight +create_tensor: loading tensor blk.22.ffn_gate.weight +create_tensor: loading tensor blk.22.ffn_down.weight +create_tensor: loading tensor blk.22.ffn_up.weight +create_tensor: loading tensor blk.23.attn_norm.weight +create_tensor: loading tensor blk.23.post_attention_norm.weight +create_tensor: loading tensor blk.23.attn_q.weight +create_tensor: loading tensor blk.23.attn_k.weight +create_tensor: loading tensor blk.23.attn_v.weight +create_tensor: loading tensor blk.23.attn_output.weight +create_tensor: loading tensor blk.23.attn_q_norm.weight +create_tensor: loading tensor blk.23.attn_k_norm.weight +create_tensor: loading tensor blk.23.ffn_gate.weight +create_tensor: loading tensor blk.23.ffn_down.weight +create_tensor: loading tensor blk.23.ffn_up.weight +create_tensor: loading tensor blk.24.attn_norm.weight +create_tensor: loading tensor blk.24.post_attention_norm.weight +create_tensor: loading tensor blk.24.attn_qkv.weight +create_tensor: loading tensor blk.24.attn_gate.weight +create_tensor: loading tensor blk.24.ssm_conv1d.weight +create_tensor: loading tensor blk.24.ssm_dt.bias +create_tensor: loading tensor blk.24.ssm_a +create_tensor: loading tensor blk.24.ssm_beta.weight +create_tensor: loading tensor blk.24.ssm_alpha.weight +create_tensor: loading tensor blk.24.ssm_norm.weight +create_tensor: loading tensor blk.24.ssm_out.weight +create_tensor: loading tensor blk.24.ffn_gate.weight +create_tensor: loading tensor blk.24.ffn_down.weight +create_tensor: loading tensor blk.24.ffn_up.weight +create_tensor: loading tensor blk.25.attn_norm.weight +create_tensor: loading tensor blk.25.post_attention_norm.weight +create_tensor: loading tensor blk.25.attn_qkv.weight +create_tensor: loading tensor blk.25.attn_gate.weight +create_tensor: loading tensor blk.25.ssm_conv1d.weight +create_tensor: loading tensor blk.25.ssm_dt.bias +create_tensor: loading tensor blk.25.ssm_a +create_tensor: loading tensor blk.25.ssm_beta.weight +create_tensor: loading tensor blk.25.ssm_alpha.weight +create_tensor: loading tensor blk.25.ssm_norm.weight +create_tensor: loading tensor blk.25.ssm_out.weight +create_tensor: loading tensor blk.25.ffn_gate.weight +create_tensor: loading tensor blk.25.ffn_down.weight +create_tensor: loading tensor blk.25.ffn_up.weight +create_tensor: loading tensor blk.26.attn_norm.weight +create_tensor: loading tensor blk.26.post_attention_norm.weight +create_tensor: loading tensor blk.26.attn_qkv.weight +create_tensor: loading tensor blk.26.attn_gate.weight +create_tensor: loading tensor blk.26.ssm_conv1d.weight +create_tensor: loading tensor blk.26.ssm_dt.bias +create_tensor: loading tensor blk.26.ssm_a +create_tensor: loading tensor blk.26.ssm_beta.weight +create_tensor: loading tensor blk.26.ssm_alpha.weight +create_tensor: loading tensor blk.26.ssm_norm.weight +create_tensor: loading tensor blk.26.ssm_out.weight +create_tensor: loading tensor blk.26.ffn_gate.weight +create_tensor: loading tensor blk.26.ffn_down.weight +create_tensor: loading tensor blk.26.ffn_up.weight +create_tensor: loading tensor blk.27.attn_norm.weight +create_tensor: loading tensor blk.27.post_attention_norm.weight +create_tensor: loading tensor blk.27.attn_q.weight +create_tensor: loading tensor blk.27.attn_k.weight +create_tensor: loading tensor blk.27.attn_v.weight +create_tensor: loading tensor blk.27.attn_output.weight +create_tensor: loading tensor blk.27.attn_q_norm.weight +create_tensor: loading tensor blk.27.attn_k_norm.weight +create_tensor: loading tensor blk.27.ffn_gate.weight +create_tensor: loading tensor blk.27.ffn_down.weight +create_tensor: loading tensor blk.27.ffn_up.weight +create_tensor: loading tensor blk.28.attn_norm.weight +create_tensor: loading tensor blk.28.post_attention_norm.weight +create_tensor: loading tensor blk.28.attn_qkv.weight +create_tensor: loading tensor blk.28.attn_gate.weight +create_tensor: loading tensor blk.28.ssm_conv1d.weight +create_tensor: loading tensor blk.28.ssm_dt.bias +create_tensor: loading tensor blk.28.ssm_a +create_tensor: loading tensor blk.28.ssm_beta.weight +create_tensor: loading tensor blk.28.ssm_alpha.weight +create_tensor: loading tensor blk.28.ssm_norm.weight +create_tensor: loading tensor blk.28.ssm_out.weight +create_tensor: loading tensor blk.28.ffn_gate.weight +create_tensor: loading tensor blk.28.ffn_down.weight +create_tensor: loading tensor blk.28.ffn_up.weight +create_tensor: loading tensor blk.29.attn_norm.weight +create_tensor: loading tensor blk.29.post_attention_norm.weight +create_tensor: loading tensor blk.29.attn_qkv.weight +create_tensor: loading tensor blk.29.attn_gate.weight +create_tensor: loading tensor blk.29.ssm_conv1d.weight +create_tensor: loading tensor blk.29.ssm_dt.bias +create_tensor: loading tensor blk.29.ssm_a +create_tensor: loading tensor blk.29.ssm_beta.weight +create_tensor: loading tensor blk.29.ssm_alpha.weight +create_tensor: loading tensor blk.29.ssm_norm.weight +create_tensor: loading tensor blk.29.ssm_out.weight +create_tensor: loading tensor blk.29.ffn_gate.weight +create_tensor: loading tensor blk.29.ffn_down.weight +create_tensor: loading tensor blk.29.ffn_up.weight +create_tensor: loading tensor blk.30.attn_norm.weight +create_tensor: loading tensor blk.30.post_attention_norm.weight +create_tensor: loading tensor blk.30.attn_qkv.weight +create_tensor: loading tensor blk.30.attn_gate.weight +create_tensor: loading tensor blk.30.ssm_conv1d.weight +create_tensor: loading tensor blk.30.ssm_dt.bias +create_tensor: loading tensor blk.30.ssm_a +create_tensor: loading tensor blk.30.ssm_beta.weight +create_tensor: loading tensor blk.30.ssm_alpha.weight +create_tensor: loading tensor blk.30.ssm_norm.weight +create_tensor: loading tensor blk.30.ssm_out.weight +create_tensor: loading tensor blk.30.ffn_gate.weight +create_tensor: loading tensor blk.30.ffn_down.weight +create_tensor: loading tensor blk.30.ffn_up.weight +create_tensor: loading tensor blk.31.attn_norm.weight +create_tensor: loading tensor blk.31.post_attention_norm.weight +create_tensor: loading tensor blk.31.attn_q.weight +create_tensor: loading tensor blk.31.attn_k.weight +create_tensor: loading tensor blk.31.attn_v.weight +create_tensor: loading tensor blk.31.attn_output.weight +create_tensor: loading tensor blk.31.attn_q_norm.weight +create_tensor: loading tensor blk.31.attn_k_norm.weight +create_tensor: loading tensor blk.31.ffn_gate.weight +create_tensor: loading tensor blk.31.ffn_down.weight +create_tensor: loading tensor blk.31.ffn_up.weight +done_getting_tensors: tensor 'token_embd.weight' (q6_K) (and 426 others) cannot be used with preferred buffer type CPU_REPACK, using CPU instead +load_tensors: offloading output layer to GPU +load_tensors: offloading 31 repeating layers to GPU +load_tensors: offloaded 33/33 layers to GPU +load_tensors: CPU model buffer size = 0.00 MiB +llama_context: constructing llama_context +llama_context: n_seq_max = 1 +llama_context: n_ctx = 4096 +llama_context: n_ctx_seq = 4096 +llama_context: n_batch = 2048 +llama_context: n_ubatch = 512 +llama_context: causal_attn = 1 +llama_context: flash_attn = enabled +llama_context: kv_unified = false +llama_context: freq_base = 10000000.0 +llama_context: freq_scale = 1 +llama_context: n_ctx_seq (4096) < n_ctx_train (262144) -- the full capacity of the model will not be utilized +set_abort_callback: call +llama_context: CPU output buffer size = 0.95 MiB +llama_kv_cache: layer 0: filtered +llama_kv_cache: layer 1: filtered +llama_kv_cache: layer 2: filtered +llama_kv_cache: layer 3: dev = CPU +llama_kv_cache: layer 4: filtered +llama_kv_cache: layer 5: filtered +llama_kv_cache: layer 6: filtered +llama_kv_cache: layer 7: dev = CPU +llama_kv_cache: layer 8: filtered +llama_kv_cache: layer 9: filtered +llama_kv_cache: layer 10: filtered +llama_kv_cache: layer 11: dev = CPU +llama_kv_cache: layer 12: filtered +llama_kv_cache: layer 13: filtered +llama_kv_cache: layer 14: filtered +llama_kv_cache: layer 15: dev = CPU +llama_kv_cache: layer 16: filtered +llama_kv_cache: layer 17: filtered +llama_kv_cache: layer 18: filtered +llama_kv_cache: layer 19: dev = CPU +llama_kv_cache: layer 20: filtered +llama_kv_cache: layer 21: filtered +llama_kv_cache: layer 22: filtered +llama_kv_cache: layer 23: dev = CPU +llama_kv_cache: layer 24: filtered +llama_kv_cache: layer 25: filtered +llama_kv_cache: layer 26: filtered +llama_kv_cache: layer 27: dev = CPU +llama_kv_cache: layer 28: filtered +llama_kv_cache: layer 29: filtered +llama_kv_cache: layer 30: filtered +llama_kv_cache: layer 31: dev = CPU +llama_kv_cache: CPU KV buffer size = 0.00 MiB +llama_kv_cache: size = 128.00 MiB ( 4096 cells, 8 layers, 1/1 seqs), K (f16): 64.00 MiB, V (f16): 64.00 MiB +llama_memory_recurrent, layer 0: dev = CPU +llama_memory_recurrent, layer 1: dev = CPU +llama_memory_recurrent, layer 2: dev = CPU +llama_memory_recurrent: layer 3: skipped +llama_memory_recurrent, layer 4: dev = CPU +llama_memory_recurrent, layer 5: dev = CPU +llama_memory_recurrent, layer 6: dev = CPU +llama_memory_recurrent: layer 7: skipped +llama_memory_recurrent, layer 8: dev = CPU +llama_memory_recurrent, layer 9: dev = CPU +llama_memory_recurrent, layer 10: dev = CPU +llama_memory_recurrent: layer 11: skipped +llama_memory_recurrent, layer 12: dev = CPU +llama_memory_recurrent, layer 13: dev = CPU +llama_memory_recurrent, layer 14: dev = CPU +llama_memory_recurrent: layer 15: skipped +llama_memory_recurrent, layer 16: dev = CPU +llama_memory_recurrent, layer 17: dev = CPU +llama_memory_recurrent, layer 18: dev = CPU +llama_memory_recurrent: layer 19: skipped +llama_memory_recurrent, layer 20: dev = CPU +llama_memory_recurrent, layer 21: dev = CPU +llama_memory_recurrent, layer 22: dev = CPU +llama_memory_recurrent: layer 23: skipped +llama_memory_recurrent, layer 24: dev = CPU +llama_memory_recurrent, layer 25: dev = CPU +llama_memory_recurrent, layer 26: dev = CPU +llama_memory_recurrent: layer 27: skipped +llama_memory_recurrent, layer 28: dev = CPU +llama_memory_recurrent, layer 29: dev = CPU +llama_memory_recurrent, layer 30: dev = CPU +llama_memory_recurrent: layer 31: skipped +llama_memory_recurrent: CPU RS buffer size = 50.25 MiB +llama_memory_recurrent: size = 50.25 MiB ( 1 cells, 32 layers, 1 seqs), R (f32): 2.25 MiB, S (f32): 48.00 MiB +llama_context: enumerating backends +llama_context: backend_ptrs.size() = 1 +sched_reserve: reserving ... +sched_reserve: max_nodes = 20480 +sched_reserve: reserving full memory module +sched_reserve: worst-case: n_tokens = 512, n_seqs = 1, n_outputs = 1 +sched_reserve: resolving fused Gated Delta Net support: +graph_reserve: reserving a graph for ubatch with n_tokens = 1, n_seqs = 1, n_outputs = 1 +sched_reserve: fused Gated Delta Net (autoregressive) enabled +graph_reserve: reserving a graph for ubatch with n_tokens = 16, n_seqs = 1, n_outputs = 16 +sched_reserve: fused Gated Delta Net (chunked) enabled +graph_reserve: reserving a graph for ubatch with n_tokens = 512, n_seqs = 1, n_outputs = 512 +graph_reserve: reserving a graph for ubatch with n_tokens = 1, n_seqs = 1, n_outputs = 1 +graph_reserve: reserving a graph for ubatch with n_tokens = 512, n_seqs = 1, n_outputs = 512 +sched_reserve: CPU compute buffer size = 495.00 MiB +sched_reserve: graph nodes = 1833 +sched_reserve: graph splits = 1 +sched_reserve: reserve took 5.66 ms, sched copies = 1 +llama_memory_breakdown_print: | memory breakdown [MiB] | total free self model context compute unaccounted | +llama_memory_breakdown_print: | - Host | 3966 = 3293 + 178 + 495 | /llama_params_fit_impl: no devices with dedicated memory found +llama_params_fit: successfully fit params to free device memory +llama_params_fit: fitting params to free memory took 0.47 seconds +llama_model_loader: loaded meta data with 53 key-value pairs and 426 tensors from D:\Documents\Github\ai-backend-studio\models\Qwen3.5-4B-Abliterated-Claude-4.6-Opus-Reasoning-Distilled.i1-Q6_K.gguf (version GGUF V3 (latest)) +llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output. +llama_model_loader: - kv 0: general.architecture str = qwen35 +llama_model_loader: - kv 1: general.type str = model +llama_model_loader: - kv 2: general.name str = Qwen3.5-4B-Abliterated-Claude-4.6-Opu... +llama_model_loader: - kv 3: general.version str = 4.6 +llama_model_loader: - kv 4: general.finetune str = Abliterated-Claude-Opus-Reasoning-Dis... +llama_model_loader: - kv 5: general.basename str = Qwen3.5 +llama_model_loader: - kv 6: general.size_label str = 4B +llama_model_loader: - kv 7: general.license str = other +llama_model_loader: - kv 8: general.base_model.count u32 = 1 +llama_model_loader: - kv 9: general.base_model.0.name str = Qwen3.5 4B Claude 4.6 Opus Reasoning ... +llama_model_loader: - kv 10: general.base_model.0.version str = 4.6 +llama_model_loader: - kv 11: general.base_model.0.organization str = Jackrong +llama_model_loader: - kv 12: general.base_model.0.repo_url str = https://huggingface.co/Jackrong/Qwen3... +llama_model_loader: - kv 13: general.tags arr[str,5] = ["abliterated", "de-censored", "reaso... +llama_model_loader: - kv 14: qwen35.block_count u32 = 32 +llama_model_loader: - kv 15: qwen35.context_length u32 = 262144 +llama_model_loader: - kv 16: qwen35.embedding_length u32 = 2560 +llama_model_loader: - kv 17: qwen35.feed_forward_length u32 = 9216 +llama_model_loader: - kv 18: qwen35.attention.head_count u32 = 16 +llama_model_loader: - kv 19: qwen35.attention.head_count_kv u32 = 4 +llama_model_loader: - kv 20: qwen35.rope.dimension_sections arr[i32,4] = [11, 11, 10, 0] +llama_model_loader: - kv 21: qwen35.rope.freq_base f32 = 10000000.000000 +llama_model_loader: - kv 22: qwen35.attention.layer_norm_rms_epsilon f32 = 0.000001 +llama_model_loader: - kv 23: qwen35.attention.key_length u32 = 256 +llama_model_loader: - kv 24: qwen35.attention.value_length u32 = 256 +llama_model_loader: - kv 25: qwen35.ssm.conv_kernel u32 = 4 +llama_model_loader: - kv 26: qwen35.ssm.state_size u32 = 128 +llama_model_loader: - kv 27: qwen35.ssm.group_count u32 = 16 +llama_model_loader: - kv 28: qwen35.ssm.time_step_rank u32 = 32 +llama_model_loader: - kv 29: qwen35.ssm.inner_size u32 = 4096 +llama_model_loader: - kv 30: qwen35.full_attention_interval u32 = 4 +llama_model_loader: - kv 31: qwen35.rope.dimension_count u32 = 64 +llama_model_loader: - kv 32: tokenizer.ggml.model str = gpt2 +llama_model_loader: - kv 33: tokenizer.ggml.pre str = qwen35 -llama_model_loader: - kv 34: tokenizer.ggml.tokens arr[str,248320] = ["!", "\"", "#", "$", "%", "&", "'", ... +llama_model_loader: - kv 35: tokenizer.ggml.token_type arr[i32,248320] = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... +llama_model_loader: - kv 36: tokenizer.ggml.merges arr[str,247587] = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",... +llama_model_loader: - kv 37: tokenizer.ggml.eos_token_id u32 = 248046 +llama_model_loader: - kv 38: tokenizer.ggml.padding_token_id u32 = 248055 +llama_model_loader: - kv 39: tokenizer.chat_template str = {%- if tools %}\n {{- '<|im_start|>... +llama_model_loader: - kv 40: general.quantization_version u32 = 2 +llama_model_loader: - kv 41: general.file_type u32 = 18 +llama_model_loader: - kv 42: general.url str = https://huggingface.co/mradermacher/Q... +llama_model_loader: - kv 43: mradermacher.quantize_version str = 2 +llama_model_loader: - kv 44: mradermacher.quantized_by str = mradermacher +llama_model_loader: - kv 45: mradermacher.quantized_at str = 2026-03-10T09:59:38+01:00 +llama_model_loader: - kv 46: mradermacher.quantized_on str = nico1 +llama_model_loader: - kv 47: general.source.url str = https://huggingface.co/Abhiray/Qwen3.... +llama_model_loader: - kv 48: mradermacher.convert_type str = hf +llama_model_loader: - kv 49: quantize.imatrix.file str = Qwen3.5-4B-Abliterated-Claude-4.6-Opu... +llama_model_loader: - kv 50: quantize.imatrix.dataset str = imatrix-training-full-3 +llama_model_loader: - kv 51: quantize.imatrix.entries_count u32 = 248 +llama_model_loader: - kv 52: quantize.imatrix.chunks_count u32 = 319 +llama_model_loader: - type f32: 177 tensors +llama_model_loader: - type q6_K: 249 tensors +print_info: file format = GGUF V3 (latest) +print_info: file type = Q6_K +print_info: file size = 3.22 GiB (6.57 BPW) \init_tokenizer: initializing tokenizer for type 2 +load: 0 unused tokens +load: control token: 248062 '<|fim_suffix|>' is not marked as EOG +load: control token: 248072 '' is not marked as EOG +load: control token: 248050 '<|box_end|>' is not marked as EOG |load: control token: 248048 '<|object_ref_end|>' is not marked as EOG +load: control token: 248055 '<|vision_pad|>' is not marked as EOG +load: control token: 248060 '<|fim_prefix|>' is not marked as EOG +load: control token: 248049 '<|box_start|>' is not marked as EOG +load: control token: 248073 '' is not marked as EOG +load: control token: 248052 '<|quad_end|>' is not marked as EOG +load: control token: 248054 '<|vision_end|>' is not marked as EOG +load: control token: 248075 '' is not marked as EOG +load: control token: 248056 '<|image_pad|>' is not marked as EOG +load: control token: 248071 '<|audio_end|>' is not marked as EOG +load: control token: 248045 '<|im_start|>' is not marked as EOG +load: control token: 248047 '<|object_ref_start|>' is not marked as EOG +load: control token: 248051 '<|quad_start|>' is not marked as EOG +load: control token: 248053 '<|vision_start|>' is not marked as EOG +load: control token: 248057 '<|video_pad|>' is not marked as EOG +load: control token: 248061 '<|fim_middle|>' is not marked as EOG +load: control token: 248070 '<|audio_start|>' is not marked as EOG +load: control token: 248074 '' is not marked as EOG +load: control token: 248076 '<|audio_pad|>' is not marked as EOG +load: printing all EOG tokens: +load: - 248044 ('<|endoftext|>') +load: - 248046 ('<|im_end|>') +load: - 248063 ('<|fim_pad|>') +load: - 248064 ('<|repo_name|>') +load: - 248065 ('<|file_sep|>') +load: special tokens cache size = 33 +load: token to piece cache size = 1.7581 MB +print_info: arch = qwen35 +print_info: vocab_only = 0 +print_info: no_alloc = 0 +print_info: n_ctx_train = 262144 +print_info: n_embd = 2560 +print_info: n_embd_inp = 2560 +print_info: n_layer = 32 +print_info: n_head = 16 +print_info: n_head_kv = 4 +print_info: n_rot = 64 +print_info: n_swa = 0 +print_info: is_swa_any = 0 +print_info: n_embd_head_k = 256 +print_info: n_embd_head_v = 256 +print_info: n_gqa = 4 +print_info: n_embd_k_gqa = 1024 +print_info: n_embd_v_gqa = 1024 +print_info: f_norm_eps = 0.0e+00 +print_info: f_norm_rms_eps = 1.0e-06 +print_info: f_clamp_kqv = 0.0e+00 +print_info: f_max_alibi_bias = 0.0e+00 +print_info: f_logit_scale = 0.0e+00 +print_info: f_attn_scale = 0.0e+00 +print_info: n_ff = 9216 +print_info: n_expert = 0 +print_info: n_expert_used = 0 +print_info: n_expert_groups = 0 +print_info: n_group_used = 0 +print_info: causal attn = 1 +print_info: pooling type = -1 +print_info: rope type = 40 +print_info: rope scaling = linear +print_info: freq_base_train = 10000000.0 +print_info: freq_scale_train = 1 +print_info: n_ctx_orig_yarn = 262144 +print_info: rope_yarn_log_mul = 0.0000 +print_info: rope_finetuned = unknown +print_info: mrope sections = [11, 11, 10, 0] +print_info: ssm_d_conv = 4 +print_info: ssm_d_inner = 4096 +print_info: ssm_d_state = 128 +print_info: ssm_dt_rank = 32 +print_info: ssm_n_group = 16 +print_info: ssm_dt_b_c_rms = 0 +print_info: model type = 4B +print_info: model params = 4.21 B +print_info: general.name = Qwen3.5-4B-Abliterated-Claude-4.6-Opus-Reasoning-Distilled +print_info: vocab type = BPE +print_info: n_vocab = 248320 +print_info: n_merges = 247587 +print_info: BOS token = 11 ',' +print_info: EOS token = 248046 '<|im_end|>' +print_info: EOT token = 248046 '<|im_end|>' +print_info: PAD token = 248055 '<|vision_pad|>' +print_info: LF token = 198 'Ċ' +print_info: FIM PRE token = 248060 '<|fim_prefix|>' +print_info: FIM SUF token = 248062 '<|fim_suffix|>' +print_info: FIM MID token = 248061 '<|fim_middle|>' +print_info: FIM PAD token = 248063 '<|fim_pad|>' +print_info: FIM REP token = 248064 '<|repo_name|>' +print_info: FIM SEP token = 248065 '<|file_sep|>' +print_info: EOG token = 248044 '<|endoftext|>' +print_info: EOG token = 248046 '<|im_end|>' +print_info: EOG token = 248063 '<|fim_pad|>' +print_info: EOG token = 248064 '<|repo_name|>' +print_info: EOG token = 248065 '<|file_sep|>' +print_info: max token length = 256 +load_tensors: loading model tensors, this can take a while... (mmap = true, direct_io = false) +load_tensors: layer 0 assigned to device CPU, is_swa = 0 +load_tensors: layer 1 assigned to device CPU, is_swa = 0 +load_tensors: layer 2 assigned to device CPU, is_swa = 0 +load_tensors: layer 3 assigned to device CPU, is_swa = 0 +load_tensors: layer 4 assigned to device CPU, is_swa = 0 +load_tensors: layer 5 assigned to device CPU, is_swa = 0 +load_tensors: layer 6 assigned to device CPU, is_swa = 0 +load_tensors: layer 7 assigned to device CPU, is_swa = 0 +load_tensors: layer 8 assigned to device CPU, is_swa = 0 +load_tensors: layer 9 assigned to device CPU, is_swa = 0 +load_tensors: layer 10 assigned to device CPU, is_swa = 0 +load_tensors: layer 11 assigned to device CPU, is_swa = 0 +load_tensors: layer 12 assigned to device CPU, is_swa = 0 +load_tensors: layer 13 assigned to device CPU, is_swa = 0 +load_tensors: layer 14 assigned to device CPU, is_swa = 0 +load_tensors: layer 15 assigned to device CPU, is_swa = 0 +load_tensors: layer 16 assigned to device CPU, is_swa = 0 +load_tensors: layer 17 assigned to device CPU, is_swa = 0 +load_tensors: layer 18 assigned to device CPU, is_swa = 0 +load_tensors: layer 19 assigned to device CPU, is_swa = 0 +load_tensors: layer 20 assigned to device CPU, is_swa = 0 +load_tensors: layer 21 assigned to device CPU, is_swa = 0 +load_tensors: layer 22 assigned to device CPU, is_swa = 0 +load_tensors: layer 23 assigned to device CPU, is_swa = 0 +load_tensors: layer 24 assigned to device CPU, is_swa = 0 +load_tensors: layer 25 assigned to device CPU, is_swa = 0 +load_tensors: layer 26 assigned to device CPU, is_swa = 0 +load_tensors: layer 27 assigned to device CPU, is_swa = 0 +load_tensors: layer 28 assigned to device CPU, is_swa = 0 +load_tensors: layer 29 assigned to device CPU, is_swa = 0 +load_tensors: layer 30 assigned to device CPU, is_swa = 0 +load_tensors: layer 31 assigned to device CPU, is_swa = 0 +load_tensors: layer 32 assigned to device CPU, is_swa = 0 +create_tensor: loading tensor token_embd.weight +create_tensor: loading tensor output_norm.weight +create_tensor: loading tensor blk.0.attn_norm.weight +create_tensor: loading tensor blk.0.post_attention_norm.weight +create_tensor: loading tensor blk.0.attn_qkv.weight +create_tensor: loading tensor blk.0.attn_gate.weight +create_tensor: loading tensor blk.0.ssm_conv1d.weight +create_tensor: loading tensor blk.0.ssm_dt.bias +create_tensor: loading tensor blk.0.ssm_a +create_tensor: loading tensor blk.0.ssm_beta.weight +create_tensor: loading tensor blk.0.ssm_alpha.weight +create_tensor: loading tensor blk.0.ssm_norm.weight +create_tensor: loading tensor blk.0.ssm_out.weight +create_tensor: loading tensor blk.0.ffn_gate.weight +create_tensor: loading tensor blk.0.ffn_down.weight +create_tensor: loading tensor blk.0.ffn_up.weight +create_tensor: loading tensor blk.1.attn_norm.weight +create_tensor: loading tensor blk.1.post_attention_norm.weight +create_tensor: loading tensor blk.1.attn_qkv.weight +create_tensor: loading tensor blk.1.attn_gate.weight +create_tensor: loading tensor blk.1.ssm_conv1d.weight +create_tensor: loading tensor blk.1.ssm_dt.bias +create_tensor: loading tensor blk.1.ssm_a +create_tensor: loading tensor blk.1.ssm_beta.weight +create_tensor: loading tensor blk.1.ssm_alpha.weight +create_tensor: loading tensor blk.1.ssm_norm.weight +create_tensor: loading tensor blk.1.ssm_out.weight +create_tensor: loading tensor blk.1.ffn_gate.weight +create_tensor: loading tensor blk.1.ffn_down.weight +create_tensor: loading tensor blk.1.ffn_up.weight +create_tensor: loading tensor blk.2.attn_norm.weight +create_tensor: loading tensor blk.2.post_attention_norm.weight +create_tensor: loading tensor blk.2.attn_qkv.weight +create_tensor: loading tensor blk.2.attn_gate.weight +create_tensor: loading tensor blk.2.ssm_conv1d.weight +create_tensor: loading tensor blk.2.ssm_dt.bias +create_tensor: loading tensor blk.2.ssm_a +create_tensor: loading tensor blk.2.ssm_beta.weight +create_tensor: loading tensor blk.2.ssm_alpha.weight +create_tensor: loading tensor blk.2.ssm_norm.weight +create_tensor: loading tensor blk.2.ssm_out.weight +create_tensor: loading tensor blk.2.ffn_gate.weight +create_tensor: loading tensor blk.2.ffn_down.weight +create_tensor: loading tensor blk.2.ffn_up.weight +create_tensor: loading tensor blk.3.attn_norm.weight +create_tensor: loading tensor blk.3.post_attention_norm.weight +create_tensor: loading tensor blk.3.attn_q.weight +create_tensor: loading tensor blk.3.attn_k.weight +create_tensor: loading tensor blk.3.attn_v.weight +create_tensor: loading tensor blk.3.attn_output.weight +create_tensor: loading tensor blk.3.attn_q_norm.weight +create_tensor: loading tensor blk.3.attn_k_norm.weight +create_tensor: loading tensor blk.3.ffn_gate.weight +create_tensor: loading tensor blk.3.ffn_down.weight +create_tensor: loading tensor blk.3.ffn_up.weight +create_tensor: loading tensor blk.4.attn_norm.weight +create_tensor: loading tensor blk.4.post_attention_norm.weight +create_tensor: loading tensor blk.4.attn_qkv.weight +create_tensor: loading tensor blk.4.attn_gate.weight +create_tensor: loading tensor blk.4.ssm_conv1d.weight +create_tensor: loading tensor blk.4.ssm_dt.bias +create_tensor: loading tensor blk.4.ssm_a +create_tensor: loading tensor blk.4.ssm_beta.weight +create_tensor: loading tensor blk.4.ssm_alpha.weight +create_tensor: loading tensor blk.4.ssm_norm.weight +create_tensor: loading tensor blk.4.ssm_out.weight +create_tensor: loading tensor blk.4.ffn_gate.weight +create_tensor: loading tensor blk.4.ffn_down.weight +create_tensor: loading tensor blk.4.ffn_up.weight +create_tensor: loading tensor blk.5.attn_norm.weight +create_tensor: loading tensor blk.5.post_attention_norm.weight +create_tensor: loading tensor blk.5.attn_qkv.weight +create_tensor: loading tensor blk.5.attn_gate.weight +create_tensor: loading tensor blk.5.ssm_conv1d.weight +create_tensor: loading tensor blk.5.ssm_dt.bias +create_tensor: loading tensor blk.5.ssm_a +create_tensor: loading tensor blk.5.ssm_beta.weight +create_tensor: loading tensor blk.5.ssm_alpha.weight +create_tensor: loading tensor blk.5.ssm_norm.weight +create_tensor: loading tensor blk.5.ssm_out.weight +create_tensor: loading tensor blk.5.ffn_gate.weight +create_tensor: loading tensor blk.5.ffn_down.weight +create_tensor: loading tensor blk.5.ffn_up.weight +create_tensor: loading tensor blk.6.attn_norm.weight +create_tensor: loading tensor blk.6.post_attention_norm.weight +create_tensor: loading tensor blk.6.attn_qkv.weight +create_tensor: loading tensor blk.6.attn_gate.weight +create_tensor: loading tensor blk.6.ssm_conv1d.weight +create_tensor: loading tensor blk.6.ssm_dt.bias +create_tensor: loading tensor blk.6.ssm_a +create_tensor: loading tensor blk.6.ssm_beta.weight +create_tensor: loading tensor blk.6.ssm_alpha.weight +create_tensor: loading tensor blk.6.ssm_norm.weight +create_tensor: loading tensor blk.6.ssm_out.weight +create_tensor: loading tensor blk.6.ffn_gate.weight +create_tensor: loading tensor blk.6.ffn_down.weight +create_tensor: loading tensor blk.6.ffn_up.weight +create_tensor: loading tensor blk.7.attn_norm.weight +create_tensor: loading tensor blk.7.post_attention_norm.weight +create_tensor: loading tensor blk.7.attn_q.weight +create_tensor: loading tensor blk.7.attn_k.weight +create_tensor: loading tensor blk.7.attn_v.weight +create_tensor: loading tensor blk.7.attn_output.weight +create_tensor: loading tensor blk.7.attn_q_norm.weight +create_tensor: loading tensor blk.7.attn_k_norm.weight +create_tensor: loading tensor blk.7.ffn_gate.weight +create_tensor: loading tensor blk.7.ffn_down.weight +create_tensor: loading tensor blk.7.ffn_up.weight +create_tensor: loading tensor blk.8.attn_norm.weight +create_tensor: loading tensor blk.8.post_attention_norm.weight +create_tensor: loading tensor blk.8.attn_qkv.weight +create_tensor: loading tensor blk.8.attn_gate.weight +create_tensor: loading tensor blk.8.ssm_conv1d.weight +create_tensor: loading tensor blk.8.ssm_dt.bias +create_tensor: loading tensor blk.8.ssm_a +create_tensor: loading tensor blk.8.ssm_beta.weight +create_tensor: loading tensor blk.8.ssm_alpha.weight +create_tensor: loading tensor blk.8.ssm_norm.weight +create_tensor: loading tensor blk.8.ssm_out.weight +create_tensor: loading tensor blk.8.ffn_gate.weight +create_tensor: loading tensor blk.8.ffn_down.weight +create_tensor: loading tensor blk.8.ffn_up.weight +create_tensor: loading tensor blk.9.attn_norm.weight +create_tensor: loading tensor blk.9.post_attention_norm.weight +create_tensor: loading tensor blk.9.attn_qkv.weight +create_tensor: loading tensor blk.9.attn_gate.weight +create_tensor: loading tensor blk.9.ssm_conv1d.weight +create_tensor: loading tensor blk.9.ssm_dt.bias +create_tensor: loading tensor blk.9.ssm_a +create_tensor: loading tensor blk.9.ssm_beta.weight +create_tensor: loading tensor blk.9.ssm_alpha.weight +create_tensor: loading tensor blk.9.ssm_norm.weight +create_tensor: loading tensor blk.9.ssm_out.weight +create_tensor: loading tensor blk.9.ffn_gate.weight +create_tensor: loading tensor blk.9.ffn_down.weight +create_tensor: loading tensor blk.9.ffn_up.weight +create_tensor: loading tensor blk.10.attn_norm.weight +create_tensor: loading tensor blk.10.post_attention_norm.weight +create_tensor: loading tensor blk.10.attn_qkv.weight +create_tensor: loading tensor blk.10.attn_gate.weight +create_tensor: loading tensor blk.10.ssm_conv1d.weight +create_tensor: loading tensor blk.10.ssm_dt.bias +create_tensor: loading tensor blk.10.ssm_a +create_tensor: loading tensor blk.10.ssm_beta.weight +create_tensor: loading tensor blk.10.ssm_alpha.weight +create_tensor: loading tensor blk.10.ssm_norm.weight +create_tensor: loading tensor blk.10.ssm_out.weight +create_tensor: loading tensor blk.10.ffn_gate.weight +create_tensor: loading tensor blk.10.ffn_down.weight +create_tensor: loading tensor blk.10.ffn_up.weight +create_tensor: loading tensor blk.11.attn_norm.weight +create_tensor: loading tensor blk.11.post_attention_norm.weight +create_tensor: loading tensor blk.11.attn_q.weight +create_tensor: loading tensor blk.11.attn_k.weight +create_tensor: loading tensor blk.11.attn_v.weight +create_tensor: loading tensor blk.11.attn_output.weight +create_tensor: loading tensor blk.11.attn_q_norm.weight +create_tensor: loading tensor blk.11.attn_k_norm.weight +create_tensor: loading tensor blk.11.ffn_gate.weight +create_tensor: loading tensor blk.11.ffn_down.weight +create_tensor: loading tensor blk.11.ffn_up.weight +create_tensor: loading tensor blk.12.attn_norm.weight +create_tensor: loading tensor blk.12.post_attention_norm.weight +create_tensor: loading tensor blk.12.attn_qkv.weight +create_tensor: loading tensor blk.12.attn_gate.weight +create_tensor: loading tensor blk.12.ssm_conv1d.weight +create_tensor: loading tensor blk.12.ssm_dt.bias +create_tensor: loading tensor blk.12.ssm_a +create_tensor: loading tensor blk.12.ssm_beta.weight +create_tensor: loading tensor blk.12.ssm_alpha.weight +create_tensor: loading tensor blk.12.ssm_norm.weight +create_tensor: loading tensor blk.12.ssm_out.weight +create_tensor: loading tensor blk.12.ffn_gate.weight +create_tensor: loading tensor blk.12.ffn_down.weight +create_tensor: loading tensor blk.12.ffn_up.weight +create_tensor: loading tensor blk.13.attn_norm.weight +create_tensor: loading tensor blk.13.post_attention_norm.weight +create_tensor: loading tensor blk.13.attn_qkv.weight +create_tensor: loading tensor blk.13.attn_gate.weight +create_tensor: loading tensor blk.13.ssm_conv1d.weight +create_tensor: loading tensor blk.13.ssm_dt.bias +create_tensor: loading tensor blk.13.ssm_a +create_tensor: loading tensor blk.13.ssm_beta.weight +create_tensor: loading tensor blk.13.ssm_alpha.weight +create_tensor: loading tensor blk.13.ssm_norm.weight +create_tensor: loading tensor blk.13.ssm_out.weight +create_tensor: loading tensor blk.13.ffn_gate.weight +create_tensor: loading tensor blk.13.ffn_down.weight +create_tensor: loading tensor blk.13.ffn_up.weight +create_tensor: loading tensor blk.14.attn_norm.weight +create_tensor: loading tensor blk.14.post_attention_norm.weight +create_tensor: loading tensor blk.14.attn_qkv.weight +create_tensor: loading tensor blk.14.attn_gate.weight +create_tensor: loading tensor blk.14.ssm_conv1d.weight +create_tensor: loading tensor blk.14.ssm_dt.bias +create_tensor: loading tensor blk.14.ssm_a +create_tensor: loading tensor blk.14.ssm_beta.weight +create_tensor: loading tensor blk.14.ssm_alpha.weight +create_tensor: loading tensor blk.14.ssm_norm.weight +create_tensor: loading tensor blk.14.ssm_out.weight +create_tensor: loading tensor blk.14.ffn_gate.weight +create_tensor: loading tensor blk.14.ffn_down.weight +create_tensor: loading tensor blk.14.ffn_up.weight +create_tensor: loading tensor blk.15.attn_norm.weight +create_tensor: loading tensor blk.15.post_attention_norm.weight +create_tensor: loading tensor blk.15.attn_q.weight +create_tensor: loading tensor blk.15.attn_k.weight +create_tensor: loading tensor blk.15.attn_v.weight +create_tensor: loading tensor blk.15.attn_output.weight +create_tensor: loading tensor blk.15.attn_q_norm.weight +create_tensor: loading tensor blk.15.attn_k_norm.weight +create_tensor: loading tensor blk.15.ffn_gate.weight +create_tensor: loading tensor blk.15.ffn_down.weight +create_tensor: loading tensor blk.15.ffn_up.weight +create_tensor: loading tensor blk.16.attn_norm.weight +create_tensor: loading tensor blk.16.post_attention_norm.weight +create_tensor: loading tensor blk.16.attn_qkv.weight +create_tensor: loading tensor blk.16.attn_gate.weight +create_tensor: loading tensor blk.16.ssm_conv1d.weight +create_tensor: loading tensor blk.16.ssm_dt.bias +create_tensor: loading tensor blk.16.ssm_a +create_tensor: loading tensor blk.16.ssm_beta.weight +create_tensor: loading tensor blk.16.ssm_alpha.weight +create_tensor: loading tensor blk.16.ssm_norm.weight +create_tensor: loading tensor blk.16.ssm_out.weight +create_tensor: loading tensor blk.16.ffn_gate.weight +create_tensor: loading tensor blk.16.ffn_down.weight +create_tensor: loading tensor blk.16.ffn_up.weight +create_tensor: loading tensor blk.17.attn_norm.weight +create_tensor: loading tensor blk.17.post_attention_norm.weight +create_tensor: loading tensor blk.17.attn_qkv.weight +create_tensor: loading tensor blk.17.attn_gate.weight +create_tensor: loading tensor blk.17.ssm_conv1d.weight +create_tensor: loading tensor blk.17.ssm_dt.bias +create_tensor: loading tensor blk.17.ssm_a +create_tensor: loading tensor blk.17.ssm_beta.weight +create_tensor: loading tensor blk.17.ssm_alpha.weight +create_tensor: loading tensor blk.17.ssm_norm.weight +create_tensor: loading tensor blk.17.ssm_out.weight +create_tensor: loading tensor blk.17.ffn_gate.weight +create_tensor: loading tensor blk.17.ffn_down.weight +create_tensor: loading tensor blk.17.ffn_up.weight +create_tensor: loading tensor blk.18.attn_norm.weight +create_tensor: loading tensor blk.18.post_attention_norm.weight +create_tensor: loading tensor blk.18.attn_qkv.weight +create_tensor: loading tensor blk.18.attn_gate.weight +create_tensor: loading tensor blk.18.ssm_conv1d.weight +create_tensor: loading tensor blk.18.ssm_dt.bias +create_tensor: loading tensor blk.18.ssm_a +create_tensor: loading tensor blk.18.ssm_beta.weight +create_tensor: loading tensor blk.18.ssm_alpha.weight +create_tensor: loading tensor blk.18.ssm_norm.weight +create_tensor: loading tensor blk.18.ssm_out.weight +create_tensor: loading tensor blk.18.ffn_gate.weight +create_tensor: loading tensor blk.18.ffn_down.weight +create_tensor: loading tensor blk.18.ffn_up.weight +create_tensor: loading tensor blk.19.attn_norm.weight +create_tensor: loading tensor blk.19.post_attention_norm.weight +create_tensor: loading tensor blk.19.attn_q.weight +create_tensor: loading tensor blk.19.attn_k.weight +create_tensor: loading tensor blk.19.attn_v.weight +create_tensor: loading tensor blk.19.attn_output.weight +create_tensor: loading tensor blk.19.attn_q_norm.weight +create_tensor: loading tensor blk.19.attn_k_norm.weight +create_tensor: loading tensor blk.19.ffn_gate.weight +create_tensor: loading tensor blk.19.ffn_down.weight +create_tensor: loading tensor blk.19.ffn_up.weight +create_tensor: loading tensor blk.20.attn_norm.weight +create_tensor: loading tensor blk.20.post_attention_norm.weight +create_tensor: loading tensor blk.20.attn_qkv.weight +create_tensor: loading tensor blk.20.attn_gate.weight +create_tensor: loading tensor blk.20.ssm_conv1d.weight +create_tensor: loading tensor blk.20.ssm_dt.bias +create_tensor: loading tensor blk.20.ssm_a +create_tensor: loading tensor blk.20.ssm_beta.weight +create_tensor: loading tensor blk.20.ssm_alpha.weight +create_tensor: loading tensor blk.20.ssm_norm.weight +create_tensor: loading tensor blk.20.ssm_out.weight +create_tensor: loading tensor blk.20.ffn_gate.weight +create_tensor: loading tensor blk.20.ffn_down.weight +create_tensor: loading tensor blk.20.ffn_up.weight +create_tensor: loading tensor blk.21.attn_norm.weight +create_tensor: loading tensor blk.21.post_attention_norm.weight +create_tensor: loading tensor blk.21.attn_qkv.weight +create_tensor: loading tensor blk.21.attn_gate.weight +create_tensor: loading tensor blk.21.ssm_conv1d.weight +create_tensor: loading tensor blk.21.ssm_dt.bias +create_tensor: loading tensor blk.21.ssm_a +create_tensor: loading tensor blk.21.ssm_beta.weight +create_tensor: loading tensor blk.21.ssm_alpha.weight +create_tensor: loading tensor blk.21.ssm_norm.weight +create_tensor: loading tensor blk.21.ssm_out.weight +create_tensor: loading tensor blk.21.ffn_gate.weight +create_tensor: loading tensor blk.21.ffn_down.weight +create_tensor: loading tensor blk.21.ffn_up.weight +create_tensor: loading tensor blk.22.attn_norm.weight +create_tensor: loading tensor blk.22.post_attention_norm.weight +create_tensor: loading tensor blk.22.attn_qkv.weight +create_tensor: loading tensor blk.22.attn_gate.weight +create_tensor: loading tensor blk.22.ssm_conv1d.weight +create_tensor: loading tensor blk.22.ssm_dt.bias +create_tensor: loading tensor blk.22.ssm_a +create_tensor: loading tensor blk.22.ssm_beta.weight +create_tensor: loading tensor blk.22.ssm_alpha.weight +create_tensor: loading tensor blk.22.ssm_norm.weight +create_tensor: loading tensor blk.22.ssm_out.weight +create_tensor: loading tensor blk.22.ffn_gate.weight +create_tensor: loading tensor blk.22.ffn_down.weight +create_tensor: loading tensor blk.22.ffn_up.weight +create_tensor: loading tensor blk.23.attn_norm.weight +create_tensor: loading tensor blk.23.post_attention_norm.weight +create_tensor: loading tensor blk.23.attn_q.weight +create_tensor: loading tensor blk.23.attn_k.weight +create_tensor: loading tensor blk.23.attn_v.weight +create_tensor: loading tensor blk.23.attn_output.weight +create_tensor: loading tensor blk.23.attn_q_norm.weight +create_tensor: loading tensor blk.23.attn_k_norm.weight +create_tensor: loading tensor blk.23.ffn_gate.weight +create_tensor: loading tensor blk.23.ffn_down.weight +create_tensor: loading tensor blk.23.ffn_up.weight +create_tensor: loading tensor blk.24.attn_norm.weight +create_tensor: loading tensor blk.24.post_attention_norm.weight +create_tensor: loading tensor blk.24.attn_qkv.weight +create_tensor: loading tensor blk.24.attn_gate.weight +create_tensor: loading tensor blk.24.ssm_conv1d.weight +create_tensor: loading tensor blk.24.ssm_dt.bias +create_tensor: loading tensor blk.24.ssm_a +create_tensor: loading tensor blk.24.ssm_beta.weight +create_tensor: loading tensor blk.24.ssm_alpha.weight +create_tensor: loading tensor blk.24.ssm_norm.weight +create_tensor: loading tensor blk.24.ssm_out.weight +create_tensor: loading tensor blk.24.ffn_gate.weight +create_tensor: loading tensor blk.24.ffn_down.weight +create_tensor: loading tensor blk.24.ffn_up.weight +create_tensor: loading tensor blk.25.attn_norm.weight +create_tensor: loading tensor blk.25.post_attention_norm.weight +create_tensor: loading tensor blk.25.attn_qkv.weight +create_tensor: loading tensor blk.25.attn_gate.weight +create_tensor: loading tensor blk.25.ssm_conv1d.weight +create_tensor: loading tensor blk.25.ssm_dt.bias +create_tensor: loading tensor blk.25.ssm_a +create_tensor: loading tensor blk.25.ssm_beta.weight +create_tensor: loading tensor blk.25.ssm_alpha.weight +create_tensor: loading tensor blk.25.ssm_norm.weight +create_tensor: loading tensor blk.25.ssm_out.weight +create_tensor: loading tensor blk.25.ffn_gate.weight +create_tensor: loading tensor blk.25.ffn_down.weight +create_tensor: loading tensor blk.25.ffn_up.weight +create_tensor: loading tensor blk.26.attn_norm.weight +create_tensor: loading tensor blk.26.post_attention_norm.weight +create_tensor: loading tensor blk.26.attn_qkv.weight +create_tensor: loading tensor blk.26.attn_gate.weight +create_tensor: loading tensor blk.26.ssm_conv1d.weight +create_tensor: loading tensor blk.26.ssm_dt.bias +create_tensor: loading tensor blk.26.ssm_a +create_tensor: loading tensor blk.26.ssm_beta.weight +create_tensor: loading tensor blk.26.ssm_alpha.weight +create_tensor: loading tensor blk.26.ssm_norm.weight +create_tensor: loading tensor blk.26.ssm_out.weight +create_tensor: loading tensor blk.26.ffn_gate.weight +create_tensor: loading tensor blk.26.ffn_down.weight +create_tensor: loading tensor blk.26.ffn_up.weight +create_tensor: loading tensor blk.27.attn_norm.weight +create_tensor: loading tensor blk.27.post_attention_norm.weight +create_tensor: loading tensor blk.27.attn_q.weight +create_tensor: loading tensor blk.27.attn_k.weight +create_tensor: loading tensor blk.27.attn_v.weight +create_tensor: loading tensor blk.27.attn_output.weight +create_tensor: loading tensor blk.27.attn_q_norm.weight +create_tensor: loading tensor blk.27.attn_k_norm.weight +create_tensor: loading tensor blk.27.ffn_gate.weight +create_tensor: loading tensor blk.27.ffn_down.weight +create_tensor: loading tensor blk.27.ffn_up.weight +create_tensor: loading tensor blk.28.attn_norm.weight +create_tensor: loading tensor blk.28.post_attention_norm.weight +create_tensor: loading tensor blk.28.attn_qkv.weight +create_tensor: loading tensor blk.28.attn_gate.weight +create_tensor: loading tensor blk.28.ssm_conv1d.weight +create_tensor: loading tensor blk.28.ssm_dt.bias +create_tensor: loading tensor blk.28.ssm_a +create_tensor: loading tensor blk.28.ssm_beta.weight +create_tensor: loading tensor blk.28.ssm_alpha.weight +create_tensor: loading tensor blk.28.ssm_norm.weight +create_tensor: loading tensor blk.28.ssm_out.weight +create_tensor: loading tensor blk.28.ffn_gate.weight +create_tensor: loading tensor blk.28.ffn_down.weight +create_tensor: loading tensor blk.28.ffn_up.weight +create_tensor: loading tensor blk.29.attn_norm.weight +create_tensor: loading tensor blk.29.post_attention_norm.weight +create_tensor: loading tensor blk.29.attn_qkv.weight +create_tensor: loading tensor blk.29.attn_gate.weight +create_tensor: loading tensor blk.29.ssm_conv1d.weight +create_tensor: loading tensor blk.29.ssm_dt.bias +create_tensor: loading tensor blk.29.ssm_a +create_tensor: loading tensor blk.29.ssm_beta.weight +create_tensor: loading tensor blk.29.ssm_alpha.weight +create_tensor: loading tensor blk.29.ssm_norm.weight +create_tensor: loading tensor blk.29.ssm_out.weight +create_tensor: loading tensor blk.29.ffn_gate.weight +create_tensor: loading tensor blk.29.ffn_down.weight +create_tensor: loading tensor blk.29.ffn_up.weight +create_tensor: loading tensor blk.30.attn_norm.weight +create_tensor: loading tensor blk.30.post_attention_norm.weight +create_tensor: loading tensor blk.30.attn_qkv.weight +create_tensor: loading tensor blk.30.attn_gate.weight +create_tensor: loading tensor blk.30.ssm_conv1d.weight +create_tensor: loading tensor blk.30.ssm_dt.bias +create_tensor: loading tensor blk.30.ssm_a +create_tensor: loading tensor blk.30.ssm_beta.weight +create_tensor: loading tensor blk.30.ssm_alpha.weight +create_tensor: loading tensor blk.30.ssm_norm.weight +create_tensor: loading tensor blk.30.ssm_out.weight +create_tensor: loading tensor blk.30.ffn_gate.weight +create_tensor: loading tensor blk.30.ffn_down.weight +create_tensor: loading tensor blk.30.ffn_up.weight +create_tensor: loading tensor blk.31.attn_norm.weight +create_tensor: loading tensor blk.31.post_attention_norm.weight +create_tensor: loading tensor blk.31.attn_q.weight +create_tensor: loading tensor blk.31.attn_k.weight +create_tensor: loading tensor blk.31.attn_v.weight +create_tensor: loading tensor blk.31.attn_output.weight +create_tensor: loading tensor blk.31.attn_q_norm.weight +create_tensor: loading tensor blk.31.attn_k_norm.weight +create_tensor: loading tensor blk.31.ffn_gate.weight +create_tensor: loading tensor blk.31.ffn_down.weight +create_tensor: loading tensor blk.31.ffn_up.weight +done_getting_tensors: tensor 'token_embd.weight' (q6_K) (and 426 others) cannot be used with preferred buffer type CPU_REPACK, using CPU instead -load_tensors: offloading output layer to GPU +load_tensors: offloading 31 repeating layers to GPU +load_tensors: offloaded 33/33 layers to GPU +load_tensors: CPU_Mapped model buffer size = 3293.12 MiB +...................................................................................... +common_init_result: added <|endoftext|> logit bias = -inf +common_init_result: added <|im_end|> logit bias = -inf +common_init_result: added <|fim_pad|> logit bias = -inf +common_init_result: added <|repo_name|> logit bias = -inf +common_init_result: added <|file_sep|> logit bias = -inf +llama_context: constructing llama_context +llama_context: n_seq_max = 1 +llama_context: n_ctx = 4096 +llama_context: n_ctx_seq = 4096 +llama_context: n_batch = 2048 +llama_context: n_ubatch = 512 +llama_context: causal_attn = 1 +llama_context: flash_attn = enabled +llama_context: kv_unified = false +llama_context: freq_base = 10000000.0 +llama_context: freq_scale = 1 +llama_context: n_ctx_seq (4096) < n_ctx_train (262144) -- the full capacity of the model will not be utilized +set_abort_callback: call +llama_context: CPU output buffer size = 0.95 MiB +llama_kv_cache: layer 0: filtered +llama_kv_cache: layer 1: filtered +llama_kv_cache: layer 2: filtered +llama_kv_cache: layer 3: dev = CPU +llama_kv_cache: layer 4: filtered +llama_kv_cache: layer 5: filtered +llama_kv_cache: layer 6: filtered +llama_kv_cache: layer 7: dev = CPU +llama_kv_cache: layer 8: filtered +llama_kv_cache: layer 9: filtered +llama_kv_cache: layer 10: filtered +llama_kv_cache: layer 11: dev = CPU +llama_kv_cache: layer 12: filtered +llama_kv_cache: layer 13: filtered +llama_kv_cache: layer 14: filtered +llama_kv_cache: layer 15: dev = CPU +llama_kv_cache: layer 16: filtered +llama_kv_cache: layer 17: filtered +llama_kv_cache: layer 18: filtered +llama_kv_cache: layer 19: dev = CPU +llama_kv_cache: layer 20: filtered +llama_kv_cache: layer 21: filtered +llama_kv_cache: layer 22: filtered +llama_kv_cache: layer 23: dev = CPU +llama_kv_cache: layer 24: filtered +llama_kv_cache: layer 25: filtered +llama_kv_cache: layer 26: filtered +llama_kv_cache: layer 27: dev = CPU +llama_kv_cache: layer 28: filtered +llama_kv_cache: layer 29: filtered +llama_kv_cache: layer 30: filtered +llama_kv_cache: layer 31: dev = CPU +llama_kv_cache: CPU KV buffer size = 128.00 MiB \llama_kv_cache: size = 128.00 MiB ( 4096 cells, 8 layers, 1/1 seqs), K (f16): 64.00 MiB, V (f16): 64.00 MiB +llama_memory_recurrent, layer 0: dev = CPU +llama_memory_recurrent, layer 1: dev = CPU +llama_memory_recurrent, layer 2: dev = CPU +llama_memory_recurrent: layer 3: skipped +llama_memory_recurrent, layer 4: dev = CPU +llama_memory_recurrent, layer 5: dev = CPU +llama_memory_recurrent, layer 6: dev = CPU +llama_memory_recurrent: layer 7: skipped +llama_memory_recurrent, layer 8: dev = CPU +llama_memory_recurrent, layer 9: dev = CPU +llama_memory_recurrent, layer 10: dev = CPU +llama_memory_recurrent: layer 11: skipped +llama_memory_recurrent, layer 12: dev = CPU +llama_memory_recurrent, layer 13: dev = CPU +llama_memory_recurrent, layer 14: dev = CPU +llama_memory_recurrent: layer 15: skipped +llama_memory_recurrent, layer 16: dev = CPU +llama_memory_recurrent, layer 17: dev = CPU +llama_memory_recurrent, layer 18: dev = CPU +llama_memory_recurrent: layer 19: skipped +llama_memory_recurrent, layer 20: dev = CPU +llama_memory_recurrent, layer 21: dev = CPU +llama_memory_recurrent, layer 22: dev = CPU +llama_memory_recurrent: layer 23: skipped +llama_memory_recurrent, layer 24: dev = CPU +llama_memory_recurrent, layer 25: dev = CPU +llama_memory_recurrent, layer 26: dev = CPU +llama_memory_recurrent: layer 27: skipped +llama_memory_recurrent, layer 28: dev = CPU +llama_memory_recurrent, layer 29: dev = CPU +llama_memory_recurrent, layer 30: dev = CPU +llama_memory_recurrent: layer 31: skipped +llama_memory_recurrent: CPU RS buffer size = 50.25 MiB +llama_memory_recurrent: size = 50.25 MiB ( 1 cells, 32 layers, 1 seqs), R (f32): 2.25 MiB, S (f32): 48.00 MiB +llama_context: enumerating backends +llama_context: backend_ptrs.size() = 1 +sched_reserve: reserving ... +sched_reserve: max_nodes = 20480 +sched_reserve: reserving full memory module +sched_reserve: worst-case: n_tokens = 512, n_seqs = 1, n_outputs = 1 +sched_reserve: resolving fused Gated Delta Net support: +graph_reserve: reserving a graph for ubatch with n_tokens = 1, n_seqs = 1, n_outputs = 1 +sched_reserve: fused Gated Delta Net (autoregressive) enabled +graph_reserve: reserving a graph for ubatch with n_tokens = 16, n_seqs = 1, n_outputs = 16 +sched_reserve: fused Gated Delta Net (chunked) enabled +graph_reserve: reserving a graph for ubatch with n_tokens = 512, n_seqs = 1, n_outputs = 512 +graph_reserve: reserving a graph for ubatch with n_tokens = 1, n_seqs = 1, n_outputs = 1 +graph_reserve: reserving a graph for ubatch with n_tokens = 512, n_seqs = 1, n_outputs = 512 +sched_reserve: CPU compute buffer size = 495.00 MiB +sched_reserve: graph nodes = 1833 +sched_reserve: graph splits = 1 +sched_reserve: reserve took 5.17 ms, sched copies = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) +set_warmup: value = 1 \set_warmup: value = 0 +srv load_model: initializing slots, n_slots = 1 |common_speculative_is_compat: the target context does not support partial sequence removal /srv load_model: speculative decoding not supported by this context +slot load_model: id 0 | task -1 | new slot, n_ctx = 4096 +slot reset: id 0 | task -1 | +srv load_model: prompt cache is enabled, size limit: 8192 MiB +srv load_model: use `--cache-ram 0` to disable the prompt cache +srv load_model: for more info see https://github.com/ggml-org/llama.cpp/pull/16391 +common_chat_templates_apply_jinja: using differential autoparser +=== Starting differential analysis === +Phase 1: Reasoning analysis +Phase 2: Content analysis +Phase 3: Tool call analysis + +--- Reasoning & Content Structure --- +reasoning_mode: TAG_BASED +reasoning_start: '' +reasoning_end: '' +content_mode: PLAIN +content_start: '' +content_end: '' + +--- Tool Call Structure --- +tool_mode: JSON_NATIVE +supports_tools: true +supports_parallel_calls: true +tool_section_start: ' +' +tool_section_end: '' +per_call_start: '' +per_call_end: '' +func_name_prefix: '' +func_name_suffix: '' +func_close: '' +arg_name_prefix: '' +arg_name_suffix: '' +arg_value_prefix: '' +arg_value_suffix: '' +name_field: 'name' +args_field: 'arguments' +id_field: '' +gen_id_field: '' +parameter_order: 'name, arguments' +=== Differential analysis complete === +common_chat_templates_apply_jinja: generated parser: +Sequence(Literal(<|im_start|>assistant +), Epsilon, Tag(content, Until()), End) + +parser generation prompt: <|im_start|>assistant + + +init: chat template, example_format: '<|im_start|>system +You are a helpful assistant<|im_end|> +<|im_start|>user +Hello<|im_end|> +<|im_start|>assistant +Hi there<|im_end|> +<|im_start|>user +How are you?<|im_end|> +<|im_start|>assistant + +' +common_chat_templates_apply_jinja: using differential autoparser +=== Starting differential analysis === +Phase 1: Reasoning analysis +Phase 2: Content analysis +Phase 3: Tool call analysis + +--- Reasoning & Content Structure --- +reasoning_mode: TAG_BASED +reasoning_start: '' +reasoning_end: '' +content_mode: PLAIN +content_start: '' +content_end: '' + +--- Tool Call Structure --- +tool_mode: JSON_NATIVE +supports_tools: true +supports_parallel_calls: true +tool_section_start: ' +' +tool_section_end: '' per_call_start: '' +per_call_end: '' +func_name_prefix: '' +func_name_suffix: '' +func_close: '' +arg_name_prefix: '' + arg_name_suffix: '' +arg_value_prefix: '' +arg_value_suffix: '' +name_field: 'name' + +args_field: 'arguments' +id_field: '' +gen_id_field: '' +parameter_order: 'name, arguments' +=== Differential analysis complete === +common_chat_templates_apply_jinja: generated parser: +Sequence(Literal(<|im_start|>assistant +), Repetition(Sequence(Literal(), Tag(reasoning, Until()), Literal(), Space), 0, 1), Tag(content, Until()), End) + +parser generation prompt: <|im_start|>assistant + + +srv init: init: chat template, thinking = 1 + + +▄▄ ▄▄ +██ ██ +██ ██ ▀▀█▄ ███▄███▄ ▀▀█▄ ▄████ ████▄ ████▄ +██ ██ ▄█▀██ ██ ██ ██ ▄█▀██ ██ ██ ██ ██ ██ +██ ██ ▀█▄██ ██ ██ ██ ▀█▄██ ██ ▀████ ████▀ ████▀ + ██ ██ + ▀▀ ▀▀ + +build : b8533-0fac87b15 +model : Qwen3.5-4B-Abliterated-Claude-4.6-Opus-Reasoning-Distilled.i1-Q6_K.gguf +modalities : text + +available commands: + /exit or Ctrl+C stop or exit + /regen regenerate the last response + /clear clear the chat history + /read add a text file + +que start_loop: processing new tasks +que start_loop: update slots +srv update_slots: all slots are idle +que start_loop: waiting for new tasks + +> Ol�! Me diga quem voc� � em uma frase. + +common_chat_templates_apply_jinja: using differential autoparser +=== Starting differential analysis === +Phase 1: Reasoning analysis +Phase 2: Content analysis +Phase 3: Tool call analysis + +--- Reasoning & Content Structure --- +reasoning_mode: TAG_BASED +reasoning_start: '' +reasoning_end: '' +content_mode: PLAIN +content_start: '' +content_end: '' + +--- Tool Call Structure --- +tool_mode: JSON_NATIVE +supports_tools: true +supports_parallel_calls: true +tool_section_start: ' +' +tool_section_end: '' +per_call_start: '' +per_call_end: '' +func_name_prefix: '' +func_name_suffix: '' +func_close: '' +arg_name_prefix: '' +arg_name_suffix: '' +arg_value_prefix: '' +arg_value_suffix: '' +name_field: 'name' +args_field: 'arguments' +id_field: '' +gen_id_field: '' +parameter_order: 'name, arguments' +=== Differential analysis complete === +common_chat_templates_apply_jinja: generated parser: +Sequence(Literal(<|im_start|>assistant +), Repetition(Sequence(Literal(), Tag(reasoning, Until()), Literal(), Space), 0, 1), Tag(content, Until()), End) + +parser generation prompt: <|im_start|>assistant + + +common_chat_templates_apply_jinja: using differential autoparser +=== Starting differential analysis === +Phase 1: Reasoning analysis +Phase 2: Content analysis +Phase 3: Tool call analysis + +--- Reasoning & Content Structure --- +reasoning_mode: TAG_BASED +reasoning_start: '' +reasoning_end: '' +content_mode: PLAIN +content_start: '' +content_end: '' + +--- Tool Call Structure --- +tool_mode: JSON_NATIVE +supports_tools: true +supports_parallel_calls: true +tool_section_start: ' +' +tool_section_end: '' +per_call_start: '' +per_call_end: '' +func_name_prefix: '' +func_name_suffix: '' +func_close: '' +arg_name_prefix: '' +arg_name_suffix: '' +arg_value_prefix: '' +arg_value_suffix: '' +name_field: 'name' +args_field: 'arguments' +id_field: '' +gen_id_field: '' +parameter_order: 'name, arguments' +=== Differential analysis complete === +common_chat_templates_apply_jinja: generated parser: +Sequence(Literal(<|im_start|>assistant +), Repetition(Sequence(Literal(), Tag(reasoning, Until()), Literal(), Space), 0, 1), Tag(content, Until()), End) + +parser generation prompt: <|im_start|>assistant + + +res add_waiting_: add task 0 to waiting list. current waiting = 0 (before add) +que post: new task, id = 0, front = 0 +/srv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +que start_loop: processing task, id = 1 +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +que post: new task, id = 2, front = 0 +slot update_slots: id 0 | task 0 | n_tokens = 19, memory_seq_rm [19, end) +slot init_sampler: id 0 | task 0 | init sampler, took 0.00 ms, tokens: text = 23, total = 23 +slot update_slots: id 0 | task 0 | prompt processing done, n_tokens = 23, batch.n_tokens = 4 +srv update_slots: decoding batch, n_tokens = 4 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 -res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 1, n_remaining = 79, next token: 9764 'Let' +srv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +srv update_chat_: Parsing chat message: Let que start_loop: processing task, id = 2 +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let +que post: new task, id = 3, front = 0 +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 24, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 + adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 [Start thinking] + +Letres send: sending result for task id = 0 + meres send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 2, n_remaining = 78, next token: 728 ' me' +srv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +srv update_chat_: Parsing chat message: Let me +que start_loop: processing task, id = 3 +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me +que post: new task, id = 4, front = 0 +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 25, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 3, n_remaining = 77, next token: 1683 ' think' +srv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +srv update_chat_: Parsing chat message: Let me think +que start_loop: processing task, id = 4 +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think +que post: new task, id = 5, front = 0 +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 26, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 + thinkadapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 4, n_remaining = 76, next token: 1472 ' through' +srv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +que start_loop: processing task, id = 5 +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +srv update_chat_: Parsing chat message: Let me think through +que post: new task, id = 6, front = 0 +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 27, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 + throughres send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 5, n_remaining = 75, next token: 411 ' this' + thissrv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +que start_loop: processing task, id = 6 +srv update_chat_: Parsing chat message: Let me think through this +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +que post: new task, id = 7, front = 0 +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 28, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 6, n_remaining = 74, next token: 15060 ' carefully' +srv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks + carefullyque start_loop: processing task, id = 7 +srv update_chat_: Parsing chat message: Let me think through this carefully +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +que post: new task, id = 8, front = 0 +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 29, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 7, n_remaining = 73, next token: 13 '.' +srv update_slots: run slots completed +.que start_loop: waiting for new tasks +que start_loop: processing new tasks +que start_loop: processing task, id = 8 +srv update_chat_: Parsing chat message: Let me think through this carefully. +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +que post: new task, id = 9, front = 0 +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 30, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 8, n_remaining = 72, next token: 271 ' + +' + + +srv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +srv update_chat_: Parsing chat message: Let me think through this carefully. + + +que start_loop: processing task, id = 9 +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + + +que post: new task, id = 10, front = 0 +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 31, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 9, n_remaining = 71, next token: 760 'The' +Thesrv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +que start_loop: processing task, id = 10 +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +que post: new task, id = 11, front = 0 +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 32, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 + questionres send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 10, n_remaining = 70, next token: 3296 ' question' +srv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +que start_loop: processing task, id = 11 +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +que post: new task, id = 12, front = 0 +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 33, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 11, n_remaining = 69, next token: 56231 '/task' +/tasksrv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task +que start_loop: processing task, id = 12 +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +que post: new task, id = 13, front = 0 +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 34, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 12, n_remaining = 68, next token: 25 ':' +srv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +que start_loop: processing task, id = 13 +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: +:que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +que post: new task, id = 14, front = 0 +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 35, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 13, n_remaining = 67, next token: 11620 ' Ol' +srv update_slots: run slots completed +que start_loop: waiting for new tasks + Olque start_loop: processing new tasks +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol +que start_loop: processing task, id = 14 +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +que post: new task, id = 15, front = 0 +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 36, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 14, n_remaining = 66, next token: 5513 '�' +srv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol� +que start_loop: processing task, id = 15 +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol� +que post: new task, id = 16, front = 0 +�slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 37, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 15, n_remaining = 65, next token: 0 '!' +srv update_slots: run slots completed +que start_loop: waiting for new tasks +!que start_loop: processing new tasks +que start_loop: processing task, id = 16 +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +que post: new task, id = 17, front = 0 +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 38, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 + Meres send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 16, n_remaining = 64, next token: 2088 ' Me' +srv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +que start_loop: processing task, id = 17 +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +que post: new task, id = 18, front = 0 +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 39, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 17, n_remaining = 63, next token: 210906 ' diga' +srv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +que start_loop: processing task, id = 18 +que start_loop: update slots + digasrv update_slots: posting NEXT_RESPONSE +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga +que post: new task, id = 19, front = 0 +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 40, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 18, n_remaining = 62, next token: 55675 ' quem' + quemsrv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem +que start_loop: processing task, id = 19 +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem +que post: new task, id = 20, front = 0 +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 41, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 19, n_remaining = 61, next token: 11643 ' voc' + vocsrv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +que start_loop: processing task, id = 20 +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc +que post: new task, id = 21, front = 0 +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 42, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +�slot process_toke: id 0 | task 0 | n_decoded = 20, n_remaining = 60, next token: 5513 '�' +srv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +que start_loop: processing task, id = 21 +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +que post: new task, id = 22, front = 0 +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 43, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 21, n_remaining = 59, next token: 28373 ' �' +srv update_slots: run slots completed +que start_loop: waiting for new tasks + �que start_loop: processing new tasks +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � +que start_loop: processing task, id = 22 +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � +que post: new task, id = 23, front = 0 +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 44, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 22, n_remaining = 58, next token: 945 ' em' +srv update_slots: run slots completed + emque start_loop: waiting for new tasks +que start_loop: processing new tasks +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em +que start_loop: processing task, id = 23 +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em +que post: new task, id = 24, front = 0 +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 45, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 23, n_remaining = 57, next token: 10293 ' uma' +srv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks + umasrv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma +que start_loop: processing task, id = 24 +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma +que post: new task, id = 25, front = 0 +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 46, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 24, n_remaining = 56, next token: 87222 ' frase' +srv update_slots: run slots completed + fraseque start_loop: waiting for new tasks +que start_loop: processing new tasks +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase +que start_loop: processing task, id = 25 +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase +que post: new task, id = 26, front = 0 +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 47, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 25, n_remaining = 55, next token: 13 '.' +srv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. +que start_loop: processing task, id = 26 +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. +que post: new task, id = 27, front = 0 +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 48, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +.set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 26, n_remaining = 54, next token: 271 ' + +' +srv update_slots: run slots completed +que start_loop: waiting for new tasks + + +que start_loop: processing new tasks +que start_loop: processing task, id = 27 +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + + +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +que post: new task, id = 28, front = 0 +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + + +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 49, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 27, n_remaining = 53, next token: 4888 'My' +srv update_slots: run slots completed +Myque start_loop: waiting for new tasks +que start_loop: processing new tasks +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My +que start_loop: processing task, id = 28 +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My +que post: new task, id = 29, front = 0 +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 50, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 28, n_remaining = 52, next token: 31626 ' reasoning' +srv update_slots: run slots completed + reasoningque start_loop: waiting for new tasks +que start_loop: processing new tasks +que start_loop: processing task, id = 29 +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +que post: new task, id = 30, front = 0 +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 51, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 29, n_remaining = 51, next token: 25 ':' +srv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: +:que start_loop: processing task, id = 30 +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: +que post: new task, id = 31, front = 0 +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 52, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 30, n_remaining = 50, next token: 179033 ' Quem' +srv update_slots: run slots completed + Quemque start_loop: waiting for new tasks +que start_loop: processing new tasks +que start_loop: processing task, id = 31 +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +que post: new task, id = 32, front = 0 +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 53, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 31, n_remaining = 49, next token: 5641 ' sou' +srv update_slots: run slots completed +que start_loop: waiting for new tasks + souque start_loop: processing new tasks +que start_loop: processing task, id = 32 +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou +que post: new task, id = 33, front = 0 +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 54, truncated = 0 +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 32, n_remaining = 48, next token: 15098 ' eu' + eusrv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu +que start_loop: processing task, id = 33 +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu +que post: new task, id = 34, front = 0 +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 55, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 + emres send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 33, n_remaining = 47, next token: 945 ' em' +srv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +que start_loop: processing task, id = 34 +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +que post: new task, id = 35, front = 0 +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 56, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 34, n_remaining = 46, next token: 10293 ' uma' +srv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma +que start_loop: processing task, id = 35 +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE + umaParsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma +que post: new task, id = 36, front = 0 +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 57, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 35, n_remaining = 45, next token: 87222 ' frase' +srv update_slots: run slots completed +que start_loop: waiting for new tasks + fraseque start_loop: processing new tasks +que start_loop: processing task, id = 36 +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +que post: new task, id = 37, front = 0 +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 58, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 36, n_remaining = 44, next token: 30 '?' +srv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? +que start_loop: processing task, id = 37 +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? +que post: new task, id = 38, front = 0 +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 59, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +?set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 37, n_remaining = 43, next token: 198 ' +' +srv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +que start_loop: processing task, id = 38 + +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +que post: new task, id = 39, front = 0 +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 60, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 38, n_remaining = 42, next token: 248069 '' +srv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +que start_loop: processing task, id = 39 +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +que post: new task, id = 40, front = 0 +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 61, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 39, n_remaining = 41, next token: 271 ' + +' +srv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + + +que start_loop: processing task, id = 40 +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +que post: new task, id = 41, front = 0 +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + + +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 62, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 40, n_remaining = 40, next token: 332 '**' +srv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +que start_loop: processing task, id = 41 + +[End thinking] + +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +** +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +que post: new task, id = 42, front = 0 +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +** +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 63, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +**res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 41, n_remaining = 39, next token: 2114 'Qu' +srv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +Quque start_loop: processing task, id = 42 +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Qu +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +que post: new task, id = 43, front = 0 +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Qu +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 64, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 42, n_remaining = 38, next token: 334 'em' +srv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem +que start_loop: processing task, id = 43 +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem +que post: new task, id = 44, front = 0 +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 65, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +emres send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 43, n_remaining = 37, next token: 5641 ' sou' + sousrv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou +que start_loop: processing task, id = 44 +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +que post: new task, id = 45, front = 0 +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 66, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 44, n_remaining = 36, next token: 15098 ' eu' +srv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +que start_loop: processing task, id = 45 +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +que post: new task, id = 46, front = 0 +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 67, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 + euset_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 45, n_remaining = 35, next token: 945 ' em' +srv update_slots: run slots completed + emque start_loop: waiting for new tasks +que start_loop: processing new tasks +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em +que start_loop: processing task, id = 46 +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em +que post: new task, id = 47, front = 0 +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 68, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 46, n_remaining = 34, next token: 10293 ' uma' + umasrv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +que start_loop: processing task, id = 47 +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma +que post: new task, id = 48, front = 0 +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 69, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 47, n_remaining = 33, next token: 87222 ' frase' + frasesrv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +que start_loop: processing task, id = 48 +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase +que post: new task, id = 49, front = 0 +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 70, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 48, n_remaining = 32, next token: 170286 '?**' +?**srv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +que start_loop: processing task, id = 49 +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +que post: new task, id = 50, front = 0 +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 71, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 49, n_remaining = 31, next token: 271 ' + +' +srv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +que start_loop: processing task, id = 50 + + +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + + +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +que post: new task, id = 51, front = 0 +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + + +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 72, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 50, n_remaining = 30, next token: 55509 'Sou' +srv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +que start_loop: processing task, id = 51 +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +que post: new task, id = 52, front = 0 +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou +Souslot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 73, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 51, n_remaining = 29, next token: 4297 ' um' +srv update_slots: run slots completed +que start_loop: waiting for new tasks + umque start_loop: processing new tasks +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um +que start_loop: processing task, id = 52 +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um +que post: new task, id = 53, front = 0 +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 74, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 52, n_remaining = 28, next token: 2972 ' **' +srv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +que start_loop: processing task, id = 53 + **que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um ** +que post: new task, id = 54, front = 0 +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 75, truncated = 0 +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um ** +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 53, n_remaining = 27, next token: 49250 'assist' +srv update_slots: run slots completed +assistque start_loop: waiting for new tasks +que start_loop: processing new tasks +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assist +que start_loop: processing task, id = 54 +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assist +que post: new task, id = 55, front = 0 +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 76, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 54, n_remaining = 26, next token: 6603 'ente' +entesrv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +que start_loop: processing task, id = 55 +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +que post: new task, id = 56, front = 0 +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 77, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 55, n_remaining = 25, next token: 401 ' de' +srv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +que start_loop: processing task, id = 56 +que start_loop: update slots +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de +srv update_slots: posting NEXT_RESPONSE +que post: new task, id = 57, front = 0 +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 78, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 + deres send: sending result for task id = 0 + inteligênciares send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 56, n_remaining = 24, next token: 235635 ' inteligência' +srv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência +que start_loop: processing task, id = 57 +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência +que post: new task, id = 58, front = 0 +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 79, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 57, n_remaining = 23, next token: 19820 ' artificial' +srv update_slots: run slots completed + artificialque start_loop: waiting for new tasks +que start_loop: processing new tasks +que start_loop: processing task, id = 58 +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial +que post: new task, id = 59, front = 0 +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 80, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +**,res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 58, n_remaining = 22, next token: 93868 '**,' +srv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +que start_loop: processing task, id = 59 +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial**, +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +que post: new task, id = 60, front = 0 +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial**, +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 81, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 59, n_remaining = 21, next token: 212482 ' desenvolvido' +srv update_slots: run slots completed + desenvolvidoque start_loop: waiting for new tasks +que start_loop: processing new tasks +que start_loop: processing task, id = 60 +que start_loop: update slots +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial**, desenvolvido +srv update_slots: posting NEXT_RESPONSE +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial**, desenvolvido +que post: new task, id = 61, front = 0 +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 82, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 60, n_remaining = 20, next token: 4018 ' por' + porsrv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial**, desenvolvido por +que start_loop: processing task, id = 61 +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +que post: new task, id = 62, front = 0 +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial**, desenvolvido por +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 83, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 61, n_remaining = 19, next token: 10293 ' uma' +srv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +que start_loop: processing task, id = 62 +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial**, desenvolvido por uma +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE + umaque post: new task, id = 63, front = 0 +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 84, truncated = 0 +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial**, desenvolvido por uma +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 62, n_remaining = 18, next token: 23828 ' grande' +srv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +que start_loop: processing task, id = 63 +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial**, desenvolvido por uma grande +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE + grandeParsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial**, desenvolvido por uma grande +que post: new task, id = 64, front = 0 +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 85, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 63, n_remaining = 17, next token: 31678 ' empresa' +srv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial**, desenvolvido por uma grande empresa +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial**, desenvolvido por uma grande empresa +que start_loop: processing task, id = 64 +que start_loop: update slots + empresasrv update_slots: posting NEXT_RESPONSE +que post: new task, id = 65, front = 0 +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 86, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 64, n_remaining = 16, next token: 401 ' de' +srv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +que start_loop: processing task, id = 65 +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial**, desenvolvido por uma grande empresa de +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +que post: new task, id = 66, front = 0 +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial**, desenvolvido por uma grande empresa de + deslot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 87, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 65, n_remaining = 15, next token: 173238 ' tecnologia' +srv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial**, desenvolvido por uma grande empresa de tecnologia +que start_loop: processing task, id = 66 +que start_loop: update slots + tecnologiasrv update_slots: posting NEXT_RESPONSE +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial**, desenvolvido por uma grande empresa de tecnologia +que post: new task, id = 67, front = 0 +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 88, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 66, n_remaining = 14, next token: 318 ' (' +srv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +que start_loop: processing task, id = 67 +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial**, desenvolvido por uma grande empresa de tecnologia ( + (que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +que post: new task, id = 68, front = 0 +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial**, desenvolvido por uma grande empresa de tecnologia ( +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 89, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 67, n_remaining = 13, next token: 41528 'prov' +srv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +que start_loop: processing task, id = 68 +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial**, desenvolvido por uma grande empresa de tecnologia (prov +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial**, desenvolvido por uma grande empresa de tecnologia (prov +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +que post: new task, id = 69, front = 0 +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 90, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +provadapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 68, n_remaining = 12, next token: 164861 'avelmente' +srv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +que start_loop: processing task, id = 69 +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial**, desenvolvido por uma grande empresa de tecnologia (provavelmente +que post: new task, id = 70, front = 0 +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial**, desenvolvido por uma grande empresa de tecnologia (provavelmente +avelmenteslot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 91, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 69, n_remaining = 11, next token: 264 ' a' +srv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks + aque start_loop: processing task, id = 70 +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial**, desenvolvido por uma grande empresa de tecnologia (provavelmente a +que post: new task, id = 71, front = 0 +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 92, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial**, desenvolvido por uma grande empresa de tecnologia (provavelmente a +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 70, n_remaining = 10, next token: 4923 ' Google' +srv update_slots: run slots completed + Googleque start_loop: waiting for new tasks +que start_loop: processing new tasks +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial**, desenvolvido por uma grande empresa de tecnologia (provavelmente a Google +que start_loop: processing task, id = 71 +que start_loop: update slots +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial**, desenvolvido por uma grande empresa de tecnologia (provavelmente a Google +srv update_slots: posting NEXT_RESPONSE +que post: new task, id = 72, front = 0 +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 93, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 71, n_remaining = 9, next token: 11 ',' +,srv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +que start_loop: processing task, id = 72 +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial**, desenvolvido por uma grande empresa de tecnologia (provavelmente a Google, +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +que post: new task, id = 73, front = 0 +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial**, desenvolvido por uma grande empresa de tecnologia (provavelmente a Google, +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 94, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 72, n_remaining = 8, next token: 55854 ' dado' +srv update_slots: run slots completed + dadoque start_loop: waiting for new tasks +que start_loop: processing new tasks +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial**, desenvolvido por uma grande empresa de tecnologia (provavelmente a Google, dado +que start_loop: processing task, id = 73 +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial**, desenvolvido por uma grande empresa de tecnologia (provavelmente a Google, dado +que post: new task, id = 74, front = 0 +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 95, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 73, n_remaining = 7, next token: 296 ' o' +srv update_slots: run slots completed + oque start_loop: waiting for new tasks +que start_loop: processing new tasks +que start_loop: processing task, id = 74 +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial**, desenvolvido por uma grande empresa de tecnologia (provavelmente a Google, dado o +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +que post: new task, id = 75, front = 0 +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial**, desenvolvido por uma grande empresa de tecnologia (provavelmente a Google, dado o +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 96, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 74, n_remaining = 6, next token: 74118 ' contexto' + contextosrv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +que start_loop: processing task, id = 75 +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial**, desenvolvido por uma grande empresa de tecnologia (provavelmente a Google, dado o contexto +que post: new task, id = 76, front = 0 +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 97, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial**, desenvolvido por uma grande empresa de tecnologia (provavelmente a Google, dado o contexto +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 75, n_remaining = 5, next token: 180024 ' comum' +srv update_slots: run slots completed + comumque start_loop: waiting for new tasks +que start_loop: processing new tasks +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial**, desenvolvido por uma grande empresa de tecnologia (provavelmente a Google, dado o contexto comum +que start_loop: processing task, id = 76 +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial**, desenvolvido por uma grande empresa de tecnologia (provavelmente a Google, dado o contexto comum +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +que post: new task, id = 77, front = 0 +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 98, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 76, n_remaining = 4, next token: 945 ' em' +srv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks + emque start_loop: processing task, id = 77 +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial**, desenvolvido por uma grande empresa de tecnologia (provavelmente a Google, dado o contexto comum em +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +que post: new task, id = 78, front = 0 +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial**, desenvolvido por uma grande empresa de tecnologia (provavelmente a Google, dado o contexto comum em +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 99, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 77, n_remaining = 3, next token: 32208 ' Portugal' +srv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks + Portugalque start_loop: processing task, id = 78 +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial**, desenvolvido por uma grande empresa de tecnologia (provavelmente a Google, dado o contexto comum em Portugal +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +que post: new task, id = 79, front = 0 +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial**, desenvolvido por uma grande empresa de tecnologia (provavelmente a Google, dado o contexto comum em Portugal +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 100, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 78, n_remaining = 2, next token: 553 ').' +srv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial**, desenvolvido por uma grande empresa de tecnologia (provavelmente a Google, dado o contexto comum em Portugal). +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial**, desenvolvido por uma grande empresa de tecnologia (provavelmente a Google, dado o contexto comum em Portugal). +que start_loop: processing task, id = 79 +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +que post: new task, id = 80, front = 0 +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 101, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +).set_adapters_lora: adapters = 0000000000000000 +adapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | n_decoded = 79, n_remaining = 1, next token: 18306 ' Pos' +srv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial**, desenvolvido por uma grande empresa de tecnologia (provavelmente a Google, dado o contexto comum em Portugal). Pos +que start_loop: processing task, id = 80 +que start_loop: update slots +srv update_slots: posting NEXT_RESPONSE +que post: new task, id = 81, front = 0 +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial**, desenvolvido por uma grande empresa de tecnologia (provavelmente a Google, dado o contexto comum em Portugal). Pos +slot update_slots: id 0 | task 0 | slot decode token, n_ctx = 4096, n_tokens = 102, truncated = 0 +srv update_slots: decoding batch, n_tokens = 1 +set_adapters_lora: adapters = 0000000000000000 + Posadapters_lora_are_same: adapters = 0000000000000000 +set_embeddings: value = 0 +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot process_toke: id 0 | task 0 | stopped by limit, n_decoded = 80, n_predict = 80 +soslot process_toke: id 0 | task 0 | n_decoded = 80, n_remaining = 0, next token: 684 'so' +slot print_timing: id 0 | task 0 | +prompt eval time = 579.36 ms / 23 tokens ( 25.19 ms per token, 39.70 tokens per second) + eval time = 9646.39 ms / 80 tokens ( 120.58 ms per token, 8.29 tokens per second) + total time = 10225.75 ms / 103 tokens + +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial**, desenvolvido por uma grande empresa de tecnologia (provavelmente a Google, dado o contexto comum em Portugal). Posso +res send: sending result for task id = 0 +res send: task id = 0 pushed to result queue +slot release: id 0 | task 0 | stop processing: n_tokens = 102, truncated = 0 +slot reset: id 0 | task 0 | +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial**, desenvolvido por uma grande empresa de tecnologia (provavelmente a Google, dado o contexto comum em Portugal). Posso +srv update_slots: run slots completed +que start_loop: waiting for new tasks +que start_loop: processing new tasks +que start_loop: processing task, id = 81 +que start_loop: update slots +srv update_slots: all slots are idle +que start_loop: waiting for new tasks +srv update_chat_: Parsing chat message: Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial**, desenvolvido por uma grande empresa de tecnologia (provavelmente a Google, dado o contexto comum em Portugal). Posso +Parsing PEG input with format peg-native: <|im_start|>assistant + +Let me think through this carefully. + +The question/task: Ol�! Me diga quem voc� � em uma frase. + +My reasoning: Quem sou eu em uma frase? + + +**Quem sou eu em uma frase?** + +Sou um **assistente de inteligência artificial**, desenvolvido por uma grande empresa de tecnologia (provavelmente a Google, dado o contexto comum em Portugal). Posso +Parsed message: {"role":"assistant","content":"**Quem sou eu em uma frase?**\n\nSou um **assistente de inteligência artificial**, desenvolvido por uma grande empresa de tecnologia (provavelmente a Google, dado o contexto comum em Portugal). Posso","reasoning_content":"\nLet me think through this carefully.\n\nThe question/task: Ol�! Me diga quem voc� � em uma frase.\n\nMy reasoning: Quem sou eu em uma frase?\n"} +res remove_waiti: remove task 0 from waiting list. current waiting = 1 (before remove) +srv stop: all tasks already finished, no need to cancel + +[ Prompt: 39.7 t/s | Generation: 8.3 t/s ] + +> \ No newline at end of file diff --git a/models/README.md b/models/README.md new file mode 100644 index 0000000..e4cfcfd --- /dev/null +++ b/models/README.md @@ -0,0 +1,10 @@ +# models/ + +# GGUF model files go here. +# They are NOT tracked by git (see .gitignore). +# +# Recommended files for development: +# qwen3.5-4b-instruct-q5_k_m.gguf ← turbo profile +# qwen3.5-9b-instruct-q4_k_m.gguf ← balanced profile +# +# Download via HuggingFace Hub or the registry service (Phase 11). diff --git a/plan.md b/plan.md new file mode 100644 index 0000000..8a47090 --- /dev/null +++ b/plan.md @@ -0,0 +1,814 @@ +Abaixo está o plano reestruturado em **Markdown**, separado por **etapas de implementação**, e já **ajustado para o seu foco real**: extrair o máximo de TPS possível com **Qwen3.5 4B/9B** em uma **RTX 3070 8 GB** ou hardware mais fraco, sem jogar qualidade fora. O plano-base do arquivo já estava correto ao manter **llama.cpp/C++ no core**, **Rust no hot path** e **Python apenas no registry/download**, com **Axum/Tokio + SSE** e comunicação por IPC. + +Também fiz um ajuste importante de foco: para o seu caso, eu **não começaria tentando ser um “LM Studio completo” logo de cara**. Eu começaria como um **runtime local de altíssimo desempenho para modelos pequenos/médios**, primeiro em **texto puro**, depois abriria multi-model, features extras e UI mais rica. Isso aumenta a chance de você realmente bater TPS alto cedo. O próprio ecossistema atual já ajuda nisso: o `llama.cpp` hoje já oferece **continuous batching**, **flash attention**, **prompt caching**, tipos de **KV cache quantizado** e **speculative decoding**; e os cards da Qwen3.5 já destacam modos de serving com foco em eficiência e modo **text-only/language-model-only** para liberar memória para KV cache. ([GitHub][1]) + +--- + +# Plano de Implementação — LocalLLM Inference Engine + +## 1. Objetivo do projeto + +Construir um backend local de inferência para LLMs com foco em: + +* **TPS alto** +* **latência baixa** +* **uso agressivamente otimizado de VRAM/RAM** +* **boa qualidade prática em modelos pequenos e médios** +* **arquitetura modular** +* **compatibilidade com API estilo OpenAI** + +### Meta prática inicial + +O alvo não deve ser “suportar tudo”. +O alvo inicial deve ser: + +* rodar **Qwen3.5-4B** como perfil principal de alto desempenho +* rodar **Qwen3.5-9B** como perfil balanceado +* extrair o máximo da **RTX 3070 8 GB** +* degradar bem para CPU/hardware mais fraco +* manter o sistema preparado para expansão futura + +A linha **Qwen3** densa aberta originalmente trouxe variantes como **4B** e **8B**; já a linha **Qwen3.5** aberta em 2026 inclui, entre outras, **4B** e **9B**, o que bate com o foco que você quer. ([Qwen][2]) + +--- + +## 2. Ajustes que eu faria no plano original + +## 2.1. Fazer a V1 em texto puro + +Mesmo que alguns modelos Qwen3.5 tenham componentes multimodais, o seu objetivo é **TPS alto em inferência textual**. Então a primeira versão deve ser: + +* **text-only** +* **single-model ativo** +* **single runtime path** +* sem visão +* sem embeddings +* sem tool-calling complexo na engine + +Os próprios materiais da Qwen3.5 destacam modo **text-only / language-model-only** justamente para liberar memória e permitir mais KV cache. ([Hugging Face][3]) + +## 2.2. Focar em throughput real, não em generalidade cedo demais + +No plano-base, a arquitetura em camadas está boa. O que eu mudaria é a ordem de implementação: + +1. engine local minimalista e muito rápida +2. benchmark e tuning +3. API compatível +4. registry/download +5. multi-request throughput +6. recursos avançados + +## 2.3. Adotar `llama.cpp` como engine principal da V1 + +Para o seu caso, isso continua sendo a melhor decisão do plano original. O `llama.cpp` já é otimizado para uma variedade grande de hardware e hoje expõe exatamente os knobs que mais importam para seu objetivo: batching contínuo, flash attention, prompt cache, speculative decoding e tuning de KV cache. ([GitHub][4]) + +## 2.4. Manter Rust no hot path + +Essa parte do plano original continua correta: **Rust + bindgen + Axum/Tokio** no hot path, e **Python só no caminho frio** de registry/Hugging Face. + +--- + +## 3. Stack final recomendada + +## 3.1. Core de inferência + +* **C/C++** +* **llama.cpp** compilado como shared library +* backend CUDA para NVIDIA + +## 3.2. Hot path do servidor + +* **Rust** +* `tokio` +* `axum` +* `hyper` +* `bindgen` + +## 3.3. Registry de modelos + +* **Python 3.11+** +* `huggingface_hub` +* processo separado +* IPC local + +## 3.4. UI de teste + +* HTML simples +* SSE nativo +* sem framework no começo + +Isso está alinhado com o documento-base enviado. + +--- + +## 4. Diretriz de performance para a RTX 3070 + +A **RTX 3070** tem **8 GB de VRAM** e compute capability **8.6**. Para esse projeto, isso significa que a build CUDA do `llama.cpp` deve ser tratada como prioridade e vale compilar direcionando a arquitetura CUDA da placa. ([NVIDIA][5]) + +### Build recomendada da engine + +* `-DGGML_CUDA=ON` +* `-DCMAKE_CUDA_ARCHITECTURES=86` + +Isso ajuda a evitar build genérica demais para a placa. ([GitHub][6]) + +### Consequência prática + +Com 8 GB, você não deve desenhar a V1 pensando em: + +* contexto gigantesco por padrão +* vários modelos carregados ao mesmo tempo +* multimodal cedo +* concorrência alta por processo logo no início + +Você deve desenhar a V1 pensando em: + +* **1 modelo ativo por vez** +* **contexto moderado** +* **alto aproveitamento de GPU** +* **prompt cache** +* **continuous batching** +* **KV cache otimizado** +* **flash attention** + +--- + +# Etapas de implementação + +## Etapa 0 — Definição do alvo técnico + +### Objetivo + +Definir o recorte exato da V1 para não desperdiçar tempo com features que não aumentam TPS. + +### Decisões + +* V1 será **texto puro** +* V1 terá **um modelo carregado por vez** +* V1 vai priorizar: + + * **Qwen3.5-4B** como modo turbo + * **Qwen3.5-9B** como modo balanceado +* API no estilo OpenAI desde cedo +* multi-model simultâneo fica para depois + +### Entregáveis + +* documento de metas +* perfis de execução +* critérios de benchmark + +### Critério de pronto + +Você consegue responder: + +* qual modelo será o “modelo de desenvolvimento” +* qual contexto padrão será usado +* qual métrica manda no projeto: TPS, latência ou concorrência + +### Minha recomendação + +Use como alvo inicial: + +* **Qwen3.5-4B** para desenvolvimento e tuning pesado +* **Qwen3.5-9B** só depois que a 4B estiver extremamente bem afinada + +--- + +## Etapa 1 — Foundation do workspace + +### Objetivo + +Criar a base do projeto com a estrutura modular correta desde o início. + +### Estrutura recomendada + +```text +localllm/ +├── Cargo.toml +├── crates/ +│ ├── llama-sys/ +│ ├── inference-core/ +│ ├── inference-server/ +│ ├── api-contracts/ +│ └── metrics/ +├── registry/ +│ ├── service.py +│ └── requirements.txt +├── ui/ +│ └── index.html +├── vendor/ +│ └── llama.cpp/ +└── models/ +``` + +Essa separação deriva da estrutura do plano-base, com ajuste pequeno de nomes para ficar mais clara para crescimento futuro. O documento-base já sugeria workspace Rust, crate de bindings, pool de inferência, servidor API, registry Python, UI e `vendor/llama.cpp` fixado como submódulo. + +### Decisões críticas + +* travar o `llama.cpp` em commit fixo +* não depender de versão flutuante +* build reproduzível + +### Critério de pronto + +* projeto compila vazio +* submodule do `llama.cpp` fixado +* layout estável + +--- + +## Etapa 2 — Build da engine CUDA + +### Objetivo + +Compilar o `llama.cpp` corretamente para sua GPU e validar carregamento básico. + +### O que implementar + +* build CMake do `llama.cpp` +* geração da shared library +* smoke test local +* script de build dev/prod + +### Recomendação + +Ter dois modos: + +* **dev build** +* **native optimized build** + +### Critério de pronto + +* shared library gerada +* modelo GGUF carregando +* primeiro token gerado localmente + +### Ajuste importante + +Nesta fase, não implemente API ainda. +Primeiro prove que o **core gera tokens rápido**. + +--- + +## Etapa 3 — FFI Rust com `bindgen` + +### Objetivo + +Criar o binding Rust ↔ `llama.cpp` sem overhead no runtime. + +O plano-base já propõe exatamente isso com `build.rs` + `bindgen`, e essa continua sendo a decisão correta. + +### O que implementar + +* crate `llama-sys` +* `build.rs` +* wrappers mínimos e seguros +* tipos Rust para: + + * model + * context + * sampler + * token stream + +### Regras desta etapa + +* wrapper fino +* sem abstrações demais +* sem inventar engine paralela antes da hora + +### Critério de pronto + +* Rust consegue: + + * abrir modelo + * tokenizar prompt + * chamar decode + * amostrar próximo token + * encerrar limpo + +--- + +## Etapa 4 — Engine local minimalista, single-model + +### Objetivo + +Ter um runtime local realmente rápido antes de colocar rede na frente. + +### O que implementar + +* `InferenceEngine` +* `LoadedModel` +* `GenerationSession` +* `SamplerConfig` +* `RuntimeProfile` + +### Regras + +* **1 modelo carregado** +* **1 caminho de execução** +* **sem hot-swap automático ainda** +* **sem múltiplos engines** + +### Perfis que eu criaria + +* `turbo_4b` +* `balanced_9b` +* `cpu_fallback` + +### Critério de pronto + +* prompt entra +* tokens saem +* TPS é medido +* memória é medida +* warmup existe + +### Otimizações obrigatórias nesta etapa + +* warmup de engine +* reuse de buffers +* evitar alocações em loop +* pré-tokenização de system prompt +* prompt cache quando aplicável + +O `llama.cpp` já suporta prompt caching e serving server-side com opções voltadas para esse tipo de fluxo. ([GitHub][1]) + +--- + +## Etapa 5 — Perf tuning de geração pura + +### Objetivo + +Ajustar o runtime até ele realmente ficar forte na 3070. + +### O que testar + +* `n_gpu_layers` +* `n_ctx` +* `n_batch` +* `ubatch` +* `n_threads` +* `flash attention` +* tipo do KV cache +* quantização do modelo + +O documento-base já previa expor `n_gpu_layers`, `n_ctx`, `n_threads` e `n_batch`. Hoje, além disso, o `llama.cpp` já documenta `--flash-attn`, `--batch-size`, `--ubatch-size` e tipos de KV cache quantizado. ([GitHub][1]) + +### Ordem de tuning recomendada + +1. acertar quantização do modelo +2. acertar `n_gpu_layers` +3. acertar `n_ctx` +4. ativar `flash attention` +5. testar KV cache mais leve +6. testar batch e ubatch +7. só depois testar concorrência + +### Critério de pronto + +Você tem uma tabela interna com: + +* TPS +* latência até primeiro token +* VRAM usada +* RAM usada +* qualidade percebida + +para pelo menos: + +* Qwen3.5-4B +* Qwen3.5-9B + +### Minha recomendação prática + +Para o seu foco, eu usaria: + +* **4B** como perfil padrão para “TPS máximo” +* **9B** como perfil “qualidade máxima viável” + +E faria os testes em quantizações médias antes de subir demais. Isso não é regra fixa do modelo; é estratégia de engenharia para sua GPU. + +--- + +## Etapa 6 — API local OpenAI-compatible + +### Objetivo + +Colocar a engine atrás de uma API limpa sem destruir a performance. + +O plano-base já propõe `Axum + SSE + contratos estilo OpenAI`, o que continua certo. + +### Endpoints iniciais + +* `POST /v1/chat/completions` +* `GET /v1/models` +* `GET /health` + +### O que deixar de fora no início + +* completions legadas se não forem necessárias +* embeddings +* files +* assistants +* websocket + +### Por que SSE + +O plano-base justifica bem SSE como fluxo unidirecional simples. Isso é compatível com seu foco e mantém o streaming barato. + +### Critério de pronto + +* request JSON entra +* stream de tokens sai em SSE +* formato é previsível +* métrica de TPS vai na resposta final + +--- + +## Etapa 7 — Scheduler de inferência e concorrência controlada + +### Objetivo + +Sair do single-request puro para throughput real multi-request. + +O documento-base já propõe um pool com semáforo e contexts dedicados. Eu manteria a ideia, mas implementaria **depois** da engine single-model estar muito boa. + +### O que implementar + +* fila de requests +* limite de concorrência +* reutilização de contexts +* backpressure +* cancelamento + +### Regra importante + +**Não escale concorrência só porque dá.** +Na 3070, concorrência excessiva pode reduzir TPS total por request, aumentar jitter e piorar latência. + +### Estratégia correta + +Ter perfis: + +* **low-latency mode** +* **high-throughput mode** + +### Critério de pronto + +* 2 a N requests simultâneos +* servidor não explode memória +* total de TPS agregado aumenta ou pelo menos não degrada absurdamente + +### Observação importante + +O `llama.cpp` server já suporta **parallel requests** e **continuous batching**, então vale estudar a arquitetura dele como referência prática antes de reinventar o scheduler inteiro. ([GitHub][1]) + +--- + +## Etapa 8 — Continuous batching + +### Objetivo + +Aumentar throughput agregado de verdade. + +### O que implementar + +* agrupamento de requests compatíveis +* união de prompts em janela de execução +* step scheduler +* contabilidade de tokens por request + +### Por que isso importa + +Para o seu foco, **continuous batching** provavelmente entrega mais ganho real do que sair adicionando feature de produto. + +O `llama.cpp` já documenta serving com **continuous batching**. ([GitHub][1]) + +### Critério de pronto + +* requests curtos simultâneos têm ganho agregado +* scheduler não mata a latência de requests isolados + +--- + +## Etapa 9 — KV cache tuning + +### Objetivo + +Aumentar capacidade útil e throughput sem depender apenas de VRAM bruta. + +### O que implementar + +* opção de tipo de KV cache +* presets de contexto +* política de truncamento inteligente +* medição de impacto em qualidade + +O `llama.cpp` já suporta tipos de KV cache quantizado para K/V, com defaults em `f16` e alternativas mais leves. ([GitHub][1]) + +### Estratégia + +No seu caso, isso é especialmente importante porque: + +* a 3070 tem só 8 GB +* Qwen3.5-9B pode ficar apertado dependendo da quantização e do contexto +* texto puro com KV cache otimizado pode render muito mais do que tentar “empurrar tudo” na VRAM + +### Critério de pronto + +* presets claros: + + * curto + * médio + * longo +* comparação objetiva entre qualidade e desempenho + +--- + +## Etapa 10 — Speculative decoding / draft model + +### Objetivo + +Buscar o próximo salto de TPS. + +O `llama.cpp` já documenta suporte a speculative decoding com modelo draft. A Qwen3.5 também já cita exemplos de serving com foco em MTP/eficiência. ([GitHub][1]) + +### Quando fazer + +Só depois que: + +* engine base estiver estável +* batching estiver bom +* cache estiver afinado + +### O que implementar + +* modelo principal +* modelo draft pequeno +* política de aceitação/rejeição +* benchmark comparando: + + * TPS + * latência + * uso extra de VRAM/RAM + +### Observação + +Essa etapa pode dar ganhos grandes, mas também pode complicar demais cedo. +Ela é etapa de **turbo**, não de fundação. + +--- + +## Etapa 11 — Registry de modelos + +### Objetivo + +Adicionar busca, download e metadata sem poluir o hot path. + +O plano-base está correto em jogar isso para um processo Python separado com `huggingface_hub`, IPC local e atualização de metadata. + +### O que implementar + +* `POST /registry/search` +* `POST /registry/download` +* `GET /registry/models` + +### Regra de arquitetura + +Nada de Python no caminho de geração de tokens. + +### Critério de pronto + +* busca GGUF +* download resumível +* checksum +* atualização do índice local + +--- + +## Etapa 12 — Observabilidade e benchmark suite + +### Objetivo + +Transformar tuning em processo técnico e não em chute. + +### O que medir + +* TTFT +* TPS +* tokens/s agregados +* VRAM +* RAM +* filas +* taxa de cancelamento +* throughput por perfil + +### O que implementar + +* endpoint de métricas +* logs estruturados +* benchmark CLI +* benchmark replayável + +### Critério de pronto + +Você consegue responder com dados: + +* qual quantização é a melhor na sua máquina +* qual contexto é o limite bom +* quando 9B vale a pena sobre 4B +* quando speculative decoding compensa + +--- + +## Etapa 13 — UI mínima de teste + +### Objetivo + +Ter um frontend simples para validar streaming, sem virar foco do projeto. + +O plano-base já propõe uma UI HTML estática simples, servida pelo próprio servidor. Para a V1, isso é o ideal. + +### O que implementar + +* seletor de perfil +* chat simples +* toggle stream +* exibição de TPS +* exibição de uso de memória +* log de benchmark rápido + +### Critério de pronto + +A UI só existe para testar o backend. Nada além disso. + +--- + +# Perfis recomendados para sua V1 + +## Perfil A — Turbo + +### Modelo + +Qwen3.5-4B + +### Objetivo + +Máximo TPS com boa qualidade prática. + +### Uso ideal + +* chat geral +* coding leve a moderado +* agentes rápidos +* máquinas mais fracas + +### Estratégia + +* quantização equilibrada +* contexto moderado +* text-only +* flash attention +* prompt cache +* batching leve + +Os cards atuais da Qwen3.5-4B mostram suporte a serving moderno e reforçam o modo text-only para eficiência de memória. ([Hugging Face][7]) + +--- + +## Perfil B — Balanceado + +### Modelo + +Qwen3.5-9B + +### Objetivo + +Subir qualidade sem matar completamente a velocidade. + +### Uso ideal + +* tarefas mais difíceis +* respostas mais consistentes +* coding mais pesado +* raciocínio melhor que 4B + +### Estratégia + +* contexto ainda controlado +* quantização mais conservadora que no 4B +* tuning fino de `n_gpu_layers` +* KV cache otimizado +* talvez speculative decoding no futuro + +Os cards públicos atuais da linha Qwen3.5 incluem 9B e destacam arquitetura voltada a serving eficiente. ([Hugging Face][8]) + +--- + +# O que eu removeria da V1 + +## Não colocaria agora + +* multimodal +* vários modelos carregados ao mesmo tempo +* websocket +* embeddings +* reranking +* LoRA dinâmica +* quantização on-the-fly +* scheduler super complexo antes do benchmark base +* UI sofisticada + +## Motivo + +Nada disso é o principal motor de TPS para o seu alvo. + +--- + +# Ordem exata que eu seguiria + +## Fase 1 + +Build CUDA + `llama.cpp` + primeiro token + +## Fase 2 + +Bindings Rust + runtime local single-model + +## Fase 3 + +Benchmark bruto de Qwen3.5-4B + +## Fase 4 + +Tuning pesado: + +* `n_gpu_layers` +* `n_ctx` +* `n_batch` +* `ubatch` +* flash attention +* KV cache + +## Fase 5 + +API OpenAI-compatible com SSE + +## Fase 6 + +Fila, scheduler e concorrência controlada + +## Fase 7 + +Continuous batching + +## Fase 8 + +Qwen3.5-9B balanceado + +## Fase 9 + +Registry Python / HF Hub + +## Fase 10 + +Speculative decoding e turbo mode + +--- + +# Conclusão técnica + +Seu plano-base está bom na arquitetura macro: **Rust no hot path**, **llama.cpp como core**, **Python só no registry**, **SSE/OpenAI-compatible**, **IPC local** e **workspace modular**. + +A principal melhoria para o seu objetivo é esta: + +> **transformar o projeto, na V1, de “infraestrutura genérica de inferência local” para “runtime de altíssimo desempenho para modelos textuais pequenos e médios”.** + +Isso significa: + +* começar por **Qwen3.5-4B** +* tratar **Qwen3.5-9B** como segundo alvo +* usar **RTX 3070** como orçamento real de VRAM +* explorar cedo: + + * **flash attention** + * **continuous batching** + * **prompt caching** + * **KV cache tuning** + * **speculative decoding depois** + +Essa é a rota com maior chance de entregar o que você quer de verdade: **TPS alto sem desandar qualidade**. +Base original: +Ajustes de foco e recursos atuais do ecossistema: ([GitHub][1]) + +Posso converter isso em seguida para um **`README.md` de projeto**, já no formato de especificação técnica pronta para repositório. + +[1]: https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md "llama.cpp/tools/server/README.md at master · ggml-org/llama.cpp · GitHub" +[2]: https://qwenlm.github.io/blog/qwen3/?utm_source=chatgpt.com "Qwen3: Think Deeper, Act Faster" +[3]: https://huggingface.co/Qwen/Qwen3.5-9B "Qwen/Qwen3.5-9B · Hugging Face" +[4]: https://github.com/ggml-org/llama.cpp "GitHub - ggml-org/llama.cpp: LLM inference in C/C++ · GitHub" +[5]: https://www.nvidia.com/en-us/geforce/graphics-cards/30-series/rtx-3070-3070ti/?utm_source=chatgpt.com "GeForce RTX 3070 Family" +[6]: https://github.com/ggml-org/llama.cpp/blob/master/docs/build.md "llama.cpp/docs/build.md at master · ggml-org/llama.cpp · GitHub" +[7]: https://huggingface.co/Qwen/Qwen3.5-4B "Qwen/Qwen3.5-4B · Hugging Face" +[8]: https://huggingface.co/Qwen/models "Qwen (Qwen)" diff --git a/registry/requirements.txt b/registry/requirements.txt new file mode 100644 index 0000000..2f51737 --- /dev/null +++ b/registry/requirements.txt @@ -0,0 +1,3 @@ +huggingface-hub>=0.23 +requests>=2.31 +tqdm>=4.66 diff --git a/registry/service.py b/registry/service.py new file mode 100644 index 0000000..aeb593a --- /dev/null +++ b/registry/service.py @@ -0,0 +1,31 @@ +""" +registry/service.py – Model registry service (Phase 1 stub). + +Responsibilities (to be completed in Phase 11): + - Search Hugging Face Hub for GGUF models + - Download models with resume support + checksum validation + - Maintain a local index of available models + - Expose a simple JSON IPC socket for the Rust server + +This file intentionally minimal in Phase 1. The structure is here so the +Rust server can already reference the IPC path in its config. +""" + +import json +import os +import socket +import sys +from pathlib import Path + +MODELS_DIR = Path(os.environ.get("LOCALLLM_MODELS_DIR", "../models")) +IPC_SOCKET = os.environ.get("LOCALLLM_REGISTRY_SOCK", "/tmp/localllm-registry.sock") + + +def main(): + print(f"[registry] models dir : {MODELS_DIR.resolve()}") + print(f"[registry] IPC socket : {IPC_SOCKET}") + print("[registry] Phase 1 stub – no-op. Full implementation in Phase 11.") + + +if __name__ == "__main__": + main() diff --git a/scripts/build-dev.ps1 b/scripts/build-dev.ps1 new file mode 100644 index 0000000..c783f37 --- /dev/null +++ b/scripts/build-dev.ps1 @@ -0,0 +1,27 @@ +# scripts/build-dev.ps1 +# Development build of llama.cpp for Windows (Debug, CUDA RTX 3070). + +$ErrorActionPreference = "Stop" + +$root = Split-Path $PSScriptRoot -Parent +$vendor = Join-Path $root "vendor\llama.cpp" +$buildDir = Join-Path $vendor "build-dev" + +if (-not (Test-Path (Join-Path $vendor "CMakeLists.txt"))) { + Write-Error "vendor/llama.cpp not initialised. Run: git submodule update --init --recursive" + exit 1 +} + +Write-Host "[INFO] Configuring llama.cpp (dev, CUDA enabled)..." +cmake -S $vendor -B $buildDir ` + -DCMAKE_BUILD_TYPE=Debug ` + -DBUILD_SHARED_LIBS=ON ` + -DGGML_CUDA=ON ` + "-DCMAKE_CUDA_ARCHITECTURES=86" ` + -DLLAMA_BUILD_TESTS=OFF ` + -DLLAMA_BUILD_EXAMPLES=OFF + +Write-Host "[INFO] Building..." +cmake --build $buildDir --parallel + +Write-Host "[OK] Dev build complete: $buildDir" diff --git a/scripts/build-dev.sh b/scripts/build-dev.sh new file mode 100644 index 0000000..b2b85ef --- /dev/null +++ b/scripts/build-dev.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# scripts/build-dev.sh +# Development build of llama.cpp (Debug, no optimization). +# Run once after `git submodule update --init --recursive`. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$SCRIPT_DIR/.." +VENDOR="$ROOT/vendor/llama.cpp" +BUILD_DIR="$VENDOR/build-dev" + +if [ ! -f "$VENDOR/CMakeLists.txt" ]; then + echo "[ERROR] vendor/llama.cpp not initialised." + echo " Run: git submodule update --init --recursive" + exit 1 +fi + +echo "[INFO] Configuring llama.cpp (dev, CUDA enabled)..." +cmake -S "$VENDOR" -B "$BUILD_DIR" \ + -DCMAKE_BUILD_TYPE=Debug \ + -DBUILD_SHARED_LIBS=ON \ + -DGGML_CUDA=ON \ + -DCMAKE_CUDA_ARCHITECTURES=86 \ + -DLLAMA_BUILD_TESTS=OFF \ + -DLLAMA_BUILD_EXAMPLES=OFF + +echo "[INFO] Building (dev)..." +cmake --build "$BUILD_DIR" --parallel "$(nproc 2>/dev/null || echo 4)" + +echo "[OK] Dev build complete: $BUILD_DIR" diff --git a/scripts/build-prod.ps1 b/scripts/build-prod.ps1 new file mode 100644 index 0000000..4815af6 --- /dev/null +++ b/scripts/build-prod.ps1 @@ -0,0 +1,27 @@ +# scripts/build-prod.ps1 +# Production build of llama.cpp for Windows (Release, CUDA RTX 3070 cc86). + +$ErrorActionPreference = "Stop" + +$root = Split-Path $PSScriptRoot -Parent +$vendor = Join-Path $root "vendor\llama.cpp" +$buildDir = Join-Path $vendor "build-prod" + +if (-not (Test-Path (Join-Path $vendor "CMakeLists.txt"))) { + Write-Error "vendor/llama.cpp not initialised. Run: git submodule update --init --recursive" + exit 1 +} + +Write-Host "[INFO] Configuring llama.cpp (prod, CUDA cc86)..." +cmake -S $vendor -B $buildDir ` + -DCMAKE_BUILD_TYPE=Release ` + -DBUILD_SHARED_LIBS=ON ` + -DGGML_CUDA=ON ` + "-DCMAKE_CUDA_ARCHITECTURES=86" ` + -DLLAMA_BUILD_TESTS=OFF ` + -DLLAMA_BUILD_EXAMPLES=OFF + +Write-Host "[INFO] Building (Release)..." +cmake --build $buildDir --config Release --parallel + +Write-Host "[OK] Prod build complete: $buildDir" diff --git a/scripts/build-prod.sh b/scripts/build-prod.sh new file mode 100644 index 0000000..79d61c1 --- /dev/null +++ b/scripts/build-prod.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# scripts/build-prod.sh +# Production build of llama.cpp (Release + LTO, RTX 3070 target). + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$SCRIPT_DIR/.." +VENDOR="$ROOT/vendor/llama.cpp" +BUILD_DIR="$VENDOR/build-prod" + +if [ ! -f "$VENDOR/CMakeLists.txt" ]; then + echo "[ERROR] vendor/llama.cpp not initialised." + echo " Run: git submodule update --init --recursive" + exit 1 +fi + +echo "[INFO] Configuring llama.cpp (prod, CUDA cc86)..." +cmake -S "$VENDOR" -B "$BUILD_DIR" \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=ON \ + -DGGML_CUDA=ON \ + -DCMAKE_CUDA_ARCHITECTURES=86 \ + -DLLAMA_BUILD_TESTS=OFF \ + -DLLAMA_BUILD_EXAMPLES=OFF \ + -DCMAKE_INTERPROCEDURAL_OPTIMIZATION=ON + +echo "[INFO] Building (prod)..." +cmake --build "$BUILD_DIR" \ + --parallel "$(nproc 2>/dev/null || echo 4)" \ + --config Release + +echo "[OK] Prod build complete: $BUILD_DIR" diff --git a/ui/index.html b/ui/index.html new file mode 100644 index 0000000..4d2f058 --- /dev/null +++ b/ui/index.html @@ -0,0 +1,163 @@ + + + + + + LocalLLM — Dev UI + + + +
+

LocalLLM

+ connecting… + — tok/s + + +
+ +
+ +
+ + +
+ + + +