evandro/ForgeLM
1
0 Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

perf: disable llama perf counters in hot path

Browse Source
This commit is contained in:
Evandro Fonseca Junior committed 2026-03-26 17:56:38 -03:00
1 parent 6eaf59dd74
commit 43e2fb01f2
2 files changed
+3 -2
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

crates/inference-core/src/engine.rs
+1 -1
View File
@@ -232,7 +232,7 @@ impl InferenceEngine {
LLAMA_FLASH_ATTN_TYPE_DISABLED LLAMA_FLASH_ATTN_TYPE_DISABLED
}; };
cparams.offload_kqv = true; cparams.offload_kqv = true;
cparams.no_perf = false; cparams.no_perf = true;
// 6. Init context // 6. Init context
let ctx_ptr = unsafe { (lib.init_from_model)(model_ptr, cparams) }; let ctx_ptr = unsafe { (lib.init_from_model)(model_ptr, cparams) };
crates/inference-core/src/session.rs
+2 -1
View File
@@ -112,7 +112,8 @@ impl GenerationSession {
debug!(n_tokens = n_prompt_tokens, "Prompt tokenised"); debug!(n_tokens = n_prompt_tokens, "Prompt tokenised");
// ── 2. Build sampler chain ───────────────────────────────────────── // ── 2. Build sampler chain ─────────────────────────────────────────
let sc_params = unsafe { (lib.sampler_chain_default_params)() }; let mut sc_params = unsafe { (lib.sampler_chain_default_params)() };
sc_params.no_perf = true;
let sampler_chain = unsafe { (lib.sampler_chain_init)(sc_params) }; let sampler_chain = unsafe { (lib.sampler_chain_init)(sc_params) };
if sampler_chain.is_null() { if sampler_chain.is_null() {
bail!("sampler_chain_init returned NULL"); bail!("sampler_chain_init returned NULL");