perf: slim turbo4b runtime profile
This commit is contained in:
1 parent
cac9227a6a
commit
0e6f5ca5a3
1 file changed
+3
-3
@@ -15,9 +15,9 @@ impl RuntimeProfile {
|
||||
match self {
|
||||
Self::Turbo4B => ProfileDefaults {
|
||||
n_gpu_layers: 999,
|
||||
n_ctx: 4096,
|
||||
n_batch: 512,
|
||||
n_ubatch: 128,
|
||||
n_ctx: 2048,
|
||||
n_batch: 256,
|
||||
n_ubatch: 64,
|
||||
flash_attn: true,
|
||||
n_threads: 8,
|
||||
},
|
||||
|
||||
Reference in new issue
Block a user