lower batch size to 512

822a0e36 · Jeffrey Morgan · 18b6b601 · 822a0e36
Commit 822a0e36 authored Jul 28, 2023 by Jeffrey Morgan
Hide whitespace changes
Inline Side-by-side

Showing with 1 addition and 1 deletion

api/types.go api/types.go +1 -1

No files found.
--- a/api/types.go
+++ b/api/types.go
@@ -190,7 +190,7 @@ func DefaultOptions() Options {
 		UseNUMA: false,

 		NumCtx:   2048,
-		NumBatch: 1024,
+		NumBatch: 512,
 		NumGPU:   1,
 		NumGQA:   1,
 		LowVRAM:  false,