server.go 32.8 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
package llm

import (
	"bufio"
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"log/slog"
	"math/rand"
	"net"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
22
	"sync"
23
24
	"time"

Daniel Hiltgen's avatar
Daniel Hiltgen committed
25
26
	"golang.org/x/sync/semaphore"

27
	"github.com/ollama/ollama/api"
28
	"github.com/ollama/ollama/build"
29
	"github.com/ollama/ollama/discover"
30
	"github.com/ollama/ollama/envconfig"
31
	"github.com/ollama/ollama/format"
32
	"github.com/ollama/ollama/llama"
33
	"github.com/ollama/ollama/runners"
34
35
)

Daniel Hiltgen's avatar
Daniel Hiltgen committed
36
37
38
39
type LlamaServer interface {
	Ping(ctx context.Context) error
	WaitUntilRunning(ctx context.Context) error
	Completion(ctx context.Context, req CompletionRequest, fn func(CompletionResponse)) error
40
	Embedding(ctx context.Context, input string) ([]float32, error)
Daniel Hiltgen's avatar
Daniel Hiltgen committed
41
42
43
	Tokenize(ctx context.Context, content string) ([]int, error)
	Detokenize(ctx context.Context, tokens []int) (string, error)
	Close() error
44
	EstimatedVRAM() uint64 // Total VRAM across all GPUs
45
	EstimatedTotal() uint64
Daniel Hiltgen's avatar
Daniel Hiltgen committed
46
	EstimatedVRAMByGPU(gpuID string) uint64
Daniel Hiltgen's avatar
Daniel Hiltgen committed
47
48
49
50
}

// llmServer is an instance of the llama.cpp server
type llmServer struct {
51
52
53
54
55
56
	port        int
	cmd         *exec.Cmd
	done        chan error // Channel to signal when the process exits
	status      *StatusWriter
	options     api.Options
	numParallel int
57
58
59
	modelPath   string
	modelLock   sync.Mutex   // Temporary until we switch fully to Go server
	model       *llama.Model // If non-nil, the runner is a new Go server
Daniel Hiltgen's avatar
Daniel Hiltgen committed
60

61
62
63
	estimate    MemoryEstimate
	totalLayers uint64
	// gpuCount     int
64
65
	gpus         discover.GpuInfoList // Recorded just before the model loaded, free space will be incorrect
	loadDuration time.Duration        // Record how long it took the model to load
66
	loadProgress float32
Daniel Hiltgen's avatar
Daniel Hiltgen committed
67
68

	sem *semaphore.Weighted
69
70
}

71
72
73
74
75
76
// LoadModel will load a model from disk. The model must be in the GGML format.
//
// It collects array values for arrays with a size less than or equal to
// maxArraySize. If maxArraySize is 0, the default value of 1024 is used. If
// the maxArraySize is negative, all arrays are collected.
func LoadModel(model string, maxArraySize int) (*GGML, error) {
Daniel Hiltgen's avatar
Daniel Hiltgen committed
77
78
79
80
	if _, err := os.Stat(model); err != nil {
		return nil, err
	}

81
82
83
84
85
86
	f, err := os.Open(model)
	if err != nil {
		return nil, err
	}
	defer f.Close()

87
	ggml, _, err := DecodeGGML(f, maxArraySize)
Daniel Hiltgen's avatar
Daniel Hiltgen committed
88
89
	return ggml, err
}
90

Daniel Hiltgen's avatar
Daniel Hiltgen committed
91
92
// NewLlamaServer will run a server for the given GPUs
// The gpu list must be a single family.
93
func NewLlamaServer(gpus discover.GpuInfoList, model string, ggml *GGML, adapters, projectors []string, opts api.Options, numParallel int) (LlamaServer, error) {
Daniel Hiltgen's avatar
Daniel Hiltgen committed
94
	var err error
95
	var cpuRunner string
96
	var estimate MemoryEstimate
97
98
	var systemTotalMemory uint64
	var systemFreeMemory uint64
99
	var systemSwapFreeMemory uint64
100

101
	systemInfo := discover.GetSystemInfo()
102
103
104
105
	systemTotalMemory = systemInfo.System.TotalMemory
	systemFreeMemory = systemInfo.System.FreeMemory
	systemSwapFreeMemory = systemInfo.System.FreeSwap
	slog.Info("system memory", "total", format.HumanBytes2(systemTotalMemory), "free", format.HumanBytes2(systemFreeMemory), "free_swap", format.HumanBytes2(systemSwapFreeMemory))
106

107
108
	// If the user wants zero GPU layers, reset the gpu list to be CPU/system ram info
	if opts.NumGPU == 0 {
109
		gpus = discover.GetCPUInfo()
110
111
	}
	if len(gpus) == 1 && gpus[0].Library == "cpu" {
112
		cpuRunner = runners.ServerForCpu()
113
		estimate = EstimateGPULayers(gpus, ggml, projectors, opts)
Daniel Hiltgen's avatar
Daniel Hiltgen committed
114
	} else {
115
		estimate = EstimateGPULayers(gpus, ggml, projectors, opts)
Daniel Hiltgen's avatar
Daniel Hiltgen committed
116

Michael Yang's avatar
Michael Yang committed
117
		switch {
118
		case gpus[0].Library == "metal" && estimate.VRAMSize > systemTotalMemory:
Daniel Hiltgen's avatar
Daniel Hiltgen committed
119
120
121
			// disable partial offloading when model is greater than total system memory as this
			// can lead to locking up the system
			opts.NumGPU = 0
122
		case gpus[0].Library != "metal" && estimate.Layers == 0:
123
			// Don't bother loading into the GPU if no layers can fit
124
			cpuRunner = runners.ServerForCpu()
125
			gpus = discover.GetCPUInfo()
126
127
		case opts.NumGPU < 0 && estimate.Layers > 0 && gpus[0].Library != "cpu":
			opts.NumGPU = estimate.Layers
128
129
130
		}
	}

131
132
133
	// On linux and windows, over-allocating CPU memory will almost always result in an error
	// Darwin has fully dynamic swap so has no direct concept of free swap space
	if runtime.GOOS != "darwin" {
134
		systemMemoryRequired := estimate.TotalSize - estimate.VRAMSize
135
		available := systemFreeMemory + systemSwapFreeMemory
136
137
138
		if systemMemoryRequired > available {
			slog.Warn("model request too large for system", "requested", format.HumanBytes2(systemMemoryRequired), "available", available, "total", format.HumanBytes2(systemTotalMemory), "free", format.HumanBytes2(systemFreeMemory), "swap", format.HumanBytes2(systemSwapFreeMemory))
			return nil, fmt.Errorf("model requires more system memory (%s) than is available (%s)", format.HumanBytes2(systemMemoryRequired), format.HumanBytes2(available))
139
140
141
		}
	}

142
143
	estimate.log()

Daniel Hiltgen's avatar
Daniel Hiltgen committed
144
	// Loop through potential servers
Michael Yang's avatar
Michael Yang committed
145
	finalErr := errors.New("no suitable llama servers found")
146

147
148
149
150
151
152
	rDir, err := runners.Refresh(build.EmbedFS)
	if err != nil {
		return nil, err
	}

	availableServers := runners.GetAvailableServers(rDir)
153
	if len(availableServers) == 0 {
154
		return nil, finalErr
155
	}
Daniel Hiltgen's avatar
Daniel Hiltgen committed
156
157
158
159
	var servers []string
	if cpuRunner != "" {
		servers = []string{cpuRunner}
	} else {
160
		servers = runners.ServersForGpu(gpus[0]) // All GPUs in the list are matching Library and Variant
Daniel Hiltgen's avatar
Daniel Hiltgen committed
161
	}
Michael Yang's avatar
string  
Michael Yang committed
162
	demandLib := envconfig.LLMLibrary()
163
164
165
166
167
168
169
	if demandLib != "" {
		serverPath := availableServers[demandLib]
		if serverPath == "" {
			slog.Info(fmt.Sprintf("Invalid OLLAMA_LLM_LIBRARY %s - not found", demandLib))
		} else {
			slog.Info("user override", "OLLAMA_LLM_LIBRARY", demandLib, "path", serverPath)
			servers = []string{demandLib}
Daniel Hiltgen's avatar
Daniel Hiltgen committed
170
171
172
173
			if strings.HasPrefix(demandLib, "cpu") {
				// Omit the GPU flag to silence the warning
				opts.NumGPU = -1
			}
174
175
176
177
		}
	}

	if len(servers) == 0 {
Daniel Hiltgen's avatar
Daniel Hiltgen committed
178
		return nil, fmt.Errorf("no servers found for %v", gpus)
179
180
181
182
	}

	params := []string{
		"--model", model,
Michael Yang's avatar
lint  
Michael Yang committed
183
184
		"--ctx-size", strconv.Itoa(opts.NumCtx),
		"--batch-size", strconv.Itoa(opts.NumBatch),
185
	}
Michael Yang's avatar
Michael Yang committed
186

Michael Yang's avatar
Michael Yang committed
187
	if opts.NumGPU >= 0 {
Michael Yang's avatar
lint  
Michael Yang committed
188
		params = append(params, "--n-gpu-layers", strconv.Itoa(opts.NumGPU))
189
190
	}

Michael Yang's avatar
Michael Yang committed
191
	if envconfig.Debug() {
192
193
194
195
		params = append(params, "--verbose")
	}

	if opts.MainGPU > 0 {
Michael Yang's avatar
lint  
Michael Yang committed
196
		params = append(params, "--main-gpu", strconv.Itoa(opts.MainGPU))
197
198
199
	}

	if len(adapters) > 0 {
200
201
202
		for _, adapter := range adapters {
			params = append(params, "--lora", adapter)
		}
203
204
205
206
207
208
209
	}

	if len(projectors) > 0 {
		// TODO: applying multiple projectors is not supported by the llama.cpp server yet
		params = append(params, "--mmproj", projectors[0])
	}

210
	defaultThreads := systemInfo.GetOptimalThreadCount()
211
	if opts.NumThread > 0 {
Michael Yang's avatar
lint  
Michael Yang committed
212
		params = append(params, "--threads", strconv.Itoa(opts.NumThread))
213
214
	} else if defaultThreads > 0 {
		params = append(params, "--threads", strconv.Itoa(defaultThreads))
215
216
	}

217
218
219
220
221
	fa := envconfig.FlashAttention()
	if fa && !gpus.FlashAttentionSupported() {
		slog.Warn("flash attention enabled but not supported by gpu")
		fa = false
	}
Sam's avatar
Sam committed
222

223
224
225
226
227
	if fa && !ggml.SupportsFlashAttention() {
		slog.Warn("flash attention enabled but not supported by model")
		fa = false
	}

228
	kvct := strings.ToLower(envconfig.KvCacheType())
229
230
231
232
233
234
235
236
237
238
239

	if fa {
		slog.Info("enabling flash attention")
		params = append(params, "--flash-attn")

		// Flash Attention also supports kv cache quantization
		// Enable if the requested and kv cache type is supported by the model
		if kvct != "" && ggml.SupportsKVCacheType(kvct) {
			params = append(params, "--kv-cache-type", kvct)
		} else {
			slog.Warn("kv cache type not supported by model", "type", kvct)
Sam's avatar
Sam committed
240
		}
241
242
243
	} else if kvct != "" && kvct != "f16" {
		slog.Warn("quantized kv cache requested but flash attention disabled", "type", kvct)
	}
244

245
246
	// mmap has issues with partial offloading on metal
	for _, g := range gpus {
247
248
249
		if g.Library == "metal" &&
			uint64(opts.NumGPU) > 0 &&
			uint64(opts.NumGPU) < ggml.KV().BlockCount()+1 {
250
251
			opts.UseMMap = new(bool)
			*opts.UseMMap = false
252
		}
Sam's avatar
Sam committed
253
	}
254

255
	// Windows CUDA should not use mmap for best performance
256
	// Linux  with a model larger than free space, mmap leads to thrashing
Daniel Hiltgen's avatar
Daniel Hiltgen committed
257
	// For CPU loads we want the memory to be allocated, not FS cache
258
259
260
261
	if (runtime.GOOS == "windows" && gpus[0].Library == "cuda" && opts.UseMMap == nil) ||
		(runtime.GOOS == "linux" && systemFreeMemory < estimate.TotalSize && opts.UseMMap == nil) ||
		(gpus[0].Library == "cpu" && opts.UseMMap == nil) ||
		(opts.UseMMap != nil && !*opts.UseMMap) {
262
263
264
265
266
267
268
		params = append(params, "--no-mmap")
	}

	if opts.UseMLock {
		params = append(params, "--mlock")
	}

269
	// TODO - NUMA support currently doesn't work properly
270

Michael Yang's avatar
lint  
Michael Yang committed
271
	params = append(params, "--parallel", strconv.Itoa(numParallel))
Daniel Hiltgen's avatar
Daniel Hiltgen committed
272

273
274
275
276
	if estimate.TensorSplit != "" {
		params = append(params, "--tensor-split", estimate.TensorSplit)
	}

277
278
279
280
	if envconfig.MultiUserCache() {
		params = append(params, "--multiuser-cache")
	}

281
	for i := range servers {
282
		dir := availableServers[servers[i]]
283
284
285
		if dir == "" {
			// Shouldn't happen
			finalErr = fmt.Errorf("[%d] server %s not listed in available servers %v", i, servers[i], availableServers)
Michael Yang's avatar
Michael Yang committed
286
			slog.Error("server list inconsistent", "error", finalErr)
287
288
			continue
		}
289

Daniel Hiltgen's avatar
Daniel Hiltgen committed
290
		if strings.HasPrefix(servers[i], "cpu") {
291
			gpus = discover.GetCPUInfo()
Daniel Hiltgen's avatar
Daniel Hiltgen committed
292
293
		}

294
		// Find an availableServers  port, retry on each iteration in case the failure was a port conflict race
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
		port := 0
		if a, err := net.ResolveTCPAddr("tcp", "localhost:0"); err == nil {
			var l *net.TCPListener
			if l, err = net.ListenTCP("tcp", a); err == nil {
				port = l.Addr().(*net.TCPAddr).Port
				l.Close()
			}
		}
		if port == 0 {
			slog.Debug("ResolveTCPAddr failed ", "error", err)
			port = rand.Intn(65535-49152) + 49152 // get a random port in the ephemeral range
		}
		finalParams := append(params, "--port", strconv.Itoa(port))

		pathEnv := "LD_LIBRARY_PATH"
		if runtime.GOOS == "windows" {
			pathEnv = "PATH"
		}
Daniel Hiltgen's avatar
Daniel Hiltgen committed
313
314
		// Start with the server directory for the LD_LIBRARY_PATH/PATH
		libraryPaths := []string{dir}
Daniel Hiltgen's avatar
Daniel Hiltgen committed
315

316
		if libraryPath, ok := os.LookupEnv(pathEnv); ok {
Daniel Hiltgen's avatar
Daniel Hiltgen committed
317
			// favor our bundled library dependencies over system libraries
Daniel Hiltgen's avatar
Daniel Hiltgen committed
318
			libraryPaths = append(libraryPaths, filepath.SplitList(libraryPath)...)
319
320
		}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
321
		// Note: we always put the dependency path first
Daniel Hiltgen's avatar
Daniel Hiltgen committed
322
		// since this was the exact version we compiled/linked against
323
		if gpus[0].DependencyPath != nil {
Daniel Hiltgen's avatar
Daniel Hiltgen committed
324
			// assume gpus from the same library have the same dependency path
325
			libraryPaths = append(gpus[0].DependencyPath, libraryPaths...)
Daniel Hiltgen's avatar
Daniel Hiltgen committed
326
327
		}

328
329
		server := filepath.Join(dir, "ollama_llama_server")
		if runtime.GOOS == "windows" {
Michael Yang's avatar
Michael Yang committed
330
			server += ".exe"
331
332
		}

333
334
335
336
		// Detect tmp cleaners wiping out the file
		_, err := os.Stat(server)
		if errors.Is(err, os.ErrNotExist) {
			slog.Warn("llama server disappeared, reinitializing payloads", "path", server, "error", err)
337
			_, err = runners.Refresh(build.EmbedFS)
338
339
340
341
342
343
			if err != nil {
				slog.Warn("failed to reinitialize payloads", "error", err)
				return nil, err
			}
		}

344
		// TODO - once fully switched to the Go runner, load the model here for tokenize/detokenize cgo access
Daniel Hiltgen's avatar
Daniel Hiltgen committed
345
		s := &llmServer{
346
347
348
349
			port:        port,
			cmd:         exec.Command(server, finalParams...),
			status:      NewStatusWriter(os.Stderr),
			options:     opts,
350
			modelPath:   model,
351
			estimate:    estimate,
352
			numParallel: numParallel,
353
354
			sem:         semaphore.NewWeighted(int64(numParallel)),
			totalLayers: ggml.KV().BlockCount() + 1,
355
			gpus:        gpus,
356
			done:        make(chan error, 1),
357
		}
Daniel Hiltgen's avatar
Daniel Hiltgen committed
358

359
		s.cmd.Env = os.Environ()
360
361
		s.cmd.Stdout = os.Stdout
		s.cmd.Stderr = s.status
362
		s.cmd.SysProcAttr = LlamaServerSysProcAttr
363

Daniel Hiltgen's avatar
Daniel Hiltgen committed
364
365
366
367
		envWorkarounds := [][2]string{}
		for _, gpu := range gpus {
			envWorkarounds = append(envWorkarounds, gpu.EnvWorkarounds...)
		}
Michael Yang's avatar
lint  
Michael Yang committed
368
		visibleDevicesEnv, visibleDevicesEnvVal := gpus.GetVisibleDevicesEnv()
369
370
371
372
373
374
375
376
377
378
379
380
381
		pathEnvVal := strings.Join(libraryPaths, string(filepath.ListSeparator))

		// Update or add the path and visible devices variable with our adjusted version
		pathNeeded := true
		devicesNeeded := visibleDevicesEnv != ""
		for i := range s.cmd.Env {
			cmp := strings.SplitN(s.cmd.Env[i], "=", 2)
			if strings.EqualFold(cmp[0], pathEnv) {
				s.cmd.Env[i] = pathEnv + "=" + pathEnvVal
				pathNeeded = false
			} else if devicesNeeded && strings.EqualFold(cmp[0], visibleDevicesEnv) {
				s.cmd.Env[i] = visibleDevicesEnv + "=" + visibleDevicesEnvVal
				devicesNeeded = false
Daniel Hiltgen's avatar
Daniel Hiltgen committed
382
383
384
385
386
387
			} else if len(envWorkarounds) != 0 {
				for _, kv := range envWorkarounds {
					if strings.EqualFold(cmp[0], kv[0]) {
						s.cmd.Env[i] = kv[0] + "=" + kv[1]
					}
				}
388
			}
Daniel Hiltgen's avatar
Daniel Hiltgen committed
389
		}
390
391
		if pathNeeded {
			s.cmd.Env = append(s.cmd.Env, pathEnv+"="+pathEnvVal)
Daniel Hiltgen's avatar
Daniel Hiltgen committed
392
		}
393
394
		if devicesNeeded {
			s.cmd.Env = append(s.cmd.Env, visibleDevicesEnv+"="+visibleDevicesEnvVal)
Daniel Hiltgen's avatar
Daniel Hiltgen committed
395
396
		}

397
		slog.Info("starting llama server", "cmd", s.cmd.String())
Michael Yang's avatar
Michael Yang committed
398
		if envconfig.Debug() {
399
400
401
			filteredEnv := []string{}
			for _, ev := range s.cmd.Env {
				if strings.HasPrefix(ev, "CUDA_") ||
Daniel Hiltgen's avatar
Daniel Hiltgen committed
402
					strings.HasPrefix(ev, "ROCR_") ||
403
404
					strings.HasPrefix(ev, "ROCM_") ||
					strings.HasPrefix(ev, "HIP_") ||
Daniel Hiltgen's avatar
Daniel Hiltgen committed
405
					strings.HasPrefix(ev, "GPU_") ||
406
407
408
409
410
411
412
413
414
415
					strings.HasPrefix(ev, "HSA_") ||
					strings.HasPrefix(ev, "GGML_") ||
					strings.HasPrefix(ev, "PATH=") ||
					strings.HasPrefix(ev, "LD_LIBRARY_PATH=") {
					filteredEnv = append(filteredEnv, ev)
				}
			}
			// Log at debug as the environment is inherited and might contain sensitive information
			slog.Debug("subprocess", "environment", filteredEnv)
		}
416
417

		if err = s.cmd.Start(); err != nil {
418
			// Detect permission denied and augment the message about noexec
419
420
421
422
			if errors.Is(err, os.ErrPermission) {
				finalErr = fmt.Errorf("unable to start server %w.  %s may have noexec set.  Set OLLAMA_TMPDIR for server to a writable executable directory", err, dir)
				continue
			}
423
424
425
426
427
428
429
430
431
			msg := ""
			if s.status != nil && s.status.LastErrMsg != "" {
				msg = s.status.LastErrMsg
			}
			err = fmt.Errorf("error starting the external llama server: %v %s", err, msg)
			finalErr = err
			continue
		}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
432
433
		// reap subprocess when it exits
		go func() {
434
435
436
437
438
439
440
			err := s.cmd.Wait()
			// Favor a more detailed message over the process exit status
			if err != nil && s.status != nil && s.status.LastErrMsg != "" {
				slog.Debug("llama runner terminated", "error", err)
				if strings.Contains(s.status.LastErrMsg, "unknown model") {
					s.status.LastErrMsg = "this model is not supported by your version of Ollama. You may need to upgrade"
				}
Michael Yang's avatar
lint  
Michael Yang committed
441
				s.done <- errors.New(s.status.LastErrMsg)
442
443
444
			} else {
				s.done <- err
			}
Daniel Hiltgen's avatar
Daniel Hiltgen committed
445
446
		}()

447
448
449
450
451
452
453
454
455
456
457
		return s, nil
	}

	slog.Error("unable to load any llama server", "error", finalErr)
	return nil, finalErr
}

type ServerStatus int

const ( // iota is reset to 0
	ServerStatusReady ServerStatus = iota
458
	ServerStatusNoSlotsAvailable
459
460
461
462
463
	ServerStatusLoadingModel
	ServerStatusNotResponding
	ServerStatusError
)

Daniel Hiltgen's avatar
Daniel Hiltgen committed
464
465
466
467
func (s ServerStatus) ToString() string {
	switch s {
	case ServerStatusReady:
		return "llm server ready"
468
	case ServerStatusNoSlotsAvailable:
Daniel Hiltgen's avatar
Daniel Hiltgen committed
469
470
471
472
473
474
475
476
477
478
		return "llm busy - no slots available"
	case ServerStatusLoadingModel:
		return "llm server loading model"
	case ServerStatusNotResponding:
		return "llm server not responding"
	default:
		return "llm server error"
	}
}

479
type ServerStatusResp struct {
Daniel Hiltgen's avatar
Daniel Hiltgen committed
480
481
482
483
484
	Status          string  `json:"status"`
	SlotsIdle       int     `json:"slots_idle"`
	SlotsProcessing int     `json:"slots_processing"`
	Error           string  `json:"error"`
	Progress        float32 `json:"progress"`
485
486
}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
487
func (s *llmServer) getServerStatus(ctx context.Context) (ServerStatus, error) {
488
489
490
491
492
493
	// Fail fast if its exited
	if s.cmd.ProcessState != nil {
		msg := ""
		if s.status != nil && s.status.LastErrMsg != "" {
			msg = s.status.LastErrMsg
		}
494
495
496
497
		if s.cmd.ProcessState.ExitCode() == -1 {
			// Most likely a signal killed it, log some more details to try to help troubleshoot
			slog.Warn("llama runner process no longer running", "sys", s.cmd.ProcessState.Sys(), "string", s.cmd.ProcessState.String())
		}
498
499
500
501
502
503
504
505
506
507
508
509
		return ServerStatusError, fmt.Errorf("llama runner process no longer running: %d %s", s.cmd.ProcessState.ExitCode(), msg)
	}

	req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("http://127.0.0.1:%d/health", s.port), nil)
	if err != nil {
		return ServerStatusError, fmt.Errorf("error creating GET request: %v", err)
	}
	req.Header.Set("Content-Type", "application/json")

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		if errors.Is(err, context.DeadlineExceeded) {
Michael Yang's avatar
Michael Yang committed
510
			return ServerStatusNotResponding, errors.New("server not responding")
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
		}
		return ServerStatusError, fmt.Errorf("health resp: %w", err)
	}
	defer resp.Body.Close()

	body, err := io.ReadAll(resp.Body)
	if err != nil {
		return ServerStatusError, fmt.Errorf("read health request: %w", err)
	}

	var status ServerStatusResp
	if err := json.Unmarshal(body, &status); err != nil {
		return ServerStatusError, fmt.Errorf("health unmarshal encode response: %w", err)
	}

	switch status.Status {
	case "ok":
		return ServerStatusReady, nil
	case "no slot available":
530
		return ServerStatusNoSlotsAvailable, nil
531
	case "loading model":
Daniel Hiltgen's avatar
Daniel Hiltgen committed
532
		s.loadProgress = status.Progress
533
534
535
536
537
538
		return ServerStatusLoadingModel, nil
	default:
		return ServerStatusError, fmt.Errorf("server error: %+v", status)
	}
}

539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
// getServerStatusRetry will retry if ServerStatusNoSlotsAvailable is received
func (s *llmServer) getServerStatusRetry(ctx context.Context) (ServerStatus, error) {
	var retries int
	for {
		status, err := s.getServerStatus(ctx)
		if err != nil {
			return status, err
		}

		if status == ServerStatusNoSlotsAvailable {
			if retries >= 10 {
				return status, fmt.Errorf("no slots available after %d retries", retries)
			}

			time.Sleep(5 * time.Millisecond)
			retries++
			continue
		}

		return status, nil
	}
}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
562
func (s *llmServer) Ping(ctx context.Context) error {
563
564
565
566
567
568
569
570
	_, err := s.getServerStatus(ctx)
	if err != nil {
		slog.Debug("server unhealthy", "error", err)
		return err
	}
	return nil
}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
571
func (s *llmServer) WaitUntilRunning(ctx context.Context) error {
572
	start := time.Now()
573
	stallDuration := envconfig.LoadTimeout()    // If no progress happens
574
	stallTimer := time.Now().Add(stallDuration) // give up if we stall
575
576
577

	slog.Info("waiting for llama runner to start responding")
	var lastStatus ServerStatus = -1
578
	fullyLoaded := false
ManniX-ITA's avatar
ManniX-ITA committed
579

580
581
	for {
		select {
Daniel Hiltgen's avatar
Daniel Hiltgen committed
582
		case <-ctx.Done():
583
			slog.Warn("client connection closed before server finished loading, aborting load")
584
			return fmt.Errorf("timed out waiting for llama runner to start: %w", ctx.Err())
585
		case err := <-s.done:
586
			return fmt.Errorf("llama runner process has terminated: %w", err)
587
588
		default:
		}
Daniel Hiltgen's avatar
Daniel Hiltgen committed
589
		if time.Now().After(stallTimer) {
ManniX-ITA's avatar
ManniX-ITA committed
590
			// timeout
591
592
593
594
			msg := ""
			if s.status != nil && s.status.LastErrMsg != "" {
				msg = s.status.LastErrMsg
			}
Daniel Hiltgen's avatar
Daniel Hiltgen committed
595
			return fmt.Errorf("timed out waiting for llama runner to start - progress %0.2f - %s", s.loadProgress, msg)
ManniX-ITA's avatar
ManniX-ITA committed
596
597
598
599
600
		}
		if s.cmd.ProcessState != nil {
			msg := ""
			if s.status != nil && s.status.LastErrMsg != "" {
				msg = s.status.LastErrMsg
601
			}
ManniX-ITA's avatar
ManniX-ITA committed
602
603
			return fmt.Errorf("llama runner process no longer running: %d %s", s.cmd.ProcessState.ExitCode(), msg)
		}
Daniel Hiltgen's avatar
Daniel Hiltgen committed
604
605
		ctx, cancel := context.WithTimeout(ctx, 200*time.Millisecond)
		defer cancel()
Daniel Hiltgen's avatar
Daniel Hiltgen committed
606
		priorProgress := s.loadProgress
Daniel Hiltgen's avatar
Daniel Hiltgen committed
607
608
609
610
611
		status, _ := s.getServerStatus(ctx)
		if lastStatus != status && status != ServerStatusReady {
			// Only log on status changes
			slog.Info("waiting for server to become available", "status", status.ToString())
		}
ManniX-ITA's avatar
ManniX-ITA committed
612
613
		switch status {
		case ServerStatusReady:
Daniel Hiltgen's avatar
Daniel Hiltgen committed
614
615
			s.loadDuration = time.Since(start)
			slog.Info(fmt.Sprintf("llama runner started in %0.2f seconds", s.loadDuration.Seconds()))
ManniX-ITA's avatar
ManniX-ITA committed
616
617
			return nil
		default:
Daniel Hiltgen's avatar
Daniel Hiltgen committed
618
			lastStatus = status
Daniel Hiltgen's avatar
Daniel Hiltgen committed
619
620
621
622
			// Reset the timer as long as we're making forward progress on the load
			if priorProgress != s.loadProgress {
				slog.Debug(fmt.Sprintf("model load progress %0.2f", s.loadProgress))
				stallTimer = time.Now().Add(stallDuration)
623
624
			} else if !fullyLoaded && int(s.loadProgress*100.0) >= 100 {
				slog.Debug("model load completed, waiting for server to become available", "status", status.ToString())
625
				stallTimer = time.Now().Add(stallDuration)
626
				fullyLoaded = true
Daniel Hiltgen's avatar
Daniel Hiltgen committed
627
			}
ManniX-ITA's avatar
ManniX-ITA committed
628
629
			time.Sleep(time.Millisecond * 250)
			continue
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
		}
	}
}

const jsonGrammar = `
root   ::= object
value  ::= object | array | string | number | ("true" | "false" | "null") ws

object ::=
  "{" ws (
            string ":" ws value
    ("," ws string ":" ws value)*
  )? "}" ws

array  ::=
  "[" ws (
            value
    ("," ws value)*
  )? "]" ws

string ::=
  "\"" (
652
    [^"\\\x7F\x00-\x1F] |
653
654
655
656
657
658
659
660
661
662
663
664
    "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) # escapes
  )* "\"" ws

number ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? ws

# Optional space: by convention, applied in this grammar after literal chars when allowed
ws ::= ([ \t\n] ws)?
`

const maxBufferSize = 512 * format.KiloByte

type ImageData struct {
665
666
667
	Data          []byte `json:"data"`
	ID            int    `json:"id"`
	AspectRatioID int    `json:"aspect_ratio_id"`
668
669
670
}

type completion struct {
671
672
673
674
675
	Content      string `json:"content"`
	Model        string `json:"model"`
	Prompt       string `json:"prompt"`
	Stop         bool   `json:"stop"`
	StoppedLimit bool   `json:"stopped_limit"`
676
677
678
679
680
681
682
683
684
685
686
687
688

	Timings struct {
		PredictedN  int     `json:"predicted_n"`
		PredictedMS float64 `json:"predicted_ms"`
		PromptN     int     `json:"prompt_n"`
		PromptMS    float64 `json:"prompt_ms"`
	}
}

type CompletionRequest struct {
	Prompt  string
	Format  string
	Images  []ImageData
Michael Yang's avatar
Michael Yang committed
689
	Options *api.Options
690
691
692
693
}

type CompletionResponse struct {
	Content            string
694
	DoneReason         string
695
696
697
698
699
700
701
	Done               bool
	PromptEvalCount    int
	PromptEvalDuration time.Duration
	EvalCount          int
	EvalDuration       time.Duration
}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
702
703
func (s *llmServer) Completion(ctx context.Context, req CompletionRequest, fn func(CompletionResponse)) error {
	if err := s.sem.Acquire(ctx, 1); err != nil {
704
705
706
707
708
		if errors.Is(err, context.Canceled) {
			slog.Info("aborting completion request due to client closing the connection")
		} else {
			slog.Error("Failed to acquire semaphore", "error", err)
		}
Daniel Hiltgen's avatar
Daniel Hiltgen committed
709
710
711
		return err
	}
	defer s.sem.Release(1)
712

713
	// put an upper limit on num_predict to avoid the model running on forever
714
715
716
717
	if req.Options.NumPredict < 0 || req.Options.NumPredict > 10*s.options.NumCtx {
		req.Options.NumPredict = 10 * s.options.NumCtx
	}

718
719
720
721
722
723
724
725
726
	request := map[string]any{
		"prompt":            req.Prompt,
		"stream":            true,
		"n_predict":         req.Options.NumPredict,
		"n_keep":            req.Options.NumKeep,
		"main_gpu":          req.Options.MainGPU,
		"temperature":       req.Options.Temperature,
		"top_k":             req.Options.TopK,
		"top_p":             req.Options.TopP,
727
		"min_p":             req.Options.MinP,
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
		"tfs_z":             req.Options.TFSZ,
		"typical_p":         req.Options.TypicalP,
		"repeat_last_n":     req.Options.RepeatLastN,
		"repeat_penalty":    req.Options.RepeatPenalty,
		"presence_penalty":  req.Options.PresencePenalty,
		"frequency_penalty": req.Options.FrequencyPenalty,
		"mirostat":          req.Options.Mirostat,
		"mirostat_tau":      req.Options.MirostatTau,
		"mirostat_eta":      req.Options.MirostatEta,
		"penalize_nl":       req.Options.PenalizeNewline,
		"seed":              req.Options.Seed,
		"stop":              req.Options.Stop,
		"image_data":        req.Images,
		"cache_prompt":      true,
	}

	// Make sure the server is ready
745
	status, err := s.getServerStatusRetry(ctx)
746
747
748
	if err != nil {
		return err
	} else if status != ServerStatusReady {
Daniel Hiltgen's avatar
Daniel Hiltgen committed
749
		return fmt.Errorf("unexpected server status: %s", status.ToString())
750
751
752
753
754
755
756
757
758
	}

	if req.Format == "json" {
		request["grammar"] = jsonGrammar
		if !strings.Contains(strings.ToLower(req.Prompt), "json") {
			slog.Warn("Prompt does not specify that the LLM should response in JSON, but JSON format is expected. For best results specify that JSON is expected in the system prompt.")
		}
	}

759
760
761
762
	// Handling JSON marshaling with special characters unescaped.
	buffer := &bytes.Buffer{}
	enc := json.NewEncoder(buffer)
	enc.SetEscapeHTML(false)
763

764
765
766
	if err := enc.Encode(request); err != nil {
		return fmt.Errorf("failed to marshal data: %v", err)
	}
767

768
769
770
771
772
773
	endpoint := fmt.Sprintf("http://127.0.0.1:%d/completion", s.port)
	serverReq, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, buffer)
	if err != nil {
		return fmt.Errorf("error creating POST request: %v", err)
	}
	serverReq.Header.Set("Content-Type", "application/json")
774

775
776
777
778
779
	res, err := http.DefaultClient.Do(serverReq)
	if err != nil {
		return fmt.Errorf("POST predict: %v", err)
	}
	defer res.Body.Close()
780

781
782
	if res.StatusCode >= 400 {
		bodyBytes, err := io.ReadAll(res.Body)
783
		if err != nil {
784
			return fmt.Errorf("failed reading llm error response: %w", err)
785
		}
786
787
788
		log.Printf("llm predict error: %s", bodyBytes)
		return fmt.Errorf("%s", bodyBytes)
	}
789

790
791
792
	scanner := bufio.NewScanner(res.Body)
	buf := make([]byte, 0, maxBufferSize)
	scanner.Buffer(buf, maxBufferSize)
793

794
795
796
	// keep track of the last token generated, this is used to abort if the model starts looping
	var lastToken string
	var tokenRepeat int
797

798
799
800
801
802
803
804
805
806
807
	for scanner.Scan() {
		select {
		case <-ctx.Done():
			// This handles the request cancellation
			return ctx.Err()
		default:
			line := scanner.Bytes()
			if len(line) == 0 {
				continue
			}
808

809
			// slog.Debug("got line", "line", string(line))
810
811
			evt, ok := bytes.CutPrefix(line, []byte("data: "))
			if !ok {
812
				evt = line
813
			}
814

815
816
			var c completion
			if err := json.Unmarshal(evt, &c); err != nil {
817
				return fmt.Errorf("error unmarshalling llm prediction response: %v", err)
818
819
820
821
822
823
824
825
			}
			switch {
			case strings.TrimSpace(c.Content) == lastToken:
				tokenRepeat++
			default:
				lastToken = strings.TrimSpace(c.Content)
				tokenRepeat = 0
			}
826

827
828
829
830
831
			// 30 picked as an arbitrary max token repeat limit, modify as needed
			if tokenRepeat > 30 {
				slog.Debug("prediction aborted, token repeat limit reached")
				return ctx.Err()
			}
832

833
834
835
836
837
			if c.Content != "" {
				fn(CompletionResponse{
					Content: c.Content,
				})
			}
838

839
			if c.Stop {
840
841
842
843
844
				doneReason := "stop"
				if c.StoppedLimit {
					doneReason = "length"
				}

845
846
				fn(CompletionResponse{
					Done:               true,
847
					DoneReason:         doneReason,
848
849
850
851
852
853
					PromptEvalCount:    c.Timings.PromptN,
					PromptEvalDuration: parseDurationMs(c.Timings.PromptMS),
					EvalCount:          c.Timings.PredictedN,
					EvalDuration:       parseDurationMs(c.Timings.PredictedMS),
				})
				return nil
854
855
			}
		}
856
	}
857

858
	if err := scanner.Err(); err != nil {
859
		if strings.Contains(err.Error(), "unexpected EOF") || strings.Contains(err.Error(), "forcibly closed") {
860
			s.Close()
861
			var msg string
862
863
			if s.status != nil && s.status.LastErrMsg != "" {
				msg = s.status.LastErrMsg
864
865
			} else {
				msg = err.Error()
866
			}
867
			return fmt.Errorf("an error was encountered while running the model: %s", msg)
868
869
		}

870
		return fmt.Errorf("error reading llm response: %v", err)
871
872
	}

873
	return nil
874
875
}

876
877
type EmbeddingRequest struct {
	Content string `json:"content"`
878
879
}

880
881
type EmbeddingResponse struct {
	Embedding []float32 `json:"embedding"`
882
883
}

884
885
func (s *llmServer) Embedding(ctx context.Context, input string) ([]float32, error) {
	if err := s.sem.Acquire(ctx, 1); err != nil {
886
887
888
889
890
		if errors.Is(err, context.Canceled) {
			slog.Info("aborting embedding request due to client closing the connection")
		} else {
			slog.Error("Failed to acquire semaphore", "error", err)
		}
Daniel Hiltgen's avatar
Daniel Hiltgen committed
891
892
		return nil, err
	}
893
	defer s.sem.Release(1)
894

895
	// Make sure the server is ready
896
	status, err := s.getServerStatusRetry(ctx)
897
898
899
	if err != nil {
		return nil, err
	} else if status != ServerStatusReady {
Daniel Hiltgen's avatar
Daniel Hiltgen committed
900
		return nil, fmt.Errorf("unexpected server status: %s", status.ToString())
901
902
	}

903
	data, err := json.Marshal(EmbeddingRequest{Content: input})
Michael Yang's avatar
Michael Yang committed
904
	if err != nil {
905
906
907
		return nil, fmt.Errorf("error marshaling embed data: %w", err)
	}

908
	r, err := http.NewRequestWithContext(ctx, http.MethodPost, fmt.Sprintf("http://127.0.0.1:%d/embedding", s.port), bytes.NewBuffer(data))
909
910
911
	if err != nil {
		return nil, fmt.Errorf("error creating embed request: %w", err)
	}
912
	r.Header.Set("Content-Type", "application/json")
913

914
	resp, err := http.DefaultClient.Do(r)
915
916
917
918
919
920
921
922
923
924
925
	if err != nil {
		return nil, fmt.Errorf("do embedding request: %w", err)
	}
	defer resp.Body.Close()

	body, err := io.ReadAll(resp.Body)
	if err != nil {
		return nil, fmt.Errorf("error reading embed response: %w", err)
	}

	if resp.StatusCode >= 400 {
926
		log.Printf("llm embedding error: %s", body)
927
928
929
		return nil, fmt.Errorf("%s", body)
	}

930
	var e EmbeddingResponse
931
	if err := json.Unmarshal(body, &e); err != nil {
932
933
934
		return nil, fmt.Errorf("unmarshal tokenize response: %w", err)
	}

935
	return e.Embedding, nil
936
937
}

Michael Yang's avatar
Michael Yang committed
938
939
940
941
942
943
944
945
type TokenizeRequest struct {
	Content string `json:"content"`
}

type TokenizeResponse struct {
	Tokens []int `json:"tokens"`
}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
946
func (s *llmServer) Tokenize(ctx context.Context, content string) ([]int, error) {
947
948
949
950
951
952
	s.modelLock.Lock()
	defer s.modelLock.Unlock()
	if s.model != nil {
		return s.model.Tokenize(content, false, true)
	}

Michael Yang's avatar
Michael Yang committed
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
	// Make sure the server is ready
	status, err := s.getServerStatus(ctx)
	if err != nil {
		return nil, err
	} else if status != ServerStatusReady && status != ServerStatusNoSlotsAvailable {
		return nil, fmt.Errorf("unexpected server status: %s", status.ToString())
	}

	data, err := json.Marshal(TokenizeRequest{Content: content})
	if err != nil {
		return nil, fmt.Errorf("marshaling encode data: %w", err)
	}

	req, err := http.NewRequestWithContext(ctx, http.MethodPost, fmt.Sprintf("http://127.0.0.1:%d/tokenize", s.port), bytes.NewBuffer(data))
	if err != nil {
		return nil, fmt.Errorf("encode request: %w", err)
	}
	req.Header.Set("Content-Type", "application/json")

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, fmt.Errorf("do encode request: %w", err)
	}
	defer resp.Body.Close()
977
978
979
	if resp.StatusCode == http.StatusNotFound {
		if s.model == nil {
			slog.Debug("new runner detected, loading model for cgo tokenization")
980
981
982
983
			m, err := llama.LoadModelFromFile(s.modelPath, llama.ModelParams{VocabOnly: true})
			if err != nil {
				return nil, err
			}
984
985
986
987
			s.model = m
		}
		return s.model.Tokenize(content, false, true)
	}
Michael Yang's avatar
Michael Yang committed
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012

	body, err := io.ReadAll(resp.Body)
	if err != nil {
		return nil, fmt.Errorf("read encode request: %w", err)
	}

	if resp.StatusCode >= 400 {
		log.Printf("llm encode error: %s", body)
		return nil, fmt.Errorf("%s", body)
	}

	var encoded TokenizeResponse
	if err := json.Unmarshal(body, &encoded); err != nil {
		return nil, fmt.Errorf("unmarshal encode response: %w", err)
	}

	return encoded.Tokens, nil
}

type DetokenizeRequest struct {
	Tokens []int `json:"tokens"`
}

type DetokenizeResponse struct {
	Content string `json:"content"`
1013
1014
}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
1015
func (s *llmServer) Detokenize(ctx context.Context, tokens []int) (string, error) {
1016
1017
1018
1019
1020
1021
1022
1023
1024
	s.modelLock.Lock()
	defer s.modelLock.Unlock()
	if s.model != nil {
		var resp string
		for _, token := range tokens {
			resp += s.model.TokenToPiece(token)
		}
		return resp, nil
	}
Michael Yang's avatar
Michael Yang committed
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
	// Make sure the server is ready
	status, err := s.getServerStatus(ctx)
	if err != nil {
		return "", err
	} else if status != ServerStatusReady && status != ServerStatusNoSlotsAvailable {
		return "", fmt.Errorf("unexpected server status: %s", status.ToString())
	}

	data, err := json.Marshal(DetokenizeRequest{Tokens: tokens})
	if err != nil {
		return "", fmt.Errorf("marshaling decode data: %w", err)
	}

	req, err := http.NewRequestWithContext(ctx, http.MethodPost, fmt.Sprintf("http://127.0.0.1:%d/detokenize", s.port), bytes.NewBuffer(data))
	if err != nil {
		return "", fmt.Errorf("decode request: %w", err)
	}
	req.Header.Set("Content-Type", "application/json")

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return "", fmt.Errorf("do decode request: %w", err)
	}
	defer resp.Body.Close()
1049
1050
1051
	if resp.StatusCode == http.StatusNotFound {
		if s.model == nil {
			slog.Debug("new runner detected, loading model for cgo tokenization")
1052
1053
1054
1055
			m, err := llama.LoadModelFromFile(s.modelPath, llama.ModelParams{VocabOnly: true})
			if err != nil {
				return "", err
			}
1056
1057
1058
1059
1060
1061
1062
1063
			s.model = m
		}
		var resp string
		for _, token := range tokens {
			resp += s.model.TokenToPiece(token)
		}
		return resp, nil
	}
Michael Yang's avatar
Michael Yang committed
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080

	body, err := io.ReadAll(resp.Body)
	if err != nil {
		return "", fmt.Errorf("read decode request: %w", err)
	}

	if resp.StatusCode >= 400 {
		log.Printf("llm decode error: %s", body)
		return "", fmt.Errorf("%s", body)
	}

	var decoded DetokenizeResponse
	if err := json.Unmarshal(body, &decoded); err != nil {
		return "", fmt.Errorf("unmarshal encode response: %w", err)
	}

	return decoded.Content, nil
1081
1082
}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
1083
func (s *llmServer) Close() error {
1084
	s.modelLock.Lock()
1085
1086
1087
1088
	if s.model != nil {
		llama.FreeModel(s.model)
		s.model = nil
	}
1089
1090
	s.modelLock.Unlock()

1091
1092
	if s.cmd != nil {
		slog.Debug("stopping llama server")
1093
1094
1095
		if err := s.cmd.Process.Kill(); err != nil {
			return err
		}
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1096
1097
1098
1099
1100
		// if ProcessState is already populated, Wait already completed, no need to wait again
		if s.cmd.ProcessState == nil {
			slog.Debug("waiting for llama server to exit")
			<-s.done
		}
1101
1102

		slog.Debug("llama server stopped")
1103
1104
1105
1106
1107
	}

	return nil
}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
1108
func (s *llmServer) EstimatedVRAM() uint64 {
1109
	return s.estimate.VRAMSize
Daniel Hiltgen's avatar
Daniel Hiltgen committed
1110
1111
}

1112
func (s *llmServer) EstimatedTotal() uint64 {
1113
	return s.estimate.TotalSize
1114
1115
}

Daniel Hiltgen's avatar
Daniel Hiltgen committed
1116
func (s *llmServer) EstimatedVRAMByGPU(gpuID string) uint64 {
1117
1118
	for i, gpu := range s.gpus {
		if gpu.ID == gpuID {
1119
1120
1121
			if i < len(s.estimate.GPUSizes) {
				return s.estimate.GPUSizes[i]
			}
1122
1123
1124
1125
1126
		}
	}
	return 0
}

1127
1128
1129
1130
1131
1132
1133
1134
func parseDurationMs(ms float64) time.Duration {
	dur, err := time.ParseDuration(fmt.Sprintf("%fms", ms))
	if err != nil {
		panic(err)
	}

	return dur
}