Merge pull request #1146 from dhiltgen/ext_server_cgo

Add cgo implementation for llama.cpp

llm/llama.cpp/patches/0004-metal-add-missing-barriers-for-mul-mat-2699.patch

-From 14b1d7e6f720dee41ce5a826376df738096d9033 Mon Sep 17 00:00:00 2001
-From: Shouzheng Liu <lshzh.hi@gmail.com>
-Date: Tue, 22 Aug 2023 02:18:40 -0400
-Subject: [PATCH] metal : add missing barriers for mul-mat (#2699)
-
----
- ggml-metal.metal | 5 +++--
- 1 file changed, 3 insertions(+), 2 deletions(-)
-
-diff --git a/ggml-metal.metal b/ggml-metal.metal
-index 88d48f6..ce3541f 100644
---- a/ggml-metal.metal
-+++ b/ggml-metal.metal
-@@ -1850,6 +1850,7 @@ kernel void kernel_mul_mm(device const  uchar * src0,
-         //load data and store to threadgroup memory
-         half4x4 temp_a;
-         dequantize_func(x, il, temp_a);
-+        threadgroup_barrier(mem_flags::mem_threadgroup);
-         #pragma unroll(16)
-         for (int i = 0; i < 16; i++) {
-             *(sa + SG_MAT_SIZE * ((tiitg / THREAD_PER_ROW / 8) \
-@@ -1895,14 +1896,14 @@ kernel void kernel_mul_mm(device const  uchar * src0,
-         }
-     } else {
-         // block is smaller than 64x32, we should avoid writing data outside of the matrix
-+        threadgroup_barrier(mem_flags::mem_threadgroup);
-         threadgroup float *temp_str = ((threadgroup float *)shared_memory) \
-                                       + 32 * (sgitg&1) + (16 * (sgitg>>1)) * BLOCK_SIZE_M;
-         for (int i = 0; i < 8; i++) {
--            threadgroup_barrier(mem_flags::mem_device);
-             simdgroup_store(c_res[i], temp_str + 8 * (i%4) + 8 * BLOCK_SIZE_M * (i/4), BLOCK_SIZE_M);
-         }
- 
--        threadgroup_barrier(mem_flags::mem_device);
-+        threadgroup_barrier(mem_flags::mem_threadgroup);
-         device float *C = dst + BLOCK_SIZE_M * r0 + (BLOCK_SIZE_N * r1) * ne0 + im*ne1*ne0;
-         if (sgitg==0) {
-             for (int i = 0; i < n_rows; i++) {
--- 
-2.41.0
-

llm/llama.cpp/patches/0005-ggml-support-CUDA-s-half-type-for-aarch64-1455-2670.patch

-From 1e3bc523d8053a77df3ac7126a84d0297ee97ef6 Mon Sep 17 00:00:00 2001
-From: Kylin <56434533+KyL0N@users.noreply.github.com>
-Date: Tue, 22 Aug 2023 15:14:23 +0800
-Subject: [PATCH] ggml : support CUDA's half type for aarch64(#1455) (#2670)
-
-* ggml: support CUDA's half type for aarch64(#1455)
-support CUDA's half type for aarch64 in ggml_fp16_t definition
-
-* ggml: use __CUDACC__ to recognise nvcc compiler
----
- ggml.h | 5 +++--
- 1 file changed, 3 insertions(+), 2 deletions(-)
-
-diff --git a/ggml.h b/ggml.h
-index 544ad2d..0ec7ec5 100644
---- a/ggml.h
-+++ b/ggml.h
-@@ -259,8 +259,9 @@
- extern "C" {
- #endif
- 
--#ifdef __ARM_NEON
--    // we use the built-in 16-bit float type
-+#if defined(__ARM_NEON) && defined(__CUDACC__)
-+    typedef half ggml_fp16_t;
-+#elif defined(__ARM_NEON)
-     typedef __fp16 ggml_fp16_t;
- #else
-     typedef uint16_t ggml_fp16_t;
--- 
-2.39.2 (Apple Git-143)
-

llm/llm.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/jmorganca/ollama/api"
 	"github.com/jmorganca/ollama/format"
+	"github.com/jmorganca/ollama/gpu"
 )
 
 type LLM interface {
@@ -18,11 +19,11 @@ type LLM interface {
 	Embedding(context.Context, string) ([]float64, error)
 	Encode(context.Context, string) ([]int, error)
 	Decode(context.Context, []int) (string, error)
-	SetOptions(api.Options)
 	Close()
-	Ping(context.Context) error
 }
 
+var AvailableShims = map[string]string{}
+
 func New(workDir, model string, adapters, projectors []string, opts api.Options) (LLM, error) {
 	if _, err := os.Stat(model); err != nil {
 		return nil, err
@@ -76,16 +77,27 @@ func New(workDir, model string, adapters, projectors []string, opts api.Options)
 		}
 	}
 
-	switch ggml.Name() {
-	case "gguf":
-		// TODO: gguf will load these options automatically from the model binary
 	opts.NumGQA = 0
 	opts.RopeFrequencyBase = 0.0
 	opts.RopeFrequencyScale = 0.0
-		return newLlama(model, adapters, projectors, chooseRunners(workDir, "gguf"), ggml.NumLayers(), opts)
-	case "ggml", "ggmf", "ggjt", "ggla":
-		return newLlama(model, adapters, projectors, chooseRunners(workDir, "ggml"), ggml.NumLayers(), opts)
-	default:
-		return nil, fmt.Errorf("unknown ggml type: %s", ggml.ModelFamily())
+	gpuInfo := gpu.GetGPUInfo()
+	return newLlmServer(gpuInfo.Library, model, adapters, projectors, ggml.NumLayers(), opts)
+}
+
+// Give any native cgo implementations an opportunity to initialize
+func Init(workdir string) error {
+	return nativeInit(workdir)
+}
+
+func newLlmServer(library, model string, adapters, projectors []string, numLayers int64, opts api.Options) (extServer, error) {
+	if _, libPresent := AvailableShims[library]; libPresent && library != "default" {
+		srv, err := newDynamicShimExtServer(AvailableShims[library], model, adapters, projectors, numLayers, opts)
+		if err == nil {
+			return srv, nil
+		}
+		log.Printf("Failed to load dynamic library - falling back to CPU mode %s", err)
 	}
+
+	return newDefaultExtServer(model, adapters, projectors, numLayers, opts)
+
 }

llm/shim_darwin.go

+package llm
+
+import (
+	"embed"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/jmorganca/ollama/api"
+)
+
+//go:embed llama.cpp/gguf/ggml-metal.metal
+var libEmbed embed.FS
+
+func newDynamicShimExtServer(library, model string, adapters, projectors []string, numLayers int64, opts api.Options) (extServer, error) {
+	// should never happen...
+	return nil, fmt.Errorf("Dynamic library loading not supported on Mac")
+}
+
+func nativeInit(workdir string) error {
+	_, err := extractDynamicLibs(workdir, "llama.cpp/gguf/ggml-metal.metal")
+	if err != nil {
+		if err == payloadMissing {
+			// TODO perhaps consider this a hard failure on arm macs?
+			log.Printf("ggml-meta.metal payload missing")
+			return nil
+		}
+		return err
+	}
+	os.Setenv("GGML_METAL_PATH_RESOURCES", workdir)
+	return nil
+}

llm/shim_ext_server.go

+//go:build !darwin
+
+package llm
+
+/*
+
+#include <stdlib.h>
+#include "dynamic_shim.h"
+
+*/
+import "C"
+import (
+	"context"
+	"embed"
+	"errors"
+	"fmt"
+	"io/fs"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"unsafe"
+
+	"github.com/jmorganca/ollama/api"
+)
+
+//go:embed llama.cpp/gguf/build/lib/*
+var libEmbed embed.FS
+
+var RocmShimMissing = fmt.Errorf("ROCm shim library not included in this build of ollama. Radeon GPUs are not supported")
+
+type shimExtServer struct {
+	s       C.struct_dynamic_llama_server
+	options api.Options
+}
+
+// Note: current implementation does not support concurrent instantiations
+var shimMutex sync.Mutex
+var llm *shimExtServer
+
+func (llm *shimExtServer) llama_server_init(sparams *C.ext_server_params_t, err *C.ext_server_resp_t) {
+	C.dynamic_shim_llama_server_init(llm.s, sparams, err)
+}
+func (llm *shimExtServer) llama_server_start() {
+	C.dynamic_shim_llama_server_start(llm.s)
+}
+func (llm *shimExtServer) llama_server_stop() {
+	C.dynamic_shim_llama_server_stop(llm.s)
+}
+
+func (llm *shimExtServer) llama_server_completion(json_req *C.char, resp *C.ext_server_resp_t) {
+	C.dynamic_shim_llama_server_completion(llm.s, json_req, resp)
+}
+func (llm *shimExtServer) llama_server_completion_next_result(task_id C.int, resp *C.ext_server_task_result_t) {
+	C.dynamic_shim_llama_server_completion_next_result(llm.s, task_id, resp)
+}
+func (llm *shimExtServer) llama_server_completion_cancel(task_id C.int, err *C.ext_server_resp_t) {
+	C.dynamic_shim_llama_server_completion_cancel(llm.s, task_id, err)
+}
+func (llm *shimExtServer) llama_server_release_task_result(result *C.ext_server_task_result_t) {
+	C.dynamic_shim_llama_server_release_task_result(llm.s, result)
+}
+
+func (llm *shimExtServer) llama_server_tokenize(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t) {
+	C.dynamic_shim_llama_server_tokenize(llm.s, json_req, json_resp, err)
+}
+func (llm *shimExtServer) llama_server_detokenize(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t) {
+	C.dynamic_shim_llama_server_detokenize(llm.s, json_req, json_resp, err)
+}
+func (llm *shimExtServer) llama_server_embedding(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t) {
+	C.dynamic_shim_llama_server_embedding(llm.s, json_req, json_resp, err)
+}
+func (llm *shimExtServer) llama_server_release_json_resp(json_resp **C.char) {
+	C.dynamic_shim_llama_server_release_json_resp(llm.s, json_resp)
+}
+
+func newDynamicShimExtServer(library, model string, adapters, projectors []string, numLayers int64, opts api.Options) (extServer, error) {
+	shimMutex.Lock()
+	defer shimMutex.Unlock()
+	libPath := C.CString(library)
+	defer C.free(unsafe.Pointer(libPath))
+	resp := newExtServerResp(128)
+	defer freeExtServerResp(resp)
+	var srv C.struct_dynamic_llama_server
+	C.dynamic_shim_init(libPath, &srv, &resp)
+	if resp.id < 0 {
+		return nil, fmt.Errorf("Unable to load dynamic library: %s", C.GoString(resp.msg))
+	}
+	llm = &shimExtServer{
+		s:       srv,
+		options: opts,
+	}
+	log.Printf("Loading Dynamic Shim llm server: %s", library)
+	return newExtServer(llm, model, adapters, projectors, numLayers, opts)
+}
+
+func (llm *shimExtServer) Predict(ctx context.Context, pred PredictOpts, fn func(PredictResult)) error {
+	return predict(llm, llm.options, ctx, pred, fn)
+}
+
+func (llm *shimExtServer) Encode(ctx context.Context, prompt string) ([]int, error) {
+	return encode(llm, ctx, prompt)
+}
+
+func (llm *shimExtServer) Decode(ctx context.Context, tokens []int) (string, error) {
+	return decode(llm, ctx, tokens)
+}
+
+func (llm *shimExtServer) Embedding(ctx context.Context, input string) ([]float64, error) {
+	return embedding(llm, ctx, input)
+}
+
+func (llm *shimExtServer) Close() {
+	close(llm)
+}
+
+func nativeInit(workdir string) error {
+	libs, err := extractDynamicLibs(workdir, "llama.cpp/gguf/build/lib/*server*")
+	if err != nil {
+		if err == payloadMissing {
+			log.Printf("%s", payloadMissing)
+			return nil
+		}
+		return err
+	}
+	for _, lib := range libs {
+		libName := strings.Split(strings.TrimPrefix(filepath.Base(lib), "lib"), ".")[0]
+		AvailableShims[libName] = lib
+	}
+
+	// Only check ROCm access if we have the dynamic lib loaded
+	if _, rocmPresent := AvailableShims["rocm_server"]; rocmPresent {
+		// Verify we have permissions - either running as root, or we have group access to the driver
+		fd, err := os.OpenFile("/dev/kfd", os.O_RDWR, 0666)
+		if err != nil {
+			if errors.Is(err, fs.ErrPermission) {
+				log.Fatalf("Radeon card detected, but permissions not set up properly.  Either run ollama as root, or add you user account to the render group.")
+				return err
+			} else if errors.Is(err, fs.ErrNotExist) {
+				// expected behavior without a radeon card
+				return nil
+			}
+
+			return fmt.Errorf("failed to check permission on /dev/kfd: %w", err)
+		}
+		fd.Close()
+
+	}
+
+	return nil
+}

scripts/build_darwin.sh

@@ -9,7 +9,7 @@ mkdir -p dist
 
 for TARGETARCH in arm64 amd64; do
     GOOS=darwin GOARCH=$TARGETARCH go generate ./...
-    GOOS=darwin GOARCH=$TARGETARCH go build -o dist/ollama-darwin-$TARGETARCH
+    CGO_ENABLED=1 GOOS=darwin GOARCH=$TARGETARCH go build -o dist/ollama-darwin-$TARGETARCH
     rm -rf llm/llama.cpp/*/build
 done
 

scripts/build_linux.sh

@@ -7,8 +7,8 @@ export GOFLAGS="'-ldflags=-w -s \"-X=github.com/jmorganca/ollama/version.Version
 
 mkdir -p dist
 
-for TARGETARCH in arm64 amd64; do
-    docker buildx build --load --platform=linux/$TARGETARCH --build-arg=VERSION --build-arg=GOFLAGS -f Dockerfile.build -t builder:$TARGETARCH .
+for TARGETARCH in amd64 arm64; do
+    docker buildx build --load --progress=plain --platform=linux/$TARGETARCH --build-arg=VERSION --build-arg=GOFLAGS -f Dockerfile.build -t builder:$TARGETARCH .
     docker create --platform linux/$TARGETARCH --name builder-$TARGETARCH builder:$TARGETARCH
     docker cp builder-$TARGETARCH:/go/src/github.com/jmorganca/ollama/ollama ./dist/ollama-linux-$TARGETARCH
     docker rm builder-$TARGETARCH

scripts/build_remote.py

+#!/usr/bin/env python3
+import subprocess
+import sys
+from urllib.parse import urlparse
+from git import Repo
+
+# Helper script to be able to build on remote repos using git to push local changes
+# (e.g. particularly helpful to target a remote windows build system)
+#
+# Typical windows remote git config looks like this:
+#
+#[remote "windows-pa"]
+#        url = jdoe@desktop-foo:C:/Users/Jdoe/code/ollama
+#        fetch = +refs/heads/*:refs/remotes/windows-pa/*
+#        uploadpack = powershell git upload-pack
+#        receivepack = powershell git receive-pack
+#
+
+# TODO - add argpare and make this more configurable 
+# - force flag becomes optional
+# - generate, build or test ...
+
+# Note: remote repo will need this run once:
+# git config --local receive.denyCurrentBranch updateInstead
+repo = Repo(".")
+
+# On linux, add links in /usr/local/bin to the go binaries to avoid needing this
+# GoCmd = "/usr/local/go/bin/go" 
+GoCmd = "go" 
+
+if repo.is_dirty():
+    print("Tree is dirty.  Commit your changes before running this script")
+    sys.exit(1)
+
+if len(sys.argv) != 2:
+    print("Please specify the remote name: " + ', '.join([r.name for r in repo.remotes]))
+    sys.exit(1)
+remote_name = sys.argv[1]
+
+remote = {r.name: r for r in repo.remotes}[remote_name]
+raw_url = list(remote.urls)[0]
+url = urlparse(raw_url)
+# Windows urls don't quite parse properly
+if url.scheme == "" and url.netloc == "":
+    url = urlparse("ssh://" + raw_url)
+print("URL: " + str(url))
+netloc = url.netloc.split(":")[0]
+path = url.path
+branch_name = repo.active_branch.name
+
+print("Force pushing content to remote...")
+# Use with care given the force push
+remote.push(force=True).raise_if_error()
+
+print("Ensuring correct branch checked out on remote via ssh...")
+subprocess.check_call(['ssh', netloc, 'cd', path, ';', 'git', 'checkout', branch_name])
+
+
+# TODO - add some hardening to try to figure out how to set up the path properly
+# subprocess.check_call(['ssh', netloc, 'cd', path, ';', 'env'])
+# TODO - or consider paramiko maybe
+
+print("Performing generate")
+subprocess.check_call(['ssh', netloc, 'cd', path, ';', GoCmd, 'generate', './...'])
+
+print("Building")
+subprocess.check_call(['ssh', netloc, 'cd', path, ';', GoCmd, 'build', '.'])
+

scripts/setup_integration_tests.sh

+#!/bin/bash
+
+# This script sets up integration tests which run the full stack to verify
+# inference locally
+set -e
+set -o pipefail
+
+REPO=$(dirname $0)/../
+export OLLAMA_MODELS=${REPO}/test_data/models
+REGISTRY_SCHEME=https
+REGISTRY=registry.ollama.ai
+TEST_MODELS=("library/orca-mini:latest" "library/llava:7b")
+ACCEPT_HEADER="Accept: application/vnd.docker.distribution.manifest.v2+json"
+
+for model in ${TEST_MODELS[@]}; do
+    TEST_MODEL=$(echo ${model} | cut -f1 -d:)
+    TEST_MODEL_TAG=$(echo ${model} | cut -f2 -d:)
+    mkdir -p ${OLLAMA_MODELS}/manifests/${REGISTRY}/${TEST_MODEL}/
+    mkdir -p ${OLLAMA_MODELS}/blobs/
+
+    echo "Pulling manifest for ${TEST_MODEL}:${TEST_MODEL_TAG}"
+    curl -s --header "${ACCEPT_HEADER}" \
+        -o ${OLLAMA_MODELS}/manifests/${REGISTRY}/${TEST_MODEL}/${TEST_MODEL_TAG} \
+        ${REGISTRY_SCHEME}://${REGISTRY}/v2/${TEST_MODEL}/manifests/${TEST_MODEL_TAG} 
+
+    CFG_HASH=$(cat ${OLLAMA_MODELS}/manifests/${REGISTRY}/${TEST_MODEL}/${TEST_MODEL_TAG} | jq -r ".config.digest")
+    echo "Pulling config blob ${CFG_HASH}"
+    curl -L -C - --header "${ACCEPT_HEADER}" \
+            -o ${OLLAMA_MODELS}/blobs/${CFG_HASH} \
+            ${REGISTRY_SCHEME}://${REGISTRY}/v2/${TEST_MODEL}/blobs/${CFG_HASH}
+
+    for LAYER in $(cat ${OLLAMA_MODELS}/manifests/${REGISTRY}/${TEST_MODEL}/${TEST_MODEL_TAG} | jq -r ".layers[].digest" ) ; do
+        echo "Pulling blob ${LAYER}"
+        curl -L -C - --header "${ACCEPT_HEADER}" \
+            -o ${OLLAMA_MODELS}/blobs/${LAYER} \
+            ${REGISTRY_SCHEME}://${REGISTRY}/v2/${TEST_MODEL}/blobs/${LAYER}
+    done
+done

server/images.go

@@ -418,6 +418,31 @@ func CreateModel(ctx context.Context, name, modelFileDir string, commands []pars
 					return err
 				}
 
+				// if the model is not in gguf format, pull the base model to try and get it in gguf format
+				if fromConfig.ModelFormat != "gguf" {
+					fn(api.ProgressResponse{Status: "updating base model"})
+					parent, err := GetModel(c.Args)
+					if err != nil {
+						return err
+					}
+					if err := PullModel(ctx, parent.OriginalModel, &RegistryOptions{}, fn); err != nil {
+						log.Printf("error pulling model: %v", err)
+					}
+					// Reset the file pointer to the beginning of the file
+					_, err = fromConfigFile.Seek(0, 0)
+					if err != nil {
+						return fmt.Errorf("update from config after pull: %w", err)
+					}
+					if err := json.NewDecoder(fromConfigFile).Decode(&fromConfig); err != nil {
+						return err
+					}
+				}
+
+				// if the model is still not in gguf format, error out
+				if fromConfig.ModelFormat != "gguf" {
+					return fmt.Errorf("%s is not in gguf format, this base model is not compatible with this version of ollama", c.Args)
+				}
+
 				config.SetModelFormat(fromConfig.ModelFormat)
 				config.SetModelFamily(append(fromConfig.ModelFamilies, fromConfig.ModelFamily)...)
 				config.SetModelType(fromConfig.ModelType)
@@ -456,16 +481,22 @@ func CreateModel(ctx context.Context, name, modelFileDir string, commands []pars
 			defer bin.Close()
 
 			var offset int64
+		CREATE:
 			for {
 				fn(api.ProgressResponse{Status: "creating model layer"})
 
 				bin.Seek(offset, io.SeekStart)
 				ggml, err := llm.DecodeGGML(bin)
-				if errors.Is(err, io.EOF) {
-					break
-				} else if err != nil {
+				if err != nil {
+					switch {
+					case errors.Is(err, io.EOF):
+						break CREATE
+					case errors.Is(err, llm.ErrUnsupportedFormat):
+						return fmt.Errorf("model binary specified in FROM field is not a valid gguf format model, %w", err)
+					default:
 						return err
 					}
+				}
 
 				config.SetModelFormat(ggml.Name())
 				config.SetModelFamily(ggml.ModelFamily())

server/llm_test.go

+package server
+
+import (
+	"context"
+	"os"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/jmorganca/ollama/api"
+	"github.com/jmorganca/ollama/llm"
+)
+
+// TODO - this would ideally be in the llm package, but that would require some refactoring of interfaces in the server
+//        package to avoid circular dependencies
+
+// WARNING - these tests will fail on mac if you don't manually copy ggml-metal.metal to this dir (./server)
+//
+// TODO - Fix this ^^
+
+var (
+	req = [2]api.GenerateRequest{
+		{
+			Model:   "orca-mini",
+			Prompt:  "tell me a short story about agi?",
+			Options: map[string]interface{}{},
+		}, {
+			Model:   "orca-mini",
+			Prompt:  "what is the origin of the us thanksgiving holiday?",
+			Options: map[string]interface{}{},
+		},
+	}
+	resp = [2]string{
+		"once upon a time",
+		"united states thanksgiving",
+	}
+)
+
+func TestIntegrationSimpleOrcaMini(t *testing.T) {
+	SkipIFNoTestData(t)
+	workDir, err := os.MkdirTemp("", "ollama")
+	require.NoError(t, err)
+	defer os.RemoveAll(workDir)
+	require.NoError(t, llm.Init(workDir))
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*60)
+	defer cancel()
+	opts := api.DefaultOptions()
+	opts.Seed = 42
+	opts.Temperature = 0.0
+	model, llmRunner := PrepareModelForPrompts(t, req[0].Model, opts)
+	defer llmRunner.Close()
+	response := OneShotPromptResponse(t, ctx, req[0], model, llmRunner)
+	assert.Contains(t, strings.ToLower(response), resp[0])
+}
+
+// TODO
+// The server always loads a new runner and closes the old one, which forces serial execution
+// At present this test case fails with concurrency problems.  Eventually we should try to
+// get true concurrency working with n_parallel support in the backend
+func TestIntegrationConcurrentPredictOrcaMini(t *testing.T) {
+	SkipIFNoTestData(t)
+
+	t.Skip("concurrent prediction on single runner not currently supported")
+
+	workDir, err := os.MkdirTemp("", "ollama")
+	require.NoError(t, err)
+	defer os.RemoveAll(workDir)
+	require.NoError(t, llm.Init(workDir))
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*60)
+	defer cancel()
+	opts := api.DefaultOptions()
+	opts.Seed = 42
+	opts.Temperature = 0.0
+	var wg sync.WaitGroup
+	wg.Add(len(req))
+	model, llmRunner := PrepareModelForPrompts(t, req[0].Model, opts)
+	defer llmRunner.Close()
+	for i := 0; i < len(req); i++ {
+		go func(i int) {
+			defer wg.Done()
+			response := OneShotPromptResponse(t, ctx, req[i], model, llmRunner)
+			t.Logf("Prompt: %s\nResponse: %s", req[0].Prompt, response)
+			assert.Contains(t, strings.ToLower(response), resp[i], "error in thread %d (%s)", i, req[i].Prompt)
+		}(i)
+	}
+	wg.Wait()
+}
+
+func TestIntegrationConcurrentRunnersOrcaMini(t *testing.T) {
+	SkipIFNoTestData(t)
+	workDir, err := os.MkdirTemp("", "ollama")
+	require.NoError(t, err)
+	defer os.RemoveAll(workDir)
+	require.NoError(t, llm.Init(workDir))
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*60)
+	defer cancel()
+	opts := api.DefaultOptions()
+	opts.Seed = 42
+	opts.Temperature = 0.0
+	var wg sync.WaitGroup
+	wg.Add(len(req))
+
+	t.Logf("Running %d concurrently", len(req))
+	for i := 0; i < len(req); i++ {
+		go func(i int) {
+			defer wg.Done()
+			model, llmRunner := PrepareModelForPrompts(t, req[0].Model, opts)
+			defer llmRunner.Close()
+			response := OneShotPromptResponse(t, ctx, req[i], model, llmRunner)
+			t.Logf("Prompt: %s\nResponse: %s", req[0].Prompt, response)
+			assert.Contains(t, strings.ToLower(response), resp[i], "error in thread %d (%s)", i, req[i].Prompt)
+		}(i)
+	}
+	wg.Wait()
+}
+
+// TODO - create a parallel test with 2 different models once we support concurrency

server/llm_utils_test.go

+package server
+
+import (
+	"context"
+	"errors"
+	"os"
+	"path"
+	"runtime"
+	"testing"
+
+	"github.com/jmorganca/ollama/api"
+	"github.com/jmorganca/ollama/llm"
+	"github.com/stretchr/testify/require"
+)
+
+func SkipIFNoTestData(t *testing.T) {
+	modelDir := getModelDir()
+	if _, err := os.Stat(modelDir); errors.Is(err, os.ErrNotExist) {
+		t.Skipf("%s does not exist - skipping integration tests", modelDir)
+	}
+}
+
+func getModelDir() string {
+	_, filename, _, _ := runtime.Caller(0)
+	return path.Dir(path.Dir(filename) + "/../test_data/models/.")
+}
+
+func PrepareModelForPrompts(t *testing.T, modelName string, opts api.Options) (*Model, llm.LLM) {
+	modelDir := getModelDir()
+	os.Setenv("OLLAMA_MODELS", modelDir)
+	model, err := GetModel(modelName)
+	require.NoError(t, err, "GetModel ")
+	err = opts.FromMap(model.Options)
+	require.NoError(t, err, "opts from model ")
+	runner, err := llm.New("unused", model.ModelPath, model.AdapterPaths, model.ProjectorPaths, opts)
+	require.NoError(t, err, "llm.New failed")
+	return model, runner
+}
+
+func OneShotPromptResponse(t *testing.T, ctx context.Context, req api.GenerateRequest, model *Model, runner llm.LLM) string {
+	prompt, err := model.Prompt(PromptVars{
+		System: req.System,
+		Prompt: req.Prompt,
+		First:  len(req.Context) == 0,
+	})
+	require.NoError(t, err, "prompt generation failed")
+	success := make(chan bool, 1)
+	response := ""
+	cb := func(r llm.PredictResult) {
+
+		if !r.Done {
+			response += r.Content
+		} else {
+			success <- true
+		}
+	}
+	predictReq := llm.PredictOpts{
+		Prompt: prompt,
+		Format: req.Format,
+		Images: req.Images,
+	}
+	err = runner.Predict(ctx, predictReq, cb)
+	require.NoError(t, err, "predict call failed")
+
+	select {
+	case <-ctx.Done():
+		t.Errorf("failed to complete before timeout: \n%s", response)
+		return ""
+	case <-success:
+		return response
+	}
+}

server/routes.go

@@ -25,6 +25,7 @@ import (
 	"github.com/gin-gonic/gin"
 
 	"github.com/jmorganca/ollama/api"
+	"github.com/jmorganca/ollama/gpu"
 	"github.com/jmorganca/ollama/llm"
 	"github.com/jmorganca/ollama/parser"
 	"github.com/jmorganca/ollama/version"
@@ -81,20 +82,6 @@ func load(c *gin.Context, modelName string, reqOpts map[string]interface{}, sess
 		return nil, err
 	}
 
-	ctx := c.Request.Context()
-
-	// check if the loaded model is still running in a subprocess, in case something unexpected happened
-	if loaded.runner != nil {
-		if err := loaded.runner.Ping(ctx); err != nil {
-			log.Print("loaded llm process not responding, closing now")
-			// the subprocess is no longer running, so close it
-			loaded.runner.Close()
-			loaded.runner = nil
-			loaded.Model = nil
-			loaded.Options = nil
-		}
-	}
-
 	needLoad := loaded.runner == nil || // is there a model loaded?
 		loaded.ModelPath != model.ModelPath || // has the base model changed?
 		!reflect.DeepEqual(loaded.AdapterPaths, model.AdapterPaths) || // have the adapters changed?
@@ -114,7 +101,7 @@ func load(c *gin.Context, modelName string, reqOpts map[string]interface{}, sess
 			// some older models are not compatible with newer versions of llama.cpp
 			// show a generalized compatibility error until there is a better way to
 			// check for model compatibility
-			if strings.Contains(err.Error(), "failed to load model") {
+			if errors.Is(llm.ErrUnsupportedFormat, err) || strings.Contains(err.Error(), "failed to load model") {
 				err = fmt.Errorf("%v: this model may be incompatible with your version of Ollama. If you previously pulled this model, try updating it by running `ollama pull %s`", err, model.ShortName)
 			}
 
@@ -126,10 +113,6 @@ func load(c *gin.Context, modelName string, reqOpts map[string]interface{}, sess
 		loaded.Options = &opts
 	}
 
-	// update options for the loaded llm
-	// TODO(mxyng): this isn't thread safe, but it should be fine for now
-	loaded.runner.SetOptions(opts)
-
 	loaded.expireAt = time.Now().Add(sessionDuration)
 
 	if loaded.expireTimer == nil {
@@ -909,9 +892,12 @@ func Serve(ln net.Listener) error {
 		os.Exit(0)
 	}()
 
-	if runtime.GOOS == "linux" {
+	if err := llm.Init(s.WorkDir); err != nil {
+		return fmt.Errorf("unable to initialize llm library %w", err)
+	}
+	if runtime.GOOS == "linux" { // TODO - windows too
 		// check compatibility to log warnings
-		if _, err := llm.CheckVRAM(); err != nil {
+		if _, err := gpu.CheckVRAM(); err != nil {
 			log.Print(err.Error())
 		}
 	}