Adapted rocm support to cgo based llama.cpp

llm/llama.cpp/gen_windows.ps1

@@ -48,4 +48,8 @@ init_vars
 git_module_setup
 apply_patches
 build
-install
\ No newline at end of file
+install
+
+# TODO - implement ROCm support on windows
+md gguf/build/winrocm/lib -ea 0
+echo $null >> gguf/build/winrocm/lib/.generated

llm/llama.cpp/generate_linux.go

 package llm
 
-//go:generate sh ./gen_linux.sh
+//go:generate bash ./gen_linux.sh

llm/llama.cpp/generate_linux_cuda.go

-//go:build cuda
-
-package llm
-
-//go:generate git submodule init
-
-//go:generate git submodule update --force ggml
-//go:generate git -C ggml apply ../patches/0001-add-detokenize-endpoint.patch
-//go:generate git -C ggml apply ../patches/0002-34B-model-support.patch
-//go:generate git -C ggml apply ../patches/0005-ggml-support-CUDA-s-half-type-for-aarch64-1455-2670.patch
-//go:generate git -C ggml apply ../patches/0001-copy-cuda-runtime-libraries.patch
-
-//go:generate rm -rf ggml/build/cuda
-//go:generate cmake -S ggml -B ggml/build/cuda -DLLAMA_CUBLAS=on -DLLAMA_ACCELERATE=on -DLLAMA_K_QUANTS=on
-//go:generate cmake --build ggml/build/cuda --target server --config Release
-//go:generate mv ggml/build/cuda/bin/server ggml/build/cuda/bin/ollama-runner
-
-//go:generate git -C gguf apply ../patches/0001-copy-cuda-runtime-libraries.patch
-//go:generate git -C gguf apply ../patches/0001-update-default-log-target.patch
-
-//go:generate rm -rf gguf/build/cuda
-//go:generate cmake -S gguf -B gguf/build/cuda -DLLAMA_CUBLAS=on -DLLAMA_ACCELERATE=on -DLLAMA_K_QUANTS=on -DLLAMA_NATIVE=off -DLLAMA_AVX=on -DLLAMA_AVX2=off -DLLAMA_AVX512=off -DLLAMA_FMA=off -DLLAMA_F16C=off
-//go:generate cmake --build gguf/build/cuda --target server --config Release
-//go:generate mv gguf/build/cuda/bin/server gguf/build/cuda/bin/ollama-runner

llm/llama.cpp/generate_linux_rocm.go

-//go:build rocm
-
-package llm
-
-//go:generate git submodule init
-
-//go:generate git submodule update --force ggml
-//go:generate git -C ggml apply ../patches/0001-add-detokenize-endpoint.patch
-//go:generate git -C ggml apply ../patches/0002-34B-model-support.patch
-//go:generate git -C ggml apply ../patches/0005-ggml-support-CUDA-s-half-type-for-aarch64-1455-2670.patch
-//go:generate git -C ggml apply ../patches/0001-copy-cuda-runtime-libraries.patch
-
-//go:generate git submodule update --force gguf
-//go:generate git -C gguf apply ../patches/0001-copy-cuda-runtime-libraries.patch
-//go:generate git -C gguf apply ../patches/0001-update-default-log-target.patch
-
-//go:generate rm -rf ggml/build/rocm
-//go:generate cmake -S ggml -B ggml/build/rocm -DLLAMA_CLBLAST=on -DLLAMA_ACCELERATE=on -DLLAMA_K_QUANTS=on
-//go:generate cmake --build ggml/build/rocm --target server --config Release
-//go:generate mv ggml/build/rocm/bin/server ggml/build/rocm/bin/ollama-runner
-
-//go:generate rm -rf gguf/build/rocm
-//go:generate cmake -S gguf -B gguf/build/rocm -DLLAMA_HIPBLAS=on -DLLAMA_ACCELERATE=on -DLLAMA_K_QUANTS=on -DLLAMA_NATIVE=off -DLLAMA_AVX=on -DLLAMA_AVX2=off -DLLAMA_AVX512=off -DLLAMA_FMA=off -DLLAMA_F16C=off -DCMAKE_C_COMPILER=$ROCM_PATH/llvm/bin/clang -DCMAKE_CXX_COMPILER=$ROCM_PATH/llvm/bin/clang++ -DAMDGPU_TARGETS='gfx803;gfx900;gfx906:xnack-;gfx908:xnack-;gfx90a:xnack+;gfx90a:xnack-;gfx1010;gfx1012;gfx1030;gfx1100;gfx1101;gfx1102' -DGPU_TARGETS='gfx803;gfx900;gfx906:xnack-;gfx908:xnack-;gfx90a:xnack+;gfx90a:xnack-;gfx1010;gfx1012;gfx1030;gfx1100;gfx1101;gfx1102'
-//go:generate cmake --build gguf/build/rocm --target server --config Release
-//go:generate mv gguf/build/rocm/bin/server gguf/build/rocm/bin/ollama-runner

llm/llama.cpp/patches/0001-Expose-callable-API-for-server.patch

-From 64b3fbb150d12b3ca63ac2fb4e57bc46f41d2ccd Mon Sep 17 00:00:00 2001
+From 087cf3300e973d7790db8f7cad01d2a790de38be Mon Sep 17 00:00:00 2001
 From: Daniel Hiltgen <daniel@ollama.com>
 Date: Mon, 13 Nov 2023 12:25:58 -0800
 Subject: [PATCH] Expose callable API for server
 
 This adds an extern "C" interface within the example server
 ---
- examples/server/CMakeLists.txt |  24 ++++
- examples/server/server.cpp     | 247 +++++++++++++++++++++++++++++++++
- examples/server/server.h       |  83 +++++++++++
+ examples/server/CMakeLists.txt |  24 +++
+ examples/server/server.cpp     | 274 +++++++++++++++++++++++++++++++++
+ examples/server/server.h       |  89 +++++++++++
  ggml-cuda.cu                   |   1 +
- 4 files changed, 355 insertions(+)
+ 4 files changed, 388 insertions(+)
  create mode 100644 examples/server/server.h
 
 diff --git a/examples/server/CMakeLists.txt b/examples/server/CMakeLists.txt
@@ -46,7 +46,7 @@ index 859cd12..4ea47a7 100644
 +endif()
 \ No newline at end of file
 diff --git a/examples/server/server.cpp b/examples/server/server.cpp
-index 895f751..f939590 100644
+index d0cd8e1..5f5d4c5 100644
 --- a/examples/server/server.cpp
 +++ b/examples/server/server.cpp
 @@ -5,6 +5,9 @@
@@ -59,7 +59,7 @@ index 895f751..f939590 100644
  
  #ifndef NDEBUG
  // crash the server in debug mode, otherwise send an http 500 error
-@@ -2631,6 +2634,7 @@ static void append_to_generated_text_from_generated_token_probs(llama_server_con
+@@ -2632,6 +2635,7 @@ static void append_to_generated_text_from_generated_token_probs(llama_server_con
      }
  }
  
@@ -67,84 +67,84 @@ index 895f751..f939590 100644
  int main(int argc, char **argv)
  {
      // own arguments required by this example
-@@ -3065,3 +3069,246 @@ int main(int argc, char **argv)
+@@ -3066,3 +3070,273 @@ int main(int argc, char **argv)
      llama_backend_free();
      return 0;
  }
 +
 +#else // LLAMA_SERVER_LIBRARY
 +// Expose the llama server as a callable extern "C" API
-+llama_server_context llama;
++llama_server_context *llama = NULL;
 +std::atomic<bool> ext_server_running(false);
 +std::thread ext_server_thread;
-+inline ext_server_err makeErr(uint32_t code, std::string msg) {
-+    if (code == 0) {
-+        return ext_server_err{0, NULL};
-+    }
-+    const std::string::size_type size = msg.size();
-+    ext_server_err ret = {
-+        code,
-+        new char[size + 1],
-+    };
-+    memcpy(ret.err, msg.c_str(), size + 1);
-+    return ret;
-+}
 +
-+ext_server_err llama_server_init(ext_server_params *sparams)
++void llama_server_init(ext_server_params *sparams, ext_server_resp_t *err)
 +{
-+    log_set_target(stdout);
-+    gpt_params params;
-+    params.n_ctx = sparams->n_ctx;
-+    params.n_batch = sparams->n_batch;
-+    params.n_threads = sparams->n_threads;
-+    params.n_parallel = sparams->n_parallel;
-+    params.rope_freq_base = sparams->rope_freq_base;
-+    params.rope_freq_scale = sparams->rope_freq_scale;
-+
-+    if (sparams->memory_f16)  {
-+        params.cache_type_k = "f16";
-+        params.cache_type_v = "f16";
-+    } else {
-+        params.cache_type_k = "f32";
-+        params.cache_type_v = "f32";
-+    }
-+    
-+    params.n_gpu_layers = sparams->n_gpu_layers;
-+    params.main_gpu = sparams->main_gpu;
-+    params.use_mlock = sparams->use_mlock;
-+    params.use_mmap = sparams->use_mmap;
-+    params.numa = sparams->numa;
-+    params.embedding = sparams->embedding;
-+    if (sparams->model != NULL) {
-+        params.model = sparams->model;
-+    }
-+
-+    for (ext_server_lora_adapter *la = sparams->lora_adapters; la != NULL; la = la->next) {
-+        params.lora_adapter.push_back(std::make_tuple(la->adapter, la->scale));
-+    }
-+    
++    assert(err != NULL && sparams != NULL);
++    err->id = 0;
++    err->msg[0] = '\0';
 +    try {
++        llama = new llama_server_context;
++        log_set_target(stdout);
++        gpt_params params;
++        params.n_ctx = sparams->n_ctx;
++        params.n_batch = sparams->n_batch;
++        params.n_threads = sparams->n_threads;
++        params.n_parallel = sparams->n_parallel;
++        params.rope_freq_base = sparams->rope_freq_base;
++        params.rope_freq_scale = sparams->rope_freq_scale;
++
++        if (sparams->memory_f16)  {
++            params.cache_type_k = "f16";
++            params.cache_type_v = "f16";
++        } else {
++            params.cache_type_k = "f32";
++            params.cache_type_v = "f32";
++        }
++
++        params.n_gpu_layers = sparams->n_gpu_layers;
++        params.main_gpu = sparams->main_gpu;
++        params.use_mlock = sparams->use_mlock;
++        params.use_mmap = sparams->use_mmap;
++        params.numa = sparams->numa;
++        params.embedding = sparams->embedding;
++        if (sparams->model != NULL) {
++            params.model = sparams->model;
++        }
++
++        for (ext_server_lora_adapter *la = sparams->lora_adapters; la != NULL; la = la->next) {
++            params.lora_adapter.push_back(std::make_tuple(la->adapter, la->scale));
++        }
++
++        if (sparams->mmproj != NULL) {
++            params.mmproj = std::string(sparams->mmproj);
++        }
++           
 +        llama_backend_init(params.numa);
 +
 +        // load the model
-+        if (!llama.load_model(params))
++        if (!llama->load_model(params))
 +        {
 +            // TODO - consider modifying the logging logic or patching load_model so we can capture more detailed error messages
 +            // and pass them back to the caller for better UX
-+            return makeErr(1, "error loading model " + params.model);
++            err->id = -1;
++            snprintf(err->msg, err->msg_len, "error loading model %s", params.model.c_str());
++            return;
 +        }
 +
-+        llama.initialize();
++        llama->initialize();
 +    } catch (std::exception &e) {
-+        return makeErr(1, e.what());
++        err->id = -1;
++        snprintf(err->msg, err->msg_len, "exception %s", e.what());
 +    } catch (...) {
-+        return makeErr(1, "Unknown Exception initializing llama server");
++        err->id = -1;
++        snprintf(err->msg, err->msg_len, "Unknown exception initializing llama server");
 +    }
-+    return makeErr(0, "");
 +}
 +
 +void llama_server_start()
 +{
++    assert(llama != NULL);
 +     // TODO mutex to protect thread creation
 +    ext_server_thread = std::thread([&]()
 +    {
@@ -154,7 +154,7 @@ index 895f751..f939590 100644
 +            ggml_time_init();
 +            while (ext_server_running.load())
 +            {
-+                if (!llama.update_slots()) {
++                if (!llama->update_slots()) {
 +                    LOG_TEE("unexpected error in llama server update_slots - exiting main loop\n");
 +                    break;
 +                }
@@ -170,124 +170,150 @@ index 895f751..f939590 100644
 +}
 +
 +void llama_server_stop() {
++    assert(llama != NULL);
 +    // TODO - too verbose, remove once things are solid
 +    LOG_TEE("requesting llama server shutdown\n");
 +    ext_server_running = false;
 +    ext_server_thread.join();
++    delete llama;
++    llama = NULL;
 +    LOG_TEE("llama server shutdown complete\n");
 +}
 +
-+ext_server_completion_resp llama_server_completion(const char *json_req) {
-+    std::string msg;
-+    ext_server_completion_resp resp = {
-+        0,
-+        NULL,
-+    };
++void llama_server_completion(const char *json_req, ext_server_resp_t *resp) {
++    assert(llama != NULL && json_req != NULL && resp != NULL);
++    resp->id = -1;
++    resp->msg[0] = '\0';
 +    try {
 +        json data = json::parse(json_req);
-+        resp.task_id = llama.request_completion(data, false, false, -1);
-+        return resp;
++        resp->id = llama->request_completion(data, false, false, -1);
 +    } catch (std::exception &e) {
-+        msg = e.what();
++        snprintf(resp->msg, resp->msg_len, "exception %s", e.what());
 +    } catch (...) {
-+        msg = "Unknown Exception during completion";
++        snprintf(resp->msg, resp->msg_len, "Unknown exception during completion");
 +    }
-+    const std::string::size_type size = msg.size();
-+    resp.task_id = 0;
-+    resp.err = new char[size + 1];
-+    memcpy(resp.err, msg.c_str(), size + 1);
-+    return resp;
 +}
 +
-+ext_task_result llama_server_completion_next_result(const int task_id) {
++void llama_server_completion_next_result(const int task_id, ext_server_task_result_t *resp) {
++    assert(llama != NULL && resp != NULL);
 +    std::string msg;
-+    ext_task_result resp = {-1,false,false,NULL};
++    resp->id = -1;
++    resp->stop = false;
++    resp->error = false;
++    resp->json_resp = NULL;
++    std::string result_json;
 +    try {
-+        task_result result = llama.next_result(task_id);
-+        std::string result_json = result.result_json.dump(-1, ' ', false, json::error_handler_t::replace);
-+        const std::string::size_type size = result_json.size();
-+        resp.id = result.id;
-+        resp.stop = result.stop;
-+        resp.error = result.error;
-+        resp.result_json = new char[size + 1];
-+        memcpy(resp.result_json, result_json.c_str(), size + 1);
++        task_result result = llama->next_result(task_id);
++        result_json = result.result_json.dump(-1, ' ', false, json::error_handler_t::replace);
++        resp->id = result.id;
++        resp->stop = result.stop;
++        resp->error = result.error;
 +        if (result.error) {
-+            llama.request_cancel(task_id);
++            llama->request_cancel(task_id);
 +        } else if (result.stop) {
-+            llama.request_cancel(task_id);
++            llama->request_cancel(task_id);
 +        }
-+        return resp;
 +    } catch (std::exception &e) {
-+        msg = e.what(); // TODO - json?
++        resp->error = true;
++        resp->id = -1;
++        result_json = "{\"error\":\"exception " + std::string(e.what()) + "\"}";
 +    } catch (...) {
-+        msg = "Unknown Exception during completion";
++        resp->error = true;
++        resp->id = -1;
++        result_json = "{\"error\":\"Unknown exception during completion\"}";
 +    }
-+    resp.error = true;
-+    const std::string::size_type size = msg.size();
-+    resp.result_json = new char[size + 1];
-+    memcpy(resp.result_json, msg.c_str(), size + 1);
-+    return resp;
++    const std::string::size_type size = result_json.size() + 1;
++    resp->json_resp = new char[size];
++    snprintf(resp->json_resp, size, "%s", result_json.c_str());
 +}
 +
-+ext_server_err llama_server_completion_cancel(const int task_id) {
++void llama_server_release_task_result(ext_server_task_result_t *result) {
++    if (result == NULL || result->json_resp == NULL) {
++        return;
++    }
++    delete[] result->json_resp;
++}
++
++void llama_server_completion_cancel(const int task_id, ext_server_resp_t *err) {
++    assert(llama != NULL && err != NULL);
++    err->id = 0;
++    err->msg[0] = '\0';
 +    try {
-+        llama.request_cancel(task_id);
++        llama->request_cancel(task_id);
 +    } catch (std::exception &e) {
-+        return makeErr(1, e.what());
++        err->id = -1;
++        snprintf(err->msg, err->msg_len, "exception %s", e.what());
 +    } catch (...) {
-+        return makeErr(1, "Unknown Exception running llama server");
++        err->id = -1;
++        snprintf(err->msg, err->msg_len, "Unknown exception completion cancel in llama server");
 +    }
-+    return makeErr(0, "");
 +}
 +
-+
-+ext_server_err llama_server_tokenize(const char *json_req, ext_server_resp *resp) {
-+    resp->json_resp = NULL;
++void llama_server_tokenize(const char *json_req, char **json_resp, ext_server_resp_t *err) {
++    assert(llama != NULL && json_req != NULL && json_resp != NULL && err != NULL);
++    *json_resp = NULL;
++    err->id = 0;
++    err->msg[0] = '\0';
 +    try {
 +        const json body = json::parse(json_req);
 +        std::vector<llama_token> tokens;
 +        if (body.count("content") != 0)
 +        {
-+            tokens = llama.tokenize(body["content"], false);
++            tokens = llama->tokenize(body["content"], false);
 +        }
 +        const json data = format_tokenizer_response(tokens);
 +        std::string result_json = data.dump();
-+        const std::string::size_type size = result_json.size();
-+        resp->json_resp = new char[size + 1];
-+        memcpy(resp->json_resp, result_json.c_str(), size + 1);
++        const std::string::size_type size = result_json.size() + 1;
++        *json_resp = new char[size];
++        snprintf(*json_resp, size, "%s", result_json.c_str());
 +    } catch (std::exception &e) {
-+        return makeErr(1, e.what());
++        err->id = -1;
++        snprintf(err->msg, err->msg_len, "exception %s", e.what());
 +    } catch (...) {
-+        return makeErr(1, "Unknown Exception during tokenize");
++        err->id = -1;
++        snprintf(err->msg, err->msg_len, "Unknown exception during tokenize");
 +    }
-+    return makeErr(0, "");
 +}
 +
-+ext_server_err llama_server_detokenize(const char *json_req, ext_server_resp *resp) {
-+    resp->json_resp = NULL;
++void llama_server_release_json_resp(char **json_resp) {
++    if (json_resp == NULL || *json_resp == NULL) {
++        return;
++    }
++    delete[] *json_resp;
++}
++
++void llama_server_detokenize(const char *json_req, char **json_resp, ext_server_resp_t *err) {
++    assert(llama != NULL && json_req != NULL && json_resp != NULL && err != NULL);
++    *json_resp = NULL;
++    err->id = 0;
++    err->msg[0] = '\0';
 +    try {
 +        const json body = json::parse(json_req);
 +        std::string content;
 +        if (body.count("tokens") != 0)
 +        {
 +            const std::vector<llama_token> tokens = body["tokens"];
-+            content = tokens_to_str(llama.ctx, tokens.cbegin(), tokens.cend());
++            content = tokens_to_str(llama->ctx, tokens.cbegin(), tokens.cend());
 +        }
 +        const json data = format_detokenized_response(content);
 +        std::string result_json = data.dump();
-+        const std::string::size_type size = result_json.size();
-+        resp->json_resp = new char[size + 1];
-+        memcpy(resp->json_resp, result_json.c_str(), size + 1);
++        const std::string::size_type size = result_json.size() + 1;
++        *json_resp = new char[size];
++        snprintf(*json_resp, size, "%s", result_json.c_str());
 +    } catch (std::exception &e) {
-+        return makeErr(1, e.what());
++        err->id = -1;
++        snprintf(err->msg, err->msg_len, "exception %s", e.what());
 +    } catch (...) {
-+        return makeErr(1, "Unknown Exception during detokenize");
++        err->id = -1;
++        snprintf(err->msg, err->msg_len, "Unknown exception during detokenize");
 +    }
-+    return makeErr(0, "");
 +}
 +
-+ext_server_err llama_server_embedding(const char *json_req, ext_server_resp *resp) {
-+    resp->json_resp = NULL;
++void llama_server_embedding(const char *json_req, char** json_resp, ext_server_resp_t *err) {
++    assert(llama != NULL && json_req != NULL && json_resp != NULL && err != NULL);
++    *json_resp = NULL;
++    err->id = 0;
++    err->msg[0] = '\0';
 +    try {
 +        const json body = json::parse(json_req);
 +        json prompt;
@@ -299,28 +325,29 @@ index 895f751..f939590 100644
 +        {
 +            prompt = "";
 +        }
-+        const int task_id = llama.request_completion({ {"prompt", prompt}, { "n_predict", 0} }, false, true, -1);
-+        task_result result = llama.next_result(task_id);
++        const int task_id = llama->request_completion({ {"prompt", prompt}, { "n_predict", 0} }, false, true, -1);
++        task_result result = llama->next_result(task_id);
 +        std::string result_json = result.result_json.dump();
-+        const std::string::size_type size = result_json.size();
-+        resp->json_resp = new char[size + 1];
-+        memcpy(resp->json_resp, result_json.c_str(), size + 1);
++        const std::string::size_type size = result_json.size() + 1;
++        *json_resp = new char[size];
++        snprintf(*json_resp, size, "%s", result_json.c_str());
 +    } catch (std::exception &e) {
-+        return makeErr(1, e.what());
++        err->id = -1;
++        snprintf(err->msg, err->msg_len, "exception %s", e.what());
 +    } catch (...) {
-+        return makeErr(1, "Unknown Exception during detokenize");
++        err->id = -1;
++        snprintf(err->msg, err->msg_len, "Unknown exception during embedding");
 +    }
-+    return makeErr(0, "");
 +}
 +
 +#endif // LLAMA_SERVER_LIBRARY
 \ No newline at end of file
 diff --git a/examples/server/server.h b/examples/server/server.h
 new file mode 100644
-index 0000000..4d03b1e
+index 0000000..d22f1b6
 --- /dev/null
 +++ b/examples/server/server.h
-@@ -0,0 +1,83 @@
+@@ -0,0 +1,89 @@
 +#if defined(LLAMA_SERVER_LIBRARY)
 +#ifndef LLAMA_SERVER_H
 +#define LLAMA_SERVER_H
@@ -336,20 +363,23 @@ index 0000000..4d03b1e
 +extern "C"
 +{
 +#endif
-+    // TODO - clean the type def's up a bit for better consistency
-+    typedef struct ext_server_err {
-+        uint32_t code; // 0 on success, > 0 on error
-+        char *err; // null if code == 0; else contains error message.  Caller responsible for freeing memory
-+    } ext_server_err;
++    typedef struct ext_server_resp {
++        int id; // < 0 on error
++        size_t msg_len; // caller must allocate msg and set msg_len
++        char *msg;
++    } ext_server_resp_t;
 +
++    // Allocated and freed by caller
 +    typedef struct ext_server_lora_adapter {
 +        char *adapter;
 +        float scale;
 +        struct ext_server_lora_adapter *next;
-+    } ext_server_lora_adapter;
++    } ext_server_lora_adapter_t;
++
++    // Allocated and freed by caller
 +    typedef struct ext_server_params
 +    {
-+        char *model;
++        char *model;            
 +        uint32_t n_ctx;         // text context, 0 = from model
 +        uint32_t n_batch;       // prompt processing maximum batch size
 +        uint32_t n_threads;     // number of threads to use for generation
@@ -363,40 +393,43 @@ index 0000000..4d03b1e
 +        bool use_mmap;          // use mmap if possible
 +        bool numa;              // attempt optimizations that help on some NUMA systems
 +        bool embedding;         // get only sentence embedding
-+        ext_server_lora_adapter* lora_adapters;
-+    } ext_server_params;
++        ext_server_lora_adapter_t* lora_adapters;
++        char *mmproj;
++    } ext_server_params_t;
++
++    typedef struct ext_server_task_result
++    {
++        int id;
++        bool stop;
++        bool error;
++        char* json_resp; // null terminated, memory managed by ext_server
++    } ext_server_task_result_t;
 +
 +    // Initialize the server once per process
-+    ext_server_err llama_server_init(ext_server_params *sparams);
++    // err->id = 0 for success and err->msg[0] = NULL
++    // err->id != 0 for failure, and err->msg contains error message
++    void llama_server_init(ext_server_params_t *sparams, ext_server_resp_t *err);
 +
-+    // Run the main loop
++    // Run the main loop, called once per init
 +    void llama_server_start();
-+    // Stop the main loop
++    // Stop the main loop and free up resources allocated in init and start.  Init must be called again to reuse
 +    void llama_server_stop();
 +
-+    typedef struct ext_task_result
-+    {
-+        int id;
-+        bool stop;
-+        bool error;
-+        char* result_json; // caller responsible to free this memory
-+    } ext_task_result;
-+    
-+    typedef struct ext_server_completion_resp {
-+        int task_id; // < 0 on error, >= 0 on success
-+        char *err; // null if task_id >= 0; else contains error message.  Caller responsible for freeing memory
-+    } ext_server_completion_resp;
-+    ext_server_completion_resp llama_server_completion(const char *json_req);
-+    ext_task_result llama_server_completion_next_result(const int task_id);
-+    ext_server_err llama_server_completion_cancel(const int task_id);
-+
-+    // Caller responsible for freeing json_resp
-+    typedef struct ext_server_resp {
-+        char *json_resp; // Caller responsible for freeing string
-+    } ext_server_resp;
-+    ext_server_err llama_server_tokenize(const char *json_req, ext_server_resp *resp);
-+    ext_server_err llama_server_detokenize(const char *json_req, ext_server_resp *resp);
-+    ext_server_err llama_server_embedding(const char *json_req, ext_server_resp *resp);
++    // json_req null terminated string, memory managed by caller
++    // resp->id >= 0 on success (task ID)
++    // resp->id < 0 on error, and resp->msg contains error message
++    void llama_server_completion(const char *json_req, ext_server_resp_t *resp);
++
++    // Caller must call llama_server_release_task_result to free resp->json_resp
++    void llama_server_completion_next_result(const int task_id, ext_server_task_result_t *result);
++    void llama_server_completion_cancel(const int task_id, ext_server_resp_t *err);
++    void llama_server_release_task_result(ext_server_task_result_t *result);
++
++    // Caller must call llama_server_releaes_json_resp to free json_resp if err.id < 0
++    void llama_server_tokenize(const char *json_req, char **json_resp, ext_server_resp_t *err);
++    void llama_server_detokenize(const char *json_req, char **json_resp, ext_server_resp_t *err);
++    void llama_server_embedding(const char *json_req, char** json_resp, ext_server_resp_t *err);
++    void llama_server_release_json_resp(char **json_resp);
 +
 +#ifdef __cplusplus
 +}
@@ -406,10 +439,10 @@ index 0000000..4d03b1e
 +#endif // LLAMA_SERVER_LIBRARY
 \ No newline at end of file
 diff --git a/ggml-cuda.cu b/ggml-cuda.cu
-index 85f7a29..ce51364 100644
+index 9e1acd3..ea64b55 100644
 --- a/ggml-cuda.cu
 +++ b/ggml-cuda.cu
-@@ -6410,6 +6410,7 @@ static cudaError_t ggml_cuda_cpy_tensor_2d(
+@@ -6505,6 +6505,7 @@ static cudaError_t ggml_cuda_cpy_tensor_2d(
          CUDA_CHECK(cudaGetDevice(&id));
          src_ptr = (char *) extra->data_device[id];
      } else {

llm/llama.go

@@ -3,6 +3,7 @@ package llm
 import (
 	"bytes"
 	"context"
+	_ "embed"
 	"errors"
 	"fmt"
 	"os"
@@ -112,12 +113,6 @@ type ImageData struct {
 	ID   int    `json:"id"`
 }
 
-type llama struct {
-	api.Options
-	ImageData []ImageData
-	Running
-}
-
 var (
 	errNvidiaSMI     = errors.New("warning: gpu support may not be enabled, check that you have installed GPU drivers: nvidia-smi command failed")
 	errAvailableVRAM = errors.New("not enough VRAM available, falling back to CPU only")
@@ -166,7 +161,8 @@ type prediction struct {
 }
 
 const maxBufferSize = 512 * format.KiloByte
-const maxRetries = 6
+const maxRetries = 3
+const retryDelay = 1 * time.Second
 
 type PredictOpts struct {
 	Prompt string

llm/llm.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/jmorganca/ollama/api"
 	"github.com/jmorganca/ollama/format"
+	"github.com/jmorganca/ollama/gpu"
 )
 
 type LLM interface {
@@ -19,7 +20,6 @@ type LLM interface {
 	Encode(context.Context, string) ([]int, error)
 	Decode(context.Context, []int) (string, error)
 	Close()
-	Ping(context.Context) error
 }
 
 func New(workDir, model string, adapters, projectors []string, opts api.Options) (LLM, error) {
@@ -78,5 +78,17 @@ func New(workDir, model string, adapters, projectors []string, opts api.Options)
 	opts.NumGQA = 0
 	opts.RopeFrequencyBase = 0.0
 	opts.RopeFrequencyScale = 0.0
-	return newLlamaExtServer(model, adapters, projectors, ggml.NumLayers(), opts)
+	gpuInfo := gpu.GetGPUInfo()
+	switch gpuInfo.Driver {
+	case "ROCM":
+		return newRocmShimExtServer(model, adapters, projectors, ggml.NumLayers(), opts)
+	default:
+		// Rely on the built-in CUDA based server which will fall back to CPU
+		return newLlamaExtServer(model, adapters, projectors, ggml.NumLayers(), opts)
+	}
+}
+
+// Give any native cgo implementations an opportunity to initialize
+func Init(workdir string) error {
+	return nativeInit(workdir)
 }

llm/rocm_shim.c

+#include "rocm_shim.h"
+
+#include <stdio.h>
+#include <string.h>
+
+#ifndef _WIN32
+#include <dlfcn.h>
+#define LOAD_LIBRARY(lib, flags) dlopen(lib, flags)
+#define LOAD_SYMBOL(handle, sym) dlsym(handle, sym)
+#define LOAD_ERR() dlerror()
+#define UNLOAD_LIBRARY(handle) dlclose(handle)
+#else
+#include <windows.h>
+#define LOAD_LIBRARY(lib, flags) LoadLibrary(lib)
+#define LOAD_SYMBOL(handle, sym) GetProcAddress(handle, sym)
+#define UNLOAD_LIBRARY(handle) FreeLibrary(handle)
+// TODO - refactor this with proper error message handling on windows
+inline static char *LOAD_ERR() {
+  static char errbuf[8];
+  snprintf(errbuf, 8, "0x%lx", GetLastError());
+  return errbuf;
+}
+#endif
+
+void rocm_shim_init(const char *libPath, struct rocm_llama_server *s,
+                    ext_server_resp_t *err) {
+  int i = 0;
+  struct lookup {
+    char *s;
+    void **p;
+  } l[] = {
+      {"llama_server_init", (void *)&s->llama_server_init},
+      {"llama_server_start", (void *)&s->llama_server_start},
+      {"llama_server_stop", (void *)&s->llama_server_stop},
+      {"llama_server_completion", (void *)&s->llama_server_completion},
+      {"llama_server_completion_next_result",
+       (void *)&s->llama_server_completion_next_result},
+      {"llama_server_completion_cancel",
+       (void *)&s->llama_server_completion_cancel},
+      {"llama_server_release_task_result",
+       (void *)&s->llama_server_release_task_result},
+      {"llama_server_tokenize", (void *)&s->llama_server_tokenize},
+      {"llama_server_detokenize", (void *)&s->llama_server_detokenize},
+      {"llama_server_embedding", (void *)&s->llama_server_embedding},
+      {"llama_server_release_json_resp",
+       (void *)&s->llama_server_release_json_resp},
+      {"", NULL},
+  };
+
+  printf("Lazy loading %s library\n", libPath);
+  s->handle = LOAD_LIBRARY(libPath, RTLD_LAZY);
+  if (!s->handle) {
+    err->id = -1;
+    snprintf(
+        err->msg, err->msg_len,
+        "Unable to load rocm server library: %s (If you have a Radeon card, "
+        "did you install the ROCM libraries?)",
+        LOAD_ERR());
+    return;
+  }
+
+  for (i = 0; l[i].p != NULL; i++) {
+    *l[i].p = LOAD_SYMBOL(s->handle, l[i].s);
+    if (!l[i].p) {
+      UNLOAD_LIBRARY(s->handle);
+      err->id = -1;
+      snprintf(err->msg, err->msg_len, "symbol lookup for %s failed: %s",
+               l[i].s, LOAD_ERR());
+      return;
+    }
+  }
+}
+
+inline void rocm_shim_llama_server_init(struct rocm_llama_server s,
+                                        ext_server_params_t *sparams,
+                                        ext_server_resp_t *err) {
+  s.llama_server_init(sparams, err);
+}
+
+inline void rocm_shim_llama_server_start(struct rocm_llama_server s) {
+  s.llama_server_start();
+}
+
+inline void rocm_shim_llama_server_stop(struct rocm_llama_server s) {
+  s.llama_server_stop();
+}
+
+inline void rocm_shim_llama_server_completion(struct rocm_llama_server s,
+                                              const char *json_req,
+                                              ext_server_resp_t *resp) {
+  s.llama_server_completion(json_req, resp);
+}
+
+inline void rocm_shim_llama_server_completion_next_result(
+    struct rocm_llama_server s, const int task_id,
+    ext_server_task_result_t *result) {
+  s.llama_server_completion_next_result(task_id, result);
+}
+
+inline void rocm_shim_llama_server_completion_cancel(struct rocm_llama_server s,
+                                                     const int task_id,
+                                                     ext_server_resp_t *err) {
+  s.llama_server_completion_cancel(task_id, err);
+}
+inline void rocm_shim_llama_server_release_task_result(
+    struct rocm_llama_server s, ext_server_task_result_t *result) {
+  s.llama_server_release_task_result(result);
+}
+
+inline void rocm_shim_llama_server_tokenize(struct rocm_llama_server s,
+                                            const char *json_req,
+                                            char **json_resp,
+                                            ext_server_resp_t *err) {
+  s.llama_server_tokenize(json_req, json_resp, err);
+}
+
+inline void rocm_shim_llama_server_detokenize(struct rocm_llama_server s,
+                                              const char *json_req,
+                                              char **json_resp,
+                                              ext_server_resp_t *err) {
+  s.llama_server_detokenize(json_req, json_resp, err);
+}
+
+inline void rocm_shim_llama_server_embedding(struct rocm_llama_server s,
+                                             const char *json_req,
+                                             char **json_resp,
+                                             ext_server_resp_t *err) {
+  s.llama_server_embedding(json_req, json_resp, err);
+}
+
+inline void rocm_shim_llama_server_release_json_resp(struct rocm_llama_server s,
+                                                     char **json_resp) {
+  s.llama_server_release_json_resp(json_resp);
+}

llm/rocm_shim.h

+#include <stdlib.h>
+
+#include "server.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+struct rocm_llama_server {
+  void *handle;
+  void (*llama_server_init)(ext_server_params_t *sparams,
+                            ext_server_resp_t *err);
+  void (*llama_server_start)();
+  void (*llama_server_stop)();
+  void (*llama_server_completion)(const char *json_req,
+                                  ext_server_resp_t *resp);
+  void (*llama_server_completion_next_result)(const int task_id,
+                                              ext_server_task_result_t *result);
+  void (*llama_server_completion_cancel)(const int task_id,
+                                         ext_server_resp_t *err);
+  void (*llama_server_release_task_result)(ext_server_task_result_t *result);
+  void (*llama_server_tokenize)(const char *json_req, char **json_resp,
+                                ext_server_resp_t *err);
+  void (*llama_server_detokenize)(const char *json_req, char **json_resp,
+                                  ext_server_resp_t *err);
+  void (*llama_server_embedding)(const char *json_req, char **json_resp,
+                                 ext_server_resp_t *err);
+  void (*llama_server_release_json_resp)(char **json_resp);
+};
+
+void rocm_shim_init(const char *libPath, struct rocm_llama_server *s,
+                    ext_server_resp_t *err);
+
+// No good way to call C function pointers from Go so inline the indirection
+void rocm_shim_llama_server_init(struct rocm_llama_server s,
+                                 ext_server_params_t *sparams,
+                                 ext_server_resp_t *err);
+
+void rocm_shim_llama_server_start(struct rocm_llama_server s);
+
+void rocm_shim_llama_server_stop(struct rocm_llama_server s);
+
+void rocm_shim_llama_server_completion(struct rocm_llama_server s,
+                                       const char *json_req,
+                                       ext_server_resp_t *resp);
+
+void rocm_shim_llama_server_completion_next_result(
+    struct rocm_llama_server s, const int task_id,
+    ext_server_task_result_t *result);
+
+void rocm_shim_llama_server_completion_cancel(struct rocm_llama_server s,
+                                              const int task_id,
+                                              ext_server_resp_t *err);
+
+void rocm_shim_llama_server_release_task_result(
+    struct rocm_llama_server s, ext_server_task_result_t *result);
+
+void rocm_shim_llama_server_tokenize(struct rocm_llama_server s,
+                                     const char *json_req, char **json_resp,
+                                     ext_server_resp_t *err);
+
+void rocm_shim_llama_server_detokenize(struct rocm_llama_server s,
+                                       const char *json_req, char **json_resp,
+                                       ext_server_resp_t *err);
+
+void rocm_shim_llama_server_embedding(struct rocm_llama_server s,
+                                      const char *json_req, char **json_resp,
+                                      ext_server_resp_t *err);
+void rocm_shim_llama_server_release_json_resp(struct rocm_llama_server s,
+                                              char **json_resp);
+
+#ifdef __cplusplus
+}
+#endif
\ No newline at end of file

llm/shim_darwin.go

+package llm
+
+import (
+	"fmt"
+
+	"github.com/jmorganca/ollama/api"
+)
+
+// no-op stubs for mac
+
+func newRocmShimExtServer(model string, adapters, projectors []string, numLayers int64, opts api.Options) (extServer, error) {
+	// should never happen...
+	return nil, fmt.Errorf("ROCM GPUs not supported on Mac")
+}
+
+func nativeInit(workDir string) error {
+	return nil
+}

llm/shim_ext_server.go

+//go:build !darwin
+
+package llm
+
+/*
+
+#include <stdlib.h>
+#include "rocm_shim.h"
+
+*/
+import "C"
+import (
+	"context"
+	"embed"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"log"
+	"os"
+	"path/filepath"
+	"runtime"
+	"sync"
+	"unsafe"
+
+	"github.com/jmorganca/ollama/api"
+)
+
+//go:embed llama.cpp/gguf/build/*/lib/*
+var libEmbed embed.FS
+
+var RocmShimMissing = fmt.Errorf("ROCm shim library not included in this build of ollama. Radeon GPUs are not supported")
+var NoShim = true
+
+type shimExtServer struct {
+	s       C.struct_rocm_llama_server
+	options api.Options
+}
+
+// Note: current implementation does not support concurrent instantiations
+var shimMutex sync.Mutex
+var llm *shimExtServer
+
+func (llm *shimExtServer) llama_server_init(sparams *C.ext_server_params_t, err *C.ext_server_resp_t) {
+	C.rocm_shim_llama_server_init(llm.s, sparams, err)
+}
+func (llm *shimExtServer) llama_server_start() {
+	C.rocm_shim_llama_server_start(llm.s)
+}
+func (llm *shimExtServer) llama_server_stop() {
+	C.rocm_shim_llama_server_stop(llm.s)
+}
+
+func (llm *shimExtServer) llama_server_completion(json_req *C.char, resp *C.ext_server_resp_t) {
+	C.rocm_shim_llama_server_completion(llm.s, json_req, resp)
+}
+func (llm *shimExtServer) llama_server_completion_next_result(task_id C.int, resp *C.ext_server_task_result_t) {
+	C.rocm_shim_llama_server_completion_next_result(llm.s, task_id, resp)
+}
+func (llm *shimExtServer) llama_server_completion_cancel(task_id C.int, err *C.ext_server_resp_t) {
+	C.rocm_shim_llama_server_completion_cancel(llm.s, task_id, err)
+}
+func (llm *shimExtServer) llama_server_release_task_result(result *C.ext_server_task_result_t) {
+	C.rocm_shim_llama_server_release_task_result(llm.s, result)
+}
+
+func (llm *shimExtServer) llama_server_tokenize(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t) {
+	C.rocm_shim_llama_server_tokenize(llm.s, json_req, json_resp, err)
+}
+func (llm *shimExtServer) llama_server_detokenize(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t) {
+	C.rocm_shim_llama_server_detokenize(llm.s, json_req, json_resp, err)
+}
+func (llm *shimExtServer) llama_server_embedding(json_req *C.char, json_resp **C.char, err *C.ext_server_resp_t) {
+	C.rocm_shim_llama_server_embedding(llm.s, json_req, json_resp, err)
+}
+func (llm *shimExtServer) llama_server_release_json_resp(json_resp **C.char) {
+	C.rocm_shim_llama_server_release_json_resp(llm.s, json_resp)
+}
+
+func newRocmShimExtServer(model string, adapters, projectors []string, numLayers int64, opts api.Options) (extServer, error) {
+	if NoShim {
+		return nil, RocmShimMissing
+	}
+	log.Printf("Loading ROCM llm server")
+	if llm == nil {
+		return nil, fmt.Errorf("nativeInit wasnt called or libary load failed")
+	}
+	llm.options = opts
+	return newExtServer(llm, model, adapters, projectors, numLayers, opts)
+}
+
+func (llm *shimExtServer) Predict(ctx context.Context, pred PredictOpts, fn func(PredictResult)) error {
+	return predict(llm, llm.options, ctx, pred, fn)
+}
+
+func (llm *shimExtServer) Encode(ctx context.Context, prompt string) ([]int, error) {
+	return encode(llm, ctx, prompt)
+}
+
+func (llm *shimExtServer) Decode(ctx context.Context, tokens []int) (string, error) {
+	return decode(llm, ctx, tokens)
+}
+
+func (llm *shimExtServer) Embedding(ctx context.Context, input string) ([]float64, error) {
+	return embedding(llm, ctx, input)
+}
+
+func (llm *shimExtServer) Close() {
+	close(llm)
+}
+
+func nativeInit(workdir string) error {
+	err := extractLib(workdir)
+	if err != nil {
+		if err == RocmShimMissing {
+			log.Printf("%s", err)
+			return nil
+		}
+		return err
+	}
+
+	// Verify we have permissions - either running as root, or we have group access to the driver
+	fd, err := os.OpenFile("/dev/kfd", os.O_RDWR, 0666)
+	if err != nil {
+		if errors.Is(err, fs.ErrPermission) {
+			log.Fatalf("Radeon card detected, but permissions not set up properly.  Either run ollama as root, or add you user account to the render group.")
+			return err
+		} else if errors.Is(err, fs.ErrNotExist) {
+			// expected behavior without a radeon card
+			return nil
+		}
+
+		return fmt.Errorf("failed to check permission on /dev/kfd: %w", err)
+	}
+	fd.Close()
+
+	shimMutex.Lock()
+	defer shimMutex.Unlock()
+	if llm != nil {
+		return nil
+	}
+	var libName string
+	switch runtime.GOOS {
+	case "darwin":
+		// shouldn't happen
+		return nil
+	case "linux":
+		libName = "librocm_server.so"
+	case "windows":
+		libName = "rocm_server.dll"
+	default:
+		// shouldn't happen
+		return nil
+	}
+	libPath := C.CString(filepath.Join(workdir, libName))
+	defer C.free(unsafe.Pointer(libPath))
+	resp := newExtServerResp(128)
+	defer freeExtServerResp(resp)
+	var srv C.struct_rocm_llama_server
+	C.rocm_shim_init(libPath, &srv, &resp)
+	if resp.id < 0 {
+		// TODO - consider softening this failure mode to allow fall-back to the CUDA based built-in llm
+		//        and run against CPU
+		return fmt.Errorf("Unable to load AMD GPU library: %s", C.GoString(resp.msg))
+	}
+	llm = &shimExtServer{
+		s:       srv,
+		options: api.DefaultOptions(),
+	}
+	return nil
+}
+
+func extractLib(workDir string) error {
+	files, err := fs.Glob(libEmbed, "llama.cpp/gguf/build/*/lib/*rocm_server*")
+	if err != nil || len(files) == 0 {
+		// this is expected, ollama may be compiled without shim library packed in
+		return RocmShimMissing
+	}
+
+	if len(files) != 1 {
+		// Shouldn't happen, but just use the first one we find
+		log.Printf("WARNING: multiple rocm libraries detected - using %s", files[0])
+	}
+
+	srcFile, err := libEmbed.Open(files[0])
+	if err != nil {
+		return fmt.Errorf("read ROCm shim %s: %v", files[0], err)
+	}
+	defer srcFile.Close()
+	if err := os.MkdirAll(workDir, 0o755); err != nil {
+		return fmt.Errorf("create ROCm shim temp dir %s: %v", workDir, err)
+	}
+
+	destFile := filepath.Join(workDir, filepath.Base(files[0]))
+
+	_, err = os.Stat(destFile)
+	switch {
+	case errors.Is(err, os.ErrNotExist):
+		destFile, err := os.OpenFile(destFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o755)
+		if err != nil {
+			return fmt.Errorf("write ROCm shim %s: %v", files[0], err)
+		}
+		defer destFile.Close()
+		if _, err := io.Copy(destFile, srcFile); err != nil {
+			return fmt.Errorf("copy ROCm shim %s: %v", files[0], err)
+		}
+	case err != nil:
+		return fmt.Errorf("stat ROCm shim %s: %v", files[0], err)
+	}
+	NoShim = false
+	return nil
+}

scripts/build_linux.sh

@@ -8,7 +8,7 @@ export GOFLAGS="'-ldflags=-w -s \"-X=github.com/jmorganca/ollama/version.Version
 mkdir -p dist
 
 for TARGETARCH in amd64 arm64; do
-    docker buildx build --load --platform=linux/$TARGETARCH --build-arg=VERSION --build-arg=GOFLAGS -f Dockerfile.build -t builder:$TARGETARCH .
+    docker buildx build --load --progress=plain --platform=linux/$TARGETARCH --build-arg=VERSION --build-arg=GOFLAGS -f Dockerfile.build -t builder:$TARGETARCH .
     docker create --platform linux/$TARGETARCH --name builder-$TARGETARCH builder:$TARGETARCH
     docker cp builder-$TARGETARCH:/go/src/github.com/jmorganca/ollama/ollama ./dist/ollama-linux-$TARGETARCH
     docker rm builder-$TARGETARCH

scripts/build_remote.py

+#!/usr/bin/env python3
+import subprocess
+import sys
+from urllib.parse import urlparse
+from git import Repo
+
+# Helper script to be able to build on remote repos using git to push local changes
+# (e.g. particularly helpful to target a remote windows build system)
+#
+# Typical windows remote git config looks like this:
+#
+#[remote "windows-pa"]
+#        url = jdoe@desktop-foo:C:/Users/Jdoe/code/ollama
+#        fetch = +refs/heads/*:refs/remotes/windows-pa/*
+#        uploadpack = powershell git upload-pack
+#        receivepack = powershell git receive-pack
+#
+
+# TODO - add argpare and make this more configurable 
+# - force flag becomes optional
+# - generate, build or test ...
+
+# Note: remote repo will need this run once:
+# git config --local receive.denyCurrentBranch updateInstead
+repo = Repo(".")
+
+# On linux, add links in /usr/local/bin to the go binaries to avoid needing this
+# GoCmd = "/usr/local/go/bin/go" 
+GoCmd = "go" 
+
+if repo.is_dirty():
+    print("Tree is dirty.  Commit your changes before running this script")
+    sys.exit(1)
+
+if len(sys.argv) != 2:
+    print("Please specify the remote name: " + ', '.join([r.name for r in repo.remotes]))
+    sys.exit(1)
+remote_name = sys.argv[1]
+
+remote = {r.name: r for r in repo.remotes}[remote_name]
+raw_url = list(remote.urls)[0]
+url = urlparse(raw_url)
+# Windows urls don't quite parse properly
+if url.scheme == "" and url.netloc == "":
+    url = urlparse("ssh://" + raw_url)
+print("URL: " + str(url))
+netloc = url.netloc.split(":")[0]
+path = url.path
+branch_name = repo.active_branch.name
+
+print("Force pushing content to remote...")
+# Use with care given the force push
+remote.push(force=True).raise_if_error()
+
+print("Ensuring correct branch checked out on remote via ssh...")
+subprocess.check_call(['ssh', netloc, 'cd', path, ';', 'git', 'checkout', branch_name])
+
+
+# TODO - add some hardening to try to figure out how to set up the path properly
+# subprocess.check_call(['ssh', netloc, 'cd', path, ';', 'env'])
+# TODO - or consider paramiko maybe
+
+print("Performing generate")
+subprocess.check_call(['ssh', netloc, 'cd', path, ';', GoCmd, 'generate', './...'])
+
+print("Building")
+subprocess.check_call(['ssh', netloc, 'cd', path, ';', GoCmd, 'build', '.'])
+

scripts/setup_integration_tests.sh

@@ -32,4 +32,4 @@ for LAYER in $(cat ${OLLAMA_MODELS}/manifests/${REGISTRY}/${TEST_MODEL}/${TEST_M
     curl -L -C - --header "${ACCEPT_HEADER}" \
         -o ${OLLAMA_MODELS}/blobs/${LAYER} \
         ${REGISTRY_SCHEME}://${REGISTRY}/v2/${TEST_MODEL}/blobs/${LAYER}
-done
\ No newline at end of file
+done

server/llm_test.go

@@ -2,14 +2,17 @@ package server
 
 import (
 	"context"
+	"os"
 	"strings"
 	"sync"
 	"testing"
 	"time"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	"github.com/jmorganca/ollama/api"
+	"github.com/jmorganca/ollama/llm"
 )
 
 // TODO - this would ideally be in the llm package, but that would require some refactoring of interfaces in the server
@@ -33,12 +36,16 @@ var (
 	}
 	resp = [2]string{
 		"once upon a time",
-		"fourth thursday",
+		"united states thanksgiving",
 	}
 )
 
 func TestIntegrationSimpleOrcaMini(t *testing.T) {
 	SkipIFNoTestData(t)
+	workDir, err := os.MkdirTemp("", "ollama")
+	require.NoError(t, err)
+	defer os.RemoveAll(workDir)
+	require.NoError(t, llm.Init(workDir))
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*60)
 	defer cancel()
 	opts := api.DefaultOptions()
@@ -56,7 +63,13 @@ func TestIntegrationSimpleOrcaMini(t *testing.T) {
 // get true concurrency working with n_parallel support in the backend
 func TestIntegrationConcurrentPredictOrcaMini(t *testing.T) {
 	SkipIFNoTestData(t)
+
 	t.Skip("concurrent prediction on single runner not currently supported")
+
+	workDir, err := os.MkdirTemp("", "ollama")
+	require.NoError(t, err)
+	defer os.RemoveAll(workDir)
+	require.NoError(t, llm.Init(workDir))
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*60)
 	defer cancel()
 	opts := api.DefaultOptions()
@@ -79,6 +92,10 @@ func TestIntegrationConcurrentPredictOrcaMini(t *testing.T) {
 
 func TestIntegrationConcurrentRunnersOrcaMini(t *testing.T) {
 	SkipIFNoTestData(t)
+	workDir, err := os.MkdirTemp("", "ollama")
+	require.NoError(t, err)
+	defer os.RemoveAll(workDir)
+	require.NoError(t, llm.Init(workDir))
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*60)
 	defer cancel()
 	opts := api.DefaultOptions()
@@ -87,6 +104,7 @@ func TestIntegrationConcurrentRunnersOrcaMini(t *testing.T) {
 	var wg sync.WaitGroup
 	wg.Add(len(req))
 
+	t.Logf("Running %d concurrently", len(req))
 	for i := 0; i < len(req); i++ {
 		go func(i int) {
 			defer wg.Done()

server/routes.go

@@ -25,6 +25,7 @@ import (
 	"github.com/gin-gonic/gin"
 
 	"github.com/jmorganca/ollama/api"
+	"github.com/jmorganca/ollama/gpu"
 	"github.com/jmorganca/ollama/llm"
 	"github.com/jmorganca/ollama/parser"
 	"github.com/jmorganca/ollama/version"
@@ -81,20 +82,6 @@ func load(c *gin.Context, modelName string, reqOpts map[string]interface{}, sess
 		return nil, err
 	}
 
-	ctx := c.Request.Context()
-
-	// check if the loaded model is still running in a subprocess, in case something unexpected happened
-	if loaded.runner != nil {
-		if err := loaded.runner.Ping(ctx); err != nil {
-			log.Print("loaded llm process not responding, closing now")
-			// the subprocess is no longer running, so close it
-			loaded.runner.Close()
-			loaded.runner = nil
-			loaded.Model = nil
-			loaded.Options = nil
-		}
-	}
-
 	needLoad := loaded.runner == nil || // is there a model loaded?
 		loaded.ModelPath != model.ModelPath || // has the base model changed?
 		!reflect.DeepEqual(loaded.AdapterPaths, model.AdapterPaths) || // have the adapters changed?
@@ -905,9 +892,12 @@ func Serve(ln net.Listener) error {
 		os.Exit(0)
 	}()
 
-	if runtime.GOOS == "linux" {
+	if err := llm.Init(s.WorkDir); err != nil {
+		return fmt.Errorf("unable to initialize llm library %w", err)
+	}
+	if runtime.GOOS == "linux" { // TODO - windows too
 		// check compatibility to log warnings
-		if _, err := llm.CheckVRAM(); err != nil {
+		if _, err := gpu.CheckVRAM(); err != nil {
 			log.Print(err.Error())
 		}
 	}