Remove submodule and shift to Go server - 0.4.0 (#7157)

* Remove llama.cpp submodule and shift new build to top

* CI: install msys and clang gcc on win

Needed for deepseek to work properly on windows

llm/generate/gen_windows.ps1

-#!powershell
-
-$ErrorActionPreference = "Stop"
-
-function amdGPUs {
-    if ($env:AMDGPU_TARGETS) {
-        return $env:AMDGPU_TARGETS
-    }
-    # Current supported rocblas list from ROCm v6.1.2 on windows
-    # https://rocm.docs.amd.com/projects/install-on-windows/en/latest/reference/system-requirements.html#windows-supported-gpus
-    $GPU_LIST = @(
-        "gfx1030"
-        "gfx1100"
-        "gfx1101"
-        "gfx1102"
-    )
-    $GPU_LIST -join ';'
-}
-
-
-function init_vars {
-    write-host "Checking for cmake..."
-    get-command cmake
-    write-host "Checking for ninja..."
-    $d=(get-command -ea 'silentlycontinue' ninja).path
-    if ($null -eq $d) {
-        $MSVC_INSTALL=(Get-CimInstance MSFT_VSInstance -Namespace root/cimv2/vs)[0].InstallLocation
-        $matches=(gci -path $MSVC_INSTALL -r -fi ninja.exe)
-        if ($matches.count -eq 0) {
-            throw "Unable to locate ninja"
-        }
-        $ninjaDir=($matches[0].FullName | split-path -parent)
-        $env:PATH="$env:PATH;$ninjaDir"
-    }
-    if (!$script:SRC_DIR) {
-        $script:SRC_DIR = $(resolve-path "..\..\")
-    }
-    if (!$script:llamacppDir) {
-        $script:llamacppDir = "../llama.cpp"
-    }
-    if (!$script:cmakeTargets) {
-        $script:cmakeTargets = @("ollama_llama_server")
-    }
-    $script:cmakeDefs = @(
-        "-DBUILD_SHARED_LIBS=on",
-        "-DGGML_NATIVE=off",
-        "-DGGML_OPENMP=off"
-        )
-    $script:commonCpuDefs = @("-DCMAKE_POSITION_INDEPENDENT_CODE=on")
-    $script:ARCH = $Env:PROCESSOR_ARCHITECTURE.ToLower()
-    $script:DIST_BASE = "${script:SRC_DIR}\dist\windows-${script:ARCH}\lib\ollama\runners"
-    md "$script:DIST_BASE" -ea 0 > $null
-    if ($env:CGO_CFLAGS -contains "-g") {
-        $script:cmakeDefs += @("-DCMAKE_VERBOSE_MAKEFILE=on", "-DLLAMA_SERVER_VERBOSE=on", "-DCMAKE_BUILD_TYPE=RelWithDebInfo")
-        $script:config = "RelWithDebInfo"
-    } else {
-        $script:cmakeDefs += @("-DLLAMA_SERVER_VERBOSE=off", "-DCMAKE_BUILD_TYPE=Release")
-        $script:config = "Release"
-    }
-    if ($null -ne $env:CMAKE_SYSTEM_VERSION) {
-        $script:cmakeDefs += @("-DCMAKE_SYSTEM_VERSION=${env:CMAKE_SYSTEM_VERSION}")
-    }
-    # Try to find the CUDA dir
-    if ($env:CUDA_LIB_DIR -eq $null) {
-        $d=(get-command -ea 'silentlycontinue' nvcc).path
-        if ($d -ne $null) {
-            $script:CUDA_LIB_DIR=($d| split-path -parent)
-            $script:CUDA_INCLUDE_DIR=($script:CUDA_LIB_DIR|split-path -parent)+"\include"
-        }
-    } else {
-        $script:CUDA_LIB_DIR=$env:CUDA_LIB_DIR
-    }
-    $script:DUMPBIN=(get-command -ea 'silentlycontinue' dumpbin).path
-    if ($null -eq $env:CMAKE_CUDA_ARCHITECTURES) {
-        $script:CMAKE_CUDA_ARCHITECTURES="50;52;61;70;75;80"
-    } else {
-        $script:CMAKE_CUDA_ARCHITECTURES=$env:CMAKE_CUDA_ARCHITECTURES
-    }
-    # Note: Windows Kits 10 signtool crashes with GCP's plugin
-    if ($null -eq $env:SIGN_TOOL) {
-        ${script:SignTool}="C:\Program Files (x86)\Windows Kits\8.1\bin\x64\signtool.exe"
-    } else {
-        ${script:SignTool}=${env:SIGN_TOOL}
-    }
-    if ("${env:KEY_CONTAINER}") {
-        ${script:OLLAMA_CERT}=$(resolve-path "${script:SRC_DIR}\ollama_inc.crt")
-    }
-}
-
-function git_module_setup {
-    # TODO add flags to skip the init/patch logic to make it easier to mod llama.cpp code in-repo
-    & git submodule init
-    if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}
-    & git submodule update --force "${script:llamacppDir}"
-    if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}
-}
-
-function apply_patches {
-    # Apply temporary patches until fix is upstream
-    foreach ($patch in $(Get-ChildItem "../patches/*.patch")) {
-        git -c 'user.name=nobody' -c 'user.email=<>' -C "${script:llamacppDir}" am $patch.FullName
-    }
-}
-
-function build {
-    write-host "generating config with: cmake -S ${script:llamacppDir} -B $script:buildDir $script:cmakeDefs"
-    & cmake --version
-    & cmake -S "${script:llamacppDir}" -B $script:buildDir $script:cmakeDefs
-    if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}
-    if ($cmakeDefs -contains "-G") {
-        $extra=@("-j8")
-    } else {
-        $extra= @("--", "/maxCpuCount:8")
-    }
-    write-host "building with: cmake --build $script:buildDir --config $script:config $($script:cmakeTargets | ForEach-Object { `"--target`", $_ }) $extra"
-    & cmake --build $script:buildDir --config $script:config ($script:cmakeTargets | ForEach-Object { "--target", $_ }) $extra
-    if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}
-    # Rearrange output to be consistent between different generators
-    if ($null -ne ${script:config} -And (test-path -path "${script:buildDir}/bin/${script:config}" ) ) {
-        mv -force "${script:buildDir}/bin/${script:config}/*" "${script:buildDir}/bin/"
-        remove-item "${script:buildDir}/bin/${script:config}"
-    }
-}
-
-function sign {
-    if ("${env:KEY_CONTAINER}") {
-        write-host "Signing ${script:buildDir}/bin/*.exe  ${script:buildDir}/bin/*.dll"
-        foreach ($file in @(get-childitem "${script:buildDir}/bin/*.exe") + @(get-childitem "${script:buildDir}/bin/*.dll")){
-            & "${script:SignTool}" sign /v /fd sha256 /t http://timestamp.digicert.com /f "${script:OLLAMA_CERT}" `
-                /csp "Google Cloud KMS Provider" /kc "${env:KEY_CONTAINER}" $file
-            if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}
-        }
-    }
-}
-
-function install {
-    write-host "Installing binaries to dist dir ${script:distDir}"
-    mkdir ${script:distDir} -ErrorAction SilentlyContinue
-    $binaries = dir "${script:buildDir}/bin/*.exe"
-    foreach ($file in $binaries) {
-        copy-item -Path $file -Destination ${script:distDir} -Force
-    }
-
-    write-host "Installing dlls to dist dir ${script:distDir}"
-    $dlls = dir "${script:buildDir}/bin/*.dll"
-    foreach ($file in $dlls) {
-        copy-item -Path $file -Destination ${script:distDir} -Force
-    }
-}
-
-function cleanup {
-    $patches = Get-ChildItem "../patches/*.diff"
-    foreach ($patch in $patches) {
-        # Extract file paths from the patch file
-        $filePaths = Get-Content $patch.FullName | Where-Object { $_ -match '^\+\+\+ ' } | ForEach-Object {
-            $parts = $_ -split ' '
-            ($parts[1] -split '/', 2)[1]
-        }
-
-        # Checkout each file
-        foreach ($file in $filePaths) {
-            git -C "${script:llamacppDir}" checkout $file
-        }
-        git -C "${script:llamacppDir}" checkout CMakeLists.txt
-    }
-}
-
-
-# -DGGML_AVX -- 2011 Intel Sandy Bridge & AMD Bulldozer
-# -DGGML_AVX2 -- 2013 Intel Haswell & 2015 AMD Excavator / 2017 AMD Zen
-# -DGGML_FMA (FMA3) -- 2013 Intel Haswell & 2012 AMD Piledriver
-
-
-function build_cpu_x64 {
-    if ((-not "${env:OLLAMA_SKIP_CPU_GENERATE}" ) -and ((-not "${env:OLLAMA_CPU_TARGET}") -or ("${env:OLLAMA_CPU_TARGET}" -eq "cpu"))) {
-        init_vars
-        $script:cmakeDefs = $script:commonCpuDefs + @("-A", "x64", "-DGGML_AVX=off", "-DGGML_AVX2=off", "-DGGML_AVX512=off", "-DGGML_FMA=off", "-DGGML_F16C=off") + $script:cmakeDefs
-        $script:buildDir="../build/windows/${script:ARCH}/cpu"
-        $script:distDir="$script:DIST_BASE\cpu"
-        write-host "Building LCD CPU"
-        build
-        sign
-        install
-    } else {
-        write-host "Skipping CPU generation step as requested"
-    }
-}
-
-function build_cpu_arm64 {
-    if ((-not "${env:OLLAMA_SKIP_CPU_GENERATE}" ) -and ((-not "${env:OLLAMA_CPU_TARGET}") -or ("${env:OLLAMA_CPU_TARGET}" -eq "cpu"))) {
-        init_vars
-        write-host "Checking for clang..."
-        get-command clang
-        $env:CFLAGS="-march=armv8.7-a -fvectorize -ffp-model=fast -fno-finite-math-only"
-        $env:CXXFLAGS="$env:CFLAGS"
-        $env:LDFLAGS="-static-libstdc++"
-        $script:cmakeDefs = $script:commonCpuDefs + @(
-            "-DCMAKE_VERBOSE_MAKEFILE=on",
-            "-DCMAKE_C_COMPILER=clang.exe",
-            "-DCMAKE_CXX_COMPILER=clang++.exe",
-            "-DMSVC_RUNTIME_LIBRARY=MultiThreaded"
-        ) + $script:cmakeDefs
-        $script:buildDir="../build/windows/${script:ARCH}/cpu"
-        $script:distDir="$script:DIST_BASE\cpu"
-        write-host "Building LCD CPU"
-        build
-        sign
-        install
-    } else {
-        write-host "Skipping CPU generation step as requested"
-    }
-}
-
-
-function build_cpu_avx() {
-    if ((-not "${env:OLLAMA_SKIP_CPU_GENERATE}" ) -and ((-not "${env:OLLAMA_CPU_TARGET}") -or ("${env:OLLAMA_CPU_TARGET}" -eq "cpu_avx"))) {
-        init_vars
-        $script:cmakeDefs = $script:commonCpuDefs + @("-A", "x64", "-DGGML_AVX=on", "-DGGML_AVX2=off", "-DGGML_AVX512=off", "-DGGML_FMA=off", "-DGGML_F16C=off") + $script:cmakeDefs
-        $script:buildDir="../build/windows/${script:ARCH}/cpu_avx"
-        $script:distDir="$script:DIST_BASE\cpu_avx"
-        write-host "Building AVX CPU"
-        build
-        sign
-        install
-    } else {
-        write-host "Skipping CPU AVX generation step as requested"
-    }
-}
-
-function build_cpu_avx2() {
-    if ((-not "${env:OLLAMA_SKIP_CPU_GENERATE}" ) -and ((-not "${env:OLLAMA_CPU_TARGET}") -or ("${env:OLLAMA_CPU_TARGET}" -eq "cpu_avx2"))) {
-        init_vars
-        $script:cmakeDefs = $script:commonCpuDefs + @("-A", "x64", "-DGGML_AVX=on", "-DGGML_AVX2=on", "-DGGML_AVX512=off", "-DGGML_FMA=on", "-DGGML_F16C=on") + $script:cmakeDefs
-        $script:buildDir="../build/windows/${script:ARCH}/cpu_avx2"
-        $script:distDir="$script:DIST_BASE\cpu_avx2"
-        write-host "Building AVX2 CPU"
-        build
-        sign
-        install
-    } else {
-        write-host "Skipping CPU AVX2 generation step as requested"
-    }
-}
-
-function build_cuda() {
-    if ((-not "${env:OLLAMA_SKIP_CUDA_GENERATE}") -and ("${script:CUDA_LIB_DIR}")) {
-        # Then build cuda as a dynamically loaded library
-        $nvcc = "$script:CUDA_LIB_DIR\nvcc.exe"
-        $script:CUDA_VERSION=((get-item ($nvcc | split-path | split-path)).Basename -Split "\.")[0]
-        if ($null -ne $script:CUDA_VERSION) {
-            $script:CUDA_VARIANT="_"+$script:CUDA_VERSION
-        }
-        init_vars
-        $script:buildDir="../build/windows/${script:ARCH}/cuda$script:CUDA_VARIANT"
-        $script:distDir="$script:DIST_BASE\cuda$script:CUDA_VARIANT"
-        $script:cmakeDefs += @(
-            "-A", "x64",
-            "-DGGML_CUDA=ON",
-            "-DGGML_AVX=on",
-            "-DGGML_AVX2=off",
-            "-DCMAKE_CUDA_FLAGS=-t6",
-            "-DCMAKE_CUDA_ARCHITECTURES=${script:CMAKE_CUDA_ARCHITECTURES}",
-            "-DCMAKE_CUDA_COMPILER_TOOLKIT_ROOT=$env:CUDA_PATH"
-            )
-        if ($null -ne $env:OLLAMA_CUSTOM_CUDA_DEFS) {
-            write-host "OLLAMA_CUSTOM_CUDA_DEFS=`"${env:OLLAMA_CUSTOM_CUDA_DEFS}`""
-            $script:cmakeDefs +=@("${env:OLLAMA_CUSTOM_CUDA_DEFS}")
-            write-host "building custom CUDA GPU"
-        }
-        build
-        sign
-        install
-
-        md "${script:SRC_DIR}\dist\windows-${script:ARCH}\lib\ollama\" -ea 0 > $null
-        write-host "copying CUDA dependencies to ${script:SRC_DIR}\dist\windows-${script:ARCH}\lib\ollama\"
-        cp "${script:CUDA_LIB_DIR}\cudart64_*.dll" "${script:SRC_DIR}\dist\windows-${script:ARCH}\lib\ollama\"
-        cp "${script:CUDA_LIB_DIR}\cublas64_*.dll" "${script:SRC_DIR}\dist\windows-${script:ARCH}\lib\ollama\"
-        cp "${script:CUDA_LIB_DIR}\cublasLt64_*.dll" "${script:SRC_DIR}\dist\windows-${script:ARCH}\lib\ollama\"
-    } else {
-        write-host "Skipping CUDA generation step"
-    }
-}
-
-function build_oneapi() {
-  if ((-not "${env:OLLAMA_SKIP_ONEAPI_GENERATE}") -and ("${env:ONEAPI_ROOT}"))  {
-    # Get oneAPI version
-    $script:ONEAPI_VERSION = icpx --version
-    $script:ONEAPI_VERSION = [regex]::Match($script:ONEAPI_VERSION, '(?<=oneAPI DPC\+\+/C\+\+ Compiler )(?<version>\d+\.\d+\.\d+)').Value
-    if ($null -ne $script:ONEAPI_VERSION) {
-      $script:ONEAPI_VARIANT = "_v" + $script:ONEAPI_VERSION
-    }
-    init_vars
-    $script:buildDir = "../build/windows/${script:ARCH}/oneapi$script:ONEAPI_VARIANT"
-    $script:distDir ="$script:DIST_BASE\oneapi$script:ONEAPI_VARIANT"
-    $script:cmakeDefs += @(
-      "-G", "MinGW Makefiles",
-      "-DGGML_SYCL=ON",
-      "-DCMAKE_C_COMPILER=icx",
-      "-DCMAKE_CXX_COMPILER=icx",
-      "-DCMAKE_BUILD_TYPE=Release"
-    )
-
-    Write-Host "Building oneAPI"
-    build
-    # Ninja doesn't prefix with config name
-    if ($null -ne $script:DUMPBIN) {
-      & "$script:DUMPBIN" /dependents "${script:buildDir}/bin/ollama_llama_server.exe" | Select-String ".dll"
-    }
-    sign
-    install
-
-    md "${script:SRC_DIR}\dist\windows-${script:ARCH}\lib\ollama\" -ea 0 > $null
-    cp "${env:ONEAPI_ROOT}\compiler\latest\bin\libirngmd.dll" "${script:SRC_DIR}\dist\windows-${script:ARCH}\lib\ollama\"
-    cp "${env:ONEAPI_ROOT}\compiler\latest\bin\libmmd.dll" "${script:SRC_DIR}\dist\windows-${script:ARCH}\lib\ollama\"
-    cp "${env:ONEAPI_ROOT}\compiler\latest\bin\pi_level_zero.dll" "${script:SRC_DIR}\dist\windows-${script:ARCH}\lib\ollama\"
-    cp "${env:ONEAPI_ROOT}\compiler\latest\bin\pi_unified_runtime.dll" "${script:SRC_DIR}\dist\windows-${script:ARCH}\lib\ollama\"
-    cp "${env:ONEAPI_ROOT}\compiler\latest\bin\pi_win_proxy_loader.dll" "${script:SRC_DIR}\dist\windows-${script:ARCH}\lib\ollama\"
-    cp "${env:ONEAPI_ROOT}\compiler\latest\bin\svml_dispmd.dll" "${script:SRC_DIR}\dist\windows-${script:ARCH}\lib\ollama\"
-    cp "${env:ONEAPI_ROOT}\compiler\latest\bin\sycl7.dll" "${script:SRC_DIR}\dist\windows-${script:ARCH}\lib\ollama\"
-    cp "${env:ONEAPI_ROOT}\mkl\latest\bin\mkl_core.2.dll" "${script:SRC_DIR}\dist\windows-${script:ARCH}\lib\ollama\"
-    cp "${env:ONEAPI_ROOT}\mkl\latest\bin\mkl_sycl_blas.4.dll" "${script:SRC_DIR}\dist\windows-${script:ARCH}\lib\ollama\"
-    cp "${env:ONEAPI_ROOT}\mkl\latest\bin\mkl_tbb_thread.2.dll" "${script:SRC_DIR}\dist\windows-${script:ARCH}\lib\ollama\"
-  } else {
-    Write-Host "Skipping oneAPI generation step"
-  }
-}
-
-function build_rocm() {
-    if ((-not "${env:OLLAMA_SKIP_ROCM_GENERATE}") -and ("${env:HIP_PATH}")) {
-        $script:ROCM_VERSION=(get-item $env:HIP_PATH).Basename
-        if ($null -ne $script:ROCM_VERSION) {
-            $script:ROCM_VARIANT="_v"+$script:ROCM_VERSION
-        }
-
-        init_vars
-        $script:buildDir="../build/windows/${script:ARCH}/rocm$script:ROCM_VARIANT"
-        $script:distDir="$script:DIST_BASE\rocm$script:ROCM_VARIANT"
-        $script:cmakeDefs += @(
-            "-G", "Ninja",
-            "-DCMAKE_C_COMPILER=clang.exe",
-            "-DCMAKE_CXX_COMPILER=clang++.exe",
-            "-DGGML_HIPBLAS=on",
-            "-DHIP_PLATFORM=amd",
-            "-DGGML_AVX=on",
-            "-DGGML_AVX2=off",
-            "-DCMAKE_POSITION_INDEPENDENT_CODE=on",
-            "-DAMDGPU_TARGETS=$(amdGPUs)",
-            "-DGPU_TARGETS=$(amdGPUs)"
-            )
-
-        # Make sure the ROCm binary dir is first in the path
-        $env:PATH="$env:HIP_PATH\bin;$env:PATH"
-
-        # We have to clobber the LIB var from the developer shell for clang to work properly
-        $env:LIB=""
-        if ($null -ne $env:OLLAMA_CUSTOM_ROCM_DEFS) {
-            write-host "OLLAMA_CUSTOM_ROCM_DEFS=`"${env:OLLAMA_CUSTOM_ROCM_DEFS}`""
-            $script:cmakeDefs += @("${env:OLLAMA_CUSTOM_ROCM_DEFS}")
-            write-host "building custom ROCM GPU"
-        }
-        write-host "Building ROCm"
-        build
-        # Ninja doesn't prefix with config name
-        ${script:config}=""
-        if ($null -ne $script:DUMPBIN) {
-            & "$script:DUMPBIN" /dependents "${script:buildDir}/bin/ollama_llama_server.exe" | select-string ".dll"
-        }
-        sign
-        install
-
-        md "${script:SRC_DIR}\dist\windows-${script:ARCH}\lib\ollama\rocblas\library\" -ea 0 > $null
-        cp "${env:HIP_PATH}\bin\hipblas.dll" "${script:SRC_DIR}\dist\windows-${script:ARCH}\lib\ollama\"
-        cp "${env:HIP_PATH}\bin\rocblas.dll" "${script:SRC_DIR}\dist\windows-${script:ARCH}\lib\ollama\"
-        # amdhip64.dll dependency comes from the driver and must be installed on the host to use AMD GPUs
-        cp "${env:HIP_PATH}\bin\rocblas\library\*" "${script:SRC_DIR}\dist\windows-${script:ARCH}\lib\ollama\rocblas\library\"
-    } else {
-        write-host "Skipping ROCm generation step"
-    }
-}
-
-init_vars
-if ($($args.count) -eq 0) {
-    git_module_setup
-    apply_patches
-    if ($script:ARCH -eq "arm64") {
-        build_cpu_arm64
-    } else { # amd64
-        build_cpu_x64
-        build_cpu_avx
-        build_cpu_avx2
-        build_cuda
-        build_oneapi
-        build_rocm
-    }
-
-    cleanup
-    write-host "`ngo generate completed.  LLM runners: $(get-childitem -path $script:DIST_BASE)"
-} else {
-    for ( $i = 0; $i -lt $args.count; $i++ ) {
-        write-host "performing $($args[$i])"
-        & $($args[$i])
-    }
-}
\ No newline at end of file

llm/generate/generate_darwin.go

-package generate
-
-//go:generate bash ./gen_darwin.sh

llm/generate/generate_linux.go

-package generate
-
-//go:generate bash ./gen_linux.sh

llm/generate/generate_windows.go

-package generate
-
-//go:generate powershell -ExecutionPolicy Bypass -File ./gen_windows.ps1

llama.cpp @ 3f1ae2e3

-Subproject commit 3f1ae2e32cde00c39b96be6d01c2997c29bae555

llm/patches/0000-cmakelist.patch

-From 7a3555098d4591c9b329c677654497ed8cee07ec Mon Sep 17 00:00:00 2001
-From: Michael Yang <mxyng@pm.me>
-Date: Fri, 23 Aug 2024 11:27:48 -0700
-Subject: [PATCH] patch cmakelist
-
----
- CMakeLists.txt | 2 ++
- 1 file changed, 2 insertions(+)
-
-diff --git a/CMakeLists.txt b/CMakeLists.txt
-index 415743c2..aaadd13e 100644
---- a/CMakeLists.txt
-+++ b/CMakeLists.txt
-@@ -210,3 +210,5 @@ if (LLAMA_BUILD_EXAMPLES)
-     add_subdirectory(examples)
-     add_subdirectory(pocs)
- endif()
-+
-+add_subdirectory(../ext_server ext_server) # ollama
--- 
-2.39.3 (Apple Git-146)
-

llm/patches/0001-load-progress.patch

-From c97ed60c3369294d5551ba099a88ddc509687df1 Mon Sep 17 00:00:00 2001
-From: Gabe Goodhart <ghart@us.ibm.com>
-Date: Thu, 19 Sep 2024 16:55:15 -0600
-Subject: [PATCH] patch load progress
-
----
- common/common.cpp | 2 ++
- common/common.h   | 7 +++++++
- 2 files changed, 9 insertions(+)
-
-diff --git a/common/common.cpp b/common/common.cpp
-index 8d0ed4f9..a09e8a53 100644
---- a/common/common.cpp
-+++ b/common/common.cpp
-@@ -955,6 +955,8 @@ struct llama_model_params llama_model_params_from_gpt_params(const gpt_params &
-     mparams.use_mmap        = params.use_mmap;
-     mparams.use_mlock       = params.use_mlock;
-     mparams.check_tensors   = params.check_tensors;
-+    mparams.progress_callback = params.progress_callback;
-+    mparams.progress_callback_user_data = params.progress_callback_user_data;
-     if (params.kv_overrides.empty()) {
-         mparams.kv_overrides = NULL;
-     } else {
-diff --git a/common/common.h b/common/common.h
-index cb87c447..818a4a4a 100644
---- a/common/common.h
-+++ b/common/common.h
-@@ -266,6 +266,13 @@ struct gpt_params {
-     std::string mmproj = "";        // path to multimodal projector                                         // NOLINT
-     std::vector<std::string> image; // path to image file(s)
- 
-+    // Called with a progress value between 0.0 and 1.0. Pass NULL to disable.
-+    // If the provided progress_callback returns true, model loading continues.
-+    // If it returns false, model loading is immediately aborted.
-+    llama_progress_callback progress_callback = NULL;
-+    // context pointer passed to the progress callback
-+    void * progress_callback_user_data;
-+
-     // embedding
-     bool embedding         = false; // get only sentence embedding
-     int32_t embd_normalize = 2;     // normalisation for embendings (-1=none, 0=max absolute int16, 1=taxicab, 2=euclidean, >2=p-norm)
--- 
-2.39.3 (Apple Git-146)
-

llm/patches/0002-clip-log.patch

-From 6fdf4268e13e56f0050fa6a29b029cbd54be49d2 Mon Sep 17 00:00:00 2001
-From: Gabe Goodhart <ghart@us.ibm.com>
-Date: Thu, 19 Sep 2024 16:58:03 -0600
-Subject: [PATCH] clip log
-
----
- examples/llava/clip.cpp | 1 +
- 1 file changed, 1 insertion(+)
-
-diff --git a/examples/llava/clip.cpp b/examples/llava/clip.cpp
-index 8aa7b075..b8941c74 100644
---- a/examples/llava/clip.cpp
-+++ b/examples/llava/clip.cpp
-@@ -3,6 +3,7 @@
- // I'll gradually clean and extend it
- // Note: Even when using identical normalized image inputs (see normalize_image_u8_to_f32()) we have a significant difference in resulting embeddings compared to pytorch
- #include "clip.h"
-+#include "common.h"
- #include "ggml.h"
- #include "ggml-alloc.h"
- #include "ggml-backend.h"
--- 
-2.39.3 (Apple Git-146)
-

llm/patches/0003-load_exception.patch

-From 4f2b9cd0f012c49f40d0784454864ad41ca418b2 Mon Sep 17 00:00:00 2001
-From: Gabe Goodhart <ghart@us.ibm.com>
-Date: Thu, 19 Sep 2024 17:00:28 -0600
-Subject: [PATCH] load exception
-
----
- src/llama.cpp | 25 ++++++++++++++++---------
- 1 file changed, 16 insertions(+), 9 deletions(-)
-
-diff --git a/src/llama.cpp b/src/llama.cpp
-index af8afd84..4d1db3d5 100644
---- a/src/llama.cpp
-+++ b/src/llama.cpp
-@@ -8871,7 +8871,7 @@ static int llama_model_load(const std::string & fname, llama_model & model, llam
-         }
-     } catch (const std::exception & err) {
-         LLAMA_LOG_ERROR("%s: error loading model: %s\n", __func__, err.what());
--        return -1;
-+        throw;
-     }
- 
-     // loading time will be recalculate after the first eval, so
-@@ -18675,16 +18675,23 @@ struct llama_model * llama_load_model_from_file(
-         }
-         model->rpc_servers.push_back(servers);
-     }
--    int status = llama_model_load(path_model, *model, params);
--    GGML_ASSERT(status <= 0);
--    if (status < 0) {
--        if (status == -1) {
--            LLAMA_LOG_ERROR("%s: failed to load model\n", __func__);
--        } else if (status == -2) {
--            LLAMA_LOG_INFO("%s: cancelled model load\n", __func__);
-+
-+    try {
-+        int status = llama_model_load(path_model, *model, params);
-+        GGML_ASSERT(status <= 0);
-+        if (status < 0) {
-+            if (status == -1) {
-+                LLAMA_LOG_ERROR("%s: failed to load model\n", __func__);
-+            } else if (status == -2) {
-+                LLAMA_LOG_INFO("%s: cancelled model load\n", __func__);
-+            }
-+            delete model;
-+            return nullptr;
-         }
-+    } catch (...) {
-+        LLAMA_LOG_ERROR("%s: exception loading model\n", __func__);
-         delete model;
--        return nullptr;
-+        throw;
-     }
- 
-     return model;
--- 
-2.39.3 (Apple Git-146)
-

llm/patches/0004-metal.patch

-From 91d3f886f1645b38d9658c0e125603e8d5338146 Mon Sep 17 00:00:00 2001
-From: nobody <>
-Date: Tue, 1 Oct 2024 13:55:01 -0600
-Subject: [PATCH] metal
-
----
- ggml/src/ggml-metal.m | 30 +++++++++++++-----------------
- 1 file changed, 13 insertions(+), 17 deletions(-)
-
-diff --git a/ggml/src/ggml-metal.m b/ggml/src/ggml-metal.m
-index 9da08fe2..3a433703 100644
---- a/ggml/src/ggml-metal.m
-+++ b/ggml/src/ggml-metal.m
-@@ -1720,27 +1720,23 @@ static void ggml_metal_encode_node(
-                 // to the matrix-vector kernel
-                 int ne11_mm_min = 1;
- 
--#if 0
-                 // the numbers below are measured on M2 Ultra for 7B and 13B models
-                 // these numbers do not translate to other devices or model sizes
-                 // TODO: need to find a better approach
--                        if ([ctx->device.name isEqualToString:@"Apple M2 Ultra"]) {
--                            switch (src0t) {
--                                case GGML_TYPE_F16:  ne11_mm_min = 2;  break;
--                                case GGML_TYPE_Q8_0: ne11_mm_min = 7;  break;
--                                case GGML_TYPE_Q2_K: ne11_mm_min = 15; break;
--                                case GGML_TYPE_Q3_K: ne11_mm_min = 7;  break;
--                                case GGML_TYPE_Q4_0:
--                                case GGML_TYPE_Q4_1: ne11_mm_min = 15; break;
--                                case GGML_TYPE_Q4_K: ne11_mm_min = 11; break;
--                                case GGML_TYPE_Q5_0:                          // not tested yet
--                                case GGML_TYPE_Q5_1: ne11_mm_min = 13; break; // not tested yet
--                                case GGML_TYPE_Q5_K: ne11_mm_min = 7;  break;
--                                case GGML_TYPE_Q6_K: ne11_mm_min = 7;  break;
--                                default:             ne11_mm_min = 1;  break;
--                            }
-+                        switch (src0t) {
-+                            case GGML_TYPE_F16:  ne11_mm_min = 2;  break;
-+                            case GGML_TYPE_Q8_0: ne11_mm_min = 7;  break;
-+                            case GGML_TYPE_Q2_K: ne11_mm_min = 15; break;
-+                            case GGML_TYPE_Q3_K: ne11_mm_min = 7;  break;
-+                            case GGML_TYPE_Q4_0:
-+                            case GGML_TYPE_Q4_1: ne11_mm_min = 15; break;
-+                            case GGML_TYPE_Q4_K: ne11_mm_min = 11; break;
-+                            case GGML_TYPE_Q5_0:                          // not tested yet
-+                            case GGML_TYPE_Q5_1: ne11_mm_min = 13; break; // not tested yet
-+                            case GGML_TYPE_Q5_K: ne11_mm_min = 7;  break;
-+                            case GGML_TYPE_Q6_K: ne11_mm_min = 7;  break;
-+                            default:             ne11_mm_min = 1;  break;
-                         }
--#endif
- 
-                         // for now the matrix-matrix multiplication kernel only works on A14+/M1+ SoCs
-                         // AMD GPU and older A-chips will reuse matrix-vector multiplication kernel
--- 
-2.39.3 (Apple Git-146)
-

llm/patches/0005-default-pretokenizer.patch

-From 0e531d69786c4a96a3a2bcf7b2d576bd6f7edf25 Mon Sep 17 00:00:00 2001
-From: Michael Yang <mxyng@pm.me>
-Date: Mon, 16 Sep 2024 15:53:13 -0700
-Subject: [PATCH] 05-default-pretokenizer.diff
-
----
- src/llama.cpp | 14 +++-----------
- 1 file changed, 3 insertions(+), 11 deletions(-)
-
-diff --git a/src/llama.cpp b/src/llama.cpp
-index 4c0a1bb6..800dfb95 100644
---- a/src/llama.cpp
-+++ b/src/llama.cpp
-@@ -6287,16 +6287,7 @@ static void llm_load_vocab(
-         if (vocab.type == LLAMA_VOCAB_TYPE_BPE) {
-             vocab.tokenizer_add_space_prefix = false;
-             vocab.tokenizer_clean_spaces = true;
--            if (tokenizer_pre.empty()) {
--                LLAMA_LOG_WARN("%s: missing pre-tokenizer type, using: 'default'\n", __func__);
--                LLAMA_LOG_WARN("%s:                                             \n", __func__);
--                LLAMA_LOG_WARN("%s: ************************************        \n", __func__);
--                LLAMA_LOG_WARN("%s: GENERATION QUALITY WILL BE DEGRADED!        \n", __func__);
--                LLAMA_LOG_WARN("%s: CONSIDER REGENERATING THE MODEL             \n", __func__);
--                LLAMA_LOG_WARN("%s: ************************************        \n", __func__);
--                LLAMA_LOG_WARN("%s:                                             \n", __func__);
--                vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
--            } else if (tokenizer_pre == "default") {
-+            if (tokenizer_pre == "default") {
-                 vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
-             } else if (
-                     tokenizer_pre == "llama3"   ||
-@@ -6398,7 +6389,8 @@ static void llm_load_vocab(
-                 vocab.tokenizer_add_bos = true;
-                 vocab.tokenizer_clean_spaces = false;
-             } else {
--                throw std::runtime_error(format("unknown pre-tokenizer type: '%s'", tokenizer_pre.c_str()));
-+                LLAMA_LOG_WARN("%s: missing or unrecognized pre-tokenizer type, using: 'default'\n", __func__);
-+                vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
-             }
-         } else if (vocab.type == LLAMA_VOCAB_TYPE_SPM) {
-             vocab.type_pre = LLAMA_VOCAB_PRE_TYPE_DEFAULT;
--- 
-2.39.3 (Apple Git-146)
-

llm/patches/0006-embeddings.patch

-From 235b6d876a74cb09abe26985fa89ebe5bfc9f562 Mon Sep 17 00:00:00 2001
-From: Gabe Goodhart <ghart@us.ibm.com>
-Date: Thu, 19 Sep 2024 17:06:17 -0600
-Subject: [PATCH] embeddings
-
----
- src/llama.cpp | 15 +++++++++------
- 1 file changed, 9 insertions(+), 6 deletions(-)
-
-diff --git a/src/llama.cpp b/src/llama.cpp
-index 1a8e0c51..e55ec3f8 100644
---- a/src/llama.cpp
-+++ b/src/llama.cpp
-@@ -16516,7 +16516,7 @@ static size_t llama_output_reserve(llama_context & lctx, size_t n_outputs) {
-     const auto n_embd  = hparams.n_embd;
- 
-     // TODO: use a per-batch flag for logits presence instead
--    const bool has_logits = !cparams.embeddings;
-+    const bool has_logits =  cparams.causal_attn;
-     const bool has_embd   =  cparams.embeddings && (cparams.pooling_type == LLAMA_POOLING_TYPE_NONE);
- 
-     const size_t logits_size = has_logits ? n_vocab*n_outputs_max : 0;
-@@ -16794,20 +16794,23 @@ static int llama_decode_internal(
-             // no output
-             res  = nullptr;
-             embd = nullptr;
--        } else if (cparams.embeddings) {
--            res  = nullptr; // do not extract logits for embedding case
--            embd = nullptr;
-+        }
-+
-+        if (cparams.embeddings) {
-             for (int i = ggml_graph_n_nodes(gf) - 1; i >= 0; --i) {
-+                embd = ggml_graph_node(gf, i);
-                 if (strcmp(ggml_graph_node(gf, i)->name, "result_embd_pooled") == 0) {
--                    embd = ggml_graph_node(gf, i);
-                     break;
-                 }
-             }
--            GGML_ASSERT(embd != nullptr && "missing embeddings tensor");
-         } else {
-             embd = nullptr; // do not extract embeddings when not needed
-             GGML_ASSERT(strcmp(res->name, "result_output") == 0 && "missing result_output tensor");
-         }
-+
-+        if (!cparams.causal_attn) {
-+            res = nullptr; // do not extract logits when not needed
-+        }
-         // LLAMA_LOG_INFO("graph build time: %.3f ms (%d nodes, %d leafs)\n", (ggml_time_us() - t_start_us)/1000.0, gf->n_nodes, gf->n_leafs);
- 
-         ggml_backend_sched_alloc_graph(lctx.sched, gf);
--- 
-2.39.3 (Apple Git-146)
-

llm/patches/0007-clip-unicode.patch

-From 01c42149cbdc194644a2f138598029938e0dd447 Mon Sep 17 00:00:00 2001
-From: Gabe Goodhart <ghart@us.ibm.com>
-Date: Thu, 19 Sep 2024 17:09:57 -0600
-Subject: [PATCH] clip unicode
-
----
- examples/llava/clip.cpp | 23 +++++++++++++++++++++++
- 1 file changed, 23 insertions(+)
-
-diff --git a/examples/llava/clip.cpp b/examples/llava/clip.cpp
-index b8941c74..3a735f17 100644
---- a/examples/llava/clip.cpp
-+++ b/examples/llava/clip.cpp
-@@ -40,6 +40,14 @@
- #include <cinttypes>
- #include <limits>
- 
-+#if defined(_WIN32)
-+#define WIN32_LEAN_AND_MEAN
-+#ifndef NOMINMAX
-+    #define NOMINMAX
-+#endif
-+#include <windows.h>
-+#endif
-+
- #define LOG_INF(...) do { fprintf(stdout, __VA_ARGS__); } while (0)
- #define LOG_WRN(...) do { fprintf(stderr, __VA_ARGS__); } while (0)
- #define LOG_ERR(...) do { fprintf(stderr, __VA_ARGS__); } while (0)
-@@ -1227,7 +1235,22 @@ struct clip_ctx * clip_model_load(const char * fname, const int verbosity = 1) {
-             return nullptr;
-         }
- 
-+#ifdef _WIN32
-+        int wlen = MultiByteToWideChar(CP_UTF8, 0, fname, -1, NULL, 0);
-+        if (!wlen) {
-+            return NULL;
-+        }
-+        wchar_t * wbuf = (wchar_t *) malloc(wlen * sizeof(wchar_t));
-+        wlen = MultiByteToWideChar(CP_UTF8, 0, fname, -1, wbuf, wlen);
-+        if (!wlen) {
-+            free(wbuf);
-+            return NULL;
-+        }
-+        auto fin = std::ifstream(wbuf, std::ios::binary);
-+        free(wbuf);
-+#else
-         auto fin = std::ifstream(fname, std::ios::binary);
-+#endif
-         if (!fin) {
-             LOG_ERR("cannot open model file for loading tensors\n");
-             clip_free(new_clip);
--- 
-2.39.3 (Apple Git-146)
-

llm/patches/0008-solar-pro.patch

-From a8fe40fa7b026d2db9bb6aeecd24fcd2027110ec Mon Sep 17 00:00:00 2001
-From: Michael Yang <mxyng@pm.me>
-Date: Mon, 16 Sep 2024 15:53:16 -0700
-Subject: [PATCH] add solar-pro support
-
-solar-pro introduces block skip connections where blocks are connected
-to other, non-sequential blocks with a scale multiple
-
-this change adds 4 new keys to store the skip connections and one new
-tensor to store the scalar. the scalar is implemented a 1-dimensional
-tensor with 2 elements dervied from the model's bskcn_tv configuration.
-in general, the values are (bskcn_tv, 1 - bskcn_tv)
----
- src/llama.cpp | 270 +++++++++++++++++++++++++++++++++++++++++++++++---
- 1 file changed, 255 insertions(+), 15 deletions(-)
-
-diff --git a/src/llama.cpp b/src/llama.cpp
-index 4c0a1bb6..c6fc0c3f 100644
---- a/src/llama.cpp
-+++ b/src/llama.cpp
-@@ -217,6 +217,7 @@ enum llm_arch {
-     LLM_ARCH_GRANITE,
-     LLM_ARCH_GRANITE_MOE,
-     LLM_ARCH_CHAMELEON,
-+    LLM_ARCH_SOLAR,
-     LLM_ARCH_UNKNOWN,
- };
- 
-@@ -270,6 +271,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
-     { LLM_ARCH_GRANITE,         "granite"      },
-     { LLM_ARCH_GRANITE_MOE,     "granitemoe"   },
-     { LLM_ARCH_CHAMELEON,       "chameleon"    },
-+    { LLM_ARCH_SOLAR,           "solar"        },
-     { LLM_ARCH_UNKNOWN,         "(unknown)"    },
- };
- 
-@@ -327,6 +329,7 @@ enum llm_kv {
-     LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT,
-     LLM_KV_ATTENTION_SLIDING_WINDOW,
-     LLM_KV_ATTENTION_SCALE,
-+    LLM_KV_ATTENTION_BLOCK_SKIP_CONNECTION,
- 
-     LLM_KV_ROPE_DIMENSION_COUNT,
-     LLM_KV_ROPE_FREQ_BASE,
-@@ -421,20 +424,21 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
-     { LLM_KV_RESIDUAL_SCALE,                    "%s.residual_scale"                    },
-     { LLM_KV_EMBEDDING_SCALE,                   "%s.embedding_scale"                   },
- 
--    { LLM_KV_ATTENTION_HEAD_COUNT,             "%s.attention.head_count"             },
--    { LLM_KV_ATTENTION_HEAD_COUNT_KV,          "%s.attention.head_count_kv"          },
--    { LLM_KV_ATTENTION_MAX_ALIBI_BIAS,         "%s.attention.max_alibi_bias"         },
--    { LLM_KV_ATTENTION_CLAMP_KQV,              "%s.attention.clamp_kqv"              },
--    { LLM_KV_ATTENTION_KEY_LENGTH,             "%s.attention.key_length"             },
--    { LLM_KV_ATTENTION_VALUE_LENGTH,           "%s.attention.value_length"           },
--    { LLM_KV_ATTENTION_LAYERNORM_EPS,          "%s.attention.layer_norm_epsilon"     },
--    { LLM_KV_ATTENTION_LAYERNORM_RMS_EPS,      "%s.attention.layer_norm_rms_epsilon" },
--    { LLM_KV_ATTENTION_CAUSAL,                 "%s.attention.causal"                 },
--    { LLM_KV_ATTENTION_Q_LORA_RANK,            "%s.attention.q_lora_rank"            },
--    { LLM_KV_ATTENTION_KV_LORA_RANK,           "%s.attention.kv_lora_rank"           },
--    { LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, "%s.attention.relative_buckets_count" },
--    { LLM_KV_ATTENTION_SLIDING_WINDOW,         "%s.attention.sliding_window"         },
--    { LLM_KV_ATTENTION_SCALE,                  "%s.attention.scale"                  },
-+    { LLM_KV_ATTENTION_HEAD_COUNT,             "%s.attention.head_count"               },
-+    { LLM_KV_ATTENTION_HEAD_COUNT_KV,          "%s.attention.head_count_kv"            },
-+    { LLM_KV_ATTENTION_MAX_ALIBI_BIAS,         "%s.attention.max_alibi_bias"           },
-+    { LLM_KV_ATTENTION_CLAMP_KQV,              "%s.attention.clamp_kqv"                },
-+    { LLM_KV_ATTENTION_KEY_LENGTH,             "%s.attention.key_length"               },
-+    { LLM_KV_ATTENTION_VALUE_LENGTH,           "%s.attention.value_length"             },
-+    { LLM_KV_ATTENTION_LAYERNORM_EPS,          "%s.attention.layer_norm_epsilon"       },
-+    { LLM_KV_ATTENTION_LAYERNORM_RMS_EPS,      "%s.attention.layer_norm_rms_epsilon"   },
-+    { LLM_KV_ATTENTION_CAUSAL,                 "%s.attention.causal"                   },
-+    { LLM_KV_ATTENTION_Q_LORA_RANK,            "%s.attention.q_lora_rank"              },
-+    { LLM_KV_ATTENTION_KV_LORA_RANK,           "%s.attention.kv_lora_rank"             },
-+    { LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, "%s.attention.relative_buckets_count"   },
-+    { LLM_KV_ATTENTION_SLIDING_WINDOW,         "%s.attention.sliding_window"           },
-+    { LLM_KV_ATTENTION_SCALE,                  "%s.attention.scale"                    },
-+    { LLM_KV_ATTENTION_BLOCK_SKIP_CONNECTION,  "%s.attention.block_skip_connection.%d" },
- 
-     { LLM_KV_ROPE_DIMENSION_COUNT,          "%s.rope.dimension_count"                 },
-     { LLM_KV_ROPE_FREQ_BASE,                "%s.rope.freq_base"                       },
-@@ -608,6 +612,7 @@ enum llm_tensor {
-     LLM_TENSOR_ENC_OUTPUT_NORM,
-     LLM_TENSOR_CLS,
-     LLM_TENSOR_CLS_OUT,
-+    LLM_TENSOR_BSKCN_TV,
- };
- 
- static const std::map<llm_arch, std::map<llm_tensor, std::string>> LLM_TENSOR_NAMES = {
-@@ -1527,6 +1532,25 @@ static const std::map<llm_arch, std::map<llm_tensor, std::string>> LLM_TENSOR_NA
-             { LLM_TENSOR_ATTN_K_NORM,     "blk.%d.attn_k_norm" },
-         },
-     },
-+
-+    {
-+        LLM_ARCH_SOLAR,
-+        {
-+            { LLM_TENSOR_TOKEN_EMBD,      "token_embd" },
-+            { LLM_TENSOR_OUTPUT_NORM,     "output_norm" },
-+            { LLM_TENSOR_OUTPUT,          "output" },
-+            { LLM_TENSOR_ATTN_NORM,       "blk.%d.attn_norm" },
-+            { LLM_TENSOR_ATTN_Q,          "blk.%d.attn_q" },
-+            { LLM_TENSOR_ATTN_K,          "blk.%d.attn_k" },
-+            { LLM_TENSOR_ATTN_V,          "blk.%d.attn_v" },
-+            { LLM_TENSOR_ATTN_OUT,        "blk.%d.attn_output" },
-+            { LLM_TENSOR_FFN_NORM,        "blk.%d.ffn_norm" },
-+            { LLM_TENSOR_FFN_GATE,        "blk.%d.ffn_gate" },
-+            { LLM_TENSOR_FFN_DOWN,        "blk.%d.ffn_down" },
-+            { LLM_TENSOR_FFN_UP,          "blk.%d.ffn_up" },
-+            { LLM_TENSOR_BSKCN_TV,        "bskcn_tv" },
-+        },
-+    },
-     {
-         LLM_ARCH_UNKNOWN,
-         {
-@@ -2360,6 +2384,7 @@ enum e_model {
-     MODEL_15B,
-     MODEL_16B,
-     MODEL_20B,
-+    MODEL_22B,
-     MODEL_30B,
-     MODEL_34B,
-     MODEL_35B,
-@@ -2409,6 +2434,8 @@ struct llama_hparams {
-     std::array<uint32_t, LLAMA_MAX_LAYERS> n_head_kv_arr;
-     std::array<uint32_t, LLAMA_MAX_LAYERS> n_ff_arr;
- 
-+    std::array<std::array<uint32_t, LLAMA_MAX_LAYERS>, 4> n_bskcn_arr;
-+
-     uint32_t n_layer_dense_lead = 0;
-     uint32_t n_lora_q = 0;
-     uint32_t n_lora_kv = 0;
-@@ -2479,6 +2506,7 @@ struct llama_hparams {
-         if (this->n_head_arr    != other.n_head_arr)    return true;
-         if (this->n_head_kv_arr != other.n_head_kv_arr) return true;
-         if (this->n_ff_arr      != other.n_ff_arr)      return true;
-+        if (this->n_bskcn_arr   != other.n_bskcn_arr)   return true;
- 
-         if (this->n_rel_attn_bkts    != other.n_rel_attn_bkts)    return true;
-         if (this->n_layer_dense_lead != other.n_layer_dense_lead) return true;
-@@ -2588,6 +2616,14 @@ struct llama_hparams {
-             return ssm_d_state * ssm_d_inner;
-         }
-     }
-+
-+    bool n_bskcn(uint32_t n, uint32_t il = 0) const {
-+        if (il < n_layer) {
-+            return n_bskcn_arr[n][il] > 0;
-+        }
-+
-+        GGML_ABORT("fatal error");
-+    }
- };
- 
- static_assert(std::is_trivially_copyable<llama_hparams>::value, "llama_hparams must be trivially copyable");
-@@ -2769,6 +2805,8 @@ struct llama_layer {
-     struct ggml_tensor * ffn_gate_scale;
-     struct ggml_tensor * ffn_up_scale;
-     struct ggml_tensor * ffn_down_scale;
-+
-+    struct ggml_tensor * bskcn_tv;
- };
- 
- // very similar to llama_batch,
-@@ -6134,6 +6172,21 @@ static void llm_load_hparams(
-                     default: model.type = e_model::MODEL_UNKNOWN;
-                }
-             } break;
-+        case LLM_ARCH_SOLAR:
-+            {
-+                ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
-+
-+                for (int i = 0; i < hparams.n_bskcn_arr.max_size(); ++i) {
-+                    auto & bskcn = hparams.n_bskcn_arr.at(i);
-+                    bskcn.fill(0);
-+                    ml.get_key_or_arr(::format(LLM_KV_NAMES.at(LLM_KV_ATTENTION_BLOCK_SKIP_CONNECTION), LLM_ARCH_NAMES.at(ml.llm_kv.arch), i), bskcn, hparams.n_layer, false);
-+                }
-+
-+                switch (hparams.n_layer) {
-+                    case 64: model.type = e_model::MODEL_22B; break;
-+                    default: model.type = e_model::MODEL_UNKNOWN;
-+                }
-+            }
-         default: (void)0;
-     }
- 
-@@ -8839,6 +8892,37 @@ static bool llm_load_tensors(
- 
-                         layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});
- 
-+                        layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd,   n_ff});
-+                        layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {  n_ff, n_embd});
-+                        layer.ffn_up   = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP,   "weight", i), {n_embd,   n_ff});
-+                    }
-+                } break;
-+            case LLM_ARCH_SOLAR:
-+                {
-+                    model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
-+
-+                    // output
-+                    {
-+                        model.output_norm = ml.create_tensor(ctx_output,       tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});
-+                        model.output      = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT,      "weight"), {n_embd, n_vocab}, llama_model_loader::TENSOR_NOT_REQUIRED);
-+                    }
-+
-+                    for (int i = 0; i < n_layer; ++i) {
-+                        ggml_context * ctx_layer = ctx_for_layer(i);
-+                        ggml_context * ctx_split = ctx_for_layer_split(i);
-+
-+                        auto & layer = model.layers[i];
-+
-+                        layer.attn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd});
-+                        layer.wq = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_Q,   "weight", i), {n_embd, n_embd_head_k * n_head});
-+                        layer.wk = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_K,   "weight", i), {n_embd, n_embd_k_gqa});
-+                        layer.wv = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_V,   "weight", i), {n_embd, n_embd_v_gqa});
-+                        layer.wo = ml.create_tensor(ctx_split, tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_head_k * n_head, n_embd});
-+
-+                        layer.ffn_norm = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd});
-+
-+                        layer.bskcn_tv = ml.create_tensor(ctx_layer, tn(LLM_TENSOR_BSKCN_TV, "weight"), {2}, llama_model_loader::TENSOR_NOT_REQUIRED | (i != 0 ? llama_model_loader::TENSOR_DUPLICATED : 0));
-+
-                         layer.ffn_gate = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd,   n_ff});
-                         layer.ffn_down = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_DOWN, "weight", i), {  n_ff, n_embd});
-                         layer.ffn_up   = ml.create_tensor(ctx_split, tn(LLM_TENSOR_FFN_UP,   "weight", i), {n_embd,   n_ff});
-@@ -16009,7 +16093,6 @@ struct llm_build_context {
- 
-         return gf;
-     }
--
-     // ref: https://github.com/facebookresearch/chameleon
-     // based on the original build_llama() function, changes:
-     //   * qk-norm
-@@ -16187,6 +16270,158 @@ struct llm_build_context {
- 
-         return gf;
-     }
-+
-+    ggml_cgraph * build_solar() {
-+        struct ggml_cgraph * gf = ggml_new_graph_custom(ctx0, llama_model_max_nodes(model), false);
-+
-+        // mutable variable, needed during the last layer of the computation to skip unused tokens
-+        int32_t n_tokens = this->n_tokens;
-+
-+        const int64_t n_embd_head = hparams.n_embd_head_v;
-+        GGML_ASSERT(n_embd_head == hparams.n_embd_head_k);
-+        GGML_ASSERT(n_embd_head == hparams.n_rot);
-+
-+        struct ggml_tensor * cur;
-+        struct ggml_tensor * inpL;
-+
-+        inpL = llm_build_inp_embd(ctx0, lctx, hparams, batch, model.tok_embd, cb);
-+
-+        // inp_pos - contains the positions
-+        struct ggml_tensor * inp_pos = build_inp_pos();
-+
-+        // KQ_mask (mask for 1 head, it will be broadcasted to all heads)
-+        struct ggml_tensor * KQ_mask = build_inp_KQ_mask();
-+
-+        struct ggml_tensor * bskcn_1;
-+        struct ggml_tensor * bskcn_2;
-+
-+        for (int il = 0; il < n_layer; ++il) {
-+            struct ggml_tensor * inpSA = inpL;
-+
-+            if (hparams.n_bskcn(0, il)) {
-+                bskcn_1 = inpSA;
-+            }
-+
-+            if (hparams.n_bskcn(1, il)) {
-+                bskcn_2 = inpSA;
-+            }
-+
-+            if (hparams.n_bskcn(2, il)) {
-+                inpSA = ggml_add(
-+                   ctx0,
-+                   ggml_mul(ctx0, bskcn_1, ggml_view_1d(ctx0, model.layers[il].bskcn_tv, 1, 0)),
-+                   ggml_mul(ctx0, inpSA, ggml_view_1d(ctx0, model.layers[il].bskcn_tv, 1, ggml_element_size(model.layers[il].bskcn_tv))));
-+            }
-+
-+            if (hparams.n_bskcn(3, il)) {
-+                inpSA = ggml_add(
-+                   ctx0,
-+                   ggml_mul(ctx0, bskcn_2, ggml_view_1d(ctx0, model.layers[il].bskcn_tv, 1, 0)),
-+                   ggml_mul(ctx0, inpSA, ggml_view_1d(ctx0, model.layers[il].bskcn_tv, 1, ggml_element_size(model.layers[il].bskcn_tv))));
-+            }
-+
-+            // norm
-+            cur = llm_build_norm(ctx0, inpL, hparams,
-+                    model.layers[il].attn_norm, NULL,
-+                    LLM_NORM_RMS, cb, il);
-+            cb(cur, "attn_norm", il);
-+
-+            // self-attention
-+            {
-+                // rope freq factors for llama3; may return nullptr for llama2 and other models
-+                struct ggml_tensor * rope_factors = build_rope_factors(il);
-+
-+                // compute Q and K and RoPE them
-+                struct ggml_tensor * Qcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wq, cur);
-+                cb(Qcur, "Qcur", il);
-+                if (model.layers[il].bq) {
-+                    Qcur = ggml_add(ctx0, Qcur, model.layers[il].bq);
-+                    cb(Qcur, "Qcur", il);
-+                }
-+
-+                struct ggml_tensor * Kcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wk, cur);
-+                cb(Kcur, "Kcur", il);
-+                if (model.layers[il].bk) {
-+                    Kcur = ggml_add(ctx0, Kcur, model.layers[il].bk);
-+                    cb(Kcur, "Kcur", il);
-+                }
-+
-+                struct ggml_tensor * Vcur = llm_build_lora_mm(lctx, ctx0, model.layers[il].wv, cur);
-+                cb(Vcur, "Vcur", il);
-+                if (model.layers[il].bv) {
-+                    Vcur = ggml_add(ctx0, Vcur, model.layers[il].bv);
-+                    cb(Vcur, "Vcur", il);
-+                }
-+
-+                Qcur = ggml_rope_ext(
-+                    ctx0, ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head, n_tokens), inp_pos, rope_factors,
-+                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
-+                    ext_factor, attn_factor, beta_fast, beta_slow
-+                );
-+                cb(Qcur, "Qcur", il);
-+
-+                Kcur = ggml_rope_ext(
-+                    ctx0, ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens), inp_pos, rope_factors,
-+                    n_rot, rope_type, n_ctx_orig, freq_base, freq_scale,
-+                    ext_factor, attn_factor, beta_fast, beta_slow
-+                );
-+                cb(Kcur, "Kcur", il);
-+
-+                cur = llm_build_kv(ctx0, lctx, kv_self, gf,
-+                        model.layers[il].wo, model.layers[il].bo,
-+                        Kcur, Vcur, Qcur, KQ_mask, n_tokens, kv_head, n_kv, 1.0f/sqrtf(float(n_embd_head)), cb, il);
-+            }
-+
-+            if (il == n_layer - 1) {
-+                // skip computing output for unused tokens
-+                struct ggml_tensor * inp_out_ids = build_inp_out_ids();
-+                n_tokens = n_outputs;
-+                cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);
-+                inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);
-+            }
-+
-+            struct ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);
-+            cb(ffn_inp, "ffn_inp", il);
-+
-+            // feed-forward network
-+            cur = llm_build_norm(ctx0, ffn_inp, hparams,
-+                    model.layers[il].ffn_norm, NULL,
-+                    LLM_NORM_RMS, cb, il);
-+            cb(cur, "ffn_norm", il);
-+
-+            cur = llm_build_ffn(ctx0, lctx, cur,
-+                    model.layers[il].ffn_up,   model.layers[il].ffn_up_b,   NULL,
-+                    model.layers[il].ffn_gate, model.layers[il].ffn_gate_b, NULL,
-+                    model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,
-+                    NULL,
-+                    LLM_FFN_SILU, LLM_FFN_PAR, cb, il);
-+            cb(cur, "ffn_out", il);
-+
-+            cur = ggml_add(ctx0, cur, ffn_inp);
-+            cb(cur, "ffn_out", il);
-+
-+            cur = lctx.cvec.apply_to(ctx0, cur, il);
-+            cb(cur, "l_out", il);
-+
-+            // input for next layer
-+            inpL = cur;
-+        }
-+
-+        cur = inpL;
-+
-+        cur = llm_build_norm(ctx0, cur, hparams,
-+                model.output_norm, NULL,
-+                LLM_NORM_RMS, cb, -1);
-+        cb(cur, "result_norm", -1);
-+
-+        // lm_head
-+        cur = llm_build_lora_mm(lctx, ctx0, model.output, cur);
-+        cb(cur, "result_output", -1);
-+
-+        ggml_build_forward_expand(gf, cur);
-+
-+        return gf;
-+    }
- };
- 
- static struct ggml_cgraph * llama_build_graph_defrag(llama_context & lctx, const std::vector<uint32_t> & ids) {
-@@ -16451,6 +16686,10 @@ static struct ggml_cgraph * llama_build_graph(
-             {
-                 result = llm.build_chameleon();
-             } break;
-+        case LLM_ARCH_SOLAR:
-+            {
-+                result = llm.build_solar();
-+            } break;
-         default:
-             GGML_ABORT("fatal error");
-     }
-@@ -19594,6 +19833,7 @@ enum llama_rope_type llama_rope_type(const struct llama_model * model) {
-         case LLM_ARCH_GRANITE:
-         case LLM_ARCH_GRANITE_MOE:
-         case LLM_ARCH_CHAMELEON:
-+        case LLM_ARCH_SOLAR:
-             return LLAMA_ROPE_TYPE_NORM;
- 
-         // the pairs of head values are offset by n_rot/2
--- 
-2.39.3 (Apple Git-146)
-

scripts/build_darwin.sh

@@ -7,15 +7,9 @@ set -e
 mkdir -p dist
 
 for TARGETARCH in arm64 amd64; do
-    if [ -n "${OLLAMA_NEW_RUNNERS}" ]; then
-        echo "Building Go runner darwin $TARGETARCH"
-        rm -rf llama/build
-        GOOS=darwin ARCH=$TARGETARCH GOARCH=$TARGETARCH make -C llama -j 8
-    else
-        echo "Building C++ runner darwin $TARGETARCH"
-        rm -rf llm/build
-        GOOS=darwin GOARCH=$TARGETARCH go generate ./...
-    fi
+    echo "Building Go runner darwin $TARGETARCH"
+    rm -rf llama/build
+    GOOS=darwin ARCH=$TARGETARCH GOARCH=$TARGETARCH make -C llama -j 8
     # These require Xcode v13 or older to target MacOS v11
     # If installed to an alternate location use the following to enable
     # export SDKROOT=/Applications/Xcode_12.5.1.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk

scripts/build_docker.sh

@@ -19,7 +19,7 @@ docker buildx build \
     ${LOAD_OR_PUSH} \
     --platform=${PLATFORM} \
     ${OLLAMA_COMMON_BUILD_ARGS} \
-    -f ${DOCKERFILE_DIR}Dockerfile \
+    -f Dockerfile \
     -t ${FINAL_IMAGE_REPO}:$VERSION \
     .
 
@@ -29,7 +29,7 @@ if echo $PLATFORM | grep "amd64" > /dev/null; then
         --platform=linux/amd64 \
         ${OLLAMA_COMMON_BUILD_ARGS} \
         --target runtime-rocm \
-        -f ${DOCKERFILE_DIR}Dockerfile \
+        -f Dockerfile \
         -t ${FINAL_IMAGE_REPO}:$VERSION-rocm \
         .
 fi
\ No newline at end of file

scripts/build_linux.sh

@@ -19,7 +19,7 @@ docker buildx build \
         --platform=${PLATFORM} \
         ${OLLAMA_COMMON_BUILD_ARGS} \
         --target dist \
-        -f ${DOCKERFILE_DIR}Dockerfile \
+        -f Dockerfile \
         .
 
 # buildx behavior changes for single vs. multiplatform

scripts/build_remote.py

-#!/usr/bin/env python3
-import subprocess
-import sys
-from urllib.parse import urlparse
-from git import Repo
-
-# Helper script to be able to build on remote repos using git to push local changes
-# (e.g. particularly helpful to target a remote windows build system)
-#
-# Typical windows remote git config looks like this:
-#
-#[remote "windows-pa"]
-#        url = jdoe@desktop-foo:C:/Users/Jdoe/code/ollama
-#        fetch = +refs/heads/*:refs/remotes/windows-pa/*
-#        uploadpack = powershell git upload-pack
-#        receivepack = powershell git receive-pack
-#
-
-# TODO - add argpare and make this more configurable 
-# - force flag becomes optional
-# - generate, build or test ...
-
-# Note: remote repo will need this run once:
-# git config --local receive.denyCurrentBranch updateInstead
-repo = Repo(".")
-
-# On linux, add links in /usr/local/bin to the go binaries to avoid needing this
-# GoCmd = "/usr/local/go/bin/go" 
-GoCmd = "go" 
-
-if repo.is_dirty():
-    print("Tree is dirty.  Commit your changes before running this script")
-    sys.exit(1)
-
-if len(sys.argv) != 2:
-    print("Please specify the remote name: " + ', '.join([r.name for r in repo.remotes]))
-    sys.exit(1)
-remote_name = sys.argv[1]
-
-remote = {r.name: r for r in repo.remotes}[remote_name]
-raw_url = list(remote.urls)[0]
-url = urlparse(raw_url)
-# Windows urls don't quite parse properly
-if url.scheme == "" and url.netloc == "":
-    url = urlparse("ssh://" + raw_url)
-print("URL: " + str(url))
-netloc = url.netloc.split(":")[0]
-path = url.path
-branch_name = repo.active_branch.name
-
-print("Force pushing content to remote...")
-# Use with care given the force push
-remote.push(force=True).raise_if_error()
-
-print("Ensuring correct branch checked out on remote via ssh...")
-subprocess.check_call(['ssh', netloc, 'cd', path, ';', 'git', 'checkout', branch_name])
-
-
-# TODO - add some hardening to try to figure out how to set up the path properly
-# subprocess.check_call(['ssh', netloc, 'cd', path, ';', 'env'])
-# TODO - or consider paramiko maybe
-
-print("Running Windows Build Script")
-subprocess.check_call(['ssh', netloc, 'cd', path, ';', "powershell", "-ExecutionPolicy", "Bypass", "-File", "./scripts/build_windows.ps1"])
-
-# print("Building")
-# subprocess.check_call(['ssh', netloc, 'cd', path, ';', GoCmd, 'build', '.'])
-
-print("Copying built result")
-subprocess.check_call(['scp', netloc +":"+ path + "/ollama.exe",  './dist/'])
-
-print("Copying installer")
-subprocess.check_call(['scp', netloc +":"+ path + "/dist/Ollama Setup.exe",  './dist/'])
-
-
-

scripts/build_windows.ps1

@@ -83,51 +83,8 @@ function buildOllama() {
     if ($null -eq ${env:OLLAMA_SKIP_GENERATE}) {
         write-host "Building ollama runners"
         Remove-Item -ea 0 -recurse -force -path "${script:SRC_DIR}\dist\windows-${script:ARCH}"
-        if ($null -eq ${env:OLLAMA_NEW_RUNNERS}) {
-            # Start by skipping CUDA to build everything else
-            write-host "Building ollama runners"
-            powershell -Command { $env:OLLAMA_SKIP_CUDA_GENERATE="1"; & go generate ./... }
-            if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}    
-
-            # Then skip everyhting else and build all the CUDA variants
-            foreach ($env:CUDA_LIB_DIR in $script:CUDA_DIRS) {
-                write-host "Building CUDA ${env:CUDA_LIB_DIR} runner"
-
-                if ($env:CUDA_LIB_DIR.Contains("v12")) {
-                    powershell -Command {
-                        $env:OLLAMA_SKIP_CUDA_GENERATE=""
-                        $env:OLLAMA_SKIP_STATIC_GENERATE="1"
-                        $env:OLLAMA_SKIP_CPU_GENERATE="1"
-                        $env:OLLAMA_SKIP_ONEAPI_GENERATE="1"
-                        $env:OLLAMA_SKIP_ROCM_GENERATE="1"
-                        $env:CMAKE_CUDA_ARCHITECTURES="60;61;62;70;72;75;80;86;87;89;90;90a"
-                        $env:OLLAMA_CUSTOM_CUDA_DEFS="-DGGML_CUDA_USE_GRAPHS=on"
-                        $env:CUDA_PATH=split-path -path $env:CUDA_LIB_DIR -parent
-                        $env:PATH="$envs:CUDA_LIB_DIR;$env:PATH"
-                        & go generate ./...
-                    }
-                } else {
-                    powershell -Command {
-                        $env:OLLAMA_SKIP_CUDA_GENERATE=""
-                        $env:OLLAMA_SKIP_STATIC_GENERATE="1"
-                        $env:OLLAMA_SKIP_CPU_GENERATE="1"
-                        $env:OLLAMA_SKIP_ONEAPI_GENERATE="1"
-                        $env:OLLAMA_SKIP_ROCM_GENERATE="1"
-                        $env:CMAKE_CUDA_ARCHITECTURES="50;52;53;60;61;62;70;72;75;80;86"
-                        $env:OLLAMA_CUSTOM_CUDA_DEFS=""
-                        $env:CUDA_PATH=split-path -path $env:CUDA_LIB_DIR -parent
-                        $env:PATH="$envs:CUDA_LIB_DIR;$env:PATH"
-                        & go generate ./...
-                    }
-                }
-                if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}
-            }
-        } else {
-            & make -C llama -j 12
-            if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}
-        }
-        
-        if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}    
+        & make -C llama -j 12
+        if ($LASTEXITCODE -ne 0) { exit($LASTEXITCODE)}
     } else {
         write-host "Skipping generate step with OLLAMA_SKIP_GENERATE set"
     }
@@ -172,7 +129,7 @@ function gatherDependencies() {
     } else {
         $depArch=$script:TARGET_ARCH
     }
-    if ($depArch -eq "amd64") {
+    if ($depArch -eq "x64") {
         cp "${env:VCToolsRedistDir}\${depArch}\Microsoft.VC*.CRT\msvcp140*.dll" "${script:DIST_DIR}\lib\ollama\"
         cp "${env:VCToolsRedistDir}\${depArch}\Microsoft.VC*.CRT\vcruntime140.dll" "${script:DIST_DIR}\lib\ollama\"
         cp "${env:VCToolsRedistDir}\${depArch}\Microsoft.VC*.CRT\vcruntime140_1.dll" "${script:DIST_DIR}\lib\ollama\"

scripts/env.sh

@@ -20,12 +20,6 @@ OLLAMA_COMMON_BUILD_ARGS="--build-arg=VERSION \
     --build-arg=CUSTOM_CPU_FLAGS \
     --build-arg=GPU_RUNNER_CPU_FLAGS \
     --build-arg=AMDGPU_TARGETS"
-OLLAMA_NEW_RUNNERS=${OLLAMA_NEW_RUNNERS:-""}
-if [ -n "${OLLAMA_NEW_RUNNERS}" ]; then
-    DOCKERFILE_DIR="./llama/"
-else
-    DOCKERFILE_DIR="./"
-fi
 
 echo "Building Ollama"
 echo "VERSION=$VERSION"