cuda graph

llama/patches/0019-metal-add-mean-kernel-14267.patch

@@ -19,7 +19,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.m b/ggml/src/ggml-metal/ggml-metal.m
 index a9eeebc6..110c9ece 100644
 --- a/ggml/src/ggml-metal/ggml-metal.m
 +++ b/ggml/src/ggml-metal/ggml-metal.m
-@@ -489,6 +489,7 @@ static void ggml_backend_metal_device_rel(struct ggml_backend_metal_device_conte
+@@ -489,6 +489,7 @@ enum ggml_metal_kernel_type {
      GGML_METAL_KERNEL_TYPE_COS,
      GGML_METAL_KERNEL_TYPE_NEG,
      GGML_METAL_KERNEL_TYPE_SUM_ROWS,
@@ -27,7 +27,7 @@ index a9eeebc6..110c9ece 100644
      GGML_METAL_KERNEL_TYPE_POOL_2D_AVG_F32,
      GGML_METAL_KERNEL_TYPE_POOL_2D_MAX_F32,
      GGML_METAL_KERNEL_TYPE_ARGMAX,
-@@ -1436,6 +1437,7 @@ @implementation GGMLMetalClass
+@@ -1436,6 +1437,7 @@ static struct ggml_backend_metal_context * ggml_metal_init(ggml_backend_dev_t de
          GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_COS,                             cos,                             true);
          GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_NEG,                             neg,                             true);
          GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_SUM_ROWS,                        sum_rows,                        true);

llama/patches/0022-BF16-macos-version-guard.patch

@@ -12,7 +12,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.m b/ggml/src/ggml-metal/ggml-metal.m
 index 110c9ece..ab46f6e3 100644
 --- a/ggml/src/ggml-metal/ggml-metal.m
 +++ b/ggml/src/ggml-metal/ggml-metal.m
-@@ -89,7 +89,11 @@
+@@ -89,7 +89,11 @@ static id<MTLDevice> ggml_backend_metal_device_acq(struct ggml_backend_metal_dev
          ctx->has_bfloat |= [ctx->mtl_device supportsFamily:MTLGPUFamilyApple6];
  
  #if defined(GGML_METAL_USE_BF16)

llama/patches/0023-MXFP4.patch

@@ -722,7 +722,7 @@ diff --git a/ggml/src/ggml-metal/ggml-metal.m b/ggml/src/ggml-metal/ggml-metal.m
 index ab46f6e3..d8e05a21 100644
 --- a/ggml/src/ggml-metal/ggml-metal.m
 +++ b/ggml/src/ggml-metal/ggml-metal.m
-@@ -40,6 +40,7 @@
+@@ -40,6 +40,7 @@ static const NSInteger MTLGPUFamilyMetal3_GGML = 5001;
  static struct ggml_backend_reg    g_ggml_backend_metal_reg;
  static struct ggml_backend_device g_ggml_backend_metal_device;
  
@@ -730,7 +730,7 @@ index ab46f6e3..d8e05a21 100644
  // information about a Metal device
  // note: assumes single GPU device - the default one
  // TODO: support multiple GPU devices
-@@ -209,6 +210,7 @@ static void ggml_backend_metal_device_rel(struct ggml_backend_metal_device_conte
+@@ -209,6 +210,7 @@ enum ggml_metal_kernel_type {
      GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_0_F32,
      GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_1_F32,
      GGML_METAL_KERNEL_TYPE_MUL_MV_Q8_0_F32,
@@ -738,7 +738,7 @@ index ab46f6e3..d8e05a21 100644
      GGML_METAL_KERNEL_TYPE_MUL_MV_EXT_F16_F32_R1_2,
      GGML_METAL_KERNEL_TYPE_MUL_MV_EXT_F16_F32_R1_3,
      GGML_METAL_KERNEL_TYPE_MUL_MV_EXT_F16_F32_R1_4,
-@@ -288,6 +290,7 @@ static void ggml_backend_metal_device_rel(struct ggml_backend_metal_device_conte
+@@ -288,6 +290,7 @@ enum ggml_metal_kernel_type {
      GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ1_M_F32,
      GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_NL_F32,
      GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_XS_F32,
@@ -746,7 +746,7 @@ index ab46f6e3..d8e05a21 100644
      GGML_METAL_KERNEL_TYPE_MUL_MM_F32_F32,
      GGML_METAL_KERNEL_TYPE_MUL_MM_F16_F32,
      GGML_METAL_KERNEL_TYPE_MUL_MM_BF16_F32,
-@@ -310,6 +313,7 @@ static void ggml_backend_metal_device_rel(struct ggml_backend_metal_device_conte
+@@ -310,6 +313,7 @@ enum ggml_metal_kernel_type {
      GGML_METAL_KERNEL_TYPE_MUL_MM_IQ1_M_F32,
      GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_NL_F32,
      GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_XS_F32,
@@ -754,7 +754,7 @@ index ab46f6e3..d8e05a21 100644
      GGML_METAL_KERNEL_TYPE_MUL_MM_ID_MAP0_F16,
      GGML_METAL_KERNEL_TYPE_MUL_MM_ID_MAP1_F32,
      GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F32_F16,
-@@ -334,6 +338,7 @@ static void ggml_backend_metal_device_rel(struct ggml_backend_metal_device_conte
+@@ -334,6 +338,7 @@ enum ggml_metal_kernel_type {
      GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ1_M_F16,
      GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_NL_F16,
      GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_XS_F16,
@@ -762,7 +762,7 @@ index ab46f6e3..d8e05a21 100644
      GGML_METAL_KERNEL_TYPE_ROPE_NORM_F32,
      GGML_METAL_KERNEL_TYPE_ROPE_NORM_F16,
      GGML_METAL_KERNEL_TYPE_ROPE_MULTI_F32,
-@@ -934,7 +939,7 @@ @implementation GGMLMetalClass
+@@ -934,7 +939,7 @@ static id<MTLLibrary> ggml_metal_load_library(id<MTLDevice> device, bool use_bfl
  
              MTLCompileOptions * options = [MTLCompileOptions new];
              options.preprocessorMacros = prep;
@@ -771,7 +771,7 @@ index ab46f6e3..d8e05a21 100644
              //[options setFastMathEnabled:false];
  
              metal_library = [device newLibraryWithSource:src options:options error:&error];
-@@ -1157,6 +1162,7 @@ @implementation GGMLMetalClass
+@@ -1157,6 +1162,7 @@ static struct ggml_backend_metal_context * ggml_metal_init(ggml_backend_dev_t de
          GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_0_F32,                 mul_mv_q5_0_f32,                 has_simdgroup_reduction);
          GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q5_1_F32,                 mul_mv_q5_1_f32,                 has_simdgroup_reduction);
          GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_Q8_0_F32,                 mul_mv_q8_0_f32,                 has_simdgroup_reduction);
@@ -779,7 +779,7 @@ index ab46f6e3..d8e05a21 100644
          GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_EXT_F16_F32_R1_2,         mul_mv_ext_f16_f32_r1_2,         has_simdgroup_reduction);
          GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_EXT_F16_F32_R1_3,         mul_mv_ext_f16_f32_r1_3,         has_simdgroup_reduction);
          GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_EXT_F16_F32_R1_4,         mul_mv_ext_f16_f32_r1_4,         has_simdgroup_reduction);
-@@ -1236,6 +1242,7 @@ @implementation GGMLMetalClass
+@@ -1236,6 +1242,7 @@ static struct ggml_backend_metal_context * ggml_metal_init(ggml_backend_dev_t de
          GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ1_M_F32,             mul_mv_id_iq1_m_f32,             has_simdgroup_reduction);
          GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_NL_F32,            mul_mv_id_iq4_nl_f32,            has_simdgroup_reduction);
          GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MV_ID_IQ4_XS_F32,            mul_mv_id_iq4_xs_f32,            has_simdgroup_reduction);
@@ -787,7 +787,7 @@ index ab46f6e3..d8e05a21 100644
          GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_F32_F32,                  mul_mm_f32_f32,                  has_simdgroup_mm);
          GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_F16_F32,                  mul_mm_f16_f32,                  has_simdgroup_mm);
          GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_BF16_F32,                 mul_mm_bf16_f32,                 has_simdgroup_mm && use_bfloat);
-@@ -1258,6 +1265,7 @@ @implementation GGMLMetalClass
+@@ -1258,6 +1265,7 @@ static struct ggml_backend_metal_context * ggml_metal_init(ggml_backend_dev_t de
          GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ1_M_F32,                mul_mm_iq1_m_f32,                has_simdgroup_mm);
          GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_NL_F32,               mul_mm_iq4_nl_f32,               has_simdgroup_mm);
          GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_IQ4_XS_F32,               mul_mm_iq4_xs_f32,               has_simdgroup_mm);
@@ -795,7 +795,7 @@ index ab46f6e3..d8e05a21 100644
          GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_MAP0_F16,              mul_mm_id_map0_f16,              has_simdgroup_mm);
          GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_MAP1_F32,              mul_mm_id_map1_f32,              has_simdgroup_mm);
          GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_F32_F16,               mul_mm_id_f32_f16,               has_simdgroup_mm);
-@@ -1282,6 +1290,7 @@ @implementation GGMLMetalClass
+@@ -1282,6 +1290,7 @@ static struct ggml_backend_metal_context * ggml_metal_init(ggml_backend_dev_t de
          GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ1_M_F16,             mul_mm_id_iq1_m_f16,             has_simdgroup_mm);
          GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_NL_F16,            mul_mm_id_iq4_nl_f16,            has_simdgroup_mm);
          GGML_METAL_ADD_KERNEL(GGML_METAL_KERNEL_TYPE_MUL_MM_ID_IQ4_XS_F16,            mul_mm_id_iq4_xs_f16,            has_simdgroup_mm);

llama/patches/0024-cuda-disable-graph-compat-check-for-OP_ADD.patch

+From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
+From: Michael Yang <git@mxy.ng>
+Date: Thu, 31 Jul 2025 12:31:58 -0700
+Subject: [PATCH] cuda: disable graph compat check for OP_ADD
+
+---
+ ggml/src/ggml-cuda/ggml-cuda.cu | 14 --------------
+ 1 file changed, 14 deletions(-)
+
+diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
+index bb19b06e..080e7467 100644
+--- a/ggml/src/ggml-cuda/ggml-cuda.cu
++++ b/ggml/src/ggml-cuda/ggml-cuda.cu
+@@ -2509,20 +2509,6 @@ static bool check_node_graph_compatibility_and_refresh_copy_ops(ggml_backend_cud
+ #endif
+         }
+ 
+-        // workarounds to exclude Gemma3n's `project_per_layer_input` operation from the batch-size heuristic, specific to ollama's implementation of gemma3n
+-        // number of layers is different for per_layer_proj between gemma3n:2b and gemma3n:4b, which is why we don't check that value here
+-        if (node->op == GGML_OP_ADD && node->src[1] && node->src[1]->ne[1] > 1 && !(node->ne[0] == 256
+-                                                                                    && node->ne[2] == 1
+-                                                                                    && node->ne[3] == 1
+-                                                                                    && node->src[0] ? std::string(node->src[0]->name).find(gemma3n_node_name) != std::string::npos : false
+-                                                                                    && node->src[1] ? node->src[1]->name == gemma3n_per_layer_proj_src1_name : false)) {
+-            // Generally, changes in batch size or context size can cause changes to the grid size of some kernels.
+-            use_cuda_graph = false;
+-#ifndef NDEBUG
+-            GGML_LOG_INFO("%s: disabling CUDA graphs due to batch size > 1 [%s] [%ld %ld %ld %ld]\n", __func__, node->name, node->ne[0], node->ne[1], node->ne[2], node->ne[3]);
+-#endif
+-        }
+-
+         if (node->op == GGML_OP_CPY) {
+ 
+             // Store the pointers which are updated for each token, such that these can be sent

ml/backend/ggml/ggml/src/ggml-cuda/ggml-cuda.cu

@@ -2509,20 +2509,6 @@ static bool check_node_graph_compatibility_and_refresh_copy_ops(ggml_backend_cud
 #endif
         }
 
-        // workarounds to exclude Gemma3n's `project_per_layer_input` operation from the batch-size heuristic, specific to ollama's implementation of gemma3n
-        // number of layers is different for per_layer_proj between gemma3n:2b and gemma3n:4b, which is why we don't check that value here
-        if (node->op == GGML_OP_ADD && node->src[1] && node->src[1]->ne[1] > 1 && !(node->ne[0] == 256
-                                                                                    && node->ne[2] == 1
-                                                                                    && node->ne[3] == 1
-                                                                                    && node->src[0] ? std::string(node->src[0]->name).find(gemma3n_node_name) != std::string::npos : false
-                                                                                    && node->src[1] ? node->src[1]->name == gemma3n_per_layer_proj_src1_name : false)) {
-            // Generally, changes in batch size or context size can cause changes to the grid size of some kernels.
-            use_cuda_graph = false;
-#ifndef NDEBUG
-            GGML_LOG_INFO("%s: disabling CUDA graphs due to batch size > 1 [%s] [%ld %ld %ld %ld]\n", __func__, node->name, node->ne[0], node->ne[1], node->ne[2], node->ne[3]);
-#endif
-        }
-
         if (node->op == GGML_OP_CPY) {
 
             // Store the pointers which are updated for each token, such that these can be sent

model/models/gptoss/model.go

@@ -146,7 +146,6 @@ func (attn *AttentionBlock) Forward(ctx ml.Context, hiddenStates, positions ml.T
 
 	query = query.Permute(ctx, 0, 2, 1, 3)
 	key = key.Permute(ctx, 0, 2, 1, 3)
-	value = value.Permute(ctx, 1, 2, 0, 3).Contiguous(ctx)
 
 	scores := key.MulmatFullPrec(ctx, query)
 	scores = scores.Scale(ctx, 1./math.Sqrt(float64(opts.headDim())))
@@ -257,10 +256,10 @@ func New(c fs.Config) (model.Model, error) {
 	}
 
 	m.Cache = kvcache.NewWrapperCache(
-		kvcache.NewSWACache(int32(c.Uint("attention.sliding_window")), m.Shift),
+		kvcache.NewSWAMemCache(int32(c.Uint("attention.sliding_window")), 4096, m.Shift),
 		kvcache.NewCausalCache(m.Shift),
 	)
-	m.Cache.SetConfig(ml.CacheConfig{})
+	m.Cache.SetConfig(ml.CacheConfig{CachePadding: 32, PermutedV: true})
 	return &m, nil
 }