Skip to content

GitLab

Unverified Commit 27f1fde4 authored by Daniel Hiltgen's avatar Daniel Hiltgen Committed by GitHub
Browse files

discovery: only retry AMD GPUs (#12894) 

* discovery: only retry AMD GPUs

CUDA and Vulkan don't crash on unsupported devices, so retry isn't necessary.
This also refactors the code to shift the Library specific logic into the ml
package.

* review comments
parent 220e133f
Hide whitespace changes
Inline Side-by-side
Showing with 96 additions and 137 deletions
discover/runner.go
View file @ 27f1fde4
...@@ -27,7 +27,6 @@ var ( ...@@ -27,7 +27,6 @@ var (
deviceMu sync.Mutex deviceMu sync.Mutex
devices []ml.DeviceInfo devices []ml.DeviceInfo
libDirs map[string]struct{} libDirs map[string]struct{}
rocmDir string
exe string exe string
bootstrapped bool bootstrapped bool
) )
...@@ -61,14 +60,6 @@ func GPUDevices(ctx context.Context, runners []ml.FilteredRunnerDiscovery) []ml. ...@@ -61,14 +60,6 @@ func GPUDevices(ctx context.Context, runners []ml.FilteredRunnerDiscovery) []ml.
libDirs[filepath.Dir(file)] = struct{}{} libDirs[filepath.Dir(file)] = struct{}{}
} }
// Our current packaging model places ggml-hip in the main directory
// but keeps rocm in an isolated directory. We have to add it to
// the [LD_LIBRARY_]PATH so ggml-hip will load properly
rocmDir = filepath.Join(ml.LibOllamaPath, "rocm")
if _, err := os.Stat(rocmDir); err != nil {
rocmDir = ""
}
if len(libDirs) == 0 { if len(libDirs) == 0 {
libDirs[""] = struct{}{} libDirs[""] = struct{}{}
} }
...@@ -82,9 +73,20 @@ func GPUDevices(ctx context.Context, runners []ml.FilteredRunnerDiscovery) []ml. ...@@ -82,9 +73,20 @@ func GPUDevices(ctx context.Context, runners []ml.FilteredRunnerDiscovery) []ml.
// are enumerated, but not actually supported. // are enumerated, but not actually supported.
// We run this in serial to avoid potentially initializing a GPU multiple // We run this in serial to avoid potentially initializing a GPU multiple
// times concurrently leading to memory contention // times concurrently leading to memory contention
// TODO refactor so we group the lib dirs and do serial per version, but parallel for different libs
for dir := range libDirs { for dir := range libDirs {
// Typically bootstrapping takes < 1s, but on some systems, with devices
// in low power/idle mode, initialization can take multiple seconds. We
// set a longer timeout just for bootstrap discovery to reduce the chance
// of giving up too quickly
bootstrapTimeout := 30 * time.Second bootstrapTimeout := 30 * time.Second
if runtime.GOOS == "windows" {
// On Windows with Defender enabled, AV scanning of the DLLs
// takes place sequentially and this can significantly increase
// the time it takes too do the initial discovery pass.
// Subsequent loads will be faster as the scan results are
// cached
bootstrapTimeout = 90 * time.Second
}
var dirs []string var dirs []string
if dir != "" { if dir != "" {
if requested != "" && filepath.Base(dir) != requested { if requested != "" && filepath.Base(dir) != requested {
...@@ -93,21 +95,11 @@ func GPUDevices(ctx context.Context, runners []ml.FilteredRunnerDiscovery) []ml. ...@@ -93,21 +95,11 @@ func GPUDevices(ctx context.Context, runners []ml.FilteredRunnerDiscovery) []ml.
} else if jetpack != "" && filepath.Base(dir) != "cuda_"+jetpack { } else if jetpack != "" && filepath.Base(dir) != "cuda_"+jetpack {
continue continue
} }
}
if dir == "" {
dirs = []string{ml.LibOllamaPath}
} else {
dirs = []string{ml.LibOllamaPath, dir} dirs = []string{ml.LibOllamaPath, dir}
} else {
dirs = []string{ml.LibOllamaPath}
} }
// ROCm can take a long time on some systems, so give it more time before giving up
if dir != "" && strings.Contains(filepath.Base(dir), "rocm") {
bootstrapTimeout = 60 * time.Second
}
// Typically bootstrapping takes < 1s, but on some systems, with devices
// in low power/idle mode, initialization can take multiple seconds. We
// set a long timeout just for bootstrap discovery to reduce the chance
// of giving up too quickly
ctx1stPass, cancel := context.WithTimeout(ctx, bootstrapTimeout) ctx1stPass, cancel := context.WithTimeout(ctx, bootstrapTimeout)
defer cancel() defer cancel()
...@@ -117,6 +109,7 @@ func GPUDevices(ctx context.Context, runners []ml.FilteredRunnerDiscovery) []ml. ...@@ -117,6 +109,7 @@ func GPUDevices(ctx context.Context, runners []ml.FilteredRunnerDiscovery) []ml.
// In the second pass, we more deeply initialize the GPUs to weed out devices that // In the second pass, we more deeply initialize the GPUs to weed out devices that
// aren't supported by a given library. We run this phase in parallel to speed up discovery. // aren't supported by a given library. We run this phase in parallel to speed up discovery.
// Only devices that need verification are included in this pass
slog.Debug("evluating which if any devices to filter out", "initial_count", len(devices)) slog.Debug("evluating which if any devices to filter out", "initial_count", len(devices))
ctx2ndPass, cancel := context.WithTimeout(ctx, 30*time.Second) ctx2ndPass, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel() defer cancel()
...@@ -125,35 +118,16 @@ func GPUDevices(ctx context.Context, runners []ml.FilteredRunnerDiscovery) []ml. ...@@ -125,35 +118,16 @@ func GPUDevices(ctx context.Context, runners []ml.FilteredRunnerDiscovery) []ml.
supportedMu := sync.Mutex{} supportedMu := sync.Mutex{}
supported := make(map[string]map[string]map[string]int) // [Library][libDir][ID] = pre-deletion devices index supported := make(map[string]map[string]map[string]int) // [Library][libDir][ID] = pre-deletion devices index
for i := range devices { for i := range devices {
libDir := devices[i].LibraryPath[len(devices[i].LibraryPath)-1] if !devices[i].NeedsInitValidation() {
if devices[i].Library == "Metal" {
continue continue
} }
slog.Debug("verifying GPU is supported", "library", libDir, "description", devices[i].Description, "compute", devices[i].Compute(), "id", devices[i].ID, "pci_id", devices[i].PCIID) libDir := devices[i].LibraryPath[len(devices[i].LibraryPath)-1]
slog.Debug("verifying device is supported", "library", libDir, "description", devices[i].Description, "compute", devices[i].Compute(), "id", devices[i].ID, "pci_id", devices[i].PCIID)
wg.Add(1) wg.Add(1)
go func(i int) { go func(i int) {
defer wg.Done() defer wg.Done()
var envVar string extraEnvs := ml.GetVisibleDevicesEnv(devices[i : i+1])
id := devices[i].ID devices[i].AddInitValidation(extraEnvs)
if devices[i].Library == "ROCm" {
if runtime.GOOS != "linux" {
envVar = "HIP_VISIBLE_DEVICES"
} else {
envVar = "ROCR_VISIBLE_DEVICES"
}
} else if devices[i].Library == "CUDA" {
envVar = "CUDA_VISIBLE_DEVICES"
} else if devices[i].Library == "Vulkan" {
id = devices[i].FilteredID
envVar = "GGML_VK_VISIBLE_DEVICES"
} else {
slog.Error("Unknown Library:" + devices[i].Library)
}
extraEnvs := map[string]string{
"GGML_CUDA_INIT": "1", // force deep initialization to trigger crash on unsupported GPUs
envVar: id, // Filter to just this one GPU
}
if len(bootstrapDevices(ctx2ndPass, devices[i].LibraryPath, extraEnvs)) == 0 { if len(bootstrapDevices(ctx2ndPass, devices[i].LibraryPath, extraEnvs)) == 0 {
slog.Debug("filtering device which didn't fully initialize", slog.Debug("filtering device which didn't fully initialize",
"id", devices[i].ID, "id", devices[i].ID,
...@@ -178,26 +152,28 @@ func GPUDevices(ctx context.Context, runners []ml.FilteredRunnerDiscovery) []ml. ...@@ -178,26 +152,28 @@ func GPUDevices(ctx context.Context, runners []ml.FilteredRunnerDiscovery) []ml.
wg.Wait() wg.Wait()
logutil.Trace("supported GPU library combinations before filtering", "supported", supported) logutil.Trace("supported GPU library combinations before filtering", "supported", supported)
filterOutVulkanThatAreSupportedByOtherGPU(needsDelete)
// Mark for deletion any overlaps - favoring the library version that can cover all GPUs if possible // Mark for deletion any overlaps - favoring the library version that can cover all GPUs if possible
filterOverlapByLibrary(supported, needsDelete) filterOverlapByLibrary(supported, needsDelete)
// TODO if we ever support multiple ROCm library versions this algorithm will need to be adjusted to keep the rocmID numeric value correct // Any Libraries that utilize numeric IDs need adjusting based on any possible filtering taking place
rocmID := 0 postFilteredID := map[string]int{}
for i := 0; i < len(needsDelete); i++ { for i := 0; i < len(needsDelete); i++ {
if needsDelete[i] { if needsDelete[i] {
logutil.Trace("removing unsupported or overlapping GPU combination", "libDir", devices[i].LibraryPath[len(devices[i].LibraryPath)-1], "description", devices[i].Description, "compute", devices[i].Compute(), "pci_id", devices[i].PCIID) logutil.Trace("removing unsupported or overlapping GPU combination", "libDir", devices[i].LibraryPath[len(devices[i].LibraryPath)-1], "description", devices[i].Description, "compute", devices[i].Compute(), "pci_id", devices[i].PCIID)
devices = append(devices[:i], devices[i+1:]...) devices = append(devices[:i], devices[i+1:]...)
needsDelete = append(needsDelete[:i], needsDelete[i+1:]...) needsDelete = append(needsDelete[:i], needsDelete[i+1:]...)
i-- i--
} else if devices[i].Library == "ROCm" { } else {
if _, ok := postFilteredID[devices[i].Library]; !ok {
postFilteredID[devices[i].Library] = 0
}
if _, err := strconv.Atoi(devices[i].ID); err == nil { if _, err := strconv.Atoi(devices[i].ID); err == nil {
// Replace the numeric ID with the post-filtered IDs // Replace the numeric ID with the post-filtered IDs
devices[i].FilteredID = devices[i].ID slog.Debug("adjusting filtering IDs", "FilterID", devices[i].ID, "new_ID", strconv.Itoa(postFilteredID[devices[i].Library]))
devices[i].ID = strconv.Itoa(rocmID) devices[i].FilterID = devices[i].ID
devices[i].ID = strconv.Itoa(postFilteredID[devices[i].Library])
} }
rocmID++ postFilteredID[devices[i].Library]++
} }
} }
...@@ -214,7 +190,7 @@ func GPUDevices(ctx context.Context, runners []ml.FilteredRunnerDiscovery) []ml. ...@@ -214,7 +190,7 @@ func GPUDevices(ctx context.Context, runners []ml.FilteredRunnerDiscovery) []ml.
case ml.DuplicateDevice: case ml.DuplicateDevice:
// Different library, choose based on priority // Different library, choose based on priority
var droppedDevice ml.DeviceInfo var droppedDevice ml.DeviceInfo
if devices[i].Library == "CUDA" || devices[i].Library == "ROCm" { if devices[i].PreferredLibrary(devices[j]) {
droppedDevice = devices[j] droppedDevice = devices[j]
} else { } else {
droppedDevice = devices[i] droppedDevice = devices[i]
...@@ -363,38 +339,6 @@ func GPUDevices(ctx context.Context, runners []ml.FilteredRunnerDiscovery) []ml. ...@@ -363,38 +339,6 @@ func GPUDevices(ctx context.Context, runners []ml.FilteredRunnerDiscovery) []ml.
return devices return devices
} }
func filterOutVulkanThatAreSupportedByOtherGPU(needsDelete []bool) {
// Filter out Vulkan devices that share a PCI ID with a non-Vulkan device that is not marked for deletion
for i := range devices {
if devices[i].Library != "Vulkan" || needsDelete[i] {
continue
}
if devices[i].PCIID == "" {
continue
}
for j := range devices {
if i == j {
continue
}
if devices[j].PCIID == "" {
continue
}
if devices[j].PCIID == devices[i].PCIID && devices[j].Library != "Vulkan" && !needsDelete[j] {
needsDelete[i] = true
slog.Debug("filtering device with duplicate PCI ID",
"id", devices[i].ID,
"library", devices[i].Library,
"libdir", devices[i].LibraryPath[len(devices[i].LibraryPath)-1],
"pci_id", devices[i].PCIID,
"kept_id", devices[j].ID,
"kept_library", devices[j].Library,
)
break
}
}
}
}
func filterOverlapByLibrary(supported map[string]map[string]map[string]int, needsDelete []bool) { func filterOverlapByLibrary(supported map[string]map[string]map[string]int, needsDelete []bool) {
// For multi-GPU systems, use the newest version that supports all the GPUs // For multi-GPU systems, use the newest version that supports all the GPUs
for _, byLibDirs := range supported { for _, byLibDirs := range supported {
......
discover/types.go
View file @ 27f1fde4
...@@ -41,7 +41,7 @@ func LogDetails(devices []ml.DeviceInfo) { ...@@ -41,7 +41,7 @@ func LogDetails(devices []ml.DeviceInfo) {
} }
slog.Info("inference compute", slog.Info("inference compute",
"id", dev.ID, "id", dev.ID,
"filtered_id", dev.FilteredID, "filter_id", dev.FilterID,
"library", dev.Library, "library", dev.Library,
"compute", dev.Compute(), "compute", dev.Compute(),
"name", dev.Name, "name", dev.Name,
......
llama/patches/0026-GPU-discovery-enhancements.patch
View file @ 27f1fde4
...@@ -14,24 +14,24 @@ Vulkan PCI and Memory ...@@ -14,24 +14,24 @@ Vulkan PCI and Memory
fix vulkan PCI ID and ID handling fix vulkan PCI ID and ID handling
--- ---
ggml/include/ggml-backend.h | 8 + ggml/include/ggml-backend.h | 6 +
ggml/src/CMakeLists.txt | 2 + ggml/src/CMakeLists.txt | 2 +
ggml/src/ggml-cuda/ggml-cuda.cu | 65 ++++ ggml/src/ggml-cuda/ggml-cuda.cu | 65 ++++
ggml/src/ggml-cuda/vendors/hip.h | 3 + ggml/src/ggml-cuda/vendors/hip.h | 3 +
ggml/src/ggml-impl.h | 8 + ggml/src/ggml-impl.h | 8 +
ggml/src/ggml-metal/ggml-metal.cpp | 2 + ggml/src/ggml-metal/ggml-metal.cpp | 2 +
ggml/src/ggml-vulkan/ggml-vulkan.cpp | 212 +++++++++++-- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 209 +++++++++++--
ggml/src/mem_hip.cpp | 452 +++++++++++++++++++++++++++ ggml/src/mem_hip.cpp | 452 +++++++++++++++++++++++++++
ggml/src/mem_nvml.cpp | 209 +++++++++++++ ggml/src/mem_nvml.cpp | 209 +++++++++++++
9 files changed, 931 insertions(+), 30 deletions(-) 9 files changed, 926 insertions(+), 30 deletions(-)
create mode 100644 ggml/src/mem_hip.cpp create mode 100644 ggml/src/mem_hip.cpp
create mode 100644 ggml/src/mem_nvml.cpp create mode 100644 ggml/src/mem_nvml.cpp
diff --git a/ggml/include/ggml-backend.h b/ggml/include/ggml-backend.h diff --git a/ggml/include/ggml-backend.h b/ggml/include/ggml-backend.h
index ba181d09d..809835243 100644 index 69223c488..6510e0cba 100644
--- a/ggml/include/ggml-backend.h --- a/ggml/include/ggml-backend.h
+++ b/ggml/include/ggml-backend.h +++ b/ggml/include/ggml-backend.h
@@ -169,6 +169,14 @@ extern "C" { @@ -169,6 +169,12 @@ extern "C" {
const char * device_id; const char * device_id;
// device capabilities // device capabilities
struct ggml_backend_dev_caps caps; struct ggml_backend_dev_caps caps;
...@@ -41,8 +41,6 @@ index ba181d09d..809835243 100644 ...@@ -41,8 +41,6 @@ index ba181d09d..809835243 100644
+ int compute_minor; + int compute_minor;
+ int integrated; + int integrated;
+ const char *library; + const char *library;
+ // number with which the devices are accessed (Vulkan)
+ const char *numeric_id;
}; };
GGML_API const char * ggml_backend_dev_name(ggml_backend_dev_t device); GGML_API const char * ggml_backend_dev_name(ggml_backend_dev_t device);
...@@ -60,7 +58,7 @@ index 0609c6503..aefe43bdd 100644 ...@@ -60,7 +58,7 @@ index 0609c6503..aefe43bdd 100644
target_include_directories(ggml-base PRIVATE .) target_include_directories(ggml-base PRIVATE .)
diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu
index 87c6c34a4..b075a18be 100644 index 5787e8cd5..d232bf828 100644
--- a/ggml/src/ggml-cuda/ggml-cuda.cu --- a/ggml/src/ggml-cuda/ggml-cuda.cu
+++ b/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu
@@ -261,6 +261,16 @@ static ggml_cuda_device_info ggml_cuda_init() { @@ -261,6 +261,16 @@ static ggml_cuda_device_info ggml_cuda_init() {
...@@ -92,7 +90,7 @@ index 87c6c34a4..b075a18be 100644 ...@@ -92,7 +90,7 @@ index 87c6c34a4..b075a18be 100644
GGML_LOG_INFO(" Device %d: %s, compute capability %d.%d, VMM: %s, ID: %s\n", GGML_LOG_INFO(" Device %d: %s, compute capability %d.%d, VMM: %s, ID: %s\n",
id, prop.name, prop.major, prop.minor, device_vmm ? "yes" : "no", id, prop.name, prop.major, prop.minor, device_vmm ? "yes" : "no",
ggml_cuda_parse_uuid(prop, id).c_str()); ggml_cuda_parse_uuid(prop, id).c_str());
@@ -3484,6 +3499,11 @@ struct ggml_backend_cuda_device_context { @@ -3476,6 +3491,11 @@ struct ggml_backend_cuda_device_context {
std::string description; std::string description;
std::string pci_bus_id; std::string pci_bus_id;
std::string id; std::string id;
...@@ -104,7 +102,7 @@ index 87c6c34a4..b075a18be 100644 ...@@ -104,7 +102,7 @@ index 87c6c34a4..b075a18be 100644
}; };
static const char * ggml_backend_cuda_device_get_name(ggml_backend_dev_t dev) { static const char * ggml_backend_cuda_device_get_name(ggml_backend_dev_t dev) {
@@ -3504,6 +3524,28 @@ static const char * ggml_backend_cuda_device_get_id(ggml_backend_dev_t dev) { @@ -3496,6 +3516,28 @@ static const char * ggml_backend_cuda_device_get_id(ggml_backend_dev_t dev) {
static void ggml_backend_cuda_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) { static void ggml_backend_cuda_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) {
ggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *)dev->context; ggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *)dev->context;
ggml_cuda_set_device(ctx->device); ggml_cuda_set_device(ctx->device);
...@@ -133,7 +131,7 @@ index 87c6c34a4..b075a18be 100644 ...@@ -133,7 +131,7 @@ index 87c6c34a4..b075a18be 100644
CUDA_CHECK(cudaMemGetInfo(free, total)); CUDA_CHECK(cudaMemGetInfo(free, total));
} }
@@ -3512,6 +3554,7 @@ static enum ggml_backend_dev_type ggml_backend_cuda_device_get_type(ggml_backend @@ -3504,6 +3546,7 @@ static enum ggml_backend_dev_type ggml_backend_cuda_device_get_type(ggml_backend
return GGML_BACKEND_DEVICE_TYPE_GPU; return GGML_BACKEND_DEVICE_TYPE_GPU;
} }
...@@ -141,7 +139,7 @@ index 87c6c34a4..b075a18be 100644 ...@@ -141,7 +139,7 @@ index 87c6c34a4..b075a18be 100644
static void ggml_backend_cuda_device_get_props(ggml_backend_dev_t dev, ggml_backend_dev_props * props) { static void ggml_backend_cuda_device_get_props(ggml_backend_dev_t dev, ggml_backend_dev_props * props) {
ggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *)dev->context; ggml_backend_cuda_device_context * ctx = (ggml_backend_cuda_device_context *)dev->context;
@@ -3525,6 +3568,19 @@ static void ggml_backend_cuda_device_get_props(ggml_backend_dev_t dev, ggml_back @@ -3517,6 +3560,19 @@ static void ggml_backend_cuda_device_get_props(ggml_backend_dev_t dev, ggml_back
// If you need the memory data, call ggml_backend_dev_memory() explicitly. // If you need the memory data, call ggml_backend_dev_memory() explicitly.
props->memory_total = props->memory_free = 0; props->memory_total = props->memory_free = 0;
...@@ -161,7 +159,7 @@ index 87c6c34a4..b075a18be 100644 ...@@ -161,7 +159,7 @@ index 87c6c34a4..b075a18be 100644
bool host_buffer = getenv("GGML_CUDA_NO_PINNED") == nullptr; bool host_buffer = getenv("GGML_CUDA_NO_PINNED") == nullptr;
#ifdef GGML_CUDA_NO_PEER_COPY #ifdef GGML_CUDA_NO_PEER_COPY
bool events = false; bool events = false;
@@ -4087,6 +4143,7 @@ ggml_backend_reg_t ggml_backend_cuda_reg() { @@ -4079,6 +4135,7 @@ ggml_backend_reg_t ggml_backend_cuda_reg() {
std::lock_guard<std::mutex> lock(mutex); std::lock_guard<std::mutex> lock(mutex);
if (!initialized) { if (!initialized) {
ggml_backend_cuda_reg_context * ctx = new ggml_backend_cuda_reg_context; ggml_backend_cuda_reg_context * ctx = new ggml_backend_cuda_reg_context;
...@@ -169,7 +167,7 @@ index 87c6c34a4..b075a18be 100644 ...@@ -169,7 +167,7 @@ index 87c6c34a4..b075a18be 100644
for (int i = 0; i < ggml_cuda_info().device_count; i++) { for (int i = 0; i < ggml_cuda_info().device_count; i++) {
ggml_backend_cuda_device_context * dev_ctx = new ggml_backend_cuda_device_context; ggml_backend_cuda_device_context * dev_ctx = new ggml_backend_cuda_device_context;
@@ -4102,6 +4159,14 @@ ggml_backend_reg_t ggml_backend_cuda_reg() { @@ -4094,6 +4151,14 @@ ggml_backend_reg_t ggml_backend_cuda_reg() {
snprintf(pci_bus_id, sizeof(pci_bus_id), "%04x:%02x:%02x.0", prop.pciDomainID, prop.pciBusID, prop.pciDeviceID); snprintf(pci_bus_id, sizeof(pci_bus_id), "%04x:%02x:%02x.0", prop.pciDomainID, prop.pciBusID, prop.pciDeviceID);
dev_ctx->pci_bus_id = pci_bus_id; dev_ctx->pci_bus_id = pci_bus_id;
...@@ -225,10 +223,10 @@ index d0fb3bcca..b63edd0c1 100644 ...@@ -225,10 +223,10 @@ index d0fb3bcca..b63edd0c1 100644
} }
#endif #endif
diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal.cpp diff --git a/ggml/src/ggml-metal/ggml-metal.cpp b/ggml/src/ggml-metal/ggml-metal.cpp
index f2ff9f322..f356e4a0a 100644 index 05ff6a5a6..032dee76d 100644
--- a/ggml/src/ggml-metal/ggml-metal.cpp --- a/ggml/src/ggml-metal/ggml-metal.cpp
+++ b/ggml/src/ggml-metal/ggml-metal.cpp +++ b/ggml/src/ggml-metal/ggml-metal.cpp
@@ -535,6 +535,7 @@ static enum ggml_backend_dev_type ggml_backend_metal_device_get_type(ggml_backen @@ -537,6 +537,7 @@ static enum ggml_backend_dev_type ggml_backend_metal_device_get_type(ggml_backen
GGML_UNUSED(dev); GGML_UNUSED(dev);
} }
...@@ -236,7 +234,7 @@ index f2ff9f322..f356e4a0a 100644 ...@@ -236,7 +234,7 @@ index f2ff9f322..f356e4a0a 100644
static void ggml_backend_metal_device_get_props(ggml_backend_dev_t dev, ggml_backend_dev_props * props) { static void ggml_backend_metal_device_get_props(ggml_backend_dev_t dev, ggml_backend_dev_props * props) {
props->name = ggml_backend_metal_device_get_name(dev); props->name = ggml_backend_metal_device_get_name(dev);
props->description = ggml_backend_metal_device_get_description(dev); props->description = ggml_backend_metal_device_get_description(dev);
@@ -543,6 +544,7 @@ static void ggml_backend_metal_device_get_props(ggml_backend_dev_t dev, ggml_bac @@ -545,6 +546,7 @@ static void ggml_backend_metal_device_get_props(ggml_backend_dev_t dev, ggml_bac
ggml_backend_metal_device_get_memory(dev, &props->memory_free, &props->memory_total); ggml_backend_metal_device_get_memory(dev, &props->memory_free, &props->memory_total);
...@@ -245,7 +243,7 @@ index f2ff9f322..f356e4a0a 100644 ...@@ -245,7 +243,7 @@ index f2ff9f322..f356e4a0a 100644
/* .async = */ true, /* .async = */ true,
/* .host_buffer = */ false, /* .host_buffer = */ false,
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
index ed83236f4..0bbcecd01 100644 index bd3ece516..7cfb14a54 100644
--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp
+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
@@ -231,6 +231,7 @@ class vk_memory_logger; @@ -231,6 +231,7 @@ class vk_memory_logger;
...@@ -286,7 +284,7 @@ index ed83236f4..0bbcecd01 100644 ...@@ -286,7 +284,7 @@ index ed83236f4..0bbcecd01 100644
// backend interface // backend interface
#define UNUSED GGML_UNUSED #define UNUSED GGML_UNUSED
@@ -12391,31 +12415,103 @@ void ggml_backend_vk_get_device_description(int device, char * description, size @@ -12392,31 +12416,102 @@ void ggml_backend_vk_get_device_description(int device, char * description, size
ggml_vk_get_device_description(dev_idx, description, description_size); ggml_vk_get_device_description(dev_idx, description, description_size);
} }
...@@ -309,7 +307,6 @@ index ed83236f4..0bbcecd01 100644 ...@@ -309,7 +307,6 @@ index ed83236f4..0bbcecd01 100644
+ std::string pci_id; + std::string pci_id;
+ std::string id; + std::string id;
+ std::string uuid; + std::string uuid;
+ std::string numeric_id;
+ int major; + int major;
+ int minor; + int minor;
+ int driver_major; + int driver_major;
...@@ -407,7 +404,7 @@ index ed83236f4..0bbcecd01 100644 ...@@ -407,7 +404,7 @@ index ed83236f4..0bbcecd01 100644
break; break;
} }
} }
@@ -12448,8 +12544,13 @@ static std::string ggml_backend_vk_get_device_pci_id(int device_idx) { @@ -12449,8 +12544,13 @@ static std::string ggml_backend_vk_get_device_pci_id(int device_idx) {
} }
} }
...@@ -422,7 +419,7 @@ index ed83236f4..0bbcecd01 100644 ...@@ -422,7 +419,7 @@ index ed83236f4..0bbcecd01 100644
} }
vk::PhysicalDeviceProperties2 props = {}; vk::PhysicalDeviceProperties2 props = {};
@@ -12466,19 +12567,24 @@ static std::string ggml_backend_vk_get_device_pci_id(int device_idx) { @@ -12467,19 +12567,24 @@ static std::string ggml_backend_vk_get_device_pci_id(int device_idx) {
char pci_bus_id[16] = {}; char pci_bus_id[16] = {};
snprintf(pci_bus_id, sizeof(pci_bus_id), "%04x:%02x:%02x.%x", pci_domain, pci_bus, pci_device, pci_function); snprintf(pci_bus_id, sizeof(pci_bus_id), "%04x:%02x:%02x.%x", pci_domain, pci_bus, pci_device, pci_function);
...@@ -456,7 +453,7 @@ index ed83236f4..0bbcecd01 100644 ...@@ -456,7 +453,7 @@ index ed83236f4..0bbcecd01 100644
static const char * ggml_backend_vk_device_get_name(ggml_backend_dev_t dev) { static const char * ggml_backend_vk_device_get_name(ggml_backend_dev_t dev) {
ggml_backend_vk_device_context * ctx = (ggml_backend_vk_device_context *)dev->context; ggml_backend_vk_device_context * ctx = (ggml_backend_vk_device_context *)dev->context;
@@ -12490,9 +12596,14 @@ static const char * ggml_backend_vk_device_get_description(ggml_backend_dev_t de @@ -12491,9 +12596,14 @@ static const char * ggml_backend_vk_device_get_description(ggml_backend_dev_t de
return ctx->description.c_str(); return ctx->description.c_str();
} }
...@@ -472,7 +469,7 @@ index ed83236f4..0bbcecd01 100644 ...@@ -472,7 +469,7 @@ index ed83236f4..0bbcecd01 100644
} }
static ggml_backend_buffer_type_t ggml_backend_vk_device_get_buffer_type(ggml_backend_dev_t dev) { static ggml_backend_buffer_type_t ggml_backend_vk_device_get_buffer_type(ggml_backend_dev_t dev) {
@@ -12516,8 +12627,9 @@ static void ggml_backend_vk_device_get_props(ggml_backend_dev_t dev, struct ggml @@ -12517,8 +12627,9 @@ static void ggml_backend_vk_device_get_props(ggml_backend_dev_t dev, struct ggml
props->name = ggml_backend_vk_device_get_name(dev); props->name = ggml_backend_vk_device_get_name(dev);
props->description = ggml_backend_vk_device_get_description(dev); props->description = ggml_backend_vk_device_get_description(dev);
...@@ -483,7 +480,7 @@ index ed83236f4..0bbcecd01 100644 ...@@ -483,7 +480,7 @@ index ed83236f4..0bbcecd01 100644
ggml_backend_vk_device_get_memory(dev, &props->memory_free, &props->memory_total); ggml_backend_vk_device_get_memory(dev, &props->memory_free, &props->memory_total);
props->caps = { props->caps = {
/* .async = */ false, /* .async = */ false,
@@ -12525,6 +12637,14 @@ static void ggml_backend_vk_device_get_props(ggml_backend_dev_t dev, struct ggml @@ -12526,6 +12637,13 @@ static void ggml_backend_vk_device_get_props(ggml_backend_dev_t dev, struct ggml
/* .buffer_from_host_ptr = */ false, /* .buffer_from_host_ptr = */ false,
/* .events = */ false, /* .events = */ false,
}; };
...@@ -494,11 +491,10 @@ index ed83236f4..0bbcecd01 100644 ...@@ -494,11 +491,10 @@ index ed83236f4..0bbcecd01 100644
+ props->driver_minor = ctx->driver_minor; + props->driver_minor = ctx->driver_minor;
+ props->integrated = ctx->is_integrated_gpu; + props->integrated = ctx->is_integrated_gpu;
+ props->library = GGML_VK_NAME; + props->library = GGML_VK_NAME;
+ props->numeric_id = ctx->numeric_id.c_str();
} }
static ggml_backend_t ggml_backend_vk_device_init(ggml_backend_dev_t dev, const char * params) { static ggml_backend_t ggml_backend_vk_device_init(ggml_backend_dev_t dev, const char * params) {
@@ -12953,6 +13073,8 @@ static ggml_backend_dev_t ggml_backend_vk_reg_get_device(ggml_backend_reg_t reg, @@ -12954,6 +13072,8 @@ static ggml_backend_dev_t ggml_backend_vk_reg_get_device(ggml_backend_reg_t reg,
static std::mutex mutex; static std::mutex mutex;
std::lock_guard<std::mutex> lock(mutex); std::lock_guard<std::mutex> lock(mutex);
if (!initialized) { if (!initialized) {
...@@ -507,7 +503,7 @@ index ed83236f4..0bbcecd01 100644 ...@@ -507,7 +503,7 @@ index ed83236f4..0bbcecd01 100644
for (int i = 0; i < ggml_backend_vk_get_device_count(); i++) { for (int i = 0; i < ggml_backend_vk_get_device_count(); i++) {
ggml_backend_vk_device_context * ctx = new ggml_backend_vk_device_context; ggml_backend_vk_device_context * ctx = new ggml_backend_vk_device_context;
char desc[256]; char desc[256];
@@ -12961,12 +13083,42 @@ static ggml_backend_dev_t ggml_backend_vk_reg_get_device(ggml_backend_reg_t reg, @@ -12962,12 +13082,41 @@ static ggml_backend_dev_t ggml_backend_vk_reg_get_device(ggml_backend_reg_t reg,
ctx->name = GGML_VK_NAME + std::to_string(i); ctx->name = GGML_VK_NAME + std::to_string(i);
ctx->description = desc; ctx->description = desc;
ctx->is_integrated_gpu = ggml_backend_vk_get_device_type(i) == vk::PhysicalDeviceType::eIntegratedGpu; ctx->is_integrated_gpu = ggml_backend_vk_get_device_type(i) == vk::PhysicalDeviceType::eIntegratedGpu;
...@@ -547,7 +543,6 @@ index ed83236f4..0bbcecd01 100644 ...@@ -547,7 +543,6 @@ index ed83236f4..0bbcecd01 100644
+ // TODO regex parse driver_props.driverInfo for a X.Y or X.Y.Z version string + // TODO regex parse driver_props.driverInfo for a X.Y or X.Y.Z version string
+ ctx->driver_major = 0; + ctx->driver_major = 0;
+ ctx->driver_minor = 0; + ctx->driver_minor = 0;
+ ctx->numeric_id = std::to_string(i);
} }
initialized = true; initialized = true;
} }
......
llama/patches/0031-Add-memory-detection-using-DXGI-PDH.patch → llama/patches/0030-Add-memory-detection-using-DXGI-PDH.patch
View file @ 27f1fde4
...@@ -38,7 +38,7 @@ index b63edd0c1..81cad8cf3 100644 ...@@ -38,7 +38,7 @@ index b63edd0c1..81cad8cf3 100644
#ifdef __cplusplus #ifdef __cplusplus
} }
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
index cc68e7968..27d6574da 100644 index 7cfb14a54..a1c46d0b3 100644
--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp
+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
@@ -73,6 +73,7 @@ DispatchLoaderDynamic & ggml_vk_default_dispatcher(); @@ -73,6 +73,7 @@ DispatchLoaderDynamic & ggml_vk_default_dispatcher();
...@@ -54,10 +54,10 @@ index cc68e7968..27d6574da 100644 ...@@ -54,10 +54,10 @@ index cc68e7968..27d6574da 100644
std::string id; std::string id;
std::string uuid; std::string uuid;
+ std::string luid; + std::string luid;
std::string numeric_id;
int major; int major;
int minor; int minor;
@@ -12449,8 +12451,22 @@ void ggml_backend_vk_get_device_memory(ggml_backend_vk_device_context *ctx, size int driver_major;
@@ -12448,8 +12450,22 @@ void ggml_backend_vk_get_device_memory(ggml_backend_vk_device_context *ctx, size
vk::PhysicalDeviceMemoryProperties memprops = vkdev.getMemoryProperties(); vk::PhysicalDeviceMemoryProperties memprops = vkdev.getMemoryProperties();
vk::PhysicalDeviceProperties2 props2; vk::PhysicalDeviceProperties2 props2;
vkdev.getProperties2(&props2); vkdev.getProperties2(&props2);
...@@ -81,7 +81,7 @@ index cc68e7968..27d6574da 100644 ...@@ -81,7 +81,7 @@ index cc68e7968..27d6574da 100644
{ {
// Use vendor specific management libraries for best VRAM reporting if available // Use vendor specific management libraries for best VRAM reporting if available
switch (props2.properties.vendorID) { switch (props2.properties.vendorID) {
@@ -12478,8 +12494,8 @@ void ggml_backend_vk_get_device_memory(ggml_backend_vk_device_context *ctx, size @@ -12477,8 +12493,8 @@ void ggml_backend_vk_get_device_memory(ggml_backend_vk_device_context *ctx, size
break; break;
} }
} }
...@@ -91,7 +91,7 @@ index cc68e7968..27d6574da 100644 ...@@ -91,7 +91,7 @@ index cc68e7968..27d6574da 100644
*total = 0; *total = 0;
*free = 0; *free = 0;
vk::PhysicalDeviceMemoryBudgetPropertiesEXT mem_budget_props; vk::PhysicalDeviceMemoryBudgetPropertiesEXT mem_budget_props;
@@ -13091,7 +13107,6 @@ static ggml_backend_dev_t ggml_backend_vk_reg_get_device(ggml_backend_reg_t reg, @@ -13089,7 +13105,6 @@ static ggml_backend_dev_t ggml_backend_vk_reg_get_device(ggml_backend_reg_t reg,
/* .reg = */ reg, /* .reg = */ reg,
/* .context = */ ctx, /* .context = */ ctx,
}); });
...@@ -99,7 +99,7 @@ index cc68e7968..27d6574da 100644 ...@@ -99,7 +99,7 @@ index cc68e7968..27d6574da 100644
// Gather additional information about the device // Gather additional information about the device
int dev_idx = vk_instance.device_indices[i]; int dev_idx = vk_instance.device_indices[i];
vk::PhysicalDeviceProperties props1; vk::PhysicalDeviceProperties props1;
@@ -13114,6 +13129,14 @@ static ggml_backend_dev_t ggml_backend_vk_reg_get_device(ggml_backend_reg_t reg, @@ -13112,6 +13127,14 @@ static ggml_backend_dev_t ggml_backend_vk_reg_get_device(ggml_backend_reg_t reg,
} }
} }
ctx->uuid = oss.str(); ctx->uuid = oss.str();
......
llama/patches/0032-interleave-multi-rope.patch → llama/patches/0031-interleave-multi-rope.patch
View file @ 27f1fde4
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: Michael Yang <git@mxy.ng> From: Michael Yang <git@mxy.ng>
Date: Web, 16 Oct 2025 20:37:19 -0700 Date: Thu, 16 Oct 2025 20:37:19 -0700
Subject: [PATCH] interleave multi rope Subject: [PATCH] interleave multi rope
since ollama doesn't use mrope for anything else, change it to mean the since ollama doesn't use mrope for anything else, change it to mean the
...@@ -85,7 +85,7 @@ index 375a0c7fd..9866c96b4 100644 ...@@ -85,7 +85,7 @@ index 375a0c7fd..9866c96b4 100644
// end of mrope // end of mrope
diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/rope_multi.comp b/ggml/src/ggml-vulkan/vulkan-shaders/rope_multi.comp diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/rope_multi.comp b/ggml/src/ggml-vulkan/vulkan-shaders/rope_multi.comp
index 111286b49..6fc2b42f8 100644 index 111286b49..633dc20ff 100644
--- a/ggml/src/ggml-vulkan/vulkan-shaders/rope_multi.comp --- a/ggml/src/ggml-vulkan/vulkan-shaders/rope_multi.comp
+++ b/ggml/src/ggml-vulkan/vulkan-shaders/rope_multi.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/rope_multi.comp
@@ -31,19 +31,13 @@ void main() { @@ -31,19 +31,13 @@ void main() {
......
ml/backend/ggml/ggml.go
View file @ 27f1fde4
...@@ -730,10 +730,6 @@ func (b *Backend) BackendDevices() []ml.DeviceInfo { ...@@ -730,10 +730,6 @@ func (b *Backend) BackendDevices() []ml.DeviceInfo {
info.PCIID = C.GoString(props.device_id) info.PCIID = C.GoString(props.device_id)
} }
info.LibraryPath = ggml.LibPaths() info.LibraryPath = ggml.LibPaths()
if props.numeric_id != nil {
info.FilteredID = C.GoString(props.numeric_id)
}
C.ggml_backend_dev_memory(dev, &props.memory_free, &props.memory_total) C.ggml_backend_dev_memory(dev, &props.memory_free, &props.memory_total)
info.TotalMemory = (uint64)(props.memory_total) info.TotalMemory = (uint64)(props.memory_total)
info.FreeMemory = (uint64)(props.memory_free) info.FreeMemory = (uint64)(props.memory_free)
......
ml/backend/ggml/ggml/include/ggml-backend.h
View file @ 27f1fde4
...@@ -175,8 +175,6 @@ extern "C" { ...@@ -175,8 +175,6 @@ extern "C" {
int compute_minor; int compute_minor;
int integrated; int integrated;
const char *library; const char *library;
// number with which the devices are accessed (Vulkan)
const char *numeric_id;
}; };
GGML_API const char * ggml_backend_dev_name(ggml_backend_dev_t device); GGML_API const char * ggml_backend_dev_name(ggml_backend_dev_t device);
......
ml/backend/ggml/ggml/src/ggml-vulkan/ggml-vulkan.cpp
View file @ 27f1fde4
...@@ -12435,7 +12435,6 @@ struct ggml_backend_vk_device_context { ...@@ -12435,7 +12435,6 @@ struct ggml_backend_vk_device_context {
std::string id; std::string id;
std::string uuid; std::string uuid;
std::string luid; std::string luid;
std::string numeric_id;
int major; int major;
int minor; int minor;
int driver_major; int driver_major;
...@@ -12661,7 +12660,6 @@ static void ggml_backend_vk_device_get_props(ggml_backend_dev_t dev, struct ggml ...@@ -12661,7 +12660,6 @@ static void ggml_backend_vk_device_get_props(ggml_backend_dev_t dev, struct ggml
props->driver_minor = ctx->driver_minor; props->driver_minor = ctx->driver_minor;
props->integrated = ctx->is_integrated_gpu; props->integrated = ctx->is_integrated_gpu;
props->library = GGML_VK_NAME; props->library = GGML_VK_NAME;
props->numeric_id = ctx->numeric_id.c_str();
} }
   
static ggml_backend_t ggml_backend_vk_device_init(ggml_backend_dev_t dev, const char * params) { static ggml_backend_t ggml_backend_vk_device_init(ggml_backend_dev_t dev, const char * params) {
...@@ -13142,7 +13140,6 @@ static ggml_backend_dev_t ggml_backend_vk_reg_get_device(ggml_backend_reg_t reg, ...@@ -13142,7 +13140,6 @@ static ggml_backend_dev_t ggml_backend_vk_reg_get_device(ggml_backend_reg_t reg,
// TODO regex parse driver_props.driverInfo for a X.Y or X.Y.Z version string // TODO regex parse driver_props.driverInfo for a X.Y or X.Y.Z version string
ctx->driver_major = 0; ctx->driver_major = 0;
ctx->driver_minor = 0; ctx->driver_minor = 0;
ctx->numeric_id = std::to_string(i);
} }
initialized = true; initialized = true;
} }
......
ml/device.go
View file @ 27f1fde4
...@@ -257,7 +257,7 @@ type DeviceInfo struct { ...@@ -257,7 +257,7 @@ type DeviceInfo struct {
// FilterID is populated with the unfiltered device ID if a numeric ID is used // FilterID is populated with the unfiltered device ID if a numeric ID is used
// so the device can be included. // so the device can be included.
FilteredID string `json:"filtered_id,omitempty"` FilterID string `json:"filter_id,omitempty"`
// Integrated is set true for integrated GPUs, false for Discrete GPUs // Integrated is set true for integrated GPUs, false for Discrete GPUs
Integrated bool `json:"integration,omitempty"` Integrated bool `json:"integration,omitempty"`
...@@ -455,6 +455,35 @@ func GetVisibleDevicesEnv(l []DeviceInfo) map[string]string { ...@@ -455,6 +455,35 @@ func GetVisibleDevicesEnv(l []DeviceInfo) map[string]string {
return env return env
} }
// NeedsInitValidation returns true if the device in question has the potential
// to crash at inference time and requires deeper validation before we include
// it in the supported devices list.
func (d DeviceInfo) NeedsInitValidation() bool {
// At this time the only library we know needs a 2nd pass is ROCm since
// rocblas will crash on unsupported devices. We want to find those crashes
// during bootstrap discovery so we can eliminate those GPUs before the user
// tries to run inference on them
return d.Library == "ROCm"
}
// Set the init validation environment variable
func (d DeviceInfo) AddInitValidation(env map[string]string) {
env["GGML_CUDA_INIT"] = "1" // force deep initialization to trigger crash on unsupported GPUs
}
// PreferredLibrary returns true if this library is preferred over the other input
// library
// Used to filter out Vulkan in favor of CUDA or ROCm
func (d DeviceInfo) PreferredLibrary(other DeviceInfo) bool {
// TODO in the future if we find Vulkan is better than ROCm on some devices
// that implementation can live here.
if d.Library == "CUDA" || d.Library == "ROCm" {
return true
}
return false
}
func (d DeviceInfo) updateVisibleDevicesEnv(env map[string]string) { func (d DeviceInfo) updateVisibleDevicesEnv(env map[string]string) {
var envVar string var envVar string
switch d.Library { switch d.Library {
...@@ -472,8 +501,8 @@ func (d DeviceInfo) updateVisibleDevicesEnv(env map[string]string) { ...@@ -472,8 +501,8 @@ func (d DeviceInfo) updateVisibleDevicesEnv(env map[string]string) {
if existing { if existing {
v = v + "," v = v + ","
} }
if d.FilteredID != "" { if d.FilterID != "" {
v = v + d.FilteredID v = v + d.FilterID
} else { } else {
v = v + d.ID v = v + d.ID
} }
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment