From 2ccee3035e480ff31f36c0aa9432b27b4f896a91 Mon Sep 17 00:00:00 2001 From: ALeXssNdR Date: Mon, 31 Aug 2026 17:16:44 +0300 Subject: [PATCH] moe-cache: pick a GPU device explicitly instead of the first non-meta device The expert-cache init selected the first non-meta entry of the model device list. On hosts where the list starts with the CPU device the cache silently disabled itself with 'no GPU device'. Select the first GPU-typed device instead, and fall back to enumerating every backend registry (including dynamically loaded ones such as libggml-cuda) when the model list has none. Measured on RTX 5080 Laptop 16GB, Ornith-1.5-35B-A3B (qwen35moe) Q4_K_M, -ngl 99 -ncmoe 99 -fa 1, 176 slots (9.8 GiB pack): baseline 56.8 t/s -> 80-92 t/s (+41-62%), single slot count that fits VRAM. --- src/llama-model.cpp | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 2dfe7eefc9fb..7970b20c6b0f 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -1705,7 +1705,24 @@ void llama_model_base::init_moe_expert_cache() { ggml_backend_dev_t dev = nullptr; for (const auto & d : devices) { - if (!d.is_meta) { dev = d.dev; break; } + if (!d.is_meta && ggml_backend_dev_type(d.dev) == GGML_BACKEND_DEVICE_TYPE_GPU) { dev = d.dev; break; } + } + if (dev == nullptr) { + // fallback: enumerate every backend registry, including dynamically + // loaded ones (e.g. libggml-cuda) absent from the default registry + for (size_t r = 0; r < ggml_backend_reg_count() && dev == nullptr; ++r) { + auto * reg = ggml_backend_reg_get(r); + for (size_t i = 0; i < ggml_backend_reg_dev_count(reg); ++i) { + auto * d = ggml_backend_reg_dev_get(reg, i); + if (ggml_backend_dev_type(d) == GGML_BACKEND_DEVICE_TYPE_GPU) { dev = d; break; } + } + } + if (dev == nullptr) { + for (size_t i = 0; i < ggml_backend_dev_count(); ++i) { + auto * d = ggml_backend_dev_get(i); + if (ggml_backend_dev_type(d) == GGML_BACKEND_DEVICE_TYPE_GPU) { dev = d; break; } + } + } } if (dev == nullptr || ggml_backend_dev_type(dev) != GGML_BACKEND_DEVICE_TYPE_GPU) { LLAMA_LOG_WARN("%s: no GPU device - expert cache disabled\n", __func__);