diff --git a/CHANGELOG.md b/CHANGELOG.md index a913548..bafa8e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Change log +## [v1.3.4] 2026-09-07 + +### Bug Fix + +- Fix scale layout handling for `groupsize=-1` in RTN fallback. This fallback is used when an MoE expert receives no routed calibration tokens, because GPTQ cannot compute activation-based statistics for that expert. The fix keeps the fallback result compatible with GPTQ's per-channel dequantization path. +- Fix MPS loading of large sharded checkpoints by loading weights on CPU before moving the model to MPS. +- Reject chunked calibration (`CalibrationConfig(batch_size=...)`) on MPS, where it is not supported. + +### Documentation + +- Clarify the confirmed Qwen3.6 `save_format="full_wrapper"` workflows, including vLLM serving and the current GGUF export workflow. + ## [v1.3.3] 2026-09-03 ### Enhancement diff --git a/docs/user-guide/basic-usage.md b/docs/user-guide/basic-usage.md index c91e9f8..62aba0c 100644 --- a/docs/user-guide/basic-usage.md +++ b/docs/user-guide/basic-usage.md @@ -155,6 +155,18 @@ runner.save_dequantized_model("./output/dequantized") runner.save_quantized_model("./output/quantized") ``` +!!! note "Qwen3.6 save format" + Qwen3.6 is quantized through its text-model layout. For confirmed + downstream workflows that require the full Hugging Face wrapper layout, + including vLLM serving and the current GGUF export workflow, save Qwen3.6 + checkpoints with `save_format="full_wrapper"`: + + `runner.save_quantized_model("./output/qwen36_quantized", save_format="full_wrapper")` + + The option is specific to Qwen3.6 and raises `RuntimeError` for other + models. Leave `save_format` at its default (`"auto"`) for other + architectures. + !!! note "vLLM serving is method-specific" `save_quantized_model()` produces a model loadable by the OneComp loader for any quantizer that supports saving (see the table below). vLLM serving, however, is only diff --git a/docs/user-guide/examples.md b/docs/user-guide/examples.md index cc7aee8..d4d238f 100644 --- a/docs/user-guide/examples.md +++ b/docs/user-guide/examples.md @@ -361,6 +361,12 @@ runner.save_quantized_model("./output/my_quantized_model") runner.save_dequantized_model("./output/my_dequantized_model") ``` +!!! note "Qwen3.6 save format" + For confirmed downstream workflows that require the full Hugging Face + wrapper layout, including vLLM serving and the current GGUF export workflow, + save Qwen3.6 models with `save_format="full_wrapper"`. See + [Basic Usage](basic-usage.md#step-5-save-the-model). + ### Load a saved quantized model ```python diff --git a/docs/user-guide/mps.md b/docs/user-guide/mps.md index 2282e5c..e9bd96e 100644 --- a/docs/user-guide/mps.md +++ b/docs/user-guide/mps.md @@ -23,6 +23,7 @@ quantization steps intentionally run on CPU for performance. See | DBF, RTN, JointQ, and other quantizers | No | — | | AutoBit DBF fallback | No | — | | Multi-GPU quantization | No | — | +| Chunked calibration | No | - | ## Installation @@ -127,6 +128,8 @@ with an NVIDIA GPU. See the [vLLM Inference guide](vllm-inference.md). - Only **GPTQ** quantizers are allowed (or **AutoBitQuantizer** whose candidates are all GPTQ). - **AutoBit DBF fallback** is rejected when the target bitwidth would require DBF-only assignment. - **`multi_gpu=True`** is not supported. +- **Chunked calibration** (`CalibrationConfig(batch_size=...)`) is not supported. + Leave `batch_size` unset when quantizing on MPS. To avoid DBF fallback on MPS, either set an explicit `wbits` within the GPTQ candidate range or ensure VRAM estimation yields a bitwidth that does not trigger DBF-only paths. diff --git a/docs/user-guide/vllm-inference.md b/docs/user-guide/vllm-inference.md index 094e6a1..c1e08b8 100644 --- a/docs/user-guide/vllm-inference.md +++ b/docs/user-guide/vllm-inference.md @@ -126,24 +126,15 @@ runner.run() runner.save_quantized_model("./Llama-3.1-8B-Instruct-gptq-4bit") ``` -!!! note "Qwen3.6: use `save_format=\"full_wrapper\"`" - Qwen3.6 quantizes as a text-only checkpoint, whose native Hugging Face - layout (`model.layers.*`) does not match what vLLM's composite - `Qwen3_5ForConditionalGeneration` loader expects (`model.language_model.layers.*`). - Pass `save_format="full_wrapper"` to `save_quantized_model()` to remap the - checkpoint for vLLM serving: - - ```python - runner.save_quantized_model("./Qwen3.6-gptq-4bit-vllm", save_format="full_wrapper") - ``` - - This option is specific to Qwen3.6 and will raise `RuntimeError` for any - other model. Leave `save_format` at its default (`"auto"`) for everything - else, including other VLMs. +!!! note "Qwen3.6 save format" + Qwen3.6 checkpoints used for vLLM serving should be saved with + `save_quantized_model(..., save_format="full_wrapper")`. See + [Basic Usage](basic-usage.md#step-5-save-the-model) for the shared + Qwen3.6 save-format guidance, including other confirmed workflows. For MoE variants (e.g. Qwen3.6-A3B), `full_wrapper` also drops each expert's trivial `g_idx` buffer, since vLLM's GPTQ `FusedMoE` kernel has no `g_idx` - parameter and an unmapped one would crash weight loading — see the `desc_act`/ + parameter and an unmapped one would crash weight loading -- see the `desc_act`/ `actorder` warning above for when this isn't safe to drop. ### 2. Serve with vLLM diff --git a/onecomp/__version__.py b/onecomp/__version__.py index 022c0c7..17be442 100644 --- a/onecomp/__version__.py +++ b/onecomp/__version__.py @@ -6,4 +6,4 @@ """ -__version__ = "1.3.3" +__version__ = "1.3.4" diff --git a/onecomp/model_config.py b/onecomp/model_config.py index 520eabb..2e1061c 100644 --- a/onecomp/model_config.py +++ b/onecomp/model_config.py @@ -11,7 +11,7 @@ import torch from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer -from .utils.device import get_default_device +from .utils.device import get_default_device, is_mps_device from .utils.dtype import needs_bfloat16 try: @@ -95,9 +95,14 @@ def load_model(self, device_map=None): If ``None`` (default), ``self.device`` is used. """ effective_device = device_map if device_map is not None else self.device + if effective_device == "auto": + target_device = get_default_device() + else: + target_device = effective_device + load_device = "cpu" if is_mps_device(target_device) else effective_device kwargs = dict( dtype=self.dtype if self.dtype == "auto" else getattr(torch, self.dtype), - device_map=effective_device, + device_map=load_device, ) config = self.load_config() @@ -135,6 +140,8 @@ def load_model(self, device_map=None): raise self.logger.info("AutoModelForCausalLM failed; trying AutoModelForImageTextToText.") model = _AutoVLM.from_pretrained(self.get_model_id_or_path(), **kwargs) + if is_mps_device(target_device): + model = model.to(target_device) model.eval() self.logger.info("Model loaded with dtype=%s", next(model.parameters()).dtype) return model diff --git a/onecomp/qep/_quantize_with_qep_arch.py b/onecomp/qep/_quantize_with_qep_arch.py index 1bd3402..811de0c 100644 --- a/onecomp/qep/_quantize_with_qep_arch.py +++ b/onecomp/qep/_quantize_with_qep_arch.py @@ -298,8 +298,15 @@ def _rtn_fallback_result(module: nn.Module, quantizer: Quantizer, name: str) -> result_dict = run_rtn(module, wbits=wbits, groupsize=groupsize, sym=quantizer.sym) - # RTN's raw scale/zero are (out_features, num_groups); GPTQResult - # expects (num_groups, out_features). + scales = result_dict["scale"] + qzeros = result_dict["zero"] + + if groupsize != -1: + # RTN's raw scale/zero are (out_features, num_groups); GPTQResult + # expects (num_groups, out_features). + scales = scales.T + qzeros = qzeros.T + return GPTQResult( dequantized_weight=result_dict["dequantized_weight"], wbits=wbits, @@ -307,8 +314,8 @@ def _rtn_fallback_result(module: nn.Module, quantizer: Quantizer, name: str) -> actorder=False, sym=quantizer.sym, qweight=result_dict["quantized_weight"], - scales=result_dict["scale"].T, - qzeros=result_dict["zero"].T, + scales=scales, + qzeros=qzeros, perm=None, ) diff --git a/onecomp/runner.py b/onecomp/runner.py index 6076714..75b32bc 100644 --- a/onecomp/runner.py +++ b/onecomp/runner.py @@ -379,6 +379,11 @@ def check(self): if is_mps_device(device): if self.multi_gpu: raise ValueError("multi_gpu is not supported on MPS device.") + if batch_size is not None: + raise ValueError( + "MPS quantization does not support calibration_config.batch_size. " + "Remove batch_size from CalibrationConfig and run without chunked calibration." + ) all_quantizers = self.quantizers if self.quantizers is not None else [self.quantizer] for i, q in enumerate(all_quantizers): label = f"quantizers[{i}]" if self.quantizers else "quantizer" diff --git a/tests/onecomp/test_model_config.py b/tests/onecomp/test_model_config.py new file mode 100644 index 0000000..8cf5271 --- /dev/null +++ b/tests/onecomp/test_model_config.py @@ -0,0 +1,114 @@ +"""Unit tests for ModelConfig MPS model-loading behavior. + +Copyright 2025-2026 Fujitsu Ltd. + +Author: Yuhki Yano +""" + +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest +import torch + +from onecomp.model_config import ModelConfig + + +def _mock_loaded_model(): + model = MagicMock(name="model") + model.parameters.side_effect = lambda: iter([torch.empty(1, dtype=torch.float16)]) + model.to.return_value = model + return model + + +def test_load_model_loads_mps_model_on_cpu_then_moves_to_mps(monkeypatch): + """MPS loads sharded checkpoints on CPU before moving the model to MPS.""" + model = _mock_loaded_model() + config = SimpleNamespace(quantization_config=None) + + load_config = MagicMock(return_value=config) + load_model = MagicMock(return_value=model) + + monkeypatch.setattr("onecomp.model_config.AutoConfig.from_pretrained", load_config) + monkeypatch.setattr( + "onecomp.model_config.AutoModelForCausalLM.from_pretrained", + load_model, + ) + + model_config = ModelConfig(model_id="test/model", device="mps") + loaded = model_config.load_model() + + assert loaded is model + load_model.assert_called_once_with( + "test/model", + dtype=torch.float16, + device_map="cpu", + ) + model.to.assert_called_once_with("mps") + model.eval.assert_called_once_with() + + +@pytest.mark.parametrize("device", ["cpu", "cuda:0"]) +def test_load_model_preserves_explicit_no_mps_device_map(monkeypatch, device): + """Explicit non-MPS device maps are passed through unchanged.""" + model = _mock_loaded_model() + config = SimpleNamespace(quantization_config=None) + + monkeypatch.setattr( + "onecomp.model_config.AutoConfig.from_pretrained", + MagicMock(return_value=config), + ) + load_model = MagicMock(return_value=model) + monkeypatch.setattr( + "onecomp.model_config.AutoModelForCausalLM.from_pretrained", + load_model, + ) + + model_config = ModelConfig(model_id="test/model", device=device) + model_config.load_model() + + load_model.assert_called_once_with( + "test/model", + dtype=torch.float16, + device_map=device, + ) + model.to.assert_not_called() + model.eval.assert_called_once_with() + + +@pytest.mark.parametrize( + "resolved_device, expected_device_map", + [ + (torch.device("cpu"), "auto"), + (torch.device("mps"), "cpu"), + ], +) +def test_load_model_auto_preserves_device_map(monkeypatch, resolved_device, expected_device_map): + """device='auto' keeps Transformers auto placement unless MPS is selected.""" + model = _mock_loaded_model() + config = SimpleNamespace(quantization_config=None) + load_model = MagicMock(return_value=model) + monkeypatch.setattr( + "onecomp.model_config.AutoConfig.from_pretrained", + MagicMock(return_value=config), + ) + monkeypatch.setattr( + "onecomp.model_config.AutoModelForCausalLM.from_pretrained", + load_model, + ) + monkeypatch.setattr( + "onecomp.model_config.get_default_device", + MagicMock(return_value=resolved_device), + ) + ModelConfig(model_id="test/model", device="auto").load_model() + + load_model.assert_called_once_with( + "test/model", + dtype=torch.float16, + device_map=expected_device_map, + ) + if resolved_device.type == "mps": + model.to.assert_called_once_with(resolved_device) + else: + model.to.assert_not_called() + model.eval.assert_called_once_with() diff --git a/tests/onecomp/test_qep_expert_recovery_integration.py b/tests/onecomp/test_qep_expert_recovery_integration.py index fae173c..276f8a5 100644 --- a/tests/onecomp/test_qep_expert_recovery_integration.py +++ b/tests/onecomp/test_qep_expert_recovery_integration.py @@ -12,6 +12,7 @@ import logging +import pytest import torch import torch.nn as nn import torch.nn.functional as F @@ -24,7 +25,7 @@ # in_features of every expert Linear must be divisible by the 4-bit pack factor # (32 // 4 == 8): gate/up consume HIDDEN, down consumes INTERMEDIATE. HIDDEN = 8 -INTERMEDIATE = 8 +INTERMEDIATE = 16 NUM_EXPERTS = 3 SEQ_LEN = 3 @@ -230,7 +231,8 @@ def test_expert_never_selected_falls_back_to_rtn(monkeypatch, caplog): assert all(quantizer.results[n].actorder is False for n in names) -def test_rtn_fallback_weight_is_actually_applied_to_the_module(monkeypatch, caplog): +@pytest.mark.parametrize("groupsize", [-1, 2]) +def test_rtn_fallback_weight_is_actually_applied_to_the_module(monkeypatch, caplog, groupsize): """The RTN-fallback ``GPTQResult`` must actually flow into the module's live weight, not just sit unused in ``quantizer.results``. A @@ -248,7 +250,7 @@ def test_rtn_fallback_weight_is_actually_applied_to_the_module(monkeypatch, capl "onecomp.qep._quantize_with_qep_arch.prepare_calibration_dataset", _fake_prepare_calibration_dataset, ) - quantizer = GPTQ(wbits=4, groupsize=2, sym=True, include_layer_keywords=["experts"]) + quantizer = GPTQ(wbits=4, groupsize=groupsize, sym=True, include_layer_keywords=["experts"]) qep_config = QEPConfig(device="cpu", percdamp=0.01, perccorr=0.5) caplog.set_level(logging.INFO, logger="onecomp.qep._quantize_with_qep_arch") diff --git a/tests/onecomp/test_rtn_fallback_result.py b/tests/onecomp/test_rtn_fallback_result.py index 4142149..2b4277a 100644 --- a/tests/onecomp/test_rtn_fallback_result.py +++ b/tests/onecomp/test_rtn_fallback_result.py @@ -8,6 +8,7 @@ Copyright 2025-2026 Fujitsu Ltd. """ +import pytest import torch import torch.nn as nn @@ -84,18 +85,26 @@ def test_dequantized_weight_matches_shape(self): result = _rtn_fallback_result(module, quantizer, "mlp.experts.0.down_proj") assert result.dequantized_weight.shape == module.weight.data.shape - def test_compute_dequantized_weight_roundtrip(self): + @pytest.mark.parametrize("groupsize", [-1, 32]) + def test_compute_dequantized_weight_roundtrip(self, groupsize): """The packaged qweight/scales/qzeros must reconstruct a weight consistent with the shapes GPTQResult.compute_dequantized_weight expects (this is what create_inference_layer / export relies on downstream). """ module = _linear(in_features=32, out_features=16) - quantizer = GPTQ(wbits=4, groupsize=16, sym=True) + quantizer = GPTQ(wbits=4, groupsize=groupsize, sym=True) result = _rtn_fallback_result(module, quantizer, "mlp.experts.0.down_proj") reconstructed = result.compute_dequantized_weight() + expected = result.dequantized_weight.to(dtype=reconstructed.dtype) assert reconstructed.shape == module.weight.data.shape + torch.testing.assert_close( + reconstructed, + expected, + rtol=0, + atol=2e-4, + ) class TestResolveGptqForRtnFallback: diff --git a/tests/onecomp/test_runner_check.py b/tests/onecomp/test_runner_check.py index 2bf4e48..96319f3 100644 --- a/tests/onecomp/test_runner_check.py +++ b/tests/onecomp/test_runner_check.py @@ -139,3 +139,16 @@ def test_autobit_low_target_on_mps_with_auto_dbf_disabled_passes(self): calibration_config=CalibrationConfig(max_length=128, num_calibration_samples=8), ) runner.check() + + def test_batch_size_is_not_supported_on_mps(self): + """MPS rejects chunked calibration requested via batch_size.""" + runner = Runner( + model_config=self._mps_model_config(), + quantizer=GPTQ(wbits=4, groupsize=128), + calibration_config=CalibrationConfig(batch_size=2), + ) + + with pytest.raises( + ValueError, match=r"MPS quantization does not support calibration_config\.batch_size" + ): + runner.check()