diff --git a/configs/nvidia-master.yaml b/configs/nvidia-master.yaml index 526fbec480..9eeeffd505 100644 --- a/configs/nvidia-master.yaml +++ b/configs/nvidia-master.yaml @@ -9081,24 +9081,19 @@ glm5.2-fp8-h200-dynamo-sglang-agentic-mtp-2p2d: # GLM-5.2's built-in nextn head (num-steps 3, # eagle-topk 1, 4 draft tokens = 3 speculative tokens), with acceptance pinned # to the golden AL 2.99 (golden_al_distribution/glm5.2_mtp.yaml, thinking_on, -# K=3) through SGLANG_SIMULATE_ACC_*. Image is bumped to v0.5.16-cu130: it is -# the first release that reads SGLANG_SIMULATE_ACC_TOKEN_MODE. +# K=3) through SGLANG_SIMULATE_ACC_*. The pinned nightly includes FlashInfer +# 0.6.18's BF16 TRTLLM MoE allocation fix for small-batch Blackwell execution +# in GLM-5.2's unquantized EAGLE draft head. # -# Cookbook low-latency arm (TP8, fp8 KV, cutedsl bf16 GEMM) only, on the AgentX -# MTP concurrency grid [1, 4, 8, 12, 16]: steps of at least 2 because -# single-step sampling cannot separate configurations by more than run-to-run -# noise on the agentic corpus, and a hard stop at conc 16. A DEP throughput arm -# is deliberately not included — its measured frontier peak is conc -# 48, well above that cap, and at conc <= 16 attention-DP leaves 2 sessions per -# rank and is strictly dominated by TP8, so it would spend a GPU job per point -# to re-measure a worse curve. This recipe offloads to host DRAM: without -# HiCache the working set outgrows HBM past conc -# 8 and the radix hit rate collapses to <0.1. +# Higher concurrencies sample the throughput curve nearer 95 P90 tokens/s/user +# while preserving the cookbook low-latency arm (TP8, fp8 KV, cutedsl bf16 GEMM). +# HiCache host-DRAM offload is required because the working set outgrows HBM +# past conc 8 without it and the radix hit rate collapses to <0.1. glm5.2-fp4-b300-sglang-agentic-mtp: - image: lmsysorg/sglang:v0.5.16-cu130 + image: lmsysorg/sglang:nightly-dev-cu13-20260901-07c8f729 model: nvidia/GLM-5.2-NVFP4 model-prefix: glm5.2 - runner: cluster:b300-nv + runner: cluster:b300-dsxe precision: fp4 framework: sglang multinode: false @@ -9106,7 +9101,7 @@ glm5.2-fp4-b300-sglang-agentic-mtp: agentic-coding: - dram-utilization: 0.80 search-space: - - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 4, 8, 12, 16] } + - { tp: 8, spec-decoding: mtp, kv-offloading: dram, kv-offload-backend: { name: hicache }, conc-list: [1, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48] } # First GLM-5.2 AgentX recipe on B200, shipped spec-decode-only per the AgentX # policy that agentic arms enable speculative decoding rather than running a @@ -9118,7 +9113,7 @@ glm5.2-fp4-b300-sglang-agentic-mtp: # nightly includes FlashInfer 0.6.18's BF16 TRTLLM MoE allocation fix, which is # required by GLM-5.2's unquantized EAGLE draft head at CUDA-graph capture. # -# Same single arm and concurrency grid as glm5.2-fp4-b300-sglang-agentic-mtp: +# Same single arm as glm5.2-fp4-b300-sglang-agentic-mtp: # cookbook low-latency TP8 with HiCache host-DRAM offload, conc # [1, 4, 8, 12, 16] (steps of at least 2, hard stop at 16). TP8-only for # memory as well as comparability -- the ~433 GB NVFP4 diff --git a/perf-changelog.yaml b/perf-changelog.yaml index b0dc6a14d4..f85ee11f69 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -7145,3 +7145,20 @@ - "Update SGLang image from lmsysorg/sglang:nightly-dev-cu13-20260907-30705c00 (2026-09-07 cu13 dev nightly, build commit sgl-project/sglang@30705c00) to the v0.5.19 release image lmsysorg/sglang:v0.5.19-cu130 (digest sha256:d6e7288627be8b02be88e4bba38e73f6d50e2826869f753c13a4c4385ab3eda9, build commit sgl-project/sglang@0bcd822377da7b5718e674eaf9c870d349424dd1, Docker Hub last pushed 2026-09-04T22:50:19Z)." - "The release image ships the same CUDA 13.0.3, FlashInfer 0.6.18 and sgl-kernel 0.4.6.post1 as the nightly. benchmarks/single_node/agentic/qwen3.5_fp8_h200_mtp.sh is unchanged: SGLANG_ENABLE_SPEC_V2 EAGLE MTP at 3 steps, golden acceptance length 3.39, flashinfer attention with allreduce fusion, fp8 quantization and fp8_e4m3 KV, HiCache kernel IO / page_first layout. TP8/EP1 DRAM HiCache concurrency 2 through 24 unchanged." pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2966 + +- config-keys: + - glm5.2-fp4-b300-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Extend the GLM-5.2 NVFP4 B300 SGLang AgentX MTP TP8 HiCache curve from concurrency 16 to 48 with new points at 20, 24, 28, 32, 40, and 48." + - "Move the recipe from the retired cluster:b300-nv fleet to cluster:b300-dsxe so the sweep can be generated and scheduled again." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2829 + +- config-keys: + - glm5.2-fp4-b300-sglang-agentic-mtp + scenario-type: + - agentic-coding + description: + - "Use the pinned SGLang nightly-dev-cu13-20260901-07c8f729 image with FlashInfer 0.6.18, which includes the BF16 TRTLLM MoE allocation fix for small-batch Blackwell execution. Model, TP8, HiCache, MTP settings and concurrency grid are unchanged." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2829