From 3e443ebcbb02523ab9e5a5b5e8f225dcb54426a7 Mon Sep 17 00:00:00 2001
From: Brandon Ros <brandonros1@gmail.com>
Date: Sun, 13 Sep 2026 16:34:17 -0400
Subject: [PATCH 01/10] Move modern backend to LLVM 21.1.8 and CUDA 13.3

---
 .github/ISSUE_TEMPLATE/bug_report.md          |  2 +-
 .github/workflows/ci_linux.yml                | 10 +--
 .github/workflows/ci_windows.yml              |  6 +-
 .github/workflows/container_images.yml        | 18 +++---
 CONTRIBUTING.md                               |  2 +-
 container/rockylinux9-cuda13/Dockerfile       |  4 +-
 .../Dockerfile                                | 42 ++++++------
 container/ubuntu24-cuda13/Dockerfile          |  4 +-
 crates/cuda_builder/Cargo.toml                |  6 +-
 crates/cuda_builder/src/lib.rs                |  8 +--
 crates/cuda_std/src/warp.rs                   |  2 +-
 crates/nvvm/Cargo.toml                        |  6 +-
 crates/nvvm/src/lib.rs                        | 16 ++---
 crates/rustc_codegen_nvvm/Cargo.toml          |  2 +-
 crates/rustc_codegen_nvvm/build.rs            | 30 ++++-----
 crates/rustc_codegen_nvvm/libintrinsics.ll    |  4 +-
 .../rustc_llvm_wrapper/PassWrapper.cpp        | 14 ++--
 .../rustc_llvm_wrapper/RustWrapper.cpp        | 10 +--
 crates/rustc_codegen_nvvm/src/abi.rs          |  4 +-
 crates/rustc_codegen_nvvm/src/back.rs         |  2 +-
 crates/rustc_codegen_nvvm/src/builder.rs      | 36 +++++------
 crates/rustc_codegen_nvvm/src/consts.rs       |  4 +-
 .../rustc_codegen_nvvm/src/ctx_intrinsics.rs  |  8 +--
 crates/rustc_codegen_nvvm/src/init.rs         |  4 +-
 crates/rustc_codegen_nvvm/src/llvm.rs         |  8 +--
 crates/rustc_codegen_nvvm/src/nvvm.rs         | 28 ++++----
 crates/rustc_codegen_nvvm/src/target.rs       |  2 +-
 crates/rustc_codegen_nvvm/src/ty.rs           | 24 +++----
 examples/vecadd/Cargo.toml                    |  4 +-
 flake.lock                                    |  6 +-
 flake.nix                                     | 64 +++++++++----------
 guide/src/guide/getting_started.md            |  6 +-
 scripts/vast-ai.sh                            |  4 +-
 33 files changed, 195 insertions(+), 195 deletions(-)
 rename container/{ubuntu24-cuda13-llvm19 => ubuntu24-cuda13-llvm21}/Dockerfile (64%)

diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
index a6cb12e1..981c224d 100644
--- a/.github/ISSUE_TEMPLATE/bug_report.md
+++ b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -22,7 +22,7 @@ A clear and concise description of what you expected to happen.
 
 - OS: [e.g. Windows 11, Ubuntu 22.04]
 - GPU: [e.g. RTX 3060]
-- CUDA Toolkit version: [e.g. 13.2]
+- CUDA Toolkit version: [e.g. 13.3]
 - cuDNN version (if applicable): [e.g. 9.x]
 - Rust toolchain: [output of `rustc --version`]
 
diff --git a/.github/workflows/ci_linux.yml b/.github/workflows/ci_linux.yml
index 11d3b38d..25e97f82 100644
--- a/.github/workflows/ci_linux.yml
+++ b/.github/workflows/ci_linux.yml
@@ -28,16 +28,16 @@ jobs:
           - name: Ubuntu-24.04 / CUDA-12.8.1 / ARM64
             image: "ghcr.io/rust-gpu/rust-cuda-ubuntu24-cuda12:latest"
             runner: ubuntu-24.04-arm
-          - name: Ubuntu-24.04 / CUDA-13.0.2 / x86_64
+          - name: Ubuntu-24.04 / CUDA-13.3.1 / x86_64
             image: "ghcr.io/rust-gpu/rust-cuda-ubuntu24-cuda13:latest"
             runner: ubuntu-latest
-          - name: Ubuntu-24.04 / CUDA-13.0.2 / ARM64
+          - name: Ubuntu-24.04 / CUDA-13.3.1 / ARM64
             image: "ghcr.io/rust-gpu/rust-cuda-ubuntu24-cuda13:latest"
             runner: ubuntu-24.04-arm
           - name: RockyLinux-9 / CUDA-12.8.1 / x86_64
             image: "ghcr.io/rust-gpu/rust-cuda-rockylinux9-cuda12:latest"
             runner: ubuntu-latest
-          - name: RockyLinux-9 / CUDA-13.0.2 / x86_64
+          - name: RockyLinux-9 / CUDA-13.3.1 / x86_64
             image: "ghcr.io/rust-gpu/rust-cuda-rockylinux9-cuda13:latest"
             runner: ubuntu-latest
 
@@ -152,8 +152,8 @@ jobs:
               --exclude cust
           '
 
-      # The `llvm19` feature on `nvvm` / `cuda_builder` / `rustc_codegen_nvvm` requires
-      # an LLVM 19 toolchain that isn't in the CI image, so we can't run a single
+      # The `llvm21` feature on `nvvm` / `cuda_builder` / `rustc_codegen_nvvm` requires
+      # an LLVM 21 toolchain that isn't in the CI image, so we can't run a single
       # `--all-features` pass over the whole workspace. Doc those three crates with
       # default features (the LLVM 7 path the CI image already supports) and the rest
       # of the workspace with `--all-features`.
diff --git a/.github/workflows/ci_windows.yml b/.github/workflows/ci_windows.yml
index 9c7d4ce9..cfdc269a 100644
--- a/.github/workflows/ci_windows.yml
+++ b/.github/workflows/ci_windows.yml
@@ -42,7 +42,7 @@ jobs:
               ]
           - os: windows-latest
             target: x86_64-pc-windows-msvc
-            cuda: "13.0.2"
+            cuda: "13.3.1"
             nvvm-dll-dir: "nvvm\\bin\\x64"
             sub-packages:
               [
@@ -123,8 +123,8 @@ jobs:
             --exclude blastoff --exclude cudnn --exclude cudnn-sys --exclude cust
 
       # Exclude crates that require cuDNN, not available on Windows CI: cudnn, cudnn-sys.
-      # The `llvm19` feature on `nvvm` / `cuda_builder` / `rustc_codegen_nvvm` requires
-      # an LLVM 19 toolchain that isn't in the CI image, so we can't run a single
+      # The `llvm21` feature on `nvvm` / `cuda_builder` / `rustc_codegen_nvvm` requires
+      # an LLVM 21 toolchain that isn't in the CI image, so we can't run a single
       # `--all-features` pass over the whole workspace. Doc those three crates with
       # default features (the LLVM 7 path the CI image already supports) and the rest
       # of the workspace with `--all-features`.
diff --git a/.github/workflows/container_images.yml b/.github/workflows/container_images.yml
index d3d089ea..0f7eb5b7 100644
--- a/.github/workflows/container_images.yml
+++ b/.github/workflows/container_images.yml
@@ -33,16 +33,16 @@ jobs:
           - name: Ubuntu-24.04/CUDA-12.8.1
             image: "rust-cuda-ubuntu24-cuda12"
             dockerfile: ./container/ubuntu24-cuda12/Dockerfile
-          - name: Ubuntu-24.04/CUDA-13.0.2
+          - name: Ubuntu-24.04/CUDA-13.3.1
             image: "rust-cuda-ubuntu24-cuda13"
             dockerfile: ./container/ubuntu24-cuda13/Dockerfile
-          - name: Ubuntu-24.04/CUDA-13.2.1/LLVM-19.1.7
-            image: "rust-cuda-ubuntu24-cuda13-llvm19"
-            dockerfile: ./container/ubuntu24-cuda13-llvm19/Dockerfile
+          - name: Ubuntu-24.04/CUDA-13.3.1/LLVM-21.1.8
+            image: "rust-cuda-ubuntu24-cuda13-llvm21"
+            dockerfile: ./container/ubuntu24-cuda13-llvm21/Dockerfile
           - name: RockyLinux-9/CUDA-12.8.1
             image: "rust-cuda-rockylinux9-cuda12"
             dockerfile: ./container/rockylinux9-cuda12/Dockerfile
-          - name: RockyLinux-9/CUDA-13.0.2
+          - name: RockyLinux-9/CUDA-13.3.1
             image: "rust-cuda-rockylinux9-cuda13"
             dockerfile: ./container/rockylinux9-cuda13/Dockerfile
     steps:
@@ -161,13 +161,13 @@ jobs:
         variance:
           - name: Ubuntu-24.04/CUDA-12.8.1
             image: "rust-cuda-ubuntu24-cuda12"
-          - name: Ubuntu-24.04/CUDA-13.0.2
+          - name: Ubuntu-24.04/CUDA-13.3.1
             image: "rust-cuda-ubuntu24-cuda13"
-          - name: Ubuntu-24.04/CUDA-13.2.1/LLVM-19.1.7
-            image: "rust-cuda-ubuntu24-cuda13-llvm19"
+          - name: Ubuntu-24.04/CUDA-13.3.1/LLVM-21.1.8
+            image: "rust-cuda-ubuntu24-cuda13-llvm21"
           - name: RockyLinux-9/CUDA-12.8.1
             image: "rust-cuda-rockylinux9-cuda12"
-          - name: RockyLinux-9/CUDA-13.0.2
+          - name: RockyLinux-9/CUDA-13.3.1
             image: "rust-cuda-rockylinux9-cuda13"
     steps:
       - name: Set lowercase repo owner
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 2890a775..9115dbeb 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -33,7 +33,7 @@ For questions, clarifications, and general help:
 ### Windows-Specific Notes
 
 - Ensure the CUDA Toolkit `bin` directory is on your `PATH` (e.g.
-  `C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.2\bin`).
+  `C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\bin`).
 - The MSVC build tools are required. Install via
   [Visual Studio Build Tools](https://visualstudio.microsoft.com/downloads/) with the
   "Desktop development with C++" workload.
diff --git a/container/rockylinux9-cuda13/Dockerfile b/container/rockylinux9-cuda13/Dockerfile
index dc428186..28137e70 100644
--- a/container/rockylinux9-cuda13/Dockerfile
+++ b/container/rockylinux9-cuda13/Dockerfile
@@ -1,4 +1,4 @@
-FROM nvcr.io/nvidia/cuda:13.0.2-cudnn-devel-rockylinux9 AS llvm-builder
+FROM nvcr.io/nvidia/cuda:13.3.1-cudnn-devel-rockylinux9 AS llvm-builder
 
 RUN dnf -y install \
     --nobest \
@@ -51,7 +51,7 @@ RUN curl -sSf -L -O https://github.com/llvm/llvm-project/releases/download/llvmo
     rm -rf llvm-7.1.0.src* && \
     dnf clean all
 
-FROM nvcr.io/nvidia/cuda:13.0.2-cudnn-devel-rockylinux9
+FROM nvcr.io/nvidia/cuda:13.3.1-cudnn-devel-rockylinux9
 
 RUN dnf -y install \
     --nobest \
diff --git a/container/ubuntu24-cuda13-llvm19/Dockerfile b/container/ubuntu24-cuda13-llvm21/Dockerfile
similarity index 64%
rename from container/ubuntu24-cuda13-llvm19/Dockerfile
rename to container/ubuntu24-cuda13-llvm21/Dockerfile
index db4edc9e..0c67b893 100644
--- a/container/ubuntu24-cuda13-llvm19/Dockerfile
+++ b/container/ubuntu24-cuda13-llvm21/Dockerfile
@@ -1,4 +1,4 @@
-FROM nvcr.io/nvidia/cuda:13.2.1-cudnn-devel-ubuntu24.04 AS llvm-builder
+FROM nvcr.io/nvidia/cuda:13.3.1-cudnn-devel-ubuntu24.04 AS llvm-builder
 
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -qq -y install \
     build-essential \
@@ -18,13 +18,13 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -qq -y install \
     zlib1g-dev && \
     rm -rf /var/lib/apt/lists/*
 
-WORKDIR /data/llvm19
+WORKDIR /data/llvm21
 
-# Download and build LLVM 19.1.7 (the active LLVM 19 pin used by `rustc_codegen_nvvm`).
+# Download and build LLVM 21.1.8 (the active LLVM 21 pin used by `rustc_codegen_nvvm`).
 # LLVM 8+ ships as a monorepo tarball; cmake source root is the `llvm/` subdir.
-RUN curl -sSf -L -O https://github.com/llvm/llvm-project/releases/download/llvmorg-19.1.7/llvm-project-19.1.7.src.tar.xz && \
-    tar -xf llvm-project-19.1.7.src.tar.xz && \
-    cd llvm-project-19.1.7.src && \
+RUN curl -sSf -L -O https://github.com/llvm/llvm-project/releases/download/llvmorg-21.1.8/llvm-project-21.1.8.src.tar.xz && \
+    tar -xf llvm-project-21.1.8.src.tar.xz && \
+    cd llvm-project-21.1.8.src && \
     mkdir build && cd build && \
     ARCH=$(dpkg --print-architecture) && \
     if [ "$ARCH" = "amd64" ]; then \
@@ -44,14 +44,14 @@ RUN curl -sSf -L -O https://github.com/llvm/llvm-project/releases/download/llvmo
         -DLLVM_INCLUDE_BENCHMARKS=OFF \
         -DLLVM_ENABLE_ZLIB=ON \
         -DLLVM_ENABLE_TERMINFO=ON \
-        -DCMAKE_INSTALL_PREFIX=/opt/llvm-19 \
+        -DCMAKE_INSTALL_PREFIX=/opt/llvm-21 \
         ../llvm && \
     ninja -j$(nproc) && \
     ninja install && \
     cd ../.. && \
-    rm -rf llvm-project-19.1.7.src*
+    rm -rf llvm-project-21.1.8.src*
 
-FROM nvcr.io/nvidia/cuda:13.2.1-cudnn-devel-ubuntu24.04
+FROM nvcr.io/nvidia/cuda:13.3.1-cudnn-devel-ubuntu24.04
 
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -qq -y install \
     build-essential \
@@ -71,10 +71,10 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -qq -y install \
     libxrandr-dev && \
     rm -rf /var/lib/apt/lists/*
 
-COPY --from=llvm-builder /opt/llvm-19 /opt/llvm-19
-RUN ln -s /opt/llvm-19/bin/llvm-config /usr/bin/llvm-config && \
-    ln -s /opt/llvm-19/bin/llvm-config /usr/bin/llvm-config-19 && \
-    ln -s /opt/llvm-19/bin/llvm-as /usr/bin/llvm-as-19
+COPY --from=llvm-builder /opt/llvm-21 /opt/llvm-21
+RUN ln -s /opt/llvm-21/bin/llvm-config /usr/bin/llvm-config && \
+    ln -s /opt/llvm-21/bin/llvm-config /usr/bin/llvm-config-21 && \
+    ln -s /opt/llvm-21/bin/llvm-as /usr/bin/llvm-as-21
 
 # Get Rust (install rustup; toolchain installed from rust-toolchain.toml below)
 RUN curl -sSf -L https://sh.rustup.rs | bash -s -- -y --profile minimal --default-toolchain none
@@ -85,15 +85,15 @@ WORKDIR /data/rust-cuda
 RUN --mount=type=bind,source=rust-toolchain.toml,target=/data/rust-cuda/rust-toolchain.toml \
     rustup show
 
-# Add nvvm + LLVM 19 dylib to the runtime linker path.
-ENV LD_LIBRARY_PATH="/opt/llvm-19/lib:/usr/local/cuda/nvvm/lib64:${LD_LIBRARY_PATH}"
+# Add nvvm + LLVM 21 dylib to the runtime linker path.
+ENV LD_LIBRARY_PATH="/opt/llvm-21/lib:/usr/local/cuda/nvvm/lib64:${LD_LIBRARY_PATH}"
 
-# `rustc_codegen_nvvm`'s build.rs probes `LLVM_CONFIG_19` to locate the LLVM 19
-# toolchain when the `llvm19` cargo feature is on. The feature itself is gated;
+# `rustc_codegen_nvvm`'s build.rs probes `LLVM_CONFIG_21` to locate the LLVM 21
+# toolchain when the `llvm21` cargo feature is on. The feature itself is gated;
 # downstream crates that depend on `cuda_builder` must build with
-# `--features llvm19` for this to take effect — that propagates through to
-# `nvvm/llvm19` (default `NvvmArch` = Blackwell) and `rustc_codegen_nvvm/llvm19`
-# (LLVM 19 codegen path) per crates/cuda_builder/Cargo.toml.
-ENV LLVM_CONFIG_19=/opt/llvm-19/bin/llvm-config
+# `--features llvm21` for this to take effect — that propagates through to
+# `nvvm/llvm21` (default `NvvmArch` = Blackwell) and `rustc_codegen_nvvm/llvm21`
+# (LLVM 21 codegen path) per crates/cuda_builder/Cargo.toml.
+ENV LLVM_CONFIG_21=/opt/llvm-21/bin/llvm-config
 ENV LLVM_LINK_STATIC=1
 ENV RUST_LOG=info
diff --git a/container/ubuntu24-cuda13/Dockerfile b/container/ubuntu24-cuda13/Dockerfile
index be2a2f73..be2fbe3c 100644
--- a/container/ubuntu24-cuda13/Dockerfile
+++ b/container/ubuntu24-cuda13/Dockerfile
@@ -1,4 +1,4 @@
-FROM nvcr.io/nvidia/cuda:13.0.2-cudnn-devel-ubuntu24.04 AS llvm-builder
+FROM nvcr.io/nvidia/cuda:13.3.1-cudnn-devel-ubuntu24.04 AS llvm-builder
 
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -qq -y install \
     build-essential \
@@ -50,7 +50,7 @@ RUN curl -sSf -L -O https://github.com/llvm/llvm-project/releases/download/llvmo
     cd ../.. && \
     rm -rf llvm-7.1.0.src*
 
-FROM nvcr.io/nvidia/cuda:13.0.2-cudnn-devel-ubuntu24.04
+FROM nvcr.io/nvidia/cuda:13.3.1-cudnn-devel-ubuntu24.04
 
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -qq -y install \
     build-essential \
diff --git a/crates/cuda_builder/Cargo.toml b/crates/cuda_builder/Cargo.toml
index bd1b4e10..3e465388 100644
--- a/crates/cuda_builder/Cargo.toml
+++ b/crates/cuda_builder/Cargo.toml
@@ -16,12 +16,12 @@ default = []
 # HACK(see rust-gpu/spirv-builder): use `dep:` to avoid Cargo auto-creating a feature
 # with the dependency name. Consumers must explicitly opt-in to compiling the backend.
 rustc_codegen_nvvm = ["dep:rustc_codegen_nvvm"]
-# Build the backend against LLVM 19 instead of LLVM 7. Propagates to `nvvm` (which
+# Build the backend against LLVM 21 instead of LLVM 7. Propagates to `nvvm` (which
 # uses it to flip the default `NvvmArch` to `Compute100`) and, when the optional
 # `rustc_codegen_nvvm` dep is also enabled, to `rustc_codegen_nvvm` itself. Even
 # when the optional dep is disabled, the build script's nested `cargo build -p
-# rustc_codegen_nvvm` reads `cfg(feature = "llvm19")` here and forwards it.
-llvm19 = ["nvvm/llvm19", "rustc_codegen_nvvm?/llvm19"]
+# rustc_codegen_nvvm` reads `cfg(feature = "llvm21")` here and forwards it.
+llvm21 = ["nvvm/llvm21", "rustc_codegen_nvvm?/llvm21"]
 
 [dependencies]
 rustc_codegen_nvvm = { version = "0.3", path = "../rustc_codegen_nvvm", optional = true }
diff --git a/crates/cuda_builder/src/lib.rs b/crates/cuda_builder/src/lib.rs
index 8e8bf2b1..17a4b893 100644
--- a/crates/cuda_builder/src/lib.rs
+++ b/crates/cuda_builder/src/lib.rs
@@ -556,11 +556,11 @@ fn build_backend_and_find(filename: &str) -> Option<PathBuf> {
         .arg(&target_dir)
         .current_dir(&workspace_dir);
 
-    // Propagate the `llvm19` cargo feature to the nested backend build. Without this
+    // Propagate the `llvm21` cargo feature to the nested backend build. Without this
     // `rustc_codegen_nvvm`'s build script falls through to the prebuilt LLVM 7
-    // download, which the LLVM 19 codegen path can't link against.
-    if cfg!(feature = "llvm19") {
-        cmd.args(["--features", "llvm19"]);
+    // download, which the LLVM 21 codegen path can't link against.
+    if cfg!(feature = "llvm21") {
+        cmd.args(["--features", "llvm21"]);
     }
 
     let status = cmd.status().ok()?;
diff --git a/crates/cuda_std/src/warp.rs b/crates/cuda_std/src/warp.rs
index 75da1d09..01f42d42 100644
--- a/crates/cuda_std/src/warp.rs
+++ b/crates/cuda_std/src/warp.rs
@@ -744,7 +744,7 @@ pub enum WarpShuffleMode {
 // The libintrinsics.ll wrappers pack their (value, predicate) result into a
 // single i64: low 32 bits = value, bit 32 = predicate. Returning a primitive
 // integer avoids the small-aggregate ABI path where rustc attaches `align N`
-// to the call's return value — an attribute LLVM 19's verifier rejects on
+// to the call's return value — an attribute LLVM 21's verifier rejects on
 // non-pointer returns.
 // Unused on host targets — every caller is `#[gpu_only]`.
 #[allow(dead_code)]
diff --git a/crates/nvvm/Cargo.toml b/crates/nvvm/Cargo.toml
index f260f35c..47998f6b 100644
--- a/crates/nvvm/Cargo.toml
+++ b/crates/nvvm/Cargo.toml
@@ -10,11 +10,11 @@ readme = "../../README.md"
 
 [features]
 default = []
-# Match the `llvm19` feature on `rustc_codegen_nvvm`. Currently only flips the
+# Match the `llvm21` feature on `rustc_codegen_nvvm`. Currently only flips the
 # default `NvvmArch` to the lowest Blackwell capability, since the LLVM 7
-# bitcode dialect can't target `compute_100+` and the LLVM 19 dialect can't
+# bitcode dialect can't target `compute_100+` and the LLVM 21 dialect can't
 # target pre-Blackwell archs.
-llvm19 = []
+llvm21 = []
 
 [dependencies]
 cust_raw = { version = "0.11.3", path = "../cust_raw", default-features = false, features = ["nvvm"] }
diff --git a/crates/nvvm/src/lib.rs b/crates/nvvm/src/lib.rs
index 9e0de2a0..8ee1df8a 100644
--- a/crates/nvvm/src/lib.rs
+++ b/crates/nvvm/src/lib.rs
@@ -310,13 +310,13 @@ pub enum NvvmArch {
     Compute73,
     /// This default value of 7.5 corresponds to Turing and later devices. We default to this
     /// because it is the minimum supported by CUDA 13.0 while being in the middle of the range
-    /// supported by CUDA 12.x. Selected as the default only when the `llvm19` feature is off;
-    /// the LLVM 19 NVVM dialect can't target pre-Blackwell archs.
+    /// supported by CUDA 12.x. Selected as the default only when the `llvm21` feature is off;
+    /// the LLVM 21 NVVM dialect can't target pre-Blackwell archs.
     // WARNING: If you change the default, consider updating:
     // - The `--target-arch` values used for compiletests in `ci_linux.yml` and
     //   `.github/workflows/ci_{linux,windows}.yml`.
     // - The CUDA versions used in `setup_cuda_environment` in `compiletests`.
-    #[cfg_attr(not(feature = "llvm19"), default)]
+    #[cfg_attr(not(feature = "llvm21"), default)]
     Compute75,
     Compute80,
     Compute86,
@@ -326,11 +326,11 @@ pub enum NvvmArch {
     Compute90,
     Compute90a,
     /// First Blackwell arch and the cutoff for NVVM's modern IR dialect — everything at
-    /// or above this capability uses the LLVM 19-flavored bitcode accepted by CUDA 12.9+
+    /// or above this capability uses the LLVM 21-flavored bitcode accepted by CUDA 12.9+
     /// `libnvvm`. See [`NvvmArch::uses_modern_ir_dialect`]. Selected as the default when
-    /// the `llvm19` feature is enabled, since the LLVM 7 dialect can't target this and
-    /// the LLVM 19 dialect can't target anything below it.
-    #[cfg_attr(feature = "llvm19", default)]
+    /// the `llvm21` feature is enabled, since the LLVM 7 dialect can't target this and
+    /// the LLVM 21 dialect can't target anything below it.
+    #[cfg_attr(feature = "llvm21", default)]
     Compute100,
     Compute100f,
     Compute100a,
@@ -758,7 +758,7 @@ impl NvvmProgram {
 
     /// Like [`verify`](Self::verify), but runs the verifier with the same `NvvmOption`s that will
     /// be passed to [`compile`](Self::compile). Passing the user-selected `-arch=compute_XXX` in
-    /// particular matters for CUDA 12.9+ / LLVM 19 bitcode: without it the verifier can fall back
+    /// particular matters for CUDA 12.9+ / LLVM 21 bitcode: without it the verifier can fall back
     /// to the legacy LLVM 7 parser and reject modern-dialect bitcode that would otherwise compile
     /// fine.
     pub fn verify_with_options(&self, options: &[NvvmOption]) -> Result<(), NvvmError> {
diff --git a/crates/rustc_codegen_nvvm/Cargo.toml b/crates/rustc_codegen_nvvm/Cargo.toml
index bab4c499..5c22cf49 100644
--- a/crates/rustc_codegen_nvvm/Cargo.toml
+++ b/crates/rustc_codegen_nvvm/Cargo.toml
@@ -16,7 +16,7 @@ crate-type = ["dylib"]
 
 [features]
 default = []
-llvm19 = []
+llvm21 = []
 
 [dependencies]
 nvvm = { version = "0.1", path = "../nvvm" }
diff --git a/crates/rustc_codegen_nvvm/build.rs b/crates/rustc_codegen_nvvm/build.rs
index 973c3499..8ac93287 100644
--- a/crates/rustc_codegen_nvvm/build.rs
+++ b/crates/rustc_codegen_nvvm/build.rs
@@ -15,7 +15,7 @@ struct LlvmFlavor {
     config_env: &'static str,
     default_binary: &'static str,
     probe_cuda_home: bool,
-    prebuilt_url: &'static str,
+    prebuilt_url: Option<&'static str>,
 }
 
 const LLVM7: LlvmFlavor = LlvmFlavor {
@@ -23,24 +23,23 @@ const LLVM7: LlvmFlavor = LlvmFlavor {
     config_env: "LLVM_CONFIG",
     default_binary: "llvm-config",
     probe_cuda_home: false,
-    prebuilt_url: PREBUILT_LLVM_URL_LLVM7,
+    prebuilt_url: Some(PREBUILT_LLVM_URL_LLVM7),
 };
 
-const LLVM19: LlvmFlavor = LlvmFlavor {
-    major: 19,
-    config_env: "LLVM_CONFIG_19",
-    default_binary: "llvm-config-19",
+const LLVM21: LlvmFlavor = LlvmFlavor {
+    major: 21,
+    config_env: "LLVM_CONFIG_21",
+    default_binary: "llvm-config-21",
     probe_cuda_home: true,
-    prebuilt_url: PREBUILT_LLVM_URL_LLVM19,
+    // No project-maintained LLVM 21 prebuilt has been published.
+    prebuilt_url: None,
 };
 
 static PREBUILT_LLVM_URL_LLVM7: &str =
     "https://github.com/rust-gpu/rustc_codegen_nvvm-llvm/releases/download/llvm-7.1.0/";
-static PREBUILT_LLVM_URL_LLVM19: &str =
-    "https://github.com/rust-gpu/rustc_codegen_nvvm-llvm/releases/download/llvm-19.1.7/";
 
 fn main() {
-    let flavor = if llvm19_enabled() { &LLVM19 } else { &LLVM7 };
+    let flavor = if llvm21_enabled() { &LLVM21 } else { &LLVM7 };
     rustc_llvm_build(flavor);
 }
 
@@ -66,8 +65,8 @@ pub fn output(cmd: &mut Command) -> String {
     String::from_utf8(output.stdout).unwrap()
 }
 
-fn llvm19_enabled() -> bool {
-    tracked_env_var_os("CARGO_FEATURE_LLVM19").is_some()
+fn llvm21_enabled() -> bool {
+    tracked_env_var_os("CARGO_FEATURE_LLVM21").is_some()
 }
 
 fn command_version(path: &Path) -> Option<String> {
@@ -97,7 +96,7 @@ fn sibling_llvm_tool(llvm_config: &Path, tool_prefix: &str) -> Option<PathBuf> {
     // into /usr/bin/ but the rest of the toolchain stays in the install prefix
     // (e.g. /usr/bin/llvm-config -> /opt/llvm-7/bin/llvm-config, with /opt/llvm-7/bin
     // off PATH). It also handles source-built toolchains where tool names are
-    // unsuffixed (`llvm-as`) versus apt-packaged ones (`llvm-as-19`).
+    // unsuffixed (`llvm-as`) versus apt-packaged ones (`llvm-as-21`).
     let output = Command::new(llvm_config).arg("--bindir").output().ok()?;
     if !output.status.success() {
         return None;
@@ -112,7 +111,7 @@ fn target_to_llvm_prebuilt(target: &str) -> String {
         "x86_64-unknown-linux-gnu" => "linux-x86_64",
         "aarch64-unknown-linux-gnu" => "linux-aarch64",
         _ => panic!(
-            "Unsupported target with no matching prebuilt LLVM: `{target}`, install LLVM and set LLVM_CONFIG (or LLVM_CONFIG_19 when the `llvm19` feature is enabled)"
+            "Unsupported target with no matching prebuilt LLVM: `{target}`, install LLVM and set LLVM_CONFIG (or LLVM_CONFIG_21 when the `llvm21` feature is enabled)"
         ),
     };
     format!("{base}.tar.xz")
@@ -202,7 +201,8 @@ fn find_llvm_config(target: &str, flavor: &LlvmFlavor) -> PathBuf {
 
     let url = tracked_env_var_os("PREBUILT_LLVM_URL")
         .map(|x| x.to_string_lossy().to_string())
-        .unwrap_or_else(|| flavor.prebuilt_url.to_string());
+        .or_else(|| flavor.prebuilt_url.map(str::to_owned))
+        .unwrap_or_else(|| fail("No LLVM 21 prebuilt is configured. Install LLVM 21.1.8 and set LLVM_CONFIG_21, or provide PREBUILT_LLVM_URL."));
     download_prebuilt_llvm(target, &url)
 }
 
diff --git a/crates/rustc_codegen_nvvm/libintrinsics.ll b/crates/rustc_codegen_nvvm/libintrinsics.ll
index ef60b884..68ded105 100644
--- a/crates/rustc_codegen_nvvm/libintrinsics.ll
+++ b/crates/rustc_codegen_nvvm/libintrinsics.ll
@@ -6,7 +6,7 @@
 ; LLVM 7 path, regenerate the .bc with `llvm-as-7` (older or newer llvm-as will emit a
 ; bitcode format libnvvm rejects).
 ;
-; The LLVM 19 path assembles this same source at build time with `llvm-as-19`; no
+; The LLVM 21 path assembles this same source at build time with `llvm-as-19`; no
 ; regeneration required, just edit and rebuild.
 source_filename = "libintrinsics"
 ; This data layout must match `DATA_LAYOUT` in `crates/rustc_codegen_nvvm/src/target.rs`.
@@ -156,7 +156,7 @@ declare {i16, i1} @llvm.umul.with.overflow.i16(i16, i16) #0
 ; small two-field aggregate — produces { i32, i8 }, which libnvvm rejects. We
 ; used to bridge by re-packing into { i32, i8 } here, but that aggregate return
 ; causes rustc's call-site ABI to attach `align N` to the return value, which
-; LLVM 19's verifier rejects (align is only valid on pointer returns). So we
+; LLVM 21's verifier rejects (align is only valid on pointer returns). So we
 ; pack into a plain i64 instead: low 32 bits = value, bit 32 = predicate.
 ; Primitive integer return ⇒ no struct ABI ⇒ no spurious return-attribute.
 
diff --git a/crates/rustc_codegen_nvvm/rustc_llvm_wrapper/PassWrapper.cpp b/crates/rustc_codegen_nvvm/rustc_llvm_wrapper/PassWrapper.cpp
index e03bdaa6..cbf74435 100644
--- a/crates/rustc_codegen_nvvm/rustc_llvm_wrapper/PassWrapper.cpp
+++ b/crates/rustc_codegen_nvvm/rustc_llvm_wrapper/PassWrapper.cpp
@@ -169,7 +169,7 @@ extern "C" void LLVMPassManagerBuilderPopulateLTOPassManager(
 extern "C" void LLVMInitializePasses()
 {
 #if LLVM_VERSION_MAJOR >= 19
-  // LLVM 19's pass pipeline is driven through PassBuilder, so the legacy
+  // LLVM 21's pass pipeline is driven through PassBuilder, so the legacy
   // registry initialization hooks are not needed here.
 #else
   PassRegistry &Registry = *PassRegistry::getPassRegistry();
@@ -1134,7 +1134,7 @@ LLVMRustWriteThinBitcodeToFile(LLVMPassManagerRef PMR,
   (void)M;
   (void)BcFile;
   (void)BcFileLen;
-  LLVMRustSetLastError("ThinLTO bitcode writing is not implemented for LLVM 19 yet");
+  LLVMRustSetLastError("ThinLTO bitcode writing is not implemented for LLVM 21 yet");
   return false;
 }
 
@@ -1159,7 +1159,7 @@ LLVMRustCreateThinLTOData(LLVMRustThinLTOModule *modules,
   (void)num_modules;
   (void)preserved_symbols;
   (void)num_symbols;
-  LLVMRustSetLastError("ThinLTO indexing is not implemented for LLVM 19 yet");
+  LLVMRustSetLastError("ThinLTO indexing is not implemented for LLVM 21 yet");
   return nullptr;
 }
 
@@ -1174,7 +1174,7 @@ LLVMRustPrepareThinLTORename(const LLVMRustThinLTOData *Data, LLVMModuleRef M)
 {
   (void)Data;
   (void)M;
-  LLVMRustSetLastError("ThinLTO rename is not implemented for LLVM 19 yet");
+  LLVMRustSetLastError("ThinLTO rename is not implemented for LLVM 21 yet");
   return false;
 }
 
@@ -1183,7 +1183,7 @@ LLVMRustPrepareThinLTOResolveWeak(const LLVMRustThinLTOData *Data, LLVMModuleRef
 {
   (void)Data;
   (void)M;
-  LLVMRustSetLastError("ThinLTO weak resolution is not implemented for LLVM 19 yet");
+  LLVMRustSetLastError("ThinLTO weak resolution is not implemented for LLVM 21 yet");
   return false;
 }
 
@@ -1192,7 +1192,7 @@ LLVMRustPrepareThinLTOInternalize(const LLVMRustThinLTOData *Data, LLVMModuleRef
 {
   (void)Data;
   (void)M;
-  LLVMRustSetLastError("ThinLTO internalization is not implemented for LLVM 19 yet");
+  LLVMRustSetLastError("ThinLTO internalization is not implemented for LLVM 21 yet");
   return false;
 }
 
@@ -1201,7 +1201,7 @@ LLVMRustPrepareThinLTOImport(const LLVMRustThinLTOData *Data, LLVMModuleRef M)
 {
   (void)Data;
   (void)M;
-  LLVMRustSetLastError("ThinLTO importing is not implemented for LLVM 19 yet");
+  LLVMRustSetLastError("ThinLTO importing is not implemented for LLVM 21 yet");
   return false;
 }
 
diff --git a/crates/rustc_codegen_nvvm/rustc_llvm_wrapper/RustWrapper.cpp b/crates/rustc_codegen_nvvm/rustc_llvm_wrapper/RustWrapper.cpp
index 5abe7900..98317f5f 100644
--- a/crates/rustc_codegen_nvvm/rustc_llvm_wrapper/RustWrapper.cpp
+++ b/crates/rustc_codegen_nvvm/rustc_llvm_wrapper/RustWrapper.cpp
@@ -285,7 +285,7 @@ static Attribute::AttrKind fromRust(LLVMRustAttribute Kind)
     return Attribute::SExt;
   case StructRet:
 #if LLVM_VERSION_MAJOR >= 19
-    report_fatal_error("StructRet not supported without a type on LLVM 19+");
+    report_fatal_error("StructRet not supported without a type on LLVM 21+");
 #else
     return Attribute::StructRet;
 #endif
@@ -444,7 +444,7 @@ extern "C" void LLVMRustAddFunctionAttributeWithType(LLVMValueRef Fn, unsigned I
   A->setAttributes(A->getAttributes().addAttributesAtIndex(Ctx, Index, B));
 #else
   // LLVM 7's StructRet/ByVal are plain attribute kinds with no type payload,
-  // so the Ty argument is only meaningful on the LLVM 19 path above. Fall through
+  // so the Ty argument is only meaningful on the LLVM 21 path above. Fall through
   // to the kind-only add on legacy LLVM.
   (void)Ty;
   Function *A = unwrap<Function>(Fn);
@@ -588,7 +588,7 @@ LLVMRustBuildAtomicLoad(LLVMBuilderRef B, LLVMValueRef Source, const char *Name,
                         LLVMAtomicOrdering Order)
 {
 #if LLVM_VERSION_MAJOR >= 19
-  report_fatal_error("LLVMRustBuildAtomicLoad requires a type-aware LLVM 19 wrapper");
+  report_fatal_error("LLVMRustBuildAtomicLoad requires a type-aware LLVM 21 wrapper");
 #else
   LoadInst *LI = new LoadInst(unwrap(Source), 0);
   LI->setAtomic(fromRust(Order));
@@ -1743,7 +1743,7 @@ static FunctionType *LLVMRustGetFunctionTypeForCallee(Value *Callee)
   if (Function *Fn = dyn_cast<Function>(Callee->stripPointerCasts()))
     return Fn->getFunctionType();
 
-  report_fatal_error("LLVMRustBuildCall requires an explicit callee type on LLVM 19");
+  report_fatal_error("LLVMRustBuildCall requires an explicit callee type on LLVM 21");
 }
 #endif
 
@@ -2030,7 +2030,7 @@ extern "C" LLVMRustModuleBuffer *
 LLVMRustModuleBufferCreate(LLVMModuleRef M)
 {
   // Longhand form avoids std::make_unique (C++14) so this compiles under
-  // LLVM 7's `-std=c++11` llvm-config cxxflags as well as LLVM 19's C++17.
+  // LLVM 7's `-std=c++11` llvm-config cxxflags as well as LLVM 21's C++17.
   auto Ret = std::unique_ptr<LLVMRustModuleBuffer>(new LLVMRustModuleBuffer());
   {
     raw_string_ostream OS(Ret->data);
diff --git a/crates/rustc_codegen_nvvm/src/abi.rs b/crates/rustc_codegen_nvvm/src/abi.rs
index b24cc543..5fb9a6f1 100644
--- a/crates/rustc_codegen_nvvm/src/abi.rs
+++ b/crates/rustc_codegen_nvvm/src/abi.rs
@@ -443,13 +443,13 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
             } => {
                 assert!(!on_stack);
                 let i = apply(attrs);
-                #[cfg(feature = "llvm19")]
+                #[cfg(feature = "llvm21")]
                 llvm::Attribute::StructRet.apply_llfn_with_type(
                     llvm::AttributePlace::Argument(i),
                     llfn,
                     self.ret.memory_ty(cx),
                 );
-                #[cfg(not(feature = "llvm19"))]
+                #[cfg(not(feature = "llvm21"))]
                 llvm::Attribute::StructRet.apply_llfn(llvm::AttributePlace::Argument(i), llfn);
             }
             _ => {}
diff --git a/crates/rustc_codegen_nvvm/src/back.rs b/crates/rustc_codegen_nvvm/src/back.rs
index 23548063..a3ce18f1 100644
--- a/crates/rustc_codegen_nvvm/src/back.rs
+++ b/crates/rustc_codegen_nvvm/src/back.rs
@@ -214,7 +214,7 @@ pub(crate) unsafe fn codegen(
     let _bc_timer =
         prof.generic_activity_with_arg("NVVM_module_codegen_make_bitcode", &module.name[..]);
 
-    #[cfg(feature = "llvm19")]
+    #[cfg(feature = "llvm21")]
     if let Err(err) = llvm::verify_module(llmod) {
         return Err(dcx.fatal(format!(
             "LLVM module verification failed for {module_name}: {err}"
diff --git a/crates/rustc_codegen_nvvm/src/builder.rs b/crates/rustc_codegen_nvvm/src/builder.rs
index b864e0ab..1c60195d 100644
--- a/crates/rustc_codegen_nvvm/src/builder.rs
+++ b/crates/rustc_codegen_nvvm/src/builder.rs
@@ -231,7 +231,7 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
                 v = transmute_llval(self.llbuilder, self.cx, v, new_ty);
             }
 
-            #[cfg(not(feature = "llvm19"))]
+            #[cfg(not(feature = "llvm21"))]
             {
                 // Get the return type.
                 let sig = llvm::LLVMGetElementType(self.val_ty(self.llfn()));
@@ -511,9 +511,9 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         trace!("Load {ty:?} {:?}", ptr);
         let ptr = self.pointercast(ptr, self.cx.type_ptr_to(ty));
         unsafe {
-            #[cfg(feature = "llvm19")]
+            #[cfg(feature = "llvm21")]
             let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED);
-            #[cfg(not(feature = "llvm19"))]
+            #[cfg(not(feature = "llvm21"))]
             let load = llvm::LLVMBuildLoad(self.llbuilder, ptr, UNNAMED);
             llvm::LLVMSetAlignment(load, align.bytes() as c_uint);
             load
@@ -524,9 +524,9 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         trace!("Volatile load `{:?}`", ptr);
         let ptr = self.pointercast(ptr, self.cx.type_ptr_to(ty));
         unsafe {
-            #[cfg(feature = "llvm19")]
+            #[cfg(feature = "llvm21")]
             let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED);
-            #[cfg(not(feature = "llvm19"))]
+            #[cfg(not(feature = "llvm21"))]
             let load = llvm::LLVMBuildLoad(self.llbuilder, ptr, UNNAMED);
             llvm::LLVMSetVolatile(load, llvm::True);
             load
@@ -1230,11 +1230,11 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
                 // Local space is only accessible to the current thread.
                 // So, there are no synchronization issues, and we can emulate it using a simple load / compare / store.
                 let load: &'ll Value = unsafe {
-                    #[cfg(feature = "llvm19")]
+                    #[cfg(feature = "llvm21")]
                     {
                         llvm::LLVMBuildLoad2(builder.llbuilder, builder.val_ty(cmp), dst, UNNAMED)
                     }
-                    #[cfg(not(feature = "llvm19"))]
+                    #[cfg(not(feature = "llvm21"))]
                     {
                         llvm::LLVMBuildLoad(builder.llbuilder, dst, UNNAMED)
                     }
@@ -1290,11 +1290,11 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
                 // Local space is only accessible to the current thread. So, there are no
                 // synchronization issues, and we can emulate it using a simple load/compare/store.
                 let load: &'ll Value = unsafe {
-                    #[cfg(feature = "llvm19")]
+                    #[cfg(feature = "llvm21")]
                     {
                         llvm::LLVMBuildLoad2(builder.llbuilder, builder.val_ty(src), dst, UNNAMED)
                     }
-                    #[cfg(not(feature = "llvm19"))]
+                    #[cfg(not(feature = "llvm21"))]
                     {
                         llvm::LLVMBuildLoad(builder.llbuilder, dst, UNNAMED)
                     }
@@ -1362,16 +1362,16 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
     }
 
     fn lifetime_start(&mut self, ptr: &'ll Value, size: Size) {
-        #[cfg(feature = "llvm19")]
+        #[cfg(feature = "llvm21")]
         self.call_lifetime_intrinsic("llvm.lifetime.start.p0", ptr, size);
-        #[cfg(not(feature = "llvm19"))]
+        #[cfg(not(feature = "llvm21"))]
         self.call_lifetime_intrinsic("llvm.lifetime.start.p0i8", ptr, size);
     }
 
     fn lifetime_end(&mut self, ptr: &'ll Value, size: Size) {
-        #[cfg(feature = "llvm19")]
+        #[cfg(feature = "llvm21")]
         self.call_lifetime_intrinsic("llvm.lifetime.end.p0", ptr, size);
-        #[cfg(not(feature = "llvm19"))]
+        #[cfg(not(feature = "llvm21"))]
         self.call_lifetime_intrinsic("llvm.lifetime.end.p0i8", ptr, size);
     }
 
@@ -1389,9 +1389,9 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         self.cx.last_call_llfn.set(None);
         let args = self.check_call("call", llty, llfn, args);
 
-        // On LLVM 7 we must ensure the callee has a pointer-to-FnTy type; LLVM 19's
+        // On LLVM 7 we must ensure the callee has a pointer-to-FnTy type; LLVM 21's
         // opaque pointers make this a no-op, so skip the cast on that path entirely.
-        #[cfg(not(feature = "llvm19"))]
+        #[cfg(not(feature = "llvm21"))]
         let llfn = unsafe {
             let llfn_ptr_ty = llvm::LLVMPointerType(llty, 0);
             if self.val_ty(llfn) == llfn_ptr_ty {
@@ -1417,11 +1417,11 @@ impl<'ll, 'tcx, 'a> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
 
         // bitcast return type if the type was remapped
         let map = self.cx.remapped_integer_args.borrow();
-        #[cfg(feature = "llvm19")]
+        #[cfg(feature = "llvm21")]
         let fn_ty = llty;
-        #[cfg(not(feature = "llvm19"))]
+        #[cfg(not(feature = "llvm21"))]
         let mut fn_ty = self.val_ty(llfn);
-        #[cfg(not(feature = "llvm19"))]
+        #[cfg(not(feature = "llvm21"))]
         while self.cx.type_kind(fn_ty) == TypeKind::Pointer {
             fn_ty = self.cx.element_type(fn_ty);
         }
diff --git a/crates/rustc_codegen_nvvm/src/consts.rs b/crates/rustc_codegen_nvvm/src/consts.rs
index b21a3b88..b8a6c834 100644
--- a/crates/rustc_codegen_nvvm/src/consts.rs
+++ b/crates/rustc_codegen_nvvm/src/consts.rs
@@ -354,13 +354,13 @@ impl<'ll> StaticCodegenMethods for CodegenCx<'ll, '_> {
             let mut val_llty = self.val_ty(v);
             let v = if val_llty == self.type_i1() {
                 val_llty = self.type_i8();
-                #[cfg(feature = "llvm19")]
+                #[cfg(feature = "llvm21")]
                 {
                     let const_int = v as *const llvm::Value as *const llvm::ConstantInt;
                     let const_val = llvm::LLVMConstIntGetZExtValue(&*const_int);
                     llvm::LLVMConstInt(val_llty, const_val, 0)
                 }
-                #[cfg(not(feature = "llvm19"))]
+                #[cfg(not(feature = "llvm21"))]
                 {
                     llvm::LLVMConstZExt(v, val_llty)
                 }
diff --git a/crates/rustc_codegen_nvvm/src/ctx_intrinsics.rs b/crates/rustc_codegen_nvvm/src/ctx_intrinsics.rs
index 3da09599..5af07124 100644
--- a/crates/rustc_codegen_nvvm/src/ctx_intrinsics.rs
+++ b/crates/rustc_codegen_nvvm/src/ctx_intrinsics.rs
@@ -137,13 +137,13 @@ impl<'ll> CodegenCx<'ll, '_> {
         ifn!(map, "llvm.cttz.i32", fn(t_i32, i1) -> t_i32);
         ifn!(map, "llvm.cttz.i64", fn(t_i64, i1) -> t_i64);
 
-        #[cfg(feature = "llvm19")]
+        #[cfg(feature = "llvm21")]
         ifn!(map, "llvm.lifetime.start.p0", fn(t_i64, i8p) -> void);
-        #[cfg(not(feature = "llvm19"))]
+        #[cfg(not(feature = "llvm21"))]
         ifn!(map, "llvm.lifetime.start.p0i8", fn(t_i64, i8p) -> void);
-        #[cfg(feature = "llvm19")]
+        #[cfg(feature = "llvm21")]
         ifn!(map, "llvm.lifetime.end.p0", fn(t_i64, i8p) -> void);
-        #[cfg(not(feature = "llvm19"))]
+        #[cfg(not(feature = "llvm21"))]
         ifn!(map, "llvm.lifetime.end.p0i8", fn(t_i64, i8p) -> void);
 
         ifn!(map, "llvm.expect.i1", fn(i1, i1) -> i1);
diff --git a/crates/rustc_codegen_nvvm/src/init.rs b/crates/rustc_codegen_nvvm/src/init.rs
index 60fef46f..5db1ec4a 100644
--- a/crates/rustc_codegen_nvvm/src/init.rs
+++ b/crates/rustc_codegen_nvvm/src/init.rs
@@ -105,8 +105,8 @@ unsafe fn configure_llvm(sess: &Session) {
         );
 
         // This tuning flag isn't guaranteed to be registered in the dylib-loaded
-        // LLVM19 backend configuration, and it is not required for correctness.
-        #[cfg(not(feature = "llvm19"))]
+        // LLVM21 backend configuration, and it is not required for correctness.
+        #[cfg(not(feature = "llvm21"))]
         add("-import-cold-multiplier=0.1", false);
 
         // for arg in sess_args {
diff --git a/crates/rustc_codegen_nvvm/src/llvm.rs b/crates/rustc_codegen_nvvm/src/llvm.rs
index 9f315cc3..cfb02efc 100644
--- a/crates/rustc_codegen_nvvm/src/llvm.rs
+++ b/crates/rustc_codegen_nvvm/src/llvm.rs
@@ -97,7 +97,7 @@ impl Attribute {
         unsafe { LLVMRustAddFunctionAttribute(llfn, idx.as_uint(), *self) }
     }
 
-    #[cfg(feature = "llvm19")]
+    #[cfg(feature = "llvm21")]
     pub fn apply_llfn_with_type(&self, idx: AttributePlace, llfn: &Value, ty: &Type) {
         unsafe { LLVMRustAddFunctionAttributeWithType(llfn, idx.as_uint(), *self, ty) }
     }
@@ -1500,7 +1500,7 @@ unsafe extern "C" {
 
     // Operations on array, pointer, and vector types (sequence types)
     pub(crate) fn LLVMRustArrayType(ElementType: &Type, ElementCount: u64) -> &Type;
-    #[cfg(feature = "llvm19")]
+    #[cfg(feature = "llvm21")]
     pub(crate) fn LLVMPointerTypeInContext(C: &Context, AddressSpace: c_uint) -> &Type;
     pub(crate) fn LLVMPointerType(ElementType: &Type, AddressSpace: c_uint) -> &Type;
     pub(crate) fn LLVMVectorType(ElementType: &Type, ElementCount: c_uint) -> &Type;
@@ -1629,7 +1629,7 @@ unsafe extern "C" {
     pub(crate) fn LLVMSetFunctionCallConv(Fn: &Value, CC: c_uint);
     pub(crate) fn LLVMRustAddAlignmentAttr(Fn: &Value, index: c_uint, bytes: u32);
     pub(crate) fn LLVMRustAddFunctionAttribute(Fn: &Value, index: c_uint, attr: Attribute);
-    #[cfg(feature = "llvm19")]
+    #[cfg(feature = "llvm21")]
     pub(crate) fn LLVMRustAddFunctionAttributeWithType(
         Fn: &Value,
         index: c_uint,
@@ -1894,7 +1894,7 @@ unsafe extern "C" {
         Val: &'a Value,
         Name: *const c_char,
     ) -> &'a Value;
-    #[cfg(feature = "llvm19")]
+    #[cfg(feature = "llvm21")]
     pub(crate) fn LLVMBuildLoad2<'a>(
         B: &Builder<'a>,
         Ty: &'a Type,
diff --git a/crates/rustc_codegen_nvvm/src/nvvm.rs b/crates/rustc_codegen_nvvm/src/nvvm.rs
index fafb4afb..70e4e858 100644
--- a/crates/rustc_codegen_nvvm/src/nvvm.rs
+++ b/crates/rustc_codegen_nvvm/src/nvvm.rs
@@ -43,9 +43,9 @@ impl Display for CodegenErr {
     }
 }
 
-#[cfg(feature = "llvm19")]
+#[cfg(feature = "llvm21")]
 fn is_known_nvvm_verify_false_negative(log: &str) -> bool {
-    log.contains("Producer: 'LLVM19")
+    log.contains("Producer: 'LLVM21")
         && log.contains("Reader: 'LLVM 7.0.1'")
         && log.contains("parse Invalid value")
 }
@@ -136,21 +136,21 @@ pub fn codegen_bitcode_modules(
     // giving it to libnvvm. Then to debug codegen failures, we can just ask the user to provide the corresponding llvm ir
     // file with --emit=llvm-ir
 
-    // On the llvm19 path, pass the same options we'll hand to `compile` so the verifier uses
+    // On the llvm21 path, pass the same options we'll hand to `compile` so the verifier uses
     // the same arch-specific parser. Without this libnvvm can default to the legacy LLVM 7
-    // reader and reject LLVM 19 dialect bitcode that would otherwise compile fine (see
+    // reader and reject LLVM 21 dialect bitcode that would otherwise compile fine (see
     // `is_known_nvvm_verify_false_negative` for the resulting log signature). On the LLVM 7
-    // path we keep the original option-less verify to avoid drift from the pre-llvm19 baseline.
-    #[cfg(feature = "llvm19")]
+    // path we keep the original option-less verify to avoid drift from the pre-llvm21 baseline.
+    #[cfg(feature = "llvm21")]
     let verification_res = prog.verify_with_options(&args.nvvm_options);
-    #[cfg(not(feature = "llvm19"))]
+    #[cfg(not(feature = "llvm21"))]
     let verification_res = prog.verify();
     if verification_res.is_err() {
         let log = prog.compiler_log().unwrap().unwrap_or_default();
-        #[cfg(feature = "llvm19")]
+        #[cfg(feature = "llvm21")]
         if target_arch.uses_modern_ir_dialect() && is_known_nvvm_verify_false_negative(&log) {
             sess.dcx().warn(
-                "libnvvm verification rejected LLVM 19 bitcode with the known legacy-reader message; proceeding to compilation anyway on the llvm19 path"
+                "libnvvm verification rejected LLVM 21 bitcode with the known legacy-reader message; proceeding to compilation anyway on the llvm21 path"
             );
         } else {
             let footer = "If you plan to submit a bug report please re-run the codegen with `RUSTFLAGS=\"--emit=llvm-ir\" and include the .ll file corresponding to the .o file mentioned in the log";
@@ -158,7 +158,7 @@ pub fn codegen_bitcode_modules(
                 "Malformed NVVM IR program rejected by libnvvm, dumping verifier log:\n\n{log}\n\n{footer}"
             );
         }
-        #[cfg(not(feature = "llvm19"))]
+        #[cfg(not(feature = "llvm21"))]
         {
             let footer = "If you plan to submit a bug report please re-run the codegen with `RUSTFLAGS=\"--emit=llvm-ir\" and include the .ll file corresponding to the .o file mentioned in the log";
             panic!(
@@ -352,17 +352,17 @@ unsafe fn internalize_pass(module: &Module, cx: &Context) {
 }
 
 unsafe fn dce_pass(module: &Module) {
-    #[cfg(feature = "llvm19")]
+    #[cfg(feature = "llvm21")]
     {
         // The legacy C API entrypoint used below (`LLVMAddGlobalDCEPass`) is not
-        // available on our current LLVM 19 runtime path. Keep the backend loadable
-        // by skipping this cleanup for now; revisit if LLVM 19 smoke tests show we
+        // available on our current LLVM 21 runtime path. Keep the backend loadable
+        // by skipping this cleanup for now; revisit if LLVM 21 smoke tests show we
         // need an explicit replacement pass.
         let _ = module;
         return;
     }
 
-    #[cfg(not(feature = "llvm19"))]
+    #[cfg(not(feature = "llvm21"))]
     unsafe {
         let pass_manager = LLVMCreatePassManager();
 
diff --git a/crates/rustc_codegen_nvvm/src/target.rs b/crates/rustc_codegen_nvvm/src/target.rs
index 562cd412..866d061d 100644
--- a/crates/rustc_codegen_nvvm/src/target.rs
+++ b/crates/rustc_codegen_nvvm/src/target.rs
@@ -2,7 +2,7 @@ use crate::llvm::{self, Type};
 use rustc_target::spec::{MergeFunctions, Target, TargetTuple};
 
 // This data layout must match `datalayout` in `crates/rustc_codegen_nvvm/libintrinsics.ll`.
-// Both LLVM 7 and LLVM 19 accept this string; the explicit specs are equivalent to the
+// Both LLVM 7 and LLVM 21 accept this string; the explicit specs are equivalent to the
 // defaults LLVM would fill in from the shorter `clang-19 -target nvptx64-nvidia-cuda`
 // output.
 pub const DATA_LAYOUT: &str = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-i128:128:128-f32:32:32-f64:64:64-v16:16:16-v32:32:32-v64:64:64-v128:128:128-n16:32:64";
diff --git a/crates/rustc_codegen_nvvm/src/ty.rs b/crates/rustc_codegen_nvvm/src/ty.rs
index b1e74365..c058d67f 100644
--- a/crates/rustc_codegen_nvvm/src/ty.rs
+++ b/crates/rustc_codegen_nvvm/src/ty.rs
@@ -56,12 +56,12 @@ impl Type {
 impl<'ll> CodegenCx<'ll, '_> {
     pub(crate) fn voidp(&self) -> &'ll Type {
         // llvm uses i8* for void ptrs, void* is invalid
-        #[cfg(feature = "llvm19")]
+        #[cfg(feature = "llvm21")]
         {
             self.type_ptr_ext(AddressSpace::ZERO)
         }
 
-        #[cfg(not(feature = "llvm19"))]
+        #[cfg(not(feature = "llvm21"))]
         {
             let i8_ty = self.type_i8();
             self.type_ptr_to_ext(i8_ty, AddressSpace::ZERO)
@@ -105,12 +105,12 @@ impl<'ll> CodegenCx<'ll, '_> {
     }
 
     pub(crate) fn type_i8p_ext(&self, address_space: AddressSpace) -> &'ll Type {
-        #[cfg(feature = "llvm19")]
+        #[cfg(feature = "llvm21")]
         {
             self.type_ptr_ext(address_space)
         }
 
-        #[cfg(not(feature = "llvm19"))]
+        #[cfg(not(feature = "llvm21"))]
         {
             self.type_ptr_to_ext(self.type_i8(), address_space)
         }
@@ -126,13 +126,13 @@ impl<'ll> CodegenCx<'ll, '_> {
     }
 
     pub(crate) fn type_ptr_to(&self, ty: &'ll Type) -> &'ll Type {
-        #[cfg(feature = "llvm19")]
+        #[cfg(feature = "llvm21")]
         {
             let _ = ty;
             self.type_ptr_ext(AddressSpace::ZERO)
         }
 
-        #[cfg(not(feature = "llvm19"))]
+        #[cfg(not(feature = "llvm21"))]
         {
             assert_ne!(
                 self.type_kind(ty),
@@ -145,13 +145,13 @@ impl<'ll> CodegenCx<'ll, '_> {
     }
 
     pub(crate) fn type_ptr_to_ext(&self, ty: &'ll Type, address_space: AddressSpace) -> &'ll Type {
-        #[cfg(feature = "llvm19")]
+        #[cfg(feature = "llvm21")]
         {
             let _ = ty;
             self.type_ptr_ext(address_space)
         }
 
-        #[cfg(not(feature = "llvm19"))]
+        #[cfg(not(feature = "llvm21"))]
         {
             unsafe { llvm::LLVMPointerType(ty, address_space.0) }
         }
@@ -245,24 +245,24 @@ impl<'ll, 'tcx> BaseTypeCodegenMethods for CodegenCx<'ll, 'tcx> {
     }
 
     fn type_ptr(&self) -> Self::Type {
-        #[cfg(feature = "llvm19")]
+        #[cfg(feature = "llvm21")]
         unsafe {
             return llvm::LLVMPointerTypeInContext(self.llcx, AddressSpace::ZERO.0);
         }
 
-        #[cfg(not(feature = "llvm19"))]
+        #[cfg(not(feature = "llvm21"))]
         {
             self.type_ptr_ext(AddressSpace::ZERO)
         }
     }
 
     fn type_ptr_ext(&self, address_space: AddressSpace) -> Self::Type {
-        #[cfg(feature = "llvm19")]
+        #[cfg(feature = "llvm21")]
         unsafe {
             return llvm::LLVMPointerTypeInContext(self.llcx, address_space.0);
         }
 
-        #[cfg(not(feature = "llvm19"))]
+        #[cfg(not(feature = "llvm21"))]
         {
             self.type_ptr_to_ext(self.type_i8(), address_space)
         }
diff --git a/examples/vecadd/Cargo.toml b/examples/vecadd/Cargo.toml
index 8686504a..eb0c4f6f 100644
--- a/examples/vecadd/Cargo.toml
+++ b/examples/vecadd/Cargo.toml
@@ -5,10 +5,10 @@ edition = "2024"
 
 [features]
 default = []
-# Build the kernels crate with the LLVM 19 backend. Forwards to `cuda_builder`,
+# Build the kernels crate with the LLVM 21 backend. Forwards to `cuda_builder`,
 # which propagates the feature to `nvvm` (default arch -> Compute100 / Blackwell)
 # and to its nested `cargo build -p rustc_codegen_nvvm`.
-llvm19 = ["cuda_builder/llvm19"]
+llvm21 = ["cuda_builder/llvm21"]
 
 [dependencies]
 cust = { path = "../../crates/cust" }
diff --git a/flake.lock b/flake.lock
index 7eaca8a8..07e24137 100644
--- a/flake.lock
+++ b/flake.lock
@@ -2,11 +2,11 @@
   "nodes": {
     "nixpkgs": {
       "locked": {
-        "lastModified": 1775888245,
-        "narHash": "sha256-nwASzrRDD1JBEu/o8ekKYEXm/oJW6EMCzCRdrwcLe90=",
+        "lastModified": 1789264731,
+        "narHash": "sha256-llGJbC0CcU8DfROr6mZjRJgMLQd/SKwfpzxJH/2lHO4=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "13043924aaa7375ce482ebe2494338e058282925",
+        "rev": "02f5696b0e6097e589076d886b317b83ff0437d7",
         "type": "github"
       },
       "original": {
diff --git a/flake.nix b/flake.nix
index c5a0f761..fb5a97cf 100644
--- a/flake.nix
+++ b/flake.nix
@@ -27,29 +27,29 @@
           # ---- CUDA toolkit (Nix-managed) ----
           # Toolkit pin chooses what PTX version NVVM emits, which then dictates
           # the minimum host driver version at runtime:
-          #   CUDA 13.2 → NVVM 22.0 → PTX 9.2 → needs driver 580.x+ (CUDA 13)
+          #   CUDA 13.3.1 → modern LLVM 21 dialect → PTX 9.3 → needs driver 580.x+ (CUDA 13)
           #   CUDA 12.9 → NVVM 21.x → PTX 8.x → runs on CUDA 12.x drivers
           # `cudatoolkit` is the kitchen-sink symlinkJoin maintained by nixpkgs —
           # every header path and lib layout is already wired correctly. The host
           # NVIDIA driver (libcuda.so.1) is needed at runtime; it is *not* shimmed
           # in here — supply it via the system or extend LD_LIBRARY_PATH yourself
           # before running CUDA programs.
-          cuda19Root = pkgs.cudaPackages_13_2.cudatoolkit;
+          cuda21Root = pkgs.cudaPackages_13_3.cudatoolkit;
           cuda7Root = pkgs.cudaPackages_12_9.cudatoolkit;
 
           toolchain = pkgs.rust-bin.fromRustupToolchainFile ./rust-toolchain.toml;
 
-          # ---- LLVM 19 (from current nixpkgs) ----
-          llvm19 = pkgs.llvmPackages_19;
-          llvm19Bin = lib.getBin llvm19.llvm;
-          llvm19Dev = lib.getDev llvm19.llvm;
-          llvm19CompatTools = pkgs.symlinkJoin {
-            name = "llvm19-compat-tools";
+          # ---- LLVM 21 (from current nixpkgs) ----
+          llvm21 = pkgs.llvmPackages_21;
+          llvm21Bin = lib.getBin llvm21.llvm;
+          llvm21Dev = lib.getDev llvm21.llvm;
+          llvm21CompatTools = pkgs.symlinkJoin {
+            name = "llvm21-compat-tools";
             paths = [
-              (pkgs.writeShellScriptBin "opt-19" ''exec ${llvm19Bin}/bin/opt "$@"'')
-              (pkgs.writeShellScriptBin "llvm-as-19" ''exec ${llvm19Bin}/bin/llvm-as "$@"'')
-              (pkgs.writeShellScriptBin "llvm-dis-19" ''exec ${llvm19Bin}/bin/llvm-dis "$@"'')
-              (pkgs.writeShellScriptBin "llc-19" ''exec ${llvm19Bin}/bin/llc "$@"'')
+              (pkgs.writeShellScriptBin "opt-21" ''exec ${llvm21Bin}/bin/opt "$@"'')
+              (pkgs.writeShellScriptBin "llvm-as-21" ''exec ${llvm21Bin}/bin/llvm-as "$@"'')
+              (pkgs.writeShellScriptBin "llvm-dis-21" ''exec ${llvm21Bin}/bin/llvm-dis "$@"'')
+              (pkgs.writeShellScriptBin "llc-21" ''exec ${llvm21Bin}/bin/llc "$@"'')
             ];
           };
 
@@ -75,10 +75,10 @@
             pkgs.cmake
             pkgs.ninja
           ];
-          # The v19 shell uses unstable's runtime libs (modern glibc). The v7 shell has
+          # The v21 shell uses unstable's runtime libs (modern glibc). The v7 shell has
           # to match LLVM 7's glibc generation (23.05), otherwise ncurses/libstdc++ from
           # unstable demand GLIBC_2.38+ symbols LLVM 7's linked glibc 2.37 doesn't have.
-          v19BuildInputs = [
+          v21BuildInputs = [
             pkgs.openssl
             pkgs.libxml2
             pkgs.zlib
@@ -116,7 +116,7 @@
             buildInputs = v7BuildInputs;
             LLVM_CONFIG = "${llvm7Dev}/bin/llvm-config";
             # Give bindgen an explicit libclang (matched to 23.05's glibc) so it doesn't
-            # fall back to scanning system paths and pick up an apt-installed LLVM 19
+            # fall back to scanning system paths and pick up an apt-installed LLVM 21
             # with deps the v7 shell's LD_LIBRARY_PATH doesn't satisfy.
             LIBCLANG_PATH = "${pkgsLlvm7.lib.getLib pkgsLlvm7.llvmPackages_7.libclang}/lib";
             shellHook = ''
@@ -129,33 +129,33 @@
             '';
           });
 
-          # ---- LLVM 19-only shell (CUDA 13.2 toolkit, the active-work shell) ----
-          v19Shell = pkgs.mkShell ((mkCudaEnv cuda19Root) // {
+          # ---- LLVM 21-only shell (CUDA 13.3.1 toolkit, the active-work shell) ----
+          v21Shell = pkgs.mkShell ((mkCudaEnv cuda21Root) // {
             nativeBuildInputs = commonNativeInputs ++ [
-              cuda19Root
-              llvm19.clang
-              llvm19.libclang
-              llvm19Bin
-              llvm19Dev
-              llvm19CompatTools
+              cuda21Root
+              llvm21.clang
+              llvm21.libclang
+              llvm21Bin
+              llvm21Dev
+              llvm21CompatTools
             ];
-            buildInputs = v19BuildInputs;
-            LLVM_CONFIG_19 = "${llvm19Dev}/bin/llvm-config";
-            LIBCLANG_PATH = "${lib.getLib llvm19.libclang}/lib";
+            buildInputs = v21BuildInputs;
+            LLVM_CONFIG_21 = "${llvm21Dev}/bin/llvm-config";
+            LIBCLANG_PATH = "${lib.getLib llvm21.libclang}/lib";
             shellHook = ''
-              export PATH="${llvm19CompatTools}/bin:${llvm19Bin}/bin:${llvm19Dev}/bin:${cuda19Root}/bin:${cuda19Root}/nvvm/bin:$PATH"
-              export LD_LIBRARY_PATH="${cuda19Root}/nvvm/lib:${cuda19Root}/nvvm/lib64:${cuda19Root}/lib64:${cuda19Root}/lib:${pkgs.ncurses.out}/lib:${pkgs.libxml2.out}/lib:${pkgs.zlib.out}/lib:${pkgs.stdenv.cc.cc.lib}/lib''${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
+              export PATH="${llvm21CompatTools}/bin:${llvm21Bin}/bin:${llvm21Dev}/bin:${cuda21Root}/bin:${cuda21Root}/nvvm/bin:$PATH"
+              export LD_LIBRARY_PATH="${cuda21Root}/nvvm/lib:${cuda21Root}/nvvm/lib64:${cuda21Root}/lib64:${cuda21Root}/lib:${pkgs.ncurses.out}/lib:${pkgs.libxml2.out}/lib:${pkgs.zlib.out}/lib:${pkgs.stdenv.cc.cc.lib}/lib''${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
 
-              echo "rust-cuda llvm19 shell (${system})"
+              echo "rust-cuda llvm21 shell (${system})"
               echo "  CUDA_HOME=$CUDA_HOME"
-              echo "  LLVM_CONFIG_19=$LLVM_CONFIG_19"
+              echo "  LLVM_CONFIG_21=$LLVM_CONFIG_21"
             '';
           });
         in
         {
-          default = v19Shell;
+          default = v21Shell;
           v7 = v7Shell;
-          v19 = v19Shell;
+          v21 = v21Shell;
         };
     in
     {
diff --git a/guide/src/guide/getting_started.md b/guide/src/guide/getting_started.md
index c5d792c8..d8a0b7b1 100644
--- a/guide/src/guide/getting_started.md
+++ b/guide/src/guide/getting_started.md
@@ -367,9 +367,9 @@ After installing the CUDA Toolkit, verify the following directories are on your
 
 ```powershell
 # CUDA 13.x
-$env:PATH += ";C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.2\bin"
-$env:PATH += ";C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.2\bin\x64"
-$env:PATH += ";C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.2\nvvm\bin\x64"
+$env:PATH += ";C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\bin"
+$env:PATH += ";C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\bin\x64"
+$env:PATH += ";C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\nvvm\bin\x64"
 
 # CUDA 12.x -- replace v12.x with your installed version
 $env:PATH += ";C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.9\bin"
diff --git a/scripts/vast-ai.sh b/scripts/vast-ai.sh
index 951c7ddd..e2018b4f 100755
--- a/scripts/vast-ai.sh
+++ b/scripts/vast-ai.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # Build vecadd on the Nix-equipped build host, then push it to a vast.ai
-# Blackwell box with CUDA 13.2+ and run it there.
+# Blackwell box with CUDA 13.3+ and run it there.
 
 set -euo pipefail
 
@@ -23,7 +23,7 @@ VAST_SSH_OPTS=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o Lo
 # in the container's glibc + the CUDA driver's libcuda.so.1.
 echo ">> Building on $BUILD_HOST"
 ssh "$BUILD_HOST" "cd '$BUILD_DIR' \
-  && nix develop .#v19 --command cargo build -p vecadd \
+  && nix develop .#v21 --command cargo build -p vecadd \
   && nix shell nixpkgs#patchelf --command patchelf \
        --set-interpreter /lib64/ld-linux-x86-64.so.2 \
        --remove-rpath '$BUILD_BIN'"

From 9016d73e6d8c784ea0c727412cbc0dd9ddf6035b Mon Sep 17 00:00:00 2001
From: Brandon Ros <brandonros1@gmail.com>
Date: Sun, 13 Sep 2026 16:57:39 -0400
Subject: [PATCH 02/10] Update CUDA installer action for Windows CUDA 13.3.1

---
 .github/workflows/ci_windows.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci_windows.yml b/.github/workflows/ci_windows.yml
index cfdc269a..b7f6bb16 100644
--- a/.github/workflows/ci_windows.yml
+++ b/.github/workflows/ci_windows.yml
@@ -65,7 +65,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Install CUDA
-        uses: Jimver/cuda-toolkit@v0.2.29
+        uses: Jimver/cuda-toolkit@v0.2.36
         id: cuda-toolkit
         with:
           cuda: ${{ matrix.cuda }}

From 5644c17fc527e479407bff21b9b4077c1dcd5bb4 Mon Sep 17 00:00:00 2001
From: Brandon Ros <brandonros1@gmail.com>
Date: Sun, 13 Sep 2026 17:44:26 -0400
Subject: [PATCH 03/10] Validate LLVM 21 Linux prebuilts with CUDA 13.3 vector
 addition

---
 .github/workflows/llvm21.yml | 74 ++++++++++++++++++++++++++++++++++++
 1 file changed, 74 insertions(+)
 create mode 100644 .github/workflows/llvm21.yml

diff --git a/.github/workflows/llvm21.yml b/.github/workflows/llvm21.yml
new file mode 100644
index 00000000..6778efaa
--- /dev/null
+++ b/.github/workflows/llvm21.yml
@@ -0,0 +1,74 @@
+name: LLVM 21 experiment
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [experiment/cuda13.3-llvm21]
+
+permissions:
+  contents: read
+  actions: read
+
+jobs:
+  vecadd:
+    name: CUDA 13.3 / LLVM 21 / ${{ matrix.arch }}
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - runner: ubuntu-24.04
+            arch: linux-x86_64
+          - runner: ubuntu-24.04-arm
+            arch: linux-aarch64
+    steps:
+      - uses: actions/checkout@v4
+
+      # Temporary experiment input until the LLVM 21 prebuilts have a release.
+      # Built from update/llvm-21.1.8 at 2cf27700bffbab35dff21a58d1e19ca12525ca6f.
+      - name: Download LLVM 21.1.8 prebuilt
+        env:
+          GH_TOKEN: ${{ github.token }}
+          LLVM_ARCH: ${{ matrix.arch }}
+        run: |
+          gh run download 34781677658 \
+            --repo brandonros/rustc_codegen_nvvm-llvm \
+            --name "$LLVM_ARCH-llvm-21.1.8" --dir llvm-prebuilt
+          tar -xf "llvm-prebuilt/$LLVM_ARCH.tar.xz" -C llvm-prebuilt
+
+      - name: Build vector-add host and PTX with CUDA 13.3.1
+        env:
+          LLVM_ARCH: ${{ matrix.arch }}
+        run: |
+          docker run --rm \
+            -v "$PWD:/workspace" -w /workspace \
+            -e LLVM_CONFIG_21="/workspace/llvm-prebuilt/$LLVM_ARCH/bin/llvm-config" \
+            -e LLVM_LINK_STATIC=1 \
+            -e RUST_CUDA_DUMP_FINAL_MODULE=1 \
+            -e RUST_CUDA_EMIT_LLVM_IR=1 \
+            nvidia/cuda:13.3.1-devel-ubuntu24.04 bash -c '
+              set -euo pipefail
+              apt-get update
+              DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+                build-essential curl ca-certificates git pkg-config libssl-dev \
+                libffi-dev libedit-dev libxml2-dev libtinfo-dev zlib1g-dev xz-utils
+              curl --proto "=https" --tlsv1.2 -sSf https://sh.rustup.rs | \
+                sh -s -- -y --profile minimal --default-toolchain none
+              export PATH="/root/.cargo/bin:$PATH"
+              export LD_LIBRARY_PATH="$(dirname "$LLVM_CONFIG_21")/../lib:/usr/local/cuda/nvvm/lib64:${LD_LIBRARY_PATH:-}"
+              export LIBRARY_PATH="/usr/local/cuda/lib64/stubs:${LIBRARY_PATH:-}"
+              test "$("$LLVM_CONFIG_21" --version)" = 21.1.8
+              nvcc --version
+              rustup show
+              cargo build -p vecadd --features llvm21
+            '
+
+      - name: Upload PTX and LLVM IR
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: vecadd-llvm21-${{ matrix.arch }}
+          path: |
+            target/**/kernels.ptx
+            target/**/final-module.ll
+          if-no-files-found: warn

From 508eddc0ac4e12b824984025bfacde3e5a8df69d Mon Sep 17 00:00:00 2001
From: Brandon Ros <brandonros1@gmail.com>
Date: Sun, 13 Sep 2026 17:58:42 -0400
Subject: [PATCH 04/10] Install libclang for bindgen in LLVM 21 validation

---
 .github/workflows/llvm21.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/llvm21.yml b/.github/workflows/llvm21.yml
index 6778efaa..15abbf45 100644
--- a/.github/workflows/llvm21.yml
+++ b/.github/workflows/llvm21.yml
@@ -50,7 +50,7 @@ jobs:
               set -euo pipefail
               apt-get update
               DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
-                build-essential curl ca-certificates git pkg-config libssl-dev \
+                build-essential libclang-dev curl ca-certificates git pkg-config libssl-dev \
                 libffi-dev libedit-dev libxml2-dev libtinfo-dev zlib1g-dev xz-utils
               curl --proto "=https" --tlsv1.2 -sSf https://sh.rustup.rs | \
                 sh -s -- -y --profile minimal --default-toolchain none

From d0acf3694975301650f22e79d35b2464de64d669 Mon Sep 17 00:00:00 2001
From: Brandon Ros <brandonros1@gmail.com>
Date: Sun, 13 Sep 2026 18:07:50 -0400
Subject: [PATCH 05/10] Move LLVM 21 validation into Linux CI using release
 prebuilts

---
 .github/workflows/ci_linux.yml | 78 ++++++++++++++++++++++++++++++++++
 .github/workflows/llvm21.yml   | 74 --------------------------------
 2 files changed, 78 insertions(+), 74 deletions(-)
 delete mode 100644 .github/workflows/llvm21.yml

diff --git a/.github/workflows/ci_linux.yml b/.github/workflows/ci_linux.yml
index 25e97f82..864379f6 100644
--- a/.github/workflows/ci_linux.yml
+++ b/.github/workflows/ci_linux.yml
@@ -1,6 +1,7 @@
 name: CI on Linux
 
 on:
+  workflow_dispatch:
   pull_request:
     paths-ignore:
       - "**.md"
@@ -25,21 +26,40 @@ jobs:
           - name: Ubuntu-24.04 / CUDA-12.8.1 / x86_64
             image: "ghcr.io/rust-gpu/rust-cuda-ubuntu24-cuda12:latest"
             runner: ubuntu-latest
+            llvm: 7
           - name: Ubuntu-24.04 / CUDA-12.8.1 / ARM64
             image: "ghcr.io/rust-gpu/rust-cuda-ubuntu24-cuda12:latest"
             runner: ubuntu-24.04-arm
+            llvm: 7
           - name: Ubuntu-24.04 / CUDA-13.3.1 / x86_64
             image: "ghcr.io/rust-gpu/rust-cuda-ubuntu24-cuda13:latest"
             runner: ubuntu-latest
+            llvm: 7
           - name: Ubuntu-24.04 / CUDA-13.3.1 / ARM64
             image: "ghcr.io/rust-gpu/rust-cuda-ubuntu24-cuda13:latest"
             runner: ubuntu-24.04-arm
+            llvm: 7
           - name: RockyLinux-9 / CUDA-12.8.1 / x86_64
             image: "ghcr.io/rust-gpu/rust-cuda-rockylinux9-cuda12:latest"
             runner: ubuntu-latest
+            llvm: 7
           - name: RockyLinux-9 / CUDA-13.3.1 / x86_64
             image: "ghcr.io/rust-gpu/rust-cuda-rockylinux9-cuda13:latest"
             runner: ubuntu-latest
+            llvm: 7
+
+          - name: Ubuntu-24.04 / CUDA-13.3.1 / LLVM-21 / x86_64
+            image: "nvidia/cuda:13.3.1-devel-ubuntu24.04"
+            runner: ubuntu-24.04
+            llvm: 21
+            artifact: linux-x86_64
+            sha256: f05360ab8f442accc18f2ff27853f98b44e857726958b2124b9fdd9e817af501
+          - name: Ubuntu-24.04 / CUDA-13.3.1 / LLVM-21 / ARM64
+            image: "nvidia/cuda:13.3.1-devel-ubuntu24.04"
+            runner: ubuntu-24.04-arm
+            llvm: 21
+            artifact: linux-aarch64
+            sha256: 7e196f66af63b99bf9801ad3693e357cbe07258222bfc12d4f2b2f2f7ec59b8e
 
     steps:
       - name: Free up space
@@ -109,32 +129,58 @@ jobs:
             sleep infinity
           docker start "$CONTAINER_NAME"
 
+      - name: Install LLVM 21 release and build dependencies
+        if: matrix.variance.llvm == 21
+        env:
+          LLVM_ARCH: ${{ matrix.variance.artifact }}
+          LLVM_SHA256: ${{ matrix.variance.sha256 }}
+        run: |
+          mkdir -p llvm-prebuilt
+          curl --fail --location --retry 3 \
+            "https://github.com/brandonros/rustc_codegen_nvvm-llvm/releases/download/llvm-21.1.8/$LLVM_ARCH.tar.xz" \
+            -o "llvm-prebuilt/$LLVM_ARCH.tar.xz"
+          echo "$LLVM_SHA256  llvm-prebuilt/$LLVM_ARCH.tar.xz" | sha256sum --check
+          tar -xf "llvm-prebuilt/$LLVM_ARCH.tar.xz" -C llvm-prebuilt
+          docker exec "$CONTAINER_NAME" bash -lc 'set -euo pipefail
+            apt-get update
+            DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+              build-essential libclang-dev curl ca-certificates git pkg-config libssl-dev \
+              libffi-dev libedit-dev libxml2-dev libtinfo-dev zlib1g-dev xz-utils
+            curl --proto "=https" --tlsv1.2 -sSf https://sh.rustup.rs | \
+              sh -s -- -y --profile minimal --default-toolchain none
+          '
+
       - name: Verify CUDA, Rust installation
         run: |
           docker exec "$CONTAINER_NAME" bash -lc 'set -euo pipefail
             nvcc --version
+            if [ -f /root/.cargo/env ]; then source /root/.cargo/env; fi
             rustup show
           '
 
       - name: Rustfmt
+        if: matrix.variance.llvm == 7
         run: |
           docker exec "$CONTAINER_NAME" bash -lc 'set -euo pipefail
             cargo fmt --all -- --check
           '
 
       - name: Build all bindings
+        if: matrix.variance.llvm == 7
         run: |
           docker exec "$CONTAINER_NAME" bash -lc 'set -euo pipefail
             cargo build --all-features -p cust_raw
           '
 
       - name: Build workspace
+        if: matrix.variance.llvm == 7
         run: |
           docker exec "$CONTAINER_NAME" bash -lc 'set -euo pipefail
             cargo build
           '
 
       - name: Clippy
+        if: matrix.variance.llvm == 7
         run: |
           docker exec "$CONTAINER_NAME" bash -lc 'set -euo pipefail
             export RUSTFLAGS=-Dwarnings
@@ -143,6 +189,7 @@ jobs:
 
       # Exclude crates with tests that require an NVIDIA GPU: blastoff, cudnn, cust.
       - name: Test
+        if: matrix.variance.llvm == 7
         run: |
           docker exec "$CONTAINER_NAME" bash -lc 'set -euo pipefail
             export RUSTFLAGS=-Dwarnings
@@ -158,6 +205,7 @@ jobs:
       # default features (the LLVM 7 path the CI image already supports) and the rest
       # of the workspace with `--all-features`.
       - name: Check documentation
+        if: matrix.variance.llvm == 7
         run: |
           docker exec "$CONTAINER_NAME" bash -lc 'set -euo pipefail
             export RUSTDOCFLAGS=-Dwarnings
@@ -167,7 +215,37 @@ jobs:
               --document-private-items --no-deps
           '
 
+      # Compile both the host and device code; execution needs an NVIDIA GPU.
+      - name: Build vector-add with LLVM 21
+        if: matrix.variance.llvm == 21
+        env:
+          LLVM_ARCH: ${{ matrix.variance.artifact }}
+        run: |
+          docker exec \
+            -e LLVM_CONFIG_21="/workspace/llvm-prebuilt/$LLVM_ARCH/bin/llvm-config" \
+            -e LLVM_LINK_STATIC=1 \
+            -e RUST_CUDA_DUMP_FINAL_MODULE=1 \
+            -e RUST_CUDA_EMIT_LLVM_IR=1 \
+            "$CONTAINER_NAME" bash -lc 'set -euo pipefail
+              source /root/.cargo/env
+              export LD_LIBRARY_PATH="$(dirname "$LLVM_CONFIG_21")/../lib:/usr/local/cuda/nvvm/lib64:${LD_LIBRARY_PATH:-}"
+              export LIBRARY_PATH="/usr/local/cuda/lib64/stubs:${LIBRARY_PATH:-}"
+              test "$("$LLVM_CONFIG_21" --version)" = 21.1.8
+              cargo build -p vecadd --features llvm21
+            '
+
+      - name: Upload LLVM 21 PTX and IR
+        if: always() && matrix.variance.llvm == 21
+        uses: actions/upload-artifact@v4
+        with:
+          name: vecadd-llvm21-${{ matrix.variance.artifact }}
+          path: |
+            target/**/kernels.ptx
+            target/**/final-module.ll
+          if-no-files-found: warn
+
       - name: Stop build container
+        if: always()
         run: |
           docker rm -f "$CONTAINER_NAME" || true
 
diff --git a/.github/workflows/llvm21.yml b/.github/workflows/llvm21.yml
deleted file mode 100644
index 15abbf45..00000000
--- a/.github/workflows/llvm21.yml
+++ /dev/null
@@ -1,74 +0,0 @@
-name: LLVM 21 experiment
-
-on:
-  workflow_dispatch:
-  push:
-    branches: [experiment/cuda13.3-llvm21]
-
-permissions:
-  contents: read
-  actions: read
-
-jobs:
-  vecadd:
-    name: CUDA 13.3 / LLVM 21 / ${{ matrix.arch }}
-    runs-on: ${{ matrix.runner }}
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - runner: ubuntu-24.04
-            arch: linux-x86_64
-          - runner: ubuntu-24.04-arm
-            arch: linux-aarch64
-    steps:
-      - uses: actions/checkout@v4
-
-      # Temporary experiment input until the LLVM 21 prebuilts have a release.
-      # Built from update/llvm-21.1.8 at 2cf27700bffbab35dff21a58d1e19ca12525ca6f.
-      - name: Download LLVM 21.1.8 prebuilt
-        env:
-          GH_TOKEN: ${{ github.token }}
-          LLVM_ARCH: ${{ matrix.arch }}
-        run: |
-          gh run download 34781677658 \
-            --repo brandonros/rustc_codegen_nvvm-llvm \
-            --name "$LLVM_ARCH-llvm-21.1.8" --dir llvm-prebuilt
-          tar -xf "llvm-prebuilt/$LLVM_ARCH.tar.xz" -C llvm-prebuilt
-
-      - name: Build vector-add host and PTX with CUDA 13.3.1
-        env:
-          LLVM_ARCH: ${{ matrix.arch }}
-        run: |
-          docker run --rm \
-            -v "$PWD:/workspace" -w /workspace \
-            -e LLVM_CONFIG_21="/workspace/llvm-prebuilt/$LLVM_ARCH/bin/llvm-config" \
-            -e LLVM_LINK_STATIC=1 \
-            -e RUST_CUDA_DUMP_FINAL_MODULE=1 \
-            -e RUST_CUDA_EMIT_LLVM_IR=1 \
-            nvidia/cuda:13.3.1-devel-ubuntu24.04 bash -c '
-              set -euo pipefail
-              apt-get update
-              DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
-                build-essential libclang-dev curl ca-certificates git pkg-config libssl-dev \
-                libffi-dev libedit-dev libxml2-dev libtinfo-dev zlib1g-dev xz-utils
-              curl --proto "=https" --tlsv1.2 -sSf https://sh.rustup.rs | \
-                sh -s -- -y --profile minimal --default-toolchain none
-              export PATH="/root/.cargo/bin:$PATH"
-              export LD_LIBRARY_PATH="$(dirname "$LLVM_CONFIG_21")/../lib:/usr/local/cuda/nvvm/lib64:${LD_LIBRARY_PATH:-}"
-              export LIBRARY_PATH="/usr/local/cuda/lib64/stubs:${LIBRARY_PATH:-}"
-              test "$("$LLVM_CONFIG_21" --version)" = 21.1.8
-              nvcc --version
-              rustup show
-              cargo build -p vecadd --features llvm21
-            '
-
-      - name: Upload PTX and LLVM IR
-        if: always()
-        uses: actions/upload-artifact@v4
-        with:
-          name: vecadd-llvm21-${{ matrix.arch }}
-          path: |
-            target/**/kernels.ptx
-            target/**/final-module.ll
-          if-no-files-found: warn

From 30fc174d7c217d7e553457821c8b9ea7ba3be649 Mon Sep 17 00:00:00 2001
From: Brandon Ros <brandonros1@gmail.com>
Date: Sun, 13 Sep 2026 18:11:20 -0400
Subject: [PATCH 06/10] Use upstream LLVM 21 release for CI and backend
 downloads

---
 .github/workflows/ci_linux.yml     |  6 +-----
 crates/rustc_codegen_nvvm/build.rs | 13 +++++++------
 2 files changed, 8 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/ci_linux.yml b/.github/workflows/ci_linux.yml
index 864379f6..8d9e85c6 100644
--- a/.github/workflows/ci_linux.yml
+++ b/.github/workflows/ci_linux.yml
@@ -53,13 +53,11 @@ jobs:
             runner: ubuntu-24.04
             llvm: 21
             artifact: linux-x86_64
-            sha256: f05360ab8f442accc18f2ff27853f98b44e857726958b2124b9fdd9e817af501
           - name: Ubuntu-24.04 / CUDA-13.3.1 / LLVM-21 / ARM64
             image: "nvidia/cuda:13.3.1-devel-ubuntu24.04"
             runner: ubuntu-24.04-arm
             llvm: 21
             artifact: linux-aarch64
-            sha256: 7e196f66af63b99bf9801ad3693e357cbe07258222bfc12d4f2b2f2f7ec59b8e
 
     steps:
       - name: Free up space
@@ -133,13 +131,11 @@ jobs:
         if: matrix.variance.llvm == 21
         env:
           LLVM_ARCH: ${{ matrix.variance.artifact }}
-          LLVM_SHA256: ${{ matrix.variance.sha256 }}
         run: |
           mkdir -p llvm-prebuilt
           curl --fail --location --retry 3 \
-            "https://github.com/brandonros/rustc_codegen_nvvm-llvm/releases/download/llvm-21.1.8/$LLVM_ARCH.tar.xz" \
+            "https://github.com/Rust-GPU/rustc_codegen_nvvm-llvm/releases/download/llvm-21.1.8/$LLVM_ARCH.tar.xz" \
             -o "llvm-prebuilt/$LLVM_ARCH.tar.xz"
-          echo "$LLVM_SHA256  llvm-prebuilt/$LLVM_ARCH.tar.xz" | sha256sum --check
           tar -xf "llvm-prebuilt/$LLVM_ARCH.tar.xz" -C llvm-prebuilt
           docker exec "$CONTAINER_NAME" bash -lc 'set -euo pipefail
             apt-get update
diff --git a/crates/rustc_codegen_nvvm/build.rs b/crates/rustc_codegen_nvvm/build.rs
index 8ac93287..59fe4da0 100644
--- a/crates/rustc_codegen_nvvm/build.rs
+++ b/crates/rustc_codegen_nvvm/build.rs
@@ -15,7 +15,7 @@ struct LlvmFlavor {
     config_env: &'static str,
     default_binary: &'static str,
     probe_cuda_home: bool,
-    prebuilt_url: Option<&'static str>,
+    prebuilt_url: &'static str,
 }
 
 const LLVM7: LlvmFlavor = LlvmFlavor {
@@ -23,7 +23,7 @@ const LLVM7: LlvmFlavor = LlvmFlavor {
     config_env: "LLVM_CONFIG",
     default_binary: "llvm-config",
     probe_cuda_home: false,
-    prebuilt_url: Some(PREBUILT_LLVM_URL_LLVM7),
+    prebuilt_url: PREBUILT_LLVM_URL_LLVM7,
 };
 
 const LLVM21: LlvmFlavor = LlvmFlavor {
@@ -31,13 +31,15 @@ const LLVM21: LlvmFlavor = LlvmFlavor {
     config_env: "LLVM_CONFIG_21",
     default_binary: "llvm-config-21",
     probe_cuda_home: true,
-    // No project-maintained LLVM 21 prebuilt has been published.
-    prebuilt_url: None,
+    prebuilt_url: PREBUILT_LLVM_URL_LLVM21,
 };
 
 static PREBUILT_LLVM_URL_LLVM7: &str =
     "https://github.com/rust-gpu/rustc_codegen_nvvm-llvm/releases/download/llvm-7.1.0/";
 
+static PREBUILT_LLVM_URL_LLVM21: &str =
+    "https://github.com/rust-gpu/rustc_codegen_nvvm-llvm/releases/download/llvm-21.1.8/";
+
 fn main() {
     let flavor = if llvm21_enabled() { &LLVM21 } else { &LLVM7 };
     rustc_llvm_build(flavor);
@@ -201,8 +203,7 @@ fn find_llvm_config(target: &str, flavor: &LlvmFlavor) -> PathBuf {
 
     let url = tracked_env_var_os("PREBUILT_LLVM_URL")
         .map(|x| x.to_string_lossy().to_string())
-        .or_else(|| flavor.prebuilt_url.map(str::to_owned))
-        .unwrap_or_else(|| fail("No LLVM 21 prebuilt is configured. Install LLVM 21.1.8 and set LLVM_CONFIG_21, or provide PREBUILT_LLVM_URL."));
+        .unwrap_or_else(|| flavor.prebuilt_url.to_owned());
     download_prebuilt_llvm(target, &url)
 }
 

From fb2b9bc31a098ef2f901e6bb4253ff94237357f7 Mon Sep 17 00:00:00 2001
From: Brandon Ros <brandonros1@gmail.com>
Date: Sun, 13 Sep 2026 22:44:02 -0400
Subject: [PATCH 07/10] fix(cust): handle CUDA 13.3 reserved graph node type

---
 crates/cust/build.rs     |  5 +++++
 crates/cust/src/graph.rs | 30 ++++++++++++++++++++++++++++++
 2 files changed, 35 insertions(+)

diff --git a/crates/cust/build.rs b/crates/cust/build.rs
index d002cdc6..319e6ba0 100644
--- a/crates/cust/build.rs
+++ b/crates/cust/build.rs
@@ -10,6 +10,11 @@ fn main() {
     if driver_version >= 12030 {
         println!("cargo::rustc-cfg=conditional_node");
     }
+    // CUDA 13.3 headers add a reserved graph node discriminant.
+    println!("cargo::rustc-check-cfg=cfg(reserved_graph_node_16)");
+    if driver_version >= 13030 {
+        println!("cargo::rustc-cfg=reserved_graph_node_16");
+    }
     // In CUDA 13.0 several pairs/trios of functions were merged:
     // ```
     // CUresult cuMemAdvise(CUdeviceptr devPtr, size_t count, CUmem_advise advice, CUdevice device);
diff --git a/crates/cust/src/graph.rs b/crates/cust/src/graph.rs
index 9f04329e..8f6cea83 100644
--- a/crates/cust/src/graph.rs
+++ b/crates/cust/src/graph.rs
@@ -175,6 +175,9 @@ pub enum GraphNodeType {
     /// Conditional node.
     #[cfg(conditional_node)]
     Conditional,
+    /// Reserved by CUDA; this does not represent a supported graph operation.
+    #[cfg(reserved_graph_node_16)]
+    Reserved16,
 }
 
 impl GraphNodeType {
@@ -197,6 +200,8 @@ impl GraphNodeType {
             CU_GRAPH_NODE_TYPE_BATCH_MEM_OP => GraphNodeType::BatchMemoryOperation,
             #[cfg(conditional_node)]
             CU_GRAPH_NODE_TYPE_CONDITIONAL => GraphNodeType::Conditional,
+            #[cfg(reserved_graph_node_16)]
+            CU_GRAPH_NODE_TYPE_RESERVED_16 => GraphNodeType::Reserved16,
         }
     }
 
@@ -219,6 +224,8 @@ impl GraphNodeType {
             Self::BatchMemoryOperation => CU_GRAPH_NODE_TYPE_BATCH_MEM_OP,
             #[cfg(conditional_node)]
             Self::Conditional => CU_GRAPH_NODE_TYPE_CONDITIONAL,
+            #[cfg(reserved_graph_node_16)]
+            Self::Reserved16 => CU_GRAPH_NODE_TYPE_RESERVED_16,
         }
     }
 }
@@ -518,3 +525,26 @@ impl Drop for Graph {
         }
     }
 }
+
+#[cfg(test)]
+mod node_type_tests {
+    use super::{GraphNodeType, driver_sys::CUgraphNodeType};
+
+    #[test]
+    fn kernel_node_round_trip() {
+        let raw = CUgraphNodeType::CU_GRAPH_NODE_TYPE_KERNEL;
+        assert_eq!(
+            GraphNodeType::from_raw(raw),
+            GraphNodeType::KernelInvocation
+        );
+        assert_eq!(GraphNodeType::from_raw(raw).to_raw(), raw);
+    }
+
+    #[cfg(reserved_graph_node_16)]
+    #[test]
+    fn reserved_node_round_trip() {
+        let raw = CUgraphNodeType::CU_GRAPH_NODE_TYPE_RESERVED_16;
+        assert_eq!(GraphNodeType::from_raw(raw), GraphNodeType::Reserved16);
+        assert_eq!(GraphNodeType::from_raw(raw).to_raw(), raw);
+    }
+}

From 9a2829b1b6e512ad5d5f8107a6544536d6c0c479 Mon Sep 17 00:00:00 2001
From: Brandon Ros <brandonros1@gmail.com>
Date: Sun, 13 Sep 2026 22:59:59 -0400
Subject: [PATCH 08/10] fix(nvvm): update wrapper APIs for LLVM 21 and verify
 capture semantics

---
 .github/workflows/ci_linux.yml                |  3 +-
 .../rustc_llvm_wrapper/PassWrapper.cpp        |  4 ++
 .../rustc_llvm_wrapper/RustWrapper.cpp        | 37 +++++++++++----
 .../rustc_llvm_wrapper/rustllvm.h             |  4 ++
 tests/llvm-wrapper/attributes.cpp             | 47 +++++++++++++++++++
 tests/llvm-wrapper/run.py                     | 35 ++++++++++++++
 6 files changed, 121 insertions(+), 9 deletions(-)
 create mode 100644 tests/llvm-wrapper/attributes.cpp
 create mode 100644 tests/llvm-wrapper/run.py

diff --git a/.github/workflows/ci_linux.yml b/.github/workflows/ci_linux.yml
index 8d9e85c6..19342483 100644
--- a/.github/workflows/ci_linux.yml
+++ b/.github/workflows/ci_linux.yml
@@ -140,7 +140,7 @@ jobs:
           docker exec "$CONTAINER_NAME" bash -lc 'set -euo pipefail
             apt-get update
             DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
-              build-essential libclang-dev curl ca-certificates git pkg-config libssl-dev \
+              build-essential libclang-dev python3 curl ca-certificates git pkg-config libssl-dev \
               libffi-dev libedit-dev libxml2-dev libtinfo-dev zlib1g-dev xz-utils
             curl --proto "=https" --tlsv1.2 -sSf https://sh.rustup.rs | \
               sh -s -- -y --profile minimal --default-toolchain none
@@ -227,6 +227,7 @@ jobs:
               export LD_LIBRARY_PATH="$(dirname "$LLVM_CONFIG_21")/../lib:/usr/local/cuda/nvvm/lib64:${LD_LIBRARY_PATH:-}"
               export LIBRARY_PATH="/usr/local/cuda/lib64/stubs:${LIBRARY_PATH:-}"
               test "$("$LLVM_CONFIG_21" --version)" = 21.1.8
+              python3 tests/llvm-wrapper/run.py
               cargo build -p vecadd --features llvm21
             '
 
diff --git a/crates/rustc_codegen_nvvm/rustc_llvm_wrapper/PassWrapper.cpp b/crates/rustc_codegen_nvvm/rustc_llvm_wrapper/PassWrapper.cpp
index cbf74435..d6c67a38 100644
--- a/crates/rustc_codegen_nvvm/rustc_llvm_wrapper/PassWrapper.cpp
+++ b/crates/rustc_codegen_nvvm/rustc_llvm_wrapper/PassWrapper.cpp
@@ -641,7 +641,11 @@ extern "C" void LLVMRustAddBuilderLibraryInfo(LLVMPassManagerBuilderRef PMBR,
 {
 #if LLVM_VERSION_MAJOR >= 19
   auto *Builder = unwrap(PMBR);
+#if LLVM_VERSION_MAJOR >= 21
+  Builder->TargetTriple = unwrap(M)->getTargetTriple().str();
+#else
   Builder->TargetTriple = unwrap(M)->getTargetTriple();
+#endif
   Builder->DisableSimplifyLibCalls = DisableSimplifyLibCalls;
 #else
   Triple TargetTriple(unwrap(M)->getTargetTriple());
diff --git a/crates/rustc_codegen_nvvm/rustc_llvm_wrapper/RustWrapper.cpp b/crates/rustc_codegen_nvvm/rustc_llvm_wrapper/RustWrapper.cpp
index 98317f5f..a5fd1fb4 100644
--- a/crates/rustc_codegen_nvvm/rustc_llvm_wrapper/RustWrapper.cpp
+++ b/crates/rustc_codegen_nvvm/rustc_llvm_wrapper/RustWrapper.cpp
@@ -197,7 +197,11 @@ extern "C" LLVMContextRef LLVMRustContextCreate(bool shouldDiscardNames)
 extern "C" void LLVMRustSetNormalizedTarget(LLVMModuleRef M,
                                             const char *Triple)
 {
-  unwrap(M)->setTargetTriple(Triple::normalize(Triple));
+#if LLVM_VERSION_MAJOR >= 21
+  unwrap(M)->setTargetTriple(llvm::Triple(llvm::Triple::normalize(Triple)));
+#else
+  unwrap(M)->setTargetTriple(llvm::Triple::normalize(Triple));
+#endif
 }
 
 extern "C" void LLVMRustPrintPassTimings()
@@ -264,7 +268,11 @@ static Attribute::AttrKind fromRust(LLVMRustAttribute Kind)
   case NoAlias:
     return Attribute::NoAlias;
   case NoCapture:
+#if LLVM_VERSION_MAJOR >= 21
+    return Attribute::Captures;
+#else
     return Attribute::NoCapture;
+#endif
   case NoInline:
     return Attribute::NoInline;
   case NonNull:
@@ -307,6 +315,17 @@ static Attribute::AttrKind fromRust(LLVMRustAttribute Kind)
   report_fatal_error("bad AttributeKind");
 }
 
+// LLVM 21 replaced the enum attribute nocapture with captures(none).
+// Construct its payload explicitly for both function and call-site attributes.
+static Attribute rustAttribute(LLVMContext &Ctx, LLVMRustAttribute Kind)
+{
+#if LLVM_VERSION_MAJOR >= 21
+  if (Kind == NoCapture)
+    return Attribute::getWithCaptureInfo(Ctx, CaptureInfo::none());
+#endif
+  return Attribute::get(Ctx, fromRust(Kind));
+}
+
 extern "C" void LLVMRustAddCallSiteAttribute(LLVMValueRef Instr, unsigned Index,
                                              LLVMRustAttribute RustAttr)
 {
@@ -314,11 +333,11 @@ extern "C" void LLVMRustAddCallSiteAttribute(LLVMValueRef Instr, unsigned Index,
   CallBase *Call = unwrap<CallBase>(Instr);
   LLVMContext &Ctx = Call->getContext();
   AttrBuilder B(Ctx);
-  B.addAttribute(Attribute::get(Ctx, fromRust(RustAttr)));
+  B.addAttribute(rustAttribute(Ctx, RustAttr));
   Call->setAttributes(Call->getAttributes().addAttributesAtIndex(Ctx, Index, B));
 #else
   CallSite Call = CallSite(unwrap<Instruction>(Instr));
-  Attribute Attr = Attribute::get(Call->getContext(), fromRust(RustAttr));
+  Attribute Attr = rustAttribute(Call->getContext(), RustAttr);
 #if LLVM_VERSION_GE(5, 0)
   Call.addAttribute(Index, Attr);
 #else
@@ -412,11 +431,11 @@ extern "C" void LLVMRustAddFunctionAttribute(LLVMValueRef Fn, unsigned Index,
   Function *A = unwrap<Function>(Fn);
   LLVMContext &Ctx = A->getContext();
   AttrBuilder B(Ctx);
-  B.addAttribute(Attribute::get(Ctx, fromRust(RustAttr)));
+  B.addAttribute(rustAttribute(Ctx, RustAttr));
   A->setAttributes(A->getAttributes().addAttributesAtIndex(Ctx, Index, B));
 #else
   Function *A = unwrap<Function>(Fn);
-  Attribute Attr = Attribute::get(A->getContext(), fromRust(RustAttr));
+  Attribute Attr = rustAttribute(A->getContext(), RustAttr);
   AttrBuilder B(Attr);
 #if LLVM_VERSION_GE(5, 0)
   A->addAttributes(Index, B);
@@ -439,7 +458,7 @@ extern "C" void LLVMRustAddFunctionAttributeWithType(LLVMValueRef Fn, unsigned I
   } else if (RustAttr == ByVal) {
     B.addByValAttr(unwrap(Ty));
   } else {
-    B.addAttribute(Attribute::get(Ctx, fromRust(RustAttr)));
+    B.addAttribute(rustAttribute(Ctx, RustAttr));
   }
   A->setAttributes(A->getAttributes().addAttributesAtIndex(Ctx, Index, B));
 #else
@@ -448,7 +467,7 @@ extern "C" void LLVMRustAddFunctionAttributeWithType(LLVMValueRef Fn, unsigned I
   // to the kind-only add on legacy LLVM.
   (void)Ty;
   Function *A = unwrap<Function>(Fn);
-  Attribute Attr = Attribute::get(A->getContext(), fromRust(RustAttr));
+  Attribute Attr = rustAttribute(A->getContext(), RustAttr);
   AttrBuilder B(Attr);
 #if LLVM_VERSION_GE(5, 0)
   A->addAttributes(Index, B);
@@ -557,7 +576,7 @@ extern "C" void LLVMRustRemoveFunctionAttributes(LLVMValueRef Fn,
   F->setAttributes(PALNew);
 #else
   Function *F = unwrap<Function>(Fn);
-  Attribute Attr = Attribute::get(F->getContext(), fromRust(RustAttr));
+  Attribute Attr = rustAttribute(F->getContext(), RustAttr);
   AttrBuilder B(Attr);
   auto PAL = F->getAttributes();
 #if LLVM_VERSION_GE(5, 0)
@@ -1643,8 +1662,10 @@ extern "C" LLVMTypeKind LLVMRustGetTypeKind(LLVMTypeRef Ty)
   case Type::VectorTyID:
     return LLVMVectorTypeKind;
 #endif
+#if LLVM_VERSION_MAJOR < 21
   case Type::X86_MMXTyID:
     return LLVMX86_MMXTypeKind;
+#endif
   case Type::TokenTyID:
     return LLVMTokenTypeKind;
   }
diff --git a/crates/rustc_codegen_nvvm/rustc_llvm_wrapper/rustllvm.h b/crates/rustc_codegen_nvvm/rustc_llvm_wrapper/rustllvm.h
index 653c7818..cefd53a7 100644
--- a/crates/rustc_codegen_nvvm/rustc_llvm_wrapper/rustllvm.h
+++ b/crates/rustc_codegen_nvvm/rustc_llvm_wrapper/rustllvm.h
@@ -49,7 +49,11 @@
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetOptions.h"
 #include "llvm/Transforms/IPO.h"
+// Only the legacy pass registry uses the instrumentation umbrella header.
+// LLVM 21 removed it; modern pipelines use PassBuilder instead.
+#if LLVM_VERSION_MAJOR < 19
 #include "llvm/Transforms/Instrumentation.h"
+#endif
 #include "llvm/Transforms/Scalar.h"
 #if LLVM_VERSION_MAJOR < 19
 #include "llvm/Transforms/Vectorize.h"
diff --git a/tests/llvm-wrapper/attributes.cpp b/tests/llvm-wrapper/attributes.cpp
new file mode 100644
index 00000000..c4328d56
--- /dev/null
+++ b/tests/llvm-wrapper/attributes.cpp
@@ -0,0 +1,47 @@
+#include "rustllvm.h"
+#include "llvm/IR/Verifier.h"
+#include <cstdlib>
+
+extern "C" void LLVMRustSetNormalizedTarget(LLVMModuleRef, const char *);
+extern "C" void LLVMRustAddFunctionAttribute(LLVMValueRef, unsigned, LLVMRustAttribute);
+extern "C" void LLVMRustAddFunctionAttributeWithType(LLVMValueRef, unsigned, LLVMRustAttribute, LLVMTypeRef);
+extern "C" void LLVMRustRemoveFunctionAttributes(LLVMValueRef, unsigned, LLVMRustAttribute);
+extern "C" void LLVMRustAddCallSiteAttribute(LLVMValueRef, unsigned, LLVMRustAttribute);
+
+// These tests never request Rust-owned string output.
+extern "C" void LLVMRustStringWriteImpl(RustStringRef, const char *, size_t) {
+  std::abort();
+}
+
+static void require(bool Condition) {
+  if (!Condition)
+    std::abort();
+}
+
+int main() {
+  llvm::LLVMContext C;
+  llvm::Module M("wrapper-test", C);
+  LLVMRustSetNormalizedTarget(llvm::wrap(&M), "nvptx64-nvidia-cuda");
+  require(M.getTargetTriple().str() == "nvptx64-nvidia-cuda");
+  auto *Ptr = llvm::PointerType::get(C, 0);
+  auto *FT = llvm::FunctionType::get(llvm::Type::getVoidTy(C), {Ptr}, false);
+  auto *F = llvm::Function::Create(FT, llvm::GlobalValue::ExternalLinkage, "callee", M);
+  auto CheckCapture = [](llvm::Attribute A) {
+    require(A.isValid());
+    require(A.getCaptureInfo() == llvm::CaptureInfo::none());
+    require(A.getAsString() == "captures(none)");
+  };
+  LLVMRustAddFunctionAttribute(llvm::wrap(F), 1, NoCapture);
+  CheckCapture(F->getAttributes().getParamAttr(0, llvm::Attribute::Captures));
+  LLVMRustRemoveFunctionAttributes(llvm::wrap(F), 1, NoCapture);
+  require(!F->hasParamAttribute(0, llvm::Attribute::Captures));
+  LLVMRustAddFunctionAttributeWithType(llvm::wrap(F), 1, NoCapture, llvm::wrap(Ptr));
+  CheckCapture(F->getAttributes().getParamAttr(0, llvm::Attribute::Captures));
+  auto *Caller = llvm::Function::Create(FT, llvm::GlobalValue::ExternalLinkage, "caller", M);
+  llvm::IRBuilder<> B(llvm::BasicBlock::Create(C, "entry", Caller));
+  auto *Call = B.CreateCall(F, {Caller->getArg(0)});
+  LLVMRustAddCallSiteAttribute(llvm::wrap(Call), 1, NoCapture);
+  CheckCapture(Call->getAttributes().getParamAttr(0, llvm::Attribute::Captures));
+  B.CreateRetVoid();
+  require(!llvm::verifyModule(M, &llvm::errs()));
+}
diff --git a/tests/llvm-wrapper/run.py b/tests/llvm-wrapper/run.py
new file mode 100644
index 00000000..a70b88e4
--- /dev/null
+++ b/tests/llvm-wrapper/run.py
@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+"""Compile both LLVM 21 wrappers and check attribute semantics without CUDA."""
+import os
+from pathlib import Path
+import shlex
+import subprocess
+import tempfile
+
+root = Path(__file__).resolve().parents[2]
+wrapper = root / 'crates/rustc_codegen_nvvm/rustc_llvm_wrapper'
+config = os.environ.get('LLVM_CONFIG_21', 'llvm-config-21')
+
+
+def llvm_flags(*args):
+    return shlex.split(subprocess.check_output([config, *args], text=True))
+
+
+version = subprocess.check_output([config, '--version'], text=True).strip()
+if not version.startswith('21.'):
+    raise RuntimeError(f'expected LLVM 21, got {version}')
+cxx = shlex.split(os.environ.get('CXX', 'c++'))
+flags = [*llvm_flags('--cxxflags'), '-DLLVM_VERSION_MAJOR=21',
+         '-DLLVM_COMPONENT_NVPTX', '-I' + str(wrapper)]
+with tempfile.TemporaryDirectory(prefix='llvm-wrapper-test-') as directory:
+    out = Path(directory)
+    for source in ('RustWrapper', 'PassWrapper'):
+        subprocess.run([*cxx, *flags, '-c', str(wrapper / (source + '.cpp')),
+                        '-o', str(out / (source + '.o'))], check=True)
+    executable = out / 'attributes'
+    subprocess.run([*cxx, *flags, str(Path(__file__).with_name('attributes.cpp')),
+                    str(out / 'RustWrapper.o'),
+                    *llvm_flags('--ldflags', '--libs', '--system-libs'),
+                    '-o', str(executable)], check=True)
+    subprocess.run([str(executable)], check=True)
+print('LLVM wrapper compilation and capture attribute checks passed')

From c62ea0fed035088a71154f41ad13aa003845a8b4 Mon Sep 17 00:00:00 2001
From: Brandon Ros <brandonros1@gmail.com>
Date: Sun, 13 Sep 2026 23:32:48 -0400
Subject: [PATCH 09/10] fix(nvvm): restore kernel annotations after LLVM 21
 bitcode upgrade

---
 .../rustc_llvm_wrapper/RustWrapper.cpp        | 33 ++++++++++++++++++
 crates/rustc_codegen_nvvm/src/llvm.rs         |  2 ++
 crates/rustc_codegen_nvvm/src/nvvm.rs         |  1 +
 tests/llvm-wrapper/attributes.cpp             | 34 +++++++++++++++++++
 4 files changed, 70 insertions(+)

diff --git a/crates/rustc_codegen_nvvm/rustc_llvm_wrapper/RustWrapper.cpp b/crates/rustc_codegen_nvvm/rustc_llvm_wrapper/RustWrapper.cpp
index a5fd1fb4..34568c15 100644
--- a/crates/rustc_codegen_nvvm/rustc_llvm_wrapper/RustWrapper.cpp
+++ b/crates/rustc_codegen_nvvm/rustc_llvm_wrapper/RustWrapper.cpp
@@ -2284,3 +2284,36 @@ extern "C" LLVMValueRef LLVMBuildInBoundsGEP2(LLVMBuilderRef B, LLVMTypeRef Ty,
 }
 
 #endif
+
+// LLVM 21 upgrades NVVM kernel metadata to PTX_Kernel while reading bitcode.
+// Keep the legacy annotations used by internalization and the NVVM handoff.
+extern "C" void LLVMRustRestoreNvvmKernelAnnotations(LLVMModuleRef Mod) {
+#if LLVM_VERSION_MAJOR >= 21
+  Module &M = *unwrap(Mod);
+  auto *Annotations = M.getOrInsertNamedMetadata("nvvm.annotations");
+  for (Function &F : M) {
+    if (F.getCallingConv() != CallingConv::PTX_Kernel)
+      continue;
+    bool Found = false;
+    for (MDNode *Node : Annotations->operands()) {
+      if (Node->getNumOperands() < 2)
+        continue;
+      auto *Value = dyn_cast_or_null<ValueAsMetadata>(Node->getOperand(0));
+      auto *Kind = dyn_cast_or_null<MDString>(Node->getOperand(1));
+      if (Value && Value->getValue() == &F && Kind && Kind->getString() == "kernel") {
+        Found = true;
+        break;
+      }
+    }
+    if (!Found) {
+      Metadata *Fields[] = {ValueAsMetadata::get(&F),
+                            MDString::get(M.getContext(), "kernel"),
+                            ConstantAsMetadata::get(ConstantInt::get(
+                                Type::getInt32Ty(M.getContext()), 1))};
+      Annotations->addOperand(MDNode::get(M.getContext(), Fields));
+    }
+  }
+#else
+  (void)Mod;
+#endif
+}
diff --git a/crates/rustc_codegen_nvvm/src/llvm.rs b/crates/rustc_codegen_nvvm/src/llvm.rs
index cfb02efc..a51641eb 100644
--- a/crates/rustc_codegen_nvvm/src/llvm.rs
+++ b/crates/rustc_codegen_nvvm/src/llvm.rs
@@ -2216,6 +2216,8 @@ unsafe extern "C" {
     pub(crate) fn LLVMRustPrepareThinLTOResolveWeak(Data: &ThinLTOData, Module: &Module) -> bool;
     pub(crate) fn LLVMRustPrepareThinLTOInternalize(Data: &ThinLTOData, Module: &Module) -> bool;
     pub(crate) fn LLVMRustFreeThinLTOData(Data: &'static mut ThinLTOData);
+    pub(crate) fn LLVMRustRestoreNvvmKernelAnnotations(M: &Module);
+
     pub(crate) fn LLVMRustParseBitcodeForLTO(
         Context: &Context,
         Data: *const u8,
diff --git a/crates/rustc_codegen_nvvm/src/nvvm.rs b/crates/rustc_codegen_nvvm/src/nvvm.rs
index 70e4e858..8ecff0cc 100644
--- a/crates/rustc_codegen_nvvm/src/nvvm.rs
+++ b/crates/rustc_codegen_nvvm/src/nvvm.rs
@@ -93,6 +93,7 @@ pub fn codegen_bitcode_modules(
 
     let module = merge_llvm_modules(modules, llcx);
     unsafe {
+        LLVMRustRestoreNvvmKernelAnnotations(module);
         internalize_pass(module, llcx);
         dce_pass(module);
 
diff --git a/tests/llvm-wrapper/attributes.cpp b/tests/llvm-wrapper/attributes.cpp
index c4328d56..0a6d6ab7 100644
--- a/tests/llvm-wrapper/attributes.cpp
+++ b/tests/llvm-wrapper/attributes.cpp
@@ -1,6 +1,8 @@
 #include "rustllvm.h"
 #include "llvm/IR/Verifier.h"
 #include <cstdlib>
+#include "llvm/AsmParser/Parser.h"
+#include "llvm/Transforms/IPO/GlobalDCE.h"
 
 extern "C" void LLVMRustSetNormalizedTarget(LLVMModuleRef, const char *);
 extern "C" void LLVMRustAddFunctionAttribute(LLVMValueRef, unsigned, LLVMRustAttribute);
@@ -18,7 +20,39 @@ static void require(bool Condition) {
     std::abort();
 }
 
+extern "C" void LLVMRustRestoreNvvmKernelAnnotations(LLVMModuleRef);
+
+static void kernelRetention() {
+  llvm::LLVMContext C;
+  llvm::SMDiagnostic Error;
+  auto M = llvm::parseAssemblyString(R"(
+    target triple = "nvptx64-nvidia-cuda"
+    define void @entry(ptr %out) { store i32 42, ptr %out ret void }
+    define internal void @dead() { ret void }
+    !nvvm.annotations = !{!0}
+    !0 = !{ptr @entry, !"kernel", i32 1}
+  )", Error, C);
+  require(bool(M));
+  auto *F = M->getFunction("entry");
+  require(F->getCallingConv() == llvm::CallingConv::PTX_Kernel);
+  require(M->getNamedMetadata("nvvm.annotations")->getNumOperands() == 0);
+  LLVMRustRestoreNvvmKernelAnnotations(llvm::wrap(M.get()));
+  LLVMRustRestoreNvvmKernelAnnotations(llvm::wrap(M.get()));
+  auto *Annotations = M->getNamedMetadata("nvvm.annotations");
+  require(Annotations->getNumOperands() == 1);
+  auto *Node = Annotations->getOperand(0);
+  require(llvm::cast<llvm::ValueAsMetadata>(Node->getOperand(0))->getValue() == F);
+  require(llvm::cast<llvm::MDString>(Node->getOperand(1))->getString() == "kernel");
+  require(llvm::mdconst::extract<llvm::ConstantInt>(Node->getOperand(2))->equalsInt(1));
+  llvm::ModuleAnalysisManager AM;
+  llvm::GlobalDCEPass().run(*M, AM);
+  require(M->getFunction("entry") == F);
+  require(M->getFunction("dead") == nullptr);
+  require(!llvm::verifyModule(*M, &llvm::errs()));
+}
+
 int main() {
+  kernelRetention();
   llvm::LLVMContext C;
   llvm::Module M("wrapper-test", C);
   LLVMRustSetNormalizedTarget(llvm::wrap(&M), "nvptx64-nvidia-cuda");

From 4decb7c5c1937bf1be90f364ee08d74f68931b90 Mon Sep 17 00:00:00 2001
From: Brandon Ros <brandonros1@gmail.com>
Date: Mon, 14 Sep 2026 21:10:19 -0400
Subject: [PATCH 10/10] Use configurable LLVM archive URLs in CI and document
 the override

---
 .github/workflows/ci_linux.yml     |  3 ++-
 crates/rustc_codegen_nvvm/build.rs |  2 +-
 guide/src/guide/getting_started.md | 22 ++++++++++++++++++++++
 3 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci_linux.yml b/.github/workflows/ci_linux.yml
index 19342483..fbd3fa11 100644
--- a/.github/workflows/ci_linux.yml
+++ b/.github/workflows/ci_linux.yml
@@ -131,10 +131,11 @@ jobs:
         if: matrix.variance.llvm == 21
         env:
           LLVM_ARCH: ${{ matrix.variance.artifact }}
+          PREBUILT_LLVM_URL: ${{ vars.PREBUILT_LLVM_URL || 'https://github.com/brandonros/rustc_codegen_nvvm-llvm/releases/download/llvm-21.1.8' }}
         run: |
           mkdir -p llvm-prebuilt
           curl --fail --location --retry 3 \
-            "https://github.com/Rust-GPU/rustc_codegen_nvvm-llvm/releases/download/llvm-21.1.8/$LLVM_ARCH.tar.xz" \
+            "${PREBUILT_LLVM_URL%/}/$LLVM_ARCH.tar.xz" \
             -o "llvm-prebuilt/$LLVM_ARCH.tar.xz"
           tar -xf "llvm-prebuilt/$LLVM_ARCH.tar.xz" -C llvm-prebuilt
           docker exec "$CONTAINER_NAME" bash -lc 'set -euo pipefail
diff --git a/crates/rustc_codegen_nvvm/build.rs b/crates/rustc_codegen_nvvm/build.rs
index 59fe4da0..55ac3965 100644
--- a/crates/rustc_codegen_nvvm/build.rs
+++ b/crates/rustc_codegen_nvvm/build.rs
@@ -121,7 +121,7 @@ fn target_to_llvm_prebuilt(target: &str) -> String {
 
 fn download_prebuilt_llvm(target: &str, base_url: &str) -> PathBuf {
     let prebuilt_name = target_to_llvm_prebuilt(target);
-    let url = format!("{base_url}{prebuilt_name}");
+    let url = format!("{}/{prebuilt_name}", base_url.trim_end_matches('/'));
 
     println!("cargo:warning=Downloading prebuilt LLVM from {url}");
 
diff --git a/guide/src/guide/getting_started.md b/guide/src/guide/getting_started.md
index d8a0b7b1..b0cf621d 100644
--- a/guide/src/guide/getting_started.md
+++ b/guide/src/guide/getting_started.md
@@ -24,6 +24,28 @@ files] are a good starting point.
 
 [Docker files]: https://github.com/Rust-GPU/rust-cuda/tree/main/container
 
+## LLVM 21 download override
+
+With the `llvm21` feature, set `LLVM_CONFIG_21` to an installed LLVM 21
+`llvm-config`, or let the backend download an archive. `PREBUILT_LLVM_URL`
+overrides the archive directory; a trailing slash is optional. The backend
+appends `linux-x86_64.tar.xz`, `linux-aarch64.tar.xz`, or
+`windows-x86_64.tar.xz` for the host platform.
+
+```sh
+export PREBUILT_LLVM_URL=https://github.com/brandonros/rustc_codegen_nvvm-llvm/releases/download/llvm-21.1.8
+USE_PREBUILT_LLVM=1 cargo build -p rustc_codegen_nvvm --features llvm21
+```
+
+`USE_PREBUILT_LLVM=1` bypasses local LLVM discovery. Without it, the URL is
+used only if local discovery fails. Select archives matching the backend's
+LLVM version; the override also applies to LLVM 7 builds.
+
+Linux CI accepts the same directory through the repository Actions variable
+`PREBUILT_LLVM_URL`. Its LLVM 21 jobs default to the release above, whose
+three archives match the artifacts from
+[run 34781677658](https://github.com/brandonros/rustc_codegen_nvvm-llvm/actions/runs/34781677658).
+
 ## CUDA basics
 
 GPU kernels are functions launched from the CPU that run on the GPU. They do not have a return