Upgrade modern backend to LLVM 21.1.8 and CUDA 13.3

.github/ISSUE_TEMPLATE/bug_report.md

@@ -22,7 +22,7 @@ A clear and concise description of what you expected to happen.
 
 - OS: [e.g. Windows 11, Ubuntu 22.04]
 - GPU: [e.g. RTX 3060]
-- CUDA Toolkit version: [e.g. 13.2]
+- CUDA Toolkit version: [e.g. 13.3]
 - cuDNN version (if applicable): [e.g. 9.x]
 - Rust toolchain: [output of `rustc --version`]
 

.github/workflows/ci_linux.yml

@@ -1,6 +1,7 @@
 name: CI on Linux
 
 on:
+  workflow_dispatch:
   pull_request:
     paths-ignore:
       - "**.md"
@@ -25,21 +26,38 @@ jobs:
           - name: Ubuntu-24.04 / CUDA-12.8.1 / x86_64
             image: "ghcr.io/rust-gpu/rust-cuda-ubuntu24-cuda12:latest"
             runner: ubuntu-latest
+            llvm: 7
           - name: Ubuntu-24.04 / CUDA-12.8.1 / ARM64
             image: "ghcr.io/rust-gpu/rust-cuda-ubuntu24-cuda12:latest"
             runner: ubuntu-24.04-arm
-          - name: Ubuntu-24.04 / CUDA-13.0.2 / x86_64
+            llvm: 7
+          - name: Ubuntu-24.04 / CUDA-13.3.1 / x86_64
             image: "ghcr.io/rust-gpu/rust-cuda-ubuntu24-cuda13:latest"
             runner: ubuntu-latest
-          - name: Ubuntu-24.04 / CUDA-13.0.2 / ARM64
+            llvm: 7
+          - name: Ubuntu-24.04 / CUDA-13.3.1 / ARM64
             image: "ghcr.io/rust-gpu/rust-cuda-ubuntu24-cuda13:latest"
             runner: ubuntu-24.04-arm
+            llvm: 7
           - name: RockyLinux-9 / CUDA-12.8.1 / x86_64
             image: "ghcr.io/rust-gpu/rust-cuda-rockylinux9-cuda12:latest"
             runner: ubuntu-latest
-          - name: RockyLinux-9 / CUDA-13.0.2 / x86_64
+            llvm: 7
+          - name: RockyLinux-9 / CUDA-13.3.1 / x86_64
             image: "ghcr.io/rust-gpu/rust-cuda-rockylinux9-cuda13:latest"
             runner: ubuntu-latest
+            llvm: 7
+
+          - name: Ubuntu-24.04 / CUDA-13.3.1 / LLVM-21 / x86_64
+            image: "nvidia/cuda:13.3.1-devel-ubuntu24.04"
+            runner: ubuntu-24.04
+            llvm: 21
+            artifact: linux-x86_64
+          - name: Ubuntu-24.04 / CUDA-13.3.1 / LLVM-21 / ARM64
+            image: "nvidia/cuda:13.3.1-devel-ubuntu24.04"
+            runner: ubuntu-24.04-arm
+            llvm: 21
+            artifact: linux-aarch64
 
     steps:
       - name: Free up space
@@ -109,32 +127,57 @@ jobs:
             sleep infinity
           docker start "$CONTAINER_NAME"
 
+      - name: Install LLVM 21 release and build dependencies
+        if: matrix.variance.llvm == 21
+        env:
+          LLVM_ARCH: ${{ matrix.variance.artifact }}
+          PREBUILT_LLVM_URL: ${{ vars.PREBUILT_LLVM_URL || 'https://github.com/brandonros/rustc_codegen_nvvm-llvm/releases/download/llvm-21.1.8' }}
+        run: |
+          mkdir -p llvm-prebuilt
+          curl --fail --location --retry 3 \
+            "${PREBUILT_LLVM_URL%/}/$LLVM_ARCH.tar.xz" \
+            -o "llvm-prebuilt/$LLVM_ARCH.tar.xz"
+          tar -xf "llvm-prebuilt/$LLVM_ARCH.tar.xz" -C llvm-prebuilt
+          docker exec "$CONTAINER_NAME" bash -lc 'set -euo pipefail
+            apt-get update
+            DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+              build-essential libclang-dev python3 curl ca-certificates git pkg-config libssl-dev \
+              libffi-dev libedit-dev libxml2-dev libtinfo-dev zlib1g-dev xz-utils
+            curl --proto "=https" --tlsv1.2 -sSf https://sh.rustup.rs | \
+              sh -s -- -y --profile minimal --default-toolchain none
+          '
+
       - name: Verify CUDA, Rust installation
         run: |
           docker exec "$CONTAINER_NAME" bash -lc 'set -euo pipefail
             nvcc --version
+            if [ -f /root/.cargo/env ]; then source /root/.cargo/env; fi
             rustup show
           '
 
       - name: Rustfmt
+        if: matrix.variance.llvm == 7
         run: |
           docker exec "$CONTAINER_NAME" bash -lc 'set -euo pipefail
             cargo fmt --all -- --check
           '
 
       - name: Build all bindings
+        if: matrix.variance.llvm == 7
         run: |
           docker exec "$CONTAINER_NAME" bash -lc 'set -euo pipefail
             cargo build --all-features -p cust_raw
           '
 
       - name: Build workspace
+        if: matrix.variance.llvm == 7
         run: |
           docker exec "$CONTAINER_NAME" bash -lc 'set -euo pipefail
             cargo build
           '
 
       - name: Clippy
+        if: matrix.variance.llvm == 7
         run: |
           docker exec "$CONTAINER_NAME" bash -lc 'set -euo pipefail
             export RUSTFLAGS=-Dwarnings
@@ -143,6 +186,7 @@ jobs:
 
       # Exclude crates with tests that require an NVIDIA GPU: blastoff, cudnn, cust.
       - name: Test
+        if: matrix.variance.llvm == 7
         run: |
           docker exec "$CONTAINER_NAME" bash -lc 'set -euo pipefail
             export RUSTFLAGS=-Dwarnings
@@ -152,12 +196,13 @@ jobs:
               --exclude cust
           '
 
-      # The `llvm19` feature on `nvvm` / `cuda_builder` / `rustc_codegen_nvvm` requires
-      # an LLVM 19 toolchain that isn't in the CI image, so we can't run a single
+      # The `llvm21` feature on `nvvm` / `cuda_builder` / `rustc_codegen_nvvm` requires
+      # an LLVM 21 toolchain that isn't in the CI image, so we can't run a single
       # `--all-features` pass over the whole workspace. Doc those three crates with
       # default features (the LLVM 7 path the CI image already supports) and the rest
       # of the workspace with `--all-features`.
       - name: Check documentation
+        if: matrix.variance.llvm == 7
         run: |
           docker exec "$CONTAINER_NAME" bash -lc 'set -euo pipefail
             export RUSTDOCFLAGS=-Dwarnings
@@ -167,7 +212,38 @@ jobs:
               --document-private-items --no-deps
           '
 
+      # Compile both the host and device code; execution needs an NVIDIA GPU.
+      - name: Build vector-add with LLVM 21
+        if: matrix.variance.llvm == 21
+        env:
+          LLVM_ARCH: ${{ matrix.variance.artifact }}
+        run: |
+          docker exec \
+            -e LLVM_CONFIG_21="/workspace/llvm-prebuilt/$LLVM_ARCH/bin/llvm-config" \
+            -e LLVM_LINK_STATIC=1 \
+            -e RUST_CUDA_DUMP_FINAL_MODULE=1 \
+            -e RUST_CUDA_EMIT_LLVM_IR=1 \
+            "$CONTAINER_NAME" bash -lc 'set -euo pipefail
+              source /root/.cargo/env
+              export LD_LIBRARY_PATH="$(dirname "$LLVM_CONFIG_21")/../lib:/usr/local/cuda/nvvm/lib64:${LD_LIBRARY_PATH:-}"
+              export LIBRARY_PATH="/usr/local/cuda/lib64/stubs:${LIBRARY_PATH:-}"
+              test "$("$LLVM_CONFIG_21" --version)" = 21.1.8
+              python3 tests/llvm-wrapper/run.py
+              cargo build -p vecadd --features llvm21
+            '
+
+      - name: Upload LLVM 21 PTX and IR
+        if: always() && matrix.variance.llvm == 21
+        uses: actions/upload-artifact@v4
+        with:
+          name: vecadd-llvm21-${{ matrix.variance.artifact }}
+          path: |
+            target/**/kernels.ptx
+            target/**/final-module.ll
+          if-no-files-found: warn
+
       - name: Stop build container
+        if: always()
         run: |
           docker rm -f "$CONTAINER_NAME" || true
 

.github/workflows/ci_windows.yml

@@ -42,7 +42,7 @@ jobs:
               ]
           - os: windows-latest
             target: x86_64-pc-windows-msvc
-            cuda: "13.0.2"
+            cuda: "13.3.1"
             nvvm-dll-dir: "nvvm\\bin\\x64"
             sub-packages:
               [
@@ -65,7 +65,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Install CUDA
-        uses: Jimver/cuda-toolkit@v0.2.29
+        uses: Jimver/cuda-toolkit@v0.2.36
         id: cuda-toolkit
         with:
           cuda: ${{ matrix.cuda }}
@@ -123,8 +123,8 @@ jobs:
             --exclude blastoff --exclude cudnn --exclude cudnn-sys --exclude cust
 
       # Exclude crates that require cuDNN, not available on Windows CI: cudnn, cudnn-sys.
-      # The `llvm19` feature on `nvvm` / `cuda_builder` / `rustc_codegen_nvvm` requires
-      # an LLVM 19 toolchain that isn't in the CI image, so we can't run a single
+      # The `llvm21` feature on `nvvm` / `cuda_builder` / `rustc_codegen_nvvm` requires
+      # an LLVM 21 toolchain that isn't in the CI image, so we can't run a single
       # `--all-features` pass over the whole workspace. Doc those three crates with
       # default features (the LLVM 7 path the CI image already supports) and the rest
       # of the workspace with `--all-features`.

.github/workflows/container_images.yml

@@ -33,16 +33,16 @@ jobs:
           - name: Ubuntu-24.04/CUDA-12.8.1
             image: "rust-cuda-ubuntu24-cuda12"
             dockerfile: ./container/ubuntu24-cuda12/Dockerfile
-          - name: Ubuntu-24.04/CUDA-13.0.2
+          - name: Ubuntu-24.04/CUDA-13.3.1
             image: "rust-cuda-ubuntu24-cuda13"
             dockerfile: ./container/ubuntu24-cuda13/Dockerfile
-          - name: Ubuntu-24.04/CUDA-13.2.1/LLVM-19.1.7
-            image: "rust-cuda-ubuntu24-cuda13-llvm19"
-            dockerfile: ./container/ubuntu24-cuda13-llvm19/Dockerfile
+          - name: Ubuntu-24.04/CUDA-13.3.1/LLVM-21.1.8
+            image: "rust-cuda-ubuntu24-cuda13-llvm21"
+            dockerfile: ./container/ubuntu24-cuda13-llvm21/Dockerfile
           - name: RockyLinux-9/CUDA-12.8.1
             image: "rust-cuda-rockylinux9-cuda12"
             dockerfile: ./container/rockylinux9-cuda12/Dockerfile
-          - name: RockyLinux-9/CUDA-13.0.2
+          - name: RockyLinux-9/CUDA-13.3.1
             image: "rust-cuda-rockylinux9-cuda13"
             dockerfile: ./container/rockylinux9-cuda13/Dockerfile
     steps:
@@ -161,13 +161,13 @@ jobs:
         variance:
           - name: Ubuntu-24.04/CUDA-12.8.1
             image: "rust-cuda-ubuntu24-cuda12"
-          - name: Ubuntu-24.04/CUDA-13.0.2
+          - name: Ubuntu-24.04/CUDA-13.3.1
             image: "rust-cuda-ubuntu24-cuda13"
-          - name: Ubuntu-24.04/CUDA-13.2.1/LLVM-19.1.7
-            image: "rust-cuda-ubuntu24-cuda13-llvm19"
+          - name: Ubuntu-24.04/CUDA-13.3.1/LLVM-21.1.8
+            image: "rust-cuda-ubuntu24-cuda13-llvm21"
           - name: RockyLinux-9/CUDA-12.8.1
             image: "rust-cuda-rockylinux9-cuda12"
-          - name: RockyLinux-9/CUDA-13.0.2
+          - name: RockyLinux-9/CUDA-13.3.1
             image: "rust-cuda-rockylinux9-cuda13"
     steps:
       - name: Set lowercase repo owner

CONTRIBUTING.md

@@ -33,7 +33,7 @@ For questions, clarifications, and general help:
 ### Windows-Specific Notes
 
 - Ensure the CUDA Toolkit `bin` directory is on your `PATH` (e.g.
-  `C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.2\bin`).
+  `C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v13.3\bin`).
 - The MSVC build tools are required. Install via
   [Visual Studio Build Tools](https://visualstudio.microsoft.com/downloads/) with the
   "Desktop development with C++" workload.

container/rockylinux9-cuda13/Dockerfile

@@ -1,4 +1,4 @@
-FROM nvcr.io/nvidia/cuda:13.0.2-cudnn-devel-rockylinux9 AS llvm-builder
+FROM nvcr.io/nvidia/cuda:13.3.1-cudnn-devel-rockylinux9 AS llvm-builder
 
 RUN dnf -y install \
     --nobest \
@@ -51,7 +51,7 @@ RUN curl -sSf -L -O https://github.com/llvm/llvm-project/releases/download/llvmo
     rm -rf llvm-7.1.0.src* && \
     dnf clean all
 
-FROM nvcr.io/nvidia/cuda:13.0.2-cudnn-devel-rockylinux9
+FROM nvcr.io/nvidia/cuda:13.3.1-cudnn-devel-rockylinux9
 
 RUN dnf -y install \
     --nobest \

container/ubuntu24-cuda13-llvm19/Dockerfile → container/ubuntu24-cuda13-llvm21/Dockerfile

@@ -1,4 +1,4 @@
-FROM nvcr.io/nvidia/cuda:13.2.1-cudnn-devel-ubuntu24.04 AS llvm-builder
+FROM nvcr.io/nvidia/cuda:13.3.1-cudnn-devel-ubuntu24.04 AS llvm-builder
 
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -qq -y install \
     build-essential \
@@ -18,13 +18,13 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -qq -y install \
     zlib1g-dev && \
     rm -rf /var/lib/apt/lists/*
 
-WORKDIR /data/llvm19
+WORKDIR /data/llvm21
 
-# Download and build LLVM 19.1.7 (the active LLVM 19 pin used by `rustc_codegen_nvvm`).
+# Download and build LLVM 21.1.8 (the active LLVM 21 pin used by `rustc_codegen_nvvm`).
 # LLVM 8+ ships as a monorepo tarball; cmake source root is the `llvm/` subdir.
-RUN curl -sSf -L -O https://github.com/llvm/llvm-project/releases/download/llvmorg-19.1.7/llvm-project-19.1.7.src.tar.xz && \
-    tar -xf llvm-project-19.1.7.src.tar.xz && \
-    cd llvm-project-19.1.7.src && \
+RUN curl -sSf -L -O https://github.com/llvm/llvm-project/releases/download/llvmorg-21.1.8/llvm-project-21.1.8.src.tar.xz && \
+    tar -xf llvm-project-21.1.8.src.tar.xz && \
+    cd llvm-project-21.1.8.src && \
     mkdir build && cd build && \
     ARCH=$(dpkg --print-architecture) && \
     if [ "$ARCH" = "amd64" ]; then \
@@ -44,14 +44,14 @@ RUN curl -sSf -L -O https://github.com/llvm/llvm-project/releases/download/llvmo
         -DLLVM_INCLUDE_BENCHMARKS=OFF \
         -DLLVM_ENABLE_ZLIB=ON \
         -DLLVM_ENABLE_TERMINFO=ON \
-        -DCMAKE_INSTALL_PREFIX=/opt/llvm-19 \
+        -DCMAKE_INSTALL_PREFIX=/opt/llvm-21 \
         ../llvm && \
     ninja -j$(nproc) && \
     ninja install && \
     cd ../.. && \
-    rm -rf llvm-project-19.1.7.src*
+    rm -rf llvm-project-21.1.8.src*
 
-FROM nvcr.io/nvidia/cuda:13.2.1-cudnn-devel-ubuntu24.04
+FROM nvcr.io/nvidia/cuda:13.3.1-cudnn-devel-ubuntu24.04
 
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -qq -y install \
     build-essential \
@@ -71,10 +71,10 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -qq -y install \
     libxrandr-dev && \
     rm -rf /var/lib/apt/lists/*
 
-COPY --from=llvm-builder /opt/llvm-19 /opt/llvm-19
-RUN ln -s /opt/llvm-19/bin/llvm-config /usr/bin/llvm-config && \
-    ln -s /opt/llvm-19/bin/llvm-config /usr/bin/llvm-config-19 && \
-    ln -s /opt/llvm-19/bin/llvm-as /usr/bin/llvm-as-19
+COPY --from=llvm-builder /opt/llvm-21 /opt/llvm-21
+RUN ln -s /opt/llvm-21/bin/llvm-config /usr/bin/llvm-config && \
+    ln -s /opt/llvm-21/bin/llvm-config /usr/bin/llvm-config-21 && \
+    ln -s /opt/llvm-21/bin/llvm-as /usr/bin/llvm-as-21
 
 # Get Rust (install rustup; toolchain installed from rust-toolchain.toml below)
 RUN curl -sSf -L https://sh.rustup.rs | bash -s -- -y --profile minimal --default-toolchain none
@@ -85,15 +85,15 @@ WORKDIR /data/rust-cuda
 RUN --mount=type=bind,source=rust-toolchain.toml,target=/data/rust-cuda/rust-toolchain.toml \
     rustup show
 
-# Add nvvm + LLVM 19 dylib to the runtime linker path.
-ENV LD_LIBRARY_PATH="/opt/llvm-19/lib:/usr/local/cuda/nvvm/lib64:${LD_LIBRARY_PATH}"
+# Add nvvm + LLVM 21 dylib to the runtime linker path.
+ENV LD_LIBRARY_PATH="/opt/llvm-21/lib:/usr/local/cuda/nvvm/lib64:${LD_LIBRARY_PATH}"
 
-# `rustc_codegen_nvvm`'s build.rs probes `LLVM_CONFIG_19` to locate the LLVM 19
-# toolchain when the `llvm19` cargo feature is on. The feature itself is gated;
+# `rustc_codegen_nvvm`'s build.rs probes `LLVM_CONFIG_21` to locate the LLVM 21
+# toolchain when the `llvm21` cargo feature is on. The feature itself is gated;
 # downstream crates that depend on `cuda_builder` must build with
-# `--features llvm19` for this to take effect — that propagates through to
-# `nvvm/llvm19` (default `NvvmArch` = Blackwell) and `rustc_codegen_nvvm/llvm19`
-# (LLVM 19 codegen path) per crates/cuda_builder/Cargo.toml.
-ENV LLVM_CONFIG_19=/opt/llvm-19/bin/llvm-config
+# `--features llvm21` for this to take effect — that propagates through to
+# `nvvm/llvm21` (default `NvvmArch` = Blackwell) and `rustc_codegen_nvvm/llvm21`
+# (LLVM 21 codegen path) per crates/cuda_builder/Cargo.toml.
+ENV LLVM_CONFIG_21=/opt/llvm-21/bin/llvm-config
 ENV LLVM_LINK_STATIC=1
 ENV RUST_LOG=info

container/ubuntu24-cuda13/Dockerfile

@@ -1,4 +1,4 @@
-FROM nvcr.io/nvidia/cuda:13.0.2-cudnn-devel-ubuntu24.04 AS llvm-builder
+FROM nvcr.io/nvidia/cuda:13.3.1-cudnn-devel-ubuntu24.04 AS llvm-builder
 
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -qq -y install \
     build-essential \
@@ -50,7 +50,7 @@ RUN curl -sSf -L -O https://github.com/llvm/llvm-project/releases/download/llvmo
     cd ../.. && \
     rm -rf llvm-7.1.0.src*
 
-FROM nvcr.io/nvidia/cuda:13.0.2-cudnn-devel-ubuntu24.04
+FROM nvcr.io/nvidia/cuda:13.3.1-cudnn-devel-ubuntu24.04
 
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -qq -y install \
     build-essential \

crates/cuda_builder/Cargo.toml

@@ -16,12 +16,12 @@ default = []
 # HACK(see rust-gpu/spirv-builder): use `dep:` to avoid Cargo auto-creating a feature
 # with the dependency name. Consumers must explicitly opt-in to compiling the backend.
 rustc_codegen_nvvm = ["dep:rustc_codegen_nvvm"]
-# Build the backend against LLVM 19 instead of LLVM 7. Propagates to `nvvm` (which
+# Build the backend against LLVM 21 instead of LLVM 7. Propagates to `nvvm` (which
 # uses it to flip the default `NvvmArch` to `Compute100`) and, when the optional
 # `rustc_codegen_nvvm` dep is also enabled, to `rustc_codegen_nvvm` itself. Even
 # when the optional dep is disabled, the build script's nested `cargo build -p
-# rustc_codegen_nvvm` reads `cfg(feature = "llvm19")` here and forwards it.
-llvm19 = ["nvvm/llvm19", "rustc_codegen_nvvm?/llvm19"]
+# rustc_codegen_nvvm` reads `cfg(feature = "llvm21")` here and forwards it.
+llvm21 = ["nvvm/llvm21", "rustc_codegen_nvvm?/llvm21"]
 
 [dependencies]
 rustc_codegen_nvvm = { version = "0.3", path = "../rustc_codegen_nvvm", optional = true }