sync CCM<>CSI test cluster setup, add integration tests on schedule

.github/workflows/csi-integration-tests.yml

@@ -0,0 +1,172 @@
+name: CSI Integration Tests
+
+on:
+  push:
+    branches:
+      - master
+
+  pull_request:
+
+  # Allow to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+  # Run this regularly, to get integration tests results against new
+  # Kubernetes releases.
+  schedule:
+    - cron: '15 3 * * *'
+
+permissions:
+  contents: read
+
+jobs:
+  test-matrix:
+    name: "Get Kubernetes Releases"
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - name: "Generate Test Matrix"
+        id: list
+        run: 'echo "tests=$(helpers/test-matrix)" >> $GITHUB_OUTPUT'
+
+    outputs:
+      tests: ${{ steps.list.outputs.tests }}
+
+  build-image:
+    name: "Build Container Image"
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - name: Evaluate image name
+        run: 'helpers/image-from-ref >> $GITHUB_ENV'
+
+      - name: Extract version
+        run: echo "VERSION=$(cat VERSION)" >> $GITHUB_ENV
+
+      - name: Build image
+        run: make build
+
+      - name: Export image
+        run: 'docker image save "$IMAGE" -o image.tar'
+
+      - name: Store hash
+        run: 'shasum -a 256 image.tar | tee image.tar.sha256'
+
+      - name: Store image
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+        with:
+          name: tested-image
+          path: |
+            image.tar
+            image.tar.sha256
+          retention-days: 30d
+
+  check-csi-integration:
+    # Preflight: verify the CLOUDSCALE_API_TOKEN at step-level and set an output
+    # so the integration job can be skipped.
+    #
+    # GitHub Actions limitations motivating this:
+    # - `secrets` are NOT available in `jobs.<id>.if` (job-level `if`), so you cannot
+    #   directly gate/skip a job by testing a secret there.
+    name: Check CSI Integration Configuration
+    runs-on: ubuntu-latest
+
+    outputs:
+      integration-enabled: ${{ steps.check.outputs.enabled }}
+
+    steps:
+      - id: check
+        name: Verify CLOUDSCALE_API_TOKEN is present
+        env:
+          CLOUDSCALE_API_TOKEN: ${{ secrets.CLOUDSCALE_API_TOKEN }}
+        run: |
+          if [ -n "$CLOUDSCALE_API_TOKEN" ]; then
+            echo "enabled=true" >> $GITHUB_OUTPUT
+            echo "CLOUDSCALE_API_TOKEN found — integration will run."
+          else
+            echo "enabled=false" >> $GITHUB_OUTPUT
+            echo "CLOUDSCALE_API_TOKEN not configured — skipping integration."
+          fi
+
+  integration:
+    name: "Kubernetes ${{ matrix.kubernetes }}"
+    runs-on: ubuntu-latest
+
+    needs:
+      - test-matrix
+      - build-image
+      - check-csi-integration
+    if: needs.check-csi-integration.outputs.integration-enabled == 'true'
+
+    strategy:
+      fail-fast: false
+      max-parallel: 1
+      matrix:
+        include: "${{ fromJson(needs.test-matrix.outputs.tests) }}"
+
+    env:
+      CLOUDSCALE_API_TOKEN: ${{ secrets.CLOUDSCALE_API_TOKEN }}
+      KUBERNETES: '${{ matrix.kubernetes }}'
+      SUBNET: '${{ matrix.subnet }}'
+      CLUSTER_PREFIX: '${{ matrix.cluster_prefix }}'
+      IMAGE_SOURCE: import
+
+    # Prevent integration tests from running in parallel. Ideally this should
+    # be seuqential, but that won't work due to the following issue:
+    #
+    # https://github.com/orgs/community/discussions/5435
+    #
+    # Instead we ensure that only one integration test per supported version
+    # is run at any given time.
+    concurrency:
+      group: integration-${{ matrix.kubernetes }}
+
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - name: Load image
+        uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0
+        with:
+          name: tested-image
+
+      - name: Validate hash
+        run: 'shasum --check image.tar.sha256'
+
+      - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
+        with:
+          go-version-file: go.mod
+
+      - name: Setup Helm
+        uses: azure/setup-helm@5119fcb9089d432beecbf79bb2c7915207344b78 # v3.5
+        with:
+          version: v4.2.4
+
+      - name: Evaluate image name
+        run: 'helpers/image-from-ref >> $GITHUB_ENV'
+
+      - name: Cleanup Leftovers
+        if: always()
+        run: helpers/cleanup
+
+      - name: Create Test Cluster
+        run: helpers/run-in-test-cluster
+
+      - name: Run Integration Tests
+        run: TESTARGS="-race" make test-integration
+
+      - name: Wait For Kubernetes-Internal Cleanup
+        if: always()
+        run: sleep 30
+
+      - name: Destroy Test Cluster
+        if: always()
+        run: helpers/cleanup

.gitignore

@@ -7,3 +7,5 @@ charts/csi-cloudscale/charts
 cmd/cloudscale-csi-plugin/cloudscale-csi-plugin
 k8test/
 bin/
+
+__pycache__/

AGENTS.md

@@ -33,14 +33,14 @@ The [`csi-spec.md`](csi-spec.md) is the authoritative source for this driver's b
 
 ## Project structure
 
-| Directory/File              | Purpose                                           |
-|-----------------------------|---------------------------------------------------|
-| `cmd/cloudscale-csi-plugin/` | Binary entry point and Dockerfile                |
-| `driver/`                    | CSI driver implementation (Controller, Node, etc.) |
-| `charts/csi-cloudscale/`     | Helm chart for deployment                        |
-| `deploy/kubernetes/releases/`| Pre-rendered Kubernetes manifests                |
-| `examples/kubernetes/`       | Example StorageClasses and PVCs                  |
-| `test/kubernetes/`           | Integration tests                                |
+| Directory/File                | Purpose                                            |
+|-------------------------------|----------------------------------------------------|
+| `cmd/cloudscale-csi-plugin/`  | Binary entry point                                 |
+| `driver/`                     | CSI driver implementation (Controller, Node, etc.) |
+| `charts/csi-cloudscale/`      | Helm chart for deployment                          |
+| `deploy/kubernetes/releases/` | Pre-rendered Kubernetes manifests                  |
+| `examples/kubernetes/`        | Example StorageClasses and PVCs                    |
+| `test/kubernetes/`            | Integration tests                                  |
 
 ## Key components
 

cmd/cloudscale-csi-plugin/Dockerfile → Dockerfile

@@ -1,3 +1,4 @@
+# Copyright cloudscale.ch
 # Copyright 2018 DigitalOcean
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -11,20 +12,44 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+FROM golang:1.26.6 AS builder
+
+WORKDIR /src
+
+# Copy go.mod/go.sum first for better layer caching
+COPY go.mod go.sum ./
+RUN go mod download
+
+# Copy all necessary source code
+COPY Makefile ./
+COPY driver/ driver/
+COPY cmd/ cmd/
+
+# Build arguments for version information
+ARG VERSION=dev
+ARG COMMIT=unknown
+ARG GIT_TREE_STATE=unknown
+
+# Build using make (ensures consistent build logic)
+RUN make compile \
+    VERSION="${VERSION}" \
+    COMMIT="${COMMIT}" \
+    GIT_TREE_STATE="${GIT_TREE_STATE}"
+
 FROM alpine:3.23.3
 
 # e2fsprogs-extra is required for resize2fs used for the resize operation
 # blkid: block device identification tool from util-linux
 RUN apk add --no-cache ca-certificates \
-                       e2fsprogs \
-                       findmnt \
-                       xfsprogs \
-                       cryptsetup \
-                       udev \
-                       blkid \
-                       e2fsprogs-extra
-
-ADD cloudscale-csi-plugin /bin/
-ADD csi-diskinfo.sh /bin/
+    e2fsprogs \
+    findmnt \
+    xfsprogs \
+    cryptsetup \
+    udev \
+    blkid \
+    e2fsprogs-extra
+
+COPY --from=builder /src/cmd/cloudscale-csi-plugin/cloudscale-csi-plugin /bin/
+COPY --from=builder /src/cmd/cloudscale-csi-plugin/csi-diskinfo.sh /bin/
 
 ENTRYPOINT ["/bin/cloudscale-csi-plugin"]

Makefile

@@ -1,11 +1,7 @@
 NAME=cloudscale-csi-plugin
 OS ?= linux
-GO_VERSION := $(shell awk '/^go/ {print $$2}' go.mod)
-ifeq ($(strip $(shell git status --porcelain 2>/dev/null)),)
-  GIT_TREE_STATE=clean
-else
-  GIT_TREE_STATE=dirty
-endif
+
+GIT_TREE_STATE ?= $(shell git status --porcelain 2>/dev/null | grep -q . && echo "dirty" || echo "clean")
 COMMIT ?= $(shell git rev-parse HEAD)
 BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD)
 LDFLAGS ?= -X github.com/cloudscale-ch/csi-cloudscale/driver.version=${VERSION} -X github.com/cloudscale-ch/csi-cloudscale/driver.commit=${COMMIT} -X github.com/cloudscale-ch/csi-cloudscale/driver.gitTreeState=${GIT_TREE_STATE}
@@ -14,6 +10,7 @@ PKG ?= github.com/cloudscale-ch/csi-cloudscale/cmd/cloudscale-csi-plugin
 VERSION ?= $(shell cat VERSION)
 CHART_VERSION ?= $(shell awk '/^version:/ {print $$2}' charts/csi-cloudscale/Chart.yaml)
 DOCKER_REPO ?= quay.io/cloudscalech/cloudscale-csi-plugin
+IMAGE ?= $(DOCKER_REPO):$(VERSION)
 
 # Setting SHELL to bash allows bash commands to be executed by recipes.
 # Options are set to exit when a recipe line exits non-zero or a piped command fails.
@@ -97,7 +94,13 @@ test: vet ## Run tests.
 .PHONY: test-integration
 test-integration: ## Run integration tests
 	@echo "==> Started integration tests"
-	@env go test -race -count 1 -v $(TESTARGS) -tags integration -parallel 4 -timeout 20m ./test/...
+	@if [ -f "$(PWD)/k8test/cluster/admin.conf" ]; then \
+		echo "==> Found k8test cluster config at $(PWD)/k8test/cluster/admin.conf, using k8test kubeconfig"; \
+		KUBECONFIG=$(PWD)/k8test/cluster/admin.conf go test -count 1 -v $(TESTARGS) -tags integration -parallel 4 -timeout 20m ./test/...; \
+	else \
+		echo "==> No k8test found, using standard kubeconfig loading"; \
+		go test -count 1 -v $(TESTARGS) -tags integration -parallel 4 -timeout 20m ./test/...; \
+	fi
 
 .PHONY: lint
 lint: golangci-lint ## Run golangci-lint linter
@@ -115,13 +118,18 @@ govulncheck: govulncheck-tool ## Run govulncheck (advisory only)
 
 .PHONY: compile
 compile: ## Build the project binary
-	@echo "==> Building the project"
-	@docker run --rm -it -e GOOS=${OS} -e GOARCH=amd64 -v ${PWD}/:/app -w /app golang:${GO_VERSION}-alpine sh -c 'apk add git && go build -o cmd/cloudscale-csi-plugin/${NAME} -ldflags "$(LDFLAGS)" ${PKG}'
+	@echo "==> Building the binary"
+	CGO_ENABLED=0 go build -o cmd/cloudscale-csi-plugin/$(NAME) -ldflags "$(LDFLAGS)" $(PKG)
 
 .PHONY: build
-build: compile ## Build the docker image
+build: ## Build the docker image
 	@echo "==> Building the docker image"
-	@docker build --platform linux/amd64 -t $(DOCKER_REPO):$(VERSION) cmd/cloudscale-csi-plugin -f cmd/cloudscale-csi-plugin/Dockerfile
+	docker build --platform linux/amd64 \
+		-t $(IMAGE) \
+		--build-arg VERSION="$(VERSION)" \
+		--build-arg COMMIT="$(COMMIT)" \
+		--build-arg GIT_TREE_STATE="$(GIT_TREE_STATE)" \
+		-f Dockerfile .
 
 .PHONY: push
 push: ## Push docker image to registry
@@ -132,9 +140,9 @@ ifeq ($(DOCKER_REPO),quay.io/cloudscalech/cloudscale-csi-plugin)
     endif
   endif
 endif
-	@echo "==> Publishing $(DOCKER_REPO):$(VERSION)"
-	@docker push $(DOCKER_REPO):$(VERSION)
-	@echo "==> Your image is now available at $(DOCKER_REPO):$(VERSION)"
+	@echo "==> Publishing $(IMAGE)"
+	@docker push $(IMAGE)
+	@echo "==> Your image is now available at $(IMAGE)"
 
 .PHONY: publish
 publish: build push clean ## Build, push, and clean

README.md

@@ -418,13 +418,7 @@ This will create a binary with version `dev` and docker image pushed to
 `cloudscalech/cloudscale-csi-plugin:dev`
 
 
-To run the integration tests run the following:
-
-```
-$ export KUBECONFIG=$(pwd)/kubeconfig 
-$ TESTARGS='-run TestPod_Single_SSD_Volume' make test-integration
-```
-
+Refer to the [deploy/README.md](deploy/README.md) for instructions on how to run the integration tests.
 
 ### Release a new version
 

charts/csi-cloudscale/templates/statefulset.yaml

@@ -18,7 +18,7 @@ spec:
     spec:
       hostNetwork: true
       priorityClassName: system-cluster-critical
-      serviceAccount: {{ include "csi-cloudscale.controller-service-account-name" . }}
+      serviceAccountName: {{ include "csi-cloudscale.controller-service-account-name" . }}
       containers:
         - name: csi-provisioner
           image: "{{ .Values.provisioner.image.registry }}/{{ .Values.provisioner.image.repository }}:{{ .Values.provisioner.image.tag }}"