diff --git a/.github/workflows/build-native-action.yml b/.github/workflows/build-native-action.yml index a4ec2bcb..65099221 100644 --- a/.github/workflows/build-native-action.yml +++ b/.github/workflows/build-native-action.yml @@ -367,3 +367,87 @@ jobs: name: ${{ steps.python-version.outputs.version }}-${{ matrix.arch }}-win.tar.xz path: build/${{ steps.python-version.outputs.version }}-${{ matrix.arch }}-win.tar.xz retention-days: 5 + + build_windows_arm64: + name: "Python Windows arm64" + # Native build: host and target are both arm64, so this runs on the + # windows-11-arm hosted runner rather than cross-compiling from the + # x64 windows-2022 image build_windows uses. WIP job for + # https://github.com/saltstack/relenv/issues/280 -- untested until a + # CI run actually exercises it; expect early iterations to surface + # missing toolchain/SDK pieces the way build_windows's history did. + runs-on: windows-11-arm + strategy: + fail-fast: false + matrix: + version: + - '3.10' + - '3.11' + - '3.12' + - '3.13' + - '3.14' + arch: + - arm64 + env: + RELENV_DATA: ${{ github.workspace }} + outputs: + version: ${{ steps.python-version.outputs.version }} + steps: + - uses: actions/checkout@v3 + + - name: Set up Python 3.11 + id: setup-python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install VS Build Tools + run: | + relenv/_scripts/install_vc_build.ps1 -CICD + + - name: Install ARM64 VC++ Tools + # install_vc_build.ps1 only requests the x86/x64 VCTools workload + # plus the legacy VC.140 toolset for the amd64/x86 job; it never + # asks for the ARM64 compiler/linker/armasm64, so request that + # component explicitly here rather than assuming the runner + # image's pre-installed VS carries it. + run: | + $vswhere = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" + $vsPath = & $vswhere -latest -products * -property installationPath + $installer = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vs_installer.exe" + & $installer modify --installPath "$vsPath" ` + --add Microsoft.VisualStudio.Component.VC.Tools.ARM64 ` + --quiet --norestart --nocache + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + + - name: Install nox + run: | + pip3 install nox + + - name: Determine Python Version + id: python-version + run: | + echo "version=$(python3 -m relenv versions --version=${{ matrix.version }})" | tee -a "$env:GITHUB_OUTPUT" + + - name: Build Python with Relenv + env: + RELENV_NATIVE_PY_VERSION: 3.10.15 + WindowsTargetPlatformVersion: "10.0.19041.0" + HOST_PYTHON: ${{ steps.setup-python.outputs.python-path }} + run: | + python -m relenv build --no-pretty --arch=${{ matrix.arch }} --python=${{ steps.python-version.outputs.version }} + + - name: Upload Build Logs + uses: actions/upload-artifact@v4 + if: always() + with: + name: ${{ steps.python-version.outputs.version }}-${{ matrix.arch }}-windows-logs + path: logs/* + retention-days: 5 + + - name: Upload Zipfile + uses: actions/upload-artifact@v4 + with: + name: ${{ steps.python-version.outputs.version }}-${{ matrix.arch }}-win.tar.xz + path: build/${{ steps.python-version.outputs.version }}-${{ matrix.arch }}-win.tar.xz + retention-days: 5 diff --git a/.github/workflows/verify-build-action.yml b/.github/workflows/verify-build-action.yml index df8c9552..949ab59e 100644 --- a/.github/workflows/verify-build-action.yml +++ b/.github/workflows/verify-build-action.yml @@ -273,3 +273,71 @@ jobs: name: ${{ steps.python-version.outputs.version }}-${{ matrix.arch }}-windows-logs path: logs/* retention-days: 5 + + test_windows_arm64: + name: "Verify Windows arm64" + # Matches build_windows_arm64's native windows-11-arm runner in + # build-native-action.yml. WIP for + # https://github.com/saltstack/relenv/issues/280. + runs-on: windows-11-arm + + strategy: + fail-fast: false + matrix: + version: + - '3.10' + - '3.11' + - '3.12' + - '3.13' + - '3.14' + arch: + - arm64 + + outputs: + version: ${{ steps.python-version.outputs.version }} + env: + RELENV_DATA: ${{ github.workspace }} + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python 3.11 + # 3.10, unlike the x64 job, has no official arm64 Windows build + # for setup-python to install ("The version '3.10' with + # architecture 'arm64' was not found for Windows Enterprise") -- + # CPython's arm64 Windows installers only started at 3.11. Use + # the same bootstrap version build_windows_arm64 already uses. + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install VS Build Tools + run: | + relenv/_scripts/install_vc_build.ps1 -CICD + + - name: Install nox + run: | + pip3 install nox + + - name: Determine Python Version + id: python-version + run: | + echo "version=$(python3 -m relenv versions --version=${{ matrix.version }})" | tee -a "$env:GITHUB_OUTPUT" + + - name: "Download artifact: build/${{ matrix.version }}-${{ matrix.arch }}-win.tar.xz" + uses: actions/download-artifact@v4 + with: + name: ${{ steps.python-version.outputs.version }}-${{ matrix.arch }}-win.tar.xz + path: build/ + + - name: Verify Build + run: | + nox -e tests -- tests/test_verify_build.py + + - name: Upload Build Logs + uses: actions/upload-artifact@v4 + if: always() + with: + name: ${{ steps.python-version.outputs.version }}-${{ matrix.arch }}-windows-logs + path: logs/* + retention-days: 5 diff --git a/relenv/_scripts/install_vc_build.ps1 b/relenv/_scripts/install_vc_build.ps1 index eb4256f2..cfb17121 100644 --- a/relenv/_scripts/install_vc_build.ps1 +++ b/relenv/_scripts/install_vc_build.ps1 @@ -97,8 +97,23 @@ try { $VS_INST_LOC = $(Get-CimInstance MSFT_VSInstance -Namespace root/cimv2/vs).InstallLocation $MSBUILD_BIN = $(Get-ChildItem "$VS_INST_LOC\MSBuild\*\Bin\msbuild.exe").FullName } catch { - # If VS is not installed, this is the fallback for this installation - $MSBUILD_BIN = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\2017\BuildTools\MSBuild\15.0\Bin\msbuild.exe" + # The MSFT_VSInstance CIM class isn't registered on every runner image + # (confirmed absent on windows-11-arm even though VS 2022 ships + # pre-installed there) -- before assuming VS needs to be installed + # from scratch, fall back to vswhere.exe, which is present on every + # GitHub-hosted Windows image regardless of CIM provider support. + $vswhere = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" + $VS_INST_LOC = $null + if ( Test-Path -Path $vswhere ) { + $VS_INST_LOC = & $vswhere -latest -products * -property installationPath + } + if ( $VS_INST_LOC ) { + $MSBUILD_BIN = $(Get-ChildItem "$VS_INST_LOC\MSBuild\*\Bin\msbuild.exe").FullName + } else { + # Genuinely no VS install found by either method - this is the + # fallback for a from-scratch installation. + $MSBUILD_BIN = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\2017\BuildTools\MSBuild\15.0\Bin\msbuild.exe" + } } #------------------------------------------------------------------------------- diff --git a/relenv/build/windows.py b/relenv/build/windows.py index 1a916b57..83bfa69e 100644 --- a/relenv/build/windows.py +++ b/relenv/build/windows.py @@ -607,6 +607,20 @@ def update_openssl(dirs: Dirs, env: EnvMapping) -> None: shutil.copy(str(target_dir / license_file), str(out_dir / "LICENSE")) break + if not is_binary: + # Bundle a copy of the OpenSSL dev tree (headers + import + # libs) into the onedir itself. Rust crates such as + # openssl-sys -- a transitive build dependency of packages + # like `cryptography` when pip falls back to building from + # source, which it does on arm64 since no prebuilt wheel + # exists yet for a target nothing has shipped wheels for -- + # have no other way to find an OpenSSL to link against. + # Consumers set OPENSSL_DIR to /OpenSSL before + # pip installing such packages. + openssl_bundle_dir = dirs.prefix / "OpenSSL" + if not openssl_bundle_dir.exists(): + shutil.copytree(str(prefix), str(openssl_bundle_dir)) + if is_binary: # Ensure include/openssl exists inc_openssl_dir = target_dir / "include" / "openssl" @@ -1108,6 +1122,7 @@ def runpip(pkg: str | os.PathLike[str]) -> None: "*.whl", "/Include/*", "/Lib/site-packages/*", + "/OpenSSL/*", ] archive = f"{dirs.prefix}.tar.xz" with tarfile.open(archive, mode="w:xz") as fp: diff --git a/relenv/common.py b/relenv/common.py index 87ddee34..6a7ac60d 100644 --- a/relenv/common.py +++ b/relenv/common.py @@ -131,7 +131,7 @@ def toolchain_root_dir() -> pathlib.Path: WIN32: ( "amd64", "x86", - # "arm64", # Python 11 should support arm. + "arm64", ), } diff --git a/tests/test_verify_build.py b/tests/test_verify_build.py index 4ba95b77..04092c64 100644 --- a/tests/test_verify_build.py +++ b/tests/test_verify_build.py @@ -750,25 +750,44 @@ def test_pip_install_pyzmq( ) -def test_pip_install_cryptography(pipexec, pyexec): +def _openssl_dir_env(build: pathlib.Path, env: dict[str, str]) -> dict[str, str]: + """ + Point OPENSSL_DIR at the OpenSSL dev tree relenv bundles into the + onedir when it built OpenSSL from source (currently windows arm64 + only). Packages like `cryptography` that fall back to compiling + openssl-sys from source -- because no prebuilt wheel exists yet for + a target nothing has shipped wheels for -- have no other way to find + an OpenSSL to link against. A no-op everywhere the bundled tree + doesn't exist (binary-openssl platforms already have prebuilt + wheels, so this never matters there). + """ + openssl_dir = build / "OpenSSL" + if openssl_dir.exists(): + env["OPENSSL_DIR"] = str(openssl_dir) + return env + + +def test_pip_install_cryptography(pipexec, pyexec, build): _install_ppbt(pyexec) packages = [ "cryptography", ] env = os.environ.copy() env["RELENV_BUILDENV"] = "yes" + env = _openssl_dir_env(build, env) for name in packages: p = subprocess.run([str(pipexec), "install", name, "--no-cache-dir"], env=env) assert p.returncode == 0, f"Failed to pip install {name}" -def test_pip_install_idem(pipexec, pyexec): +def test_pip_install_idem(pipexec, pyexec, build): _install_ppbt(pyexec) packages = [ "idem", ] env = os.environ.copy() env["RELENV_BUILDENV"] = "yes" + env = _openssl_dir_env(build, env) for name in packages: p = subprocess.run([str(pipexec), "install", name, "--no-cache-dir"], env=env) assert p.returncode == 0, f"Failed to pip install {name}" @@ -797,6 +816,15 @@ def test_pip_install_salt_pip_dir(pipexec, pyexec, build, build_version, arch): if sys.platform == "win32" and arch == "amd64": pytest.xfail("Known failure on windows amd64") + if sys.platform == "win32" and arch == "arm64": + # Windows arm64: salt's transitive C-extension deps + # (pymssql, cryptography, cffi at their older pinned versions) + # have no arm64 wheels on PyPI and their source builds do not + # succeed under the MSVC arm64 toolchain (no FreeTDS/OpenSSL + # for arm64 in this environment, and cffi <1.17 predates arm64 + # Windows support). + pytest.xfail("Known failure on windows arm64") + if sys.platform == "darwin" and ("3.13" in build_version or "3.14" in build_version): pytest.xfail("Salt does not work with 3.13+ on macos yet") @@ -1489,7 +1517,7 @@ def test_install_with_target_uninstall(pipexec, build): assert not (extras / "bin" / "cowsay").exists() -def test_install_with_target_cffi_versions(pipexec, pyexec, build, build_version): +def test_install_with_target_cffi_versions(pipexec, pyexec, build, build_version, arch): env = os.environ.copy() env["RELENV_DEBUG"] = "yes" extras = build / "extras" @@ -1497,7 +1525,15 @@ def test_install_with_target_cffi_versions(pipexec, pyexec, build, build_version cffi_version = "2.0.0" else: cffi_version = "1.17.1" - if build_version[:4] not in ["3.13", "3.14"]: + # cffi 1.14.6 / 1.16.0 have no Windows arm64 wheels on PyPI and their + # source builds fail under the MSVC arm64 toolchain (arm64 support + # landed upstream in cffi 1.17). Skip the older-cffi coverage on + # win-arm64; the ``cffi_version`` path below still exercises the + # arm64-supported release. + old_cffi_supported = build_version[:4] not in ["3.13", "3.14"] and not ( + sys.platform == "win32" and arch == "arm64" + ) + if old_cffi_supported: subprocess.run( [str(pipexec), "install", "cffi==1.14.6"], check=True, @@ -1523,8 +1559,20 @@ def test_install_with_target_cffi_versions(pipexec, pyexec, build, build_version proc.stdout.decode().strip() == "1.17.1" -def test_install_with_target_no_ignore_installed(pipexec, pyexec, build, build_version): - if build_version.startswith("3.14"): +def test_install_with_target_no_ignore_installed(pipexec, pyexec, build, build_version, arch): + # On Windows arm64, cffi <1.17 and pygit2 <1.16 have no wheels on + # PyPI and their source builds do not support the MSVC arm64 + # toolchain. Pin to the earliest arm64-supported release for each + # Python version so this test exercises the same "install cffi, + # then --target pygit2 sees it as already-satisfied" flow. + if sys.platform == "win32" and arch == "arm64": + if build_version.startswith("3.14"): + cffi = "cffi==2.0.0" + pygit2 = "pygit2==1.19.2" + else: + cffi = "cffi==1.17.1" + pygit2 = "pygit2==1.16.0" + elif build_version.startswith("3.14"): cffi = "cffi==2.0.0" pygit2 = "pygit2==1.19.2" elif build_version.startswith("3.13"): @@ -1559,8 +1607,17 @@ def test_install_with_target_no_ignore_installed(pipexec, pyexec, build, build_v assert "installed cffi" not in out -def test_install_with_target_ignore_installed(pipexec, pyexec, build, build_version): - if build_version.startswith("3.14"): +def test_install_with_target_ignore_installed(pipexec, pyexec, build, build_version, arch): + # See test_install_with_target_no_ignore_installed for why win-arm64 + # needs its own version selection. + if sys.platform == "win32" and arch == "arm64": + if build_version.startswith("3.14"): + cffi = "cffi==2.0.0" + pygit2 = "pygit2==1.19.2" + else: + cffi = "cffi==1.17.1" + pygit2 = "pygit2==1.16.0" + elif build_version.startswith("3.14"): cffi = "cffi==2.0.0" pygit2 = "pygit2==1.19.2" elif build_version.startswith("3.13"):