diff --git a/.github/workflows/test_latest_versions.yml b/.github/workflows/test_latest_versions.yml index 1944108..90215aa 100644 --- a/.github/workflows/test_latest_versions.yml +++ b/.github/workflows/test_latest_versions.yml @@ -25,6 +25,22 @@ jobs: matrix: os: [ubuntu-latest, macos-latest] python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + include: + # Every other macOS cell uses Apple clang, which has no OpenMP of its + # own and so exercises only the Homebrew-libomp path. This one pins + # Homebrew LLVM to cover the other branch -- a compiler that ships its + # own libomp -- which is the case setup.py has to detect and prefer. + # Without it that path has no coverage here at all. + # + # `cc` is part of the identity rather than an extra key on an existing + # cell: an include entry matching an existing os/python-version pair + # merges into it instead of adding a job, which would have replaced the + # Apple clang coverage for that version rather than adding to it. Every + # other cell therefore carries cc: system. + - os: macos-latest + python-version: "3.14" + cc: llvm + cc: [system] steps: - uses: actions/checkout@v7 with: @@ -48,6 +64,21 @@ jobs: # default library path. run: | brew install open-mpi libomp openblas + - name: Pin Homebrew LLVM as the compiler (macOS) + if: matrix.cc == 'llvm' + shell: bash + # Homebrew's llvm builds the openmp runtime, so its clang carries a + # libomp of its own. setup.py must notice and use that one: adding a + # second from the standalone formula would put two same-named runtimes in + # one process, which aborts at the first parallel region with + # "OMP: Error #15". + run: | + brew install llvm + LLVM_PREFIX="$(brew --prefix llvm)" + { + echo "CC=${LLVM_PREFIX}/bin/clang" + echo "CXX=${LLVM_PREFIX}/bin/clang++" + } >> "$GITHUB_ENV" - name: Install dependencies run: | python -m pip install --upgrade pip @@ -61,4 +92,40 @@ jobs: # Not --parallel: the environments would then race to install the same wheel, # and the MPI tests want the runner's cores to themselves. run: | - tox -e py,notebook,mpi + # -vv surfaces the build backend's output. At default verbosity, and + # at -v, tox prints a single "build_wheel>" line and swallows + # everything setup.py reports -- so a macOS build that silently picked + # the wrong OpenMP left no trace of which one it chose. Checked + # locally: -v yields nothing, -vv shows it. + tox -vv -e py,notebook,mpi + - name: Assert exactly one OpenMP runtime is loaded (macOS) + if: runner.os == 'macOS' + shell: bash + # A passing suite is not enough to protect the compiler-owned-libomp + # path: an earlier version of that detection silently found nothing and + # fell back to the standalone formula, which still built and still passed + # here -- it only aborted once another OpenMP consumer shared the process. + # Count the runtimes actually mapped after the backend loads instead. + run: | + .tox/py/bin/python - <<'PY' + import ctypes, sys + + libc = ctypes.CDLL(None) + libc._dyld_image_count.restype = ctypes.c_uint32 + libc._dyld_get_image_name.restype = ctypes.c_char_p + libc._dyld_get_image_name.argtypes = [ctypes.c_uint32] + + import sbd + sbd.get_backend() # backends load lazily, on first use + + omp = sorted({ + n for i in range(libc._dyld_image_count()) + if (n := libc._dyld_get_image_name(i).decode("utf-8", "replace")) + and any(k in n.lower() for k in ("libomp", "libiomp", "libgomp")) + }) + for name in omp: + print(f" {name}") + if len(omp) != 1: + sys.exit(f"expected exactly 1 OpenMP runtime, found {len(omp)}") + print("OK: one OpenMP runtime") + PY diff --git a/setup.py b/setup.py index 63bff45..4987893 100644 --- a/setup.py +++ b/setup.py @@ -473,6 +473,79 @@ def _homebrew_prefix(): return None +def _resolve_darwin_cxx(): + """(path, version_line) of the C++ compiler this build will actually use. + + distutils takes CC/CXX from the environment, else from sysconfig -- where + conda records a bare 'clang++' that is resolved through PATH, so a Homebrew + LLVM silently wins over both Apple clang and a conda toolchain. Knowing + which one it is decides where OpenMP comes from, so this is resolved before + the libomp search rather than merely reported afterwards. + """ + import shutil + cxx = (os.environ.get('CXX') or sysconfig.get_config_var('CXX') + or 'clang++').split()[0] + path = shutil.which(cxx) or cxx + try: + version = subprocess.check_output( + [path, '--version'], universal_newlines=True, + stderr=subprocess.STDOUT).splitlines()[0] + except Exception: + version = '(version unknown)' + return path, version + + +def _compiler_openmp(cxx_path): + """(include_dir, lib_dir) of an OpenMP shipped with cxx_path, or None. + + An LLVM that builds the openmp runtime -- Homebrew's llvm formula, and the + conda-forge clang packages -- carries a libomp.dylib in its own tree, and + `-fopenmp` links THAT copy. Adding a second libomp from elsewhere then puts + two same-named runtimes in one process, which aborts at the first parallel + region with "OMP: Error #15". So when the compiler brings its own, that is + the one to build against. + + Do not guess the layout. omp.h is installed into the clang RESOURCE + directory (lib/clang//include), not /include -- Homebrew's own + formula test compiles `#include ` with no -I at all -- while + libomp.dylib does land in /lib. Ask the driver for the resource + directory rather than hardcoding a version number into the path. + """ + if not cxx_path or not os.path.isabs(cxx_path): + return None + # Resolve symlinks first: CC/CXX is commonly Homebrew's opt/ alias + # (/opt/homebrew/opt/llvm/bin/clang++), while -print-resource-dir answers + # with the real Cellar path. Comparing or joining the two forms without + # normalising invites mismatches. + cxx_real = os.path.realpath(cxx_path) + lib_dir = os.path.join(os.path.dirname(os.path.dirname(cxx_real)), 'lib') + have_lib = any(os.path.exists(os.path.join(lib_dir, name)) + for name in ('libomp.dylib', 'libomp.a')) + try: + resource_dir = subprocess.check_output( + [cxx_real, '-print-resource-dir'], universal_newlines=True, + stderr=subprocess.DEVNULL).strip() + except Exception as exc: + resource_dir = None + print(f"Notice: {cxx_real} -print-resource-dir failed: {exc!r}", + file=sys.stderr) + inc_dir = os.path.join(resource_dir, 'include') if resource_dir else None + have_header = bool(inc_dir) and os.path.exists(os.path.join(inc_dir, 'omp.h')) + # Say what was found either way: a silent None here sends the build to a + # different OpenMP, which still compiles and still passes its own tests, and + # only aborts once another OpenMP consumer shares the process. + # stderr, not stdout: tox hides the build backend's stdout at default + # verbosity, which is why an earlier version of this probe printed nothing + # in CI and left the fallback looking like a mystery again. + print(f"Darwin OpenMP probe: compiler={cxx_real}\n" + f" lib_dir={lib_dir} libomp={have_lib}\n" + f" resource_dir={resource_dir} omp.h={have_header}", + file=sys.stderr) + if not (have_lib and have_header): + return None + return inc_dir, lib_dir + + def find_nvidia_hpc_sdk(): nvhpc_home = os.environ.get('NVHPC_HOME', None) if nvhpc_home: @@ -750,17 +823,50 @@ def detect_gpu_toolchain(): if build_cpu: print("\nConfiguring CPU backend (_core_cpu)") if platform.system() == 'Darwin': + # Which compiler is used decides where OpenMP may come from, so resolve + # it first. Printing it is also the difference between a reproducible + # build and a mystery, since a bare 'clang++' from sysconfig is resolved + # through PATH. + _cxx_path, _cxx_ver = _resolve_darwin_cxx() + print(f"Darwin C++ compiler: {_cxx_path}\n" + f" {_cxx_ver} (pin it with CC/CXX)", + file=sys.stderr) + # macOS has no system OpenMP, so libomp comes from a package manager. - # Prefer the conda env when it has one: those are the libraries actually - # LOADED at import time (resolved via the python executable's - # @loader_path/../lib), so building against Homebrew's copies instead - # means compiling against different libraries than the process runs on. + # + # Order matters, and it is about which libomp ends up in the PROCESS, + # not merely which one satisfies the compile: + # + # 1. The compiler's own runtime, when it has one. `-fopenmp` links that + # copy no matter what else is on the link line, so naming a second + # libomp here is how you get two same-named runtimes in one process + # and an "OMP: Error #15" abort at the first parallel region. + # 2. Otherwise the conda env, whose libraries are the ones actually + # LOADED at import time (resolved via the python executable's + # @loader_path/../lib). + # 3. Otherwise Homebrew's standalone libomp, which is what Apple clang + # needs, having no OpenMP of its own. conda_prefix = os.environ.get('CONDA_PREFIX') - if conda_prefix and os.path.exists( + compiler_omp = _compiler_openmp(_cxx_path) + if compiler_omp: + omp_inc, omp_lib = compiler_omp + # BLAS is a separate question from OpenMP: the compiler tree has no + # OpenBLAS, so keep taking that from conda or Homebrew. + if conda_prefix and os.path.isdir(os.path.join(conda_prefix, 'lib')): + openblas_lib = os.path.join(conda_prefix, 'lib') + else: + openblas_lib = os.path.join(_homebrew_prefix() or '/opt/homebrew', + 'opt', 'openblas', 'lib') + print(f"Darwin: libomp from the compiler's own tree\n" + f" headers {omp_inc}\n" + f" library {omp_lib}\n" + f" BLAS {openblas_lib}", file=sys.stderr) + elif conda_prefix and os.path.exists( os.path.join(conda_prefix, 'include', 'omp.h')): omp_inc = os.path.join(conda_prefix, 'include') omp_lib = openblas_lib = os.path.join(conda_prefix, 'lib') - print(f"Darwin: libomp and BLAS from conda env {conda_prefix}") + print(f"Darwin: libomp and BLAS from conda env {conda_prefix}", + file=sys.stderr) else: # Ask brew for its prefix rather than assuming: it is /opt/homebrew # on Apple silicon and /usr/local on Intel, so either one hardcoded @@ -775,31 +881,15 @@ def detect_gpu_toolchain(): # Fail here with the fix, rather than 100 lines later with # "'omp.h' file not found" from the middle of a compile. print("Error: no OpenMP runtime found on this macOS host.\n" - f" Looked in $CONDA_PREFIX/include and {omp_inc}.\n" + f" Looked beside {_cxx_path}, in $CONDA_PREFIX/include, " + f"and in {omp_inc}.\n" " Apple clang ships without OpenMP, so install one:\n" " conda install -c conda-forge llvm-openmp (preferred)\n" " brew install libomp") sys.exit(1) print(f"Darwin: libomp and BLAS from Homebrew at {brew_prefix} " - "(no conda libomp found)") - - # Say WHICH clang is compiling. distutils takes CC/CXX from the - # environment, else from sysconfig -- where conda records a bare - # 'clang++' that is resolved through PATH, so a Homebrew LLVM silently - # wins over both Apple clang and a conda toolchain. Printing it is the - # difference between a reproducible build and a mystery. - import shutil - _cxx = (os.environ.get('CXX') or sysconfig.get_config_var('CXX') - or 'clang++').split()[0] - _cxx_path = shutil.which(_cxx) or _cxx - try: - _cxx_ver = subprocess.check_output( - [_cxx_path, '--version'], universal_newlines=True, - stderr=subprocess.STDOUT).splitlines()[0] - except Exception: - _cxx_ver = '(version unknown)' - print(f"Darwin C++ compiler: {_cxx_path}\n" - f" {_cxx_ver} (pin it with CC/CXX)") + "(no conda libomp found)", file=sys.stderr) + cpu_compile_args = [ '-DSBD_TRADMODE', '-DOMPI_SKIP_MPICXX', @@ -813,9 +903,8 @@ def detect_gpu_toolchain(): # Not extra_link_args: that carries a bare `-fopenmp`, which Apple # clang rejects at link time the same way it does when compiling. # Re-derive the rpath entries over the libomp/BLAS directories chosen - # above -- whether they came from conda or Homebrew -- so those dylibs - # resolve at import time. Apple's linker wants `-rpath`, not GNU ld's - # `--rpath`. + # above, so those dylibs resolve at import time. Apple's linker wants + # `-rpath`, not GNU ld's `--rpath`. cpu_link_args = [f'-L{d}' for d in (omp_lib, openblas_lib)] cpu_link_args += [f'-Wl,-rpath,{d}' for d in cpu_lib_dirs] else: