From ab4ea84679a9af5fa6172a4b4ce55c86d7a1ac5c Mon Sep 17 00:00:00 2001 From: Mohamed Gaber Date: Sat, 28 Sep 2024 21:03:43 +0300 Subject: [PATCH] wheels: more compatibility * Update manylinux images * FFI now built as a per-platform static library * Explicitly set minimum macOS deployment target, use clang on macOS * Try enabling Windows (as an experiment) * Disable aarch64-linux, aarch64-windows --- .github/workflows/_run_cibw_linux.py | 44 ++++++++++++ .github/workflows/cibw_before_all.sh | 21 ++++++ .github/workflows/cibw_before_build.sh | 23 +++++++ .github/workflows/wheels.yml | 92 +++++++++++++------------- .gitignore | 1 + 5 files changed, 134 insertions(+), 47 deletions(-) create mode 100644 .github/workflows/_run_cibw_linux.py create mode 100644 .github/workflows/cibw_before_all.sh create mode 100644 .github/workflows/cibw_before_build.sh diff --git a/.github/workflows/_run_cibw_linux.py b/.github/workflows/_run_cibw_linux.py new file mode 100644 index 000000000..675817ae0 --- /dev/null +++ b/.github/workflows/_run_cibw_linux.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +# Copyright (C) 2024 Efabless Corporation +# +# Permission to use, copy, modify, and/or distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +""" +This runs the cibuildwheel step from the wheels workflow locally. +""" + +import os +import yaml +import platform +import subprocess + +__dir__ = os.path.dirname(os.path.abspath(__file__)) + + +workflow = yaml.safe_load(open(os.path.join(__dir__, "wheels.yml"))) + +env = os.environ.copy() + +steps = workflow["jobs"]["build_wheels"]["steps"] +cibw_step = None +for step in steps: + if (step.get("uses") or "").startswith("pypa/cibuildwheel"): + cibw_step = step + break + +for key, value in cibw_step["env"].items(): + if key.endswith("WIN") or key.endswith("MAC"): + continue + env[key] = value + +env["CIBW_ARCHS"] = os.getenv("CIBW_ARCHS") or platform.machine() +subprocess.check_call(["cibuildwheel"], env=env) diff --git a/.github/workflows/cibw_before_all.sh b/.github/workflows/cibw_before_all.sh new file mode 100644 index 000000000..934902380 --- /dev/null +++ b/.github/workflows/cibw_before_all.sh @@ -0,0 +1,21 @@ +set -e +set -x + +# Build-time dependencies +## Linux Docker Images +if command -v yum &> /dev/null; then + yum install -y flex bison +fi + +if command -v apk &> /dev/null; then + apk add flex bison +fi + +## macOS/Windows -- install in GitHub Action itself, not container + +# Build Static FFI (platform-dependent but not Python version dependent) +cd ffi +SHELL=bash CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --prefix=$PWD/pfx + +make install -j$(getconf _NPROCESSORS_ONLN 2>/dev/null || sysctl -n hw.ncpu) +sed -i.bak 's@-L${toolexeclibdir} -lffi@${toolexeclibdir}/libffi.a@' ./pfx/lib/pkgconfig/libffi.pc diff --git a/.github/workflows/cibw_before_build.sh b/.github/workflows/cibw_before_build.sh new file mode 100644 index 000000000..03baf110b --- /dev/null +++ b/.github/workflows/cibw_before_build.sh @@ -0,0 +1,23 @@ +set -e +set -x + +# Build boost +cd ./boost +## Delete the artefacts from previous builds (if any) +rm -rf ./pfx +## Bootstrap bjam +./bootstrap.sh --prefix=./pfx +## Build Boost against current version of Python, only for +## static linkage (Boost is statically linked because system boost packages +## wildly vary in versions, including the libboost_python3 version) +./b2\ + -j$(getconf _NPROCESSORS_ONLN 2>/dev/null || sysctl -n hw.ncpu)\ + --prefix=./pfx\ + --with-filesystem\ + --with-system\ + --with-python\ + cxxflags="$(python3-config --includes) -std=c++17 -fPIC"\ + cflags="$(python3-config --includes) -fPIC"\ + link=static\ + variant=release\ + install diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 9f3394d84..5961f364a 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -6,23 +6,25 @@ on: jobs: build_wheels: strategy: + fail-fast: false matrix: os: [ - # You can't cross-compile on Linux, so might as well do the emulated - # workload on its own { name: "Ubuntu 22.04", family: "linux", runner: "ubuntu-22.04", archs: "x86_64", }, - { - name: "Ubuntu 22.04", - family: "linux", - runner: "ubuntu-22.04", - archs: "aarch64", - }, - # While you can cross-compile on macOS, this is less of a headache + ## Aarch64 is disabled for now: GitHub is committing to EOY + ## for free aarch64 runners for open-source projects and + ## emulation times out: + ## https://github.com/orgs/community/discussions/19197#discussioncomment-10550689 + # { + # name: "Ubuntu 22.04", + # family: "linux", + # runner: "ubuntu-22.04", + # archs: "aarch64", + # }, { name: "macOS 13", family: "macos", @@ -35,13 +37,12 @@ jobs: runner: "macos-14", archs: "arm64", }, - # TODO: Make Windows Work - # { - # name: "Windows Server 2019", - # family: "windows", - # runner: "windows-2019", - # archs: "AMD64 ARM64", - # }, + { + name: "Windows Server 2019", + family: "windows", + runner: "windows-2019", + archs: "AMD64", + }, ] name: Build Wheels | ${{ matrix.os.name }} | ${{ matrix.os.archs }} runs-on: ${{ matrix.os.runner }} @@ -55,22 +56,27 @@ jobs: uses: docker/setup-qemu-action@v2 - uses: actions/setup-python@v3 - name: Get Boost Source + shell: bash run: | mkdir -p boost - curl -L https://sourceforge.net/projects/boost/files/boost/1.83.0/boost_1_83_0.tar.bz2 | tar --strip-components=1 -xjC boost - - name: Seed Makefile.bak - # For every sed, a .bak is created so it can be copied over Makefile to - # rever the change for the next Python version to be built. - # - # This creates a .bak for the first Python version's cp to consume. + curl -L https://github.com/boostorg/boost/releases/download/boost-1.86.0/boost-1.86.0-b2-nodocs.tar.gz | tar --strip-components=1 -xzC boost + - name: Get FFI + shell: bash run: | - cp Makefile Makefile.bak + mkdir -p ffi + curl -L https://github.com/libffi/libffi/releases/download/v3.4.6/libffi-3.4.6.tar.gz | tar --strip-components=1 -xzC ffi + ## Software installed by default in GitHub Action Runner VMs: + ## https://github.com/actions/runner-images - if: ${{ matrix.os.family == 'macos' }} name: "[macOS] Flex/Bison" run: | brew install flex bison echo "PATH=$(brew --prefix flex)/bin:$PATH" >> $GITHUB_ENV echo "PATH=$(brew --prefix bison)/bin:$PATH" >> $GITHUB_ENV + - if : ${{ matrix.os.family == 'windows' }} + name: "[Windows] Flex/Bison" + run: | + choco install winflexbison3 - if: ${{ matrix.os.family == 'macos' && matrix.os.archs == 'arm64' }} name: "[macOS/arm64] Install Python 3.8 (see: https://cibuildwheel.pypa.io/en/stable/faq/#macos-building-cpython-38-wheels-on-arm64)" uses: actions/setup-python@v5 @@ -79,33 +85,25 @@ jobs: - name: Build wheels uses: pypa/cibuildwheel@v2.20.0 env: - # Explaining the build steps: - # 1. Revert previous seds (if any) - # 2. Delete the libboost static archives from previous builds (if any) - # 3. Navigate to boost source extracted in previous GHA step - # 4-5. Build Boost against current version of Python, only for - # static linkage - # (Boost is statically linked because system boost packages - # wildly vary in versions, including the libboost_python3 - # version) - # 6-7. Return to package directory and sed the Makefile to point at - # the newly compiled libboost_python CIBW_SKIP: pp* # The Makefile requires python3-config which is not in pypy CIBW_ARCHS: ${{ matrix.os.archs }} CIBW_BUILD_VERBOSITY: "1" - CIBW_BEFORE_ALL_LINUX: yum install -y libffi-devel flex bison || apk add libffi-dev flex bison - CIBW_BEFORE_ALL_MAC: brew install libffi - CIBW_BEFORE_BUILD: | - cp Makefile.bak Makefile &&\ - cd ./boost &&\ - rm -rf ./pfx &&\ - ./bootstrap.sh --prefix=./pfx &&\ - ./b2 --prefix=./pfx --with-filesystem --with-system --with-python cxxflags="$(python3-config --includes) -std=c++17 -fPIC" cflags="$(python3-config --includes) -fPIC" link=static variant=release install &&\ - ls ./pfx/lib/libboost_python*.a &&\ - cd {package} &&\ - sed -i'.bak' -e "1i\\ - LINKFLAGS = -L./boost/pfx/lib" -e "1i\\ - CXXFLAGS = -I./boost/pfx/include" -e "s@BOOST_PYTHON_LIB ?=@BOOST_PYTHON_LIB = $(ls ./boost/pfx/lib/libboost_python*.a)\nTrash =@" Makefile + # manylinux2014 (default) does not have a modern enough C++ compiler for Yosys + CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 + CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28 + CIBW_BEFORE_ALL: bash ./.github/workflows/cibw_before_all.sh + CIBW_ENVIRONMENT: > + CXXFLAGS=-I./boost/pfx/include + LINKFLAGS=-L./boost/pfx/lib + PKG_CONFIG_PATH=./ffi/pfx/lib/pkgconfig + makeFlags='BOOST_PYTHON_LIB=./boost/pfx/lib/libboost_python*.a' + CIBW_ENVIRONMENT_MACOS: > + CXXFLAGS=-I./boost/pfx/include + LINKFLAGS=-L./boost/pfx/lib + PKG_CONFIG_PATH=./ffi/pfx/lib/pkgconfig + MACOSX_DEPLOYMENT_TARGET=11 + makeFlags='BOOST_PYTHON_LIB=./boost/pfx/lib/libboost_python*.a CONFIG=clang' + CIBW_BEFORE_BUILD: bash ./.github/workflows/cibw_before_build.sh - uses: actions/upload-artifact@v3 with: path: ./dist/*.whl diff --git a/.gitignore b/.gitignore index 5d6a466db..5c3d8f4e9 100644 --- a/.gitignore +++ b/.gitignore @@ -51,3 +51,4 @@ __pycache__ /build /venv /boost +/ffi