wheels: more compatibility

* Update manylinux images * FFI now built as a per-platform static library * Explicitly set minimum macOS deployment target, use clang on macOS * Try enabling Windows (as an experiment) * Disable aarch64-linux, aarch64-windows
2024-09-28 21:03:43 +03:00 · 2024-09-28 21:03:43 +03:00 · ab4ea84679
parent 407343a7a1
commit ab4ea84679
5 changed files with 134 additions and 47 deletions
--- a/.github/workflows/_run_cibw_linux.py
+++ b/.github/workflows/_run_cibw_linux.py
@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+# Copyright (C) 2024 Efabless Corporation
+#
+# Permission to use, copy, modify, and/or distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+"""
+This runs the cibuildwheel step from the wheels workflow locally.
+"""
+
+import os
+import yaml
+import platform
+import subprocess
+
+__dir__ = os.path.dirname(os.path.abspath(__file__))
+
+
+workflow = yaml.safe_load(open(os.path.join(__dir__, "wheels.yml")))
+
+env = os.environ.copy()
+
+steps = workflow["jobs"]["build_wheels"]["steps"]
+cibw_step = None
+for step in steps:
+    if (step.get("uses") or "").startswith("pypa/cibuildwheel"):
+        cibw_step = step
+        break
+
+for key, value in cibw_step["env"].items():
+    if key.endswith("WIN") or key.endswith("MAC"):
+        continue
+    env[key] = value
+
+env["CIBW_ARCHS"] = os.getenv("CIBW_ARCHS") or platform.machine()
+subprocess.check_call(["cibuildwheel"], env=env)
--- a/.github/workflows/cibw_before_all.sh
+++ b/.github/workflows/cibw_before_all.sh
@ -0,0 +1,21 @@
+set -e
+set -x
+
+# Build-time dependencies
+## Linux Docker Images
+if command -v yum &> /dev/null; then
+    yum install -y flex bison
+fi
+
+if command -v apk &> /dev/null; then
+    apk add flex bison
+fi
+
+## macOS/Windows -- install in GitHub Action itself, not container
+
+# Build Static FFI (platform-dependent but not Python version dependent)
+cd ffi
+SHELL=bash CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --prefix=$PWD/pfx
+
+make install -j$(getconf _NPROCESSORS_ONLN 2>/dev/null || sysctl -n hw.ncpu)
+sed -i.bak 's@-L${toolexeclibdir} -lffi@${toolexeclibdir}/libffi.a@' ./pfx/lib/pkgconfig/libffi.pc
--- a/.github/workflows/cibw_before_build.sh
+++ b/.github/workflows/cibw_before_build.sh
@ -0,0 +1,23 @@
+set -e
+set -x
+
+# Build boost
+cd ./boost
+## Delete the artefacts from previous builds (if any)
+rm -rf ./pfx
+## Bootstrap bjam
+./bootstrap.sh --prefix=./pfx
+## Build Boost against current version of Python, only for
+## static linkage (Boost is statically linked because system boost packages
+## wildly vary in versions, including the libboost_python3 version)
+./b2\
+    -j$(getconf _NPROCESSORS_ONLN 2>/dev/null || sysctl -n hw.ncpu)\
+    --prefix=./pfx\
+    --with-filesystem\
+    --with-system\
+    --with-python\
+    cxxflags="$(python3-config --includes) -std=c++17 -fPIC"\
+    cflags="$(python3-config --includes) -fPIC"\
+    link=static\
+    variant=release\
+    install
--- a/.github/workflows/wheels.yml
+++ b/.github/workflows/wheels.yml
@ -6,23 +6,25 @@ on:
 jobs:
  build_wheels:
    strategy:
+      fail-fast: false
      matrix:
        os: [
-            # You can't cross-compile on Linux, so might as well do the emulated
-            # workload on its own
            {
              name: "Ubuntu 22.04",
              family: "linux",
              runner: "ubuntu-22.04",
              archs: "x86_64",
            },
-            {
-              name: "Ubuntu 22.04",
-              family: "linux",
-              runner: "ubuntu-22.04",
-              archs: "aarch64",
-            },
-            # While you can cross-compile on macOS, this is less of a headache
+            ## Aarch64 is disabled for now: GitHub is committing to EOY
+            ## for free aarch64 runners for open-source projects and
+            ## emulation times out:
+            ## https://github.com/orgs/community/discussions/19197#discussioncomment-10550689
+            # {
+            #   name: "Ubuntu 22.04",
+            #   family: "linux",
+            #   runner: "ubuntu-22.04",
+            #   archs: "aarch64",
+            # },
            {
              name: "macOS 13",
              family: "macos",
@ -35,13 +37,12 @@ jobs:
              runner: "macos-14",
              archs: "arm64",
            },
-            # TODO: Make Windows Work
-            # {
-            #   name: "Windows Server 2019",
-            #   family: "windows",
-            #   runner: "windows-2019",
-            #   archs: "AMD64 ARM64",
-            # },
+            {
+              name: "Windows Server 2019",
+              family: "windows",
+              runner: "windows-2019",
+              archs: "AMD64",
+            },
          ]
    name: Build Wheels | ${{ matrix.os.name }} | ${{ matrix.os.archs }}
    runs-on: ${{ matrix.os.runner }}
@ -55,22 +56,27 @@ jobs:
        uses: docker/setup-qemu-action@v2
      - uses: actions/setup-python@v3
      - name: Get Boost Source
+        shell: bash
        run: |
          mkdir -p boost
-          curl -L https://sourceforge.net/projects/boost/files/boost/1.83.0/boost_1_83_0.tar.bz2 | tar --strip-components=1 -xjC boost
-      - name: Seed Makefile.bak
-        # For every sed, a .bak is created so it can be copied over Makefile to
-        # rever the change for the next Python version to be built.
-        #
-        # This creates a .bak for the first Python version's cp to consume.
+          curl -L https://github.com/boostorg/boost/releases/download/boost-1.86.0/boost-1.86.0-b2-nodocs.tar.gz | tar --strip-components=1 -xzC boost
+      - name: Get FFI
+        shell: bash
        run: |
-          cp Makefile Makefile.bak
+          mkdir -p ffi
+          curl -L https://github.com/libffi/libffi/releases/download/v3.4.6/libffi-3.4.6.tar.gz | tar --strip-components=1 -xzC ffi
+      ## Software installed by default in GitHub Action Runner VMs:
+      ##  https://github.com/actions/runner-images
      - if: ${{ matrix.os.family == 'macos' }}
        name: "[macOS] Flex/Bison"
        run: |
          brew install flex bison
          echo "PATH=$(brew --prefix flex)/bin:$PATH" >> $GITHUB_ENV
          echo "PATH=$(brew --prefix bison)/bin:$PATH" >> $GITHUB_ENV
+      - if : ${{ matrix.os.family == 'windows' }}
+        name: "[Windows] Flex/Bison"
+        run: |
+          choco install winflexbison3
      - if: ${{ matrix.os.family == 'macos' && matrix.os.archs == 'arm64' }}
        name: "[macOS/arm64] Install Python 3.8 (see: https://cibuildwheel.pypa.io/en/stable/faq/#macos-building-cpython-38-wheels-on-arm64)"
        uses: actions/setup-python@v5
@ -79,33 +85,25 @@ jobs:
      - name: Build wheels
        uses: pypa/cibuildwheel@v2.20.0
        env:
-          # Explaining the build steps:
-          #   1. Revert previous seds (if any)
-          #   2. Delete the libboost static archives from previous builds (if any)
-          #   3. Navigate to boost source extracted in previous GHA step
-          #   4-5. Build Boost against current version of Python, only for
-          #        static linkage
-          #        (Boost is statically linked because system boost packages
-          #         wildly vary in versions, including the libboost_python3
-          #         version)
-          #   6-7. Return to package directory and sed the Makefile to point at
-          #        the newly compiled libboost_python
          CIBW_SKIP: pp* # The Makefile requires python3-config which is not in pypy
          CIBW_ARCHS: ${{ matrix.os.archs }}
          CIBW_BUILD_VERBOSITY: "1"
-          CIBW_BEFORE_ALL_LINUX: yum install -y libffi-devel flex bison || apk add libffi-dev flex bison
-          CIBW_BEFORE_ALL_MAC: brew install libffi
-          CIBW_BEFORE_BUILD: |
-            cp Makefile.bak Makefile &&\
-              cd ./boost &&\
-              rm -rf ./pfx &&\
-              ./bootstrap.sh --prefix=./pfx &&\
-              ./b2 --prefix=./pfx --with-filesystem --with-system --with-python cxxflags="$(python3-config --includes) -std=c++17 -fPIC" cflags="$(python3-config --includes) -fPIC" link=static variant=release install &&\
-              ls ./pfx/lib/libboost_python*.a &&\
-              cd {package} &&\
-              sed -i'.bak' -e "1i\\
-                LINKFLAGS = -L./boost/pfx/lib" -e "1i\\
-                CXXFLAGS = -I./boost/pfx/include" -e "s@BOOST_PYTHON_LIB ?=@BOOST_PYTHON_LIB = $(ls ./boost/pfx/lib/libboost_python*.a)\nTrash =@" Makefile
+          # manylinux2014 (default) does not have a modern enough C++ compiler for Yosys
+          CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28
+          CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28
+          CIBW_BEFORE_ALL: bash ./.github/workflows/cibw_before_all.sh
+          CIBW_ENVIRONMENT: >
+            CXXFLAGS=-I./boost/pfx/include
+            LINKFLAGS=-L./boost/pfx/lib
+            PKG_CONFIG_PATH=./ffi/pfx/lib/pkgconfig
+            makeFlags='BOOST_PYTHON_LIB=./boost/pfx/lib/libboost_python*.a'
+          CIBW_ENVIRONMENT_MACOS: >
+            CXXFLAGS=-I./boost/pfx/include
+            LINKFLAGS=-L./boost/pfx/lib
+            PKG_CONFIG_PATH=./ffi/pfx/lib/pkgconfig
+            MACOSX_DEPLOYMENT_TARGET=11
+            makeFlags='BOOST_PYTHON_LIB=./boost/pfx/lib/libboost_python*.a CONFIG=clang'
+          CIBW_BEFORE_BUILD: bash ./.github/workflows/cibw_before_build.sh
      - uses: actions/upload-artifact@v3
        with:
          path: ./dist/*.whl
--- a/.gitignore
+++ b/.gitignore
@ -51,3 +51,4 @@ __pycache__
 /build
 /venv
 /boost
+/ffi