wheels: more compatibility

* Update manylinux images
* FFI now built as a per-platform static library
* Explicitly set minimum macOS deployment target, use clang on macOS
* Try enabling Windows (as an experiment)
* Disable aarch64-linux, aarch64-windows

.github/workflows/_run_cibw_linux.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+# Copyright (C) 2024 Efabless Corporation
+#
+# Permission to use, copy, modify, and/or distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+"""
+This runs the cibuildwheel step from the wheels workflow locally.
+"""
+
+import os
+import yaml
+import platform
+import subprocess
+
+__dir__ = os.path.dirname(os.path.abspath(__file__))
+
+
+workflow = yaml.safe_load(open(os.path.join(__dir__, "wheels.yml")))
+
+env = os.environ.copy()
+
+steps = workflow["jobs"]["build_wheels"]["steps"]
+cibw_step = None
+for step in steps:
+    if (step.get("uses") or "").startswith("pypa/cibuildwheel"):
+        cibw_step = step
+        break
+
+for key, value in cibw_step["env"].items():
+    if key.endswith("WIN") or key.endswith("MAC"):
+        continue
+    env[key] = value
+
+env["CIBW_ARCHS"] = os.getenv("CIBW_ARCHS") or platform.machine()
+subprocess.check_call(["cibuildwheel"], env=env)

.github/workflows/cibw_before_all.sh

@@ -0,0 +1,21 @@
+set -e
+set -x
+
+# Build-time dependencies
+## Linux Docker Images
+if command -v yum &> /dev/null; then
+    yum install -y flex bison
+fi
+
+if command -v apk &> /dev/null; then
+    apk add flex bison
+fi
+
+## macOS/Windows -- install in GitHub Action itself, not container
+
+# Build Static FFI (platform-dependent but not Python version dependent)
+cd ffi
+SHELL=bash CFLAGS=-fPIC CXXFLAGS=-fPIC ./configure --prefix=$PWD/pfx
+
+make install -j$(getconf _NPROCESSORS_ONLN 2>/dev/null || sysctl -n hw.ncpu)
+sed -i.bak 's@-L${toolexeclibdir} -lffi@${toolexeclibdir}/libffi.a@' ./pfx/lib/pkgconfig/libffi.pc

.github/workflows/cibw_before_build.sh

@@ -0,0 +1,23 @@
+set -e
+set -x
+
+# Build boost
+cd ./boost
+## Delete the artefacts from previous builds (if any)
+rm -rf ./pfx
+## Bootstrap bjam
+./bootstrap.sh --prefix=./pfx
+## Build Boost against current version of Python, only for
+## static linkage (Boost is statically linked because system boost packages
+## wildly vary in versions, including the libboost_python3 version)
+./b2\
+    -j$(getconf _NPROCESSORS_ONLN 2>/dev/null || sysctl -n hw.ncpu)\
+    --prefix=./pfx\
+    --with-filesystem\
+    --with-system\
+    --with-python\
+    cxxflags="$(python3-config --includes) -std=c++17 -fPIC"\
+    cflags="$(python3-config --includes) -fPIC"\
+    link=static\
+    variant=release\
+    install

.github/workflows/wheels.yml

@@ -6,23 +6,25 @@ on:
 jobs:
   build_wheels:
     strategy:
+      fail-fast: false
       matrix:
         os: [
-            # You can't cross-compile on Linux, so might as well do the emulated
-            # workload on its own
             {
               name: "Ubuntu 22.04",
               family: "linux",
               runner: "ubuntu-22.04",
               archs: "x86_64",
             },
-            {
+            ## Aarch64 is disabled for now: GitHub is committing to EOY
-              name: "Ubuntu 22.04",
+            ## for free aarch64 runners for open-source projects and
-              family: "linux",
+            ## emulation times out:
-              runner: "ubuntu-22.04",
+            ## https://github.com/orgs/community/discussions/19197#discussioncomment-10550689
-              archs: "aarch64",
+            # {
-            },
+            #   name: "Ubuntu 22.04",
-            # While you can cross-compile on macOS, this is less of a headache
+            #   family: "linux",
+            #   runner: "ubuntu-22.04",
+            #   archs: "aarch64",
+            # },
             {
               name: "macOS 13",
               family: "macos",
@@ -35,13 +37,12 @@ jobs:
               runner: "macos-14",
               archs: "arm64",
             },
-            # TODO: Make Windows Work
+            {
-            # {
+              name: "Windows Server 2019",
-            #   name: "Windows Server 2019",
+              family: "windows",
-            #   family: "windows",
+              runner: "windows-2019",
-            #   runner: "windows-2019",
+              archs: "AMD64",
-            #   archs: "AMD64 ARM64",
+            },
-            # },
           ]
     name: Build Wheels | ${{ matrix.os.name }} | ${{ matrix.os.archs }}
     runs-on: ${{ matrix.os.runner }}
@@ -55,22 +56,27 @@ jobs:
         uses: docker/setup-qemu-action@v2
       - uses: actions/setup-python@v3
       - name: Get Boost Source
+        shell: bash
         run: |
           mkdir -p boost
-          curl -L https://sourceforge.net/projects/boost/files/boost/1.83.0/boost_1_83_0.tar.bz2 | tar --strip-components=1 -xjC boost
+          curl -L https://github.com/boostorg/boost/releases/download/boost-1.86.0/boost-1.86.0-b2-nodocs.tar.gz | tar --strip-components=1 -xzC boost
-      - name: Seed Makefile.bak
+      - name: Get FFI
-        # For every sed, a .bak is created so it can be copied over Makefile to
+        shell: bash
-        # rever the change for the next Python version to be built.
-        #
-        # This creates a .bak for the first Python version's cp to consume.
         run: |
-          cp Makefile Makefile.bak
+          mkdir -p ffi
+          curl -L https://github.com/libffi/libffi/releases/download/v3.4.6/libffi-3.4.6.tar.gz | tar --strip-components=1 -xzC ffi
+      ## Software installed by default in GitHub Action Runner VMs:
+      ##  https://github.com/actions/runner-images
       - if: ${{ matrix.os.family == 'macos' }}
         name: "[macOS] Flex/Bison"
         run: |
           brew install flex bison
           echo "PATH=$(brew --prefix flex)/bin:$PATH" >> $GITHUB_ENV
           echo "PATH=$(brew --prefix bison)/bin:$PATH" >> $GITHUB_ENV
+      - if : ${{ matrix.os.family == 'windows' }}
+        name: "[Windows] Flex/Bison"
+        run: |
+          choco install winflexbison3
       - if: ${{ matrix.os.family == 'macos' && matrix.os.archs == 'arm64' }}
         name: "[macOS/arm64] Install Python 3.8 (see: https://cibuildwheel.pypa.io/en/stable/faq/#macos-building-cpython-38-wheels-on-arm64)"
         uses: actions/setup-python@v5
@@ -79,33 +85,25 @@ jobs:
       - name: Build wheels
         uses: pypa/cibuildwheel@v2.20.0
         env:
-          # Explaining the build steps:
-          #   1. Revert previous seds (if any)
-          #   2. Delete the libboost static archives from previous builds (if any)
-          #   3. Navigate to boost source extracted in previous GHA step
-          #   4-5. Build Boost against current version of Python, only for
-          #        static linkage
-          #        (Boost is statically linked because system boost packages
-          #         wildly vary in versions, including the libboost_python3
-          #         version)
-          #   6-7. Return to package directory and sed the Makefile to point at
-          #        the newly compiled libboost_python
           CIBW_SKIP: pp* # The Makefile requires python3-config which is not in pypy
           CIBW_ARCHS: ${{ matrix.os.archs }}
           CIBW_BUILD_VERBOSITY: "1"
-          CIBW_BEFORE_ALL_LINUX: yum install -y libffi-devel flex bison || apk add libffi-dev flex bison
+          # manylinux2014 (default) does not have a modern enough C++ compiler for Yosys
-          CIBW_BEFORE_ALL_MAC: brew install libffi
+          CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28
-          CIBW_BEFORE_BUILD: |
+          CIBW_MANYLINUX_AARCH64_IMAGE: manylinux_2_28
-            cp Makefile.bak Makefile &&\
+          CIBW_BEFORE_ALL: bash ./.github/workflows/cibw_before_all.sh
-              cd ./boost &&\
+          CIBW_ENVIRONMENT: >
-              rm -rf ./pfx &&\
+            CXXFLAGS=-I./boost/pfx/include
-              ./bootstrap.sh --prefix=./pfx &&\
+            LINKFLAGS=-L./boost/pfx/lib
-              ./b2 --prefix=./pfx --with-filesystem --with-system --with-python cxxflags="$(python3-config --includes) -std=c++17 -fPIC" cflags="$(python3-config --includes) -fPIC" link=static variant=release install &&\
+            PKG_CONFIG_PATH=./ffi/pfx/lib/pkgconfig
-              ls ./pfx/lib/libboost_python*.a &&\
+            makeFlags='BOOST_PYTHON_LIB=./boost/pfx/lib/libboost_python*.a'
-              cd {package} &&\
+          CIBW_ENVIRONMENT_MACOS: >
-              sed -i'.bak' -e "1i\\
+            CXXFLAGS=-I./boost/pfx/include
-                LINKFLAGS = -L./boost/pfx/lib" -e "1i\\
+            LINKFLAGS=-L./boost/pfx/lib
-                CXXFLAGS = -I./boost/pfx/include" -e "s@BOOST_PYTHON_LIB ?=@BOOST_PYTHON_LIB = $(ls ./boost/pfx/lib/libboost_python*.a)\nTrash =@" Makefile
+            PKG_CONFIG_PATH=./ffi/pfx/lib/pkgconfig
+            MACOSX_DEPLOYMENT_TARGET=11
+            makeFlags='BOOST_PYTHON_LIB=./boost/pfx/lib/libboost_python*.a CONFIG=clang'
+          CIBW_BEFORE_BUILD: bash ./.github/workflows/cibw_before_build.sh
       - uses: actions/upload-artifact@v3
         with:
           path: ./dist/*.whl

.gitignore

@@ -51,3 +51,4 @@ __pycache__
 /build
 /venv
 /boost
+/ffi