From 407343a7a12385bba40e406f68270b95afabb09b Mon Sep 17 00:00:00 2001 From: Mohamed Gaber Date: Mon, 12 Aug 2024 14:21:05 +0300 Subject: [PATCH] Pyosys Wheels * Created `setup.py`: Python package manifest to build `pyosys` wheels with a custom extension to build and include `libyosys.so` using Make * `.gitignore`: Added byproducts of the Python wheel build process * `Makefile`: Added `-undefined dynamic_lookup` to `libyosys.so` so missing symbols can be resolved by importing into a Python interpreter * `kernel/yosys.cc`: Gated `PyImport_AppendInittab` with `!Py_IsInitialized`; as of Python 3.12, the interpreter is already initialized and `PyImport_AppendInittab` would cause an exception to be raised * Created `wheels.yml`: CI workflow for building wheels for CPython on: * Linux (glibc, musl) and Darwin * x86-64 and arm64 --- .github/workflows/wheels.yml | 111 +++++++++++++++++++++++++++++++++++ .gitignore | 7 ++- Makefile | 2 +- kernel/yosys.cc | 13 ++-- setup.py | 97 ++++++++++++++++++++++++++++++ 5 files changed, 224 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/wheels.yml create mode 100644 setup.py diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml new file mode 100644 index 000000000..9f3394d84 --- /dev/null +++ b/.github/workflows/wheels.yml @@ -0,0 +1,111 @@ +name: Build Wheels for PyPI +on: + workflow_dispatch: + push: + # tags: ["yosys-*"] +jobs: + build_wheels: + strategy: + matrix: + os: [ + # You can't cross-compile on Linux, so might as well do the emulated + # workload on its own + { + name: "Ubuntu 22.04", + family: "linux", + runner: "ubuntu-22.04", + archs: "x86_64", + }, + { + name: "Ubuntu 22.04", + family: "linux", + runner: "ubuntu-22.04", + archs: "aarch64", + }, + # While you can cross-compile on macOS, this is less of a headache + { + name: "macOS 13", + family: "macos", + runner: "macos-13", + archs: "x86_64", + }, + { + name: "macOS 14", + family: "macos", + runner: "macos-14", + archs: "arm64", + }, + # TODO: Make Windows Work + # { + # name: "Windows Server 2019", + # family: "windows", + # runner: "windows-2019", + # archs: "AMD64 ARM64", + # }, + ] + name: Build Wheels | ${{ matrix.os.name }} | ${{ matrix.os.archs }} + runs-on: ${{ matrix.os.runner }} + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + submodules: true + - if: ${{ matrix.os.family == 'linux' }} + name: "[Linux] Set up QEMU" + uses: docker/setup-qemu-action@v2 + - uses: actions/setup-python@v3 + - name: Get Boost Source + run: | + mkdir -p boost + curl -L https://sourceforge.net/projects/boost/files/boost/1.83.0/boost_1_83_0.tar.bz2 | tar --strip-components=1 -xjC boost + - name: Seed Makefile.bak + # For every sed, a .bak is created so it can be copied over Makefile to + # rever the change for the next Python version to be built. + # + # This creates a .bak for the first Python version's cp to consume. + run: | + cp Makefile Makefile.bak + - if: ${{ matrix.os.family == 'macos' }} + name: "[macOS] Flex/Bison" + run: | + brew install flex bison + echo "PATH=$(brew --prefix flex)/bin:$PATH" >> $GITHUB_ENV + echo "PATH=$(brew --prefix bison)/bin:$PATH" >> $GITHUB_ENV + - if: ${{ matrix.os.family == 'macos' && matrix.os.archs == 'arm64' }} + name: "[macOS/arm64] Install Python 3.8 (see: https://cibuildwheel.pypa.io/en/stable/faq/#macos-building-cpython-38-wheels-on-arm64)" + uses: actions/setup-python@v5 + with: + python-version: 3.8 + - name: Build wheels + uses: pypa/cibuildwheel@v2.20.0 + env: + # Explaining the build steps: + # 1. Revert previous seds (if any) + # 2. Delete the libboost static archives from previous builds (if any) + # 3. Navigate to boost source extracted in previous GHA step + # 4-5. Build Boost against current version of Python, only for + # static linkage + # (Boost is statically linked because system boost packages + # wildly vary in versions, including the libboost_python3 + # version) + # 6-7. Return to package directory and sed the Makefile to point at + # the newly compiled libboost_python + CIBW_SKIP: pp* # The Makefile requires python3-config which is not in pypy + CIBW_ARCHS: ${{ matrix.os.archs }} + CIBW_BUILD_VERBOSITY: "1" + CIBW_BEFORE_ALL_LINUX: yum install -y libffi-devel flex bison || apk add libffi-dev flex bison + CIBW_BEFORE_ALL_MAC: brew install libffi + CIBW_BEFORE_BUILD: | + cp Makefile.bak Makefile &&\ + cd ./boost &&\ + rm -rf ./pfx &&\ + ./bootstrap.sh --prefix=./pfx &&\ + ./b2 --prefix=./pfx --with-filesystem --with-system --with-python cxxflags="$(python3-config --includes) -std=c++17 -fPIC" cflags="$(python3-config --includes) -fPIC" link=static variant=release install &&\ + ls ./pfx/lib/libboost_python*.a &&\ + cd {package} &&\ + sed -i'.bak' -e "1i\\ + LINKFLAGS = -L./boost/pfx/lib" -e "1i\\ + CXXFLAGS = -I./boost/pfx/include" -e "s@BOOST_PYTHON_LIB ?=@BOOST_PYTHON_LIB = $(ls ./boost/pfx/lib/libboost_python*.a)\nTrash =@" Makefile + - uses: actions/upload-artifact@v3 + with: + path: ./dist/*.whl diff --git a/.gitignore b/.gitignore index e82343e89..5d6a466db 100644 --- a/.gitignore +++ b/.gitignore @@ -45,4 +45,9 @@ __pycache__ /tests/unit/bintest/ /tests/unit/objtest/ /tests/ystests -/result \ No newline at end of file +/result +/dist +/*.egg-info +/build +/venv +/boost diff --git a/Makefile b/Makefile index 5b6fc3ff2..ebea193cf 100644 --- a/Makefile +++ b/Makefile @@ -742,7 +742,7 @@ $(PROGRAM_PREFIX)yosys$(EXE): $(OBJS) libyosys.so: $(filter-out kernel/driver.o,$(OBJS)) ifeq ($(OS), Darwin) - $(P) $(CXX) -o libyosys.so -shared -Wl,-install_name,$(LIBDIR)/libyosys.so $(LINKFLAGS) $^ $(LIBS) $(LIBS_VERIFIC) + $(P) $(CXX) -o libyosys.so -shared -undefined dynamic_lookup -Wl,-install_name,$(LIBDIR)/libyosys.so $(LINKFLAGS) $^ $(LIBS) $(LIBS_VERIFIC) else $(P) $(CXX) -o libyosys.so -shared -Wl,-soname,$(LIBDIR)/libyosys.so $(LINKFLAGS) $^ $(LIBS) $(LIBS_VERIFIC) endif diff --git a/kernel/yosys.cc b/kernel/yosys.cc index efdc54b79..510151479 100644 --- a/kernel/yosys.cc +++ b/kernel/yosys.cc @@ -555,10 +555,15 @@ void yosys_setup() #undef X #ifdef WITH_PYTHON - PyImport_AppendInittab((char*)"libyosys", INIT_MODULE); - Py_Initialize(); - PyRun_SimpleString("import sys"); - signal(SIGINT, SIG_DFL); + // With Python 3.12, calling PyImport_AppendInittab on an already + // initialized platform fails (such as when libyosys is imported + // from a Python interpreter) + if (!Py_IsInitialized()) { + PyImport_AppendInittab((char*)"libyosys", INIT_MODULE); + Py_Initialize(); + PyRun_SimpleString("import sys"); + signal(SIGINT, SIG_DFL); + } #endif Pass::init_register(); diff --git a/setup.py b/setup.py new file mode 100644 index 000000000..02e6e5843 --- /dev/null +++ b/setup.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +# Copyright (C) 2024 Efabless Corporation +# +# Permission to use, copy, modify, and/or distribute this software for any +# purpose with or without fee is hereby granted, provided that the above +# copyright notice and this permission notice appear in all copies. +# +# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +import os +import re +import shlex +import shutil +from setuptools import setup, Extension +from setuptools.command.build_ext import build_ext + +__dir__ = os.path.dirname(os.path.abspath(__file__)) + +yosys_version_rx = re.compile(r"YOSYS_VER\s*:=\s*([\w\-\+\.]+)") + +version = yosys_version_rx.search( + open(os.path.join(__dir__, "Makefile"), encoding="utf8").read() +)[1] + + +class libyosys_so_ext(Extension): + def __init__( + self, + ) -> None: + super().__init__( + "libyosys.so", + [], + ) + self.args = [ + "ENABLE_PYOSYS=1", + # Wheel meant to be imported from interpreter + "ENABLE_PYTHON_CONFIG_EMBED=0", + # Would need to be installed separately by the user + "ENABLE_TCL=0", + # Would need to be installed separately by the user + "ENABLE_READLINE=0", + # Show compile commands + "PRETTY=0", + ] + + def custom_build(self, bext: build_ext): + bext.spawn( + ["make", f"-j{os.cpu_count() or 1}", self.name] + + shlex.split(os.getenv("makeFlags", "")) + + self.args + ) + build_path = os.path.dirname(os.path.dirname(bext.get_ext_fullpath(self.name))) + pyosys_path = os.path.join(build_path, "pyosys") + target = os.path.join(pyosys_path, os.path.basename(self.name)) + os.makedirs(pyosys_path, exist_ok=True) + shutil.copyfile(self.name, target) + + # I don't know how debug info is getting here. + bext.spawn(["strip", "-S", target]) + + +class custom_build_ext(build_ext): + def build_extension(self, ext) -> None: + if not hasattr(ext, "custom_build"): + return super().build_extension(ext) + return ext.custom_build(self) + + +setup( + name="pyosys", + packages=["pyosys"], + version=version, + description="Python access to libyosys", + long_description=open(os.path.join(__dir__, "README.md")).read(), + long_description_content_type="text/markdown", + author="Claire Xenia Wolf", + author_email="claire@yosyshq.com", + install_requires=["wheel", "setuptools"], + classifiers=[ + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Intended Audience :: Developers", + "Operating System :: POSIX :: Linux", + "Operating System :: MacOS :: MacOS X", + ], + package_dir={"pyosys": "misc"}, + python_requires=">=3.8", + ext_modules=[libyosys_so_ext()], + cmdclass={ + "build_ext": custom_build_ext, + }, +)