Merge pull request #383 from isuruf/arm64

MRG: Support building arm64 and universal2 wheels on x86_64

Add `arm64` and `universal2` PLAT options.  `universal2` works by seperate Intel and ARM64 builds, and fusing the resulting wheels.

README.rst

@@ -272,6 +272,9 @@ To use these scripts
     # and test containers, via docker.
     dist: xenial
 
+    # osx image that enables building Apple silicon libraries
+    osx_image: xcode12.2
+
     matrix:
       include:
         - os: linux
@@ -326,6 +329,10 @@ To use these scripts
         - os: osx
           env:
             - MB_PYTHON_VERSION=3.8
+        - os: osx
+          env:
+            - MB_PYTHON_VERSION=3.9
+            - PLAT="universal2"
         - os: osx
           env:
             - MB_PYTHON_VERSION=3.9
@@ -419,6 +426,36 @@ To use these scripts
 * Make sure your project is set up to build on AppVeyor, and you should now
   be ready (for what could be another round of slow debugging).
 
+* For Apple silicon support you can either create an ``arm64`` wheel or
+  a ``universal2`` wheel by supplying ``PLAT`` env variable.
+  ``universal2`` builds work on both ``arm64`` and ``x86_64`` platforms
+  and also make it possible for the wheel code to work when switching the
+  architecture on Apple silicon machines where ``x86_64`` can be run
+  using Rosetta2 emulation.
+
+  There are two ways to build ``universal2`` builds.
+  1. Build with ``-arch x86_64 -arch arm64``.
+     These flags instruct the C/C++ compiler to compile twice and create a
+     fat object/executable/library. This is the easiest, but has several
+     drawbacks. If you are using C/C++ libraries that are built using
+     library_builders, it's highly likely that they don't build correctly
+     because most build systems and packages don't support building fat binaries.
+     We could possibly build them separately and fuse them, but the headers might
+     not be identical which is required when building the wheel as a ``universal2``
+     wheel. If you are using Fortran, ``gfortran`` doesn't support fat binaries.
+
+  2. Build ``arm64`` and ``x86_64`` wheels separately and fuse them.
+     For this to work, we need to build the C/C++ libraries twice. Therefore,
+     the library building is once called with ``BUILD_PREFIX=${BUILD_PREFIX:-/usr/local}``
+     for ``x86_64`` and then called again with ``BUILD_PREFIX=/opt/arm64-builds``.
+     Once the two wheels are created, these two are merged. Both the
+     ``arm64`` and ``universal2`` wheels are outputs for this build.
+
+  In multibuild we are going with option 2. You can override this behaviour by
+  overriding the function ``wrap_wheel_builder``.
+  To build Apple silicon builds, you should use a CI service with Xcode 12 with
+  universal build support and make sure that xcode is the default.
+
 If your project depends on NumPy, you will want to build against the earliest
 NumPy that your project supports - see `forward, backward NumPy compatibility
 <https://stackoverflow.com/questions/17709641/valueerror-numpy-dtype-has-the-wrong-size-try-recompiling/18369312#18369312>`_.

common_utils.sh

@@ -270,22 +270,27 @@ function bdist_wheel_cmd {
     cp dist/*.whl $abs_wheelhouse
 }
 
+function wrap_wheel_builder {
+    # Wrapper for build commands, overwritten by macOS for universal2 or arm64 wheel building
+    $@
+}
+
 function build_pip_wheel {
     # Standard wheel building command with pip wheel
-    build_wheel_cmd "pip_wheel_cmd" $@
+    wrap_wheel_builder build_wheel_cmd "pip_wheel_cmd" $@
 }
 
 function build_bdist_wheel {
     # Wheel building with bdist_wheel. See bdist_wheel_cmd
-    build_wheel_cmd "bdist_wheel_cmd" $@
+    wrap_wheel_builder build_wheel_cmd "bdist_wheel_cmd" $@
 }
 
 function build_wheel {
     # Set default building method to pip
-    build_pip_wheel $@
+    wrap_wheel_builder build_pip_wheel $@
 }
 
-function build_index_wheel {
+function build_index_wheel_cmd {
     # Builds wheel from some index, usually pypi
     #
     # Parameters:
@@ -316,6 +321,10 @@ function build_index_wheel {
     repair_wheelhouse $wheelhouse
 }
 
+function build_index_wheel {
+    wrap_wheel_builder build_index_wheel_cmd $@
+}
+
 function pip_opts {
     [ -n "$MANYLINUX_URL" ] && echo "--find-links $MANYLINUX_URL"
 }

configure_build.sh

@@ -21,8 +21,13 @@ if [ -n "$IS_MACOS" ]; then
         ARCH_FLAGS=${ARCH_FLAGS:-"-arch i386 -arch x86_64"}
     elif [[ $PLAT == x86_64 ]]; then
         ARCH_FLAGS=${ARCH_FLAGS:-"-arch x86_64"}
+    elif [[ $PLAT == arm64 ]]; then
+        ARCH_FLAGS=${ARCH_FLAGS:-"-arch arm64"}
+    elif [[ $PLAT == universal2 ]]; then
+        # Do nothing as we are going with fusing wheels
+        ARCH_FLAGS=${ARCH_FLAGS:-}
     else
-        echo "invalid platform = '$PLAT', supported values are 'intel' or 'x86_64'"
+        echo "Invalid platform = '$PLAT'. Supported values are 'intel', 'x86_64', 'arm64' or 'universal2'"
         exit 1
     fi
     # Only set CFLAGS, FFLAGS if they are not already defined.  Build functions
@@ -53,9 +58,17 @@ else
     fi
 fi
 
-# Promote BUILD_PREFIX on search path to any newly built libs
-export CPPFLAGS="-I$BUILD_PREFIX/include $CPPFLAGS"
-export LIBRARY_PATH="$BUILD_PREFIX/lib:$LIBRARY_PATH"
-export PKG_CONFIG_PATH="$BUILD_PREFIX/lib/pkgconfig/:$PKG_CONFIG_PATH"
-# Add binary path for configure utils etc
-export PATH="$BUILD_PREFIX/bin:$PATH"
+export CPPFLAGS_BACKUP="$CPPFLAGS"
+export LIBRARY_PATH_BACKUP="$LIBRARY_PATH"
+export PKG_CONFIG_PATH_BACKUP="$PKG_CONFIG_PATH"
+
+function update_env_for_build_prefix {
+  # Promote BUILD_PREFIX on search path to any newly built libs
+  export CPPFLAGS="-I$BUILD_PREFIX/include $CPPFLAGS_BACKUP"
+  export LIBRARY_PATH="$BUILD_PREFIX/lib:$LIBRARY_PATH_BACKUP"
+  export PKG_CONFIG_PATH="$BUILD_PREFIX/lib/pkgconfig/:$PKG_CONFIG_PATH_BACKUP"
+  # Add binary path for configure utils etc
+  export PATH="$BUILD_PREFIX/bin:$PATH"
+}
+
+update_env_for_build_prefix

osx_utils.sh

@@ -103,7 +103,9 @@ function macpython_sdk_list_for_version {
     local _major=${_ver%%.*}
     local _return
 
-    if [ "$_major" -eq "2" ]; then
+    if [ "${PLAT}" = "arm64" ]; then
+        _return="11.0"
+    elif [ "$_major" -eq "2" ]; then
         [ $(lex_ver $_ver) -lt $(lex_ver 2.7.18) ] && _return="10.6"
         [ $(lex_ver $_ver) -ge $(lex_ver 2.7.15) ] && _return="$_return 10.9"
     elif [ "$_major" -eq "3" ]; then
@@ -155,7 +157,11 @@ function pyinst_fname_for_version {
     local py_version=$1
     local py_osx_ver=${2:-$(macpython_sdk_for_version $py_version)}
     local inst_ext=$(pyinst_ext_for_version $py_version)
-    echo "python-${py_version}-macosx${py_osx_ver}.${inst_ext}"
+    if [ "${PLAT:-}" == "arm64" ] || [ "${PLAT:-}" == "universal2" ]; then
+      echo "python-${py_version}-macos11.0.${inst_ext}"
+    else
+      echo "python-${py_version}-macosx${py_osx_ver}.${inst_ext}"
+    fi
 }
 
 function get_macpython_arch {
@@ -168,7 +174,7 @@ function get_macpython_arch {
     # Note: MUST only be called after the version of Python used to build the
     # target wheel has been installed and is on the path
     local distutils_plat=${1:-$(get_distutils_platform)}
-    if [[ $distutils_plat =~ macosx-(10\.[0-9]+)-(.*) ]]; then
+    if [[ $distutils_plat =~ macosx-(1[0-9]\.[0-9]+)-(.*) ]]; then
         echo ${BASH_REMATCH[2]}
     else
         echo "Error parsing macOS distutils platform '$distutils_plat'"
@@ -186,7 +192,7 @@ function get_macpython_osx_ver {
     # Note: MUST only be called after the version of Python used to build the
     # target wheel has been installed and is on the path
     local distutils_plat=${1:-$(get_distutils_platform)}
-    if [[ $distutils_plat =~ macosx-(10\.[0-9]+)-(.*) ]]; then
+    if [[ $distutils_plat =~ macosx-(1[0-9]\.[0-9]+)-(.*) ]]; then
         echo ${BASH_REMATCH[1]}
     else
         echo "Error parsing macOS distutils platform '$distutils_plat'"
@@ -417,3 +423,65 @@ function activate_ccache {
     # Prove to the developer that ccache is activated
     echo "Using C compiler: $(which clang)"
 }
+
+function macos_intel_build_setup {
+    # Setup build for single arch x86_64 wheels
+    export PLAT="x86_64"
+    export _PYTHON_HOST_PLATFORM="macosx-${MB_PYTHON_OSX_VER}-x86_64"
+    export CFLAGS+=" -arch x86_64"
+    export CXXFLAGS+=" -arch x86_64"
+    export ARCHFLAGS+=" -arch x86_64"
+    export CPPFLAGS+=" -arch x86_64"
+    export LDFLAGS+=" -arch x86_64"
+}
+
+function macos_arm64_build_setup {
+    # Setup build for single arch arm_64 wheels
+    export PLAT="arm64"
+    export BUILD_PREFIX=/opt/arm64-builds
+    sudo mkdir -p $BUILD_PREFIX
+    sudo chown -R $USER $BUILD_PREFIX
+    update_env_for_build_prefix
+    export _PYTHON_HOST_PLATFORM="macosx-11.0-arm64"
+    export CFLAGS+=" -arch arm64"
+    export CXXFLAGS+=" -arch arm64"
+    export CPPFLAGS+=" -arch arm64"
+    export ARCHFLAGS+=" -arch arm64"
+    export FCFLAGS+=" -arch arm64"
+    export FC=$FC_ARM64
+    export LDFLAGS+=" -arch arm64 -L$BUILD_PREFIX/lib -Wl,-rpath,$BUILD_PREFIX/lib ${FC_ARM64_LDFLAGS:-}"
+    # This would automatically let autoconf know that we are cross compiling for arm64 darwin
+    export host_alias="aarch64-apple-darwin20.0.0"
+}
+
+function fuse_macos_intel_arm64 {
+    local wheelhouse=$(abspath ${WHEEL_SDIR:-wheelhouse})
+    local py_osx_ver=$(echo ${MB_PYTHON_OSX_VER} | sed "s/\./_/g")
+    mkdir -p tmp_fused_wheelhouse
+    for whl in $wheelhouse/*.whl; do
+       if [[ "$whl" == *macosx_${py_osx_ver}_x86_64.whl ]]; then
+           whl_base=$(echo $whl | rev | cut -c 23- | rev)
+           if [[ -f "${whl_base}macosx_11_0_arm64.whl" ]]; then
+               delocate-fuse $whl "${whl_base}macosx_11_0_arm64.whl" -w tmp_fused_wheelhouse
+               mv tmp_fused_wheelhouse/$(basename $whl) $wheelhouse/$(basename ${whl_base})macosx_${py_osx_ver}_universal2.whl
+               # Since we want one wheel that's installable for testing we are deleting the *_x86_64 wheel.
+               # We are not deleting arm64 wheel because the size is lower and homebrew/conda-forge python
+               # will use them by default
+               rm $whl
+           fi
+       fi
+    done
+}
+
+function wrap_wheel_builder {
+    if [[ "${PLAT:-}" == "universal2" ]]; then
+        (macos_intel_build_setup && $@)
+        rm -rf *-stamp
+        (macos_arm64_build_setup && $@)
+        fuse_macos_intel_arm64
+    elif [[ "${PLAT:-}" == "arm64" ]]; then
+        (macos_arm64_build_setup && $@)
+    else
+        $@
+    fi
+}

tests/test_osx_utils.sh

@@ -57,6 +57,10 @@
 [ "$(macpython_sdk_list_for_version 2.7.17)" == "10.6 10.9" ] || ingest
 [ "$(macpython_sdk_list_for_version 2.7.18)" == "10.9" ] || ingest
 
+(PLAT="arm64";      [ "$(macpython_sdk_for_version 3.9)" == "11.0" ] || ingest)
+(PLAT="universal2"; [ "$(macpython_sdk_for_version 3.9)" == "10.9" ] || ingest)
+(PLAT="x86_64";     [ "$(macpython_sdk_for_version 3.9)" == "10.9" ] || ingest)
+[ "$(macpython_sdk_for_version 3.9)"    == "10.9" ] || ingest
 [ "$(macpython_sdk_for_version 3.8)"    == "10.9" ] || ingest
 [ "$(macpython_sdk_for_version 3.5)"    == "10.6" ] || ingest
 [ "$(macpython_sdk_for_version 2.7)"    == "10.9" ] || ingest