Merge branch 'devel' into focal-images-for-64-bit

.appveyor.yml

+# https://www.appveyor.com/docs/windows-images-software
+
+image: Visual Studio 2022
+
 environment:
   global:
     REPO_DIR: python-appveyor-demo
@@ -7,30 +11,15 @@ environment:
     TEST_DEPENDS: "nose"
 
   matrix:
-    - PYTHON: "C:\\Miniconda"
-      PYTHON_VERSION: "2.7"
-      PYTHON_ARCH: "32"
-    - PYTHON: "C:\\Miniconda-x64"
-      PYTHON_VERSION: "2.7"
-      PYTHON_ARCH: "64"
-    - PYTHON: "C:\\Miniconda35"
-      PYTHON_VERSION: "3.5"
-      PYTHON_ARCH: "32"
-    - PYTHON: "C:\\Miniconda35-x64"
-      PYTHON_VERSION: "3.5"
-      PYTHON_ARCH: "64"
-    - PYTHON: "C:\\Miniconda36"
-      PYTHON_VERSION: "3.6"
-      PYTHON_ARCH: "32"
-    - PYTHON: "C:\\Miniconda36-x64"
-      PYTHON_VERSION: "3.6"
-      PYTHON_ARCH: "64"
     - PYTHON: "C:\\Miniconda37"
       PYTHON_VERSION: "3.7"
       PYTHON_ARCH: "32"
     - PYTHON: "C:\\Miniconda37-x64"
       PYTHON_VERSION: "3.7"
       PYTHON_ARCH: "64"
+    - PYTHON: "C:\\Miniconda38-x64"
+      PYTHON_VERSION: "3.8"
+      PYTHON_ARCH: "64"
 
 # We always use a 64-bit machine, but can build x86 distributions
 # with the TARGET_ARCH variable.
@@ -38,20 +27,27 @@ platform:
     - x64
 
 matrix:
-    fast_finish: true
+    fast_finish: false
 
 install:
     # Install miniconda and fix headers
+    - where python
+    - where py  # On Windows, py might be a better way to find and run CPython
+    - py --list  # py -3.10-32, -3.10-64 with same syntax for 3.{9-3}, 2.{7,6}
     - SET PATH=%PYTHON%;%PYTHON%\Scripts;%PYTHON%\Library\bin;%PATH%
+    - echo %PATH%
+    - where conda
     - conda info
 
     # Check that we have the expected version and architecture for Python
     - python --version
     - python -c "import struct; print(struct.calcsize('P') * 8)"
+    - py --version
+    - py -c "import struct; print(struct.calcsize('P') * 8)"
 
     # clone a origsel/python-appveyor-demo
     # this repo includes a simple package to test appveyor
-    - git clone git://github.com/ogrisel/python-appveyor-demo.git
+    - git clone https://github.com/ogrisel/python-appveyor-demo.git
 
 build_script:
     # Install build requirements
@@ -85,7 +81,11 @@ test_script:
     - ps: .\install_python.ps1
     - set PYTHON=C:\Python38
     - ps: .\install_python.ps1
-    - set PYTHON=C:\Python39rc1-x64
+    - set PYTHON=C:\Python39-x64
+    - ps: .\install_python.ps1
+    - set PYTHON=C:\Python39
+    - ps: .\install_python.ps1
+    - set PYTHON=C:\Python310-x64
     - ps: .\install_python.ps1
-    - set PYTHON=C:\Python39rc1
+    - set PYTHON=C:\Python310
     - ps: .\install_python.ps1

.github/workflows/lint_python.yml

+name: lint_python
+on: [pull_request, push, workflow_dispatch]
+jobs:
+  lint_python:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+      - run: pip install --upgrade pip wheel
+      - run: pip install bandit black codespell flake8 flake8-2020 flake8-bugbear
+                         flake8-comprehensions isort mypy pytest pyupgrade safety
+      - run: bandit --recursive --skip B101 .  # B101 is assert statements
+      - run: black --check . || true
+      - run: codespell --ignore-words-list="commend"  # --skip="*.css,*.js,*.lock"
+      - run: flake8 . --count --max-complexity=10 --max-line-length=88
+                      --show-source --statistics
+      - run: isort --check-only --profile black .
+      - run: pip install -r requirements.txt || pip install --editable . || true
+      - run: mkdir --parents --verbose .mypy_cache
+      - run: mypy --ignore-missing-imports --install-types --non-interactive .
+      - run: pytest . || pytest --doctest-modules . || true
+      - run: shopt -s globstar && pyupgrade --py36-plus **/*.py || true
+      - run: safety check

.github/workflows/test.yml

+# This is a basic workflow to help you get started with Actions
+
+name: Test
+
+# Controls when the workflow will run
+on:
+  # Triggers the workflow on push or pull request events but only for the devel branch
+  push:
+    branches: [ devel ]
+  pull_request:
+    branches: [ devel ]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  # This workflow contains a single job called "build"
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]       # [ubuntu-latest, macos-latest, windows-latest]
+        python-version: ["3.10"]  # ["2.7", "3.7", "3.8", "3.9", "3.10", "3.11", "pypy3"]
+
+    # The type of runner that the job will run on
+    runs-on: ${{ matrix.os }}
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      # Runs a single command using the runner's Python
+      - name: Run a one-line script
+        shell: python
+        run: print("Hello, world!")
+
+      # Runs a set of commands using the runner's shell
+      - name: Run a multi-line script
+        run: |
+          echo Add other actions to build,
+          echo test, and deploy your project.

LICENSE

@@ -7,7 +7,7 @@ Copyright and License
 The multibuild package, including all examples, code snippets and attached
 documentation is covered by the 2-clause BSD license.
 
-    Copyright (c) 2013-2021, Matt Terry and Matthew Brett; all rights
+    Copyright (c) 2013-2022, Matt Terry and Matthew Brett; all rights
     reserved.
 
     Redistribution and use in source and binary forms, with or without

README.rst

@@ -70,9 +70,9 @@ See https://github.com/MacPython/nipy-wheels for a simple example.
 
    ``wheelhouse/*.whl`` defines the files you want to upload.
 
-*   You might also want to build and upload from Appveyor.  To encrypt the API
+*   You might also want to build and upload from AppVeyor.  To encrypt the API
     key above, go to https://ci.appveyor.com/account/matthew-brett/settings
-    (where ``matthew-brett`` is the account from which your Appveyor job runs.
+    (where ``matthew-brett`` is the account from which your AppVeyor job runs.
     Click on "Encrypt YaML" on the left.  Type in your API key value (e.g.
     ``ni-1234abcd-12ab-34dc-1234-d1e1f3a4b5c6``) as the value to encrypt. Click "Encrypt" and note the text it suggests.  Now, at the top of your ``appveyor.yml`` file, add something like::
 
@@ -179,7 +179,7 @@ shell scripts listed above are available for your build and test.
 Build options are controlled mainly by the following environment
 variables:
 
-* ``MB_PYTHON_VER`` sets the Python version targetted: ``major.minor.patch``
+* ``MB_PYTHON_VER`` sets the Python version targeted: ``major.minor.patch``
   for CPython, or ``pypy-major.minor`` for PyPy.
 * ``MB_PYTHON_OSX_VER`` sets the minimum macOS SDK version for any C
   extensions. For CPython targets it may be set to 10.6 or 10.9, provided a
@@ -220,7 +220,7 @@ Build phase
 -----------
 
 Specify the Manylinux version to build for with the ``MB_ML_VER`` environment
-variable. The default version is ``1``.  Versions that are currently valid are:
+variable. The default version is ``2014``.  Versions that are currently valid are:
 
 * ``1`` corresponding to manylinux1 (see `PEP 513 <https://www.python.org/dev/peps/pep-0513>`_).
 * ``2010``  corresponding to manylinux2010 (see `PEP 571 <https://www.python.org/dev/peps/pep-0571>`_).
@@ -338,7 +338,7 @@ To use these scripts
 
 * Add this (here) repository as a submodule::
 
-    git submodule add https://github.com/matthew-brett/multibuild.git
+    git submodule add https://github.com/multi-build/multibuild.git
 
 * Add your own project repository as another submodule::
 
@@ -595,7 +595,7 @@ Multibuild development
 **********************
 
 The main multibuild repository is always at
-https://github.com/matthew-brett/multibuild
+https://github.com/multi-build/multibuild
 
 We try to keep the ``master`` branch stable and do testing and development
 in the ``devel`` branch.  From time to time we merge ``devel`` into ``master``.

common_utils.sh

@@ -24,9 +24,6 @@ GET_PIP_URL=https://bootstrap.pypa.io/get-pip.py
 # with, so it is passed in when calling "docker run" for tests.
 UNICODE_WIDTH=${UNICODE_WIDTH:-32}
 
-# Default Manylinux version
-MB_ML_VER=${MB_ML_VER:-2014}
-
 if [ $(uname) == "Darwin" ]; then
   IS_MACOS=1; IS_OSX=1;
 else
@@ -35,6 +32,13 @@ else
   which python || export PATH=/opt/python/cp39-cp39/bin:$PATH
 fi
 
+if [ "$MB_ML_LIBC" == "musllinux" ]; then
+  IS_ALPINE=1;
+  MB_ML_VER=${MB_ML_VER:-"_1_1"}
+else
+  # Default Manylinux version
+  MB_ML_VER=${MB_ML_VER:-2014}
+fi
 
 # Work round bug in travis xcode image described at
 # https://github.com/direnv/direnv/issues/210
@@ -220,6 +224,8 @@ function install_rsync {
     if [ -n "$IS_MACOS" ]; then
         # macOS. The colon in the next line is the null command
         :
+    elif [ -n "$IS_ALPINE" ]; then
+        [[ $(type -P rsync) ]] || apk add rsync
     elif [[ $MB_ML_VER == "_2_24" ]]; then
         # debian:9 based distro
         [[ $(type -P rsync) ]] || apt-get install -y rsync
@@ -308,6 +314,7 @@ function build_wheel_cmd {
     local repo_dir=${2:-$REPO_DIR}
     [ -z "$repo_dir" ] && echo "repo_dir not defined" && exit 1
     local wheelhouse=$(abspath ${WHEEL_SDIR:-wheelhouse})
+    mkdir -p "$wheelhouse"
     start_spinner
     if [ -n "$(is_function "pre_build")" ]; then pre_build; fi
     stop_spinner

configure_build.sh

@@ -58,8 +58,12 @@ else
         # centos based distro
         yum install -y libtool wget
     elif [ "${MB_PYTHON_VERSION:0:4}" == "pypy" ]; then
-        # centos based distro
-        yum install -y wget
+        if [ -n "$IS_ALPINE" ]; then
+          apk add wget
+        else
+          # centos based distro
+          yum install -y wget
+        fi
     fi
 fi
 

library_builders.sh

@@ -18,6 +18,7 @@ BZIP2_VERSION="${BZIP2_VERSION:-1.0.7}"
 FREETYPE_VERSION="${FREETYPE_VERSION:-2.11.0}"
 TIFF_VERSION="${TIFF_VERSION:-4.1.0}"
 JPEG_VERSION="${JPEG_VERSION:-9b}"
+JPEGTURBO_VERSION="${JPEGTURBO_VERSION:-2.1.3}"
 OPENJPEG_VERSION="${OPENJPEG_VERSION:-2.1}"
 LCMS2_VERSION="${LCMS2_VERSION:-2.9}"
 GIFLIB_VERSION="${GIFLIB_VERSION:-5.1.3}"
@@ -133,7 +134,9 @@ function build_zlib {
     # Gives an old but safe version
     if [ -n "$IS_MACOS" ]; then return; fi  # OSX has zlib already
     if [ -e zlib-stamp ]; then return; fi
-    if [[ $MB_ML_VER == "_2_24" ]]; then
+    if [ -n "$IS_ALPINE" ]; then
+        apk add zlib-dev
+    elif [[ $MB_ML_VER == "_2_24" ]]; then
         # debian:9 based distro
         apt-get install -y zlib1g-dev
     else
@@ -159,6 +162,18 @@ function build_jpeg {
     touch jpeg-stamp
 }
 
+function build_libjpeg_turbo {
+    if [ -e jpeg-stamp ]; then return; fi
+    local cmake=$(get_modern_cmake)
+    fetch_unpack https://download.sourceforge.net/libjpeg-turbo/libjpeg-turbo-${JPEGTURBO_VERSION}.tar.gz
+    (cd libjpeg-turbo-${JPEGTURBO_VERSION} \
+        && $cmake -G"Unix Makefiles" -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_INSTALL_LIBDIR=/usr/local/lib . \
+        && make install)
+
+    # Prevent build_jpeg
+    touch jpeg-stamp
+}
+
 function build_libpng {
     build_zlib
     build_simple libpng $LIBPNG_VERSION https://download.sourceforge.net/libpng
@@ -186,6 +201,8 @@ function get_modern_cmake {
     local cmake=cmake
     if [ -n "$IS_MACOS" ]; then
         brew install cmake > /dev/null
+    elif [ -n "$IS_ALPINE" ]; then
+        apk add cmake > /dev/null
     elif [[ $MB_ML_VER == "_2_24" ]]; then
         # debian:9 based distro
         apt-get install -y cmake
@@ -411,6 +428,8 @@ function build_suitesparse {
     if [ -e suitesparse-stamp ]; then return; fi
     if [ -n "$IS_MACOS" ]; then
         brew install suite-sparse > /dev/null
+    elif [ -n "$IS_ALPINE" ]; then
+        apk add suitesparse-dev
     elif [[ $MB_ML_VER == "_2_24" ]]; then
         # debian:9 based distro
         apt-get install -y libsuitesparse-dev > /dev/null

manylinux_utils.sh

@@ -44,24 +44,30 @@ function activate_ccache {
     ln -s /parent-home/.ccache $HOME/.ccache
 
     # Now install ccache
-    if [[ $MB_ML_VER == "_2_24" ]]; then
-        # debian:9 based distro
-        suppress apt-get install -y ccache
+    local ccache_dir
+    if [ -n "$IS_ALPINE" ]; then
+        suppress apk add ccache
+        ccache_dir='/usr/lib/ccache/bin'
     else
-        # centos based distro
-        suppress yum_install ccache
-    fi
+        if [[ $MB_ML_VER == "_2_24" ]]; then
+            # debian:9 based distro
+            suppress apt-get install -y ccache
+        else
+            # centos based distro
+            suppress yum_install ccache
+        fi
 
-    # Create fake compilers and prepend them to the PATH
-    # Note that yum is supposed to create these for us,
-    # but I had trouble finding them
-    local ccache_dir=/usr/lib/ccache/compilers
-    mkdir -p $ccache_dir
-    ln -s /usr/bin/ccache $ccache_dir/gcc
-    ln -s /usr/bin/ccache $ccache_dir/g++
-    ln -s /usr/bin/ccache $ccache_dir/cc
-    ln -s /usr/bin/ccache $ccache_dir/c++
-    export PATH=$ccache_dir:$PATH
+        # Create fake compilers and prepend them to the PATH
+        # Note that yum is supposed to create these for us,
+        # but I had trouble finding them
+        ccache_dir='/usr/lib/ccache/compilers'
+        mkdir -p $ccache_dir
+        ln -s /usr/bin/ccache $ccache_dir/gcc
+        ln -s /usr/bin/ccache $ccache_dir/g++
+        ln -s /usr/bin/ccache $ccache_dir/cc
+        ln -s /usr/bin/ccache $ccache_dir/c++
+    fi
+    export PATH="$ccache_dir:$PATH"
 
     # Prove to the developer that ccache is activated
     echo "Using C compiler: $(which gcc)"

osx_utils.sh

@@ -10,16 +10,16 @@ MACPYTHON_URL=https://www.python.org/ftp/python
 MACPYTHON_PY_PREFIX=/Library/Frameworks/Python.framework/Versions
 WORKING_SDIR=working
 
-# As of 29 June 2021 - latest Python of each version with binary download
+# As of 24 March 2022 - latest Python of each version with binary download
 # available.
-# See: https://www.python.org/downloads/mac-osx/
+# See: https://www.python.org/downloads/macos/
 LATEST_2p7=2.7.18
 LATEST_3p5=3.5.4
 LATEST_3p6=3.6.8
 LATEST_3p7=3.7.9
 LATEST_3p8=3.8.10
-LATEST_3p9=3.9.6
-LATEST_3p10=3.10.0rc1
+LATEST_3p9=3.9.12
+LATEST_3p10=3.10.3
 
 
 function check_python {
@@ -440,7 +440,7 @@ function repair_wheelhouse {
 function install_pkg_config {
     # Install pkg-config avoiding error from homebrew
     # See :
-    # https://github.com/matthew-brett/multibuild/issues/24#issue-221951587
+    # https://github.com/multi-build/multibuild/issues/24#issue-221951587
     command -v pkg-config > /dev/null 2>&1 || brew install pkg-config
 }
 
@@ -484,6 +484,8 @@ function macos_arm64_cross_build_setup {
     export ARCHFLAGS+=" -arch arm64"
     export FCFLAGS+=" -arch arm64"
     export FC=$FC_ARM64
+    export F90=${F90_ARM64:-${FC}}
+    export F77=${F77_ARM64:-${FC}}
     export MACOSX_DEPLOYMENT_TARGET="11.0"
     export CROSS_COMPILING=1
     export LDFLAGS+=" -arch arm64 -L$BUILD_PREFIX/lib -Wl,-rpath,$BUILD_PREFIX/lib ${FC_ARM64_LDFLAGS:-}"
@@ -537,6 +539,12 @@ function wrap_wheel_builder {
         else
             (macos_arm64_cross_build_setup && $@)
         fi
+    elif [[ "${PLAT:-}" == "x86_64" ]]; then
+        if [[ "$(uname -m)" == "x86_64" ]]; then
+            (macos_intel_native_build_setup && $@)
+        else
+            (macos_intel_cross_build_setup && $@)
+        fi
     else
         $@
     fi

supported_wheels.py

@@ -26,13 +26,14 @@ def tags_for(fname):
 
 
 def main():
-    supported = {(tag.interpreter, tag.abi, tag.platform)
-                 for tag in sys_tags()
+    supported = {
+        (tag.interpreter, tag.abi, tag.platform) for tag in sys_tags()
     }
     for fname in sys.argv[1:]:
         tags = set(tags_for(fname))
         if supported.intersection(tags):
             print(fname)
 
+
 if __name__ == '__main__':
     main()

tests/test_library_builders.sh

@@ -27,7 +27,6 @@ fetch_unpack https://github.com/harfbuzz/harfbuzz/releases/download/2.7.4/harfbu
 suppress build_bzip2
 suppress build_openssl
 suppress build_libpng
-suppress build_libwebp
 suppress build_szip
 suppress build_swig
 # We need to find a failable test for build_github
@@ -35,7 +34,7 @@ suppress build_swig
 # E.g. arb (below) requires a couple of other libraries.
 # Run here just for the output, even though they fail.
 (set +e ;
-    build_github fredrik-johansson/arb 2.20.0 ;
+    build_github fredrik-johansson/arb 2.21.1 ;
     build_github glennrp/libpng v1.6.37 ;
     build_github wbhart/mpir mpir-3.0.0
     )
@@ -43,9 +42,10 @@ suppress build_flex
 if [[ $MB_ML_VER != "_2_24" ]]; then
     suppress build_openblas
 fi
+suppress ensure_xz
 suppress build_tiff
+suppress build_libwebp
 suppress build_lcms2
-suppress ensure_xz
 suppress build_freetype
 suppress build_libyaml
 if [ -z "$IS_MACOS" ]; then
@@ -61,6 +61,8 @@ if [ -z "$IS_MACOS" ]; then
 fi
 suppress build_new_zlib
 suppress build_hdf5
+rm jpeg-stamp
+suppress build_libjpeg_turbo
 suppress get_modern_cmake
 
 [ ${MB_PYTHON_VERSION+x} ] || ingest "\$MB_PYTHON_VERSION is not set"

tests/test_osx_utils.sh

@@ -68,3 +68,10 @@
 
 # Test pkg-config install
 install_pkg_config
+
+function echo_host_platform {
+    echo $_PYTHON_HOST_PLATFORM
+}
+
+# Make sure `_PYTHON_HOST_PLATFORM` is set when building x86_64 thin wheel
+(PLAT="x86_64"; MB_PYTHON_OSX_VER="10.9"; [ "$(wrap_wheel_builder echo_host_platform)" == "macosx-10.9-x86_64" ])

travis_linux_steps.sh

@@ -69,6 +69,7 @@ function build_multilinux {
     # Depends on
     #     MB_PYTHON_VERSION
     #     MB_ML_VER
+    #     MB_ML_LIBC (optional)
     #     UNICODE_WIDTH (optional)
     #     BUILD_DEPENDS (optional)
     #     DOCKER_IMAGE (optional)
@@ -77,7 +78,8 @@ function build_multilinux {
     local plat=$1
     [ -z "$plat" ] && echo "plat not defined" && exit 1
     local build_cmds="$2"
-    local docker_image=${DOCKER_IMAGE:-quay.io/pypa/manylinux${MB_ML_VER}_\$plat}
+    local libc=${MB_ML_LIBC:-manylinux}
+    local docker_image=${DOCKER_IMAGE:-quay.io/pypa/${libc}${MB_ML_VER}_\$plat}
     docker_image=$(eval echo "$docker_image")
     retry docker pull $docker_image
     docker run --rm \
@@ -95,6 +97,7 @@ function build_multilinux {
         -e REPO_DIR="$repo_dir" \
         -e PLAT="$PLAT" \
         -e MB_ML_VER="$MB_ML_VER" \
+        -e MB_ML_LIBC="$libc" \
         -v $PWD:/io \
         -v $HOME:/parent-home \
         $docker_image /io/$MULTIBUILD_DIR/docker_build_wrap.sh
@@ -108,6 +111,7 @@ function install_run {
     # Depends on
     #  PLAT (can be passed in as argument)
     #  MB_PYTHON_VERSION
+    #  MB_ML_LIBC (optional)
     #  UNICODE_WIDTH (optional)
     #  WHEEL_SDIR (optional)
     #  MANYLINUX_URL (optional)
@@ -115,11 +119,15 @@ function install_run {
     #  MB_TEST_VER (optional)
     local plat=${1:-${PLAT:-x86_64}}
     if [ -z "$DOCKER_TEST_IMAGE" ]; then
-        local bitness=$([ "$plat" == i686 ] && echo 32 || echo 64)
-        if [ "$bitness" == "32" ]; then
-            local docker_image="matthewbrett/trusty:$bitness"
+        if [ "$MB_ML_LIBC" == "musllinux" ]; then
+            local docker_image="multibuild/alpine3.14_$plat"
         else
-            local docker_image="multibuild/focal_x86_64"
+            local bitness=$([ "$plat" == i686 ] && echo 32 || echo 64)
+            if [ "$bitness" == "32" ]; then
+                local docker_image="matthewbrett/trusty:$bitness"
+            else
+                local docker_image="multibuild/focal_x86_64"
+            fi
         fi
     else
         # aarch64 is called arm64v8 in Ubuntu