Unverified Commit fd61f503 authored by Henry Schreiner's avatar Henry Schreiner Committed by GitHub
Browse files

feat: setup.py redesign and helpers (#2433)

* feat: setup.py redesign and helpers

* refactor: simpler design with two outputs

* refactor: helper file update and Windows support

* fix: review points from @YannickJadoul

* refactor: fixes to naming and more docs

* feat: more customization points

* feat: add entry point pybind11-config

* refactor: Try Extension-focused method

* refactor: rename alt/inplace to global

* fix: allow usage with git modules, better docs

* feat: global as an extra (@YannickJadoul's suggestion)

* feat: single version location

* fix: remove the requirement that setuptools must be imported first

* fix: some review points from @wjacob

* fix: use .in, add procedure to docs

* refactor: avoid monkeypatch copy

* docs: minor typos corrected

* fix: minor points from @YannickJadoul

* fix: typo on Windows C++ mode

* fix: MSVC 15 update 3+ have c++14 flag

See <https://docs.microsoft.com/en-us/cpp/build/reference/std-specify-language-standard-version?view=vs-2019>

* docs: discuss making SDists by hand

* ci: use pep517.build instead of manual setup.py

* refactor: more comments from @YannickJadoul

* docs: updates from @ktbarrett

* fix: change to newly recommended tool instead of pep517.build

This was intended as a proof of concept; build seems to be the correct replacement.

See https://github.com/pypa/pep517/pull/83

* docs: updates from @wjakob

* refactor: dual version locations

* docs: typo spotted by @wjakob
parent 41aa9260
......@@ -3,128 +3,113 @@
# Setup script for PyPI; use CMakeFile.txt to build extension modules
from setuptools import setup
from distutils.command.install_headers import install_headers
from distutils.command.build_py import build_py
from pybind11 import __version__
import contextlib
import os
import re
import shutil
import string
import subprocess
import sys
import tempfile
package_data = [
'include/pybind11/detail/class.h',
'include/pybind11/detail/common.h',
'include/pybind11/detail/descr.h',
'include/pybind11/detail/init.h',
'include/pybind11/detail/internals.h',
'include/pybind11/detail/typeid.h',
'include/pybind11/attr.h',
'include/pybind11/buffer_info.h',
'include/pybind11/cast.h',
'include/pybind11/chrono.h',
'include/pybind11/common.h',
'include/pybind11/complex.h',
'include/pybind11/eigen.h',
'include/pybind11/embed.h',
'include/pybind11/eval.h',
'include/pybind11/functional.h',
'include/pybind11/iostream.h',
'include/pybind11/numpy.h',
'include/pybind11/operators.h',
'include/pybind11/options.h',
'include/pybind11/pybind11.h',
'include/pybind11/pytypes.h',
'include/pybind11/stl.h',
'include/pybind11/stl_bind.h',
]
# Prevent installation of pybind11 headers by setting
# PYBIND11_USE_CMAKE.
if os.environ.get('PYBIND11_USE_CMAKE'):
headers = []
else:
headers = package_data
class InstallHeaders(install_headers):
"""Use custom header installer because the default one flattens subdirectories"""
def run(self):
if not self.distribution.headers:
return
for header in self.distribution.headers:
subdir = os.path.dirname(os.path.relpath(header, 'include/pybind11'))
install_dir = os.path.join(self.install_dir, subdir)
self.mkpath(install_dir)
(out, _) = self.copy_file(header, install_dir)
self.outfiles.append(out)
# Install the headers inside the package as well
class BuildPy(build_py):
def build_package_data(self):
build_py.build_package_data(self)
for header in package_data:
target = os.path.join(self.build_lib, 'pybind11', header)
self.mkpath(os.path.dirname(target))
self.copy_file(header, target, preserve_mode=False)
def get_outputs(self, include_bytecode=1):
outputs = build_py.get_outputs(self, include_bytecode=include_bytecode)
for header in package_data:
target = os.path.join(self.build_lib, 'pybind11', header)
outputs.append(target)
return outputs
setup(
name='pybind11',
version=__version__,
description='Seamless operability between C++11 and Python',
author='Wenzel Jakob',
author_email='wenzel.jakob@epfl.ch',
url='https://github.com/pybind/pybind11',
download_url='https://github.com/pybind/pybind11/tarball/v' + __version__,
packages=['pybind11'],
license='BSD',
headers=headers,
zip_safe=False,
cmdclass=dict(install_headers=InstallHeaders, build_py=BuildPy),
classifiers=[
'Development Status :: 5 - Production/Stable',
'Intended Audience :: Developers',
'Topic :: Software Development :: Libraries :: Python Modules',
'Topic :: Utilities',
'Programming Language :: C++',
'Programming Language :: Python :: 2.7',
'Programming Language :: Python :: 3',
'Programming Language :: Python :: 3.2',
'Programming Language :: Python :: 3.3',
'Programming Language :: Python :: 3.4',
'Programming Language :: Python :: 3.5',
'Programming Language :: Python :: 3.6',
'License :: OSI Approved :: BSD License'
],
keywords='C++11, Python bindings',
long_description="""pybind11 is a lightweight header-only library that
exposes C++ types in Python and vice versa, mainly to create Python bindings of
existing C++ code. Its goals and syntax are similar to the excellent
Boost.Python by David Abrahams: to minimize boilerplate code in traditional
extension modules by inferring type information using compile-time
introspection.
The main issue with Boost.Python-and the reason for creating such a similar
project-is Boost. Boost is an enormously large and complex suite of utility
libraries that works with almost every C++ compiler in existence. This
compatibility has its cost: arcane template tricks and workarounds are
necessary to support the oldest and buggiest of compiler specimens. Now that
C++11-compatible compilers are widely available, this heavy machinery has
become an excessively large and unnecessary dependency.
Think of this library as a tiny self-contained version of Boost.Python with
everything stripped away that isn't relevant for binding generation. Without
comments, the core header files only require ~4K lines of code and depend on
Python (2.7 or 3.x, or PyPy2.7 >= 5.7) and the C++ standard library. This
compact implementation was possible thanks to some of the new C++11 language
features (specifically: tuples, lambda functions and variadic templates). Since
its creation, this library has grown beyond Boost.Python in many ways, leading
to dramatically simpler binding code in many common situations.""")
import setuptools.command.sdist
DIR = os.path.abspath(os.path.dirname(__file__))
VERSION_REGEX = re.compile(
r"^\s*#\s*define\s+PYBIND11_VERSION_([A-Z]+)\s+(.*)$", re.MULTILINE
)
# PYBIND11_GLOBAL_SDIST will build a different sdist, with the python-headers
# files, and the sys.prefix files (CMake and headers).
global_sdist = os.environ.get("PYBIND11_GLOBAL_SDIST", False)
setup_py = "tools/setup_global.py.in" if global_sdist else "tools/setup_main.py.in"
extra_cmd = 'cmdclass["sdist"] = SDist\n'
to_src = (
("pyproject.toml", "tools/pyproject.toml"),
("setup.py", setup_py),
)
# Read the listed version
with open("pybind11/_version.py") as f:
code = compile(f.read(), "pybind11/_version.py", "exec")
loc = {}
exec(code, loc)
version = loc["__version__"]
# Verify that the version matches the one in C++
with open("include/pybind11/detail/common.h") as f:
matches = dict(VERSION_REGEX.findall(f.read()))
cpp_version = "{MAJOR}.{MINOR}.{PATCH}".format(**matches)
if version != cpp_version:
msg = "Python version {} does not match C++ version {}!".format(
version, cpp_version
)
raise RuntimeError(msg)
def get_and_replace(filename, binary=False, **opts):
with open(filename, "rb" if binary else "r") as f:
contents = f.read()
# Replacement has to be done on text in Python 3 (both work in Python 2)
if binary:
return string.Template(contents.decode()).substitute(opts).encode()
else:
return string.Template(contents).substitute(opts)
# Use our input files instead when making the SDist (and anything that depends
# on it, like a wheel)
class SDist(setuptools.command.sdist.sdist):
def make_release_tree(self, base_dir, files):
setuptools.command.sdist.sdist.make_release_tree(self, base_dir, files)
for to, src in to_src:
txt = get_and_replace(src, binary=True, version=version, extra_cmd="")
dest = os.path.join(base_dir, to)
# This is normally linked, so unlink before writing!
os.unlink(dest)
with open(dest, "wb") as f:
f.write(txt)
# Backport from Python 3
@contextlib.contextmanager
def TemporaryDirectory(): # noqa: N802
"Prepare a temporary directory, cleanup when done"
try:
tmpdir = tempfile.mkdtemp()
yield tmpdir
finally:
shutil.rmtree(tmpdir)
# Remove the CMake install directory when done
@contextlib.contextmanager
def remove_output(*sources):
try:
yield
finally:
for src in sources:
shutil.rmtree(src)
with remove_output("pybind11/include", "pybind11/share"):
# Generate the files if they are not present.
with TemporaryDirectory() as tmpdir:
cmd = ["cmake", "-S", ".", "-B", tmpdir] + [
"-DCMAKE_INSTALL_PREFIX=pybind11",
"-DBUILD_TESTING=OFF",
"-DPYBIND11_NOPYTHON=ON",
]
cmake_opts = dict(cwd=DIR, stdout=sys.stdout, stderr=sys.stderr)
subprocess.check_call(cmd, **cmake_opts)
subprocess.check_call(["cmake", "--install", tmpdir], **cmake_opts)
txt = get_and_replace(setup_py, version=version, extra_cmd=extra_cmd)
code = compile(txt, setup_py, "exec")
exec(code, {"SDist": SDist})
# -*- coding: utf-8 -*-
import contextlib
import os
import string
import subprocess
import sys
import tarfile
import zipfile
# These tests must be run explicitly
# They require CMake 3.15+ (--install)
DIR = os.path.abspath(os.path.dirname(__file__))
MAIN_DIR = os.path.dirname(os.path.dirname(DIR))
main_headers = {
"include/pybind11/attr.h",
"include/pybind11/buffer_info.h",
"include/pybind11/cast.h",
"include/pybind11/chrono.h",
"include/pybind11/common.h",
"include/pybind11/complex.h",
"include/pybind11/eigen.h",
"include/pybind11/embed.h",
"include/pybind11/eval.h",
"include/pybind11/functional.h",
"include/pybind11/iostream.h",
"include/pybind11/numpy.h",
"include/pybind11/operators.h",
"include/pybind11/options.h",
"include/pybind11/pybind11.h",
"include/pybind11/pytypes.h",
"include/pybind11/stl.h",
"include/pybind11/stl_bind.h",
}
detail_headers = {
"include/pybind11/detail/class.h",
"include/pybind11/detail/common.h",
"include/pybind11/detail/descr.h",
"include/pybind11/detail/init.h",
"include/pybind11/detail/internals.h",
"include/pybind11/detail/typeid.h",
}
cmake_files = {
"share/cmake/pybind11/FindPythonLibsNew.cmake",
"share/cmake/pybind11/pybind11Common.cmake",
"share/cmake/pybind11/pybind11Config.cmake",
"share/cmake/pybind11/pybind11ConfigVersion.cmake",
"share/cmake/pybind11/pybind11NewTools.cmake",
"share/cmake/pybind11/pybind11Targets.cmake",
"share/cmake/pybind11/pybind11Tools.cmake",
}
py_files = {
"__init__.py",
"__main__.py",
"_version.py",
"commands.py",
"setup_helpers.py",
}
headers = main_headers | detail_headers
src_files = headers | cmake_files
all_files = src_files | py_files
sdist_files = {
"pybind11",
"pybind11/include",
"pybind11/include/pybind11",
"pybind11/include/pybind11/detail",
"pybind11/share",
"pybind11/share/cmake",
"pybind11/share/cmake/pybind11",
"pyproject.toml",
"setup.cfg",
"setup.py",
"LICENSE",
"MANIFEST.in",
"README.md",
"PKG-INFO",
}
local_sdist_files = {
".egg-info",
".egg-info/PKG-INFO",
".egg-info/SOURCES.txt",
".egg-info/dependency_links.txt",
".egg-info/not-zip-safe",
".egg-info/top_level.txt",
}
def test_build_sdist(monkeypatch, tmpdir):
monkeypatch.chdir(MAIN_DIR)
out = subprocess.check_output(
[
sys.executable,
"setup.py",
"sdist",
"--formats=tar",
"--dist-dir",
str(tmpdir),
]
)
if hasattr(out, "decode"):
out = out.decode()
(sdist,) = tmpdir.visit("*.tar")
with tarfile.open(str(sdist)) as tar:
start = tar.getnames()[0] + "/"
version = start[9:-1]
simpler = set(n.split("/", 1)[-1] for n in tar.getnames()[1:])
with contextlib.closing(
tar.extractfile(tar.getmember(start + "setup.py"))
) as f:
setup_py = f.read()
with contextlib.closing(
tar.extractfile(tar.getmember(start + "pyproject.toml"))
) as f:
pyproject_toml = f.read()
files = set("pybind11/{}".format(n) for n in all_files)
files |= sdist_files
files |= set("pybind11{}".format(n) for n in local_sdist_files)
files.add("pybind11.egg-info/entry_points.txt")
files.add("pybind11.egg-info/requires.txt")
assert simpler == files
with open(os.path.join(MAIN_DIR, "tools", "setup_main.py.in"), "rb") as f:
contents = (
string.Template(f.read().decode())
.substitute(version=version, extra_cmd="")
.encode()
)
assert setup_py == contents
with open(os.path.join(MAIN_DIR, "tools", "pyproject.toml"), "rb") as f:
contents = f.read()
assert pyproject_toml == contents
def test_build_global_dist(monkeypatch, tmpdir):
monkeypatch.chdir(MAIN_DIR)
monkeypatch.setenv("PYBIND11_GLOBAL_SDIST", "1")
out = subprocess.check_output(
[
sys.executable,
"setup.py",
"sdist",
"--formats=tar",
"--dist-dir",
str(tmpdir),
]
)
if hasattr(out, "decode"):
out = out.decode()
(sdist,) = tmpdir.visit("*.tar")
with tarfile.open(str(sdist)) as tar:
start = tar.getnames()[0] + "/"
version = start[16:-1]
simpler = set(n.split("/", 1)[-1] for n in tar.getnames()[1:])
with contextlib.closing(
tar.extractfile(tar.getmember(start + "setup.py"))
) as f:
setup_py = f.read()
with contextlib.closing(
tar.extractfile(tar.getmember(start + "pyproject.toml"))
) as f:
pyproject_toml = f.read()
files = set("pybind11/{}".format(n) for n in all_files)
files |= sdist_files
files |= set("pybind11_global{}".format(n) for n in local_sdist_files)
assert simpler == files
with open(os.path.join(MAIN_DIR, "tools", "setup_global.py.in"), "rb") as f:
contents = (
string.Template(f.read().decode())
.substitute(version=version, extra_cmd="")
.encode()
)
assert setup_py == contents
with open(os.path.join(MAIN_DIR, "tools", "pyproject.toml"), "rb") as f:
contents = f.read()
assert pyproject_toml == contents
def tests_build_wheel(monkeypatch, tmpdir):
monkeypatch.chdir(MAIN_DIR)
subprocess.check_output(
[sys.executable, "-m", "pip", "wheel", ".", "-w", str(tmpdir)]
)
(wheel,) = tmpdir.visit("*.whl")
files = set("pybind11/{}".format(n) for n in all_files)
files |= {
"dist-info/LICENSE",
"dist-info/METADATA",
"dist-info/RECORD",
"dist-info/WHEEL",
"dist-info/entry_points.txt",
"dist-info/top_level.txt",
}
with zipfile.ZipFile(str(wheel)) as z:
names = z.namelist()
trimmed = set(n for n in names if "dist-info" not in n)
trimmed |= set(
"dist-info/{}".format(n.split("/", 1)[-1]) for n in names if "dist-info" in n
)
assert files == trimmed
def tests_build_global_wheel(monkeypatch, tmpdir):
monkeypatch.chdir(MAIN_DIR)
monkeypatch.setenv("PYBIND11_GLOBAL_SDIST", "1")
subprocess.check_output(
[sys.executable, "-m", "pip", "wheel", ".", "-w", str(tmpdir)]
)
(wheel,) = tmpdir.visit("*.whl")
files = set("data/data/{}".format(n) for n in src_files)
files |= set("data/headers/{}".format(n[8:]) for n in headers)
files |= {
"dist-info/LICENSE",
"dist-info/METADATA",
"dist-info/WHEEL",
"dist-info/top_level.txt",
"dist-info/RECORD",
}
with zipfile.ZipFile(str(wheel)) as z:
names = z.namelist()
beginning = names[0].split("/", 1)[0].rsplit(".", 1)[0]
trimmed = set(n[len(beginning) + 1 :] for n in names)
assert files == trimmed
# -*- coding: utf-8 -*-
import os
import sys
import subprocess
from textwrap import dedent
import pytest
DIR = os.path.abspath(os.path.dirname(__file__))
MAIN_DIR = os.path.dirname(os.path.dirname(DIR))
@pytest.mark.parametrize("std", [11, 0])
def test_simple_setup_py(monkeypatch, tmpdir, std):
monkeypatch.chdir(tmpdir)
monkeypatch.syspath_prepend(MAIN_DIR)
(tmpdir / "setup.py").write_text(
dedent(
u"""\
import sys
sys.path.append({MAIN_DIR!r})
from setuptools import setup, Extension
from pybind11.setup_helpers import build_ext, Pybind11Extension
std = {std}
ext_modules = [
Pybind11Extension(
"simple_setup",
sorted(["main.cpp"]),
cxx_std=std,
),
]
cmdclass = dict()
if std == 0:
cmdclass["build_ext"] = build_ext
setup(
name="simple_setup_package",
cmdclass=cmdclass,
ext_modules=ext_modules,
)
"""
).format(MAIN_DIR=MAIN_DIR, std=std),
encoding="ascii",
)
(tmpdir / "main.cpp").write_text(
dedent(
u"""\
#include <pybind11/pybind11.h>
int f(int x) {
return x * 3;
}
PYBIND11_MODULE(simple_setup, m) {
m.def("f", &f);
}
"""
),
encoding="ascii",
)
subprocess.check_call(
[sys.executable, "setup.py", "build_ext", "--inplace"],
stdout=sys.stdout,
stderr=sys.stderr,
)
# Debug helper printout, normally hidden
for item in tmpdir.listdir():
print(item.basename)
assert (
len([f for f in tmpdir.listdir() if f.basename.startswith("simple_setup")]) == 1
)
assert len(list(tmpdir.listdir())) == 4 # two files + output + build_dir
(tmpdir / "test.py").write_text(
dedent(
u"""\
import simple_setup
assert simple_setup.f(3) == 9
"""
),
encoding="ascii",
)
subprocess.check_call(
[sys.executable, "test.py"], stdout=sys.stdout, stderr=sys.stderr
)
[pytest]
minversion = 3.1
norecursedirs = test_cmake_build test_embed
norecursedirs = test_* extra_*
xfail_strict = True
addopts =
# show summary of skipped tests
......
......@@ -300,7 +300,7 @@ _pybind11_generate_lto(pybind11::thin_lto TRUE)
# ---------------------- pybind11_strip -----------------------------
function(pybind11_strip target_name)
# Strip unnecessary sections of the binary on Linux/Mac OS
# Strip unnecessary sections of the binary on Linux/macOS
if(CMAKE_STRIP)
if(APPLE)
set(x_opt -x)
......
......@@ -197,7 +197,7 @@ function(pybind11_add_module target_name)
endif()
if(NOT MSVC AND NOT ${CMAKE_BUILD_TYPE} MATCHES Debug|RelWithDebInfo)
# Strip unnecessary sections of the binary on Linux/Mac OS
# Strip unnecessary sections of the binary on Linux/macOS
pybind11_strip(${target_name})
endif()
......
[build-system]
requires = ["setuptools", "wheel"]
build-backend = "setuptools.build_meta"
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Setup script for pybind11-global (in the sdist or in tools/setup_global.py in the repository)
# This package is targeted for easy use from CMake.
import contextlib
import glob
import os
import re
import shutil
import subprocess
import sys
import tempfile
# Setuptools has to be before distutils
from setuptools import setup
from distutils.command.install_headers import install_headers
class InstallHeadersNested(install_headers):
def run(self):
headers = self.distribution.headers or []
for header in headers:
# Remove pybind11/include/
short_header = header.split("/", 2)[-1]
dst = os.path.join(self.install_dir, os.path.dirname(short_header))
self.mkpath(dst)
(out, _) = self.copy_file(header, dst)
self.outfiles.append(out)
main_headers = glob.glob("pybind11/include/pybind11/*.h")
detail_headers = glob.glob("pybind11/include/pybind11/detail/*.h")
cmake_files = glob.glob("pybind11/share/cmake/pybind11/*.cmake")
headers = main_headers + detail_headers
cmdclass = {"install_headers": InstallHeadersNested}
$extra_cmd
setup(
name="pybind11_global",
version="$version",
packages=[],
headers=headers,
data_files=[
("share/cmake/pybind11", cmake_files),
("include/pybind11", main_headers),
("include/pybind11/detail", detail_headers),
],
cmdclass=cmdclass,
)
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Setup script (in the sdist or in tools/setup_main.py in the repository)
from setuptools import setup
cmdclass = {}
$extra_cmd
setup(
name="pybind11",
version="$version",
download_url='https://github.com/pybind/pybind11/tarball/v$version',
packages=[
"pybind11",
"pybind11.include.pybind11",
"pybind11.include.pybind11.detail",
"pybind11.share.cmake.pybind11",
],
package_data={
"pybind11.include.pybind11": ["*.h"],
"pybind11.include.pybind11.detail": ["*.h"],
"pybind11.share.cmake.pybind11": ["*.cmake"],
},
extras_require={
"global": ["pybind11_global==$version"]
},
entry_points={
"console_scripts": [
"pybind11-config = pybind11.__main__:main",
]
},
cmdclass=cmdclass
)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment