Merge branch 'master' into stable

include/pybind11/detail/common.h

@@ -10,12 +10,12 @@
 #pragma once
 
 #define PYBIND11_VERSION_MAJOR 2
-#define PYBIND11_VERSION_MINOR 8
-#define PYBIND11_VERSION_PATCH 1
+#define PYBIND11_VERSION_MINOR 9
+#define PYBIND11_VERSION_PATCH 0
 
 // Similar to Python's convention: https://docs.python.org/3/c-api/apiabiversion.html
 // Additional convention: 0xD = dev
-#define PYBIND11_VERSION_HEX 0x02080100
+#define PYBIND11_VERSION_HEX 0x02090000
 
 #define PYBIND11_NAMESPACE_BEGIN(name) namespace name {
 #define PYBIND11_NAMESPACE_END(name) }
@@ -154,6 +154,14 @@
 // C4505: 'PySlice_GetIndicesEx': unreferenced local function has been removed (PyPy only)
 #  pragma warning(disable: 4505)
 #  if defined(_DEBUG) && !defined(Py_DEBUG)
+// Workaround for a VS 2022 issue.
+// NOTE: This workaround knowingly violates the Python.h include order requirement:
+// https://docs.python.org/3/c-api/intro.html#include-files
+// See https://github.com/pybind/pybind11/pull/3497 for full context.
+#    include <yvals.h>
+#    if _MSVC_STL_VERSION >= 143
+#      include <crtdefs.h>
+#    endif
 #    define PYBIND11_DEBUG_MARKER
 #    undef _DEBUG
 #  endif
@@ -183,6 +191,21 @@
 #  define PYBIND11_HAS_VARIANT 1
 #endif
 
+#if defined(PYBIND11_CPP17)
+#  if defined(__has_include)
+#    if __has_include(<string_view>)
+#      define PYBIND11_HAS_STRING_VIEW
+#    endif
+#  elif defined(_MSC_VER)
+#    define PYBIND11_HAS_STRING_VIEW
+#  endif
+#endif
+
+#if defined(__cpp_lib_char8_t) && __cpp_lib_char8_t >= 201811L
+#  define PYBIND11_HAS_U8STRING
+#endif
+
+
 #include <Python.h>
 #include <frameobject.h>
 #include <pythread.h>

include/pybind11/detail/descr.h

@@ -54,8 +54,8 @@ constexpr descr<N1 + N2, Ts1..., Ts2...> operator+(const descr<N1, Ts1...> &a, c
 }
 
 template <size_t N>
-constexpr descr<N - 1> _(char const(&text)[N]) { return descr<N - 1>(text); }
-constexpr descr<0> _(char const(&)[1]) { return {}; }
+constexpr descr<N - 1> const_name(char const(&text)[N]) { return descr<N - 1>(text); }
+constexpr descr<0> const_name(char const(&)[1]) { return {}; }
 
 template <size_t Rem, size_t... Digits> struct int_to_str : int_to_str<Rem/10, Rem%10, Digits...> { };
 template <size_t...Digits> struct int_to_str<0, Digits...> {
@@ -64,25 +64,50 @@ template <size_t...Digits> struct int_to_str<0, Digits...> {
 
 // Ternary description (like std::conditional)
 template <bool B, size_t N1, size_t N2>
-constexpr enable_if_t<B, descr<N1 - 1>> _(char const(&text1)[N1], char const(&)[N2]) {
-    return _(text1);
+constexpr enable_if_t<B, descr<N1 - 1>> const_name(char const(&text1)[N1], char const(&)[N2]) {
+    return const_name(text1);
 }
 template <bool B, size_t N1, size_t N2>
-constexpr enable_if_t<!B, descr<N2 - 1>> _(char const(&)[N1], char const(&text2)[N2]) {
-    return _(text2);
+constexpr enable_if_t<!B, descr<N2 - 1>> const_name(char const(&)[N1], char const(&text2)[N2]) {
+    return const_name(text2);
 }
 
 template <bool B, typename T1, typename T2>
-constexpr enable_if_t<B, T1> _(const T1 &d, const T2 &) { return d; }
+constexpr enable_if_t<B, T1> const_name(const T1 &d, const T2 &) { return d; }
 template <bool B, typename T1, typename T2>
-constexpr enable_if_t<!B, T2> _(const T1 &, const T2 &d) { return d; }
+constexpr enable_if_t<!B, T2> const_name(const T1 &, const T2 &d) { return d; }
 
 template <size_t Size>
-auto constexpr _() -> remove_cv_t<decltype(int_to_str<Size / 10, Size % 10>::digits)> {
+auto constexpr const_name() -> remove_cv_t<decltype(int_to_str<Size / 10, Size % 10>::digits)> {
     return int_to_str<Size / 10, Size % 10>::digits;
 }
 
-template <typename Type> constexpr descr<1, Type> _() { return {'%'}; }
+template <typename Type> constexpr descr<1, Type> const_name() { return {'%'}; }
+
+// The "_" might be defined as a macro - don't define it if so.
+// Repeating the const_name code to avoid introducing a #define.
+#ifndef _
+template <size_t N>
+constexpr descr<N-1> _(char const(&text)[N]) { return const_name<N>(text); }
+template <bool B, size_t N1, size_t N2>
+constexpr enable_if_t<B, descr<N1 - 1>> _(char const(&text1)[N1], char const(&text2)[N2]) {
+    return const_name<B,N1,N2>(text1, text2);
+}
+template <bool B, size_t N1, size_t N2>
+constexpr enable_if_t<!B, descr<N2 - 1>> _(char const(&text1)[N1], char const(&text2)[N2]) {
+    return const_name<B,N1,N2>(text1, text2);
+}
+template <bool B, typename T1, typename T2>
+constexpr enable_if_t<B, T1> _(const T1 &d1, const T2 &d2) { return const_name<B,T1,T2>(d1, d2); }
+template <bool B, typename T1, typename T2>
+constexpr enable_if_t<!B, T2> _(const T1 &d1, const T2 &d2) { return const_name<B,T1,T2>(d1, d2); }
+
+template <size_t Size>
+auto constexpr _() -> remove_cv_t<decltype(int_to_str<Size / 10, Size % 10>::digits)> {
+    return const_name<Size>();
+}
+template <typename Type> constexpr descr<1, Type> _() { return const_name<Type>(); }
+#endif
 
 constexpr descr<0> concat() { return {}; }
 
@@ -92,12 +117,12 @@ constexpr descr<N, Ts...> concat(const descr<N, Ts...> &descr) { return descr; }
 template <size_t N, typename... Ts, typename... Args>
 constexpr auto concat(const descr<N, Ts...> &d, const Args &...args)
     -> decltype(std::declval<descr<N + 2, Ts...>>() + concat(args...)) {
-    return d + _(", ") + concat(args...);
+    return d + const_name(", ") + concat(args...);
 }
 
 template <size_t N, typename... Ts>
 constexpr descr<N + 2, Ts...> type_descr(const descr<N, Ts...> &descr) {
-    return _("{") + descr + _("}");
+    return const_name("{") + descr + const_name("}");
 }
 
 PYBIND11_NAMESPACE_END(detail)

include/pybind11/detail/init.h

@@ -24,7 +24,7 @@ public:
 
     template <typename> using cast_op_type = value_and_holder &;
     explicit operator value_and_holder &() { return *value; }
-    static constexpr auto name = _<value_and_holder>();
+    static constexpr auto name = const_name<value_and_holder>();
 
 private:
     value_and_holder *value = nullptr;

include/pybind11/detail/type_caster_base.h

@@ -897,7 +897,7 @@ template <typename type> class type_caster_base : public type_caster_generic {
     using itype = intrinsic_t<type>;
 
 public:
-    static constexpr auto name = _<type>();
+    static constexpr auto name = const_name<type>();
 
     type_caster_base() : type_caster_base(typeid(type)) { }
     explicit type_caster_base(const std::type_info &info) : type_caster_generic(info) { }

include/pybind11/eigen.h

@@ -50,8 +50,12 @@ PYBIND11_NAMESPACE_BEGIN(detail)
 
 #if EIGEN_VERSION_AT_LEAST(3,3,0)
 using EigenIndex = Eigen::Index;
+template<typename Scalar, int Flags, typename StorageIndex>
+using EigenMapSparseMatrix = Eigen::Map<Eigen::SparseMatrix<Scalar, Flags, StorageIndex>>;
 #else
 using EigenIndex = EIGEN_DEFAULT_DENSE_INDEX_TYPE;
+template<typename Scalar, int Flags, typename StorageIndex>
+using EigenMapSparseMatrix = Eigen::MappedSparseMatrix<Scalar, Flags, StorageIndex>;
 #endif
 
 // Matches Eigen::Map, Eigen::Ref, blocks, etc:
@@ -80,14 +84,12 @@ template <bool EigenRowMajor> struct EigenConformable {
     // Matrix type:
     EigenConformable(EigenIndex r, EigenIndex c,
             EigenIndex rstride, EigenIndex cstride) :
-        conformable{true}, rows{r}, cols{c} {
-        // TODO: when Eigen bug #747 is fixed, remove the tests for non-negativity. http://eigen.tuxfamily.org/bz/show_bug.cgi?id=747
-        if (rstride < 0 || cstride < 0) {
-            negativestrides = true;
-        } else {
-            stride = {EigenRowMajor ? rstride : cstride /* outer stride */,
-                      EigenRowMajor ? cstride : rstride /* inner stride */ };
-        }
+        conformable{true}, rows{r}, cols{c},
+        //TODO: when Eigen bug #747 is fixed, remove the tests for non-negativity. http://eigen.tuxfamily.org/bz/show_bug.cgi?id=747
+        stride{EigenRowMajor ? (rstride > 0 ? rstride : 0) : (cstride > 0 ? cstride : 0) /* outer stride */,
+               EigenRowMajor ? (cstride > 0 ? cstride : 0) : (rstride > 0 ? rstride : 0) /* inner stride */ },
+        negativestrides{rstride < 0 || cstride < 0} {
+
     }
     // Vector type:
     EigenConformable(EigenIndex r, EigenIndex c, EigenIndex stride)
@@ -190,20 +192,20 @@ template <typename Type_> struct EigenProps {
     static constexpr bool show_f_contiguous = !show_c_contiguous && show_order && requires_col_major;
 
     static constexpr auto descriptor =
-        _("numpy.ndarray[") + npy_format_descriptor<Scalar>::name +
-        _("[")  + _<fixed_rows>(_<(size_t) rows>(), _("m")) +
-        _(", ") + _<fixed_cols>(_<(size_t) cols>(), _("n")) +
-        _("]") +
+        const_name("numpy.ndarray[") + npy_format_descriptor<Scalar>::name +
+        const_name("[")  + const_name<fixed_rows>(const_name<(size_t) rows>(), const_name("m")) +
+        const_name(", ") + const_name<fixed_cols>(const_name<(size_t) cols>(), const_name("n")) +
+        const_name("]") +
         // For a reference type (e.g. Ref<MatrixXd>) we have other constraints that might need to be
         // satisfied: writeable=True (for a mutable reference), and, depending on the map's stride
         // options, possibly f_contiguous or c_contiguous.  We include them in the descriptor output
         // to provide some hint as to why a TypeError is occurring (otherwise it can be confusing to
         // see that a function accepts a 'numpy.ndarray[float64[3,2]]' and an error message that you
         // *gave* a numpy.ndarray of the right type and dimensions.
-        _<show_writeable>(", flags.writeable", "") +
-        _<show_c_contiguous>(", flags.c_contiguous", "") +
-        _<show_f_contiguous>(", flags.f_contiguous", "") +
-        _("]");
+        const_name<show_writeable>(", flags.writeable", "") +
+        const_name<show_c_contiguous>(", flags.c_contiguous", "") +
+        const_name<show_f_contiguous>(", flags.f_contiguous", "") +
+        const_name("]");
 };
 
 // Casts an Eigen type to numpy array.  If given a base, the numpy array references the src data,
@@ -573,7 +575,7 @@ struct type_caster<Type, enable_if_t<is_eigen_sparse<Type>::value>> {
         if (!values || !innerIndices || !outerIndices)
             return false;
 
-        value = Eigen::MappedSparseMatrix<Scalar,
+        value = EigenMapSparseMatrix<Scalar,
                                      Type::Flags & (Eigen::RowMajor | Eigen::ColMajor),
                                      StorageIndex>(
             shape[0].cast<Index>(), shape[1].cast<Index>(), nnz,
@@ -598,8 +600,8 @@ struct type_caster<Type, enable_if_t<is_eigen_sparse<Type>::value>> {
         ).release();
     }
 
-    PYBIND11_TYPE_CASTER(Type, _<(Type::IsRowMajor) != 0>("scipy.sparse.csr_matrix[", "scipy.sparse.csc_matrix[")
-            + npy_format_descriptor<Scalar>::name + _("]"));
+    PYBIND11_TYPE_CASTER(Type, const_name<(Type::IsRowMajor) != 0>("scipy.sparse.csr_matrix[", "scipy.sparse.csc_matrix[")
+            + npy_format_descriptor<Scalar>::name + const_name("]"));
 };
 
 PYBIND11_NAMESPACE_END(detail)

include/pybind11/embed.h

@@ -102,6 +102,13 @@ inline wchar_t *widen_chars(const char *safe_arg) {
     wchar_t *widened_arg = Py_DecodeLocale(safe_arg, nullptr);
 #else
     wchar_t *widened_arg = nullptr;
+
+// warning C4996: 'mbstowcs': This function or variable may be unsafe.
+#if defined(_MSC_VER)
+#pragma warning(push)
+#pragma warning(disable:4996)
+#endif
+
 #    if defined(HAVE_BROKEN_MBSTOWCS) && HAVE_BROKEN_MBSTOWCS
     size_t count = strlen(safe_arg);
 #    else
@@ -111,6 +118,11 @@ inline wchar_t *widen_chars(const char *safe_arg) {
         widened_arg = new wchar_t[count + 1];
         mbstowcs(widened_arg, safe_arg, count + 1);
     }
+
+#if defined(_MSC_VER)
+#pragma warning(pop)
+#endif
+
 #endif
     return widened_arg;
 }

include/pybind11/eval.h

@@ -19,11 +19,11 @@ PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
 PYBIND11_NAMESPACE_BEGIN(detail)
 
 inline void ensure_builtins_in_globals(object &global) {
-    #if PY_VERSION_HEX < 0x03080000
+    #if defined(PYPY_VERSION) || PY_VERSION_HEX < 0x03080000
         // Running exec and eval on Python 2 and 3 adds `builtins` module under
         // `__builtins__` key to globals if not yet present.
         // Python 3.8 made PyRun_String behave similarly. Let's also do that for
-        // older versions, for consistency.
+        // older versions, for consistency. This was missing from PyPy3.8 7.3.7.
         if (!global.contains("__builtins__"))
             global["__builtins__"] = module_::import(PYBIND11_BUILTINS_MODULE);
     #else

include/pybind11/functional.h

@@ -113,8 +113,8 @@ public:
         return cpp_function(std::forward<Func>(f_), policy).release();
     }
 
-    PYBIND11_TYPE_CASTER(type, _("Callable[[") + concat(make_caster<Args>::name...) + _("], ")
-                               + make_caster<retval_type>::name + _("]"));
+    PYBIND11_TYPE_CASTER(type, const_name("Callable[[") + concat(make_caster<Args>::name...) + const_name("], ")
+                               + make_caster<retval_type>::name + const_name("]"));
 };
 
 PYBIND11_NAMESPACE_END(detail)

include/pybind11/numpy.h

@@ -39,7 +39,7 @@ class array; // Forward declaration
 
 PYBIND11_NAMESPACE_BEGIN(detail)
 
-template <> struct handle_type_name<array> { static constexpr auto name = _("numpy.ndarray"); };
+template <> struct handle_type_name<array> { static constexpr auto name = const_name("numpy.ndarray"); };
 
 template <typename type, typename SFINAE = void> struct npy_format_descriptor;
 
@@ -290,7 +290,7 @@ template <typename T> struct array_info_scalar {
     using type = T;
     static constexpr bool is_array = false;
     static constexpr bool is_empty = false;
-    static constexpr auto extents = _("");
+    static constexpr auto extents = const_name("");
     static void append_extents(list& /* shape */) { }
 };
 // Computes underlying type and a comma-separated list of extents for array
@@ -309,8 +309,8 @@ template <typename T, size_t N> struct array_info<std::array<T, N>> {
         array_info<T>::append_extents(shape);
     }
 
-    static constexpr auto extents = _<array_info<T>::is_array>(
-        concat(_<N>(), array_info<T>::extents), _<N>()
+    static constexpr auto extents = const_name<array_info<T>::is_array>(
+        concat(const_name<N>(), array_info<T>::extents), const_name<N>()
     );
 };
 // For numpy we have special handling for arrays of characters, so we don't include
@@ -1021,7 +1021,7 @@ template <typename T>
 struct format_descriptor<T, detail::enable_if_t<detail::array_info<T>::is_array>> {
     static std::string format() {
         using namespace detail;
-        static constexpr auto extents = _("(") + array_info<T>::extents + _(")");
+        static constexpr auto extents = const_name("(") + array_info<T>::extents + const_name(")");
         return extents.text + format_descriptor<remove_all_extents_t<T>>::format();
     }
 };
@@ -1056,28 +1056,28 @@ struct npy_format_descriptor_name;
 
 template <typename T>
 struct npy_format_descriptor_name<T, enable_if_t<std::is_integral<T>::value>> {
-    static constexpr auto name = _<std::is_same<T, bool>::value>(
-        _("bool"), _<std::is_signed<T>::value>("numpy.int", "numpy.uint") + _<sizeof(T)*8>()
+    static constexpr auto name = const_name<std::is_same<T, bool>::value>(
+        const_name("bool"), const_name<std::is_signed<T>::value>("numpy.int", "numpy.uint") + const_name<sizeof(T)*8>()
     );
 };
 
 template <typename T>
 struct npy_format_descriptor_name<T, enable_if_t<std::is_floating_point<T>::value>> {
-    static constexpr auto name = _<std::is_same<T, float>::value
+    static constexpr auto name = const_name<std::is_same<T, float>::value
                                    || std::is_same<T, const float>::value
                                    || std::is_same<T, double>::value
                                    || std::is_same<T, const double>::value>(
-        _("numpy.float") + _<sizeof(T)*8>(), _("numpy.longdouble")
+        const_name("numpy.float") + const_name<sizeof(T)*8>(), const_name("numpy.longdouble")
     );
 };
 
 template <typename T>
 struct npy_format_descriptor_name<T, enable_if_t<is_complex<T>::value>> {
-    static constexpr auto name = _<std::is_same<typename T::value_type, float>::value
+    static constexpr auto name = const_name<std::is_same<typename T::value_type, float>::value
                                    || std::is_same<typename T::value_type, const float>::value
                                    || std::is_same<typename T::value_type, double>::value
                                    || std::is_same<typename T::value_type, const double>::value>(
-        _("numpy.complex") + _<sizeof(typename T::value_type)*16>(), _("numpy.longcomplex")
+        const_name("numpy.complex") + const_name<sizeof(typename T::value_type)*16>(), const_name("numpy.longcomplex")
     );
 };
 
@@ -1105,7 +1105,7 @@ public:
 };
 
 #define PYBIND11_DECL_CHAR_FMT \
-    static constexpr auto name = _("S") + _<N>(); \
+    static constexpr auto name = const_name("S") + const_name<N>(); \
     static pybind11::dtype dtype() { return pybind11::dtype(std::string("S") + std::to_string(N)); }
 template <size_t N> struct npy_format_descriptor<char[N]> { PYBIND11_DECL_CHAR_FMT };
 template <size_t N> struct npy_format_descriptor<std::array<char, N>> { PYBIND11_DECL_CHAR_FMT };
@@ -1117,7 +1117,7 @@ private:
 public:
     static_assert(!array_info<T>::is_empty, "Zero-sized arrays are not supported");
 
-    static constexpr auto name = _("(") + array_info<T>::extents + _(")") + base_descr::name;
+    static constexpr auto name = const_name("(") + array_info<T>::extents + const_name(")") + base_descr::name;
     static pybind11::dtype dtype() {
         list shape;
         array_info<T>::append_extents(shape);
@@ -1705,7 +1705,7 @@ vectorize_extractor(const Func &f, Return (*) (Args ...)) {
 }
 
 template <typename T, int Flags> struct handle_type_name<array_t<T, Flags>> {
-    static constexpr auto name = _("numpy.ndarray[") + npy_format_descriptor<T>::name + _("]");
+    static constexpr auto name = const_name("numpy.ndarray[") + npy_format_descriptor<T>::name + const_name("]");
 };
 
 PYBIND11_NAMESPACE_END(detail)

include/pybind11/pybind11.h

@@ -115,7 +115,7 @@ public:
     template <typename Return, typename Class, typename... Arg, typename... Extra>
     // NOLINTNEXTLINE(google-explicit-constructor)
     cpp_function(Return (Class::*f)(Arg...)&, const Extra&... extra) {
-        initialize([f](Class *c, Arg... args) -> Return { return (c->*f)(args...); },
+        initialize([f](Class *c, Arg... args) -> Return { return (c->*f)(std::forward<Arg>(args)...); },
                    (Return (*) (Class *, Arg...)) nullptr, extra...);
     }
 
@@ -133,7 +133,7 @@ public:
     template <typename Return, typename Class, typename... Arg, typename... Extra>
     // NOLINTNEXTLINE(google-explicit-constructor)
     cpp_function(Return (Class::*f)(Arg...) const&, const Extra&... extra) {
-        initialize([f](const Class *c, Arg... args) -> Return { return (c->*f)(args...); },
+        initialize([f](const Class *c, Arg... args) -> Return { return (c->*f)(std::forward<Arg>(args)...); },
                    (Return (*)(const Class *, Arg ...)) nullptr, extra...);
     }
 
@@ -203,7 +203,7 @@ protected:
             conditional_t<std::is_void<Return>::value, void_type, Return>
         >;
 
-        static_assert(expected_num_args<Extra...>(sizeof...(Args), cast_in::has_args, cast_in::has_kwargs),
+        static_assert(expected_num_args<Extra...>(sizeof...(Args), cast_in::args_pos >= 0, cast_in::has_kwargs),
                       "The number of argument annotations does not match the number of function arguments");
 
         /* Dispatch code which converts function arguments and performs the actual function call */
@@ -238,30 +238,37 @@ protected:
             return result;
         };
 
+        rec->nargs_pos = cast_in::args_pos >= 0
+            ? static_cast<std::uint16_t>(cast_in::args_pos)
+            : sizeof...(Args) - cast_in::has_kwargs; // Will get reduced more if we have a kw_only
+        rec->has_args = cast_in::args_pos >= 0;
+        rec->has_kwargs = cast_in::has_kwargs;
+
         /* Process any user-provided function attributes */
         process_attributes<Extra...>::init(extra..., rec);
 
         {
             constexpr bool has_kw_only_args = any_of<std::is_same<kw_only, Extra>...>::value,
                            has_pos_only_args = any_of<std::is_same<pos_only, Extra>...>::value,
-                           has_args = any_of<std::is_same<args, Args>...>::value,
                            has_arg_annotations = any_of<is_keyword<Extra>...>::value;
             static_assert(has_arg_annotations || !has_kw_only_args, "py::kw_only requires the use of argument annotations");
             static_assert(has_arg_annotations || !has_pos_only_args, "py::pos_only requires the use of argument annotations (for docstrings and aligning the annotations to the argument)");
-            static_assert(!(has_args && has_kw_only_args), "py::kw_only cannot be combined with a py::args argument");
+
+            static_assert(constexpr_sum(is_kw_only<Extra>::value...) <= 1, "py::kw_only may be specified only once");
+            static_assert(constexpr_sum(is_pos_only<Extra>::value...) <= 1, "py::pos_only may be specified only once");
+            constexpr auto kw_only_pos = constexpr_first<is_kw_only, Extra...>();
+            constexpr auto pos_only_pos = constexpr_first<is_pos_only, Extra...>();
+            static_assert(!(has_kw_only_args && has_pos_only_args) || pos_only_pos < kw_only_pos, "py::pos_only must come before py::kw_only");
         }
 
         /* Generate a readable signature describing the function's arguments and return value types */
-        static constexpr auto signature = _("(") + cast_in::arg_names + _(") -> ") + cast_out::name;
+        static constexpr auto signature = const_name("(") + cast_in::arg_names + const_name(") -> ") + cast_out::name;
         PYBIND11_DESCR_CONSTEXPR auto types = decltype(signature)::types();
 
         /* Register the function with Python from generic (non-templated) code */
         // Pass on the ownership over the `unique_rec` to `initialize_generic`. `rec` stays valid.
         initialize_generic(std::move(unique_rec), signature.text, types.data(), sizeof...(Args));
 
-        if (cast_in::has_args) rec->has_args = true;
-        if (cast_in::has_kwargs) rec->has_kwargs = true;
-
         /* Stash some additional information used by an important optimization in 'functional.h' */
         using FunctionType = Return (*)(Args...);
         constexpr bool is_function_ptr =
@@ -340,16 +347,18 @@ protected:
         /* Generate a proper function signature */
         std::string signature;
         size_t type_index = 0, arg_index = 0;
+        bool is_starred = false;
         for (auto *pc = text; *pc != '\0'; ++pc) {
             const auto c = *pc;
 
             if (c == '{') {
                 // Write arg name for everything except *args and **kwargs.
-                if (*(pc + 1) == '*')
+                is_starred = *(pc + 1) == '*';
+                if (is_starred)
                     continue;
                 // Separator for keyword-only arguments, placed before the kw
-                // arguments start
-                if (rec->nargs_kw_only > 0 && arg_index + rec->nargs_kw_only == args)
+                // arguments start (unless we are already putting an *args)
+                if (!rec->has_args && arg_index == rec->nargs_pos)
                     signature += "*, ";
                 if (arg_index < rec->args.size() && rec->args[arg_index].name) {
                     signature += rec->args[arg_index].name;
@@ -361,7 +370,7 @@ protected:
                 signature += ": ";
             } else if (c == '}') {
                 // Write default value if available.
-                if (arg_index < rec->args.size() && rec->args[arg_index].descr) {
+                if (!is_starred && arg_index < rec->args.size() && rec->args[arg_index].descr) {
                     signature += " = ";
                     signature += rec->args[arg_index].descr;
                 }
@@ -369,6 +378,7 @@ protected:
                 // argument, rather than before like *
                 if (rec->nargs_pos_only > 0 && (arg_index + 1) == rec->nargs_pos_only)
                     signature += ", /";
+                if (!is_starred)
                     arg_index++;
             } else if (c == '%') {
                 const std::type_info *t = types[type_index++];
@@ -395,7 +405,7 @@ protected:
             }
         }
 
-        if (arg_index != args || types[type_index] != nullptr)
+        if (arg_index != args - rec->has_args - rec->has_kwargs || types[type_index] != nullptr)
             pybind11_fail("Internal error while parsing type signature (2)");
 
 #if PY_MAJOR_VERSION < 3
@@ -631,7 +641,7 @@ protected:
                       named positional arguments weren't *also* specified via kwarg.
                    2. If we weren't given enough, try to make up the omitted ones by checking
                       whether they were provided by a kwarg matching the `py::arg("name")` name.  If
-                      so, use it (and remove it from kwargs; if not, see if the function binding
+                      so, use it (and remove it from kwargs); if not, see if the function binding
                       provided a default that we can use.
                    3. Ensure that either all keyword arguments were "consumed", or that the function
                       takes a kwargs argument to accept unconsumed kwargs.
@@ -649,7 +659,7 @@ protected:
                 size_t num_args = func.nargs;    // Number of positional arguments that we need
                 if (func.has_args) --num_args;   // (but don't count py::args
                 if (func.has_kwargs) --num_args; //  or py::kwargs)
-                size_t pos_args = num_args - func.nargs_kw_only;
+                size_t pos_args = func.nargs_pos;
 
                 if (!func.has_args && n_args_in > pos_args)
                     continue; // Too many positional arguments for this overload
@@ -695,6 +705,10 @@ protected:
                 if (bad_arg)
                     continue; // Maybe it was meant for another overload (issue #688)
 
+                // Keep track of how many position args we copied out in case we need to come back
+                // to copy the rest into a py::args argument.
+                size_t positional_args_copied = args_copied;
+
                 // We'll need to copy this if we steal some kwargs for defaults
                 dict kwargs = reinterpret_borrow<dict>(kwargs_in);
 
@@ -747,6 +761,10 @@ protected:
                         }
 
                         if (value) {
+                            // If we're at the py::args index then first insert a stub for it to be replaced later
+                            if (func.has_args && call.args.size() == func.nargs_pos)
+                                call.args.push_back(none());
+
                             call.args.push_back(value);
                             call.args_convert.push_back(arg_rec.convert);
                         }
@@ -769,16 +787,19 @@ protected:
                         // We didn't copy out any position arguments from the args_in tuple, so we
                         // can reuse it directly without copying:
                         extra_args = reinterpret_borrow<tuple>(args_in);
-                    } else if (args_copied >= n_args_in) {
+                    } else if (positional_args_copied >= n_args_in) {
                         extra_args = tuple(0);
                     } else {
-                        size_t args_size = n_args_in - args_copied;
+                        size_t args_size = n_args_in - positional_args_copied;
                         extra_args = tuple(args_size);
                         for (size_t i = 0; i < args_size; ++i) {
-                            extra_args[i] = PyTuple_GET_ITEM(args_in, args_copied + i);
+                            extra_args[i] = PyTuple_GET_ITEM(args_in, positional_args_copied + i);
                         }
                     }
+                    if (call.args.size() <= func.nargs_pos)
                         call.args.push_back(extra_args);
+                    else
+                        call.args[func.nargs_pos] = extra_args;
                     call.args_convert.push_back(false);
                     call.args_ref = std::move(extra_args);
                 }
@@ -1958,6 +1979,16 @@ inline std::pair<decltype(internals::registered_types_py)::iterator, bool> all_t
         // gets destroyed:
         weakref((PyObject *) type, cpp_function([type](handle wr) {
             get_internals().registered_types_py.erase(type);
+
+            // TODO consolidate the erasure code in pybind11_meta_dealloc() in class.h
+            auto &cache = get_internals().inactive_override_cache;
+            for (auto it = cache.begin(), last = cache.end(); it != last; ) {
+                if (it->first == reinterpret_cast<PyObject *>(type))
+                    it = cache.erase(it);
+                else
+                    ++it;
+            }
+
             wr.dec_ref();
         })).release();
     }
@@ -2334,8 +2365,9 @@ inline function get_type_override(const void *this_ptr, const type_info *this_ty
 
     /* Don't call dispatch code if invoked from overridden function.
        Unfortunately this doesn't work on PyPy. */
-#if !defined(PYPY_VERSION)
-
+#if !defined(PYPY_VERSION) && PY_VERSION_HEX < 0x030B0000
+    // TODO: Remove PyPy workaround for Python 3.11.
+    // Current API fails on 3.11 since co_varnames can be null.
 #if PY_VERSION_HEX >= 0x03090000
     PyFrameObject *frame = PyThreadState_GetFrame(PyThreadState_Get());
     if (frame != nullptr) {
@@ -2343,7 +2375,7 @@ inline function get_type_override(const void *this_ptr, const type_info *this_ty
         // f_code is guaranteed to not be NULL
         if ((std::string) str(f_code->co_name) == name && f_code->co_argcount > 0) {
             PyObject* locals = PyEval_GetLocals();
-            if (locals != nullptr) {
+            if (locals != nullptr && f_code->co_varnames != nullptr) {
                 PyObject *self_caller = dict_getitem(
                     locals, PyTuple_GET_ITEM(f_code->co_varnames, 0)
                 );

include/pybind11/pytypes.h

@@ -18,6 +18,10 @@
 #  include <optional>
 #endif
 
+#ifdef PYBIND11_HAS_STRING_VIEW
+#  include <string_view>
+#endif
+
 PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
 
 /* A few forward declarations */
@@ -287,10 +291,10 @@ protected:
     struct borrowed_t { };
     struct stolen_t { };
 
-#ifndef DOXYGEN_SHOULD_SKIP_THIS  // Issue in breathe 4.26.1
+    /// @cond BROKEN
     template <typename T> friend T reinterpret_borrow(handle);
     template <typename T> friend T reinterpret_steal(handle);
-#endif
+    /// @endcond
 
 public:
     // Only accessible from derived classes and the reinterpret_* functions
@@ -431,7 +435,7 @@ inline void raise_from(error_already_set& err, PyObject *type, const char *messa
 
 #endif
 
-/** \defgroup python_builtins _
+/** \defgroup python_builtins const_name
     Unless stated otherwise, the following C++ functions behave the same
     as their Python counterparts.
  */
@@ -1085,6 +1089,20 @@ public:
     // NOLINTNEXTLINE(google-explicit-constructor)
     str(const std::string &s) : str(s.data(), s.size()) { }
 
+#ifdef PYBIND11_HAS_STRING_VIEW
+    // enable_if is needed to avoid "ambiguous conversion" errors (see PR #3521).
+    template <typename T, detail::enable_if_t<std::is_same<T, std::string_view>::value, int> = 0>
+    // NOLINTNEXTLINE(google-explicit-constructor)
+    str(T s) : str(s.data(), s.size()) { }
+
+# ifdef PYBIND11_HAS_U8STRING
+    // reinterpret_cast here is safe (C++20 guarantees char8_t has the same size/alignment as char)
+    // NOLINTNEXTLINE(google-explicit-constructor)
+    str(std::u8string_view s) : str(reinterpret_cast<const char*>(s.data()), s.size()) { }
+# endif
+
+#endif
+
     explicit str(const bytes &b);
 
     /** \rst
@@ -1167,6 +1185,26 @@ public:
             pybind11_fail("Unable to extract bytes contents!");
         return std::string(buffer, (size_t) length);
     }
+
+#ifdef PYBIND11_HAS_STRING_VIEW
+    // enable_if is needed to avoid "ambiguous conversion" errors (see PR #3521).
+    template <typename T, detail::enable_if_t<std::is_same<T, std::string_view>::value, int> = 0>
+    // NOLINTNEXTLINE(google-explicit-constructor)
+    bytes(T s) : bytes(s.data(), s.size()) { }
+
+    // Obtain a string view that views the current `bytes` buffer value.  Note that this is only
+    // valid so long as the `bytes` instance remains alive and so generally should not outlive the
+    // lifetime of the `bytes` instance.
+    // NOLINTNEXTLINE(google-explicit-constructor)
+    operator std::string_view() const {
+        char *buffer = nullptr;
+        ssize_t length = 0;
+        if (PYBIND11_BYTES_AS_STRING_AND_SIZE(m_ptr, &buffer, &length))
+            pybind11_fail("Unable to extract bytes contents!");
+        return {buffer, static_cast<size_t>(length)};
+    }
+#endif
+
 };
 // Note: breathe >= 4.17.0 will fail to build docs if the below two constructors
 // are included in the doxygen group; close here and reopen after as a workaround
@@ -1714,10 +1752,17 @@ public:
     static memoryview from_memory(const void *mem, ssize_t size) {
         return memoryview::from_memory(const_cast<void*>(mem), size, true);
     }
+
+#ifdef PYBIND11_HAS_STRING_VIEW
+    static memoryview from_memory(std::string_view mem) {
+        return from_memory(const_cast<char*>(mem.data()), static_cast<ssize_t>(mem.size()), true);
+    }
+#endif
+
 #endif
 };
 
-#ifndef DOXYGEN_SHOULD_SKIP_THIS
+/// @cond DUPLICATE
 inline memoryview memoryview::from_buffer(
     void *ptr, ssize_t itemsize, const char* format,
     detail::any_container<ssize_t> shape,
@@ -1745,7 +1790,7 @@ inline memoryview memoryview::from_buffer(
         throw error_already_set();
     return memoryview(object(obj, stolen_t{}));
 }
-#endif  // DOXYGEN_SHOULD_SKIP_THIS
+/// @endcond
 /// @} pytypes
 
 /// \addtogroup python_builtins

include/pybind11/stl.h

@@ -78,7 +78,7 @@ template <typename Type, typename Key> struct set_caster {
         return s.release();
     }
 
-    PYBIND11_TYPE_CASTER(type, _("Set[") + key_conv::name + _("]"));
+    PYBIND11_TYPE_CASTER(type, const_name("Set[") + key_conv::name + const_name("]"));
 };
 
 template <typename Type, typename Key, typename Value> struct map_caster {
@@ -120,7 +120,7 @@ template <typename Type, typename Key, typename Value> struct map_caster {
         return d.release();
     }
 
-    PYBIND11_TYPE_CASTER(Type, _("Dict[") + key_conv::name + _(", ") + value_conv::name + _("]"));
+    PYBIND11_TYPE_CASTER(Type, const_name("Dict[") + key_conv::name + const_name(", ") + value_conv::name + const_name("]"));
 };
 
 template <typename Type, typename Value> struct list_caster {
@@ -166,7 +166,7 @@ public:
         return l.release();
     }
 
-    PYBIND11_TYPE_CASTER(Type, _("List[") + value_conv::name + _("]"));
+    PYBIND11_TYPE_CASTER(Type, const_name("List[") + value_conv::name + const_name("]"));
 };
 
 template <typename Type, typename Alloc> struct type_caster<std::vector<Type, Alloc>>
@@ -223,7 +223,7 @@ public:
         return l.release();
     }
 
-    PYBIND11_TYPE_CASTER(ArrayType, _("List[") + value_conv::name + _<Resizable>(_(""), _("[") + _<Size>() + _("]")) + _("]"));
+    PYBIND11_TYPE_CASTER(ArrayType, const_name("List[") + value_conv::name + const_name<Resizable>(const_name(""), const_name("[") + const_name<Size>() + const_name("]")) + const_name("]"));
 };
 
 template <typename Type, size_t Size> struct type_caster<std::array<Type, Size>>
@@ -273,7 +273,7 @@ template<typename Type, typename Value = typename Type::value_type> struct optio
         return true;
     }
 
-    PYBIND11_TYPE_CASTER(Type, _("Optional[") + value_conv::name + _("]"));
+    PYBIND11_TYPE_CASTER(Type, const_name("Optional[") + value_conv::name + const_name("]"));
 };
 
 #if defined(PYBIND11_HAS_OPTIONAL)
@@ -353,7 +353,7 @@ struct variant_caster<V<Ts...>> {
     }
 
     using Type = V<Ts...>;
-    PYBIND11_TYPE_CASTER(Type, _("Union[") + detail::concat(make_caster<Ts>::name...) + _("]"));
+    PYBIND11_TYPE_CASTER(Type, const_name("Union[") + detail::concat(make_caster<Ts>::name...) + const_name("]"));
 };
 
 #if defined(PYBIND11_HAS_VARIANT)

include/pybind11/stl/filesystem.h

@@ -92,7 +92,7 @@ public:
         return true;
     }
 
-    PYBIND11_TYPE_CASTER(T, _("os.PathLike"));
+    PYBIND11_TYPE_CASTER(T, const_name("os.PathLike"));
 };
 
 template<> struct type_caster<std::filesystem::path>

noxfile.py

@@ -57,10 +57,10 @@ def docs(session: nox.Session) -> None:
     session.chdir("docs")
 
     if "pdf" in session.posargs:
-        session.run("sphinx-build", "-M", "latexpdf", ".", "_build")
+        session.run("sphinx-build", "-b", "latexpdf", ".", "_build")
         return
 
-    session.run("sphinx-build", "-M", "html", ".", "_build")
+    session.run("sphinx-build", "-b", "html", ".", "_build")
 
     if "serve" in session.posargs:
         session.log("Launching docs at http://localhost:8000/ - use Ctrl-C to quit")
@@ -85,5 +85,9 @@ def build(session: nox.Session) -> None:
     """
 
     session.install("build")
-    session.run("python", "-m", "build")
-    session.run("python", "-m", "build", env={"PYBIND11_GLOBAL_SDIST": "1"})
+    session.log("Building normal files")
+    session.run("python", "-m", "build", *session.posargs)
+    session.log("Building pybind11-global files (PYBIND11_GLOBAL_SDIST=1)")
+    session.run(
+        "python", "-m", "build", *session.posargs, env={"PYBIND11_GLOBAL_SDIST": "1"}
+    )

pybind11/_version.py

@@ -8,5 +8,5 @@ def _to_int(s):
         return s
 
 
-__version__ = "2.8.1"
+__version__ = "2.9.0"
 version_info = tuple(_to_int(s) for s in __version__.split("."))

pybind11/setup_helpers.py

@@ -42,6 +42,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 import contextlib
 import os
 import platform
+import shlex
 import shutil
 import sys
 import sysconfig
@@ -143,7 +144,12 @@ class Pybind11Extension(_Extension):
         if WIN:
             cflags += ["/EHsc", "/bigobj"]
         else:
-            cflags += ["-fvisibility=hidden", "-g0"]
+            cflags += ["-fvisibility=hidden"]
+            env_cflags = os.environ.get("CFLAGS", "")
+            env_cppflags = os.environ.get("CPPFLAGS", "")
+            c_cpp_flags = shlex.split(env_cflags) + shlex.split(env_cppflags)
+            if not any(opt.startswith("-g") for opt in c_cpp_flags):
+                cflags += ["-g0"]
             if MACOS:
                 cflags += ["-stdlib=libc++"]
                 ldflags += ["-stdlib=libc++"]
@@ -460,8 +466,14 @@ class ParallelCompile(object):
                     threads = 1
 
             if threads > 1:
-                for _ in ThreadPool(threads).imap_unordered(_single_compile, objects):
+                pool = ThreadPool(threads)
+                # In Python 2, ThreadPool can't be used as a context manager.
+                # Once we are no longer supporting it, this can be 'with pool:'
+                try:
+                    for _ in pool.imap_unordered(_single_compile, objects):
                         pass
+                finally:
+                    pool.terminate()
             else:
                 for ob in objects:
                     _single_compile(ob)

tests/CMakeLists.txt

@@ -429,6 +429,14 @@ foreach(target ${test_targets})
   endif()
 endforeach()
 
+# Provide nice organisation in IDEs
+if(NOT CMAKE_VERSION VERSION_LESS 3.8)
+  source_group(
+    TREE "${CMAKE_CURRENT_SOURCE_DIR}/../include"
+    PREFIX "Header Files"
+    FILES ${PYBIND11_HEADERS})
+endif()
+
 # Make sure pytest is found or produce a warning
 pybind11_find_import(pytest VERSION 3.1)
 

tests/extra_setuptools/test_setuphelper.py

@@ -8,6 +8,7 @@ import pytest
 
 DIR = os.path.abspath(os.path.dirname(__file__))
 MAIN_DIR = os.path.dirname(os.path.dirname(DIR))
+WIN = sys.platform.startswith("win32") or sys.platform.startswith("cygwin")
 
 
 @pytest.mark.parametrize("parallel", [False, True])
@@ -71,13 +72,20 @@ def test_simple_setup_py(monkeypatch, tmpdir, parallel, std):
         encoding="ascii",
     )
 
-    subprocess.check_call(
+    out = subprocess.check_output(
         [sys.executable, "setup.py", "build_ext", "--inplace"],
-        stdout=sys.stdout,
-        stderr=sys.stderr,
     )
+    if not WIN:
+        assert b"-g0" in out
+    out = subprocess.check_output(
+        [sys.executable, "setup.py", "build_ext", "--inplace", "--force"],
+        env=dict(os.environ, CFLAGS="-g"),
+    )
+    if not WIN:
+        assert b"-g0" not in out
 
     # Debug helper printout, normally hidden
+    print(out)
     for item in tmpdir.listdir():
         print(item.basename)
 

tests/pybind11_tests.h

@@ -65,7 +65,7 @@ PYBIND11_NAMESPACE_BEGIN(pybind11)
 PYBIND11_NAMESPACE_BEGIN(detail)
 template<> class type_caster<RValueCaster> {
 public:
-    PYBIND11_TYPE_CASTER(RValueCaster, _("RValueCaster"));
+    PYBIND11_TYPE_CASTER(RValueCaster, const_name("RValueCaster"));
     static handle cast(RValueCaster &&, return_value_policy, handle) { return py::str("rvalue").release(); }
     static handle cast(const RValueCaster &, return_value_policy, handle) { return py::str("lvalue").release(); }
 };

tests/requirements.txt

@@ -2,7 +2,7 @@ numpy==1.16.6; python_version<"3.6" and sys_platform!="win32" and platform_pytho
 numpy==1.19.0; platform_python_implementation=="PyPy" and sys_platform=="linux" and python_version=="3.6"
 numpy==1.20.0; platform_python_implementation=="PyPy" and sys_platform=="linux" and python_version=="3.7"
 numpy==1.19.3; platform_python_implementation!="PyPy" and python_version=="3.6"
-numpy==1.21.3; platform_python_implementation!="PyPy" and python_version>="3.7"
+numpy==1.21.3; platform_python_implementation!="PyPy" and python_version>="3.7" and python_version<"3.11"
 py @ git+https://github.com/pytest-dev/py; python_version>="3.11"
 pytest==4.6.9; python_version<"3.5"
 pytest==6.1.2; python_version=="3.5"