Merge branch 'master' into stable

include/pybind11/numpy.h

@@ -518,7 +518,7 @@ public:
     }
 
     /// Single-character for dtype's type.
-    /// For example, ``float`` is 'f', ``double`` 'd', ``int`` 'i', and ``long`` 'd'.
+    /// For example, ``float`` is 'f', ``double`` 'd', ``int`` 'i', and ``long`` 'l'.
     char char_() const {
         // Note: The signature, `dtype::char_` follows the naming of NumPy's
         // public Python API (i.e., ``dtype.char``), rather than its internal

include/pybind11/pybind11.h

@@ -1124,6 +1124,15 @@ inline dict globals() {
     return reinterpret_borrow<dict>(p ? p : module_::import("__main__").attr("__dict__").ptr());
 }
 
+#if PY_VERSION_HEX >= 0x03030000
+template <typename... Args,
+          typename = detail::enable_if_t<args_are_all_keyword_or_ds<Args...>()>>
+PYBIND11_DEPRECATED("make_simple_namespace should be replaced with py::module_::import(\"types\").attr(\"SimpleNamespace\") ")
+object make_simple_namespace(Args&&... args_) {
+    return module_::import("types").attr("SimpleNamespace")(std::forward<Args>(args_)...);
+}
+#endif
+
 PYBIND11_NAMESPACE_BEGIN(detail)
 /// Generic support for creating new Python heap types
 class generic_type : public object {
@@ -1967,29 +1976,54 @@ struct iterator_state {
 };
 
 // Note: these helpers take the iterator by non-const reference because some
-// iterators in the wild can't be dereferenced when const. C++ needs the extra parens in decltype
-// to enforce an lvalue. The & after Iterator is required for MSVC < 16.9. SFINAE cannot be
-// reused for result_type due to bugs in ICC, NVCC, and PGI compilers. See PR #3293.
-template <typename Iterator, typename SFINAE = decltype((*std::declval<Iterator &>()))>
+// iterators in the wild can't be dereferenced when const. The & after Iterator
+// is required for MSVC < 16.9. SFINAE cannot be reused for result_type due to
+// bugs in ICC, NVCC, and PGI compilers. See PR #3293.
+template <typename Iterator, typename SFINAE = decltype(*std::declval<Iterator &>())>
 struct iterator_access {
-    using result_type = decltype((*std::declval<Iterator &>()));
+    using result_type = decltype(*std::declval<Iterator &>());
     // NOLINTNEXTLINE(readability-const-return-type) // PR #3263
     result_type operator()(Iterator &it) const {
         return *it;
     }
 };
 
-template <typename Iterator, typename SFINAE = decltype(((*std::declval<Iterator &>()).first)) >
-struct iterator_key_access {
-    using result_type = decltype(((*std::declval<Iterator &>()).first));
+template <typename Iterator, typename SFINAE = decltype((*std::declval<Iterator &>()).first) >
+class iterator_key_access {
+private:
+    using pair_type = decltype(*std::declval<Iterator &>());
+
+public:
+    /* If either the pair itself or the element of the pair is a reference, we
+     * want to return a reference, otherwise a value. When the decltype
+     * expression is parenthesized it is based on the value category of the
+     * expression; otherwise it is the declared type of the pair member.
+     * The use of declval<pair_type> in the second branch rather than directly
+     * using *std::declval<Iterator &>() is a workaround for nvcc
+     * (it's not used in the first branch because going via decltype and back
+     * through declval does not perfectly preserve references).
+     */
+    using result_type = conditional_t<
+        std::is_reference<decltype(*std::declval<Iterator &>())>::value,
+        decltype(((*std::declval<Iterator &>()).first)),
+        decltype(std::declval<pair_type>().first)
+    >;
     result_type operator()(Iterator &it) const {
         return (*it).first;
     }
 };
 
-template <typename Iterator, typename SFINAE = decltype(((*std::declval<Iterator &>()).second))>
-struct iterator_value_access {
-    using result_type = decltype(((*std::declval<Iterator &>()).second));
+template <typename Iterator, typename SFINAE = decltype((*std::declval<Iterator &>()).second)>
+class iterator_value_access {
+private:
+    using pair_type = decltype(*std::declval<Iterator &>());
+
+public:
+    using result_type = conditional_t<
+        std::is_reference<decltype(*std::declval<Iterator &>())>::value,
+        decltype(((*std::declval<Iterator &>()).second)),
+        decltype(std::declval<pair_type>().second)
+    >;
     result_type operator()(Iterator &it) const {
         return (*it).second;
     }
@@ -2301,6 +2335,29 @@ inline function get_type_override(const void *this_ptr, const type_info *this_ty
     /* Don't call dispatch code if invoked from overridden function.
        Unfortunately this doesn't work on PyPy. */
 #if !defined(PYPY_VERSION)
+
+#if PY_VERSION_HEX >= 0x03090000
+    PyFrameObject *frame = PyThreadState_GetFrame(PyThreadState_Get());
+    if (frame != nullptr) {
+        PyCodeObject *f_code = PyFrame_GetCode(frame);
+        // f_code is guaranteed to not be NULL
+        if ((std::string) str(f_code->co_name) == name && f_code->co_argcount > 0) {
+            PyObject* locals = PyEval_GetLocals();
+            if (locals != nullptr) {
+                PyObject *self_caller = dict_getitem(
+                    locals, PyTuple_GET_ITEM(f_code->co_varnames, 0)
+                );
+                if (self_caller == self.ptr()) {
+                    Py_DECREF(f_code);
+                    Py_DECREF(frame);
+                    return function();
+                }
+            }
+        }
+        Py_DECREF(f_code);
+        Py_DECREF(frame);
+    }
+#else
     PyFrameObject *frame = PyThreadState_Get()->frame;
     if (frame != nullptr && (std::string) str(frame->f_code->co_name) == name
         && frame->f_code->co_argcount > 0) {
@@ -2310,6 +2367,8 @@ inline function get_type_override(const void *this_ptr, const type_info *this_ty
         if (self_caller == self.ptr())
             return function();
     }
+#endif
+
 #else
     /* PyPy currently doesn't provide a detailed cpyext emulation of
        frame objects, so we have to emulate this using Python. This

include/pybind11/stl.h

@@ -245,17 +245,17 @@ template <typename Key, typename Value, typename Hash, typename Equal, typename
   : map_caster<std::unordered_map<Key, Value, Hash, Equal, Alloc>, Key, Value> { };
 
 // This type caster is intended to be used for std::optional and std::experimental::optional
-template<typename T> struct optional_caster {
-    using value_conv = make_caster<typename T::value_type>;
+template<typename Type, typename Value = typename Type::value_type> struct optional_caster {
+    using value_conv = make_caster<Value>;
 
-    template <typename T_>
-    static handle cast(T_ &&src, return_value_policy policy, handle parent) {
+    template <typename T>
+    static handle cast(T &&src, return_value_policy policy, handle parent) {
         if (!src)
             return none().inc_ref();
         if (!std::is_lvalue_reference<T>::value) {
-            policy = return_value_policy_override<T>::policy(policy);
+            policy = return_value_policy_override<Value>::policy(policy);
         }
-        return value_conv::cast(*std::forward<T_>(src), policy, parent);
+        return value_conv::cast(*std::forward<T>(src), policy, parent);
     }
 
     bool load(handle src, bool convert) {
@@ -269,11 +269,11 @@ template<typename T> struct optional_caster {
         if (!inner_caster.load(src, convert))
             return false;
 
-        value.emplace(cast_op<typename T::value_type &&>(std::move(inner_caster)));
+        value.emplace(cast_op<Value &&>(std::move(inner_caster)));
         return true;
     }
 
-    PYBIND11_TYPE_CASTER(T, _("Optional[") + value_conv::name + _("]"));
+    PYBIND11_TYPE_CASTER(Type, _("Optional[") + value_conv::name + _("]"));
 };
 
 #if defined(PYBIND11_HAS_OPTIONAL)

noxfile.py

@@ -2,7 +2,7 @@ import nox
 
 nox.options.sessions = ["lint", "tests", "tests_packaging"]
 
-PYTHON_VERISONS = ["2.7", "3.5", "3.6", "3.7", "3.8", "3.9", "3.10"]
+PYTHON_VERISONS = ["2.7", "3.5", "3.6", "3.7", "3.8", "3.9", "3.10", "3.11"]
 
 
 @nox.session(reuse_venv=True)
@@ -20,7 +20,8 @@ def tests(session: nox.Session) -> None:
     Run the tests (requires a compiler).
     """
     tmpdir = session.create_tmp()
-    session.install("pytest", "cmake")
+    session.install("cmake")
+    session.install("-r", "tests/requirements.txt")
     session.run(
         "cmake",
         "-S",

pybind11/_version.py

@@ -8,5 +8,5 @@ def _to_int(s):
         return s
 
 
-__version__ = "2.8.0"
+__version__ = "2.8.1"
 version_info = tuple(_to_int(s) for s in __version__.split("."))

tests/CMakeLists.txt

@@ -174,10 +174,14 @@ set(PYBIND11_CROSS_MODULE_GIL_TESTS test_gil_scoped.py)
 set(PYBIND11_EIGEN_REPO
     "https://gitlab.com/libeigen/eigen.git"
     CACHE STRING "Eigen repository to use for tests")
-# This hash is for 3.3.8, using a hash for security reasons
-set(PYBIND11_EIGEN_VERSION
-    "dc252fbf00079ccab57948a164b1421703fe4361"
-    CACHE STRING "Eigen version to use for tests")
+# Always use a hash for reconfigure speed and security reasons
+# Include the version number for pretty printing (keep in sync)
+set(PYBIND11_EIGEN_VERSION_AND_HASH
+    "3.4.0;929bc0e191d0927b1735b9a1ddc0e8b77e3a25ec"
+    CACHE STRING "Eigen version to use for tests, format: VERSION;HASH")
+
+list(GET PYBIND11_EIGEN_VERSION_AND_HASH 0 PYBIND11_EIGEN_VERSION_STRING)
+list(GET PYBIND11_EIGEN_VERSION_AND_HASH 1 PYBIND11_EIGEN_VERSION_HASH)
 
 # Check if Eigen is available; if not, remove from PYBIND11_TEST_FILES (but
 # keep it in PYBIND11_PYTEST_FILES, so that we get the "eigen is not installed"
@@ -196,16 +200,22 @@ if(PYBIND11_TEST_FILES_EIGEN_I GREATER -1)
     FetchContent_Declare(
       eigen
       GIT_REPOSITORY "${PYBIND11_EIGEN_REPO}"
-      GIT_TAG "${PYBIND11_EIGEN_VERSION}")
+      GIT_TAG "${PYBIND11_EIGEN_VERSION_HASH}")
 
     FetchContent_GetProperties(eigen)
     if(NOT eigen_POPULATED)
-      message(STATUS "Downloading Eigen")
+      message(
+        STATUS
+          "Downloading Eigen ${PYBIND11_EIGEN_VERSION_STRING} (${PYBIND11_EIGEN_VERSION_HASH}) from ${PYBIND11_EIGEN_REPO}"
+      )
       FetchContent_Populate(eigen)
     endif()
 
     set(EIGEN3_INCLUDE_DIR ${eigen_SOURCE_DIR})
     set(EIGEN3_FOUND TRUE)
+    # When getting locally, the version is not visible from a superprojet,
+    # so just force it.
+    set(EIGEN3_VERSION "${PYBIND11_EIGEN_VERSION_STRING}")
 
   else()
     find_package(Eigen3 3.2.7 QUIET CONFIG)
@@ -256,7 +266,9 @@ if(Boost_FOUND)
 endif()
 
 # Check if we need to add -lstdc++fs or -lc++fs or nothing
-if(MSVC)
+if(DEFINED CMAKE_CXX_STANDARD AND CMAKE_CXX_STANDARD LESS 17)
+  set(STD_FS_NO_LIB_NEEDED TRUE)
+elseif(MSVC)
   set(STD_FS_NO_LIB_NEEDED TRUE)
 else()
   file(
@@ -286,7 +298,7 @@ elseif(${STD_FS_NEEDS_CXXFS})
 elseif(${STD_FS_NO_LIB_NEEDED})
   set(STD_FS_LIB "")
 else()
-  message(WARNING "Unknown compiler - not passing -lstdc++fs")
+  message(WARNING "Unknown C++17 compiler - not passing -lstdc++fs")
   set(STD_FS_LIB "")
 endif()
 

tests/requirements.txt

---extra-index-url https://antocuni.github.io/pypy-wheels/manylinux2010/
-numpy==1.16.6; python_version<"3.6" and sys_platform!="win32"
-numpy==1.18.0; platform_python_implementation=="PyPy" and sys_platform=="darwin" and python_version>="3.6"
-numpy==1.19.3; (platform_python_implementation!="PyPy" or sys_platform=="linux") and python_version=="3.6"
-numpy==1.21.2; (platform_python_implementation!="PyPy" or sys_platform=="linux") and python_version>="3.7" and python_version<"3.10"
-numpy==1.21.2; platform_python_implementation!="PyPy" and sys_platform=="linux" and python_version=="3.10"
+numpy==1.16.6; python_version<"3.6" and sys_platform!="win32" and platform_python_implementation!="PyPy"
+numpy==1.19.0; platform_python_implementation=="PyPy" and sys_platform=="linux" and python_version=="3.6"
+numpy==1.20.0; platform_python_implementation=="PyPy" and sys_platform=="linux" and python_version=="3.7"
+numpy==1.19.3; platform_python_implementation!="PyPy" and python_version=="3.6"
+numpy==1.21.3; platform_python_implementation!="PyPy" and python_version>="3.7"
+py @ git+https://github.com/pytest-dev/py; python_version>="3.11"
 pytest==4.6.9; python_version<"3.5"
 pytest==6.1.2; python_version=="3.5"
 pytest==6.2.4; python_version>="3.6"
 pytest-timeout
-scipy==1.2.3; (platform_python_implementation!="PyPy" or sys_platform=="linux") and python_version<"3.6"
-scipy==1.5.4; (platform_python_implementation!="PyPy" or sys_platform=="linux") and python_version>="3.6" and python_version<"3.10"
+scipy==1.2.3; platform_python_implementation!="PyPy" and python_version<"3.6"
+scipy==1.5.4; platform_python_implementation!="PyPy" and python_version>="3.6" and python_version<"3.10"

tests/test_buffers.py

@@ -5,7 +5,7 @@ import struct
 
 import pytest
 
-import env  # noqa: F401
+import env
 from pybind11_tests import ConstructorStats
 from pybind11_tests import buffers as m
 

tests/test_builtin_casters.py

 # -*- coding: utf-8 -*-
 import pytest
 
-import env  # noqa: F401
+import env
 from pybind11_tests import IncType, UserType
 from pybind11_tests import builtin_casters as m
 

tests/test_callbacks.py

@@ -4,7 +4,7 @@ from threading import Thread
 
 import pytest
 
-import env  # NOQA: F401
+import env  # noqa: F401
 from pybind11_tests import callbacks as m
 
 

tests/test_eigen.cpp

@@ -13,6 +13,9 @@
 #include <pybind11/stl.h>
 
 #if defined(_MSC_VER)
+#if _MSC_VER < 1910  // VS 2015's MSVC
+#  pragma warning(disable: 4127) // C4127: conditional expression is constant
+#endif
 #  pragma warning(disable: 4996) // C4996: std::unary_negation is deprecated
 #endif
 

tests/test_enum.py

@@ -96,13 +96,13 @@ Members:
         y >= object()  # noqa: B015
 
     with pytest.raises(TypeError):
-        y | object()  # noqa: B015
+        y | object()
 
     with pytest.raises(TypeError):
-        y & object()  # noqa: B015
+        y & object()
 
     with pytest.raises(TypeError):
-        y ^ object()  # noqa: B015
+        y ^ object()
 
     assert int(m.UnscopedEnum.ETwo) == 2
     assert str(m.UnscopedEnum(2)) == "UnscopedEnum.ETwo"

tests/test_methods_and_attributes.py

@@ -102,7 +102,7 @@ def test_properties():
     assert instance.def_property == 3
 
     with pytest.raises(AttributeError) as excinfo:
-        dummy = instance.def_property_writeonly  # noqa: F841 unused var
+        dummy = instance.def_property_writeonly  # unused var
     assert "unreadable attribute" in str(excinfo.value)
 
     instance.def_property_writeonly = 4
@@ -127,7 +127,7 @@ def test_static_properties():
     assert m.TestProperties.def_readwrite_static == 2
 
     with pytest.raises(AttributeError) as excinfo:
-        dummy = m.TestProperties.def_writeonly_static  # noqa: F841 unused var
+        dummy = m.TestProperties.def_writeonly_static  # unused var
     assert "unreadable attribute" in str(excinfo.value)
 
     m.TestProperties.def_writeonly_static = 3

tests/test_pickling.py

 # -*- coding: utf-8 -*-
 import pytest
 
-import env  # noqa: F401
+import env
 from pybind11_tests import pickling as m
 
 try:

tests/test_pytypes.cpp

@@ -84,7 +84,7 @@ TEST_SUBMODULE(pytypes, m) {
 #if PY_VERSION_HEX >= 0x03030000
     // test_simple_namespace
     m.def("get_simple_namespace", []() {
-        auto ns = py::make_simple_namespace("attr"_a=42, "x"_a="foo", "wrong"_a=1);
+        auto ns = py::module_::import("types").attr("SimpleNamespace")("attr"_a=42, "x"_a="foo", "wrong"_a=1);
         py::delattr(ns, "wrong");
         py::setattr(ns, "right", py::int_(2));
         return ns;

tests/test_pytypes.py

@@ -5,7 +5,7 @@ import sys
 
 import pytest
 
-import env  # noqa: F401
+import env
 from pybind11_tests import debug_enabled
 from pybind11_tests import pytypes as m
 
@@ -610,7 +610,7 @@ def test_weakref(create_weakref, create_weakref_with_callback):
 
     obj = WeaklyReferenced()
     assert getweakrefcount(obj) == 0
-    wr = create_weakref(obj)  # noqa: F841
+    wr = create_weakref(obj)
     assert getweakrefcount(obj) == 1
 
     obj = WeaklyReferenced()

tests/test_sequences_and_iterators.cpp

@@ -38,6 +38,17 @@ bool operator==(const NonZeroIterator<std::pair<A, B>>& it, const NonZeroSentine
     return !(*it).first || !(*it).second;
 }
 
+/* Iterator where dereferencing returns prvalues instead of references. */
+template<typename T>
+class NonRefIterator {
+    const T* ptr_;
+public:
+    explicit NonRefIterator(const T *ptr) : ptr_(ptr) {}
+    T operator*() const { return T(*ptr_); }
+    NonRefIterator& operator++() { ++ptr_; return *this; }
+    bool operator==(const NonRefIterator &other) const { return ptr_ == other.ptr_; }
+};
+
 class NonCopyableInt {
 public:
     explicit NonCopyableInt(int value) : value_(value) {}
@@ -331,7 +342,7 @@ TEST_SUBMODULE(sequences_and_iterators, m) {
     py::class_<IntPairs>(m, "IntPairs")
         .def(py::init<std::vector<std::pair<int, int>>>())
         .def("nonzero", [](const IntPairs& s) {
-                return py::make_iterator(NonZeroIterator<std::pair<int, int>>(s.begin()), NonZeroSentinel());
+            return py::make_iterator(NonZeroIterator<std::pair<int, int>>(s.begin()), NonZeroSentinel());
         }, py::keep_alive<0, 1>())
         .def("nonzero_keys", [](const IntPairs& s) {
             return py::make_key_iterator(NonZeroIterator<std::pair<int, int>>(s.begin()), NonZeroSentinel());
@@ -340,6 +351,20 @@ TEST_SUBMODULE(sequences_and_iterators, m) {
             return py::make_value_iterator(NonZeroIterator<std::pair<int, int>>(s.begin()), NonZeroSentinel());
         }, py::keep_alive<0, 1>())
 
+        // test iterator that returns values instead of references
+        .def("nonref", [](const IntPairs& s) {
+             return py::make_iterator(NonRefIterator<std::pair<int, int>>(s.begin()),
+                                      NonRefIterator<std::pair<int, int>>(s.end()));
+        }, py::keep_alive<0, 1>())
+        .def("nonref_keys", [](const IntPairs& s) {
+             return py::make_key_iterator(NonRefIterator<std::pair<int, int>>(s.begin()),
+                                          NonRefIterator<std::pair<int, int>>(s.end()));
+        }, py::keep_alive<0, 1>())
+        .def("nonref_values", [](const IntPairs& s) {
+             return py::make_value_iterator(NonRefIterator<std::pair<int, int>>(s.begin()),
+                                            NonRefIterator<std::pair<int, int>>(s.end()));
+        }, py::keep_alive<0, 1>())
+
         // test single-argument make_iterator
         .def("simple_iterator", [](IntPairs& self) {
             return py::make_iterator(self);

tests/test_sequences_and_iterators.py

@@ -52,6 +52,13 @@ def test_generalized_iterators():
             next(it)
 
 
+def test_nonref_iterators():
+    pairs = m.IntPairs([(1, 2), (3, 4), (0, 5)])
+    assert list(pairs.nonref()) == [(1, 2), (3, 4), (0, 5)]
+    assert list(pairs.nonref_keys()) == [1, 3, 0]
+    assert list(pairs.nonref_values()) == [2, 4, 5]
+
+
 def test_generalized_iterators_simple():
     assert list(m.IntPairs([(1, 2), (3, 4), (0, 5)]).simple_iterator()) == [
         (1, 2),

tests/test_stl.cpp

@@ -19,6 +19,18 @@
 #include <vector>
 #include <string>
 
+#if defined(PYBIND11_TEST_BOOST)
+#include <boost/optional.hpp>
+
+namespace pybind11 { namespace detail {
+template <typename T>
+struct type_caster<boost::optional<T>> : optional_caster<boost::optional<T>> {};
+
+template <>
+struct type_caster<boost::none_t> : void_caster<boost::none_t> {};
+}} // namespace pybind11::detail
+#endif
+
 // Test with `std::variant` in C++17 mode, or with `boost::variant` in C++11/14
 #if defined(PYBIND11_HAS_VARIANT)
 using std::variant;
@@ -59,7 +71,8 @@ namespace std {
 template <template <typename> class OptionalImpl, typename T>
 struct OptionalHolder
 {
-    OptionalHolder() = default;
+    // NOLINTNEXTLINE(modernize-use-equals-default): breaks GCC 4.8
+    OptionalHolder() {};
     bool member_initialized() const {
         return member && member->initialized;
     }
@@ -67,6 +80,95 @@ struct OptionalHolder
 };
 
 
+enum class EnumType {
+  kSet = 42,
+  kUnset = 85,
+};
+
+// This is used to test that return-by-ref and return-by-copy policies are
+// handled properly for optional types. This is a regression test for a dangling
+// reference issue. The issue seemed to require the enum value type to
+// reproduce - it didn't seem to happen if the value type is just an integer.
+template <template <typename> class OptionalImpl>
+class OptionalProperties {
+public:
+    using OptionalEnumValue = OptionalImpl<EnumType>;
+
+    OptionalProperties() : value(EnumType::kSet) {}
+    ~OptionalProperties() {
+        // Reset value to detect use-after-destruction.
+        // This is set to a specific value rather than nullopt to ensure that
+        // the memory that contains the value gets re-written.
+        value = EnumType::kUnset;
+    }
+
+    OptionalEnumValue& access_by_ref() { return value; }
+    OptionalEnumValue access_by_copy() { return value; }
+
+private:
+    OptionalEnumValue value;
+};
+
+// This type mimics aspects of boost::optional from old versions of Boost,
+// which exposed a dangling reference bug in Pybind11. Recent versions of
+// boost::optional, as well as libstdc++'s std::optional, don't seem to be
+// affected by the same issue. This is meant to be a minimal implementation
+// required to reproduce the issue, not fully standard-compliant.
+// See issue #3330 for more details.
+template <typename T>
+class ReferenceSensitiveOptional {
+public:
+    using value_type = T;
+
+    ReferenceSensitiveOptional() = default;
+    // NOLINTNEXTLINE(google-explicit-constructor)
+    ReferenceSensitiveOptional(const T& value) : storage{value} {}
+    // NOLINTNEXTLINE(google-explicit-constructor)
+    ReferenceSensitiveOptional(T&& value) : storage{std::move(value)} {}
+    ReferenceSensitiveOptional& operator=(const T& value) {
+        storage = {value};
+        return *this;
+    }
+    ReferenceSensitiveOptional& operator=(T&& value) {
+        storage = {std::move(value)};
+        return *this;
+    }
+
+    template <typename... Args>
+    T& emplace(Args&&... args) {
+        storage.clear();
+        storage.emplace_back(std::forward<Args>(args)...);
+        return storage.back();
+    }
+
+    const T& value() const noexcept {
+        assert(!storage.empty());
+        return storage[0];
+    }
+
+    const T& operator*() const noexcept {
+        return value();
+    }
+
+    const T* operator->() const noexcept {
+        return &value();
+    }
+
+    explicit operator bool() const noexcept {
+        return !storage.empty();
+    }
+
+private:
+    std::vector<T> storage;
+};
+
+namespace pybind11 { namespace detail {
+template <typename T>
+struct type_caster<ReferenceSensitiveOptional<T>> : optional_caster<ReferenceSensitiveOptional<T>> {};
+} // namespace detail
+} // namespace pybind11
+
+
 TEST_SUBMODULE(stl, m) {
     // test_vector
     m.def("cast_vector", []() { return std::vector<int>{1}; });
@@ -145,6 +247,10 @@ TEST_SUBMODULE(stl, m) {
         return v;
     });
 
+    pybind11::enum_<EnumType>(m, "EnumType")
+        .value("kSet", EnumType::kSet)
+        .value("kUnset", EnumType::kUnset);
+
     // test_move_out_container
     struct MoveOutContainer {
         struct Value { int value; };
@@ -213,6 +319,12 @@ TEST_SUBMODULE(stl, m) {
         .def(py::init<>())
         .def_readonly("member", &opt_holder::member)
         .def("member_initialized", &opt_holder::member_initialized);
+
+    using opt_props = OptionalProperties<std::optional>;
+    pybind11::class_<opt_props>(m, "OptionalProperties")
+        .def(pybind11::init<>())
+        .def_property_readonly("access_by_ref", &opt_props::access_by_ref)
+        .def_property_readonly("access_by_copy", &opt_props::access_by_copy);
 #endif
 
 #ifdef PYBIND11_HAS_EXP_OPTIONAL
@@ -239,8 +351,75 @@ TEST_SUBMODULE(stl, m) {
         .def(py::init<>())
         .def_readonly("member", &opt_exp_holder::member)
         .def("member_initialized", &opt_exp_holder::member_initialized);
+
+    using opt_exp_props = OptionalProperties<std::experimental::optional>;
+    pybind11::class_<opt_exp_props>(m, "OptionalExpProperties")
+        .def(pybind11::init<>())
+        .def_property_readonly("access_by_ref", &opt_exp_props::access_by_ref)
+        .def_property_readonly("access_by_copy", &opt_exp_props::access_by_copy);
 #endif
 
+#if defined(PYBIND11_TEST_BOOST)
+    // test_boost_optional
+    m.attr("has_boost_optional") = true;
+
+    using boost_opt_int = boost::optional<int>;
+    using boost_opt_no_assign = boost::optional<NoAssign>;
+    m.def("double_or_zero_boost", [](const boost_opt_int& x) -> int {
+        return x.value_or(0) * 2;
+    });
+    m.def("half_or_none_boost", [](int x) -> boost_opt_int {
+        return x != 0 ? boost_opt_int(x / 2) : boost_opt_int();
+    });
+    m.def("test_nullopt_boost", [](boost_opt_int x) {
+        return x.value_or(42);
+    }, py::arg_v("x", boost::none, "None"));
+    m.def("test_no_assign_boost", [](const boost_opt_no_assign &x) {
+        return x ? x->value : 42;
+    }, py::arg_v("x", boost::none, "None"));
+
+    using opt_boost_holder = OptionalHolder<boost::optional, MoveOutDetector>;
+    py::class_<opt_boost_holder>(m, "OptionalBoostHolder", "Class with optional member")
+        .def(py::init<>())
+        .def_readonly("member", &opt_boost_holder::member)
+        .def("member_initialized", &opt_boost_holder::member_initialized);
+
+    using opt_boost_props = OptionalProperties<boost::optional>;
+    pybind11::class_<opt_boost_props>(m, "OptionalBoostProperties")
+        .def(pybind11::init<>())
+        .def_property_readonly("access_by_ref", &opt_boost_props::access_by_ref)
+        .def_property_readonly("access_by_copy", &opt_boost_props::access_by_copy);
+#endif
+
+    // test_refsensitive_optional
+    using refsensitive_opt_int = ReferenceSensitiveOptional<int>;
+    using refsensitive_opt_no_assign = ReferenceSensitiveOptional<NoAssign>;
+    m.def("double_or_zero_refsensitive", [](const refsensitive_opt_int& x) -> int {
+        return (x ? x.value() : 0) * 2;
+    });
+    m.def("half_or_none_refsensitive", [](int x) -> refsensitive_opt_int {
+        return x != 0 ? refsensitive_opt_int(x / 2) : refsensitive_opt_int();
+    });
+    // NOLINTNEXTLINE(performance-unnecessary-value-param)
+    m.def("test_nullopt_refsensitive", [](refsensitive_opt_int x) {
+        return x ? x.value() : 42;
+    }, py::arg_v("x", refsensitive_opt_int(), "None"));
+    m.def("test_no_assign_refsensitive", [](const refsensitive_opt_no_assign &x) {
+        return x ? x->value : 42;
+    }, py::arg_v("x", refsensitive_opt_no_assign(), "None"));
+
+    using opt_refsensitive_holder = OptionalHolder<ReferenceSensitiveOptional, MoveOutDetector>;
+    py::class_<opt_refsensitive_holder>(m, "OptionalRefSensitiveHolder", "Class with optional member")
+        .def(py::init<>())
+        .def_readonly("member", &opt_refsensitive_holder::member)
+        .def("member_initialized", &opt_refsensitive_holder::member_initialized);
+
+    using opt_refsensitive_props = OptionalProperties<ReferenceSensitiveOptional>;
+    pybind11::class_<opt_refsensitive_props>(m, "OptionalRefSensitiveProperties")
+        .def(pybind11::init<>())
+        .def_property_readonly("access_by_ref", &opt_refsensitive_props::access_by_ref)
+        .def_property_readonly("access_by_copy", &opt_refsensitive_props::access_by_copy);
+
 #ifdef PYBIND11_HAS_FILESYSTEM
     // test_fs_path
     m.attr("has_filesystem") = true;
@@ -280,8 +459,12 @@ TEST_SUBMODULE(stl, m) {
     m.def("tpl_ctor_set", [](std::unordered_set<TplCtorClass> &) {});
 #if defined(PYBIND11_HAS_OPTIONAL)
     m.def("tpl_constr_optional", [](std::optional<TplCtorClass> &) {});
-#elif defined(PYBIND11_HAS_EXP_OPTIONAL)
-    m.def("tpl_constr_optional", [](std::experimental::optional<TplCtorClass> &) {});
+#endif
+#if defined(PYBIND11_HAS_EXP_OPTIONAL)
+    m.def("tpl_constr_optional_exp", [](std::experimental::optional<TplCtorClass> &) {});
+#endif
+#if defined(PYBIND11_TEST_BOOST)
+    m.def("tpl_constr_optional_boost", [](boost::optional<TplCtorClass> &) {});
 #endif
 
     // test_vec_of_reference_wrapper

tests/test_stl.py

@@ -132,6 +132,10 @@ def test_optional():
     assert mvalue.initialized
     assert holder.member_initialized()
 
+    props = m.OptionalProperties()
+    assert int(props.access_by_ref) == 42
+    assert int(props.access_by_copy) == 42
+
 
 @pytest.mark.skipif(
     not hasattr(m, "has_exp_optional"), reason="no <experimental/optional>"
@@ -160,6 +164,69 @@ def test_exp_optional():
     assert mvalue.initialized
     assert holder.member_initialized()
 
+    props = m.OptionalExpProperties()
+    assert int(props.access_by_ref) == 42
+    assert int(props.access_by_copy) == 42
+
+
+@pytest.mark.skipif(not hasattr(m, "has_boost_optional"), reason="no <boost/optional>")
+def test_boost_optional():
+    assert m.double_or_zero_boost(None) == 0
+    assert m.double_or_zero_boost(42) == 84
+    pytest.raises(TypeError, m.double_or_zero_boost, "foo")
+
+    assert m.half_or_none_boost(0) is None
+    assert m.half_or_none_boost(42) == 21
+    pytest.raises(TypeError, m.half_or_none_boost, "foo")
+
+    assert m.test_nullopt_boost() == 42
+    assert m.test_nullopt_boost(None) == 42
+    assert m.test_nullopt_boost(42) == 42
+    assert m.test_nullopt_boost(43) == 43
+
+    assert m.test_no_assign_boost() == 42
+    assert m.test_no_assign_boost(None) == 42
+    assert m.test_no_assign_boost(m.NoAssign(43)) == 43
+    pytest.raises(TypeError, m.test_no_assign_boost, 43)
+
+    holder = m.OptionalBoostHolder()
+    mvalue = holder.member
+    assert mvalue.initialized
+    assert holder.member_initialized()
+
+    props = m.OptionalBoostProperties()
+    assert int(props.access_by_ref) == 42
+    assert int(props.access_by_copy) == 42
+
+
+def test_reference_sensitive_optional():
+    assert m.double_or_zero_refsensitive(None) == 0
+    assert m.double_or_zero_refsensitive(42) == 84
+    pytest.raises(TypeError, m.double_or_zero_refsensitive, "foo")
+
+    assert m.half_or_none_refsensitive(0) is None
+    assert m.half_or_none_refsensitive(42) == 21
+    pytest.raises(TypeError, m.half_or_none_refsensitive, "foo")
+
+    assert m.test_nullopt_refsensitive() == 42
+    assert m.test_nullopt_refsensitive(None) == 42
+    assert m.test_nullopt_refsensitive(42) == 42
+    assert m.test_nullopt_refsensitive(43) == 43
+
+    assert m.test_no_assign_refsensitive() == 42
+    assert m.test_no_assign_refsensitive(None) == 42
+    assert m.test_no_assign_refsensitive(m.NoAssign(43)) == 43
+    pytest.raises(TypeError, m.test_no_assign_refsensitive, 43)
+
+    holder = m.OptionalRefSensitiveHolder()
+    mvalue = holder.member
+    assert mvalue.initialized
+    assert holder.member_initialized()
+
+    props = m.OptionalRefSensitiveProperties()
+    assert int(props.access_by_ref) == 42
+    assert int(props.access_by_copy) == 42
+
 
 @pytest.mark.skipif(not hasattr(m, "has_filesystem"), reason="no <filesystem>")
 def test_fs_path():