Merge pull request #308 from aldanor/recarray

py::dtype, buffer protocol improvements, structured types support

.gitignore

@@ -3,7 +3,7 @@ CMakeFiles
 Makefile
 cmake_install.cmake
 .DS_Store
-/example/example.so
+/example/example*.so
 /example/example.cpython*.so
 /example/example.pyd
 /example/example*.dll
@@ -31,3 +31,4 @@ MANIFEST
 .DS_Store
 /dist
 /build
+/cmake/

README.md

@@ -106,8 +106,9 @@ Tomasz Miąsko,
 Dean Moldovan,
 Ben Pritchard,
 Jason Rhinelander,
-Boris Schäling, and
-Pim Schellart.
+Boris Schäling,
+Pim Schellart,
+Ivan Smirnov.
 
 ### License
 

docs/advanced.rst

@@ -1224,12 +1224,12 @@ completely avoid copy operations with Python expressions like
     py::class_<Matrix>(m, "Matrix")
        .def_buffer([](Matrix &m) -> py::buffer_info {
             return py::buffer_info(
-                m.data(),                            /* Pointer to buffer */
-                sizeof(float),                       /* Size of one scalar */
-                py::format_descriptor<float>::value, /* Python struct-style format descriptor */
-                2,                                   /* Number of dimensions */
-                { m.rows(), m.cols() },              /* Buffer dimensions */
-                { sizeof(float) * m.rows(),          /* Strides (in bytes) for each index */
+                m.data(),                               /* Pointer to buffer */
+                sizeof(float),                          /* Size of one scalar */
+                py::format_descriptor<float>::format(), /* Python struct-style format descriptor */
+                2,                                      /* Number of dimensions */
+                { m.rows(), m.cols() },                 /* Buffer dimensions */
+                { sizeof(float) * m.rows(),             /* Strides (in bytes) for each index */
                   sizeof(float) }
             );
         });
@@ -1273,7 +1273,7 @@ buffer objects (e.g. a NumPy matrix).
             py::buffer_info info = b.request();
 
             /* Some sanity checks ... */
-            if (info.format != py::format_descriptor<Scalar>::value)
+            if (info.format != py::format_descriptor<Scalar>::format())
                 throw std::runtime_error("Incompatible format: expected a double array!");
 
             if (info.ndim != 2)
@@ -1299,7 +1299,7 @@ as follows:
             m.data(),                /* Pointer to buffer */
             sizeof(Scalar),          /* Size of one scalar */
             /* Python struct-style format descriptor */
-            py::format_descriptor<Scalar>::value,
+            py::format_descriptor<Scalar>::format(),
             /* Number of dimensions */
             2,
             /* Buffer dimensions */
@@ -1358,6 +1358,30 @@ template paramenter, and it ensures that non-conforming arguments are converted
 into an array satisfying the specified requirements instead of trying the next
 function overload.
 
+NumPy structured types
+======================
+
+In order for ``py::array_t`` to work with structured (record) types, we first need
+to register the memory layout of the type. This can be done via ``PYBIND11_NUMPY_DTYPE``
+macro which expects the type followed by field names:
+
+.. code-block:: cpp
+
+    struct A {
+        int x;
+        double y;
+    };
+
+    struct B {
+        int z;
+        A a;
+    };
+
+    PYBIND11_NUMPY_DTYPE(A, x, y);
+    PYBIND11_NUMPY_DTYPE(B, z, a);
+
+    /* now both A and B can be used as template arguments to py::array_t */
+
 Vectorizing functions
 =====================
 
@@ -1433,17 +1457,11 @@ simply using ``vectorize``).
         if (buf1.ndim != 1 || buf2.ndim != 1)
             throw std::runtime_error("Number of dimensions must be one");
 
-        if (buf1.shape[0] != buf2.shape[0])
+        if (buf1.size != buf2.size)
             throw std::runtime_error("Input shapes must match");
 
-        auto result = py::array(py::buffer_info(
-            nullptr,            /* Pointer to data (nullptr -> ask NumPy to allocate!) */
-            sizeof(double),     /* Size of one item */
-            py::format_descriptor<double>::value, /* Buffer format */
-            buf1.ndim,          /* How many dimensions? */
-            { buf1.shape[0] },  /* Number of elements for each dimension */
-            { sizeof(double) }  /* Strides for each dimension */
-        ));
+        /* No pointer is passed, so NumPy will allocate the buffer */
+        auto result = py::array_t<double>(buf1.size);
 
         auto buf3 = result.request();
 
@@ -1830,4 +1848,3 @@ is always ``none``).
 
     // Evaluate the statements in an separate Python file on disk
     py::eval_file("script.py", scope);
-

docs/changelog.rst

@@ -27,6 +27,23 @@ Breaking changes queued for v2.0.0 (Not yet released)
 * ``make_iterator()`` improvements for better compatibility with various types
   (now uses prefix increment operator)
 * ``arg()`` now accepts a wider range of argument types for default values
+* Added support for registering structured dtypes via ``PYBIND11_NUMPY_DTYPE()`` macro.
+* Added ``PYBIND11_STR_TYPE`` macro which maps to the ``builtins.str`` type.
+* Added a simplified ``buffer_info`` constructor for 1-dimensional buffers.
+* Format descriptor strings should now be accessed via ``format_descriptor::format()``
+  (for compatibility purposes, the old syntax ``format_descriptor::value`` will still
+  work for non-structured data types).
+* Added a class wrapping NumPy array descriptors: ``dtype``.
+* Added buffer/NumPy support for ``char[N]`` and ``std::array<char, N>`` types.
+* ``array`` gained new constructors accepting dtype objects.
+* Added constructors for ``array`` and ``array_t`` explicitly accepting shape and
+  strides; if strides are not provided, they are deduced assuming C-contiguity.
+  Also added simplified constructors for 1-dimensional case.
+* Added constructors for ``str`` from ``bytes`` and for ``bytes`` from ``str``.
+  This will do the UTF-8 decoding/encoding as required.
+* Added constructors for ``str`` and ``bytes`` from zero-terminated char pointers,
+  and from char pointers and length.
+* Added ``memoryview`` wrapper type which is constructible from ``buffer_info``.
 * Various minor improvements of library internals (no user-visible changes)
 
 1.8.1 (July 12, 2016)

example/CMakeLists.txt

@@ -26,6 +26,7 @@ set(PYBIND11_EXAMPLES
   example-stl-binder-vector.cpp
   example-eval.cpp
   example-custom-exceptions.cpp
+  example-numpy-dtypes.cpp
   issues.cpp
 )
 
@@ -65,4 +66,3 @@ foreach(VALUE ${PYBIND11_EXAMPLES})
   string(REGEX REPLACE "^(.+).cpp$" "\\1" EXAMPLE_NAME "${VALUE}")
   add_test(NAME ${EXAMPLE_NAME} COMMAND ${RUN_TEST} ${EXAMPLE_NAME})
 endforeach()
-

example/example-buffers.cpp

@@ -81,7 +81,7 @@ void init_ex_buffers(py::module &m) {
         /// Construct from a buffer
         .def("__init__", [](Matrix &v, py::buffer b) {
             py::buffer_info info = b.request();
-            if (info.format != py::format_descriptor<float>::value || info.ndim != 2)
+            if (info.format != py::format_descriptor<float>::format() || info.ndim != 2)
                 throw std::runtime_error("Incompatible buffer format!");
             new (&v) Matrix(info.shape[0], info.shape[1]);
             memcpy(v.data(), info.ptr, sizeof(float) * v.rows() * v.cols());
@@ -104,12 +104,12 @@ void init_ex_buffers(py::module &m) {
        /// Provide buffer access
        .def_buffer([](Matrix &m) -> py::buffer_info {
             return py::buffer_info(
-                m.data(),                            /* Pointer to buffer */
-                sizeof(float),                       /* Size of one scalar */
-                py::format_descriptor<float>::value, /* Python struct-style format descriptor */
-                2,                                   /* Number of dimensions */
-                { m.rows(), m.cols() },              /* Buffer dimensions */
-                { sizeof(float) * m.rows(),          /* Strides (in bytes) for each index */
+                m.data(),                               /* Pointer to buffer */
+                sizeof(float),                          /* Size of one scalar */
+                py::format_descriptor<float>::format(), /* Python struct-style format descriptor */
+                2,                                      /* Number of dimensions */
+                { m.rows(), m.cols() },                 /* Buffer dimensions */
+                { sizeof(float) * m.rows(),             /* Strides (in bytes) for each index */
                   sizeof(float) }
             );
         })

example/example-numpy-dtypes.cpp

+/*
+  example/example-numpy-dtypes.cpp -- Structured and compound NumPy dtypes
+
+  Copyright (c) 2016 Ivan Smirnov
+
+  All rights reserved. Use of this source code is governed by a
+  BSD-style license that can be found in the LICENSE file.
+*/
+
+#include "example.h"
+
+#include <pybind11/numpy.h>
+#include <cstdint>
+#include <iostream>
+
+#ifdef __GNUC__
+#define PYBIND11_PACKED(cls) cls __attribute__((__packed__))
+#else
+#define PYBIND11_PACKED(cls) __pragma(pack(push, 1)) cls __pragma(pack(pop))
+#endif
+
+namespace py = pybind11;
+
+struct SimpleStruct {
+    bool x;
+    uint32_t y;
+    float z;
+};
+
+std::ostream& operator<<(std::ostream& os, const SimpleStruct& v) {
+    return os << "s:" << v.x << "," << v.y << "," << v.z;
+}
+
+PYBIND11_PACKED(struct PackedStruct {
+    bool x;
+    uint32_t y;
+    float z;
+});
+
+std::ostream& operator<<(std::ostream& os, const PackedStruct& v) {
+    return os << "p:" << v.x << "," << v.y << "," << v.z;
+}
+
+PYBIND11_PACKED(struct NestedStruct {
+    SimpleStruct a;
+    PackedStruct b;
+});
+
+std::ostream& operator<<(std::ostream& os, const NestedStruct& v) {
+    return os << "n:a=" << v.a << ";b=" << v.b;
+}
+
+struct PartialStruct {
+    bool x;
+    uint32_t y;
+    float z;
+    uint64_t dummy2;
+};
+
+struct PartialNestedStruct {
+    uint64_t dummy1;
+    PartialStruct a;
+    uint64_t dummy2;
+};
+
+struct UnboundStruct { };
+
+struct StringStruct {
+    char a[3];
+    std::array<char, 3> b;
+};
+
+std::ostream& operator<<(std::ostream& os, const StringStruct& v) {
+    os << "a='";
+    for (size_t i = 0; i < 3 && v.a[i]; i++) os << v.a[i];
+    os << "',b='";
+    for (size_t i = 0; i < 3 && v.b[i]; i++) os << v.b[i];
+    return os << "'";
+}
+
+template <typename T>
+py::array mkarray_via_buffer(size_t n) {
+    return py::array(py::buffer_info(nullptr, sizeof(T),
+                                     py::format_descriptor<T>::format(),
+                                     1, { n }, { sizeof(T) }));
+}
+
+template <typename S>
+py::array_t<S, 0> create_recarray(size_t n) {
+    auto arr = mkarray_via_buffer<S>(n);
+    auto req = arr.request();
+    auto ptr = static_cast<S*>(req.ptr);
+    for (size_t i = 0; i < n; i++) {
+        ptr[i].x = i % 2 != 0; ptr[i].y = (uint32_t) i; ptr[i].z = (float) i * 1.5f;
+    }
+    return arr;
+}
+
+std::string get_format_unbound() {
+    return py::format_descriptor<UnboundStruct>::format();
+}
+
+py::array_t<NestedStruct, 0> create_nested(size_t n) {
+    auto arr = mkarray_via_buffer<NestedStruct>(n);
+    auto req = arr.request();
+    auto ptr = static_cast<NestedStruct*>(req.ptr);
+    for (size_t i = 0; i < n; i++) {
+        ptr[i].a.x = i % 2 != 0; ptr[i].a.y = (uint32_t) i; ptr[i].a.z = (float) i * 1.5f;
+        ptr[i].b.x = (i + 1) % 2 != 0; ptr[i].b.y = (uint32_t) (i + 1); ptr[i].b.z = (float) (i + 1) * 1.5f;
+    }
+    return arr;
+}
+
+py::array_t<PartialNestedStruct, 0> create_partial_nested(size_t n) {
+    auto arr = mkarray_via_buffer<PartialNestedStruct>(n);
+    auto req = arr.request();
+    auto ptr = static_cast<PartialNestedStruct*>(req.ptr);
+    for (size_t i = 0; i < n; i++) {
+        ptr[i].a.x = i % 2 != 0; ptr[i].a.y = (uint32_t) i; ptr[i].a.z = (float) i * 1.5f;
+    }
+    return arr;
+}
+
+py::array_t<StringStruct, 0> create_string_array(bool non_empty) {
+    auto arr = mkarray_via_buffer<StringStruct>(non_empty ? 4 : 0);
+    if (non_empty) {
+        auto req = arr.request();
+        auto ptr = static_cast<StringStruct*>(req.ptr);
+        for (size_t i = 0; i < req.size * req.itemsize; i++)
+            static_cast<char*>(req.ptr)[i] = 0;
+        ptr[1].a[0] = 'a'; ptr[1].b[0] = 'a';
+        ptr[2].a[0] = 'a'; ptr[2].b[0] = 'a';
+        ptr[3].a[0] = 'a'; ptr[3].b[0] = 'a';
+
+        ptr[2].a[1] = 'b'; ptr[2].b[1] = 'b';
+        ptr[3].a[1] = 'b'; ptr[3].b[1] = 'b';
+
+        ptr[3].a[2] = 'c'; ptr[3].b[2] = 'c';
+    }
+    return arr;
+}
+
+template <typename S>
+void print_recarray(py::array_t<S, 0> arr) {
+    auto req = arr.request();
+    auto ptr = static_cast<S*>(req.ptr);
+    for (size_t i = 0; i < req.size; i++)
+        std::cout << ptr[i] << std::endl;
+}
+
+void print_format_descriptors() {
+    std::cout << py::format_descriptor<SimpleStruct>::format() << std::endl;
+    std::cout << py::format_descriptor<PackedStruct>::format() << std::endl;
+    std::cout << py::format_descriptor<NestedStruct>::format() << std::endl;
+    std::cout << py::format_descriptor<PartialStruct>::format() << std::endl;
+    std::cout << py::format_descriptor<PartialNestedStruct>::format() << std::endl;
+    std::cout << py::format_descriptor<StringStruct>::format() << std::endl;
+}
+
+void print_dtypes() {
+    std::cout << (std::string) py::dtype::of<SimpleStruct>().str() << std::endl;
+    std::cout << (std::string) py::dtype::of<PackedStruct>().str() << std::endl;
+    std::cout << (std::string) py::dtype::of<NestedStruct>().str() << std::endl;
+    std::cout << (std::string) py::dtype::of<PartialStruct>().str() << std::endl;
+    std::cout << (std::string) py::dtype::of<PartialNestedStruct>().str() << std::endl;
+    std::cout << (std::string) py::dtype::of<StringStruct>().str() << std::endl;
+}
+
+py::array_t<int32_t, 0> test_array_ctors(int i) {
+    using arr_t = py::array_t<int32_t, 0>;
+
+    std::vector<int32_t> data { 1, 2, 3, 4, 5, 6 };
+    std::vector<size_t> shape { 3, 2 };
+    std::vector<size_t> strides { 8, 4 };
+
+    auto ptr = data.data();
+    auto vptr = (void *) ptr;
+    auto dtype = py::dtype("int32");
+
+    py::buffer_info buf_ndim1(vptr, 4, "i", 6);
+    py::buffer_info buf_ndim1_null(nullptr, 4, "i", 6);
+    py::buffer_info buf_ndim2(vptr, 4, "i", 2, shape, strides);
+    py::buffer_info buf_ndim2_null(nullptr, 4, "i", 2, shape, strides);
+
+    auto fill = [](py::array arr) {
+        auto req = arr.request();
+        for (int i = 0; i < 6; i++) ((int32_t *) req.ptr)[i] = i + 1;
+        return arr;
+    };
+
+    switch (i) {
+    // shape: (3, 2)
+    case 10: return arr_t(shape, strides, ptr);
+    case 11: return py::array(shape, strides, ptr);
+    case 12: return py::array(dtype, shape, strides, vptr);
+    case 13: return arr_t(shape, ptr);
+    case 14: return py::array(shape, ptr);
+    case 15: return py::array(dtype, shape, vptr);
+    case 16: return arr_t(buf_ndim2);
+    case 17: return py::array(buf_ndim2);
+    // shape: (3, 2) - post-fill
+    case 20: return fill(arr_t(shape, strides));
+    case 21: return py::array(shape, strides, ptr); // can't have nullptr due to templated ctor
+    case 22: return fill(py::array(dtype, shape, strides));
+    case 23: return fill(arr_t(shape));
+    case 24: return py::array(shape, ptr); // can't have nullptr due to templated ctor
+    case 25: return fill(py::array(dtype, shape));
+    case 26: return fill(arr_t(buf_ndim2_null));
+    case 27: return fill(py::array(buf_ndim2_null));
+    // shape: (6, )
+    case 30: return arr_t(6, ptr);
+    case 31: return py::array(6, ptr);
+    case 32: return py::array(dtype, 6, vptr);
+    case 33: return arr_t(buf_ndim1);
+    case 34: return py::array(buf_ndim1);
+    // shape: (6, )
+    case 40: return fill(arr_t(6));
+    case 41: return py::array(6, ptr);  // can't have nullptr due to templated ctor
+    case 42: return fill(py::array(dtype, 6));
+    case 43: return fill(arr_t(buf_ndim1_null));
+    case 44: return fill(py::array(buf_ndim1_null));
+    }
+    return arr_t();
+}
+
+py::list test_dtype_ctors() {
+    py::list list;
+    list.append(py::dtype("int32"));
+    list.append(py::dtype(std::string("float64")));
+    list.append(py::dtype::from_args(py::str("bool")));
+    py::list names, offsets, formats;
+    py::dict dict;
+    names.append(py::str("a")); names.append(py::str("b")); dict["names"] = names;
+    offsets.append(py::int_(1)); offsets.append(py::int_(10)); dict["offsets"] = offsets;
+    formats.append(py::dtype("int32")); formats.append(py::dtype("float64")); dict["formats"] = formats;
+    dict["itemsize"] = py::int_(20);
+    list.append(py::dtype::from_args(dict));
+    list.append(py::dtype(names, formats, offsets, 20));
+    list.append(py::dtype(py::buffer_info((void *) 0, 1, "I", 1)));
+    list.append(py::dtype(py::buffer_info((void *) 0, 1, "T{i:a:f:b:}", 1)));
+    return list;
+}
+
+py::list test_dtype_methods() {
+    py::list list;
+    auto dt1 = py::dtype::of<int32_t>();
+    auto dt2 = py::dtype::of<SimpleStruct>();
+    list.append(dt1); list.append(dt2);
+    list.append(py::bool_(dt1.has_fields())); list.append(py::bool_(dt2.has_fields()));
+    list.append(py::int_(dt1.itemsize())); list.append(py::int_(dt2.itemsize()));
+    return list;
+}
+
+void init_ex_numpy_dtypes(py::module &m) {
+    PYBIND11_NUMPY_DTYPE(SimpleStruct, x, y, z);
+    PYBIND11_NUMPY_DTYPE(PackedStruct, x, y, z);
+    PYBIND11_NUMPY_DTYPE(NestedStruct, a, b);
+    PYBIND11_NUMPY_DTYPE(PartialStruct, x, y, z);
+    PYBIND11_NUMPY_DTYPE(PartialNestedStruct, a);
+    PYBIND11_NUMPY_DTYPE(StringStruct, a, b);
+
+    m.def("create_rec_simple", &create_recarray<SimpleStruct>);
+    m.def("create_rec_packed", &create_recarray<PackedStruct>);
+    m.def("create_rec_nested", &create_nested);
+    m.def("create_rec_partial", &create_recarray<PartialStruct>);
+    m.def("create_rec_partial_nested", &create_partial_nested);
+    m.def("print_format_descriptors", &print_format_descriptors);
+    m.def("print_rec_simple", &print_recarray<SimpleStruct>);
+    m.def("print_rec_packed", &print_recarray<PackedStruct>);
+    m.def("print_rec_nested", &print_recarray<NestedStruct>);
+    m.def("print_dtypes", &print_dtypes);
+    m.def("get_format_unbound", &get_format_unbound);
+    m.def("create_string_array", &create_string_array);
+    m.def("print_string_array", &print_recarray<StringStruct>);
+    m.def("test_array_ctors", &test_array_ctors);
+    m.def("test_dtype_ctors", &test_dtype_ctors);
+    m.def("test_dtype_methods", &test_dtype_methods);
+}
+
+#undef PYBIND11_PACKED

example/example-numpy-dtypes.py

+#!/usr/bin/env python
+from __future__ import print_function
+
+import numpy as np
+from example import (
+    create_rec_simple, create_rec_packed, create_rec_nested, print_format_descriptors,
+    print_rec_simple, print_rec_packed, print_rec_nested, print_dtypes, get_format_unbound,
+    create_rec_partial, create_rec_partial_nested, create_string_array, print_string_array,
+    test_array_ctors, test_dtype_ctors, test_dtype_methods
+)
+
+
+def check_eq(arr, data, dtype):
+    np.testing.assert_equal(arr, np.array(data, dtype=dtype))
+
+try:
+    get_format_unbound()
+    raise Exception
+except RuntimeError as e:
+    assert 'unsupported buffer format' in str(e)
+
+print_format_descriptors()
+print_dtypes()
+
+simple_dtype = np.dtype({'names': ['x', 'y', 'z'],
+                         'formats': ['?', 'u4', 'f4'],
+                         'offsets': [0, 4, 8]})
+packed_dtype = np.dtype([('x', '?'), ('y', 'u4'), ('z', 'f4')])
+
+elements = [(False, 0, 0.0), (True, 1, 1.5), (False, 2, 3.0)]
+
+for func, dtype in [(create_rec_simple, simple_dtype), (create_rec_packed, packed_dtype)]:
+    arr = func(0)
+    assert arr.dtype == dtype
+    check_eq(arr, [], simple_dtype)
+    check_eq(arr, [], packed_dtype)
+
+    arr = func(3)
+    assert arr.dtype == dtype
+    check_eq(arr, elements, simple_dtype)
+    check_eq(arr, elements, packed_dtype)
+
+    if dtype == simple_dtype:
+        print_rec_simple(arr)
+    else:
+        print_rec_packed(arr)
+
+arr = create_rec_partial(3)
+print(arr.dtype)
+partial_dtype = arr.dtype
+assert '' not in arr.dtype.fields
+assert partial_dtype.itemsize > simple_dtype.itemsize
+check_eq(arr, elements, simple_dtype)
+check_eq(arr, elements, packed_dtype)
+
+arr = create_rec_partial_nested(3)
+print(arr.dtype)
+assert '' not in arr.dtype.fields
+assert '' not in arr.dtype.fields['a'][0].fields
+assert arr.dtype.itemsize > partial_dtype.itemsize
+np.testing.assert_equal(arr['a'], create_rec_partial(3))
+
+nested_dtype = np.dtype([('a', simple_dtype), ('b', packed_dtype)])
+
+arr = create_rec_nested(0)
+assert arr.dtype == nested_dtype
+check_eq(arr, [], nested_dtype)
+
+arr = create_rec_nested(3)
+assert arr.dtype == nested_dtype
+check_eq(arr, [((False, 0, 0.0), (True, 1, 1.5)),
+               ((True, 1, 1.5), (False, 2, 3.0)),
+               ((False, 2, 3.0), (True, 3, 4.5))], nested_dtype)
+print_rec_nested(arr)
+
+assert create_rec_nested.__doc__.strip().endswith('numpy.ndarray[NestedStruct]')
+
+arr = create_string_array(True)
+print(arr.dtype)
+print_string_array(arr)
+dtype = arr.dtype
+assert arr['a'].tolist() == [b'', b'a', b'ab', b'abc']
+assert arr['b'].tolist() == [b'', b'a', b'ab', b'abc']
+arr = create_string_array(False)
+assert dtype == arr.dtype
+
+data = np.arange(1, 7, dtype='int32')
+for i in range(8):
+    np.testing.assert_array_equal(test_array_ctors(10 + i), data.reshape((3, 2)))
+    np.testing.assert_array_equal(test_array_ctors(20 + i), data.reshape((3, 2)))
+for i in range(5):
+    np.testing.assert_array_equal(test_array_ctors(30 + i), data)
+    np.testing.assert_array_equal(test_array_ctors(40 + i), data)
+
+d1 = np.dtype({'names': ['a', 'b'], 'formats': ['int32', 'float64'],
+               'offsets': [1, 10], 'itemsize': 20})
+d2 = np.dtype([('a', 'i4'), ('b', 'f4')])
+assert test_dtype_ctors() == [np.dtype('int32'), np.dtype('float64'),
+                              np.dtype('bool'), d1, d1, np.dtype('uint32'), d2]
+
+assert test_dtype_methods() == [np.dtype('int32'), simple_dtype, False, True,
+                                np.dtype('int32').itemsize, simple_dtype.itemsize]

example/example-numpy-dtypes.ref

+T{=?:x:3x=I:y:=f:z:}
+T{=?:x:=I:y:=f:z:}
+T{=T{=?:x:3x=I:y:=f:z:}:a:=T{=?:x:=I:y:=f:z:}:b:}
+T{=?:x:3x=I:y:=f:z:12x}
+T{8x=T{=?:x:3x=I:y:=f:z:12x}:a:8x}
+T{=3s:a:=3s:b:}
+{'names':['x','y','z'], 'formats':['?','<u4','<f4'], 'offsets':[0,4,8], 'itemsize':12}
+[('x', '?'), ('y', '<u4'), ('z', '<f4')]
+[('a', {'names':['x','y','z'], 'formats':['?','<u4','<f4'], 'offsets':[0,4,8], 'itemsize':12}), ('b', [('x', '?'), ('y', '<u4'), ('z', '<f4')])]
+{'names':['x','y','z'], 'formats':['?','<u4','<f4'], 'offsets':[0,4,8], 'itemsize':24}
+{'names':['a'], 'formats':[{'names':['x','y','z'], 'formats':['?','<u4','<f4'], 'offsets':[0,4,8], 'itemsize':24}], 'offsets':[8], 'itemsize':40}
+[('a', 'S3'), ('b', 'S3')]
+s:0,0,0
+s:1,1,1.5
+s:0,2,3
+p:0,0,0
+p:1,1,1.5
+p:0,2,3
+{'names':['x','y','z'], 'formats':['?','<u4','<f4'], 'offsets':[0,4,8], 'itemsize':24}
+{'names':['a'], 'formats':[{'names':['x','y','z'], 'formats':['?','<u4','<f4'], 'offsets':[0,4,8], 'itemsize':24}], 'offsets':[8], 'itemsize':40}
+n:a=s:0,0,0;b=p:1,1,1.5
+n:a=s:1,1,1.5;b=p:0,2,3
+n:a=s:0,2,3;b=p:1,3,4.5
+[('a', 'S3'), ('b', 'S3')]
+a='',b=''
+a='a',b='a'
+a='ab',b='ab'
+a='abc',b='abc'

example/example-python-types.cpp

@@ -139,6 +139,22 @@ public:
         throw std::runtime_error("This exception was intentionally thrown.");
     }
 
+    py::bytes get_bytes_from_string() {
+        return (py::bytes) std::string("foo");
+    }
+
+    py::bytes get_bytes_from_str() {
+        return (py::bytes) py::str("bar", 3);
+    }
+
+    py::str get_str_from_string() {
+        return (py::str) std::string("baz");
+    }
+
+    py::str get_str_from_bytes() {
+        return (py::str) py::bytes("boo", 3);
+    }
+
     static int value;
     static const int value2;
 };
@@ -167,6 +183,10 @@ void init_ex_python_types(py::module &m) {
         .def("pair_passthrough", &ExamplePythonTypes::pair_passthrough, "Return a pair in reversed order")
         .def("tuple_passthrough", &ExamplePythonTypes::tuple_passthrough, "Return a triple in reversed order")
         .def("throw_exception", &ExamplePythonTypes::throw_exception, "Throw an exception")
+        .def("get_bytes_from_string", &ExamplePythonTypes::get_bytes_from_string, "py::bytes from std::string")
+        .def("get_bytes_from_str", &ExamplePythonTypes::get_bytes_from_str, "py::bytes from py::str")
+        .def("get_str_from_string", &ExamplePythonTypes::get_str_from_string, "py::str from std::string")
+        .def("get_str_from_bytes", &ExamplePythonTypes::get_str_from_bytes, "py::str from py::bytes")
         .def_static("new_instance", &ExamplePythonTypes::new_instance, "Return an instance")
         .def_readwrite_static("value", &ExamplePythonTypes::value, "Static value member")
         .def_readonly_static("value2", &ExamplePythonTypes::value2, "Static value member (readonly)")

example/example-python-types.py

@@ -66,6 +66,11 @@ print("__module__(example.ExamplePythonTypes) = %s" % ExamplePythonTypes.__modul
 print("__name__(example.ExamplePythonTypes.get_set) = %s" % ExamplePythonTypes.get_set.__name__)
 print("__module__(example.ExamplePythonTypes.get_set) = %s" % ExamplePythonTypes.get_set.__module__)
 
+print(instance.get_bytes_from_string().decode())
+print(instance.get_bytes_from_str().decode())
+print(instance.get_str_from_string().encode().decode())
+print(instance.get_str_from_bytes().encode().decode())
+
 from example import ConstructorStats
 
 cstats = ConstructorStats.get(ExamplePythonTypes)

example/example-python-types.ref

@@ -28,106 +28,106 @@ Help on class ExamplePythonTypes in module example
 
 class EExxaammpplleePPyytthhoonnTTyyppeess(__builtin__.object)
  |  Example 2 documentation
- |  
+ |
  |  Methods defined here:
- |  
+ |
  |  ____iinniitt____(...)
  |      x.__init__(...) initializes x; see help(type(x)) for signature
- |  
+ |
  |  ggeett__aarrrraayy(...)
- |      
+ |
  |      Signature : (example.ExamplePythonTypes) -> List[unicode[2]]
  |      Return a C++ array
- |  
+ |
  |  ggeett__ddiicctt(...)
  |      Signature : (example.ExamplePythonTypes) -> dict
- |      
+ |
  |      Return a Python dictionary
- |  
+ |
  |  ggeett__ddiicctt__22(...)
- |      
+ |
  |      Signature : (example.ExamplePythonTypes) -> Dict[unicode, unicode]
  |      Return a C++ dictionary
- |  
+ |
  |  ggeett__lliisstt(...)
  |      Signature : (example.ExamplePythonTypes) -> list
- |      
+ |
  |      Return a Python list
- |  
+ |
  |  ggeett__lliisstt__22(...)
- |      
+ |
  |      Signature : (example.ExamplePythonTypes) -> List[unicode]
  |      Return a C++ list
- |  
+ |
  |  ggeett__sseett(...)
  |      Signature : (example.ExamplePythonTypes) -> set
- |      
+ |
  |      Return a Python set
- |  
+ |
  |  ggeett__sseett22(...)
  |      Signature : (example.ExamplePythonTypes) -> set
- |      
+ |
  |      Return a C++ set
- |  
+ |
  |  ppaaiirr__ppaasssstthhrroouugghh(...)
- |      
+ |
  |      Signature : (example.ExamplePythonTypes, Tuple[bool, unicode]) -> Tuple[unicode, bool]
  |      Return a pair in reversed order
- |  
+ |
  |  pprriinntt__aarrrraayy(...)
- |      
+ |
  |      Signature : (example.ExamplePythonTypes, List[unicode[2]]) -> None
  |      Print entries of a C++ array
- |  
+ |
  |  pprriinntt__ddiicctt(...)
- |      
+ |
  |      Signature : (example.ExamplePythonTypes, dict) -> None
  |      Print entries of a Python dictionary
- |  
+ |
  |  pprriinntt__ddiicctt__22(...)
- |      
+ |
  |      Signature : (example.ExamplePythonTypes, Dict[unicode, unicode]) -> None
  |      Print entries of a C++ dictionary
- |  
+ |
  |  pprriinntt__lliisstt(...)
- |      
+ |
  |      Signature : (example.ExamplePythonTypes, list) -> None
  |      Print entries of a Python list
- |  
+ |
  |  pprriinntt__lliisstt__22(...)
- |      
+ |
  |      Signature : (example.ExamplePythonTypes, List[unicode]) -> None
  |      Print entries of a C++ list
- |  
+ |
  |  pprriinntt__sseett(...)
- |      
+ |
  |      Signature : (example.ExamplePythonTypes, set) -> None
  |      Print entries of a Python set
- |  
+ |
  |  pprriinntt__sseett__22(...)
- |      
+ |
  |      Signature : (example.ExamplePythonTypes, Set[unicode]) -> None
  |      Print entries of a C++ set
- |  
+ |
  |  tthhrrooww__eexxcceeppttiioonn(...)
- |      
+ |
  |      Signature : (example.ExamplePythonTypes) -> None
  |      Throw an exception
- |  
+ |
  |  ttuuppllee__ppaasssstthhrroouugghh(...)
- |      
+ |
  |      Signature : (example.ExamplePythonTypes, Tuple[bool, unicode, int]) -> Tuple[int, unicode, bool]
  |      Return a triple in reversed order
- |  
+ |
  |  ----------------------------------------------------------------------
  |  Data and other attributes defined here:
- |  
+ |
  |  ____nneeww____ = <built-in method __new__ of example.ExamplePythonTypes__Meta object>
  |      T.__new__(S, ...) -> a new object with type S, a subtype of T
- |  
+ |
  |  nneeww__iinnssttaannccee = <built-in method new_instance of PyCapsule object>
  |      Signature : () -> example.ExamplePythonTypes
- |      
+ |
  |      Return an instance
 
 __name__(example) = example
@@ -135,6 +135,10 @@ __name__(example.ExamplePythonTypes) = ExamplePythonTypes
 __module__(example.ExamplePythonTypes) = example
 __name__(example.ExamplePythonTypes.get_set) = get_set
 __module__(example.ExamplePythonTypes.get_set) = example
+foo
+bar
+baz
+boo
 Instances not destroyed: 1
 ### ExamplePythonTypes @ 0x1045b80 destroyed
 Instances not destroyed: 0

example/example.cpp

@@ -29,6 +29,7 @@ void init_ex_inheritance(py::module &);
 void init_ex_stl_binder_vector(py::module &);
 void init_ex_eval(py::module &);
 void init_ex_custom_exceptions(py::module &);
+void init_ex_numpy_dtypes(py::module &);
 void init_issues(py::module &);
 
 #if defined(PYBIND11_TEST_EIGEN)
@@ -72,6 +73,7 @@ PYBIND11_PLUGIN(example) {
     init_ex_stl_binder_vector(m);
     init_ex_eval(m);
     init_ex_custom_exceptions(m);
+    init_ex_numpy_dtypes(m);
     init_issues(m);
 
     #if defined(PYBIND11_TEST_EIGEN)

example/run_test.py

@@ -46,12 +46,11 @@ name = sys.argv[1]
 try:
     output_bytes = subprocess.check_output([sys.executable, "-u", name + ".py"],
                                            stderr=subprocess.STDOUT)
-except subprocess.CalledProcessError as e:
-    if e.returncode == 99:
-        print('Test "%s" could not be run.' % name)
-        exit(0)
-    else:
-        raise
+except subprocess.CalledProcessError as exc:
+    print('Test `{}` failed:\n{}\n'.format(name, '-' * 50))
+    print(exc.output.decode())
+    print('-' * 50)
+    sys.exit(1)
 
 output    = sanitize(output_bytes.decode('utf-8'))
 reference = sanitize(open(name + '.ref', 'r').read())

include/pybind11/common.h

@@ -95,6 +95,7 @@
 #define PYBIND11_STRING_NAME "str"
 #define PYBIND11_SLICE_OBJECT PyObject
 #define PYBIND11_FROM_STRING PyUnicode_FromString
+#define PYBIND11_STR_TYPE ::pybind11::str
 #define PYBIND11_OB_TYPE(ht_type) (ht_type).ob_base.ob_base.ob_type
 #define PYBIND11_PLUGIN_IMPL(name) \
     extern "C" PYBIND11_EXPORT PyObject *PyInit_##name()
@@ -113,6 +114,7 @@
 #define PYBIND11_STRING_NAME "unicode"
 #define PYBIND11_SLICE_OBJECT PySliceObject
 #define PYBIND11_FROM_STRING PyString_FromString
+#define PYBIND11_STR_TYPE ::pybind11::bytes
 #define PYBIND11_OB_TYPE(ht_type) (ht_type).ob_type
 #define PYBIND11_PLUGIN_IMPL(name) \
     extern "C" PYBIND11_EXPORT PyObject *init##name()
@@ -204,12 +206,13 @@ struct buffer_info {
     void *ptr;                   // Pointer to the underlying storage
     size_t itemsize;             // Size of individual items in bytes
     size_t size;                 // Total number of entries
-    std::string format;          // For homogeneous buffers, this should be set to format_descriptor<T>::value
+    std::string format;          // For homogeneous buffers, this should be set to format_descriptor<T>::format()
     size_t ndim;                 // Number of dimensions
     std::vector<size_t> shape;   // Shape of the tensor (1 entry per dimension)
     std::vector<size_t> strides; // Number of entries between adjacent entries (for each per dimension)
 
     buffer_info() : ptr(nullptr), view(nullptr) {}
+
     buffer_info(void *ptr, size_t itemsize, const std::string &format, size_t ndim,
                 const std::vector<size_t> &shape, const std::vector<size_t> &strides)
         : ptr(ptr), itemsize(itemsize), size(1), format(format),
@@ -218,6 +221,10 @@ struct buffer_info {
             size *= shape[i];
     }
 
+    buffer_info(void *ptr, size_t itemsize, const std::string &format, size_t size)
+    : buffer_info(ptr, itemsize, format, 1, std::vector<size_t> { size },
+                  std::vector<size_t> { itemsize }) { }
+
     buffer_info(Py_buffer *view)
         : ptr(view->buf), itemsize((size_t) view->itemsize), size(1), format(view->format),
           ndim((size_t) view->ndim), shape((size_t) view->ndim), strides((size_t) view->ndim), view(view) {
@@ -231,6 +238,7 @@ struct buffer_info {
     ~buffer_info() {
         if (view) { PyBuffer_Release(view); delete view; }
     }
+
 private:
     Py_buffer *view = nullptr;
 };
@@ -323,14 +331,23 @@ PYBIND11_RUNTIME_EXCEPTION(reference_cast_error) /// Used internally
 [[noreturn]] PYBIND11_NOINLINE inline void pybind11_fail(const std::string &reason) { throw std::runtime_error(reason); }
 
 /// Format strings for basic number types
-#define PYBIND11_DECL_FMT(t, v) template<> struct format_descriptor<t> { static constexpr const char *value = v; }
+#define PYBIND11_DECL_FMT(t, v) template<> struct format_descriptor<t> \
+    { static constexpr const char* value = v; /* for backwards compatibility */ \
+      static constexpr const char* format() { return value; } }
+
 template <typename T, typename SFINAE = void> struct format_descriptor { };
+
 template <typename T> struct format_descriptor<T, typename std::enable_if<std::is_integral<T>::value>::type> {
     static constexpr const char value[2] =
         { "bBhHiIqQ"[detail::log2(sizeof(T))*2 + (std::is_unsigned<T>::value ? 1 : 0)], '\0' };
+    static constexpr const char* format() { return value; }
 };
+
 template <typename T> constexpr const char format_descriptor<
     T, typename std::enable_if<std::is_integral<T>::value>::type>::value[2];
-PYBIND11_DECL_FMT(float, "f"); PYBIND11_DECL_FMT(double, "d"); PYBIND11_DECL_FMT(bool, "?");
+
+PYBIND11_DECL_FMT(float, "f");
+PYBIND11_DECL_FMT(double, "d");
+PYBIND11_DECL_FMT(bool, "?");
 
 NAMESPACE_END(pybind11)

include/pybind11/descr.h

@@ -181,7 +181,7 @@ PYBIND11_NOINLINE inline descr concat(descr &&d) { return d; }
 template <typename... Args> PYBIND11_NOINLINE descr concat(descr &&d, Args&&... args) { return std::move(d) + _(", ") + concat(std::forward<Args>(args)...); }
 PYBIND11_NOINLINE inline descr type_descr(descr&& d) { return _("{") + std::move(d) + _("}"); }
 
-#define PYBIND11_DESCR descr
+#define PYBIND11_DESCR ::pybind11::detail::descr
 #endif
 
 NAMESPACE_END(detail)

include/pybind11/eigen.h

@@ -133,7 +133,7 @@ struct type_caster<Type, typename std::enable_if<is_eigen_dense<Type>::value &&
                 /* Size of one scalar */
                 sizeof(Scalar),
                 /* Python struct-style format descriptor */
-                format_descriptor<Scalar>::value,
+                format_descriptor<Scalar>::format(),
                 /* Number of dimensions */
                 1,
                 /* Buffer dimensions */
@@ -148,7 +148,7 @@ struct type_caster<Type, typename std::enable_if<is_eigen_dense<Type>::value &&
                 /* Size of one scalar */
                 sizeof(Scalar),
                 /* Python struct-style format descriptor */
-                format_descriptor<Scalar>::value,
+                format_descriptor<Scalar>::format(),
                 /* Number of dimensions */
                 isVector ? 1 : 2,
                 /* Buffer dimensions */
@@ -233,7 +233,7 @@ struct type_caster<Type, typename std::enable_if<is_eigen_sparse<Type>::value>::
             try {
                 obj = matrix_type(obj);
             } catch (const error_already_set &) {
-                PyErr_Clear(); 
+                PyErr_Clear();
                 return false;
             }
         }
@@ -276,7 +276,7 @@ struct type_caster<Type, typename std::enable_if<is_eigen_sparse<Type>::value>::
             // Size of one scalar
             sizeof(Scalar),
             // Python struct-style format descriptor
-            format_descriptor<Scalar>::value,
+            format_descriptor<Scalar>::format(),
             // Number of dimensions
             1,
             // Buffer dimensions
@@ -291,7 +291,7 @@ struct type_caster<Type, typename std::enable_if<is_eigen_sparse<Type>::value>::
             // Size of one scalar
             sizeof(StorageIndex),
             // Python struct-style format descriptor
-            format_descriptor<StorageIndex>::value,
+            format_descriptor<StorageIndex>::format(),
             // Number of dimensions
             1,
             // Buffer dimensions
@@ -306,7 +306,7 @@ struct type_caster<Type, typename std::enable_if<is_eigen_sparse<Type>::value>::
             // Size of one scalar
             sizeof(StorageIndex),
             // Python struct-style format descriptor
-            format_descriptor<StorageIndex>::value,
+            format_descriptor<StorageIndex>::format(),
             // Number of dimensions
             1,
             // Buffer dimensions

include/pybind11/pytypes.h

@@ -339,12 +339,14 @@ inline iterator handle::begin() const { return iterator(PyObject_GetIter(ptr()),
 inline iterator handle::end() const { return iterator(nullptr, false); }
 inline detail::args_proxy handle::operator*() const { return detail::args_proxy(*this); }
 
+class bytes;
+
 class str : public object {
 public:
     PYBIND11_OBJECT_DEFAULT(str, object, detail::PyUnicode_Check_Permissive)
 
-    str(const std::string &s)
-        : object(PyUnicode_FromStringAndSize(s.c_str(), (ssize_t) s.length()), false) {
+    str(const char *c, size_t n)
+    : object(PyUnicode_FromStringAndSize(c, (ssize_t) n), false) {
         if (!m_ptr) pybind11_fail("Could not allocate string object!");
     }
 
@@ -353,6 +355,10 @@ public:
         if (!m_ptr) pybind11_fail("Could not allocate string object!");
     }
 
+    str(const std::string &s) : str(s.data(), s.size()) { }
+
+    str(const bytes &b);
+
     operator std::string() const {
         object temp = *this;
         if (PyUnicode_Check(m_ptr)) {
@@ -362,8 +368,7 @@ public:
         }
         char *buffer;
         ssize_t length;
-        int err = PYBIND11_BYTES_AS_STRING_AND_SIZE(temp.ptr(), &buffer, &length);
-        if (err == -1)
+        if (PYBIND11_BYTES_AS_STRING_AND_SIZE(temp.ptr(), &buffer, &length))
             pybind11_fail("Unable to extract string contents! (invalid type)");
         return std::string(buffer, (size_t) length);
     }
@@ -382,21 +387,57 @@ class bytes : public object {
 public:
     PYBIND11_OBJECT_DEFAULT(bytes, object, PYBIND11_BYTES_CHECK)
 
-    bytes(const std::string &s)
-        : object(PYBIND11_BYTES_FROM_STRING_AND_SIZE(s.data(), (ssize_t) s.size()), false) {
+    bytes(const char *c)
+    : object(PYBIND11_BYTES_FROM_STRING(c), false) {
+        if (!m_ptr) pybind11_fail("Could not allocate bytes object!");
+    }
+
+    bytes(const char *c, size_t n)
+    : object(PYBIND11_BYTES_FROM_STRING_AND_SIZE(c, (ssize_t) n), false) {
         if (!m_ptr) pybind11_fail("Could not allocate bytes object!");
     }
 
+    bytes(const std::string &s) : bytes(s.data(), s.size()) { }
+
+    bytes(const pybind11::str &s);
+
     operator std::string() const {
         char *buffer;
         ssize_t length;
-        int err = PYBIND11_BYTES_AS_STRING_AND_SIZE(m_ptr, &buffer, &length);
-        if (err == -1)
+        if (PYBIND11_BYTES_AS_STRING_AND_SIZE(m_ptr, &buffer, &length))
             pybind11_fail("Unable to extract bytes contents!");
         return std::string(buffer, (size_t) length);
     }
 };
 
+inline bytes::bytes(const pybind11::str &s) {
+    object temp = s;
+    if (PyUnicode_Check(s.ptr())) {
+        temp = object(PyUnicode_AsUTF8String(s.ptr()), false);
+        if (!temp)
+            pybind11_fail("Unable to extract string contents! (encoding issue)");
+    }
+    char *buffer;
+    ssize_t length;
+    if (PYBIND11_BYTES_AS_STRING_AND_SIZE(temp.ptr(), &buffer, &length))
+        pybind11_fail("Unable to extract string contents! (invalid type)");
+    auto obj = object(PYBIND11_BYTES_FROM_STRING_AND_SIZE(buffer, length), false);
+    if (!obj)
+        pybind11_fail("Could not allocate bytes object!");
+    m_ptr = obj.release().ptr();
+}
+
+inline str::str(const bytes& b) {
+    char *buffer;
+    ssize_t length;
+    if (PYBIND11_BYTES_AS_STRING_AND_SIZE(b.ptr(), &buffer, &length))
+        pybind11_fail("Unable to extract bytes contents!");
+    auto obj = object(PyUnicode_FromStringAndSize(buffer, (ssize_t) length), false);
+    if (!obj)
+        pybind11_fail("Could not allocate string object!");
+    m_ptr = obj.release().ptr();
+}
+
 class none : public object {
 public:
     PYBIND11_OBJECT(none, object, detail::PyNone_Check)
@@ -570,6 +611,38 @@ public:
     }
 };
 
+class memoryview : public object {
+public:
+    memoryview(const buffer_info& info) {
+        static Py_buffer buf { };
+        // Py_buffer uses signed sizes, strides and shape!..
+        static std::vector<Py_ssize_t> py_strides { };
+        static std::vector<Py_ssize_t> py_shape { };
+        buf.buf = info.ptr;
+        buf.itemsize = (Py_ssize_t) info.itemsize;
+        buf.format = const_cast<char *>(info.format.c_str());
+        buf.ndim = (int) info.ndim;
+        buf.len = (Py_ssize_t) info.size;
+        py_strides.clear();
+        py_shape.clear();
+        for (size_t i = 0; i < info.ndim; ++i) {
+            py_strides.push_back((Py_ssize_t) info.strides[i]);
+            py_shape.push_back((Py_ssize_t) info.shape[i]);
+        }
+        buf.strides = py_strides.data();
+        buf.shape = py_shape.data();
+        buf.suboffsets = nullptr;
+        buf.readonly = false;
+        buf.internal = nullptr;
+
+        m_ptr = PyMemoryView_FromBuffer(&buf);
+        if (!m_ptr)
+            pybind11_fail("Unable to create memoryview from buffer descriptor");
+    }
+
+    PYBIND11_OBJECT_DEFAULT(memoryview, object, PyMemoryView_Check)
+};
+
 inline size_t len(handle h) {
     ssize_t result = PyObject_Length(h.ptr());
     if (result < 0)