Merge pull request #308 from aldanor/recarray

py::dtype, buffer protocol improvements, structured types support

.gitignore

@@ -3,7 +3,7 @@ CMakeFiles
 Makefile
 cmake_install.cmake
 .DS_Store
-/example/example.so
+/example/example*.so
 /example/example.cpython*.so
 /example/example.pyd
 /example/example*.dll
@@ -31,3 +31,4 @@ MANIFEST
 .DS_Store
 /dist
 /build
+/cmake/

README.md

@@ -106,8 +106,9 @@ Tomasz Miąsko,
 Dean Moldovan,
 Ben Pritchard,
 Jason Rhinelander,
-Boris Schäling, and
-Pim Schellart.
+Boris Schäling,
+Pim Schellart,
+Ivan Smirnov.
 
 ### License
 

docs/advanced.rst

@@ -1226,7 +1226,7 @@ completely avoid copy operations with Python expressions like
             return py::buffer_info(
                 m.data(),                               /* Pointer to buffer */
                 sizeof(float),                          /* Size of one scalar */
-                py::format_descriptor<float>::value, /* Python struct-style format descriptor */
+                py::format_descriptor<float>::format(), /* Python struct-style format descriptor */
                 2,                                      /* Number of dimensions */
                 { m.rows(), m.cols() },                 /* Buffer dimensions */
                 { sizeof(float) * m.rows(),             /* Strides (in bytes) for each index */
@@ -1273,7 +1273,7 @@ buffer objects (e.g. a NumPy matrix).
             py::buffer_info info = b.request();
 
             /* Some sanity checks ... */
-            if (info.format != py::format_descriptor<Scalar>::value)
+            if (info.format != py::format_descriptor<Scalar>::format())
                 throw std::runtime_error("Incompatible format: expected a double array!");
 
             if (info.ndim != 2)
@@ -1299,7 +1299,7 @@ as follows:
             m.data(),                /* Pointer to buffer */
             sizeof(Scalar),          /* Size of one scalar */
             /* Python struct-style format descriptor */
-            py::format_descriptor<Scalar>::value,
+            py::format_descriptor<Scalar>::format(),
             /* Number of dimensions */
             2,
             /* Buffer dimensions */
@@ -1358,6 +1358,30 @@ template paramenter, and it ensures that non-conforming arguments are converted
 into an array satisfying the specified requirements instead of trying the next
 function overload.
 
+NumPy structured types
+======================
+
+In order for ``py::array_t`` to work with structured (record) types, we first need
+to register the memory layout of the type. This can be done via ``PYBIND11_NUMPY_DTYPE``
+macro which expects the type followed by field names:
+
+.. code-block:: cpp
+
+    struct A {
+        int x;
+        double y;
+    };
+
+    struct B {
+        int z;
+        A a;
+    };
+
+    PYBIND11_NUMPY_DTYPE(A, x, y);
+    PYBIND11_NUMPY_DTYPE(B, z, a);
+
+    /* now both A and B can be used as template arguments to py::array_t */
+
 Vectorizing functions
 =====================
 
@@ -1433,17 +1457,11 @@ simply using ``vectorize``).
         if (buf1.ndim != 1 || buf2.ndim != 1)
             throw std::runtime_error("Number of dimensions must be one");
 
-        if (buf1.shape[0] != buf2.shape[0])
+        if (buf1.size != buf2.size)
             throw std::runtime_error("Input shapes must match");
 
-        auto result = py::array(py::buffer_info(
-            nullptr,            /* Pointer to data (nullptr -> ask NumPy to allocate!) */
-            sizeof(double),     /* Size of one item */
-            py::format_descriptor<double>::value, /* Buffer format */
-            buf1.ndim,          /* How many dimensions? */
-            { buf1.shape[0] },  /* Number of elements for each dimension */
-            { sizeof(double) }  /* Strides for each dimension */
-        ));
+        /* No pointer is passed, so NumPy will allocate the buffer */
+        auto result = py::array_t<double>(buf1.size);
 
         auto buf3 = result.request();
 
@@ -1830,4 +1848,3 @@ is always ``none``).
 
     // Evaluate the statements in an separate Python file on disk
     py::eval_file("script.py", scope);
-

docs/changelog.rst

@@ -27,6 +27,23 @@ Breaking changes queued for v2.0.0 (Not yet released)
 * ``make_iterator()`` improvements for better compatibility with various types
   (now uses prefix increment operator)
 * ``arg()`` now accepts a wider range of argument types for default values
+* Added support for registering structured dtypes via ``PYBIND11_NUMPY_DTYPE()`` macro.
+* Added ``PYBIND11_STR_TYPE`` macro which maps to the ``builtins.str`` type.
+* Added a simplified ``buffer_info`` constructor for 1-dimensional buffers.
+* Format descriptor strings should now be accessed via ``format_descriptor::format()``
+  (for compatibility purposes, the old syntax ``format_descriptor::value`` will still
+  work for non-structured data types).
+* Added a class wrapping NumPy array descriptors: ``dtype``.
+* Added buffer/NumPy support for ``char[N]`` and ``std::array<char, N>`` types.
+* ``array`` gained new constructors accepting dtype objects.
+* Added constructors for ``array`` and ``array_t`` explicitly accepting shape and
+  strides; if strides are not provided, they are deduced assuming C-contiguity.
+  Also added simplified constructors for 1-dimensional case.
+* Added constructors for ``str`` from ``bytes`` and for ``bytes`` from ``str``.
+  This will do the UTF-8 decoding/encoding as required.
+* Added constructors for ``str`` and ``bytes`` from zero-terminated char pointers,
+  and from char pointers and length.
+* Added ``memoryview`` wrapper type which is constructible from ``buffer_info``.
 * Various minor improvements of library internals (no user-visible changes)
 
 1.8.1 (July 12, 2016)

example/CMakeLists.txt

@@ -26,6 +26,7 @@ set(PYBIND11_EXAMPLES
   example-stl-binder-vector.cpp
   example-eval.cpp
   example-custom-exceptions.cpp
+  example-numpy-dtypes.cpp
   issues.cpp
 )
 
@@ -65,4 +66,3 @@ foreach(VALUE ${PYBIND11_EXAMPLES})
   string(REGEX REPLACE "^(.+).cpp$" "\\1" EXAMPLE_NAME "${VALUE}")
   add_test(NAME ${EXAMPLE_NAME} COMMAND ${RUN_TEST} ${EXAMPLE_NAME})
 endforeach()
-

example/example-buffers.cpp

@@ -81,7 +81,7 @@ void init_ex_buffers(py::module &m) {
         /// Construct from a buffer
         .def("__init__", [](Matrix &v, py::buffer b) {
             py::buffer_info info = b.request();
-            if (info.format != py::format_descriptor<float>::value || info.ndim != 2)
+            if (info.format != py::format_descriptor<float>::format() || info.ndim != 2)
                 throw std::runtime_error("Incompatible buffer format!");
             new (&v) Matrix(info.shape[0], info.shape[1]);
             memcpy(v.data(), info.ptr, sizeof(float) * v.rows() * v.cols());
@@ -106,7 +106,7 @@ void init_ex_buffers(py::module &m) {
             return py::buffer_info(
                 m.data(),                               /* Pointer to buffer */
                 sizeof(float),                          /* Size of one scalar */
-                py::format_descriptor<float>::value, /* Python struct-style format descriptor */
+                py::format_descriptor<float>::format(), /* Python struct-style format descriptor */
                 2,                                      /* Number of dimensions */
                 { m.rows(), m.cols() },                 /* Buffer dimensions */
                 { sizeof(float) * m.rows(),             /* Strides (in bytes) for each index */

example/example-numpy-dtypes.cpp

+/*
+  example/example-numpy-dtypes.cpp -- Structured and compound NumPy dtypes
+
+  Copyright (c) 2016 Ivan Smirnov
+
+  All rights reserved. Use of this source code is governed by a
+  BSD-style license that can be found in the LICENSE file.
+*/
+
+#include "example.h"
+
+#include <pybind11/numpy.h>
+#include <cstdint>
+#include <iostream>
+
+#ifdef __GNUC__
+#define PYBIND11_PACKED(cls) cls __attribute__((__packed__))
+#else
+#define PYBIND11_PACKED(cls) __pragma(pack(push, 1)) cls __pragma(pack(pop))
+#endif
+
+namespace py = pybind11;
+
+struct SimpleStruct {
+    bool x;
+    uint32_t y;
+    float z;
+};
+
+std::ostream& operator<<(std::ostream& os, const SimpleStruct& v) {
+    return os << "s:" << v.x << "," << v.y << "," << v.z;
+}
+
+PYBIND11_PACKED(struct PackedStruct {
+    bool x;
+    uint32_t y;
+    float z;
+});
+
+std::ostream& operator<<(std::ostream& os, const PackedStruct& v) {
+    return os << "p:" << v.x << "," << v.y << "," << v.z;
+}
+
+PYBIND11_PACKED(struct NestedStruct {
+    SimpleStruct a;
+    PackedStruct b;
+});
+
+std::ostream& operator<<(std::ostream& os, const NestedStruct& v) {
+    return os << "n:a=" << v.a << ";b=" << v.b;
+}
+
+struct PartialStruct {
+    bool x;
+    uint32_t y;
+    float z;
+    uint64_t dummy2;
+};
+
+struct PartialNestedStruct {
+    uint64_t dummy1;
+    PartialStruct a;
+    uint64_t dummy2;
+};
+
+struct UnboundStruct { };
+
+struct StringStruct {
+    char a[3];
+    std::array<char, 3> b;
+};
+
+std::ostream& operator<<(std::ostream& os, const StringStruct& v) {
+    os << "a='";
+    for (size_t i = 0; i < 3 && v.a[i]; i++) os << v.a[i];
+    os << "',b='";
+    for (size_t i = 0; i < 3 && v.b[i]; i++) os << v.b[i];
+    return os << "'";
+}
+
+template <typename T>
+py::array mkarray_via_buffer(size_t n) {
+    return py::array(py::buffer_info(nullptr, sizeof(T),
+                                     py::format_descriptor<T>::format(),
+                                     1, { n }, { sizeof(T) }));
+}
+
+template <typename S>
+py::array_t<S, 0> create_recarray(size_t n) {
+    auto arr = mkarray_via_buffer<S>(n);
+    auto req = arr.request();
+    auto ptr = static_cast<S*>(req.ptr);
+    for (size_t i = 0; i < n; i++) {
+        ptr[i].x = i % 2 != 0; ptr[i].y = (uint32_t) i; ptr[i].z = (float) i * 1.5f;
+    }
+    return arr;
+}
+
+std::string get_format_unbound() {
+    return py::format_descriptor<UnboundStruct>::format();
+}
+
+py::array_t<NestedStruct, 0> create_nested(size_t n) {
+    auto arr = mkarray_via_buffer<NestedStruct>(n);
+    auto req = arr.request();
+    auto ptr = static_cast<NestedStruct*>(req.ptr);
+    for (size_t i = 0; i < n; i++) {
+        ptr[i].a.x = i % 2 != 0; ptr[i].a.y = (uint32_t) i; ptr[i].a.z = (float) i * 1.5f;
+        ptr[i].b.x = (i + 1) % 2 != 0; ptr[i].b.y = (uint32_t) (i + 1); ptr[i].b.z = (float) (i + 1) * 1.5f;
+    }
+    return arr;
+}
+
+py::array_t<PartialNestedStruct, 0> create_partial_nested(size_t n) {
+    auto arr = mkarray_via_buffer<PartialNestedStruct>(n);
+    auto req = arr.request();
+    auto ptr = static_cast<PartialNestedStruct*>(req.ptr);
+    for (size_t i = 0; i < n; i++) {
+        ptr[i].a.x = i % 2 != 0; ptr[i].a.y = (uint32_t) i; ptr[i].a.z = (float) i * 1.5f;
+    }
+    return arr;
+}
+
+py::array_t<StringStruct, 0> create_string_array(bool non_empty) {
+    auto arr = mkarray_via_buffer<StringStruct>(non_empty ? 4 : 0);
+    if (non_empty) {
+        auto req = arr.request();
+        auto ptr = static_cast<StringStruct*>(req.ptr);
+        for (size_t i = 0; i < req.size * req.itemsize; i++)
+            static_cast<char*>(req.ptr)[i] = 0;
+        ptr[1].a[0] = 'a'; ptr[1].b[0] = 'a';
+        ptr[2].a[0] = 'a'; ptr[2].b[0] = 'a';
+        ptr[3].a[0] = 'a'; ptr[3].b[0] = 'a';
+
+        ptr[2].a[1] = 'b'; ptr[2].b[1] = 'b';
+        ptr[3].a[1] = 'b'; ptr[3].b[1] = 'b';
+
+        ptr[3].a[2] = 'c'; ptr[3].b[2] = 'c';
+    }
+    return arr;
+}
+
+template <typename S>
+void print_recarray(py::array_t<S, 0> arr) {
+    auto req = arr.request();
+    auto ptr = static_cast<S*>(req.ptr);
+    for (size_t i = 0; i < req.size; i++)
+        std::cout << ptr[i] << std::endl;
+}
+
+void print_format_descriptors() {
+    std::cout << py::format_descriptor<SimpleStruct>::format() << std::endl;
+    std::cout << py::format_descriptor<PackedStruct>::format() << std::endl;
+    std::cout << py::format_descriptor<NestedStruct>::format() << std::endl;
+    std::cout << py::format_descriptor<PartialStruct>::format() << std::endl;
+    std::cout << py::format_descriptor<PartialNestedStruct>::format() << std::endl;
+    std::cout << py::format_descriptor<StringStruct>::format() << std::endl;
+}
+
+void print_dtypes() {
+    std::cout << (std::string) py::dtype::of<SimpleStruct>().str() << std::endl;
+    std::cout << (std::string) py::dtype::of<PackedStruct>().str() << std::endl;
+    std::cout << (std::string) py::dtype::of<NestedStruct>().str() << std::endl;
+    std::cout << (std::string) py::dtype::of<PartialStruct>().str() << std::endl;
+    std::cout << (std::string) py::dtype::of<PartialNestedStruct>().str() << std::endl;
+    std::cout << (std::string) py::dtype::of<StringStruct>().str() << std::endl;
+}
+
+py::array_t<int32_t, 0> test_array_ctors(int i) {
+    using arr_t = py::array_t<int32_t, 0>;
+
+    std::vector<int32_t> data { 1, 2, 3, 4, 5, 6 };
+    std::vector<size_t> shape { 3, 2 };
+    std::vector<size_t> strides { 8, 4 };
+
+    auto ptr = data.data();
+    auto vptr = (void *) ptr;
+    auto dtype = py::dtype("int32");
+
+    py::buffer_info buf_ndim1(vptr, 4, "i", 6);
+    py::buffer_info buf_ndim1_null(nullptr, 4, "i", 6);
+    py::buffer_info buf_ndim2(vptr, 4, "i", 2, shape, strides);
+    py::buffer_info buf_ndim2_null(nullptr, 4, "i", 2, shape, strides);
+
+    auto fill = [](py::array arr) {
+        auto req = arr.request();
+        for (int i = 0; i < 6; i++) ((int32_t *) req.ptr)[i] = i + 1;
+        return arr;
+    };
+
+    switch (i) {
+    // shape: (3, 2)
+    case 10: return arr_t(shape, strides, ptr);
+    case 11: return py::array(shape, strides, ptr);
+    case 12: return py::array(dtype, shape, strides, vptr);
+    case 13: return arr_t(shape, ptr);
+    case 14: return py::array(shape, ptr);
+    case 15: return py::array(dtype, shape, vptr);
+    case 16: return arr_t(buf_ndim2);
+    case 17: return py::array(buf_ndim2);
+    // shape: (3, 2) - post-fill
+    case 20: return fill(arr_t(shape, strides));
+    case 21: return py::array(shape, strides, ptr); // can't have nullptr due to templated ctor
+    case 22: return fill(py::array(dtype, shape, strides));
+    case 23: return fill(arr_t(shape));
+    case 24: return py::array(shape, ptr); // can't have nullptr due to templated ctor
+    case 25: return fill(py::array(dtype, shape));
+    case 26: return fill(arr_t(buf_ndim2_null));
+    case 27: return fill(py::array(buf_ndim2_null));
+    // shape: (6, )
+    case 30: return arr_t(6, ptr);
+    case 31: return py::array(6, ptr);
+    case 32: return py::array(dtype, 6, vptr);
+    case 33: return arr_t(buf_ndim1);
+    case 34: return py::array(buf_ndim1);
+    // shape: (6, )
+    case 40: return fill(arr_t(6));
+    case 41: return py::array(6, ptr);  // can't have nullptr due to templated ctor
+    case 42: return fill(py::array(dtype, 6));
+    case 43: return fill(arr_t(buf_ndim1_null));
+    case 44: return fill(py::array(buf_ndim1_null));
+    }
+    return arr_t();
+}
+
+py::list test_dtype_ctors() {
+    py::list list;
+    list.append(py::dtype("int32"));
+    list.append(py::dtype(std::string("float64")));
+    list.append(py::dtype::from_args(py::str("bool")));
+    py::list names, offsets, formats;
+    py::dict dict;
+    names.append(py::str("a")); names.append(py::str("b")); dict["names"] = names;
+    offsets.append(py::int_(1)); offsets.append(py::int_(10)); dict["offsets"] = offsets;
+    formats.append(py::dtype("int32")); formats.append(py::dtype("float64")); dict["formats"] = formats;
+    dict["itemsize"] = py::int_(20);
+    list.append(py::dtype::from_args(dict));
+    list.append(py::dtype(names, formats, offsets, 20));
+    list.append(py::dtype(py::buffer_info((void *) 0, 1, "I", 1)));
+    list.append(py::dtype(py::buffer_info((void *) 0, 1, "T{i:a:f:b:}", 1)));
+    return list;
+}
+
+py::list test_dtype_methods() {
+    py::list list;
+    auto dt1 = py::dtype::of<int32_t>();
+    auto dt2 = py::dtype::of<SimpleStruct>();
+    list.append(dt1); list.append(dt2);
+    list.append(py::bool_(dt1.has_fields())); list.append(py::bool_(dt2.has_fields()));
+    list.append(py::int_(dt1.itemsize())); list.append(py::int_(dt2.itemsize()));
+    return list;
+}
+
+void init_ex_numpy_dtypes(py::module &m) {
+    PYBIND11_NUMPY_DTYPE(SimpleStruct, x, y, z);
+    PYBIND11_NUMPY_DTYPE(PackedStruct, x, y, z);
+    PYBIND11_NUMPY_DTYPE(NestedStruct, a, b);
+    PYBIND11_NUMPY_DTYPE(PartialStruct, x, y, z);
+    PYBIND11_NUMPY_DTYPE(PartialNestedStruct, a);
+    PYBIND11_NUMPY_DTYPE(StringStruct, a, b);
+
+    m.def("create_rec_simple", &create_recarray<SimpleStruct>);
+    m.def("create_rec_packed", &create_recarray<PackedStruct>);
+    m.def("create_rec_nested", &create_nested);
+    m.def("create_rec_partial", &create_recarray<PartialStruct>);
+    m.def("create_rec_partial_nested", &create_partial_nested);
+    m.def("print_format_descriptors", &print_format_descriptors);
+    m.def("print_rec_simple", &print_recarray<SimpleStruct>);
+    m.def("print_rec_packed", &print_recarray<PackedStruct>);
+    m.def("print_rec_nested", &print_recarray<NestedStruct>);
+    m.def("print_dtypes", &print_dtypes);
+    m.def("get_format_unbound", &get_format_unbound);
+    m.def("create_string_array", &create_string_array);
+    m.def("print_string_array", &print_recarray<StringStruct>);
+    m.def("test_array_ctors", &test_array_ctors);
+    m.def("test_dtype_ctors", &test_dtype_ctors);
+    m.def("test_dtype_methods", &test_dtype_methods);
+}
+
+#undef PYBIND11_PACKED

example/example-numpy-dtypes.py

+#!/usr/bin/env python
+from __future__ import print_function
+
+import numpy as np
+from example import (
+    create_rec_simple, create_rec_packed, create_rec_nested, print_format_descriptors,
+    print_rec_simple, print_rec_packed, print_rec_nested, print_dtypes, get_format_unbound,
+    create_rec_partial, create_rec_partial_nested, create_string_array, print_string_array,
+    test_array_ctors, test_dtype_ctors, test_dtype_methods
+)
+
+
+def check_eq(arr, data, dtype):
+    np.testing.assert_equal(arr, np.array(data, dtype=dtype))
+
+try:
+    get_format_unbound()
+    raise Exception
+except RuntimeError as e:
+    assert 'unsupported buffer format' in str(e)
+
+print_format_descriptors()
+print_dtypes()
+
+simple_dtype = np.dtype({'names': ['x', 'y', 'z'],
+                         'formats': ['?', 'u4', 'f4'],
+                         'offsets': [0, 4, 8]})
+packed_dtype = np.dtype([('x', '?'), ('y', 'u4'), ('z', 'f4')])
+
+elements = [(False, 0, 0.0), (True, 1, 1.5), (False, 2, 3.0)]
+
+for func, dtype in [(create_rec_simple, simple_dtype), (create_rec_packed, packed_dtype)]:
+    arr = func(0)
+    assert arr.dtype == dtype
+    check_eq(arr, [], simple_dtype)
+    check_eq(arr, [], packed_dtype)
+
+    arr = func(3)
+    assert arr.dtype == dtype
+    check_eq(arr, elements, simple_dtype)
+    check_eq(arr, elements, packed_dtype)
+
+    if dtype == simple_dtype:
+        print_rec_simple(arr)
+    else:
+        print_rec_packed(arr)
+
+arr = create_rec_partial(3)
+print(arr.dtype)
+partial_dtype = arr.dtype
+assert '' not in arr.dtype.fields
+assert partial_dtype.itemsize > simple_dtype.itemsize
+check_eq(arr, elements, simple_dtype)
+check_eq(arr, elements, packed_dtype)
+
+arr = create_rec_partial_nested(3)
+print(arr.dtype)
+assert '' not in arr.dtype.fields
+assert '' not in arr.dtype.fields['a'][0].fields
+assert arr.dtype.itemsize > partial_dtype.itemsize
+np.testing.assert_equal(arr['a'], create_rec_partial(3))
+
+nested_dtype = np.dtype([('a', simple_dtype), ('b', packed_dtype)])
+
+arr = create_rec_nested(0)
+assert arr.dtype == nested_dtype
+check_eq(arr, [], nested_dtype)
+
+arr = create_rec_nested(3)
+assert arr.dtype == nested_dtype
+check_eq(arr, [((False, 0, 0.0), (True, 1, 1.5)),
+               ((True, 1, 1.5), (False, 2, 3.0)),
+               ((False, 2, 3.0), (True, 3, 4.5))], nested_dtype)
+print_rec_nested(arr)
+
+assert create_rec_nested.__doc__.strip().endswith('numpy.ndarray[NestedStruct]')
+
+arr = create_string_array(True)
+print(arr.dtype)
+print_string_array(arr)
+dtype = arr.dtype
+assert arr['a'].tolist() == [b'', b'a', b'ab', b'abc']
+assert arr['b'].tolist() == [b'', b'a', b'ab', b'abc']
+arr = create_string_array(False)
+assert dtype == arr.dtype
+
+data = np.arange(1, 7, dtype='int32')
+for i in range(8):
+    np.testing.assert_array_equal(test_array_ctors(10 + i), data.reshape((3, 2)))
+    np.testing.assert_array_equal(test_array_ctors(20 + i), data.reshape((3, 2)))
+for i in range(5):
+    np.testing.assert_array_equal(test_array_ctors(30 + i), data)
+    np.testing.assert_array_equal(test_array_ctors(40 + i), data)
+
+d1 = np.dtype({'names': ['a', 'b'], 'formats': ['int32', 'float64'],
+               'offsets': [1, 10], 'itemsize': 20})
+d2 = np.dtype([('a', 'i4'), ('b', 'f4')])
+assert test_dtype_ctors() == [np.dtype('int32'), np.dtype('float64'),
+                              np.dtype('bool'), d1, d1, np.dtype('uint32'), d2]
+
+assert test_dtype_methods() == [np.dtype('int32'), simple_dtype, False, True,
+                                np.dtype('int32').itemsize, simple_dtype.itemsize]

example/example-numpy-dtypes.ref

+T{=?:x:3x=I:y:=f:z:}
+T{=?:x:=I:y:=f:z:}
+T{=T{=?:x:3x=I:y:=f:z:}:a:=T{=?:x:=I:y:=f:z:}:b:}
+T{=?:x:3x=I:y:=f:z:12x}
+T{8x=T{=?:x:3x=I:y:=f:z:12x}:a:8x}
+T{=3s:a:=3s:b:}
+{'names':['x','y','z'], 'formats':['?','<u4','<f4'], 'offsets':[0,4,8], 'itemsize':12}
+[('x', '?'), ('y', '<u4'), ('z', '<f4')]
+[('a', {'names':['x','y','z'], 'formats':['?','<u4','<f4'], 'offsets':[0,4,8], 'itemsize':12}), ('b', [('x', '?'), ('y', '<u4'), ('z', '<f4')])]
+{'names':['x','y','z'], 'formats':['?','<u4','<f4'], 'offsets':[0,4,8], 'itemsize':24}
+{'names':['a'], 'formats':[{'names':['x','y','z'], 'formats':['?','<u4','<f4'], 'offsets':[0,4,8], 'itemsize':24}], 'offsets':[8], 'itemsize':40}
+[('a', 'S3'), ('b', 'S3')]
+s:0,0,0
+s:1,1,1.5
+s:0,2,3
+p:0,0,0
+p:1,1,1.5
+p:0,2,3
+{'names':['x','y','z'], 'formats':['?','<u4','<f4'], 'offsets':[0,4,8], 'itemsize':24}
+{'names':['a'], 'formats':[{'names':['x','y','z'], 'formats':['?','<u4','<f4'], 'offsets':[0,4,8], 'itemsize':24}], 'offsets':[8], 'itemsize':40}
+n:a=s:0,0,0;b=p:1,1,1.5
+n:a=s:1,1,1.5;b=p:0,2,3
+n:a=s:0,2,3;b=p:1,3,4.5
+[('a', 'S3'), ('b', 'S3')]
+a='',b=''
+a='a',b='a'
+a='ab',b='ab'
+a='abc',b='abc'

example/example-python-types.cpp

@@ -139,6 +139,22 @@ public:
         throw std::runtime_error("This exception was intentionally thrown.");
     }
 
+    py::bytes get_bytes_from_string() {
+        return (py::bytes) std::string("foo");
+    }
+
+    py::bytes get_bytes_from_str() {
+        return (py::bytes) py::str("bar", 3);
+    }
+
+    py::str get_str_from_string() {
+        return (py::str) std::string("baz");
+    }
+
+    py::str get_str_from_bytes() {
+        return (py::str) py::bytes("boo", 3);
+    }
+
     static int value;
     static const int value2;
 };
@@ -167,6 +183,10 @@ void init_ex_python_types(py::module &m) {
         .def("pair_passthrough", &ExamplePythonTypes::pair_passthrough, "Return a pair in reversed order")
         .def("tuple_passthrough", &ExamplePythonTypes::tuple_passthrough, "Return a triple in reversed order")
         .def("throw_exception", &ExamplePythonTypes::throw_exception, "Throw an exception")
+        .def("get_bytes_from_string", &ExamplePythonTypes::get_bytes_from_string, "py::bytes from std::string")
+        .def("get_bytes_from_str", &ExamplePythonTypes::get_bytes_from_str, "py::bytes from py::str")
+        .def("get_str_from_string", &ExamplePythonTypes::get_str_from_string, "py::str from std::string")
+        .def("get_str_from_bytes", &ExamplePythonTypes::get_str_from_bytes, "py::str from py::bytes")
         .def_static("new_instance", &ExamplePythonTypes::new_instance, "Return an instance")
         .def_readwrite_static("value", &ExamplePythonTypes::value, "Static value member")
         .def_readonly_static("value2", &ExamplePythonTypes::value2, "Static value member (readonly)")

example/example-python-types.py

@@ -66,6 +66,11 @@ print("__module__(example.ExamplePythonTypes) = %s" % ExamplePythonTypes.__modul
 print("__name__(example.ExamplePythonTypes.get_set) = %s" % ExamplePythonTypes.get_set.__name__)
 print("__module__(example.ExamplePythonTypes.get_set) = %s" % ExamplePythonTypes.get_set.__module__)
 
+print(instance.get_bytes_from_string().decode())
+print(instance.get_bytes_from_str().decode())
+print(instance.get_str_from_string().encode().decode())
+print(instance.get_str_from_bytes().encode().decode())
+
 from example import ConstructorStats
 
 cstats = ConstructorStats.get(ExamplePythonTypes)

example/example-python-types.ref

@@ -135,6 +135,10 @@ __name__(example.ExamplePythonTypes) = ExamplePythonTypes
 __module__(example.ExamplePythonTypes) = example
 __name__(example.ExamplePythonTypes.get_set) = get_set
 __module__(example.ExamplePythonTypes.get_set) = example
+foo
+bar
+baz
+boo
 Instances not destroyed: 1
 ### ExamplePythonTypes @ 0x1045b80 destroyed
 Instances not destroyed: 0

example/example.cpp

@@ -29,6 +29,7 @@ void init_ex_inheritance(py::module &);
 void init_ex_stl_binder_vector(py::module &);
 void init_ex_eval(py::module &);
 void init_ex_custom_exceptions(py::module &);
+void init_ex_numpy_dtypes(py::module &);
 void init_issues(py::module &);
 
 #if defined(PYBIND11_TEST_EIGEN)
@@ -72,6 +73,7 @@ PYBIND11_PLUGIN(example) {
     init_ex_stl_binder_vector(m);
     init_ex_eval(m);
     init_ex_custom_exceptions(m);
+    init_ex_numpy_dtypes(m);
     init_issues(m);
 
     #if defined(PYBIND11_TEST_EIGEN)

example/run_test.py

@@ -46,12 +46,11 @@ name = sys.argv[1]
 try:
     output_bytes = subprocess.check_output([sys.executable, "-u", name + ".py"],
                                            stderr=subprocess.STDOUT)
-except subprocess.CalledProcessError as e:
-    if e.returncode == 99:
-        print('Test "%s" could not be run.' % name)
-        exit(0)
-    else:
-        raise
+except subprocess.CalledProcessError as exc:
+    print('Test `{}` failed:\n{}\n'.format(name, '-' * 50))
+    print(exc.output.decode())
+    print('-' * 50)
+    sys.exit(1)
 
 output    = sanitize(output_bytes.decode('utf-8'))
 reference = sanitize(open(name + '.ref', 'r').read())

include/pybind11/common.h

@@ -95,6 +95,7 @@
 #define PYBIND11_STRING_NAME "str"
 #define PYBIND11_SLICE_OBJECT PyObject
 #define PYBIND11_FROM_STRING PyUnicode_FromString
+#define PYBIND11_STR_TYPE ::pybind11::str
 #define PYBIND11_OB_TYPE(ht_type) (ht_type).ob_base.ob_base.ob_type
 #define PYBIND11_PLUGIN_IMPL(name) \
     extern "C" PYBIND11_EXPORT PyObject *PyInit_##name()
@@ -113,6 +114,7 @@
 #define PYBIND11_STRING_NAME "unicode"
 #define PYBIND11_SLICE_OBJECT PySliceObject
 #define PYBIND11_FROM_STRING PyString_FromString
+#define PYBIND11_STR_TYPE ::pybind11::bytes
 #define PYBIND11_OB_TYPE(ht_type) (ht_type).ob_type
 #define PYBIND11_PLUGIN_IMPL(name) \
     extern "C" PYBIND11_EXPORT PyObject *init##name()
@@ -204,12 +206,13 @@ struct buffer_info {
     void *ptr;                   // Pointer to the underlying storage
     size_t itemsize;             // Size of individual items in bytes
     size_t size;                 // Total number of entries
-    std::string format;          // For homogeneous buffers, this should be set to format_descriptor<T>::value
+    std::string format;          // For homogeneous buffers, this should be set to format_descriptor<T>::format()
     size_t ndim;                 // Number of dimensions
     std::vector<size_t> shape;   // Shape of the tensor (1 entry per dimension)
     std::vector<size_t> strides; // Number of entries between adjacent entries (for each per dimension)
 
     buffer_info() : ptr(nullptr), view(nullptr) {}
+
     buffer_info(void *ptr, size_t itemsize, const std::string &format, size_t ndim,
                 const std::vector<size_t> &shape, const std::vector<size_t> &strides)
         : ptr(ptr), itemsize(itemsize), size(1), format(format),
@@ -218,6 +221,10 @@ struct buffer_info {
             size *= shape[i];
     }
 
+    buffer_info(void *ptr, size_t itemsize, const std::string &format, size_t size)
+    : buffer_info(ptr, itemsize, format, 1, std::vector<size_t> { size },
+                  std::vector<size_t> { itemsize }) { }
+
     buffer_info(Py_buffer *view)
         : ptr(view->buf), itemsize((size_t) view->itemsize), size(1), format(view->format),
           ndim((size_t) view->ndim), shape((size_t) view->ndim), strides((size_t) view->ndim), view(view) {
@@ -231,6 +238,7 @@ struct buffer_info {
     ~buffer_info() {
         if (view) { PyBuffer_Release(view); delete view; }
     }
+
 private:
     Py_buffer *view = nullptr;
 };
@@ -323,14 +331,23 @@ PYBIND11_RUNTIME_EXCEPTION(reference_cast_error) /// Used internally
 [[noreturn]] PYBIND11_NOINLINE inline void pybind11_fail(const std::string &reason) { throw std::runtime_error(reason); }
 
 /// Format strings for basic number types
-#define PYBIND11_DECL_FMT(t, v) template<> struct format_descriptor<t> { static constexpr const char *value = v; }
+#define PYBIND11_DECL_FMT(t, v) template<> struct format_descriptor<t> \
+    { static constexpr const char* value = v; /* for backwards compatibility */ \
+      static constexpr const char* format() { return value; } }
+
 template <typename T, typename SFINAE = void> struct format_descriptor { };
+
 template <typename T> struct format_descriptor<T, typename std::enable_if<std::is_integral<T>::value>::type> {
     static constexpr const char value[2] =
         { "bBhHiIqQ"[detail::log2(sizeof(T))*2 + (std::is_unsigned<T>::value ? 1 : 0)], '\0' };
+    static constexpr const char* format() { return value; }
 };
+
 template <typename T> constexpr const char format_descriptor<
     T, typename std::enable_if<std::is_integral<T>::value>::type>::value[2];
-PYBIND11_DECL_FMT(float, "f"); PYBIND11_DECL_FMT(double, "d"); PYBIND11_DECL_FMT(bool, "?");
+
+PYBIND11_DECL_FMT(float, "f");
+PYBIND11_DECL_FMT(double, "d");
+PYBIND11_DECL_FMT(bool, "?");
 
 NAMESPACE_END(pybind11)

include/pybind11/descr.h

@@ -181,7 +181,7 @@ PYBIND11_NOINLINE inline descr concat(descr &&d) { return d; }
 template <typename... Args> PYBIND11_NOINLINE descr concat(descr &&d, Args&&... args) { return std::move(d) + _(", ") + concat(std::forward<Args>(args)...); }
 PYBIND11_NOINLINE inline descr type_descr(descr&& d) { return _("{") + std::move(d) + _("}"); }
 
-#define PYBIND11_DESCR descr
+#define PYBIND11_DESCR ::pybind11::detail::descr
 #endif
 
 NAMESPACE_END(detail)

include/pybind11/eigen.h

@@ -133,7 +133,7 @@ struct type_caster<Type, typename std::enable_if<is_eigen_dense<Type>::value &&
                 /* Size of one scalar */
                 sizeof(Scalar),
                 /* Python struct-style format descriptor */
-                format_descriptor<Scalar>::value,
+                format_descriptor<Scalar>::format(),
                 /* Number of dimensions */
                 1,
                 /* Buffer dimensions */
@@ -148,7 +148,7 @@ struct type_caster<Type, typename std::enable_if<is_eigen_dense<Type>::value &&
                 /* Size of one scalar */
                 sizeof(Scalar),
                 /* Python struct-style format descriptor */
-                format_descriptor<Scalar>::value,
+                format_descriptor<Scalar>::format(),
                 /* Number of dimensions */
                 isVector ? 1 : 2,
                 /* Buffer dimensions */
@@ -276,7 +276,7 @@ struct type_caster<Type, typename std::enable_if<is_eigen_sparse<Type>::value>::
             // Size of one scalar
             sizeof(Scalar),
             // Python struct-style format descriptor
-            format_descriptor<Scalar>::value,
+            format_descriptor<Scalar>::format(),
             // Number of dimensions
             1,
             // Buffer dimensions
@@ -291,7 +291,7 @@ struct type_caster<Type, typename std::enable_if<is_eigen_sparse<Type>::value>::
             // Size of one scalar
             sizeof(StorageIndex),
             // Python struct-style format descriptor
-            format_descriptor<StorageIndex>::value,
+            format_descriptor<StorageIndex>::format(),
             // Number of dimensions
             1,
             // Buffer dimensions
@@ -306,7 +306,7 @@ struct type_caster<Type, typename std::enable_if<is_eigen_sparse<Type>::value>::
             // Size of one scalar
             sizeof(StorageIndex),
             // Python struct-style format descriptor
-            format_descriptor<StorageIndex>::value,
+            format_descriptor<StorageIndex>::format(),
             // Number of dimensions
             1,
             // Buffer dimensions

include/pybind11/numpy.h

@@ -13,6 +13,11 @@
 #include "complex.h"
 #include <numeric>
 #include <algorithm>
+#include <array>
+#include <cstdlib>
+#include <cstring>
+#include <sstream>
+#include <initializer_list>
 
 #if defined(_MSC_VER)
 #pragma warning(push)
@@ -20,18 +25,12 @@
 #endif
 
 NAMESPACE_BEGIN(pybind11)
-namespace detail { template <typename type, typename SFINAE = void> struct npy_format_descriptor { }; }
-
-class array : public buffer {
-public:
-    struct API {
-        enum Entries {
-            API_PyArray_Type = 2,
-            API_PyArray_DescrFromType = 45,
-            API_PyArray_FromAny = 69,
-            API_PyArray_NewCopy = 85,
-            API_PyArray_NewFromDescr = 94,
+namespace detail {
+template <typename type, typename SFINAE = void> struct npy_format_descriptor { };
+template <typename type> struct is_pod_struct;
 
+struct npy_api {
+    enum constants {
         NPY_C_CONTIGUOUS_ = 0x0001,
         NPY_F_CONTIGUOUS_ = 0x0002,
         NPY_ARRAY_FORCECAST_ = 0x0010,
@@ -43,100 +42,258 @@ public:
         NPY_LONG_, NPY_ULONG_,
         NPY_LONGLONG_, NPY_ULONGLONG_,
         NPY_FLOAT_, NPY_DOUBLE_, NPY_LONGDOUBLE_,
-            NPY_CFLOAT_, NPY_CDOUBLE_, NPY_CLONGDOUBLE_
+        NPY_CFLOAT_, NPY_CDOUBLE_, NPY_CLONGDOUBLE_,
+        NPY_OBJECT_ = 17,
+        NPY_STRING_, NPY_UNICODE_, NPY_VOID_
     };
 
-        static API lookup() {
-            module m = module::import("numpy.core.multiarray");
-            object c = (object) m.attr("_ARRAY_API");
-#if PY_MAJOR_VERSION >= 3
-            void **api_ptr = (void **) (c ? PyCapsule_GetPointer(c.ptr(), NULL) : nullptr);
-#else
-            void **api_ptr = (void **) (c ? PyCObject_AsVoidPtr(c.ptr()) : nullptr);
-#endif
-            API api;
-            api.PyArray_Type_          = (decltype(api.PyArray_Type_))          api_ptr[API_PyArray_Type];
-            api.PyArray_DescrFromType_ = (decltype(api.PyArray_DescrFromType_)) api_ptr[API_PyArray_DescrFromType];
-            api.PyArray_FromAny_       = (decltype(api.PyArray_FromAny_))       api_ptr[API_PyArray_FromAny];
-            api.PyArray_NewCopy_       = (decltype(api.PyArray_NewCopy_))       api_ptr[API_PyArray_NewCopy];
-            api.PyArray_NewFromDescr_  = (decltype(api.PyArray_NewFromDescr_))  api_ptr[API_PyArray_NewFromDescr];
+    static npy_api& get() {
+        static npy_api api = lookup();
         return api;
     }
 
-        bool PyArray_Check_(PyObject *obj) const { return (bool) PyObject_TypeCheck(obj, PyArray_Type_); }
+    bool PyArray_Check_(PyObject *obj) const {
+        return (bool) PyObject_TypeCheck(obj, PyArray_Type_);
+    }
+    bool PyArrayDescr_Check_(PyObject *obj) const {
+        return (bool) PyObject_TypeCheck(obj, PyArrayDescr_Type_);
+    }
 
     PyObject *(*PyArray_DescrFromType_)(int);
     PyObject *(*PyArray_NewFromDescr_)
         (PyTypeObject *, PyObject *, int, Py_intptr_t *,
          Py_intptr_t *, void *, int, PyObject *);
+    PyObject *(*PyArray_DescrNewFromType_)(int);
     PyObject *(*PyArray_NewCopy_)(PyObject *, int);
     PyTypeObject *PyArray_Type_;
+    PyTypeObject *PyArrayDescr_Type_;
     PyObject *(*PyArray_FromAny_) (PyObject *, PyObject *, int, int, int, PyObject *);
+    int (*PyArray_DescrConverter_) (PyObject *, PyObject **);
+    bool (*PyArray_EquivTypes_) (PyObject *, PyObject *);
+    int (*PyArray_GetArrayParamsFromObject_)(PyObject *, PyObject *, char, PyObject **, int *,
+                                             Py_ssize_t *, PyObject **, PyObject *);
+private:
+    enum functions {
+        API_PyArray_Type = 2,
+        API_PyArrayDescr_Type = 3,
+        API_PyArray_DescrFromType = 45,
+        API_PyArray_FromAny = 69,
+        API_PyArray_NewCopy = 85,
+        API_PyArray_NewFromDescr = 94,
+        API_PyArray_DescrNewFromType = 9,
+        API_PyArray_DescrConverter = 174,
+        API_PyArray_EquivTypes = 182,
+        API_PyArray_GetArrayParamsFromObject = 278,
     };
 
-    PYBIND11_OBJECT_DEFAULT(array, buffer, lookup_api().PyArray_Check_)
+    static npy_api lookup() {
+        module m = module::import("numpy.core.multiarray");
+        object c = (object) m.attr("_ARRAY_API");
+#if PY_MAJOR_VERSION >= 3
+        void **api_ptr = (void **) (c ? PyCapsule_GetPointer(c.ptr(), NULL) : nullptr);
+#else
+        void **api_ptr = (void **) (c ? PyCObject_AsVoidPtr(c.ptr()) : nullptr);
+#endif
+        npy_api api;
+#define DECL_NPY_API(Func) api.Func##_ = (decltype(api.Func##_)) api_ptr[API_##Func];
+        DECL_NPY_API(PyArray_Type);
+        DECL_NPY_API(PyArrayDescr_Type);
+        DECL_NPY_API(PyArray_DescrFromType);
+        DECL_NPY_API(PyArray_FromAny);
+        DECL_NPY_API(PyArray_NewCopy);
+        DECL_NPY_API(PyArray_NewFromDescr);
+        DECL_NPY_API(PyArray_DescrNewFromType);
+        DECL_NPY_API(PyArray_DescrConverter);
+        DECL_NPY_API(PyArray_EquivTypes);
+        DECL_NPY_API(PyArray_GetArrayParamsFromObject);
+#undef DECL_NPY_API
+        return api;
+    }
+};
+}
+
+class dtype : public object {
+public:
+    PYBIND11_OBJECT_DEFAULT(dtype, object, detail::npy_api::get().PyArrayDescr_Check_);
+
+    dtype(const buffer_info &info) {
+        dtype descr(_dtype_from_pep3118()(PYBIND11_STR_TYPE(info.format)));
+        m_ptr = descr.strip_padding().release().ptr();
+    }
+
+    dtype(std::string format) {
+        m_ptr = from_args(pybind11::str(format)).release().ptr();
+    }
+
+    dtype(const char *format) : dtype(std::string(format)) { }
+
+    dtype(list names, list formats, list offsets, size_t itemsize) {
+        dict args;
+        args["names"] = names;
+        args["formats"] = formats;
+        args["offsets"] = offsets;
+        args["itemsize"] = int_(itemsize);
+        m_ptr = from_args(args).release().ptr();
+    }
+
+    static dtype from_args(object args) {
+        // This is essentially the same as calling np.dtype() constructor in Python
+        PyObject *ptr = nullptr;
+        if (!detail::npy_api::get().PyArray_DescrConverter_(args.release().ptr(), &ptr) || !ptr)
+            pybind11_fail("NumPy: failed to create structured dtype");
+        return object(ptr, false);
+    }
+
+    template <typename T> static dtype of() {
+        return detail::npy_format_descriptor<T>::dtype();
+    }
+
+    size_t itemsize() const {
+        return (size_t) attr("itemsize").cast<int_>();
+    }
+
+    bool has_fields() const {
+        return attr("fields").cast<object>().ptr() != Py_None;
+    }
+
+    std::string kind() const {
+        return (std::string) attr("kind").cast<pybind11::str>();
+    }
+
+private:
+    static object _dtype_from_pep3118() {
+        static PyObject *obj = module::import("numpy.core._internal")
+            .attr("_dtype_from_pep3118").cast<object>().release().ptr();
+        return object(obj, true);
+    }
+
+    dtype strip_padding() {
+        // Recursively strip all void fields with empty names that are generated for
+        // padding fields (as of NumPy v1.11).
+        auto fields = attr("fields").cast<object>();
+        if (fields.ptr() == Py_None)
+            return *this;
+
+        struct field_descr { PYBIND11_STR_TYPE name; object format; int_ offset; };
+        std::vector<field_descr> field_descriptors;
+
+        auto items = fields.attr("items").cast<object>();
+        for (auto field : items()) {
+            auto spec = object(field, true).cast<tuple>();
+            auto name = spec[0].cast<pybind11::str>();
+            auto format = spec[1].cast<tuple>()[0].cast<dtype>();
+            auto offset = spec[1].cast<tuple>()[1].cast<int_>();
+            if (!len(name) && format.kind() == "V")
+                continue;
+            field_descriptors.push_back({(PYBIND11_STR_TYPE) name, format.strip_padding(), offset});
+        }
+
+        std::sort(field_descriptors.begin(), field_descriptors.end(),
+                  [](const field_descr& a, const field_descr& b) {
+                      return (int) a.offset < (int) b.offset;
+                  });
+
+        list names, formats, offsets;
+        for (auto& descr : field_descriptors) {
+            names.append(descr.name);
+            formats.append(descr.format);
+            offsets.append(descr.offset);
+        }
+        return dtype(names, formats, offsets, itemsize());
+    }
+};
+
+class array : public buffer {
+public:
+    PYBIND11_OBJECT_DEFAULT(array, buffer, detail::npy_api::get().PyArray_Check_)
 
     enum {
-        c_style = API::NPY_C_CONTIGUOUS_,
-        f_style = API::NPY_F_CONTIGUOUS_,
-        forcecast = API::NPY_ARRAY_FORCECAST_
+        c_style = detail::npy_api::NPY_C_CONTIGUOUS_,
+        f_style = detail::npy_api::NPY_F_CONTIGUOUS_,
+        forcecast = detail::npy_api::NPY_ARRAY_FORCECAST_
     };
 
-    template <typename Type> array(size_t size, const Type *ptr) {
-        API& api = lookup_api();
-        PyObject *descr = api.PyArray_DescrFromType_(detail::npy_format_descriptor<Type>::value);
-        if (descr == nullptr)
-            pybind11_fail("NumPy: unsupported buffer format!");
-        Py_intptr_t shape = (Py_intptr_t) size;
-        object tmp = object(api.PyArray_NewFromDescr_(
-            api.PyArray_Type_, descr, 1, &shape, nullptr, (void *) ptr, 0, nullptr), false);
-        if (ptr && tmp)
-            tmp = object(api.PyArray_NewCopy_(tmp.ptr(), -1 /* any order */), false);
+    array(const pybind11::dtype& dt, const std::vector<size_t>& shape,
+          const std::vector<size_t>& strides, void *ptr = nullptr) {
+        auto& api = detail::npy_api::get();
+        auto ndim = shape.size();
+        if (shape.size() != strides.size())
+            pybind11_fail("NumPy: shape ndim doesn't match strides ndim");
+        auto descr = dt;
+        object tmp(api.PyArray_NewFromDescr_(
+            api.PyArray_Type_, descr.release().ptr(), (int) ndim, (Py_intptr_t *) shape.data(),
+            (Py_intptr_t *) strides.data(), ptr, 0, nullptr), false);
         if (!tmp)
             pybind11_fail("NumPy: unable to create array!");
+        if (ptr)
+            tmp = object(api.PyArray_NewCopy_(tmp.ptr(), -1 /* any order */), false);
         m_ptr = tmp.release().ptr();
     }
 
-    array(const buffer_info &info) {
-        API& api = lookup_api();
-        if ((info.format.size() < 1) || (info.format.size() > 2))
-            pybind11_fail("Unsupported buffer format!");
-        int fmt = (int) info.format[0];
-        if (info.format == "Zd")      fmt = API::NPY_CDOUBLE_;
-        else if (info.format == "Zf") fmt = API::NPY_CFLOAT_;
+    array(const pybind11::dtype& dt, const std::vector<size_t>& shape, void *ptr = nullptr)
+    : array(dt, shape, default_strides(shape, dt.itemsize()), ptr) { }
 
-        PyObject *descr = api.PyArray_DescrFromType_(fmt);
-        if (descr == nullptr)
-            pybind11_fail("NumPy: unsupported buffer format '" + info.format + "'!");
-        object tmp(api.PyArray_NewFromDescr_(
-            api.PyArray_Type_, descr, (int) info.ndim, (Py_intptr_t *) &info.shape[0],
-            (Py_intptr_t *) &info.strides[0], info.ptr, 0, nullptr), false);
-        if (info.ptr && tmp)
-            tmp = object(api.PyArray_NewCopy_(tmp.ptr(), -1 /* any order */), false);
-        if (!tmp)
-            pybind11_fail("NumPy: unable to create array!");
-        m_ptr = tmp.release().ptr();
+    array(const pybind11::dtype& dt, size_t size, void *ptr = nullptr)
+    : array(dt, std::vector<size_t> { size }, ptr) { }
+
+    template<typename T> array(const std::vector<size_t>& shape,
+                               const std::vector<size_t>& strides, T* ptr)
+    : array(pybind11::dtype::of<T>(), shape, strides, (void *) ptr) { }
+
+    template<typename T> array(const std::vector<size_t>& shape, T* ptr)
+    : array(shape, default_strides(shape, sizeof(T)), ptr) { }
+
+    template<typename T> array(size_t size, T* ptr)
+    : array(std::vector<size_t> { size }, ptr) { }
+
+    array(const buffer_info &info)
+    : array(pybind11::dtype(info), info.shape, info.strides, info.ptr) { }
+
+    pybind11::dtype dtype() {
+        return attr("dtype").cast<pybind11::dtype>();
     }
 
 protected:
-    static API &lookup_api() {
-        static API api = API::lookup();
-        return api;
+    template <typename T, typename SFINAE> friend struct detail::npy_format_descriptor;
+
+    static std::vector<size_t> default_strides(const std::vector<size_t>& shape, size_t itemsize) {
+        auto ndim = shape.size();
+        std::vector<size_t> strides(ndim);
+        if (ndim) {
+            std::fill(strides.begin(), strides.end(), itemsize);
+            for (size_t i = 0; i < ndim - 1; i++)
+                for (size_t j = 0; j < ndim - 1 - i; j++)
+                    strides[j] *= shape[ndim - 1 - i];
+        }
+        return strides;
     }
 };
 
 template <typename T, int ExtraFlags = array::forcecast> class array_t : public array {
 public:
     PYBIND11_OBJECT_CVT(array_t, array, is_non_null, m_ptr = ensure(m_ptr));
+
     array_t() : array() { }
-    array_t(const buffer_info& info) : array(info) {}
+
+    array_t(const buffer_info& info) : array(info) { }
+
+    array_t(const std::vector<size_t>& shape, const std::vector<size_t>& strides, T* ptr = nullptr)
+    : array(shape, strides, ptr) { }
+
+    array_t(const std::vector<size_t>& shape, T* ptr = nullptr)
+    : array(shape, ptr) { }
+
+    array_t(size_t size, T* ptr = nullptr)
+    : array(size, ptr) { }
+
     static bool is_non_null(PyObject *ptr) { return ptr != nullptr; }
+
     static PyObject *ensure(PyObject *ptr) {
         if (ptr == nullptr)
             return nullptr;
-        API &api = lookup_api();
-        PyObject *descr = api.PyArray_DescrFromType_(detail::npy_format_descriptor<T>::value);
-        PyObject *result = api.PyArray_FromAny_(ptr, descr, 0, 0, API::NPY_ENSURE_ARRAY_ | ExtraFlags, nullptr);
+        auto& api = detail::npy_api::get();
+        PyObject *result = api.PyArray_FromAny_(ptr, pybind11::dtype::of<T>().release().ptr(), 0, 0,
+                                                detail::npy_api::NPY_ENSURE_ARRAY_ | ExtraFlags, nullptr);
         if (!result)
             PyErr_Clear();
         Py_DECREF(ptr);
@@ -144,15 +301,47 @@ public:
     }
 };
 
+template <typename T>
+struct format_descriptor<T, typename std::enable_if<detail::is_pod_struct<T>::value>::type> {
+    static const char *format() { return detail::npy_format_descriptor<T>::format(); }
+};
+
+template <size_t N> struct format_descriptor<char[N]> {
+    static const char *format() { PYBIND11_DESCR s = detail::_<N>() + detail::_("s"); return s.text(); }
+};
+template <size_t N> struct format_descriptor<std::array<char, N>> {
+    static const char *format() { PYBIND11_DESCR s = detail::_<N>() + detail::_("s"); return s.text(); }
+};
+
 NAMESPACE_BEGIN(detail)
+template <typename T> struct is_std_array : std::false_type { };
+template <typename T, size_t N> struct is_std_array<std::array<T, N>> : std::true_type { };
+
+template <typename T>
+struct is_pod_struct {
+    enum { value = std::is_pod<T>::value && // offsetof only works correctly for POD types
+           !std::is_array<T>::value &&
+           !is_std_array<T>::value &&
+           !std::is_integral<T>::value &&
+           !std::is_same<T, float>::value &&
+           !std::is_same<T, double>::value &&
+           !std::is_same<T, bool>::value &&
+           !std::is_same<T, std::complex<float>>::value &&
+           !std::is_same<T, std::complex<double>>::value };
+};
 
 template <typename T> struct npy_format_descriptor<T, typename std::enable_if<std::is_integral<T>::value>::type> {
 private:
     constexpr static const int values[8] = {
-        array::API::NPY_BYTE_, array::API::NPY_UBYTE_, array::API::NPY_SHORT_,    array::API::NPY_USHORT_,
-        array::API::NPY_INT_,  array::API::NPY_UINT_,  array::API::NPY_LONGLONG_, array::API::NPY_ULONGLONG_ };
+        npy_api::NPY_BYTE_, npy_api::NPY_UBYTE_, npy_api::NPY_SHORT_,    npy_api::NPY_USHORT_,
+        npy_api::NPY_INT_,  npy_api::NPY_UINT_,  npy_api::NPY_LONGLONG_, npy_api::NPY_ULONGLONG_ };
 public:
     enum { value = values[detail::log2(sizeof(T)) * 2 + (std::is_unsigned<T>::value ? 1 : 0)] };
+    static pybind11::dtype dtype() {
+        if (auto ptr = npy_api::get().PyArray_DescrFromType_(value))
+            return object(ptr, true);
+        pybind11_fail("Unsupported buffer format!");
+    }
     template <typename T2 = T, typename std::enable_if<std::is_signed<T2>::value, int>::type = 0>
     static PYBIND11_DESCR name() { return _("int") + _<sizeof(T)*8>(); }
     template <typename T2 = T, typename std::enable_if<!std::is_signed<T2>::value, int>::type = 0>
@@ -162,12 +351,149 @@ template <typename T> constexpr const int npy_format_descriptor<
     T, typename std::enable_if<std::is_integral<T>::value>::type>::values[8];
 
 #define DECL_FMT(Type, NumPyName, Name) template<> struct npy_format_descriptor<Type> { \
-    enum { value = array::API::NumPyName }; \
+    enum { value = npy_api::NumPyName }; \
+    static pybind11::dtype dtype() { \
+        if (auto ptr = npy_api::get().PyArray_DescrFromType_(value)) \
+            return object(ptr, true); \
+        pybind11_fail("Unsupported buffer format!"); \
+    } \
     static PYBIND11_DESCR name() { return _(Name); } }
-DECL_FMT(float, NPY_FLOAT_, "float32"); DECL_FMT(double, NPY_DOUBLE_, "float64"); DECL_FMT(bool, NPY_BOOL_, "bool");
-DECL_FMT(std::complex<float>, NPY_CFLOAT_, "complex64"); DECL_FMT(std::complex<double>, NPY_CDOUBLE_, "complex128");
+DECL_FMT(float, NPY_FLOAT_, "float32");
+DECL_FMT(double, NPY_DOUBLE_, "float64");
+DECL_FMT(bool, NPY_BOOL_, "bool");
+DECL_FMT(std::complex<float>, NPY_CFLOAT_, "complex64");
+DECL_FMT(std::complex<double>, NPY_CDOUBLE_, "complex128");
 #undef DECL_FMT
 
+#define DECL_CHAR_FMT \
+    static PYBIND11_DESCR name() { return _("S") + _<N>(); } \
+    static pybind11::dtype dtype() { \
+        PYBIND11_DESCR fmt = _("S") + _<N>(); \
+        return pybind11::dtype(fmt.text());  \
+    } \
+    static const char *format() { PYBIND11_DESCR s = _<N>() + _("s"); return s.text(); }
+template <size_t N> struct npy_format_descriptor<char[N]> { DECL_CHAR_FMT };
+template <size_t N> struct npy_format_descriptor<std::array<char, N>> { DECL_CHAR_FMT };
+#undef DECL_CHAR_FMT
+
+struct field_descriptor {
+    const char *name;
+    size_t offset;
+    size_t size;
+    const char *format;
+    dtype descr;
+};
+
+template <typename T>
+struct npy_format_descriptor<T, typename std::enable_if<is_pod_struct<T>::value>::type> {
+    static PYBIND11_DESCR name() { return _("struct"); }
+
+    static pybind11::dtype dtype() {
+        if (!dtype_())
+            pybind11_fail("NumPy: unsupported buffer format!");
+        return object(dtype_(), true);
+    }
+
+    static const char* format() {
+        if (!dtype_())
+            pybind11_fail("NumPy: unsupported buffer format!");
+        return format_().c_str();
+    }
+
+    static void register_dtype(std::initializer_list<field_descriptor> fields) {
+        list names, formats, offsets;
+        for (auto field : fields) {
+            if (!field.descr)
+                pybind11_fail("NumPy: unsupported field dtype");
+            names.append(PYBIND11_STR_TYPE(field.name));
+            formats.append(field.descr);
+            offsets.append(int_(field.offset));
+        }
+        dtype_() = pybind11::dtype(names, formats, offsets, sizeof(T)).release().ptr();
+
+        // There is an existing bug in NumPy (as of v1.11): trailing bytes are
+        // not encoded explicitly into the format string. This will supposedly
+        // get fixed in v1.12; for further details, see these:
+        // - https://github.com/numpy/numpy/issues/7797
+        // - https://github.com/numpy/numpy/pull/7798
+        // Because of this, we won't use numpy's logic to generate buffer format
+        // strings and will just do it ourselves.
+        std::vector<field_descriptor> ordered_fields(fields);
+        std::sort(ordered_fields.begin(), ordered_fields.end(),
+                  [](const field_descriptor& a, const field_descriptor &b) {
+                      return a.offset < b.offset;
+                  });
+        size_t offset = 0;
+        std::ostringstream oss;
+        oss << "T{";
+        for (auto& field : ordered_fields) {
+            if (field.offset > offset)
+                oss << (field.offset - offset) << 'x';
+            // note that '=' is required to cover the case of unaligned fields
+            oss << '=' << field.format << ':' << field.name << ':';
+            offset = field.offset + field.size;
+        }
+        if (sizeof(T) > offset)
+            oss << (sizeof(T) - offset) << 'x';
+        oss << '}';
+        format_() = oss.str();
+
+        // Sanity check: verify that NumPy properly parses our buffer format string
+        auto& api = npy_api::get();
+        auto arr =  array(buffer_info(nullptr, sizeof(T), format(), 1, { 0 }, { sizeof(T) }));
+        if (!api.PyArray_EquivTypes_(dtype_(), arr.dtype().ptr()))
+            pybind11_fail("NumPy: invalid buffer descriptor!");
+    }
+
+private:
+    static inline PyObject*& dtype_() { static PyObject *ptr = nullptr; return ptr; }
+    static inline std::string& format_() { static std::string s; return s; }
+};
+
+// Extract name, offset and format descriptor for a struct field
+#define PYBIND11_FIELD_DESCRIPTOR(Type, Field) \
+    ::pybind11::detail::field_descriptor { \
+        #Field, offsetof(Type, Field), sizeof(decltype(static_cast<Type*>(0)->Field)), \
+        ::pybind11::format_descriptor<decltype(static_cast<Type*>(0)->Field)>::format(), \
+        ::pybind11::detail::npy_format_descriptor<decltype(static_cast<Type*>(0)->Field)>::dtype() \
+    }
+
+// The main idea of this macro is borrowed from https://github.com/swansontec/map-macro
+// (C) William Swanson, Paul Fultz
+#define PYBIND11_EVAL0(...) __VA_ARGS__
+#define PYBIND11_EVAL1(...) PYBIND11_EVAL0 (PYBIND11_EVAL0 (PYBIND11_EVAL0 (__VA_ARGS__)))
+#define PYBIND11_EVAL2(...) PYBIND11_EVAL1 (PYBIND11_EVAL1 (PYBIND11_EVAL1 (__VA_ARGS__)))
+#define PYBIND11_EVAL3(...) PYBIND11_EVAL2 (PYBIND11_EVAL2 (PYBIND11_EVAL2 (__VA_ARGS__)))
+#define PYBIND11_EVAL4(...) PYBIND11_EVAL3 (PYBIND11_EVAL3 (PYBIND11_EVAL3 (__VA_ARGS__)))
+#define PYBIND11_EVAL(...)  PYBIND11_EVAL4 (PYBIND11_EVAL4 (PYBIND11_EVAL4 (__VA_ARGS__)))
+#define PYBIND11_MAP_END(...)
+#define PYBIND11_MAP_OUT
+#define PYBIND11_MAP_COMMA ,
+#define PYBIND11_MAP_GET_END() 0, PYBIND11_MAP_END
+#define PYBIND11_MAP_NEXT0(test, next, ...) next PYBIND11_MAP_OUT
+#define PYBIND11_MAP_NEXT1(test, next) PYBIND11_MAP_NEXT0 (test, next, 0)
+#define PYBIND11_MAP_NEXT(test, next)  PYBIND11_MAP_NEXT1 (PYBIND11_MAP_GET_END test, next)
+#ifdef _MSC_VER // MSVC is not as eager to expand macros, hence this workaround
+#define PYBIND11_MAP_LIST_NEXT1(test, next) \
+    PYBIND11_EVAL0 (PYBIND11_MAP_NEXT0 (test, PYBIND11_MAP_COMMA next, 0))
+#else
+#define PYBIND11_MAP_LIST_NEXT1(test, next) \
+    PYBIND11_MAP_NEXT0 (test, PYBIND11_MAP_COMMA next, 0)
+#endif
+#define PYBIND11_MAP_LIST_NEXT(test, next) \
+    PYBIND11_MAP_LIST_NEXT1 (PYBIND11_MAP_GET_END test, next)
+#define PYBIND11_MAP_LIST0(f, t, x, peek, ...) \
+    f(t, x) PYBIND11_MAP_LIST_NEXT (peek, PYBIND11_MAP_LIST1) (f, t, peek, __VA_ARGS__)
+#define PYBIND11_MAP_LIST1(f, t, x, peek, ...) \
+    f(t, x) PYBIND11_MAP_LIST_NEXT (peek, PYBIND11_MAP_LIST0) (f, t, peek, __VA_ARGS__)
+// PYBIND11_MAP_LIST(f, t, a1, a2, ...) expands to f(t, a1), f(t, a2), ...
+#define PYBIND11_MAP_LIST(f, t, ...) \
+    PYBIND11_EVAL (PYBIND11_MAP_LIST1 (f, t, __VA_ARGS__, (), 0))
+
+#define PYBIND11_NUMPY_DTYPE(Type, ...) \
+    ::pybind11::detail::npy_format_descriptor<Type>::register_dtype \
+        ({PYBIND11_MAP_LIST (PYBIND11_FIELD_DESCRIPTOR, Type, __VA_ARGS__)})
+
 template  <class T>
 using array_iterator = typename std::add_pointer<T>::type;
 
@@ -348,7 +674,7 @@ struct vectorize_helper {
             return cast(f(*((Args *) buffers[Index].ptr)...));
 
         array result(buffer_info(nullptr, sizeof(Return),
-            format_descriptor<Return>::value,
+            format_descriptor<Return>::format(),
             ndim, shape, strides));
 
         buffer_info buf = result.request();

include/pybind11/pytypes.h

@@ -339,12 +339,14 @@ inline iterator handle::begin() const { return iterator(PyObject_GetIter(ptr()),
 inline iterator handle::end() const { return iterator(nullptr, false); }
 inline detail::args_proxy handle::operator*() const { return detail::args_proxy(*this); }
 
+class bytes;
+
 class str : public object {
 public:
     PYBIND11_OBJECT_DEFAULT(str, object, detail::PyUnicode_Check_Permissive)
 
-    str(const std::string &s)
-        : object(PyUnicode_FromStringAndSize(s.c_str(), (ssize_t) s.length()), false) {
+    str(const char *c, size_t n)
+    : object(PyUnicode_FromStringAndSize(c, (ssize_t) n), false) {
         if (!m_ptr) pybind11_fail("Could not allocate string object!");
     }
 
@@ -353,6 +355,10 @@ public:
         if (!m_ptr) pybind11_fail("Could not allocate string object!");
     }
 
+    str(const std::string &s) : str(s.data(), s.size()) { }
+
+    str(const bytes &b);
+
     operator std::string() const {
         object temp = *this;
         if (PyUnicode_Check(m_ptr)) {
@@ -362,8 +368,7 @@ public:
         }
         char *buffer;
         ssize_t length;
-        int err = PYBIND11_BYTES_AS_STRING_AND_SIZE(temp.ptr(), &buffer, &length);
-        if (err == -1)
+        if (PYBIND11_BYTES_AS_STRING_AND_SIZE(temp.ptr(), &buffer, &length))
             pybind11_fail("Unable to extract string contents! (invalid type)");
         return std::string(buffer, (size_t) length);
     }
@@ -382,21 +387,57 @@ class bytes : public object {
 public:
     PYBIND11_OBJECT_DEFAULT(bytes, object, PYBIND11_BYTES_CHECK)
 
-    bytes(const std::string &s)
-        : object(PYBIND11_BYTES_FROM_STRING_AND_SIZE(s.data(), (ssize_t) s.size()), false) {
+    bytes(const char *c)
+    : object(PYBIND11_BYTES_FROM_STRING(c), false) {
+        if (!m_ptr) pybind11_fail("Could not allocate bytes object!");
+    }
+
+    bytes(const char *c, size_t n)
+    : object(PYBIND11_BYTES_FROM_STRING_AND_SIZE(c, (ssize_t) n), false) {
         if (!m_ptr) pybind11_fail("Could not allocate bytes object!");
     }
 
+    bytes(const std::string &s) : bytes(s.data(), s.size()) { }
+
+    bytes(const pybind11::str &s);
+
     operator std::string() const {
         char *buffer;
         ssize_t length;
-        int err = PYBIND11_BYTES_AS_STRING_AND_SIZE(m_ptr, &buffer, &length);
-        if (err == -1)
+        if (PYBIND11_BYTES_AS_STRING_AND_SIZE(m_ptr, &buffer, &length))
             pybind11_fail("Unable to extract bytes contents!");
         return std::string(buffer, (size_t) length);
     }
 };
 
+inline bytes::bytes(const pybind11::str &s) {
+    object temp = s;
+    if (PyUnicode_Check(s.ptr())) {
+        temp = object(PyUnicode_AsUTF8String(s.ptr()), false);
+        if (!temp)
+            pybind11_fail("Unable to extract string contents! (encoding issue)");
+    }
+    char *buffer;
+    ssize_t length;
+    if (PYBIND11_BYTES_AS_STRING_AND_SIZE(temp.ptr(), &buffer, &length))
+        pybind11_fail("Unable to extract string contents! (invalid type)");
+    auto obj = object(PYBIND11_BYTES_FROM_STRING_AND_SIZE(buffer, length), false);
+    if (!obj)
+        pybind11_fail("Could not allocate bytes object!");
+    m_ptr = obj.release().ptr();
+}
+
+inline str::str(const bytes& b) {
+    char *buffer;
+    ssize_t length;
+    if (PYBIND11_BYTES_AS_STRING_AND_SIZE(b.ptr(), &buffer, &length))
+        pybind11_fail("Unable to extract bytes contents!");
+    auto obj = object(PyUnicode_FromStringAndSize(buffer, (ssize_t) length), false);
+    if (!obj)
+        pybind11_fail("Could not allocate string object!");
+    m_ptr = obj.release().ptr();
+}
+
 class none : public object {
 public:
     PYBIND11_OBJECT(none, object, detail::PyNone_Check)
@@ -570,6 +611,38 @@ public:
     }
 };
 
+class memoryview : public object {
+public:
+    memoryview(const buffer_info& info) {
+        static Py_buffer buf { };
+        // Py_buffer uses signed sizes, strides and shape!..
+        static std::vector<Py_ssize_t> py_strides { };
+        static std::vector<Py_ssize_t> py_shape { };
+        buf.buf = info.ptr;
+        buf.itemsize = (Py_ssize_t) info.itemsize;
+        buf.format = const_cast<char *>(info.format.c_str());
+        buf.ndim = (int) info.ndim;
+        buf.len = (Py_ssize_t) info.size;
+        py_strides.clear();
+        py_shape.clear();
+        for (size_t i = 0; i < info.ndim; ++i) {
+            py_strides.push_back((Py_ssize_t) info.strides[i]);
+            py_shape.push_back((Py_ssize_t) info.shape[i]);
+        }
+        buf.strides = py_strides.data();
+        buf.shape = py_shape.data();
+        buf.suboffsets = nullptr;
+        buf.readonly = false;
+        buf.internal = nullptr;
+
+        m_ptr = PyMemoryView_FromBuffer(&buf);
+        if (!m_ptr)
+            pybind11_fail("Unable to create memoryview from buffer descriptor");
+    }
+
+    PYBIND11_OBJECT_DEFAULT(memoryview, object, PyMemoryView_Check)
+};
+
 inline size_t len(handle h) {
     ssize_t result = PyObject_Length(h.ptr());
     if (result < 0)