tests: cleanup and ci hardening (#2397)

* tests: refactor and cleanup

* refactor: more consistent

* tests: vendor six

* tests: more xfails, nicer system

* tests: simplify to info

* tests: suggestions from @YannickJadoul and @bstaletic

* tests: restore some pypy tests that now pass

* tests: rename info to env

* tests: strict False/True

* tests: drop explicit strict=True again

* tests: reduce minimum PyTest to 3.1

tests/test_numpy_dtypes.py

 # -*- coding: utf-8 -*-
 import re
+
 import pytest
-from pybind11_tests import numpy_dtypes as m
 
-pytestmark = pytest.requires_numpy
+import env  # noqa: F401
+
+from pybind11_tests import numpy_dtypes as m
 
-with pytest.suppress(ImportError):
-    import numpy as np
+np = pytest.importorskip("numpy")
 
 
 @pytest.fixture(scope='module')
@@ -294,7 +295,7 @@ def test_register_dtype():
     assert 'dtype is already registered' in str(excinfo.value)
 
 
-@pytest.unsupported_on_pypy
+@pytest.mark.xfail("env.PYPY")
 def test_str_leak():
     from sys import getrefcount
     fmt = "f4"

tests/test_numpy_vectorize.py

@@ -2,10 +2,7 @@
 import pytest
 from pybind11_tests import numpy_vectorize as m
 
-pytestmark = pytest.requires_numpy
-
-with pytest.suppress(ImportError):
-    import numpy as np
+np = pytest.importorskip("numpy")
 
 
 def test_vectorize(capture):

tests/test_pickling.py

 # -*- coding: utf-8 -*-
 import pytest
+
+import env  # noqa: F401
+
 from pybind11_tests import pickling as m
 
 try:
@@ -22,7 +25,7 @@ def test_roundtrip(cls_name):
     assert p2.extra2() == p.extra2()
 
 
-@pytest.unsupported_on_pypy
+@pytest.mark.xfail("env.PYPY")
 @pytest.mark.parametrize("cls_name", ["PickleableWithDict", "PickleableWithDictNew"])
 def test_roundtrip_with_dict(cls_name):
     cls = getattr(m, cls_name)

tests/test_pytypes.py

@@ -3,6 +3,8 @@ from __future__ import division
 import pytest
 import sys
 
+import env  # noqa: F401
+
 from pybind11_tests import pytypes as m
 from pybind11_tests import debug_enabled
 
@@ -113,7 +115,7 @@ def test_bytes(doc):
     assert m.bytes_from_str().decode() == "bar"
 
     assert doc(m.bytes_from_str) == "bytes_from_str() -> {}".format(
-        "str" if pytest.PY2 else "bytes"
+        "str" if env.PY2 else "bytes"
     )
 
 
@@ -224,7 +226,7 @@ def test_pybind11_str_raw_str():
     # specifically to exercise pybind11::str::raw_str
     cvt = m.convert_to_pybind11_str
     assert cvt(u"Str") == u"Str"
-    assert cvt(b'Bytes') == u"Bytes" if pytest.PY2 else "b'Bytes'"
+    assert cvt(b'Bytes') == u"Bytes" if env.PY2 else "b'Bytes'"
     assert cvt(None) == u"None"
     assert cvt(False) == u"False"
     assert cvt(True) == u"True"
@@ -237,8 +239,8 @@ def test_pybind11_str_raw_str():
     assert cvt([28]) == u"[28]"
     assert cvt({}) == u"{}"
     assert cvt({3: 4}) == u"{3: 4}"
-    assert cvt(set()) == u"set([])" if pytest.PY2 else "set()"
-    assert cvt({3, 3}) == u"set([3])" if pytest.PY2 else "{3}"
+    assert cvt(set()) == u"set([])" if env.PY2 else "set()"
+    assert cvt({3, 3}) == u"set([3])" if env.PY2 else "{3}"
 
     valid_orig = u"DZ"
     valid_utf8 = valid_orig.encode("utf-8")
@@ -324,7 +326,7 @@ def test_memoryview(method, args, fmt, expected_view):
     view = method(*args)
     assert isinstance(view, memoryview)
     assert view.format == fmt
-    if isinstance(expected_view, bytes) or not pytest.PY2:
+    if isinstance(expected_view, bytes) or not env.PY2:
         view_as_list = list(view)
     else:
         # Using max to pick non-zero byte (big-endian vs little-endian).
@@ -332,9 +334,7 @@ def test_memoryview(method, args, fmt, expected_view):
     assert view_as_list == list(expected_view)
 
 
-@pytest.mark.skipif(
-    not hasattr(sys, 'getrefcount'),
-    reason='getrefcount is not available')
+@pytest.mark.xfail("env.PYPY", reason="getrefcount is not available")
 @pytest.mark.parametrize('method', [
     m.test_memoryview_object,
     m.test_memoryview_buffer_info,
@@ -352,7 +352,7 @@ def test_memoryview_from_buffer_empty_shape():
     view = m.test_memoryview_from_buffer_empty_shape()
     assert isinstance(view, memoryview)
     assert view.format == 'B'
-    if pytest.PY2:
+    if env.PY2:
         # Python 2 behavior is weird, but Python 3 (the future) is fine.
         # PyPy3 has <memoryview, while CPython 2 has <memory
         assert bytes(view).startswith(b'<memory')
@@ -366,14 +366,14 @@ def test_test_memoryview_from_buffer_invalid_strides():
 
 
 def test_test_memoryview_from_buffer_nullptr():
-    if pytest.PY2:
+    if env.PY2:
         m.test_memoryview_from_buffer_nullptr()
     else:
         with pytest.raises(ValueError):
             m.test_memoryview_from_buffer_nullptr()
 
 
-@pytest.unsupported_on_py2
+@pytest.mark.skipif("env.PY2")
 def test_memoryview_from_memory():
     view = m.test_memoryview_from_memory()
     assert isinstance(view, memoryview)

tests/test_stl_binders.py

 # -*- coding: utf-8 -*-
 import pytest
-from pybind11_tests import stl_binders as m
 
-with pytest.suppress(ImportError):
-    import numpy as np
+import env  # noqa: F401
+
+from pybind11_tests import stl_binders as m
 
 
 def test_vector_int():
@@ -68,15 +68,14 @@ def test_vector_int():
     assert len(v_int2) == 0
 
 
-# related to the PyPy's buffer protocol.
-@pytest.unsupported_on_pypy
+# Older PyPy's failed here, related to the PyPy's buffer protocol.
 def test_vector_buffer():
     b = bytearray([1, 2, 3, 4])
     v = m.VectorUChar(b)
     assert v[1] == 2
     v[2] = 5
     mv = memoryview(v)  # We expose the buffer interface
-    if not pytest.PY2:
+    if not env.PY2:
         assert mv[2] == 5
         mv[2] = 6
     else:
@@ -84,7 +83,7 @@ def test_vector_buffer():
         mv[2] = '\x06'
     assert v[2] == 6
 
-    if not pytest.PY2:
+    if not env.PY2:
         mv = memoryview(b)
         v = m.VectorUChar(mv[::2])
         assert v[1] == 3
@@ -94,9 +93,8 @@ def test_vector_buffer():
     assert "NumPy type info missing for " in str(excinfo.value)
 
 
-@pytest.unsupported_on_pypy
-@pytest.requires_numpy
 def test_vector_buffer_numpy():
+    np = pytest.importorskip("numpy")
     a = np.array([1, 2, 3, 4], dtype=np.int32)
     with pytest.raises(TypeError):
         m.VectorInt(a)

tests/test_virtual_functions.py

 # -*- coding: utf-8 -*-
 import pytest
 
+import env  # noqa: F401
+
 from pybind11_tests import virtual_functions as m
 from pybind11_tests import ConstructorStats
 
@@ -160,7 +162,7 @@ def test_alias_delay_initialization2(capture):
 
 # PyPy: Reference count > 1 causes call with noncopyable instance
 # to fail in ncv1.print_nc()
-@pytest.unsupported_on_pypy
+@pytest.mark.xfail("env.PYPY")
 @pytest.mark.skipif(not hasattr(m, "NCVirt"), reason="NCVirt test broken on ICPC")
 def test_move_support():
     class NCVirtExt(m.NCVirt):