Support building python bindings for multiple python versions (#655)

* Target multiple python versions

* Fix overriding c++ standard

* Add a cmake module for python

* Remove python 3 stage

* Suppress warnings

* Formatting
Co-authored-by: mvermeulen <5479696+mvermeulen@users.noreply.github.com>

CMakeLists.txt

@@ -43,6 +43,7 @@ else()
     set(MIGRAPHX_ENABLE_GPU Off CACHE BOOL "")
 endif()
 
+set(CMAKE_CXX_STANDARD_DEFAULT "")
 add_compile_options(-std=c++14)
 
 list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake)

Jenkinsfile

@@ -127,10 +127,6 @@ rocmtest tidy: rocmnode('rocmtest') { cmake_build ->
         cmake_build("hcc", "-DCMAKE_BUILD_TYPE=release")
         stash includes: 'build/*.deb', name: 'migraphx-package'
     }
-}, clang_release_py3: rocmnode('vega') { cmake_build ->
-    stage('Clang Release Python 3') {
-        cmake_build("hcc", "-DCMAKE_BUILD_TYPE=release -DPYTHON_EXECUTABLE=/usr/local/bin/python3")
-    }
 }, hip_clang_release: rocmhipclangnode('vega') { cmake_build ->
     stage('Hip Clang Release') {
         cmake_build("/opt/rocm/llvm/bin/clang++", "-DCMAKE_BUILD_TYPE=release")

cmake/PythonModules.cmake

+if(COMMAND find_python)
+    return()
+endif()
+
+set(PYBIND11_NOPYTHON On)
+find_package(pybind11 REQUIRED)
+macro(find_python version)
+    find_program(PYTHON_CONFIG_${version} python${version}-config)
+    if(EXISTS ${PYTHON_CONFIG_${version}})
+        execute_process(COMMAND ${PYTHON_CONFIG_${version}} --includes OUTPUT_VARIABLE _python_include_args)
+        separate_arguments(_python_includes UNIX_COMMAND "${_python_include_args}")
+        string(REPLACE "-I" "" _python_includes "${_python_includes}")
+        add_library(python${version}::headers INTERFACE IMPORTED GLOBAL)
+        set_target_properties(python${version}::headers PROPERTIES
+            INTERFACE_INCLUDE_DIRECTORIES "${_python_includes}"
+        )
+        execute_process(COMMAND ${PYTHON_CONFIG_${version}} --prefix OUTPUT_VARIABLE _python_prefix)
+        string(STRIP "${_python_prefix}" _python_prefix)
+        set(PYTHON_${version}_EXECUTABLE "${_python_prefix}/bin/python${version}" CACHE PATH "")
+    endif()
+endmacro()
+function(py_extension name version)
+    set(_python_module_extension)
+    execute_process(COMMAND ${PYTHON_CONFIG_${version}} --extension-suffix OUTPUT_VARIABLE _python_module_extension)
+    string(STRIP "${_python_module_extension}" _python_module_extension)
+    set_target_properties(${name} PROPERTIES PREFIX "" SUFFIX "${_python_module_extension}")
+endfunction()
+function(py_add_module NAME)
+    set(options)
+    set(oneValueArgs PYTHON_VERSION PYTHON_MODULE)
+    set(multiValueArgs)
+
+    cmake_parse_arguments(PARSE "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
+    set(PYTHON_VERSION ${PARSE_PYTHON_VERSION})
+
+    add_library(${NAME} MODULE ${PARSE_UNPARSED_ARGUMENTS})
+    pybind11_strip(${NAME})
+    py_extension(${NAME} ${PYTHON_VERSION})
+    target_link_libraries(${NAME} PRIVATE pybind11::module pybind11::lto python${PYTHON_VERSION}::headers)
+    set_target_properties(${NAME} PROPERTIES 
+        OUTPUT_NAME ${PARSE_PYTHON_MODULE}
+        C_VISIBILITY_PRESET hidden
+        CXX_VISIBILITY_PRESET hidden
+    )
+
+endfunction()
+set(PYTHON_SEARCH_VERSIONS 2.7 3.5 3.6 3.7 3.8 3.9)
+
+set(PYTHON_VERSIONS)
+foreach(PYTHON_VERSION ${PYTHON_SEARCH_VERSIONS})
+    find_python(${PYTHON_VERSION})
+    if(TARGET python${PYTHON_VERSION}::headers)
+        message(STATUS "Python ${PYTHON_VERSION} found.")
+        list(APPEND PYTHON_VERSIONS ${PYTHON_VERSION})
+    else()
+        message(STATUS "Python ${PYTHON_VERSION} not found.")
+    endif()
+endforeach()
\ No newline at end of file

requirements.txt

@@ -5,5 +5,5 @@ ROCmSoftwarePlatform/MIOpen@2.4.0
 nlohmann/json@v3.8.0
 blaze,https://bitbucket.org/blaze-lib/blaze/get/f0755dea0e03.tar.gz -X header -DHEADER_DIR=blaze
 half,https://github.com/pfultz2/half/archive/1.12.0.tar.gz -X header -H sha256:0a08660b68abb176ebc2a0cdf8de46e3182a7f46c66443bb80dbfaaec98cf969
-pybind/pybind11@v2.2.4 -DPYBIND11_TEST=Off --build
+pybind/pybind11@d159a563383d10c821ba7b2a71905d1207db6de4 --build
 msgpack/msgpack-c@cpp-3.3.0 -DMSGPACK_BUILD_TESTS=Off
\ No newline at end of file

src/py/CMakeLists.txt

 
 option(MIGRAPHX_ENABLE_PYTHON "Enable python bindings" ON)
 if(MIGRAPHX_ENABLE_PYTHON)
-    find_program(DEFAULT_PYTHON_EXE python)
-    if(DEFAULT_PYTHON_EXE)
-        set(PYTHON_EXECUTABLE ${DEFAULT_PYTHON_EXE} CACHE PATH "Path to python executable")
-    endif()
-    find_package(pybind11 REQUIRED)
-    pybind11_add_module(migraphx_py migraphx_py.cpp)
-    set_target_properties(migraphx_py PROPERTIES 
-        OUTPUT_NAME migraphx
-        C_VISIBILITY_PRESET hidden
-        CXX_VISIBILITY_PRESET hidden
-    )
-    target_link_libraries(migraphx_py PRIVATE migraphx migraphx_tf migraphx_onnx migraphx_cpu)
-    if(MIGRAPHX_ENABLE_GPU)
-        target_link_libraries(migraphx_py PRIVATE migraphx_gpu)
-        target_compile_definitions(migraphx_py PRIVATE -DHAVE_GPU)
-    endif()
-    rocm_install_targets(TARGETS migraphx_py)
+    include(PythonModules)
+
+    add_custom_target(migraphx_py)
+
+    foreach(PYTHON_VERSION ${PYTHON_VERSIONS})
+        py_add_module(migraphx_py_${PYTHON_VERSION} migraphx_py.cpp PYTHON_VERSION ${PYTHON_VERSION} PYTHON_MODULE migraphx)
+        target_link_libraries(migraphx_py_${PYTHON_VERSION} PRIVATE migraphx migraphx_tf migraphx_onnx migraphx_cpu)
+        if(MIGRAPHX_ENABLE_GPU)
+            target_link_libraries(migraphx_py_${PYTHON_VERSION} PRIVATE migraphx_gpu)
+            target_compile_definitions(migraphx_py_${PYTHON_VERSION} PRIVATE -DHAVE_GPU)
+        endif()
+        rocm_install_targets(TARGETS migraphx_py_${PYTHON_VERSION})
+        add_dependencies(migraphx_py migraphx_py_${PYTHON_VERSION})
+    endforeach()
 endif()

src/py/migraphx_py.cpp

@@ -22,6 +22,20 @@
 using half   = half_float::half;
 namespace py = pybind11;
 
+#ifdef __clang__
+#define MIGRAPHX_PUSH_UNUSED_WARNING \
+    _Pragma("clang diagnostic push") \
+        _Pragma("clang diagnostic ignored \"-Wused-but-marked-unused\"")
+#define MIGRAPHX_POP_WARNING _Pragma("clang diagnostic pop")
+#else
+#define MIGRAPHX_PUSH_UNUSED_WARNING
+#define MIGRAPHX_POP_WARNING
+#endif
+#define MIGRAPHX_PYBIND11_MODULE(...) \
+    MIGRAPHX_PUSH_UNUSED_WARNING      \
+    PYBIND11_MODULE(__VA_ARGS__)      \
+    MIGRAPHX_POP_WARNING
+
 namespace migraphx {
 
 migraphx::value to_value(py::kwargs kwargs);
@@ -194,7 +208,7 @@ migraphx::shape to_shape(const py::buffer_info& info)
     }
 }
 
-PYBIND11_MODULE(migraphx, m)
+MIGRAPHX_PYBIND11_MODULE(migraphx, m)
 {
     py::class_<migraphx::shape>(m, "shape")
         .def(py::init<>())

test/py/CMakeLists.txt

 
-find_package(PythonInterp)
+include(PythonModules)
 
 function(add_py_test NAME SCRIPT)
-    set (ENV_COMMAND ${CMAKE_COMMAND} -E env 
-        "PYTHONPATH=$<TARGET_FILE_DIR:migraphx_py>"
-        "PYTHONMALLOC=debug"
-        "MALLOC_CHECK_=3"
-    )
-    add_test(
-        NAME test_py_${NAME} 
-        COMMAND ${ENV_COMMAND} ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/${SCRIPT} ${ARGN})
-    add_custom_target(test_py_${NAME} 
-        COMMAND ${ENV_COMMAND} ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/${SCRIPT} ${ARGN} 
-        COMMENT "${PYTHON_EXECUTABLE} ${SCRIPT}")
+    foreach(PYTHON_VERSION ${PYTHON_VERSIONS})
+        set (ENV_COMMAND ${CMAKE_COMMAND} -E env 
+            "PYTHONPATH=$<TARGET_FILE_DIR:migraphx_py_${PYTHON_VERSION}>"
+            "PYTHONMALLOC=debug"
+            "MALLOC_CHECK_=3"
+        )
+        set(PYTHON_EXECUTABLE ${PYTHON_${PYTHON_VERSION}_EXECUTABLE})
+        add_test(
+            NAME test_py_${PYTHON_VERSION}_${NAME} 
+            COMMAND ${ENV_COMMAND} ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/${SCRIPT} ${ARGN})
+        add_custom_target(test_py_${PYTHON_VERSION}_${NAME} 
+            COMMAND ${ENV_COMMAND} ${PYTHON_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/${SCRIPT} ${ARGN} 
+            COMMENT "${PYTHON_EXECUTABLE} ${SCRIPT}")
+
+    endforeach()
 endfunction()
-add_dependencies(tests migraphx_py)
-add_dependencies(check migraphx_py)
+foreach(PYTHON_VERSION ${PYTHON_VERSIONS})
+    add_dependencies(tests migraphx_py_${PYTHON_VERSION})
+    add_dependencies(check migraphx_py_${PYTHON_VERSION})
+endforeach()
 
 add_py_test(cpu test_cpu.py WORKING_DIRECTORY ${TEST_ONNX_DIR})
 add_py_test(save_load test_save_load.py WORKING_DIRECTORY ${TEST_ONNX_DIR})