Merge pull request #34 from RadeonOpenCompute/fixLinkType

Associate a link only when its info is present

rocm_bandwidth_test.hpp

@@ -433,6 +433,7 @@ class RocmBandwidthTest : public BaseTest {
   static const uint32_t LINK_TYPE_SELF = 0x00;
   static const uint32_t LINK_TYPE_PCIE = 0x01;
   static const uint32_t LINK_TYPE_XGMI = 0x02;
+  static const uint32_t LINK_TYPE_MULTI_HOPS = 0x03;
   static const uint32_t LINK_TYPE_NO_PATH = 0xFFFFFFFF;
 
   // List used to store transactions per user request

rocm_bandwidth_test_print.cpp

@@ -185,7 +185,7 @@ void RocmBandwidthTest::PrintLinkTypeMatrix() const {
   std::cout.width(format);
   std::cout << "";
   std::cout.width(format);
-  std::cout << "Device Link Types: P == PCIe, X == xGMI, NP == No Path, N/A = Not Applicable";
+  std::cout << "Device Link Types: P = PCIe, X = xGMI, N/A = Not Applicable";
   std::cout << std::endl;
   std::cout << std::endl;
 
@@ -208,14 +208,14 @@ void RocmBandwidthTest::PrintLinkTypeMatrix() const {
     for (uint32_t dst_idx = 0; dst_idx < agent_index_; dst_idx++) {
       uint32_t link_type = link_type_matrix_[(src_idx * agent_index_) + dst_idx];
       std::cout.width(format);
-      if (link_type == LINK_TYPE_SELF) {
-        std::cout << "N/A";
-      } else if (link_type == LINK_TYPE_XGMI) {
+      if (link_type == LINK_TYPE_XGMI) {
         std::cout << "X";
       } else if (link_type == LINK_TYPE_PCIE) {
         std::cout << "P";
-      } else if (link_type == LINK_TYPE_NO_PATH) {
-        std::cout << "NP";
+      } else if ((link_type == LINK_TYPE_SELF) ||
+                 (link_type == LINK_TYPE_NO_PATH) ||
+                 (link_type == LINK_TYPE_MULTI_HOPS)) {
+        std::cout << "N/A";
       }
     }
     std::cout << std::endl;

rocm_bandwidth_test_topology.cpp

@@ -278,14 +278,33 @@ void RocmBandwidthTest::BindLinkType(uint32_t idx1, uint32_t idx2) {
   memset(link_info, 0, (hops * sizeof(hsa_amd_memory_pool_link_info_t)));
   err_ = hsa_amd_agent_memory_pool_get_info(agent1, pool,
                  HSA_AMD_AGENT_MEMORY_POOL_INFO_LINK_INFO, link_info);
-  link_type_matrix_[(idx1 * agent_index_) + idx2] = 0;
-  for(uint32_t hopIdx = 0; hopIdx < hops; hopIdx++) {
-    if ((link_info[hopIdx]).link_type != HSA_AMD_LINK_INFO_TYPE_XGMI) {
+  
+  // Initialize link type based on Src and Dst devices plus link
+  // type reported by ROCr library
+  hsa_device_type_t src_dev_type = agent_list_[idx1].device_type_;
+  hsa_device_type_t dst_dev_type = agent_list_[idx2].device_type_;
+  link_type_matrix_[(idx1 * agent_index_) + idx2] = LINK_TYPE_NO_PATH;
+  
+  // Update link matrix if there is one hop. Currently Thunk
+  // accumulates numa weight of the multiple hops into one link
+  if (hops == 1) {
+    if ((link_info[0]).link_type == HSA_AMD_LINK_INFO_TYPE_XGMI) {
+      link_type_matrix_[(idx1 * agent_index_) + idx2] = LINK_TYPE_XGMI;
+      free(link_info); 
+      return;
+    }
+
+    // Update link type to be PCIE if one or both devices are GPU's
+    if ((src_dev_type == HSA_DEVICE_TYPE_GPU) ||
+        (dst_dev_type == HSA_DEVICE_TYPE_GPU)) {
       link_type_matrix_[(idx1 * agent_index_) + idx2] = LINK_TYPE_PCIE;
-      break;
+      free(link_info); 
+      return;
     }
-    link_type_matrix_[(idx1 * agent_index_) + idx2] = LINK_TYPE_XGMI;
   }
+  
+  // This should not be happening
+  link_type_matrix_[(idx1 * agent_index_) + idx2] = LINK_TYPE_MULTI_HOPS;
   free(link_info); 
 }