Print Link Type between nodes

rocm_bandwidth_test.cpp

@@ -533,9 +533,10 @@ RocmBandwidthTest::RocmBandwidthTest(int argc, char** argv) : BaseTest() {
   req_copy_all_bidir_ = REQ_INVALID;
   req_copy_all_unidir_ = REQ_INVALID;
   
-  link_matrix_ = NULL;
   access_matrix_ = NULL;
+  link_type_matrix_ = NULL;
   active_agents_list_ = NULL;
+  link_weight_matrix_ = NULL;
   
   latency_ = false;
   validate_ = false;
@@ -543,8 +544,8 @@ RocmBandwidthTest::RocmBandwidthTest(int argc, char** argv) : BaseTest() {
 
   // Initialize version of the test
   version_.major_id = 2;
-  version_.minor_id = 0;
-  version_.step_id = 1;
+  version_.minor_id = 1;
+  version_.step_id = 0;
   version_.reserved = 0;
 
   bw_iter_cnt_ = getenv("ROCM_BW_ITER_CNT");
@@ -563,7 +564,13 @@ RocmBandwidthTest::RocmBandwidthTest(int argc, char** argv) : BaseTest() {
   exit_value_ = 0;
 }
 
-RocmBandwidthTest::~RocmBandwidthTest() { }
+RocmBandwidthTest::~RocmBandwidthTest() {
+
+  delete access_matrix_;
+  delete link_type_matrix_;
+  delete link_weight_matrix_;
+  delete active_agents_list_;
+}
 
 std::string RocmBandwidthTest::GetVersion() const {
 

rocm_bandwidth_test.hpp

@@ -207,6 +207,10 @@ class RocmBandwidthTest : public BaseTest {
   // @brief: Discover the topology of pools on Rocm Platform
   void DiscoverTopology();
 
+  // @brief: Populate link type for the set of agents
+  void DiscoverLinkType();
+  void BindLinkType(uint32_t idx1, uint32_t idx2);
+
   // @brief: Populate link weight for the set of agents
   void DiscoverLinkWeight();
   void BindLinkWeight(uint32_t idx1, uint32_t idx2);
@@ -217,8 +221,11 @@ class RocmBandwidthTest : public BaseTest {
   // @brief: Print topology info
   void PrintTopology();
 
-  // @brief: Print link matrix
-  void PrintLinkMatrix() const;
+  // @brief: Print link type matrix
+  void PrintLinkTypeMatrix() const;
+
+  // @brief: Print link weight matrix
+  void PrintLinkWeightMatrix() const;
 
   // @brief: Print access matrix
   void PrintAccessMatrix() const;
@@ -420,6 +427,11 @@ class RocmBandwidthTest : public BaseTest {
   static const uint32_t DEV_COPY_LATENCY = 0x04;
   static const uint32_t VALIDATE_COPY_OP = 0x08;
 
+  static const uint32_t LINK_TYPE_SELF = 0x00;
+  static const uint32_t LINK_TYPE_PCIE = 0x01;
+  static const uint32_t LINK_TYPE_XGMI = 0x02;
+  static const uint32_t LINK_TYPE_NO_PATH = 0xFFFFFFFF;
+
   // List used to store transactions per user request
   vector<async_trans_t> trans_list_;
 
@@ -428,7 +440,8 @@ class RocmBandwidthTest : public BaseTest {
 
   // Matrix used to track Access among agents
   uint32_t* access_matrix_;
-  uint32_t* link_matrix_;
+  uint32_t* link_type_matrix_;
+  uint32_t* link_weight_matrix_;
   
   // Env key to determine if Fine-grained or
   // Coarse-grained pool should be filtered out

rocm_bandwidth_test_parse.cpp

@@ -319,7 +319,8 @@ void RocmBandwidthTest::ParseArguments() {
   if (print_topology) {
     PrintTopology();
     PrintAccessMatrix();
-    PrintLinkMatrix();
+    PrintLinkWeightMatrix();
+    PrintLinkTypeMatrix();
     exit(0);
   }
 

rocm_bandwidth_test_print.cpp

@@ -177,13 +177,58 @@ void RocmBandwidthTest::PrintAccessMatrix() const {
   std::cout << std::endl;
 }
 
-void RocmBandwidthTest::PrintLinkMatrix() const {
+void RocmBandwidthTest::PrintLinkTypeMatrix() const {
+
+  uint32_t format = 10;
+  std::cout.setf(ios::left);
+
+  std::cout.width(format);
+  std::cout << "";
+  std::cout.width(format);
+  std::cout << "Device Link Types: P == PCIe, X == xGMI, NP == No Path, N/A = Not Applicable";
+  std::cout << std::endl;
+  std::cout << std::endl;
+
+  std::cout.width(format);
+  std::cout << "";
+  std::cout.width(format);
+  std::cout << "D/D";
+  for (uint32_t idx0 = 0; idx0 < agent_index_; idx0++) {
+    std::cout.width(format);
+    std::cout << idx0;
+  }
+  std::cout << std::endl;
+  std::cout << std::endl;
+
+  for (uint32_t src_idx = 0; src_idx < agent_index_; src_idx++) {
+    std::cout.width(format);
+    std::cout << "";
+    std::cout.width(format);
+    std::cout << src_idx;
+    for (uint32_t dst_idx = 0; dst_idx < agent_index_; dst_idx++) {
+      uint32_t link_type = link_type_matrix_[(src_idx * agent_index_) + dst_idx];
+      std::cout.width(format);
+      if (link_type == LINK_TYPE_SELF) {
+        std::cout << "N/A";
+      } else if (link_type == LINK_TYPE_XGMI) {
+        std::cout << "X";
+      } else if (link_type == LINK_TYPE_PCIE) {
+        std::cout << "P";
+      } else if (link_type == LINK_TYPE_NO_PATH) {
+        std::cout << "NP";
+      }
+    }
+    std::cout << std::endl;
+    std::cout << std::endl;
+  }
+  std::cout << std::endl;
+}
+
+void RocmBandwidthTest::PrintLinkWeightMatrix() const {
 
-  // uint32_t format = 12;
   uint32_t format = 10;
   std::cout.setf(ios::left);
 
-  // std::cout << std::endl;
   std::cout.width(format);
   std::cout << "";
   std::cout.width(format);
@@ -208,7 +253,7 @@ void RocmBandwidthTest::PrintLinkMatrix() const {
     std::cout.width(format);
     std::cout << src_idx;
     for (uint32_t dst_idx = 0; dst_idx < agent_index_; dst_idx++) {
-      uint32_t link_weight = link_matrix_[(src_idx * agent_index_) + dst_idx];
+      uint32_t link_weight = link_weight_matrix_[(src_idx * agent_index_) + dst_idx];
       std::cout.width(format);
       if (link_weight == 0xFFFFFFFF) {
         std::cout << "N/A";

rocm_bandwidth_test_report.cpp

@@ -161,7 +161,8 @@ void RocmBandwidthTest::Display() const {
     PrintVersion();
     DisplayDevInfo();
     PrintAccessMatrix();
-    PrintLinkMatrix();
+    PrintLinkWeightMatrix();
+    PrintLinkTypeMatrix();
     DisplayCopyTimeMatrix(true);
     return;
   }
@@ -171,7 +172,8 @@ void RocmBandwidthTest::Display() const {
       PrintVersion();
       DisplayDevInfo();
       PrintAccessMatrix();
-      PrintLinkMatrix();
+      PrintLinkWeightMatrix();
+      PrintLinkTypeMatrix();
     }
     DisplayCopyTimeMatrix(true);
     return;

rocm_bandwidth_test_topology.cpp

@@ -247,15 +247,72 @@ void RocmBandwidthTest::DiscoverTopology() {
   // Populate the lists of agents and pools
   err_ = hsa_iterate_agents(AgentInfo, this);
 
-  // Populate the access matrix
+  // Populate the access, link type and weight matrices
+  // Access matrix must be populated first
   PopulateAccessMatrix();
+  DiscoverLinkType();
   DiscoverLinkWeight();
 }
 
+void RocmBandwidthTest::BindLinkType(uint32_t idx1, uint32_t idx2) {
+  
+  // Agent has no pools so no need to look for link type distance
+  if (agent_pool_list_[idx2].pool_list.size() == 0) {
+    link_type_matrix_[(idx1 * agent_index_) + idx2] = LINK_TYPE_NO_PATH;
+    return;
+  }
+  
+  uint32_t hops = 0;
+  hsa_agent_t agent1 = agent_list_[idx1].agent_;
+  hsa_amd_memory_pool_t& pool = agent_pool_list_[idx2].pool_list[0].pool_;
+  err_ = hsa_amd_agent_memory_pool_get_info(agent1, pool,
+                   HSA_AMD_AGENT_MEMORY_POOL_INFO_NUM_LINK_HOPS, &hops);
+  if (hops < 1) {
+    link_type_matrix_[(idx1 * agent_index_) + idx2] = LINK_TYPE_NO_PATH;
+    return;
+  }
+
+  hsa_amd_memory_pool_link_info_t* link_info;
+  uint32_t link_info_sz = hops * sizeof(hsa_amd_memory_pool_link_info_t);
+  link_info = (hsa_amd_memory_pool_link_info_t *)malloc(link_info_sz);
+  memset(link_info, 0, (hops * sizeof(hsa_amd_memory_pool_link_info_t)));
+  err_ = hsa_amd_agent_memory_pool_get_info(agent1, pool,
+                 HSA_AMD_AGENT_MEMORY_POOL_INFO_LINK_INFO, link_info);
+  link_type_matrix_[(idx1 * agent_index_) + idx2] = 0;
+  for(uint32_t hopIdx = 0; hopIdx < hops; hopIdx++) {
+    if ((link_info[hopIdx]).link_type != HSA_AMD_LINK_INFO_TYPE_XGMI) {
+      link_type_matrix_[(idx1 * agent_index_) + idx2] = LINK_TYPE_PCIE;
+      break;
+    }
+    link_type_matrix_[(idx1 * agent_index_) + idx2] = LINK_TYPE_XGMI;
+  }
+  free(link_info); 
+}
+
+void RocmBandwidthTest::DiscoverLinkType() {
+
+  // Allocate space if it is first time
+  if (link_type_matrix_ == NULL) {
+    link_type_matrix_ = new uint32_t[agent_index_ * agent_index_]();
+  }
+
+  agent_info_t agent_info;
+  for (uint32_t idx1 = 0; idx1 < agent_index_; idx1++) {
+    for (uint32_t idx2 = 0; idx2 < agent_index_; idx2++) {
+      if (idx1 == idx2) {
+        link_type_matrix_[(idx1 * agent_index_) + idx2] = LINK_TYPE_SELF;
+        continue;
+      }
+      BindLinkType(idx1, idx2);
+    }
+  }
+}
+
 void RocmBandwidthTest::BindLinkWeight(uint32_t idx1, uint32_t idx2) {
   
   // Agent has no pools so no need to look for numa distance
   if (agent_pool_list_[idx2].pool_list.size() == 0) {
+    link_weight_matrix_[(idx1 * agent_index_) + idx2] = 0xFFFFFFFF;
     return;
   }
   
@@ -265,7 +322,7 @@ void RocmBandwidthTest::BindLinkWeight(uint32_t idx1, uint32_t idx2) {
   err_ = hsa_amd_agent_memory_pool_get_info(agent1, pool,
                    HSA_AMD_AGENT_MEMORY_POOL_INFO_NUM_LINK_HOPS, &hops);
   if (hops < 1) {
-    link_matrix_[(idx1 * agent_index_) + idx2] = 0xFFFFFFFF;
+    link_weight_matrix_[(idx1 * agent_index_) + idx2] = 0xFFFFFFFF;
     return;
   }
 
@@ -275,10 +332,9 @@ void RocmBandwidthTest::BindLinkWeight(uint32_t idx1, uint32_t idx2) {
   memset(link_info, 0, (hops * sizeof(hsa_amd_memory_pool_link_info_t)));
   err_ = hsa_amd_agent_memory_pool_get_info(agent1, pool,
                  HSA_AMD_AGENT_MEMORY_POOL_INFO_LINK_INFO, link_info);
-  link_matrix_[(idx1 *agent_index_) + idx2] = 0;
+  link_weight_matrix_[(idx1 * agent_index_) + idx2] = 0;
   for(uint32_t hopIdx = 0; hopIdx < hops; hopIdx++) {
-    link_matrix_[(idx1 *agent_index_) + idx2] += (link_info[hopIdx]).numa_distance;
-
+    link_weight_matrix_[(idx1 * agent_index_) + idx2] += (link_info[hopIdx]).numa_distance;
   }
   free(link_info); 
 }
@@ -286,15 +342,15 @@ void RocmBandwidthTest::BindLinkWeight(uint32_t idx1, uint32_t idx2) {
 void RocmBandwidthTest::DiscoverLinkWeight() {
 
   // Allocate space if it is first time
-  if (link_matrix_ == NULL) {
-    link_matrix_ = new uint32_t[agent_index_ * agent_index_]();
+  if (link_weight_matrix_ == NULL) {
+    link_weight_matrix_ = new uint32_t[agent_index_ * agent_index_]();
   }
 
   agent_info_t agent_info;
   for (uint32_t idx1 = 0; idx1 < agent_index_; idx1++) {
     for (uint32_t idx2 = 0; idx2 < agent_index_; idx2++) {
       if (idx1 == idx2) {
-        link_matrix_[(idx1 *agent_index_) + idx2] = 0;
+        link_weight_matrix_[(idx1 *agent_index_) + idx2] = 0;
         continue;
       }
       BindLinkWeight(idx1, idx2);