impr: Library/Client build organization

Change code organization and build options

Code changes related to the following:
  * Build files
    * Options to build client, shared, and static libraries
  * Source code directories
  * Modern C++20 changes
  * Based on TB 1.6.4
  * Formatting
Signed-off-by: Oliveira, Daniel <daniel.oliveira@amd.com>

client/include/Topology.hpp

+/*
+Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#pragma once
+
+#include "TransferBench.hpp"
+
+#include <vector>
+
+
+static int RemappedCpuIndex(int origIdx)
+{
+    static std::vector<int> remappingCpu;
+
+    // Build CPU remapping on first use
+    // Skip numa nodes that are not configured
+    if (remappingCpu.empty()) {
+        for (int node = 0; node <= numa_max_node(); node++) {
+            if (numa_bitmask_isbitset(numa_get_mems_allowed(), node)) {
+                remappingCpu.push_back(node);
+            }
+        }
+    }
+    return remappingCpu[origIdx];
+}
+
+static void PrintNicToGPUTopo([[maybe_unused]] bool outputToCsv)
+{
+#ifdef NIC_EXEC_ENABLED
+    printf(
+        " NIC | Device Name | Active | PCIe Bus ID  | NUMA | Closest GPU(s) | GID Index | GID "
+        "Descriptor\n");
+    if (!outputToCsv) {
+        printf(
+            "-----+-------------+--------+--------------+------+----------------+-----------+------"
+            "-------------\n");
+    }
+
+    int numGpus               = TransferBench::GetNumExecutors(EXE_GPU_GFX);
+    auto const& ibvDeviceList = GetIbvDeviceList();
+    for (int i = 0; i < ibvDeviceList.size(); i++) {
+        std::string closestGpusStr = "";
+        for (int j = 0; j < numGpus; j++) {
+            if (TransferBench::GetClosestNicToGpu(j) == i) {
+                if (closestGpusStr != "") { closestGpusStr += ","; }
+                closestGpusStr += std::to_string(j);
+            }
+        }
+
+        printf(" %-3d | %-11s | %-6s | %-12s | %-4d | %-14s | %-9s | %-20s\n",
+               i,
+               ibvDeviceList[i].name.c_str(),
+               ibvDeviceList[i].hasActivePort ? "Yes" : "No",
+               ibvDeviceList[i].busId.c_str(),
+               ibvDeviceList[i].numaNode,
+               closestGpusStr.c_str(),
+               ibvDeviceList[i].isRoce && ibvDeviceList[i].hasActivePort
+                   ? std::to_string(ibvDeviceList[i].gidIndex).c_str()
+                   : "N/A",
+               ibvDeviceList[i].isRoce && ibvDeviceList[i].hasActivePort
+                   ? ibvDeviceList[i].gidDescriptor.c_str()
+                   : "N/A");
+    }
+    printf("\n");
+#endif
+}
+
+void DisplayTopology(bool outputToCsv)
+{
+    int numCpus = TransferBench::GetNumExecutors(EXE_CPU);
+    int numGpus = TransferBench::GetNumExecutors(EXE_GPU_GFX);
+    int numNics = TransferBench::GetNumExecutors(EXE_NIC);
+    char sep    = (outputToCsv ? ',' : '|');
+
+    if (outputToCsv) {
+        printf("NumCpus,%d\n", numCpus);
+        printf("NumGpus,%d\n", numGpus);
+        printf("NumNics,%d\n", numNics);
+    } else {
+        printf("\nDetected Topology:\n");
+        printf("==================\n");
+        printf("  %d configured CPU NUMA node(s) [%d total]\n", numCpus, numa_max_node() + 1);
+        printf("  %d GPU device(s)\n", numGpus);
+        printf("  %d Supported NIC device(s)\n", numNics);
+    }
+
+    // Print out detected CPU topology
+    printf("\n            %c", sep);
+    for (int j = 0; j < numCpus; j++) { printf("NUMA %02d%c", j, sep); }
+    printf(" #Cpus %c Closest GPU(s)\n", sep);
+
+    if (!outputToCsv) {
+        printf("------------+");
+        for (int j = 0; j <= numCpus; j++) { printf("-------+"); }
+        printf("---------------\n");
+    }
+
+    for (int i = 0; i < numCpus; i++) {
+        int nodeI = RemappedCpuIndex(i);
+        printf("NUMA %02d (%02d)%c", i, nodeI, sep);
+        for (int j = 0; j < numCpus; j++) {
+            int nodeJ    = RemappedCpuIndex(j);
+            int numaDist = numa_distance(nodeI, nodeJ);
+            printf(" %5d %c", numaDist, sep);
+        }
+
+        int numCpuCores = 0;
+        for (int j = 0; j < numa_num_configured_cpus(); j++) {
+            if (numa_node_of_cpu(j) == nodeI) { numCpuCores++; }
+        }
+        printf(" %5d %c", numCpuCores, sep);
+
+        for (int j = 0; j < numGpus; j++) {
+            if (TransferBench::GetClosestCpuNumaToGpu(j) == nodeI) { printf(" %d", j); }
+        }
+        printf("\n");
+    }
+    printf("\n");
+
+    // Print out detected NIC topology
+    PrintNicToGPUTopo(outputToCsv);
+
+    // Print out detected GPU topology
+#if defined(__NVCC__)
+    for (int i = 0; i < numGpus; i++) {
+        hipDeviceProp_t prop;
+        HIP_CALL(hipGetDeviceProperties(&prop, i));
+        printf(" GPU %02d | %s\n", i, prop.name);
+    }
+    // No further topology detection done for NVIDIA platforms
+    return;
+#else
+    // Print headers
+    if (!outputToCsv) {
+        printf("        |");
+        for (int j = 0; j < numGpus; j++) {
+            hipDeviceProp_t prop;
+            HIP_CALL(hipGetDeviceProperties(&prop, j));
+            std::string fullName = prop.gcnArchName;
+            std::string archName = fullName.substr(0, fullName.find(':'));
+            printf(" %6s |", archName.c_str());
+        }
+        printf("\n");
+    }
+
+    printf("        %c", sep);
+    for (int j = 0; j < numGpus; j++) { printf(" GPU %02d %c", j, sep); }
+    printf(" PCIe Bus ID  %c #CUs %c NUMA %c #DMA %c #XCC %c NIC\n", sep, sep, sep, sep, sep);
+
+    if (!outputToCsv) {
+        for (int j = 0; j <= numGpus; j++) { printf("--------+"); }
+        printf("--------------+------+------+------+------+------\n");
+    }
+
+    // Loop over each GPU device
+    for (int i = 0; i < numGpus; i++) {
+        printf(" GPU %02d %c", i, sep);
+
+        // Print off link information
+        for (int j = 0; j < numGpus; j++) {
+            if (i == j) {
+                printf("    N/A %c", sep);
+            } else {
+                uint32_t linkType, hopCount;
+                HIP_CALL(hipExtGetLinkTypeAndHopCount(i, j, &linkType, &hopCount));
+                printf(" %s-%d %c",
+                       linkType == HSA_AMD_LINK_INFO_TYPE_HYPERTRANSPORT ? "  HT"
+                       : linkType == HSA_AMD_LINK_INFO_TYPE_QPI          ? " QPI"
+                       : linkType == HSA_AMD_LINK_INFO_TYPE_PCIE         ? "PCIE"
+                       : linkType == HSA_AMD_LINK_INFO_TYPE_INFINBAND    ? "INFB"
+                       : linkType == HSA_AMD_LINK_INFO_TYPE_XGMI         ? "XGMI"
+                                                                         : "????",
+                       hopCount,
+                       sep);
+            }
+        }
+
+        char pciBusId[20];
+        HIP_CALL(hipDeviceGetPCIBusId(pciBusId, 20, i));
+        printf(" %-11s %c %-4d %c %-4d %c %-4d %c %-4d %c %-4d\n",
+               pciBusId,
+               sep,
+               TransferBench::GetNumSubExecutors({EXE_GPU_GFX, i}),
+               sep,
+               TransferBench::GetClosestCpuNumaToGpu(i),
+               sep,
+               TransferBench::GetNumExecutorSubIndices({EXE_GPU_DMA, i}),
+               sep,
+               TransferBench::GetNumExecutorSubIndices({EXE_GPU_GFX, i}),
+               sep,
+               TransferBench::GetClosestNicToGpu(i));
+    }
+#endif
+}

client/src/Client.cpp

+/*
+Copyright (c) 2019-2024 Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+#include "Client.hpp"
+#include "EnvVars.hpp"
+#include "Presets.hpp"
+#include "Topology.hpp"
+
+#include <algorithm>
+#include <cstddef>
+#include <cstdint>
+#include <fstream>
+#include <map>
+#include <string>
+#include <vector>
+
+
+int main(int argc, char** argv)
+{
+    // Collect environment variables
+    EnvVars ev;
+
+    // Display usage instructions and detected topology
+    if (argc <= 1) {
+        if (!ev.outputToCsv) {
+            DisplayUsage(argv[0]);
+            DisplayPresets();
+        }
+        DisplayTopology(ev.outputToCsv);
+        exit(0);
+    }
+
+    // Determine number of bytes to run per Transfer
+    size_t numBytesPerTransfer = argc > 2 ? atoll(argv[2]) : DEFAULT_BYTES_PER_TRANSFER;
+    if (argc > 2) {
+        // Adjust bytes if unit specified
+        char units = argv[2][strlen(argv[2]) - 1];
+        switch (units) {
+            case 'G':
+            case 'g': numBytesPerTransfer *= 1024;
+            case 'M':
+            case 'm': numBytesPerTransfer *= 1024;
+            case 'K':
+            case 'k': numBytesPerTransfer *= 1024;
+        }
+    }
+    if (numBytesPerTransfer % 4) {
+        printf("[ERROR] numBytesPerTransfer (%lu) must be a multiple of 4\n", numBytesPerTransfer);
+        exit(1);
+    }
+
+    // Run preset benchmark if requested
+    if (RunPreset(ev, numBytesPerTransfer, argc, argv)) { exit(0); }
+
+    // Read input from command line or configuration file
+    std::vector<std::string> lines;
+    {
+        std::string line;
+        if (!strcmp(argv[1], "cmdline")) {
+            for (int i = 3; i < argc; i++) { line += std::string(argv[i]) + " "; }
+            lines.push_back(line);
+        } else {
+            std::ifstream cfgFile(argv[1]);
+            if (!cfgFile.is_open()) {
+                printf("[ERROR] Unable to open transfer configuration file: [%s]\n", argv[1]);
+                exit(1);
+            }
+            while (std::getline(cfgFile, line)) { lines.push_back(line); }
+            cfgFile.close();
+        }
+    }
+
+    // Print environment variables and CSV header
+    ev.DisplayEnvVars();
+    if (ev.outputToCsv) {
+        printf("Test#,Transfer#,NumBytes,Src,Exe,Dst,CUs,BW(GB/s),Time(ms),SrcAddr,DstAddr\n");
+    }
+
+    TransferBench::ConfigOptions cfgOptions = ev.ToConfigOptions();
+    TransferBench::TestResults results;
+    std::vector<ErrResult> errors;
+
+    // Process each line as a Test
+    int testNum = 0;
+    for (std::string const& line : lines) {
+        // Check if line is a comment to be echoed to output (starts with ##)
+        if (!ev.outputToCsv && line[0] == '#' && line[1] == '#') { printf("%s\n", line.c_str()); }
+
+        // Parse set of parallel Transfers to execute
+        std::vector<Transfer> transfers;
+        CheckForError(TransferBench::ParseTransfers(line, transfers));
+        if (transfers.empty()) { continue; }
+
+        // Check for variable sub-executors Transfers
+        auto numVariableTransfers = std::size_t(0);
+        int maxVarCount           = 0;
+        {
+            std::map<ExeDevice, int> varTransferCount;
+            for (auto const& t : transfers) {
+                if (t.numSubExecs == 0) {
+                    if (t.exeDevice.exeType != EXE_GPU_GFX) {
+                        printf(
+                            "[ERROR] Variable number of subexecutors is only supported on GFX "
+                            "executors\n");
+                        exit(1);
+                    }
+                    numVariableTransfers++;
+                    varTransferCount[t.exeDevice]++;
+                    maxVarCount = max(maxVarCount, varTransferCount[t.exeDevice]);
+                }
+            }
+            if (numVariableTransfers > 0 && numVariableTransfers != transfers.size()) {
+                printf(
+                    "[ERROR] All or none of the Transfers in the Test must use variable number of "
+                    "Subexecutors\n");
+                exit(1);
+            }
+        }
+
+        // Track which transfers have already numBytes specified
+        std::vector<bool> bytesSpecified(transfers.size());
+        int hasUnspecified = false;
+        for (auto i = std::size_t(0); i < transfers.size(); i++) {
+            bytesSpecified[i] = (transfers[i].numBytes != 0);
+            if (transfers[i].numBytes == 0) { hasUnspecified = true; }
+        }
+
+        // Run the specified numbers of bytes otherwise generate a range of values
+        for (size_t bytes = (1 << 10); bytes <= (1 << 29); bytes *= 2) {
+            size_t deltaBytes = std::max(1UL, bytes / ev.samplingFactor);
+            size_t currBytes  = (numBytesPerTransfer == 0) ? bytes : numBytesPerTransfer;
+            do {
+                for (auto i = std::size_t(0); i < transfers.size(); i++) {
+                    if (!bytesSpecified[i]) { transfers[i].numBytes = currBytes; }
+                }
+
+                if (maxVarCount == 0) {
+                    if (TransferBench::RunTransfers(cfgOptions, transfers, results)) {
+                        PrintResults(ev, ++testNum, transfers, results);
+                    }
+                    PrintErrors(results.errResults);
+                } else {
+                    // Variable subexecutors - Determine how many subexecutors to sweep up to
+                    int maxNumVarSubExec = ev.maxNumVarSubExec;
+                    if (maxNumVarSubExec == 0) {
+                        maxNumVarSubExec = TransferBench::GetNumSubExecutors({EXE_GPU_GFX, 0}) /
+                                           maxVarCount;
+                    }
+
+                    TransferBench::TestResults bestResults;
+                    std::vector<Transfer> bestTransfers;
+                    for (int numSubExecs = ev.minNumVarSubExec; numSubExecs <= maxNumVarSubExec;
+                         numSubExecs++) {
+                        std::vector<Transfer> tempTransfers = transfers;
+                        for (auto& t : tempTransfers) {
+                            if (t.numSubExecs == 0) { t.numSubExecs = numSubExecs; }
+                        }
+
+                        TransferBench::TestResults tempResults;
+                        if (!TransferBench::RunTransfers(cfgOptions, tempTransfers, tempResults)) {
+                            PrintErrors(tempResults.errResults);
+                        } else {
+                            if (tempResults.avgTotalBandwidthGbPerSec >
+                                bestResults.avgTotalBandwidthGbPerSec) {
+                                bestResults   = tempResults;
+                                bestTransfers = tempTransfers;
+                            }
+                        }
+                    }
+                    PrintResults(ev, ++testNum, bestTransfers, bestResults);
+                    PrintErrors(bestResults.errResults);
+                }
+                if (numBytesPerTransfer != 0 || !hasUnspecified) { break; }
+                currBytes += deltaBytes;
+            } while (currBytes < bytes * 2);
+            if (numBytesPerTransfer != 0 || !hasUnspecified) { break; }
+        }
+    }
+}
+
+void DisplayUsage(char const* cmdName)
+{
+    std::string nicSupport = "";
+#if NIC_EXEC_ENABLED
+    nicSupport = " (with NIC support)";
+#endif
+    printf("TransferBench v%s.(%s)[%s]\n",
+           TransferBench::GetTransferBenchVersion().c_str(),
+           GetClientVersion().c_str(),
+           nicSupport.c_str());
+    printf("========================================\n");
+
+    if (numa_available() == -1) {
+        printf(
+            "[ERROR] NUMA library not supported. Check to see if libnuma has been installed on "
+            "this system\n");
+        exit(1);
+    }
+
+    printf("Usage: %s config <N>\n", cmdName);
+    printf("  config: Either:\n");
+    printf(
+        "          - Filename of configFile containing Transfers to execute (see example.cfg for "
+        "format)\n");
+    printf("          - Name of preset config:\n");
+    printf("  N     : (Optional) Number of bytes to copy per Transfer.\n");
+    printf("          If not specified, defaults to %lu bytes. Must be a multiple of 4 bytes\n",
+           DEFAULT_BYTES_PER_TRANSFER);
+    printf("          If 0 is specified, a range of Ns will be benchmarked\n");
+    printf("          May append a suffix ('K', 'M', 'G') for kilobytes / megabytes / gigabytes\n");
+    printf("\n");
+
+    EnvVars::DisplayUsage();
+}
+
+std::string MemDevicesToStr(std::vector<MemDevice> const& memDevices)
+{
+    if (memDevices.empty()) { return "N"; }
+    std::stringstream ss;
+    for (auto const& m : memDevices) { ss << TransferBench::MemTypeStr[m.memType] << m.memIndex; }
+    return ss.str();
+}
+
+void PrintResults(EnvVars const& ev,
+                  int const testNum,
+                  std::vector<Transfer> const& transfers,
+                  TransferBench::TestResults const& results)
+{
+    char sep                  = ev.outputToCsv ? ',' : '|';
+    size_t numTimedIterations = results.numTimedIterations;
+
+    if (!ev.outputToCsv) { printf("Test %d:\n", testNum); }
+
+    // Loop over each executor
+    for (auto exeInfoPair : results.exeResults) {
+        ExeDevice const& exeDevice = exeInfoPair.first;
+        ExeResult const& exeResult = exeInfoPair.second;
+        ExeType const exeType      = exeDevice.exeType;
+        int32_t const exeIndex     = exeDevice.exeIndex;
+
+        printf(
+            " Executor: %3s %02d %c %8.3f GB/s %c %8.3f ms %c %12lu bytes %c %-7.3f GB/s (sum)\n",
+            ExeTypeName[exeType],
+            exeIndex,
+            sep,
+            exeResult.avgBandwidthGbPerSec,
+            sep,
+            exeResult.avgDurationMsec,
+            sep,
+            exeResult.numBytes,
+            sep,
+            exeResult.sumBandwidthGbPerSec);
+
+        // Loop over each executor
+        for (int idx : exeResult.transferIdx) {
+            Transfer const& t       = transfers[idx];
+            TransferResult const& r = results.tfrResults[idx];
+
+            char exeSubIndexStr[32] = "";
+            if (t.exeSubIndex != -1) { sprintf(exeSubIndexStr, ".%d", t.exeSubIndex); }
+            printf(
+                "     Transfer %02d  %c %8.3f GB/s %c %8.3f ms %c %12lu bytes %c %s -> "
+                "%c%03d%s:%03d -> %s\n",
+                idx,
+                sep,
+                r.avgBandwidthGbPerSec,
+                sep,
+                r.avgDurationMsec,
+                sep,
+                r.numBytes,
+                sep,
+                MemDevicesToStr(t.srcs).c_str(),
+                TransferBench::ExeTypeStr[t.exeDevice.exeType],
+                t.exeDevice.exeIndex,
+                exeSubIndexStr,
+                t.numSubExecs,
+                MemDevicesToStr(t.dsts).c_str());
+
+            // Show per-iteration timing information
+            if (ev.showIterations) {
+                // Check that per-iteration information exists
+                if (r.perIterMsec.size() != numTimedIterations) {
+                    printf(
+                        "[ERROR] Per iteration timing data unavailable: Expected %lu data points, "
+                        "but have %lu\n",
+                        numTimedIterations,
+                        r.perIterMsec.size());
+                    exit(1);
+                }
+
+                // Compute standard deviation and track iterations by speed
+                std::set<std::pair<double, int>> times;
+                double stdDevTime = 0;
+                double stdDevBw   = 0;
+                for (auto i = std::size_t(0); i < numTimedIterations; i++) {
+                    times.insert(std::make_pair(r.perIterMsec[i], i + 1));
+                    double const varTime = fabs(r.avgDurationMsec - r.perIterMsec[i]);
+                    stdDevTime += varTime * varTime;
+
+                    double iterBandwidthGbs = (t.numBytes / 1.0E9) / r.perIterMsec[i] * 1000.0f;
+                    double const varBw      = fabs(iterBandwidthGbs - r.avgBandwidthGbPerSec);
+                    stdDevBw += varBw * varBw;
+                }
+                stdDevTime = sqrt(stdDevTime / numTimedIterations);
+                stdDevBw   = sqrt(stdDevBw / numTimedIterations);
+
+                // Loop over iterations (fastest to slowest)
+                for (auto& time : times) {
+                    double iterDurationMsec = time.first;
+                    double iterBandwidthGbs = (t.numBytes / 1.0E9) / iterDurationMsec * 1000.0f;
+                    printf("      Iter %03d    %c %8.3f GB/s %c %8.3f ms %c",
+                           time.second,
+                           sep,
+                           iterBandwidthGbs,
+                           sep,
+                           iterDurationMsec,
+                           sep);
+
+                    std::set<int> usedXccs;
+                    if ((time.second - 1) < static_cast<std::int32_t>(r.perIterCUs.size())) {
+                        printf(" CUs:");
+                        for (auto x : r.perIterCUs[time.second - 1]) {
+                            printf(" %02d:%02d", x.first, x.second);
+                            usedXccs.insert(x.first);
+                        }
+                    }
+
+                    printf(" XCCs:");
+                    for (auto x : usedXccs) { printf(" %02d", x); }
+                    printf("\n");
+                }
+                printf("      StandardDev %c %8.3f GB/s %c %8.3f ms %c\n",
+                       sep,
+                       stdDevBw,
+                       sep,
+                       stdDevTime,
+                       sep);
+            }
+        }
+    }
+    printf(" Aggregate (CPU)  %c %8.3f GB/s %c %8.3f ms %c %12lu bytes %c Overhead: %.3f ms\n",
+           sep,
+           results.avgTotalBandwidthGbPerSec,
+           sep,
+           results.avgTotalDurationMsec,
+           sep,
+           results.totalBytesTransferred,
+           sep,
+           results.overheadMsec);
+}
+
+void CheckForError(ErrResult const& error)
+{
+    switch (error.errType) {
+        case ERR_NONE: return;
+        case ERR_WARN: printf("[WARN] %s\n", error.errMsg.c_str()); return;
+        case ERR_FATAL: printf("[ERROR] %s\n", error.errMsg.c_str()); exit(1);
+        default: break;
+    }
+}
+
+void PrintErrors(std::vector<ErrResult> const& errors)
+{
+    bool isFatal = false;
+    for (auto const& err : errors) {
+        printf("[%s] %s\n", err.errType == ERR_FATAL ? "ERROR" : "WARN", err.errMsg.c_str());
+        isFatal |= (err.errType == ERR_FATAL);
+    }
+    if (isFatal) { exit(1); }
+}
+
+auto GetClientVersion() -> const std::string
+{
+    static constexpr auto TB_UNKNOWN_CLIENT_VERSION = std::string_view("Unknown");
+    auto tb_client_version                          = std::string(TRANSFERBENCH_CLIENT_VERSION);
+    if (tb_client_version.empty()) { tb_client_version = std::string(TB_UNKNOWN_CLIENT_VERSION); }
+
+    return tb_client_version;
+}

cmake/modules/Dependencies.cmake

+# MIT License
+#
+# Copyright (c) 2023-25 Advanced Micro Devices, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
+#
+
+# Test dependencies
+#==================================================================================================
+include(FetchContent)
+
+set(ROCM_WARN_TOOLCHAIN_VAR OFF CACHE BOOL "")
+
+# Find or download/install rocm-cmake project
+#==================================================================================================
+find_package(ROCmCMakeBuildTools 0.11.0 CONFIG QUIET PATHS "${ROCM_PATH}")
+if((NOT ROCmCMakeBuildTools_FOUND) OR INSTALL_DEPENDENCIES)
+    message(STATUS "ROCmCMakeBuildTools not found. Checking for ROCM (deprecated)")
+    find_package(ROCM 0.7.3 CONFIG QUIET PATHS "${ROCM_PATH}") # deprecated fallback
+    if((NOT ROCM_FOUND) OR INSTALL_DEPENDENCIES)
+        message(STATUS "ROCM (deprecated) not found. Downloading and building ROCmCMakeBuildTools")
+        set(PROJECT_EXTERN_DIR ${CMAKE_CURRENT_BINARY_DIR}/extern)
+        set(rocm_cmake_tag "rocm-6.4.0" CACHE STRING "rocm-cmake tag to download")
+        FetchContent_Declare(
+            rocm-cmake
+            GIT_REPOSITORY https://github.com/ROCm/rocm-cmake.git
+            GIT_TAG ${rocm_cmake_tag}
+            SOURCE_SUBDIR "DISABLE ADDING TO BUILD"
+        )
+        FetchContent_MakeAvailable(rocm-cmake)
+        message(STATUS "rocm-cmake_SOURCE_DIR: ${rocm-cmake_SOURCE_DIR}")
+        find_package(ROCmCMakeBuildTools CONFIG REQUIRED NO_DEFAULT_PATH PATHS "${rocm-cmake_SOURCE_DIR}")
+        message(STATUS "Found ROCmCmakeBuildTools version: ${ROCmCMakeBuildTools_VERSION}")
+    endif()
+elseif(ROCmCMakeBuildTools_FOUND)
+    message(STATUS "Found ROCmCmakeBuildTools version: ${ROCmCMakeBuildTools_VERSION}")
+endif()
+
+
+# Find available local ROCM targets
+# NOTE: This will eventually be part of ROCm-CMake and should be removed at that time
+#==================================================================================================
+function(rocm_local_targets VARIABLE)
+    set(${VARIABLE} "NOTFOUND" PARENT_SCOPE)
+    find_program(_rocm_agent_enumerator rocm_agent_enumerator HINTS /opt/rocm/bin ENV ROCM_PATH)
+    if(NOT _rocm_agent_enumerator STREQUAL "_rocm_agent_enumerator-NOTFOUND")
+        execute_process(
+            COMMAND "${_rocm_agent_enumerator}"
+            RESULT_VARIABLE _found_agents
+            OUTPUT_VARIABLE _rocm_agents
+            ERROR_QUIET
+        )
+        if (_found_agents EQUAL 0)
+            string(REPLACE "\n" ";" _rocm_agents "${_rocm_agents}")
+            unset(result)
+            foreach (agent IN LISTS _rocm_agents)
+                if (NOT agent STREQUAL "gfx000")
+                    list(APPEND result "${agent}")
+                endif()
+            endforeach()
+            if(result)
+                list(REMOVE_DUPLICATES result)
+                set(${VARIABLE} "${result}" PARENT_SCOPE)
+            endif()
+        endif()
+    endif()
+endfunction()
+
+include(ROCMSetupVersion)
+include(ROCMCreatePackage)
+include(ROCMInstallTargets)
+include(ROCMPackageConfigHelpers)
+include(ROCMInstallSymlinks)
+include(ROCMCheckTargetIds)
+include(ROCMClients)
+include(ROCMHeaderWrapper)

cmake/modules/FindNUMA.cmake

+# Copyright (c) 2020 - 2021 Advanced Micro Devices, Inc. All rights reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+find_path(NUMA_INCLUDE_DIR numa.h)
+find_library(NUMA_LIBRARIES numa)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(NUMA
+  DEFAULT_MSG
+  NUMA_LIBRARIES NUMA_INCLUDE_DIR)
+
+mark_as_advanced(NUMA_LIBRARIES NUMA_INCLUDE_DIR)

cmake/modules/FindPackageMessage.cmake

+# Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
+# file Copyright.txt or https://cmake.org/licensing for details.
+
+#[=======================================================================[.rst:
+FindPackageMessage
+------------------
+
+.. code-block:: cmake
+
+  find_package_message(<name> "message for user" "find result details")
+
+This function is intended to be used in FindXXX.cmake modules files.
+It will print a message once for each unique find result.  This is
+useful for telling the user where a package was found.  The first
+argument specifies the name (XXX) of the package.  The second argument
+specifies the message to display.  The third argument lists details
+about the find result so that if they change the message will be
+displayed again.  The macro also obeys the QUIET argument to the
+find_package command.
+
+Example:
+
+.. code-block:: cmake
+
+  if(X11_FOUND)
+    find_package_message(X11 "Found X11: ${X11_X11_LIB}"
+      "[${X11_X11_LIB}][${X11_INCLUDE_DIR}]")
+  else()
+   ...
+  endif()
+#]=======================================================================]
+
+function(find_package_message pkg msg details)
+  # Avoid printing a message repeatedly for the same find result.
+  if(NOT ${pkg}_FIND_QUIETLY)
+    string(REPLACE "\n" "" details "${details}")
+    set(DETAILS_VAR FIND_PACKAGE_MESSAGE_DETAILS_${pkg})
+    if(NOT "${details}" STREQUAL "${${DETAILS_VAR}}")
+      # The message has not yet been printed.
+      message(STATUS "${msg}")
+
+      # Save the find details in the cache to avoid printing the same
+      # message again.
+      set("${DETAILS_VAR}" "${details}"
+        CACHE INTERNAL "Details about finding ${pkg}")
+    endif()
+  endif()
+endfunction()
\ No newline at end of file

cmake/tbclient_version.hpp.in

+/*
+MIT License
+
+Copyright (c) Advanced Micro Devices, Inc. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+*/
+
+#pragma once
+
+/*
+    CMake auto-generated file: Do not edit it.
+*/
+#define TRANSFERBENCH_CLIENT_VERSION "@TRANSFERBENCH_CLIENT_TARGET_VERSION@"
+