nic block logging

lib/simbricks/nicbm/nicbm.cc

@@ -35,13 +35,18 @@
 #include <ctime>
 #include <iostream>
 
+#include <fstream>
+#include <iterator>
+#include <chrono>
+
 extern "C" {
 #include <simbricks/proto/base.h>
 }
 
 //#define DEBUG_NICBM 1
-#define STAT_NICBM 1
+// #define STAT_NICBM 1
 #define DMA_MAX_PENDING 64
+#define NICBM_BLOCK_LOGGING
 
 namespace nicbm {
 
@@ -68,6 +73,11 @@ static uint64_t s_n2d_poll_sync = 0;
 static int stat_flag = 0;
 #endif
 
+#ifdef NICBM_BLOCK_LOGGING
+static bool working_flag_host = false;
+static bool working_flag_net = false;
+#endif
+
 static void sigint_handler(int dummy) {
   exiting = 1;
 }
@@ -316,7 +326,14 @@ void Runner::EthSend(const void *data, size_t len) {
       SIMBRICKS_PROTO_NET_D2N_MSG_SEND | SIMBRICKS_PROTO_NET_D2N_OWN_NET;
 }
 
-void Runner::PollH2D() {
+int64_t rdtsc_cycle() { return __builtin_ia32_rdtsc(); }
+
+int64_t get_time() {
+  using namespace std::chrono;
+  return duration_cast<microseconds>(steady_clock::now().time_since_epoch()).count();
+}
+
+void Runner::PollH2D(std::vector<int64_t> *block_logging) {
   volatile union SimbricksProtoPcieH2D *msg =
       SimbricksNicIfH2DPoll(&nsparams_, main_time);
   uint8_t type;
@@ -328,6 +345,13 @@ void Runner::PollH2D() {
   }
 #endif
 
+#ifdef NICBM_BLOCK_LOGGING
+    if (working_flag_host != SimbricksNicIfH2DPollStatus()) {
+      block_logging->push_back(rdtsc_cycle());
+      working_flag_host = !working_flag_host;
+    }
+#endif
+
   if (msg == NULL)
     return;
 
@@ -377,7 +401,7 @@ void Runner::PollH2D() {
   SimbricksNicIfH2DNext();
 }
 
-void Runner::PollN2D() {
+void Runner::PollN2D(std::vector<int64_t> *block_logging) {
   volatile union SimbricksProtoNetN2D *msg =
       SimbricksNicIfN2DPoll(&nsparams_, main_time);
   uint8_t t;
@@ -389,6 +413,13 @@ void Runner::PollN2D() {
   }
 #endif
 
+#ifdef NICBM_BLOCK_LOGGING
+    if (working_flag_net != SimbricksNicIfN2DPollStatus()) {
+      block_logging->push_back(rdtsc_cycle());
+      working_flag_net = !working_flag_net;
+    }
+#endif
+
   if (msg == NULL)
     return;
 
@@ -520,6 +551,9 @@ int Runner::RunMain(int argc, char *argv[]) {
 
   bool is_sync = nsparams_.sync_pci || nsparams_.sync_eth;
 
+  std::vector<int64_t> host_block_logging = {rdtsc_cycle()};
+  std::vector<int64_t> net_block_logging = {rdtsc_cycle()};
+
   while (!exiting) {
     while (SimbricksNicIfSync(&nsparams_, main_time)) {
       fprintf(stderr, "warn: SimbricksNicIfSync failed (t=%lu)\n", main_time);
@@ -527,8 +561,8 @@ int Runner::RunMain(int argc, char *argv[]) {
     SimbricksNicIfAdvanceEpoch(&nsparams_, main_time);
 
     do {
-      PollH2D();
-      PollN2D();
+      PollH2D(&host_block_logging);
+      PollN2D(&net_block_logging);
       EventTrigger();
 
       if (is_sync) {
@@ -587,6 +621,20 @@ int Runner::RunMain(int argc, char *argv[]) {
       (double)(s_h2d_poll_sync + s_n2d_poll_sync) / (s_h2d_poll_suc + s_n2d_poll_suc));
 #endif
 
+  std::clock_t total = std::clock();
+
+  std::string file_name = std::string(nsparams_.pci_socket_path) + std::string("_nicbm_host_block_logging.txt");
+  std::ofstream output_file(file_name);
+  output_file << "CLOCKS_PER_SEC: " << CLOCKS_PER_SEC << '\n';
+  output_file << "Total clocks: " << total << '\n';
+  std::copy(host_block_logging.begin(), host_block_logging.end(), std::ostream_iterator<int64_t>(output_file, "\n"));
+
+  file_name = std::string(nsparams_.pci_socket_path) + std::string("_nicbm_net_block_logging.txt");
+  std::ofstream output_file1(file_name);
+  output_file1 << "CLOCKS_PER_SEC: " << CLOCKS_PER_SEC << '\n';
+  output_file1 << "Total clocks: " << total << '\n';
+  std::copy(net_block_logging.begin(), net_block_logging.end(), std::ostream_iterator<int64_t>(output_file1, "\n"));
+
   SimbricksNicIfCleanup();
   return 0;
 }

lib/simbricks/nicbm/nicbm.h

@@ -29,6 +29,7 @@
 #include <cstring>
 #include <deque>
 #include <set>
+#include <vector>
 
 extern "C" {
 #include <simbricks/nicif/nicif.h>
@@ -133,10 +134,10 @@ class Runner {
   void H2DReadcomp(volatile struct SimbricksProtoPcieH2DReadcomp *rc);
   void H2DWritecomp(volatile struct SimbricksProtoPcieH2DWritecomp *wc);
   void H2DDevctrl(volatile struct SimbricksProtoPcieH2DDevctrl *dc);
-  void PollH2D();
+  void PollH2D(std::vector<int64_t> *block_logging);
 
   void EthRecv(volatile struct SimbricksProtoNetN2DRecv *recv);
-  void PollN2D();
+  void PollN2D(std::vector<int64_t> *block_logging);
 
   bool EventNext(uint64_t &retval);
   void EventTrigger();

lib/simbricks/nicif/nicif.c

@@ -74,6 +74,12 @@ static int shm_fd = -1;
 static int pci_cfd = -1;
 static int eth_cfd = -1;
 
+static bool pci_received = false;
+static bool eth_received = false;
+
+bool SimbricksNicIfH2DPollStatus() {return pci_received;}
+bool SimbricksNicIfN2DPollStatus() {return eth_received;}
+
 static int accept_pci(struct SimbricksProtoPcieDevIntro *di, int pci_lfd,
                       int *sync_pci) {
   if ((pci_cfd = accept(pci_lfd, NULL, NULL)) < 0) {
@@ -365,8 +371,11 @@ volatile union SimbricksProtoPcieH2D *SimbricksNicIfH2DPoll(
 
   /* message not ready */
   if ((msg->dummy.own_type & SIMBRICKS_PROTO_PCIE_H2D_OWN_MASK) !=
-      SIMBRICKS_PROTO_PCIE_H2D_OWN_DEV)
+      SIMBRICKS_PROTO_PCIE_H2D_OWN_DEV) {
+    pci_received = false;
     return NULL;
+  }
+  pci_received = true;
 
   /* if in sync mode, wait till message is ready */
   pci_last_rx_time = msg->dummy.timestamp;
@@ -413,8 +422,11 @@ volatile union SimbricksProtoNetN2D *SimbricksNicIfN2DPoll(
 
   /* message not ready */
   if ((msg->dummy.own_type & SIMBRICKS_PROTO_NET_N2D_OWN_MASK) !=
-      SIMBRICKS_PROTO_NET_N2D_OWN_DEV)
+      SIMBRICKS_PROTO_NET_N2D_OWN_DEV) {
+    eth_received = false;
     return NULL;
+  }
+  eth_received = true;
 
   /* if in sync mode, wait till message is ready */
   eth_last_rx_time = msg->dummy.timestamp;

lib/simbricks/nicif/nicif.h

@@ -28,6 +28,10 @@
 #include <simbricks/proto/network.h>
 #include <simbricks/proto/pcie.h>
 
+#include <stdbool.h>
+bool SimbricksNicIfH2DPollStatus();
+bool SimbricksNicIfN2DPollStatus();
+
 struct SimbricksNicIfParams {
   const char *pci_socket_path;
   const char *eth_socket_path;