net_switch.cc 10.7 KB
Newer Older
Antoine Kaufmann's avatar
Antoine Kaufmann committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
/*
 * Copyright 2021 Max Planck Institute for Software Systems, and
 * National University of Singapore
 *
 * Permission is hereby granted, free of charge, to any person obtaining
 * a copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be
 * included in all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */

25
#include <unistd.h>
Jialin Li's avatar
Jialin Li committed
26
#include <pcap/pcap.h>
27
28
29
#include <linux/ip.h>
#include <linux/if_ether.h>
#include <arpa/inet.h>
30
31
32
33

#include <cassert>
#include <climits>
#include <csignal>
Jialin Li's avatar
Jialin Li committed
34
35
36
37
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <unordered_map>
38
#include <vector>
Jialin Li's avatar
Jialin Li committed
39

40
41
#include <unistd.h>
#include <simbricks/proto/npy.hpp>
Zhiqiang Xie's avatar
Zhiqiang Xie committed
42

Jialin Li's avatar
Jialin Li committed
43
extern "C" {
44
#include <simbricks/netif/netif.h>
45
#include <simbricks/proto/base.h>
Jialin Li's avatar
Jialin Li committed
46
47
};

48
//#define NETSWITCH_DEBUG
Zhiqiang Xie's avatar
Zhiqiang Xie committed
49
50
//#define NETSWITCH_STAT
#define NETSWITCH_BLOCK_LOGGING
51

Antoine Kaufmann's avatar
Antoine Kaufmann committed
52
53
static uint64_t sync_period = (500 * 1000ULL);  // 500ns
static uint64_t eth_latency = (500 * 1000ULL);  // 500ns
Jialin Li's avatar
Jialin Li committed
54
static pcap_dumper_t *dumpfile = nullptr;
Jialin Li's avatar
Jialin Li committed
55

Hejing Li's avatar
Hejing Li committed
56
57
58
59
60
61
62
63
64
65
66
67
68

#ifdef NETSWITCH_STAT
static uint64_t d2n_poll_total = 0;
static uint64_t d2n_poll_suc = 0;
static uint64_t d2n_poll_sync = 0;

static uint64_t s_d2n_poll_total = 0;
static uint64_t s_d2n_poll_suc = 0;
static uint64_t s_d2n_poll_sync = 0;

static int stat_flag = 0;
#endif

Zhiqiang Xie's avatar
Zhiqiang Xie committed
69
70
71
72
73
74
#ifdef NETSWITCH_BLOCK_LOGGING
static bool working_flag = false;
// #define SYNC_IDLE
int64_t rdtsc_cycle() { return __builtin_ia32_rdtsc(); }
#endif

Jialin Li's avatar
Jialin Li committed
75
76
/* MAC address type */
struct MAC {
77
  const volatile uint8_t *data;
Jialin Li's avatar
Jialin Li committed
78

79
  explicit MAC(const volatile uint8_t *data) : data(data) {
80
  }
Jialin Li's avatar
Jialin Li committed
81

82
83
84
85
86
  bool operator==(const MAC &other) const {
    for (int i = 0; i < 6; i++) {
      if (data[i] != other.data[i]) {
        return false;
      }
Jialin Li's avatar
Jialin Li committed
87
    }
88
89
    return true;
  }
Jialin Li's avatar
Jialin Li committed
90
91
};
namespace std {
Antoine Kaufmann's avatar
Antoine Kaufmann committed
92
template <>
93
94
struct hash<MAC> {
  size_t operator()(const MAC &m) const {
Antoine Kaufmann's avatar
Antoine Kaufmann committed
95
96
    size_t res = 0;
    for (int i = 0; i < 6; i++) {
97
      res = (res << 4) | (res ^ m.data[i]);
Antoine Kaufmann's avatar
Antoine Kaufmann committed
98
99
    }
    return res;
100
  }
Antoine Kaufmann's avatar
Antoine Kaufmann committed
101
102
};
}  // namespace std
Jialin Li's avatar
Jialin Li committed
103
104
105
106
107
108

/* Global variables */
static uint64_t cur_ts = 0;
static int exiting = 0;
static const volatile uint8_t bcast[6] = {0xFF};
static const MAC bcast_addr(bcast);
109
static std::vector<struct SimbricksNetIf> nsifs;
Jialin Li's avatar
Jialin Li committed
110
111
static std::unordered_map<MAC, int> mac_table;

112
113
static void sigint_handler(int dummy) {
  exiting = 1;
Jialin Li's avatar
Jialin Li committed
114
115
}

Hejing Li's avatar
Hejing Li committed
116
117
118
119
120
121
122
123
124
125
static void sigusr1_handler(int dummy) {
  fprintf(stderr, "main_time = %lu\n", cur_ts);
}

#ifdef NETSWITCH_STAT
static void sigusr2_handler(int dummy) {
  stat_flag = 1;
}
#endif

126
static void forward_pkt(volatile struct SimbricksProtoNetD2NSend *tx,
127
                        size_t port) {
128
  volatile union SimbricksProtoNetN2D *msg_to;
Jialin Li's avatar
Jialin Li committed
129
130
  struct pcap_pkthdr ph;

131

Jialin Li's avatar
Jialin Li committed
132
133
134
135
136
137
138
139
140
  // log to pcap file if initialized
  if (dumpfile) {
      memset(&ph, 0, sizeof(ph));
      ph.ts.tv_sec = cur_ts / 1000000000000ULL;
      ph.ts.tv_usec = (cur_ts % 1000000000000ULL) / 1000ULL;
      ph.caplen = tx->len;
      ph.len = tx->len;
      pcap_dump((unsigned char *)dumpfile, &ph, (unsigned char *)tx->data);
  }
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
  // print sending tick: [packet type] source_IP -> dest_IP len:
  
#ifdef NETSWITCH_DEBUG
  uint16_t eth_proto;
  struct ethhdr *hdr;
  struct iphdr *iph;
  hdr = (struct ethhdr*)tx->data; 
  eth_proto = ntohs(hdr->h_proto);
  iph = (struct iphdr *)(hdr + 1);
  fprintf(stderr, "%20lu: ", cur_ts);
  if (eth_proto == ETH_P_IP){
    fprintf(stderr, "[ IP] ");
    
  } 
  else if(eth_proto == ETH_P_ARP){
    fprintf(stderr, "[ARP] ");
  } 
  else{
    fprintf(stderr, "unkwon eth type\n");
  }

  fprintf(stderr, "%8X -> %8X len: %lu\n ", iph->saddr, iph->daddr, iph->tot_len + sizeof(struct ethhdr));
#endif

Jialin Li's avatar
Jialin Li committed
165

166
  msg_to = SimbricksNetIfN2DAlloc(&nsifs[port], cur_ts, eth_latency);
167
  if (msg_to != NULL) {
168
    volatile struct SimbricksProtoNetN2DRecv *rx;
169
170
171
172
173
174
    rx = &msg_to->recv;
    rx->len = tx->len;
    rx->port = 0;
    memcpy((void *)rx->data, (void *)tx->data, tx->len);

    // WMB();
175
176
    rx->own_type =
        SIMBRICKS_PROTO_NET_N2D_MSG_RECV | SIMBRICKS_PROTO_NET_N2D_OWN_DEV;
177
178
179
  } else {
    fprintf(stderr, "forward_pkt: dropping packet\n");
  }
Jialin Li's avatar
Jialin Li committed
180
181
}

Zhiqiang Xie's avatar
Zhiqiang Xie committed
182
183
static void switch_pkt(struct SimbricksNetIf *nsif, size_t iport,
    std::vector<int64_t> *block_logging) {
184
  volatile union SimbricksProtoNetD2N *msg_from =
185
      SimbricksNetIfD2NPoll(nsif, cur_ts);
Hejing Li's avatar
Hejing Li committed
186
187
188
189
190
191
192
#ifdef NETSWITCH_STAT
  d2n_poll_total += 1;
  if (stat_flag){
    s_d2n_poll_total += 1;
  }
#endif

193
  if (msg_from == NULL) {
Zhiqiang Xie's avatar
Zhiqiang Xie committed
194
195
196
197
198
199
#ifdef NETSWITCH_BLOCK_LOGGING
    if (working_flag) {
      block_logging->push_back(rdtsc_cycle());
      working_flag = false;
    }
#endif
200
201
202
    return;
  }

Hejing Li's avatar
Hejing Li committed
203
204
205
206
207
208
209
#ifdef NETSWITCH_STAT
  d2n_poll_suc += 1;
  if (stat_flag){
    s_d2n_poll_suc += 1;
  }
#endif

210
211
  uint8_t type = msg_from->dummy.own_type & SIMBRICKS_PROTO_NET_D2N_MSG_MASK;
  if (type == SIMBRICKS_PROTO_NET_D2N_MSG_SEND) {
Zhiqiang Xie's avatar
Zhiqiang Xie committed
212
213
214
215
216
217
#ifdef NETSWITCH_BLOCK_LOGGING
    if (!working_flag) {
      block_logging->push_back(rdtsc_cycle());
      working_flag = true;
    }
#endif
218
    volatile struct SimbricksProtoNetD2NSend *tx;
219
220
221
222
223
224
    tx = &msg_from->send;
    // Get MAC addresses
    MAC dst(tx->data), src(tx->data + 6);
    // MAC learning
    if (!(src == bcast_addr)) {
      mac_table[src] = iport;
Jialin Li's avatar
Jialin Li committed
225
    }
226
227
    // L2 forwarding
    if (mac_table.count(dst) > 0) {
228
      size_t eport = mac_table.at(dst);
229
      forward_pkt(tx, eport);
Jialin Li's avatar
Jialin Li committed
230
    } else {
231
      // Broadcast
232
      for (size_t eport = 0; eport < nsifs.size(); eport++) {
233
234
235
236
237
        if (eport != iport) {
          // Do not forward to ingress port
          forward_pkt(tx, eport);
        }
      }
Jialin Li's avatar
Jialin Li committed
238
    }
239
  } else if (type == SIMBRICKS_PROTO_NET_D2N_MSG_SYNC) {
Zhiqiang Xie's avatar
Zhiqiang Xie committed
240
241
242
243
244
245
246
247
248
249
250
251
252
#ifdef NETSWITCH_BLOCK_LOGGING
#ifdef SYNC_IDLE
    if (working_flag) {
      block_logging->push_back(rdtsc_cycle());
      working_flag = false;
    }
#else
    if (!working_flag) {
      block_logging->push_back(rdtsc_cycle());
      working_flag = true;
    }
#endif
#endif
Hejing Li's avatar
Hejing Li committed
253
254
255
256
257
258
#ifdef NETSWITCH_STAT
    d2n_poll_sync += 1;
    if (stat_flag){
      s_d2n_poll_sync += 1;
    }
#endif
259
260
261
262
  } else {
    fprintf(stderr, "switch_pkt: unsupported type=%u\n", type);
    abort();
  }
263
  SimbricksNetIfD2NDone(nsif, msg_from);
Jialin Li's avatar
Jialin Li committed
264
265
}

266
267
268
int main(int argc, char *argv[]) {
  int c;
  int bad_option = 0;
269
  int sync_eth = 1;
270
  int sync_mode = SIMBRICKS_PROTO_SYNC_SIMBRICKS;
Jialin Li's avatar
Jialin Li committed
271
  pcap_t *pc = nullptr;
272
273

  // Parse command line argument
274
  while ((c = getopt(argc, argv, "s:uS:E:m:p:")) != -1 && !bad_option) {
275
276
    switch (c) {
      case 's': {
277
        struct SimbricksNetIf nsif;
278
279
        int sync = sync_eth;
        fprintf(stderr, "Switch connecting to: %s\n", optarg);
280
        if (SimbricksNetIfInit(&nsif, optarg, &sync) != 0) {
281
282
          fprintf(stderr, "connecting to %s failed\n", optarg);
          return EXIT_FAILURE;
Jialin Li's avatar
Jialin Li committed
283
        }
284
285
286
287
        nsifs.push_back(nsif);
        break;
      }

288
289
290
291
      case 'u':
        sync_eth = 0;
        break;

292
293
294
295
296
297
298
299
300
301
      case 'S':
        sync_period = strtoull(optarg, NULL, 0) * 1000ULL;
        break;

      case 'E':
        eth_latency = strtoull(optarg, NULL, 0) * 1000ULL;
        break;

      case 'm':
        sync_mode = strtol(optarg, NULL, 0);
302
        assert(sync_mode == SIMBRICKS_PROTO_SYNC_SIMBRICKS ||
303
               sync_mode == SIMBRICKS_PROTO_SYNC_BARRIER);
304
305
        break;

Jialin Li's avatar
Jialin Li committed
306
307
308
309
310
311
312
313
314
315
316
      case 'p':
        pc = pcap_open_dead_with_tstamp_precision(DLT_EN10MB, 65535,
                                                  PCAP_TSTAMP_PRECISION_NANO);
        if (pc == nullptr) {
            perror("pcap_open_dead failed");
            return EXIT_FAILURE;
        }

        dumpfile = pcap_dump_open(pc, optarg);
        break;

317
318
319
320
      default:
        fprintf(stderr, "unknown option %c\n", c);
        bad_option = 1;
        break;
Jialin Li's avatar
Jialin Li committed
321
    }
322
323
324
325
326
327
328
329
330
331
332
  }

  if (nsifs.empty() || bad_option) {
    fprintf(stderr,
            "Usage: net_switch [-S SYNC-PERIOD] [-E ETH-LATENCY] "
            "-s SOCKET-A [-s SOCKET-B ...]\n");
    return EXIT_FAILURE;
  }

  signal(SIGINT, sigint_handler);
  signal(SIGTERM, sigint_handler);
Hejing Li's avatar
Hejing Li committed
333
334
335
336
337
338
  signal(SIGUSR1, sigusr1_handler);

#ifdef NETSWITCH_STAT
  signal(SIGUSR2, sigusr2_handler);
#endif

339
340

  printf("start polling\n");
341
342
  // to reduce the overhead, replace 'block_logging' with statically allocated memory and
  // employ another thread to log data into disks
Zhiqiang Xie's avatar
Zhiqiang Xie committed
343
  std::vector<int64_t> block_logging = {rdtsc_cycle()};
344
345
346
  while (!exiting) {
    // Sync all interfaces
    for (auto &nsif : nsifs) {
347
      if (SimbricksNetIfN2DSync(&nsif, cur_ts, eth_latency, sync_period,
348
                                sync_mode) != 0) {
349
        fprintf(stderr, "SimbricksNetIfN2DSync failed\n");
350
351
        abort();
      }
Jialin Li's avatar
Jialin Li committed
352
    }
353
    SimbricksNetIfAdvanceEpoch(cur_ts, sync_period, sync_mode);
354
355
356
357
358

    // Switch packets
    uint64_t min_ts;
    do {
      min_ts = ULLONG_MAX;
359
      for (size_t port = 0; port < nsifs.size(); port++) {
360
        auto &nsif = nsifs.at(port);
Zhiqiang Xie's avatar
Zhiqiang Xie committed
361
        switch_pkt(&nsif, port, &block_logging);
362
        if (nsif.sync) {
363
          uint64_t ts = SimbricksNetIfD2NTimestamp(&nsif);
364
          min_ts = ts < min_ts ? ts : min_ts;
Jialin Li's avatar
Jialin Li committed
365
        }
366
367
368
369
370
      }
    } while (!exiting && (min_ts <= cur_ts));

    // Update cur_ts
    if (min_ts < ULLONG_MAX) {
371
      cur_ts = SimbricksNetIfAdvanceTime(min_ts, sync_period, sync_mode);
Jialin Li's avatar
Jialin Li committed
372
    }
373
  }
Jialin Li's avatar
Jialin Li committed
374

Hejing Li's avatar
Hejing Li committed
375
376
377
378
379
380
381
382
383
384
385
386
387
388
#ifdef NETSWITCH_STAT
  fprintf(stderr, "%20s: %22lu %20s: %22lu  poll_suc_rate: %f\n",
          "d2n_poll_total", d2n_poll_total, "d2n_poll_suc", d2n_poll_suc,
          (double)d2n_poll_suc / d2n_poll_total);
  fprintf(stderr, "%65s: %22lu  sync_rate: %f\n", "d2n_poll_sync",
          d2n_poll_sync, (double)d2n_poll_sync / d2n_poll_suc);

  fprintf(stderr, "%20s: %22lu %20s: %22lu  poll_suc_rate: %f\n",
          "s_d2n_poll_total", s_d2n_poll_total, "s_d2n_poll_suc", s_d2n_poll_suc,
          (double)s_d2n_poll_suc / s_d2n_poll_total);
  fprintf(stderr, "%65s: %22lu  sync_rate: %f\n", "s_d2n_poll_sync",
          s_d2n_poll_sync, (double)s_d2n_poll_sync / s_d2n_poll_suc);
#endif

389
390
391
392
  std::string pid = std::to_string(getpid());
  std::string file_name = std::string("net_switch_block_logging_") + pid + std::string(".npy");
  const long unsigned npy_shape[1] = {block_logging.size()};
  npy::SaveArrayAsNumpy(file_name, false, 1, npy_shape, block_logging);
Zhiqiang Xie's avatar
Zhiqiang Xie committed
393

394
  return 0;
Jialin Li's avatar
Jialin Li committed
395
}