net_switch.cc 12.7 KB
Newer Older
Antoine Kaufmann's avatar
Antoine Kaufmann committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
/*
 * Copyright 2021 Max Planck Institute for Software Systems, and
 * National University of Singapore
 *
 * Permission is hereby granted, free of charge, to any person obtaining
 * a copy of this software and associated documentation files (the
 * "Software"), to deal in the Software without restriction, including
 * without limitation the rights to use, copy, modify, merge, publish,
 * distribute, sublicense, and/or sell copies of the Software, and to
 * permit persons to whom the Software is furnished to do so, subject to
 * the following conditions:
 *
 * The above copyright notice and this permission notice shall be
 * included in all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
 * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */

25
#include <unistd.h>
Jialin Li's avatar
Jialin Li committed
26
#include <pcap/pcap.h>
27
28
29
#include <linux/ip.h>
#include <linux/if_ether.h>
#include <arpa/inet.h>
30
31
32
33

#include <cassert>
#include <climits>
#include <csignal>
Jialin Li's avatar
Jialin Li committed
34
35
36
37
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <unordered_map>
38
#include <vector>
Jialin Li's avatar
Jialin Li committed
39

40
#include <simbricks/base/cxxatomicfix.h>
Jialin Li's avatar
Jialin Li committed
41
extern "C" {
42
#include <simbricks/network/if.h>
43
#include <simbricks/nicif/nicif.h>
Jialin Li's avatar
Jialin Li committed
44
45
};

46
//#define NETSWITCH_DEBUG
Hejing Li's avatar
Hejing Li committed
47
#define NETSWITCH_STAT
48

49
struct SimbricksBaseIfParams netParams;
Jialin Li's avatar
Jialin Li committed
50
static pcap_dumper_t *dumpfile = nullptr;
Jialin Li's avatar
Jialin Li committed
51

Hejing Li's avatar
Hejing Li committed
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
#ifdef NETSWITCH_STAT
#endif

#ifdef NETSWITCH_STAT
static uint64_t d2n_poll_total = 0;
static uint64_t d2n_poll_suc = 0;
static uint64_t d2n_poll_sync = 0;

static uint64_t s_d2n_poll_total = 0;
static uint64_t s_d2n_poll_suc = 0;
static uint64_t s_d2n_poll_sync = 0;

static int stat_flag = 0;
#endif

Jialin Li's avatar
Jialin Li committed
67
68
/* MAC address type */
struct MAC {
69
  const uint8_t *data;
Jialin Li's avatar
Jialin Li committed
70

71
  explicit MAC(const uint8_t *data) : data(data) {
72
  }
Jialin Li's avatar
Jialin Li committed
73

74
75
76
77
78
  bool operator==(const MAC &other) const {
    for (int i = 0; i < 6; i++) {
      if (data[i] != other.data[i]) {
        return false;
      }
Jialin Li's avatar
Jialin Li committed
79
    }
80
81
    return true;
  }
Jialin Li's avatar
Jialin Li committed
82
83
};
namespace std {
Antoine Kaufmann's avatar
Antoine Kaufmann committed
84
template <>
85
86
struct hash<MAC> {
  size_t operator()(const MAC &m) const {
Antoine Kaufmann's avatar
Antoine Kaufmann committed
87
88
    size_t res = 0;
    for (int i = 0; i < 6; i++) {
89
      res = (res << 4) | (res ^ m.data[i]);
Antoine Kaufmann's avatar
Antoine Kaufmann committed
90
91
    }
    return res;
92
  }
Antoine Kaufmann's avatar
Antoine Kaufmann committed
93
94
};
}  // namespace std
Jialin Li's avatar
Jialin Li committed
95

96

97
/** Abstract base switch port */
98
class Port {
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
 public:
  enum RxPollState {
    kRxPollSuccess = 0,
    kRxPollFail = 1,
    kRxPollSync = 2,
  };

  virtual ~Port() = default;

  virtual bool Connect(const char *path, int sync) = 0;
  virtual bool IsSync() = 0;
  virtual void Sync(uint64_t cur_ts) = 0;
  virtual uint64_t NextTimestamp() = 0;
  virtual enum RxPollState RxPacket(
      const void *& data, size_t &len, uint64_t cur_ts) = 0;
  virtual void RxDone() = 0;
  virtual bool TxPacket(const void *data, size_t len, uint64_t cur_ts) = 0;
};

118

119
120
/** Normal network switch port (conneting to a NIC) */
class NetPort : public Port {
121
 protected:
122
123
124
  struct SimbricksNetIf netifObj_;
  struct SimbricksNetIf *netif_;
  volatile union SimbricksProtoNetMsg *rx_;
125
126
127
  int sync_;

 public:
128
129
  NetPort() : netif_(&netifObj_), rx_(nullptr), sync_(0) {
    memset(&netifObj_, 0, sizeof(netifObj_));
130
131
  }

132
133
  NetPort(const NetPort &other) : netifObj_(other.netifObj_),
      netif_(&netifObj_), rx_(other.rx_), sync_(other.sync_) {}
134

135
  virtual bool Connect(const char *path, int sync) override {
136
    sync_ = sync;
137
    return SimbricksNetIfInit(netif_, &netParams, path, &sync_) == 0;
138
139
  }

140
  virtual bool IsSync() override {
141
142
143
    return sync_;
  }

144
  virtual void Sync(uint64_t cur_ts) override {
145
    while (SimbricksNetIfOutSync(netif_, cur_ts));
146
147
  }

148
  virtual uint64_t NextTimestamp() override {
149
    return SimbricksNetIfInTimestamp(netif_);
150
151
152
  }

  virtual enum RxPollState RxPacket(
153
      const void *& data, size_t &len, uint64_t cur_ts) override {
154
155
    assert(rx_ == nullptr);

156
    rx_ = SimbricksNetIfInPoll(netif_, cur_ts);
157
158
159
    if (!rx_)
      return kRxPollFail;

160
161
162
163
    uint8_t type = SimbricksNetIfInType(netif_, rx_);
    if (type == SIMBRICKS_PROTO_NET_MSG_PACKET) {
      data = (const void *)rx_->packet.data;
      len = rx_->packet.len;
164
      return kRxPollSuccess;
165
    } else if (type == SIMBRICKS_PROTO_MSG_TYPE_SYNC) {
166
167
168
169
170
171
172
      return kRxPollSync;
    } else {
      fprintf(stderr, "switch_pkt: unsupported type=%u\n", type);
      abort();
    }
  }

173
  virtual void RxDone() override {
174
175
    assert(rx_ != nullptr);

176
    SimbricksNetIfInDone(netif_, rx_);
177
178
179
    rx_ = nullptr;
  }

180
181
  virtual bool TxPacket(
      const void *data, size_t len, uint64_t cur_ts) override {
182
183
    volatile union SimbricksProtoNetMsg *msg_to =
      SimbricksNetIfOutAlloc(netif_, cur_ts);
184
    if (!msg_to && !sync_) {
185
      return false;
186
187
    } else if (!msg_to && sync_) {
      while (!msg_to)
188
        msg_to = SimbricksNetIfOutAlloc(netif_, cur_ts);
189
    }
190
191
    volatile struct SimbricksProtoNetMsgPacket *rx;
    rx = &msg_to->packet;
192
193
194
195
    rx->len = len;
    rx->port = 0;
    memcpy((void *)rx->data, data, len);

196
    SimbricksNetIfOutSend(netif_, msg_to, SIMBRICKS_PROTO_NET_MSG_PACKET);
197
198
199
200
    return true;
  }
};

201
202

/** Hosting network switch port (connected to another network) */
203
class NetHostPort : public NetPort {
204
205
206
207
 protected:
  struct SimbricksNicIf nicif_;

 public:
208
209
  NetHostPort() {
    netif_ = &nicif_.net;
210
211
212
    memset(&nicif_, 0, sizeof(nicif_));
  }

213
214
215
  NetHostPort(const NetHostPort &other) : NetPort(other), nicif_(other.nicif_) {
    netif_ = &nicif_.net;
  }
216
217
218
219
220

  virtual bool Connect(const char *path, int sync) override {
    sync_ = sync;
    std::string shm_path = path;
    shm_path += "-shm";
221
222
223
224
225
226
227
    struct SimbricksBaseIfParams params = netParams;
    netParams.sock_path = path;
    if (!sync)
      netParams.sync_mode = kSimbricksBaseIfSyncDisabled;
    int ret = SimbricksNicIfInit(&nicif_, shm_path.c_str(), &params, nullptr,
        nullptr);
    sync_ = SimbricksBaseIfSyncEnabled(&netif_->base);
228
229
230
231
232
233
234
235
236
    return ret == 0;
  }

  virtual bool IsSync() override {
    return sync_;
  }
};


Jialin Li's avatar
Jialin Li committed
237
238
239
/* Global variables */
static uint64_t cur_ts = 0;
static int exiting = 0;
240
static const uint8_t bcast[6] = {0xFF};
Jialin Li's avatar
Jialin Li committed
241
static const MAC bcast_addr(bcast);
242
static std::vector<Port *> ports;
Jialin Li's avatar
Jialin Li committed
243
244
static std::unordered_map<MAC, int> mac_table;

245
246
static void sigint_handler(int dummy) {
  exiting = 1;
Jialin Li's avatar
Jialin Li committed
247
248
}

Hejing Li's avatar
Hejing Li committed
249
250
251
252
253
254
255
256
257
258
static void sigusr1_handler(int dummy) {
  fprintf(stderr, "main_time = %lu\n", cur_ts);
}

#ifdef NETSWITCH_STAT
static void sigusr2_handler(int dummy) {
  stat_flag = 1;
}
#endif

259
260
static void forward_pkt(const void *pkt_data, size_t pkt_len, size_t port_id,
                        size_t iport_id) {
Jialin Li's avatar
Jialin Li committed
261
  struct pcap_pkthdr ph;
262
  Port &dest_port = *ports[port_id];
263

Jialin Li's avatar
Jialin Li committed
264
265
266
267
268
  // log to pcap file if initialized
  if (dumpfile) {
      memset(&ph, 0, sizeof(ph));
      ph.ts.tv_sec = cur_ts / 1000000000000ULL;
      ph.ts.tv_usec = (cur_ts % 1000000000000ULL) / 1000ULL;
269
270
271
      ph.caplen = pkt_len;
      ph.len = pkt_len;
      pcap_dump((unsigned char *)dumpfile, &ph, (unsigned char *)pkt_data);
Jialin Li's avatar
Jialin Li committed
272
  }
273
  // print sending tick: [packet type] source_IP -> dest_IP len:
274

275
276
277
278
#ifdef NETSWITCH_DEBUG
  uint16_t eth_proto;
  struct ethhdr *hdr;
  struct iphdr *iph;
279
  hdr = (struct ethhdr*)pkt_data;
280
281
  eth_proto = ntohs(hdr->h_proto);
  iph = (struct iphdr *)(hdr + 1);
282
283
284
285
  uint64_t dmac = (*(uint64_t *) hdr->h_dest) & 0xFFFFFFFFFFULL;
  uint64_t smac = (*(uint64_t *) hdr->h_source) & 0xFFFFFFFFFFULL;
  fprintf(stderr, "%20lu: [P %zu -> %zu] %lx -> %lx ", cur_ts, iport_id,
          port_id, smac, dmac);
286
287
  if (eth_proto == ETH_P_IP){
    fprintf(stderr, "[ IP] ");
288
289
    fprintf(stderr, "%8X -> %8X len: %lu\n", iph->saddr, iph->daddr,
            ntohs(iph->tot_len) + sizeof(struct ethhdr));
290
291
  } 
  else if(eth_proto == ETH_P_ARP){
292
293
294
    fprintf(stderr, "[ARP] %8X -> %8X\n",
            *(uint32_t *) ((uint8_t *) pkt_data + 28),
            *(uint32_t *) ((uint8_t *) pkt_data + 38) );
295
296
  } 
  else{
297
    fprintf(stderr, "unknown eth type\n");
298
299
300
  }
#endif

301
302
  if (!dest_port.TxPacket(pkt_data, pkt_len, cur_ts))
    fprintf(stderr, "forward_pkt: dropping packet on port %zu\n", port_id);
Jialin Li's avatar
Jialin Li committed
303
304
}

305
306
307
308
static void switch_pkt(Port &port, size_t iport) {
  const void *pkt_data;
  size_t pkt_len;

Hejing Li's avatar
Hejing Li committed
309
310
311
312
313
314
315
#ifdef NETSWITCH_STAT
  d2n_poll_total += 1;
  if (stat_flag){
    s_d2n_poll_total += 1;
  }
#endif

316
317
  enum Port::RxPollState poll = port.RxPacket(pkt_data, pkt_len, cur_ts);
  if (poll == Port::kRxPollFail) {
318
319
320
    return;
  }

Hejing Li's avatar
Hejing Li committed
321
322
323
324
325
326
327
#ifdef NETSWITCH_STAT
  d2n_poll_suc += 1;
  if (stat_flag){
    s_d2n_poll_suc += 1;
  }
#endif

328
  if (poll == Port::kRxPollSuccess) {
329
    // Get MAC addresses
330
    MAC dst((const uint8_t *)pkt_data), src((const uint8_t *)pkt_data + 6);
331
332
333
    // MAC learning
    if (!(src == bcast_addr)) {
      mac_table[src] = iport;
Jialin Li's avatar
Jialin Li committed
334
    }
335
    // L2 forwarding
336
337
338
    auto i = mac_table.find(dst);
    if (i != mac_table.end()) {
      size_t eport = i->second;
339
      if (eport != iport)
340
        forward_pkt(pkt_data, pkt_len, eport, iport);
Jialin Li's avatar
Jialin Li committed
341
    } else {
342
      // Broadcast
343
      for (size_t eport = 0; eport < ports.size(); eport++) {
344
345
        if (eport != iport) {
          // Do not forward to ingress port
346
          forward_pkt(pkt_data, pkt_len, eport, iport);
347
348
        }
      }
Jialin Li's avatar
Jialin Li committed
349
    }
350
  } else if (poll == Port::kRxPollSync) {
Hejing Li's avatar
Hejing Li committed
351
352
353
354
355
356
#ifdef NETSWITCH_STAT
    d2n_poll_sync += 1;
    if (stat_flag){
      s_d2n_poll_sync += 1;
    }
#endif
357
  } else {
358
    fprintf(stderr, "switch_pkt: unsupported poll result=%u\n", poll);
359
360
    abort();
  }
361
  port.RxDone();
Jialin Li's avatar
Jialin Li committed
362
363
}

364
365
366
int main(int argc, char *argv[]) {
  int c;
  int bad_option = 0;
367
  int sync_eth = 1;
Jialin Li's avatar
Jialin Li committed
368
  pcap_t *pc = nullptr;
369

370
371
  SimbricksNetIfDefaultParams(&netParams);

372
  // Parse command line argument
373
  while ((c = getopt(argc, argv, "s:h:uS:E:p:")) != -1 && !bad_option) {
374
375
    switch (c) {
      case 's': {
376
        NetPort *port = new NetPort;
377
        fprintf(stderr, "Switch connecting to: %s\n", optarg);
378
        if (!port->Connect(optarg, sync_eth)) {
379
380
          fprintf(stderr, "connecting to %s failed\n", optarg);
          return EXIT_FAILURE;
Jialin Li's avatar
Jialin Li committed
381
        }
382
        ports.push_back(port);
383
384
385
        break;
      }

386
387
388
389
390
391
392
393
394
395
396
      case 'h': {
        NetHostPort *port = new NetHostPort;
        fprintf(stderr, "Switch listening on: %s\n", optarg);
        if (!port->Connect(optarg, sync_eth)) {
          fprintf(stderr, "listening on %s failed\n", optarg);
          return EXIT_FAILURE;
        }
        ports.push_back(port);
        break;
      }

397
398
399
400
      case 'u':
        sync_eth = 0;
        break;

401
      case 'S':
402
        netParams.sync_interval = strtoull(optarg, NULL, 0) * 1000ULL;
403
404
405
        break;

      case 'E':
406
        netParams.link_latency = strtoull(optarg, NULL, 0) * 1000ULL;
407
408
        break;

Jialin Li's avatar
Jialin Li committed
409
410
411
412
413
414
415
416
417
418
419
      case 'p':
        pc = pcap_open_dead_with_tstamp_precision(DLT_EN10MB, 65535,
                                                  PCAP_TSTAMP_PRECISION_NANO);
        if (pc == nullptr) {
            perror("pcap_open_dead failed");
            return EXIT_FAILURE;
        }

        dumpfile = pcap_dump_open(pc, optarg);
        break;

420
421
422
423
      default:
        fprintf(stderr, "unknown option %c\n", c);
        bad_option = 1;
        break;
Jialin Li's avatar
Jialin Li committed
424
    }
425
426
  }

427
  if (ports.empty() || bad_option) {
428
429
430
431
432
433
434
435
    fprintf(stderr,
            "Usage: net_switch [-S SYNC-PERIOD] [-E ETH-LATENCY] "
            "-s SOCKET-A [-s SOCKET-B ...]\n");
    return EXIT_FAILURE;
  }

  signal(SIGINT, sigint_handler);
  signal(SIGTERM, sigint_handler);
Hejing Li's avatar
Hejing Li committed
436
437
438
439
440
441
  signal(SIGUSR1, sigusr1_handler);

#ifdef NETSWITCH_STAT
  signal(SIGUSR2, sigusr2_handler);
#endif

442
443
444
445

  printf("start polling\n");
  while (!exiting) {
    // Sync all interfaces
446
447
    for (auto port : ports)
      port->Sync(cur_ts);
448
449
450
451
452

    // Switch packets
    uint64_t min_ts;
    do {
      min_ts = ULLONG_MAX;
453
      for (size_t port_i = 0; port_i < ports.size(); port_i++) {
454
        auto &port = *ports[port_i];
455
456
457
        switch_pkt(port, port_i);
        if (port.IsSync()) {
          uint64_t ts = port.NextTimestamp();
458
          min_ts = ts < min_ts ? ts : min_ts;
Jialin Li's avatar
Jialin Li committed
459
        }
460
461
462
463
464
      }
    } while (!exiting && (min_ts <= cur_ts));

    // Update cur_ts
    if (min_ts < ULLONG_MAX) {
465
      cur_ts = min_ts;
Jialin Li's avatar
Jialin Li committed
466
    }
467
  }
Jialin Li's avatar
Jialin Li committed
468

Hejing Li's avatar
Hejing Li committed
469
470
471
472
473
474
475
476
477
478
479
480
481
482
#ifdef NETSWITCH_STAT
  fprintf(stderr, "%20s: %22lu %20s: %22lu  poll_suc_rate: %f\n",
          "d2n_poll_total", d2n_poll_total, "d2n_poll_suc", d2n_poll_suc,
          (double)d2n_poll_suc / d2n_poll_total);
  fprintf(stderr, "%65s: %22lu  sync_rate: %f\n", "d2n_poll_sync",
          d2n_poll_sync, (double)d2n_poll_sync / d2n_poll_suc);

  fprintf(stderr, "%20s: %22lu %20s: %22lu  poll_suc_rate: %f\n",
          "s_d2n_poll_total", s_d2n_poll_total, "s_d2n_poll_suc", s_d2n_poll_suc,
          (double)s_d2n_poll_suc / s_d2n_poll_total);
  fprintf(stderr, "%65s: %22lu  sync_rate: %f\n", "s_d2n_poll_sync",
          s_d2n_poll_sync, (double)s_d2n_poll_sync / s_d2n_poll_suc);
#endif

483
  return 0;
Jialin Li's avatar
Jialin Li committed
484
}