/* * Copyright 2022 Max Planck Institute for Software Systems, and * National University of Singapore * * Permission is hereby granted, free of charge, to any person obtaining * a copy of this software and associated documentation files (the * "Software"), to deal in the Software without restriction, including * without limitation the rights to use, copy, modify, merge, publish, * distribute, sublicense, and/or sell copies of the Software, and to * permit persons to whom the Software is furnished to do so, subject to * the following conditions: * * The above copyright notice and this permission notice shall be * included in all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include extern "C" { #include #include #include }; //#define MEMNIC_DEBUG 1 static int exiting = 0; static uint64_t cur_ts = 0; uint16_t src_port = 1; uint16_t dest_port = 1; uint32_t ip_addr = 0x0F0E0D0C; union mac_addr_{ uint64_t mac_64; uint8_t mac_byte[6]; }; union mac_addr_ mac_addr; static void sigint_handler(int dummy) { exiting = 1; } static void sigusr1_handler(int dummy) { fprintf(stderr, "main_time = %lu\n", cur_ts); } bool MemNicIfInit(struct SimbricksMemIf *memif, struct SimbricksNetIf *netif, const char *shm_path, struct SimbricksBaseIfParams *memParams, struct SimbricksBaseIfParams *netParams) { struct SimbricksBaseIf *membase = &memif->base; struct SimbricksBaseIf *netbase = &netif->base; // first allocate pool size_t shm_size = 0; if (memParams){ shm_size += memParams->in_num_entries * memParams->in_entries_size; shm_size += memParams->out_num_entries * memParams->out_entries_size; } if (netParams){ shm_size += netParams->in_num_entries * netParams->in_entries_size; shm_size += netParams->out_num_entries * netParams->out_entries_size; } std::string shm_path_ = shm_path; struct SimbricksBaseIfSHMPool pool_; memset(&pool_, 0, sizeof(pool_)); if (SimbricksBaseIfSHMPoolCreate(&pool_, shm_path_.c_str(), shm_size) != 0) { perror("MemNicIfInit: SimbricksBaseIfSHMPoolCreate failed"); return false; } struct SimBricksBaseIfEstablishData ests[2]; struct SimbricksProtoMemHostIntro mem_intro; struct SimbricksProtoNetIntro net_intro; unsigned n_bifs = 0; memset(&net_intro, 0, sizeof(net_intro)); // MemIf Init if (SimbricksBaseIfInit(membase, memParams)) { perror("MemIfInit: SimbricksBaseIfInit failed"); } if (SimbricksBaseIfListen(membase, &pool_) != 0) { perror("MemifInit: SimbricksBaseIfListen failed"); return false; } memset(&mem_intro, 0, sizeof(mem_intro)); ests[n_bifs].base_if = membase; ests[n_bifs].tx_intro = &mem_intro; ests[n_bifs].tx_intro_len = sizeof(mem_intro); ests[n_bifs].rx_intro = &mem_intro; ests[n_bifs].rx_intro_len = sizeof(mem_intro); n_bifs++; // NetIf Init if (SimbricksBaseIfInit(netbase, netParams)) { perror("NetIfInit: SimbricksBaseIfInit failed"); } if (SimbricksBaseIfListen(netbase, &pool_) != 0) { perror("NetIfInit: SimbricksBaseIfListen failed"); return false; } memset(&net_intro, 0, sizeof(net_intro)); ests[n_bifs].base_if = netbase; ests[n_bifs].tx_intro = &net_intro; ests[n_bifs].tx_intro_len = sizeof(net_intro); ests[n_bifs].rx_intro = &net_intro; ests[n_bifs].rx_intro_len = sizeof(net_intro); n_bifs++; if (SimBricksBaseIfEstablish(ests, 2)) { fprintf(stderr, "SimBricksBaseIfEstablish failed\n"); return false; } printf("done connecting\n"); return true; } static inline int SimbricksMemNicIfSync(struct SimbricksMemIf *memif, struct SimbricksNetIf *netif, uint64_t cur_ts) { return ((SimbricksMemIfM2HOutSync(memif, cur_ts) == 0 && SimbricksNetIfOutSync(netif, cur_ts) == 0) ? 0 : -1); } static inline uint64_t SimbricksMemNicIfNextTimestamp( struct SimbricksMemIf *memif, struct SimbricksNetIf *netif) { uint64_t net_in = SimbricksNetIfInTimestamp(netif); uint64_t mem_in = SimbricksMemIfH2MInTimestamp(memif); return (net_in < mem_in ? net_in : mem_in); } void ForwardToETH(SimbricksNetIf *netif, volatile union SimbricksProtoMemH2M *data, uint8_t type) { volatile union SimbricksProtoNetMsg *msg = SimbricksNetIfOutAlloc(netif, cur_ts); if (msg == NULL) return; volatile struct SimbricksProtoNetMsgPacket *packet = &msg->packet; // Add Ethernet header struct ethhdr *eth_hdr = (struct ethhdr *)packet->data; //memcpy(eth_hdr->h_source, mac_addr.mac_byte, ETH_ALEN); int i = 0; for (i = 0; i < ETH_ALEN; i++){ eth_hdr->h_source[i] = mac_addr.mac_byte[i]; eth_hdr->h_dest[i] = 0xFF; // Keep destination to broadcast for now } eth_hdr->h_proto = htons(ETH_P_IP); // Add IP header struct iphdr *ip_hdr = (struct iphdr *)(eth_hdr + 1); ip_hdr->daddr = 0xFFFFFFFF; ip_hdr->saddr = ip_addr; ip_hdr->tot_len = sizeof(struct iphdr) + sizeof(struct udphdr) + sizeof(struct MemOp); if (type == SIMBRICKS_PROTO_MEM_H2M_MSG_WRITE){ ip_hdr->tot_len += data->write.len; } ip_hdr->tot_len = htons(ip_hdr->tot_len); // Add UDP header struct udphdr *udp_hdr = (struct udphdr *)(ip_hdr + 1); udp_hdr->uh_sport = src_port; udp_hdr->uh_dport = dest_port; udp_hdr->uh_ulen = sizeof(struct udphdr) + sizeof(struct MemOp); if (type == SIMBRICKS_PROTO_MEM_H2M_MSG_WRITE){ udp_hdr->uh_ulen += data->write.len; } udp_hdr->uh_sum = 0; // To update later packet->len = sizeof(struct ethhdr) + sizeof(struct iphdr) + sizeof(struct udphdr) + sizeof(struct MemOp); // Fill the MemOps struct in the payload struct MemOp *memop = (struct MemOp *)(udp_hdr + 1); void *payload; switch (type) { case SIMBRICKS_PROTO_MEM_H2M_MSG_READ: memop->OpType = type; memop->req_id = data->read.req_id; memop->as_id = data->read.as_id; memop->addr = data->read.addr; memop->len = data->read.len; break; case SIMBRICKS_PROTO_MEM_H2M_MSG_WRITE: memop->OpType = type; memop->req_id = data->write.req_id; memop->as_id = data->write.as_id; memop->addr = data->write.addr; memop->len = data->write.len; payload = (void *)(memop + 1); memcpy((void *)payload, (void *)data->write.data, data->write.len); packet->len += data->write.len; break; default: fprintf(stderr, "ForwardToETH: unsupported type=%u\n", type); } SimbricksNetIfOutSend(netif, msg, SIMBRICKS_PROTO_NET_MSG_PACKET); } void ForwardToMEM(SimbricksMemIf *memif, volatile struct SimbricksProtoNetMsgPacket *packet) { volatile union SimbricksProtoMemM2H *msg = SimbricksMemIfM2HOutAlloc(memif, cur_ts); if (msg == NULL) return; uint8_t type; struct ethhdr *eth_hdr = (struct ethhdr *)packet->data; struct iphdr *ip_hdr = (struct iphdr *)(eth_hdr + 1); struct udphdr *udp_hdr = (struct udphdr *)(ip_hdr + 1); struct MemOp *memop = (struct MemOp *)(udp_hdr + 1); void *data = (void *)(memop + 1); type = memop->OpType; switch (type){ case SIMBRICKS_PROTO_MEM_M2H_MSG_READCOMP: volatile struct SimbricksProtoMemM2HReadcomp *rc; rc = &msg->readcomp; rc->req_id = memop->req_id; memcpy((void*)rc->data, (void*)data, memop->len); SimbricksMemIfM2HOutSend(memif, msg, SIMBRICKS_PROTO_MEM_M2H_MSG_READCOMP); break; case SIMBRICKS_PROTO_MEM_M2H_MSG_WRITECOMP: volatile struct SimbricksProtoMemM2HWritecomp *wc; wc = &msg->writecomp; wc->req_id = memop->req_id; SimbricksMemIfM2HOutSend(memif, msg, SIMBRICKS_PROTO_MEM_M2H_MSG_WRITECOMP); break; case SIMBRICKS_PROTO_MSG_TYPE_SYNC: break; default: fprintf(stderr, "poll_m2h: unsupported type=%u\n", type); } } void PollN2M(SimbricksNetIf *netif, struct SimbricksMemIf *memif, uint64_t cur_ts) { volatile union SimbricksProtoNetMsg *msg = SimbricksNetIfInPoll(netif, cur_ts); if (msg == NULL){ return; } uint8_t type; type = SimbricksNetIfInType(netif, msg); switch (type) { case SIMBRICKS_PROTO_NET_MSG_PACKET: ForwardToMEM(memif, &msg->packet); break; case SIMBRICKS_PROTO_MSG_TYPE_SYNC: break; default: fprintf(stderr, "poll_n2m: unsupported type=%u\n", type); } SimbricksNetIfInDone(netif, msg); } void PollH2M(struct SimbricksMemIf *memif, SimbricksNetIf *netif, uint64_t cur_ts) { volatile union SimbricksProtoMemH2M *msg = SimbricksMemIfH2MInPoll(memif, cur_ts); if (msg == NULL) { return; } uint8_t type; type = SimbricksMemIfH2MInType(memif, msg); switch (type) { case SIMBRICKS_PROTO_MEM_H2M_MSG_READ: //printf("received read request\n"); ForwardToETH(netif, msg, type); break; case SIMBRICKS_PROTO_MEM_H2M_MSG_WRITE: //printf("received write request\n"); ForwardToETH(netif, msg, type); break; case SIMBRICKS_PROTO_MSG_TYPE_SYNC: break; default: fprintf(stderr, "poll_h2m: unsupported type=%u\n", type); } SimbricksMemIfH2MInDone(memif, msg); } int main(int argc, char *argv[]) { signal(SIGINT, sigint_handler); signal(SIGUSR1, sigusr1_handler); int sync_mem = 1, sync_net = 1; uint64_t ts_mem = 0; uint64_t ts_net = 0; const char *shmPath; struct SimbricksBaseIfParams memParams; struct SimbricksBaseIfParams netParams; struct SimbricksMemIf memif; struct SimbricksNetIf netif; SimbricksMemIfDefaultParams(&memParams); SimbricksNetIfDefaultParams(&netParams); if (argc < 4 || argc > 10) { fprintf(stderr, "Usage: memnic MEM-SOCKET NET-SOCKET" "SHM [MAC-ADDR] [SYNC-MODE] [START-TICK] [SYNC-PERIOD] [MEM-LATENCY]" "[ETH-LATENCY]\n"); return -1; } if (argc >= 7) cur_ts = strtoull(argv[6], NULL, 0); if (argc >= 8) memParams.sync_interval = netParams.sync_interval = strtoull(argv[7], NULL, 0) * 1000ULL; if (argc >= 9) memParams.link_latency = strtoull(argv[8], NULL, 0) * 1000ULL; if (argc >= 10) netParams.link_latency = strtoull(argv[9], NULL, 0) * 1000ULL; memParams.sock_path = argv[1]; netParams.sock_path = argv[2]; shmPath = argv[3]; mac_addr.mac_64 = strtoull(argv[4], NULL, 16); printf("mac_addr=%lx\n", mac_addr.mac_64); printf("mac_8: %X:%X:%X:%X:%X:%X\n", mac_addr.mac_byte[0], mac_addr.mac_byte[1],mac_addr.mac_byte[2],mac_addr.mac_byte[3],mac_addr.mac_byte[4],mac_addr.mac_byte[5]); memParams.sync_mode = kSimbricksBaseIfSyncOptional; netParams.sync_mode = kSimbricksBaseIfSyncOptional; memParams.blocking_conn = false; memif.base.sync = sync_mem; netif.base.sync = sync_net; if (!MemNicIfInit(&memif, &netif, shmPath, &memParams, &netParams)){ fprintf(stderr, "MemNicIf init error happens"); return -1; } fprintf(stderr, "start polling\n"); while (!exiting){ while (SimbricksMemNicIfSync(&memif, &netif, cur_ts)) { fprintf(stderr, "warn: SimbricksMemNicIfSync failed (memif=%lu)\n", cur_ts); } do { PollH2M(&memif, &netif, cur_ts); PollN2M(&netif, &memif, cur_ts); ts_mem = SimbricksMemIfH2MInTimestamp(&memif); ts_net = SimbricksNetIfInTimestamp(&netif); } while (!exiting && ((sync_mem && ts_mem <= cur_ts) || (sync_net && ts_net <= cur_ts))); if (sync_mem && sync_net) cur_ts = ts_mem <= ts_net ? ts_mem : ts_net; else if (sync_mem) cur_ts = ts_mem; else if (sync_net) cur_ts = ts_net; } // Todo: cleanup return 0; }