lib/simbricks: big refactor of protocol and API

High-level idea is to separate a out a base-protocol and library that
the PCIe and network protocols layer over. The base-protocol includes
synchronization and the basics of setting up queues, and transferring
messages. So this will enable us to build protocol-agnostic pieces like
proxies, loggers etc. A secondary goal down the line is to remove the
fixed roles of  "nics listen on pcie & network" while hosts and networks
connect, instead making this dynamic.

Unfortunately this is a big change touching just about every piece
interfacing with simbricks. So this is also consolidating and
restructuring our simbricks libraries and adapters, reducing copying of
code etc. After this commit all simulators, ours and external ones, are
broken till the respective commits fixing them.

lib/simbricks/base/cxxatomicfix.h

+/*
+ * Copyright 2022 Max Planck Institute for Software Systems, and
+ * National University of Singapore
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef SIMBRICKS_BASE_CXXATOMICFIX_H_
+#define SIMBRICKS_BASE_CXXATOMICFIX_H_
+
+/**
+ * FIXME: This is a worklaround till we fix all simbricks headers to be
+ * compatible with C++, elliminating the need for extern "C", making it possible
+ * to include this in the generic header where we use the atomics.
+ *
+ * until then, this needs to be included before the generic header.
+ */
+ 
+#include <atomic>
+#define _Atomic(T) std::atomic<T>
+using std::atomic_load_explicit;
+using std::atomic_store_explicit;
+using std::memory_order_acquire;
+using std::memory_order_release;
+
+#endif  // SIMBRICKS_BASE_CXXATOMICFIX_H_

lib/simbricks/base/generic.h

+/*
+ * Copyright 2022 Max Planck Institute for Software Systems, and
+ * National University of Singapore
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef SIMBRICKS_BASEIF_GENERIC_H_
+#define SIMBRICKS_BASEIF_GENERIC_H_
+
+#include <simbricks/base/if.h>
+
+/**
+ * Generates static inline functions specialized for message types of a specific
+ * SimBricks protocol. These are thin wrappers around the baseif functions but
+ * insert appropriate casts.
+ *
+ * Generates the following function with the specified prefix:
+ *  - In: prefixInPeek (wraps `SimbricksBaseIfInPeek`)
+ *  - In: prefixInPoll (wraps `SimbricksBaseIfInPoll`)
+ *  - In: prefixInType (wraps `SimbricksBaseIfInType`)
+ *  - In: prefixInDone (wraps `SimbricksBaseIfInDone`)
+ *  - In: prefixInTimestamp (wraps `SimbricksBaseIfInTimestamp`)
+ *  - Out: prefixOutAlloc (wraps `SimbricksBaseIfOutAlloc`)
+ *  - Out: prefixOutSend (wraps `SimbricksBaseIfOutSend`)
+ *  - Out: prefixOutSync (wraps `SimbricksBaseIfOutSync`)
+ *  - Out: prefixOutNextSync (wraps `SimbricksBaseIfOutNextSync`)
+ *  - Out: prefixOutMsgLen (wraps `SimBricksBaseIfOutMsgLen`)
+ *
+ * @param prefix    Name prefix for all the functions
+ * @param msg_union Union name for the message type of the protocol. (not
+ *                  including the "union" keyword).
+ * @param if_struct Interfacing struct, (member base must be
+ *                  `struct SimBricksBaseIf`).
+ */
+#define SIMBRICKS_BASEIF_GENERIC(prefix, msg_union, if_struct) \
+    \
+    static inline volatile union msg_union *prefix##InPeek( \
+        struct if_struct *base_if, uint64_t ts) { \
+      return (volatile union msg_union *) SimbricksBaseIfInPeek( \
+          &base_if->base, ts); \
+    } \
+    \
+    static inline volatile union msg_union *prefix##InPoll( \
+        struct if_struct *base_if, uint64_t ts) { \
+      return (volatile union msg_union *) SimbricksBaseIfInPoll( \
+          &base_if->base, ts); \
+    } \
+    \
+    static inline uint8_t prefix##InType( \
+        struct if_struct *base_if, volatile union msg_union *msg) { \
+      return SimbricksBaseIfInType(&base_if->base, &msg->base); \
+    } \
+    \
+    static inline void prefix##InDone( \
+        struct if_struct *base_if, volatile union msg_union *msg) { \
+      SimbricksBaseIfInDone(&base_if->base, &msg->base); \
+    } \
+    \
+    static inline uint64_t prefix##InTimestamp( struct if_struct *base_if) { \
+      return SimbricksBaseIfInTimestamp(&base_if->base); \
+    } \
+    \
+    static inline volatile union msg_union *prefix##OutAlloc( \
+        struct if_struct *base_if, \
+        uint64_t timestamp) { \
+      return (volatile union msg_union *) SimbricksBaseIfOutAlloc( \
+          &base_if->base, timestamp); \
+    } \
+    \
+    static inline void prefix##OutSend(struct if_struct *base_if, \
+                                    volatile union msg_union *msg, \
+                                    uint8_t msg_type) { \
+      SimbricksBaseIfOutSend(&base_if->base, &msg->base, msg_type); \
+    } \
+    \
+    static inline int prefix##OutSync(struct if_struct *base_if, \
+                                         uint64_t timestamp) { \
+      return SimbricksBaseIfOutSync(&base_if->base, timestamp); \
+    } \
+    \
+    static inline uint64_t prefix##OutNextSync(struct if_struct *base_if) { \
+      return SimbricksBaseIfOutNextSync(&base_if->base); \
+    } \
+    \
+    static inline size_t prefix##OutMsgLen(struct if_struct *base_if) { \
+      return SimbricksBaseIfOutMsgLen(&base_if->base); \
+    }
+
+#endif  // SIMBRICKS_BASEIF_BASEIF_H_

lib/simbricks/base/if.h

+/*
+ * Copyright 2022 Max Planck Institute for Software Systems, and
+ * National University of Singapore
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef SIMBRICKS_BASE_IF_H_
+#define SIMBRICKS_BASE_IF_H_
+
+#ifndef __cplusplus
+// FIXME (see net_switch)
+#include <stdatomic.h>
+#endif
+#include <stddef.h>
+#include <stdint.h>
+#include <stdbool.h>
+
+#include <simbricks/base/proto.h>
+
+/** Handle for a SHM pool. Treat as opaque. */
+struct SimbricksBaseIfSHMPool {
+  const char *path;
+  int fd;
+  void *base;
+  size_t size;
+  size_t pos;
+};
+
+enum SimbricksBaseIfSyncMode {
+  /** No synchronization enabled. */
+  kSimbricksBaseIfSyncDisabled,
+  /** Synchronization enabled if both peers request it. */
+  kSimbricksBaseIfSyncOptional,
+  /** Enable synchronization and error if not both support it. */
+  kSimbricksBaseIfSyncRequired,
+};
+
+/** Parameters for a SimBricks interface */
+struct SimbricksBaseIfParams {
+  /** Link latency/propagation delay [picoseconds] */
+  uint64_t link_latency;
+  /** Maximum gap between sync messages [picoseconds] */
+  uint64_t sync_interval;
+  /** Unix socket path to listen on/connect to */
+  const char *sock_path;
+  /** Synchronization mode: disabled, optional, required */
+  enum SimbricksBaseIfSyncMode sync_mode;
+
+  /** for connecters and listeners choose blocking vs. non-blocking. */
+  bool blocking_conn;
+
+  /** For listeners: Number of entries in incoming queue*/
+  size_t in_num_entries;
+  /** For listeners: Size of individual entries in incoming queue */
+  size_t in_entries_size;
+  /** For listeners: Number of entries in outgoing queue */
+  size_t out_num_entries;
+  /** For listeners: Size of individual entries in outgoing queue */
+  size_t out_entries_size;
+
+  uint64_t upper_layer_proto;
+};
+
+/** Handle for a SimBricks base interface. Treat as opaque. */
+struct SimbricksBaseIf {
+  void *in_queue;
+  size_t in_pos;
+  size_t in_elen;
+  size_t in_enum;
+  uint64_t in_timestamp;
+
+  void *out_queue;
+  size_t out_pos;
+  size_t out_elen;
+  size_t out_enum;
+  uint64_t out_timestamp;
+
+  int conn_state;
+  int sync;
+  struct SimbricksBaseIfParams params;
+  struct SimbricksBaseIfSHMPool *shm;
+  int listen_fd;
+  int conn_fd;
+  bool listener;
+};
+
+
+/** Create and map a new shared memory pool with the specified path and size. */
+int SimbricksBaseIfSHMPoolCreate(struct SimbricksBaseIfSHMPool *pool,
+                                 const char *path, size_t pool_size);
+/** Map existing shared memory pool by file descriptor. */
+int SimbricksBaseIfSHMPoolMapFd(struct SimbricksBaseIfSHMPool *pool,
+                                int fd);
+/** Map existing shared memory pool by path. */
+int SimbricksBaseIfSHMPoolMap(struct SimbricksBaseIfSHMPool *pool,
+                              const char *path);
+/** Unmap shared memory pool, without unlinking it. */
+int SimbricksBaseIfSHMPoolUnmap(struct SimbricksBaseIfSHMPool *pool);
+/** Delete but don't unmap shared memory pool. */
+int SimbricksBaseIfSHMPoolUnlink(struct SimbricksBaseIfSHMPool *pool);
+
+
+/** Initialize params struct with default values */
+void SimbricksBaseIfDefaultParams(struct SimbricksBaseIfParams *params);
+
+int SimbricksBaseIfInit(struct SimbricksBaseIf *base_if,
+                        struct SimbricksBaseIfParams *params);
+
+/** Create listening base interface. Note this does not wait for a connector. */
+int SimbricksBaseIfListen(struct SimbricksBaseIf *base_if,
+                          struct SimbricksBaseIfSHMPool *pool);
+/** Initiate connection for base interface. Note this is asynchronous. */
+int SimbricksBaseIfConnect(struct SimbricksBaseIf *base_if);
+/** Check if incoming/outgoing connection is established . (non-blocking) */
+int SimbricksBaseIfConnected(struct SimbricksBaseIf *base_if);
+/** FD to wait on for listen or connect event. */
+int SimbricksBaseIfConnFd(struct SimbricksBaseIf *base_if);
+/** Block till base_if is connected or failed */
+int SimbricksBaseIfConnsWait(struct SimbricksBaseIf **base_ifs, unsigned n);
+
+/** Send intro. */
+int SimbricksBaseIfIntroSend(struct SimbricksBaseIf *base_if,
+                             const void *payload, size_t payload_len);
+/** Receive intro. */
+int SimbricksBaseIfIntroRecv(struct SimbricksBaseIf *base_if,
+                             void *payload, size_t *payload_len);
+/** FD to wait on for intro events. */
+int SimbricksBaseIfIntroFd(struct SimbricksBaseIf *base_if);
+
+
+void SimbricksBaseIfClose(struct SimbricksBaseIf *base_if);
+void SimbricksBaseIfUnlink(struct SimbricksBaseIf *base_if);
+
+/**
+ * Poll for an incoming message without advancing the position if one is found.
+ * Message must be retrieved again with a call to `SimbricksBaseIfInPoll`
+ *
+ * @param base_if   Base interface handle (connected).
+ * @param timestamp Current timestamp (in picoseconds).
+ * @return Pointer to the message struct if successfull, NULL otherwise.
+ */
+static inline volatile union SimbricksProtoBaseMsg *SimbricksBaseIfInPeek(
+    struct SimbricksBaseIf *base_if,
+    uint64_t timestamp) {
+  volatile union SimbricksProtoBaseMsg *msg =
+      (volatile union SimbricksProtoBaseMsg *) (void *) (
+        (uint8_t *) base_if->in_queue + base_if->in_pos * base_if->in_elen);
+  uint8_t own_type = atomic_load_explicit(
+      (volatile _Atomic(uint8_t) *) &msg->header.own_type,
+      memory_order_acquire);
+
+  /* message not ready */
+  if ((own_type & SIMBRICKS_PROTO_MSG_OWN_MASK) !=
+      SIMBRICKS_PROTO_MSG_OWN_CON)
+    return NULL;
+
+  /* if in sync mode, wait till message is ready */
+  base_if->in_timestamp = msg->header.timestamp;
+  if (base_if->sync && base_if->in_timestamp > timestamp)
+    return NULL;
+
+  return msg;
+}
+
+/**
+ * Poll for an incoming message. After processing the message must be freed by
+ * calling `SimbricksBaseIfInDone`.
+ *
+ * @param base_if   Base interface handle (connected).
+ * @param timestamp Current timestamp (in picoseconds).
+ * @return Pointer to the message struct if successfull, NULL otherwise.
+ */
+static inline volatile union SimbricksProtoBaseMsg *SimbricksBaseIfInPoll(
+    struct SimbricksBaseIf *base_if,
+    uint64_t timestamp) {
+  volatile union SimbricksProtoBaseMsg *msg =
+      SimbricksBaseIfInPeek(base_if, timestamp);
+
+  if (msg != NULL)
+    base_if->in_pos = (base_if->in_pos + 1) % base_if->in_enum;
+  return msg;
+}
+
+/**
+ * Read message type from received message.
+ *
+ * @param base_if  Base interface handle (connected).
+ * @param msg      Pointer to the previously received message.
+ */
+static inline uint8_t SimbricksBaseIfInType(
+    struct SimbricksBaseIf *base_if,
+    volatile union SimbricksProtoBaseMsg *msg) {
+  return (msg->header.own_type & ~SIMBRICKS_PROTO_MSG_OWN_MASK);
+
+}
+
+/**
+ * Mark received message as processed and pass ownership of the slot back to the
+ * sender.
+ *
+ * @param base_if  Base interface handle (connected).
+ * @param msg      Pointer to the previously received message.
+ */
+static inline void SimbricksBaseIfInDone(
+    struct SimbricksBaseIf *base_if,
+    volatile union SimbricksProtoBaseMsg *msg) {
+  atomic_store_explicit(
+      (volatile _Atomic(uint8_t) *) &msg->header.own_type,
+      (uint8_t) ((msg->header.own_type & ~SIMBRICKS_PROTO_MSG_OWN_MASK) |
+        SIMBRICKS_PROTO_MSG_OWN_PRO),
+      memory_order_release);
+}
+
+/**
+ * Message timestamp of the next. Valid only after a poll failed because of a
+ * future timestamp.
+ *
+ * @param base_if Base interface handle (connected).
+ * @return Input timestamp.
+ */
+static inline uint64_t SimbricksBaseIfInTimestamp(
+    struct SimbricksBaseIf *base_if) {
+  return base_if->in_timestamp;
+}
+
+/**
+ * Allocate a new message in the queue. Must be followed by a call to
+ * `SimbricksBaseIfOutSend`.
+ *
+ * @param base_if   Base interface handle (connected).
+ * @param timestamp Current timestamp (in picoseconds).
+ * @return Pointer to the message struct if successfull, NULL otherwise.
+ */
+static inline volatile union SimbricksProtoBaseMsg *SimbricksBaseIfOutAlloc(
+    struct SimbricksBaseIf *base_if,
+    uint64_t timestamp) {
+  volatile union SimbricksProtoBaseMsg *msg =
+      (volatile union SimbricksProtoBaseMsg *) (void *) (
+        (uint8_t *) base_if->out_queue + base_if->out_pos * base_if->out_elen);
+
+  uint8_t own_type = atomic_load_explicit(
+      (volatile _Atomic(uint8_t) *) &msg->header.own_type,
+      memory_order_acquire);
+  if ((own_type & SIMBRICKS_PROTO_MSG_OWN_MASK) !=
+      SIMBRICKS_PROTO_MSG_OWN_PRO) {
+    return NULL;
+  }
+
+  msg->header.timestamp = timestamp + base_if->params.link_latency;
+  base_if->out_timestamp = timestamp;
+
+  base_if->out_pos = (base_if->out_pos + 1) % base_if->out_enum;
+  return msg;  
+}
+
+/**
+ * Send out a fully filled message. Sets the message type and ownership flag.
+ * Also acts as a compiler barrier to avoid other writes to the message being
+ * reordered after this.
+ *
+ * @param base_if  Base interface handle (connected).
+ * @param msg      Pointer to the previously allocated and fully initialized
+                   message (other than the type.).
+ * @param msg_type Message type to set (without ownership flag).
+ */
+static inline void SimbricksBaseIfOutSend(
+    struct SimbricksBaseIf *base_if, volatile union SimbricksProtoBaseMsg *msg,
+    uint8_t msg_type) {
+  atomic_store_explicit((volatile _Atomic(uint8_t) *) &msg->header.own_type,
+                        (uint8_t) (msg_type | SIMBRICKS_PROTO_MSG_OWN_CON),
+                        memory_order_release);
+}
+
+/**
+ * Send a synchronization dummy message if necessary.
+ *
+ * @param base_if   Base interface handle (connected).
+ * @param timestamp Current timestamp (in picoseconds).
+ * @return 0 if sync successfully sent, 1 if sync was unnecessary, -1 if a
+ * necessary sync message could not be sent because the queue is full.
+ */
+static inline int SimbricksBaseIfOutSync(struct SimbricksBaseIf *base_if,
+                                         uint64_t timestamp) {
+  if (!base_if->sync || (base_if->out_timestamp > 0 &&
+                         timestamp - base_if->out_timestamp <
+                         base_if->params.sync_interval))
+    return 0;
+
+  volatile union SimbricksProtoBaseMsg *msg =
+      SimbricksBaseIfOutAlloc(base_if, timestamp);
+  if (!msg)
+    return -1;
+
+  SimbricksBaseIfOutSend(base_if, msg, SIMBRICKS_PROTO_MSG_TYPE_SYNC);
+  return 0;  
+}
+
+/**
+ * Timestamp when the next sync or data packet must be sent.
+ *
+ * @param base_if   Base interface handle (connected).
+ * @return Timestamp. Undefined if synchronization is disabled.
+ */
+static inline uint64_t SimbricksBaseIfOutNextSync(
+    struct SimbricksBaseIf *base_if)
+{
+  return base_if->out_timestamp + base_if->params.sync_interval;
+}
+
+/**
+ * Retrieve maximal total message length for outgoing messages.
+ * 
+ * @param base_if Base interface handle (connected).
+ * @return Maximal message length in bytes.
+ */
+static inline size_t SimbricksBaseIfOutMsgLen(struct SimbricksBaseIf *base_if) {
+  return base_if->out_elen;
+}
+
+/**
+ * Check if synchronization is enabled for this connection.
+ *
+ * @param base_if Base interface handle (connected).
+ * @return true if synchronized, false otherwise.
+ */
+static inline bool SimbricksBaseIfSyncEnabled(struct SimbricksBaseIf *base_if) {
+  return base_if->sync;
+}
+
+#endif  // SIMBRICKS_BASEIF_BASEIF_H_

lib/simbricks/proto/network.h → lib/simbricks/base/proto.h

@@ -22,134 +22,112 @@
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#ifndef SIMBRICKS_PROTO_NETWORK_H_
-#define SIMBRICKS_PROTO_NETWORK_H_
+#ifndef SIMBRICKS_BASE_PROTO_H_
+#define SIMBRICKS_BASE_PROTO_H_
 
+#include <assert.h>
 #include <stdint.h>
 
-/******************************************************************************/
-/* Initialization messages on Unix socket */
+#define SIMBRICKS_PROTO_MSG_SZCHECK(s) static_assert(sizeof(s) == 64, \
+                                           "SimBrick message size check failed")
 
-/** in dev_intro.flags to indicate that sender supports issuing syncs. */
-#define SIMBRICKS_PROTO_NET_FLAGS_DI_SYNC (1 << 0)
+#define SIMBRICKS_PROTO_VERSION 1
 
-/**
- * welcome message sent by device to network. This message comes with the shared
- * memory file descriptor attached.
- */
-struct SimbricksProtoNetDevIntro {
-  /** flags: see SIMBRICKS_PROTO_NET_FLAGS_DI_* */
-  uint64_t flags;
+#define SIMBRICKS_PROTO_ID_BASE   0x00
+#define SIMBRICKS_PROTO_ID_NET    0x01
+#define SIMBRICKS_PROTO_ID_PCIE   0x02
 
-  /** offset of the device-to-network queue in shared memory region */
-  uint64_t d2n_offset;
-  /** size of an entry in the device-to-network queue in bytes */
-  uint64_t d2n_elen;
-  /** total device-to-network queue length in #entries */
-  uint64_t d2n_nentries;
-
-  /** offset of the net-to-device queue in shared memory region */
-  uint64_t n2d_offset;
-  /** size of an entry in the net-to-device queue in bytes */
-  uint64_t n2d_elen;
-  /** total net-to-device queue length in #entries */
-  uint64_t n2d_nentries;
-} __attribute__((packed));
+/** Listener requests synchronization */
+#define SIMBRICKS_PROTO_FLAGS_LI_SYNC (1 << 0)
+/** Listener forces synchronization */
+#define SIMBRICKS_PROTO_FLAGS_LI_SYNC_FORCE (1 << 1)
 
-#define SIMBRICKS_PROTO_NET_FLAGS_NI_SYNC (1 << 0)
+/**
+ * Welcome message that the listener sends to the connector on the unix socket.
+ * The message specifies if and what synchronization is enabled, the shared
+ * memory queues information, and information on the upper layer protocol. The
+ * message on the Unix socket also includes the shared memory file descriptor
+ * with this message. Finally the intro also contains the upper-layer intro.
+ */
+struct SimbricksProtoListenerIntro {
+  /** simbricks protocol version */
+  uint64_t version;
 
-/** welcome message sent by network to device */
-struct SimbricksProtoNetNetIntro {
-  /** flags: see SIMBRICKS_PROTO_NET_FLAGS_IN_* */
+  /** flags: see SIMBRICKS_PROTO_FLAGS_LI_* */
   uint64_t flags;
-} __attribute__((packed));
 
-/******************************************************************************/
-/* Messages on in-memory device to network channel */
-
-/** Mask for ownership bit in own_type field */
-#define SIMBRICKS_PROTO_NET_D2N_OWN_MASK 0x80
-/** Message is owned by device */
-#define SIMBRICKS_PROTO_NET_D2N_OWN_DEV 0x00
-/** Message is owned by network */
-#define SIMBRICKS_PROTO_NET_D2N_OWN_NET 0x80
-
-/** Mask for type value in own_type field */
-#define SIMBRICKS_PROTO_NET_D2N_MSG_MASK 0x7f
-#define SIMBRICKS_PROTO_NET_D2N_MSG_SYNC 0x1
-#define SIMBRICKS_PROTO_NET_D2N_MSG_SEND 0x2
-
-struct SimbricksProtoNetD2NDummy {
-  uint8_t pad[48];
-  uint64_t timestamp;
-  uint8_t pad_[7];
-  uint8_t own_type;
+  /** offset of the listener-to-connecter queue in shared memory region */
+  uint64_t l2c_offset;
+  /** size of an entry in the listener-to-connecter queue in bytes */
+  uint64_t l2c_elen;
+  /** total listener-to-connecter queue length in #entries */
+  uint64_t l2c_nentries;
+
+  /** offset of the connecter-to-listener queue in shared memory region */
+  uint64_t c2l_offset;
+  /** size of an entry in the host-to-device queue in bytes */
+  uint64_t c2l_elen;
+  /** total host-to-device queue length in #entries */
+  uint64_t c2l_nentries;
+
+  /** upper layer protocol identifier: see SIMBRICKS_PROTO_ID_* */
+  uint64_t upper_layer_proto;
+  /** offset of upper layer intro from beginning of this message */
+  uint64_t upper_layer_intro_off;
 } __attribute__((packed));
 
-struct SimbricksProtoNetD2NSync {
-  uint8_t pad[48];
-  uint64_t timestamp;
-  uint8_t pad_[7];
-  uint8_t own_type;
-} __attribute__((packed));
 
-struct SimbricksProtoNetD2NSend {
-  uint16_t len;
-  uint8_t port;
-  uint8_t pad[45];
-  uint64_t timestamp;
-  uint8_t pad_[7];
-  uint8_t own_type;
-  uint8_t data[];
-} __attribute__((packed));
+/** Connecter has synchronization enabled */
+#define SIMBRICKS_PROTO_FLAGS_CO_SYNC (1 << 0)
+/** Connecter forces synchronization */
+#define SIMBRICKS_PROTO_FLAGS_CO_SYNC_FORCE (1 << 1)
 
-union SimbricksProtoNetD2N {
-  struct SimbricksProtoNetD2NDummy dummy;
-  struct SimbricksProtoNetD2NSync sync;
-  struct SimbricksProtoNetD2NSend send;
-};
+struct SimbricksProtoConnecterIntro {
+  /** simbricks protocol version */
+  uint64_t version;
 
-/******************************************************************************/
-/* Messages on in-memory network to device channel */
+  /** flags: see SIMBRICKS_PROTO_FLAGS_CO_* */
+  uint64_t flags;
 
-#define SIMBRICKS_PROTO_NET_N2D_OWN_MASK 0x80
-/** Message is owned by host */
-#define SIMBRICKS_PROTO_NET_N2D_OWN_NET 0x00
-/** Message is owned by device */
-#define SIMBRICKS_PROTO_NET_N2D_OWN_DEV 0x80
+  /** upper layer protocol identifier: see SIMBRICKS_PROTO_ID_* */
+  uint64_t upper_layer_proto;
+  /** offset of upper layer intro from beginning of this message */
+  uint64_t upper_layer_intro_off;
+} __attribute__((packed));
 
-#define SIMBRICKS_PROTO_NET_N2D_MSG_MASK 0x7f
-#define SIMBRICKS_PROTO_NET_N2D_MSG_SYNC 0x1
-#define SIMBRICKS_PROTO_NET_N2D_MSG_RECV 0x2
 
-struct SimbricksProtoNetN2DDummy {
+/** Mask for ownership bit in own_type field */
+#define SIMBRICKS_PROTO_MSG_OWN_MASK 0x80
+/** Message is owned by producer */
+#define SIMBRICKS_PROTO_MSG_OWN_PRO 0x00
+/** Message is owned by consumer */
+#define SIMBRICKS_PROTO_MSG_OWN_CON 0x80
+
+/** Mask for messsage type in own_type field */
+#define SIMBRICKS_PROTO_MSG_TYPE_MASK 0x7f
+
+/** Pure Sync Message, no upper layer data */
+#define SIMBRICKS_PROTO_MSG_TYPE_SYNC 0x00
+/* values in between are reserved for future extensions */
+/** first message type reserved for upper layer protocols */
+#define SIMBRICKS_PROTO_MSG_TYPE_UPPER_START 0x40
+
+struct SimbricksProtoBaseMsgHeader {
   uint8_t pad[48];
   uint64_t timestamp;
   uint8_t pad_[7];
   uint8_t own_type;
 } __attribute__((packed));
+SIMBRICKS_PROTO_MSG_SZCHECK(struct SimbricksProtoBaseMsgHeader);
 
-struct SimbricksProtoNetN2DSync {
-  uint8_t pad[48];
-  uint64_t timestamp;
-  uint8_t pad_[7];
-  uint8_t own_type;
+union SimbricksProtoBaseMsg {
+  struct SimbricksProtoBaseMsgHeader header;
+  struct SimbricksProtoBaseMsgHeader sync;
 } __attribute__((packed));
+SIMBRICKS_PROTO_MSG_SZCHECK(union SimbricksProtoBaseMsg);
 
-struct SimbricksProtoNetN2DRecv {
-  uint16_t len;
-  uint8_t port;
-  uint8_t pad[45];
-  uint64_t timestamp;
-  uint8_t pad_[7];
-  uint8_t own_type;
-  uint8_t data[];
-};
-
-union SimbricksProtoNetN2D {
-  struct SimbricksProtoNetN2DDummy dummy;
-  struct SimbricksProtoNetN2DSync sync;
-  struct SimbricksProtoNetN2DRecv recv;
-};
+/* deprecated */
+#define SIMBRICKS_PROTO_SYNC_SIMBRICKS 0
+#define SIMBRICKS_PROTO_SYNC_BARRIER 1
 
-#endif  // SIMBRICKS_PROTO_NETWORK_H_
+#endif  // SIMBRICKS_BASE_PROTO_H_

lib/simbricks/base/rules.mk

+# Copyright 2021 Max Planck Institute for Software Systems, and
+# National University of Singapore
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+include mk/subdir_pre.mk
+
+lib_base := $(d)libbase.a
+
+OBJS := $(addprefix $(d),if.o)
+
+libsimbricks_objs += $(OBJS)
+
+$(lib_base): $(OBJS)
+
+CLEAN := $(lib_base) $(OBJS)
+include mk/subdir_post.mk

lib/simbricks/netif/netif.c

-/*
- * Copyright 2021 Max Planck Institute for Software Systems, and
- * National University of Singapore
- *
- * Permission is hereby granted, free of charge, to any person obtaining
- * a copy of this software and associated documentation files (the
- * "Software"), to deal in the Software without restriction, including
- * without limitation the rights to use, copy, modify, merge, publish,
- * distribute, sublicense, and/or sell copies of the Software, and to
- * permit persons to whom the Software is furnished to do so, subject to
- * the following conditions:
- *
- * The above copyright notice and this permission notice shall be
- * included in all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
- * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
- * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
- * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
- * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- */
-
-#include "lib/simbricks/netif/netif.h"
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/mman.h>
-#include <sys/socket.h>
-#include <unistd.h>
-
-#include <simbricks/proto/base.h>
-
-#include "lib/simbricks/netif/internal.h"
-
-static uint64_t current_epoch = 0;
-
-int SimbricksNetIfInit(struct SimbricksNetIf *nsif, const char *eth_socket_path,
-                       int *sync_eth) {
-  struct SimbricksProtoNetDevIntro di;
-  struct SimbricksProtoNetNetIntro ni;
-  int cfd, shm_fd;
-  void *p;
-
-  if ((cfd = uxsocket_connect(eth_socket_path)) < 0) {
-    return -1;
-  }
-
-  memset(&ni, 0, sizeof(ni));
-
-  if (*sync_eth)
-    ni.flags |= SIMBRICKS_PROTO_NET_FLAGS_NI_SYNC;
-
-  if (send(cfd, &ni, sizeof(ni), 0) != sizeof(ni)) {
-    perror("sending net intro failed");
-    return -1;
-  }
-
-  if (uxsocket_recv(cfd, &di, sizeof(di), &shm_fd)) {
-    return -1;
-  }
-
-  if ((p = shm_map(shm_fd)) == NULL) {
-    return -1;
-  }
-  close(shm_fd);
-
-  if ((di.flags & SIMBRICKS_PROTO_NET_FLAGS_DI_SYNC) == 0) {
-    *sync_eth = 0;
-    nsif->sync = 0;
-  } else {
-    nsif->sync = *sync_eth;
-  }
-
-  nsif->d2n_queue = (uint8_t *)p + di.d2n_offset;
-  nsif->n2d_queue = (uint8_t *)p + di.n2d_offset;
-  nsif->d2n_elen = di.d2n_elen;
-  nsif->n2d_elen = di.n2d_elen;
-  nsif->d2n_enum = di.d2n_nentries;
-  nsif->n2d_enum = di.n2d_nentries;
-  nsif->d2n_pos = nsif->n2d_pos = 0;
-  nsif->d2n_timestamp = nsif->n2d_timestamp = 0;
-
-  return 0;
-}
-
-void SimbricksNetIfCleanup(struct SimbricksNetIf *nsif) {
-  fprintf(stderr, "SimbricksNetIfCleanup: TODO\n");
-  abort();
-}
-
-volatile union SimbricksProtoNetD2N *SimbricksNetIfD2NPoll(
-    struct SimbricksNetIf *nsif, uint64_t timestamp) {
-  volatile union SimbricksProtoNetD2N *msg =
-      (volatile union SimbricksProtoNetD2N *)(nsif->d2n_queue +
-                                              nsif->d2n_pos * nsif->d2n_elen);
-
-  /* message not ready */
-  if ((msg->dummy.own_type & SIMBRICKS_PROTO_NET_D2N_OWN_MASK) !=
-      SIMBRICKS_PROTO_NET_D2N_OWN_NET)
-    return NULL;
-
-  /* if in sync mode, wait till message is ready */
-  nsif->d2n_timestamp = msg->dummy.timestamp;
-  if (nsif->sync && nsif->d2n_timestamp > timestamp)
-    return NULL;
-
-  nsif->d2n_pos = (nsif->d2n_pos + 1) % nsif->d2n_enum;
-  return msg;
-}
-
-void SimbricksNetIfD2NDone(struct SimbricksNetIf *nsif,
-                           volatile union SimbricksProtoNetD2N *msg) {
-  msg->dummy.own_type =
-      (msg->dummy.own_type & SIMBRICKS_PROTO_NET_D2N_MSG_MASK) |
-      SIMBRICKS_PROTO_NET_D2N_OWN_DEV;
-}
-
-volatile union SimbricksProtoNetN2D *SimbricksNetIfN2DAlloc(
-    struct SimbricksNetIf *nsif, uint64_t timestamp, uint64_t latency) {
-  volatile union SimbricksProtoNetN2D *msg =
-      (volatile union SimbricksProtoNetN2D *)(nsif->n2d_queue +
-                                              nsif->n2d_pos * nsif->n2d_elen);
-
-  if ((msg->dummy.own_type & SIMBRICKS_PROTO_NET_N2D_OWN_MASK) !=
-      SIMBRICKS_PROTO_NET_N2D_OWN_NET) {
-    return NULL;
-  }
-
-  msg->dummy.timestamp = timestamp + latency;
-  nsif->n2d_timestamp = timestamp;
-
-  nsif->n2d_pos = (nsif->n2d_pos + 1) % nsif->n2d_enum;
-  return msg;
-}
-
-int SimbricksNetIfN2DSync(struct SimbricksNetIf *nsif, uint64_t timestamp,
-                          uint64_t latency, uint64_t sync_delay,
-                          int sync_mode) {
-  volatile union SimbricksProtoNetN2D *msg;
-  volatile struct SimbricksProtoNetN2DSync *sync;
-  int do_sync;
-
-  if (!nsif->sync)
-    return 0;
-
-  switch (sync_mode) {
-    case SIMBRICKS_PROTO_SYNC_SIMBRICKS:
-      do_sync = nsif->n2d_timestamp == 0 ||
-                timestamp - nsif->n2d_timestamp >= sync_delay;
-      break;
-    case SIMBRICKS_PROTO_SYNC_BARRIER:
-      do_sync = current_epoch == 0 || timestamp - current_epoch >= sync_delay;
-      break;
-    default:
-      fprintf(stderr, "unsupported sync mode=%u\n", sync_mode);
-      return 0;
-  }
-
-  if (!do_sync) {
-    return 0;
-  }
-
-  msg = SimbricksNetIfN2DAlloc(nsif, timestamp, latency);
-  if (msg == NULL)
-    return -1;
-
-  sync = &msg->sync;
-  // WMB();
-  sync->own_type =
-      SIMBRICKS_PROTO_NET_N2D_MSG_SYNC | SIMBRICKS_PROTO_NET_N2D_OWN_DEV;
-
-  return 0;
-}
-
-void SimbricksNetIfAdvanceEpoch(uint64_t timestamp, uint64_t sync_delay,
-                                int sync_mode) {
-  if (sync_mode == SIMBRICKS_PROTO_SYNC_BARRIER) {
-    if (timestamp - current_epoch >= sync_delay) {
-      current_epoch = timestamp;
-    }
-  }
-}
-
-uint64_t SimbricksNetIfAdvanceTime(uint64_t timestamp, uint64_t sync_delay,
-                                   int sync_mode) {
-  switch (sync_mode) {
-    case SIMBRICKS_PROTO_SYNC_SIMBRICKS:
-      return timestamp;
-    case SIMBRICKS_PROTO_SYNC_BARRIER:
-      return timestamp < current_epoch + sync_delay
-                 ? timestamp
-                 : current_epoch + sync_delay;
-    default:
-      fprintf(stderr, "unsupported sync mode=%u\n", sync_mode);
-      return timestamp;
-  }
-}

lib/simbricks/netif/utils.c → lib/simbricks/network/if.c

@@ -22,95 +22,74 @@
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include <fcntl.h>
-#include <pthread.h>
+#include "lib/simbricks/network/if.h"
+
+#include <poll.h>
 #include <stdio.h>
-#include <stdlib.h>
 #include <string.h>
-#include <sys/ioctl.h>
-#include <sys/mman.h>
-#include <sys/socket.h>
-#include <sys/stat.h>
-#include <sys/un.h>
-#include <unistd.h>
 
-#include "lib/simbricks/netif/internal.h"
+void SimbricksNetIfDefaultParams(struct SimbricksBaseIfParams *params)
+{
+  SimbricksBaseIfDefaultParams(params);
+  params->in_entries_size = params->out_entries_size = 1536 + 64;
+  params->upper_layer_proto = SIMBRICKS_PROTO_ID_NET;
+}
+
+int SimbricksNetIfInit(struct SimbricksNetIf *nsif,
+                       struct SimbricksBaseIfParams *params,
+                       const char *eth_socket_path,
+                       int *sync_eth)
+{
+  // some threaded code using this interface
+  struct SimbricksBaseIfParams params_ = *params;
+  struct SimbricksBaseIf *bif = &nsif->base;
 
-int uxsocket_connect(const char *path) {
-  int fd;
-  struct sockaddr_un saun;
+  if (sync_eth && *sync_eth)
+    params_.sync_mode = kSimbricksBaseIfSyncOptional;
+  else
+    params_.sync_mode = kSimbricksBaseIfSyncDisabled;
 
-  /* prepare and connect socket */
-  memset(&saun, 0, sizeof(saun));
-  saun.sun_family = AF_UNIX;
-  strcpy(saun.sun_path, path);
+  params_.sock_path = eth_socket_path;
 
-  if ((fd = socket(AF_UNIX, SOCK_STREAM, 0)) == -1) {
-    perror("socket failed");
+  if (SimbricksBaseIfInit(bif, &params_)) {
+    perror("SimbricksNetIfInit: SimbricksBaseIfInit failed");
     return -1;
   }
 
-  if (connect(fd, (struct sockaddr *)&saun, sizeof(saun)) != 0) {
-    perror("connect failed");
+  if (SimbricksBaseIfConnect(bif)) {
+    perror("SimbricksNetIfInit: SimbricksBaseIfConnect failed");
     return -1;
   }
 
-  return fd;
-}
-
-int uxsocket_recv(int fd, void *data, size_t len, int *pfd) {
-  int *ppfd;
-  ssize_t ret;
-  struct cmsghdr *cmsg;
-  union {
-    char buf[CMSG_SPACE(sizeof(int))];
-    struct cmsghdr align;
-  } u;
-  struct iovec iov = {
-      .iov_base = data,
-      .iov_len = len,
-  };
-  struct msghdr msg = {
-      .msg_name = NULL,
-      .msg_namelen = 0,
-      .msg_iov = &iov,
-      .msg_iovlen = 1,
-      .msg_control = u.buf,
-      .msg_controllen = sizeof(u.buf),
-      .msg_flags = 0,
-  };
-
-  if ((ret = recvmsg(fd, &msg, 0)) != (ssize_t)len) {
-    perror("recvmsg failed");
+  if (SimbricksBaseIfConnsWait(&bif, 1)) {
+    perror("SimbricksNetIfInit: SimbricksBaseIfConnect failed");
     return -1;
   }
 
-  cmsg = CMSG_FIRSTHDR(&msg);
-  ppfd = (int *)CMSG_DATA(cmsg);
-  if (msg.msg_controllen <= 0 || cmsg->cmsg_len != CMSG_LEN(sizeof(int))) {
-    fprintf(stderr, "accessing ancillary data failed\n");
+  struct SimbricksProtoNetIntro intro;
+  memset(&intro, 0, sizeof(intro));
+  if (SimbricksBaseIfIntroSend(bif, &intro, sizeof(intro))) {
+    perror("SimbricksNetIfInit: SimbricksBaseIfIntroSend failed");
     return -1;
   }
 
-  *pfd = *ppfd;
-  return 0;
-}
-
-void *shm_map(int shm_fd) {
-  void *p;
-  struct stat statbuf;
-
-  if (fstat(shm_fd, &statbuf) != 0) {
-    perror("shm_map: fstat failed");
-    return NULL;
+  struct pollfd pfd;
+  pfd.fd = SimbricksBaseIfIntroFd(bif);
+  pfd.events = POLLIN;
+  pfd.revents = 0;
+  if (poll(&pfd, 1, -1) != 1) {
+    perror("SimbricksNetIfInit: poll failed");
+    return -1;
   }
 
-  p = mmap(NULL, statbuf.st_size, PROT_READ | PROT_WRITE, MAP_SHARED, shm_fd,
-           0);
-  if (p == MAP_FAILED) {
-    perror("shm_map: mmap failed");
-    return NULL;
+  size_t plen = sizeof(intro);
+  if (SimbricksBaseIfIntroRecv(bif, &intro, &plen)) {
+    perror("SimbricksNetIfInit: SimbricksBaseIfIntroRecv failed");
+    return -1;
   }
 
-  return p;
-}
+  if (sync_eth)
+    *sync_eth = SimbricksBaseIfSyncEnabled(bif);
+
+  return 0;
+}
\ No newline at end of file

lib/simbricks/proto/base.h → lib/simbricks/network/if.h

@@ -22,10 +22,27 @@
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#ifndef SIMBRICKS_PROTO_BASE_H_
-#define SIMBRICKS_PROTO_BASE_H_
+#ifndef SIMBRICKS_NETWORK_IF_H_
+#define SIMBRICKS_NETWORK_IF_H_
 
-#define SIMBRICKS_PROTO_SYNC_SIMBRICKS 0
-#define SIMBRICKS_PROTO_SYNC_BARRIER 1
+#include <stddef.h>
+#include <stdint.h>
 
-#endif  // SIMBRICKS_PROTO_BASE_H_
+#include <simbricks/network/proto.h>
+#include <simbricks/base/generic.h>
+
+
+struct SimbricksNetIf {
+    struct SimbricksBaseIf base;
+};
+
+void SimbricksNetIfDefaultParams(struct SimbricksBaseIfParams *params);
+int SimbricksNetIfInit(struct SimbricksNetIf *nsif,
+                       struct SimbricksBaseIfParams *params,
+                       const char *eth_socket_path,
+                       int *sync_eth);
+
+/** Generate queue access functions */
+SIMBRICKS_BASEIF_GENERIC(SimbricksNetIf, SimbricksProtoNetMsg, SimbricksNetIf);
+
+#endif  // SIMBRICKS_NETWORK_IF_H_

lib/simbricks/netif/netif.h → lib/simbricks/network/proto.h

@@ -22,49 +22,39 @@
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#ifndef SIMBRICKS_NETIF_NETIF_H_
-#define SIMBRICKS_NETIF_NETIF_H_
+#ifndef SIMBRICKS_NETWORK_PROTO_H_
+#define SIMBRICKS_NETWORK_PROTO_H_
 
-#include <stddef.h>
 #include <stdint.h>
 
-#include <simbricks/proto/network.h>
+#include <simbricks/base/proto.h>
 
-struct SimbricksNetIf {
-  uint8_t *d2n_queue;
-  size_t d2n_pos;
-  size_t d2n_elen;
-  size_t d2n_enum;
-  uint64_t d2n_timestamp;
+/******************************************************************************/
+/* Initialization messages on Unix socket */
 
-  uint8_t *n2d_queue;
-  size_t n2d_pos;
-  size_t n2d_elen;
-  size_t n2d_enum;
-  uint64_t n2d_timestamp;
+/** welcome message sent by network devices to eachother. */
+struct SimbricksProtoNetIntro {
+} __attribute__((packed));
 
-  int sync;
-};
+/******************************************************************************/
+/* The network protocol is symmetric */
 
-int SimbricksNetIfInit(struct SimbricksNetIf *nsif, const char *eth_socket_path,
-                       int *sync_eth);
-void SimbricksNetIfCleanup(struct SimbricksNetIf *nsif);
+/** a network packet */
+#define SIMBRICKS_PROTO_NET_MSG_PACKET 0x40
 
-volatile union SimbricksProtoNetD2N *SimbricksNetIfD2NPoll(
-    struct SimbricksNetIf *nsif, uint64_t timestamp);
-void SimbricksNetIfD2NDone(struct SimbricksNetIf *nsif,
-                           volatile union SimbricksProtoNetD2N *msg);
-static inline uint64_t SimbricksNetIfD2NTimestamp(struct SimbricksNetIf *nsif) {
-  return nsif->d2n_timestamp;
-}
+struct SimbricksProtoNetMsgPacket {
+  uint16_t len;
+  uint8_t port;
+  uint8_t pad[45];
+  uint64_t timestamp;
+  uint8_t pad_[7];
+  uint8_t own_type;
+  uint8_t data[];
+} __attribute__((packed));
 
-volatile union SimbricksProtoNetN2D *SimbricksNetIfN2DAlloc(
-    struct SimbricksNetIf *nsif, uint64_t timestamp, uint64_t latency);
-int SimbricksNetIfN2DSync(struct SimbricksNetIf *nsif, uint64_t timestamp,
-                          uint64_t latency, uint64_t sync_delay, int sync_mode);
-void SimbricksNetIfAdvanceEpoch(uint64_t timestamp, uint64_t sync_delay,
-                                int sync_mode);
-uint64_t SimbricksNetIfAdvanceTime(uint64_t timestamp, uint64_t sync_delay,
-                                   int sync_mode);
+union SimbricksProtoNetMsg {
+  union SimbricksProtoBaseMsg base;
+  struct SimbricksProtoNetMsgPacket packet;
+};
 
-#endif  // SIMBRICKS_NETIF_NETIF_H_
+#endif  // SIMBRICKS_NETWORK_PROTO_H_

lib/simbricks/netif/rules.mk → lib/simbricks/network/rules.mk

@@ -22,9 +22,11 @@
 
 include mk/subdir_pre.mk
 
-lib_netif := $(d)libnetif_common.a
+lib_netif := $(d)libnetwork.a
 
-OBJS := $(addprefix $(d),netif.o utils.o)
+OBJS := $(addprefix $(d),if.o)
+
+libsimbricks_objs += $(OBJS)
 
 $(lib_netif): $(OBJS)
 

lib/simbricks/nicif/nicif.c

@@ -26,429 +26,137 @@
 
 #include <poll.h>
 #include <stdio.h>
-#include <stdlib.h>
 #include <string.h>
-#include <sys/socket.h>
-#include <unistd.h>
 
-#include <simbricks/proto/base.h>
-
-#include "lib/simbricks/nicif/internal.h"
-
-#define D2H_ELEN (9024 + 64)
-#define D2H_ENUM 8192
-
-#define H2D_ELEN (9024 + 64)
-#define H2D_ENUM 8192
-
-#define D2N_ELEN (1536 + 64)
-#define D2N_ENUM 8192
-
-#define N2D_ELEN (1536 + 64)
-#define N2D_ENUM 8192
-
-
-
-
-
-
-static int accept_pci(struct SimbricksNicIf *nicif,
-                      struct SimbricksProtoPcieDevIntro *di,
-                      int pci_lfd,
-                      int *sync_pci) {
-  if ((nicif->pci_cfd = accept(pci_lfd, NULL, NULL)) < 0) {
-    return -1;
-  }
-  close(pci_lfd);
-  printf("pci connection accepted\n");
-
-  di->d2h_offset = nicif->d2h_off;
-  di->d2h_elen = D2H_ELEN;
-  di->d2h_nentries = D2H_ENUM;
-
-  di->h2d_offset = nicif->h2d_off;
-  di->h2d_elen = H2D_ELEN;
-  di->h2d_nentries = H2D_ENUM;
-
-  if (*sync_pci)
-    di->flags |= SIMBRICKS_PROTO_PCIE_FLAGS_DI_SYNC;
-  else
-    di->flags &= ~((uint64_t)SIMBRICKS_PROTO_PCIE_FLAGS_DI_SYNC);
-
-  if (uxsocket_send(nicif->pci_cfd, di, sizeof(*di), nicif->shm_fd)) {
-    return -1;
-  }
-  printf("pci intro sent\n");
-  return 0;
-}
-
-static int accept_eth(struct SimbricksNicIf *nicif,
-                      int eth_lfd,
-                      int *sync_eth) {
-  struct SimbricksProtoNetDevIntro di;
-
-  if ((nicif->eth_cfd = accept(eth_lfd, NULL, NULL)) < 0) {
-    return -1;
-  }
-  close(eth_lfd);
-  printf("eth connection accepted\n");
-
-  memset(&di, 0, sizeof(di));
-  di.flags = 0;
-  if (*sync_eth)
-    di.flags |= SIMBRICKS_PROTO_NET_FLAGS_DI_SYNC;
-
-  di.d2n_offset = nicif->d2n_off;
-  di.d2n_elen = D2N_ELEN;
-  di.d2n_nentries = D2N_ENUM;
-
-  di.n2d_offset = nicif->n2d_off;
-  di.n2d_elen = N2D_ELEN;
-  di.n2d_nentries = N2D_ENUM;
-
-  if (uxsocket_send(nicif->eth_cfd, &di, sizeof(di), nicif->shm_fd)) {
+int SimbricksNicIfInit(struct SimbricksNicIf *nicif,
+                       const char *shm_path,
+                       struct SimbricksBaseIfParams *netParams,
+                       struct SimbricksBaseIfParams *pcieParams,
+                       struct SimbricksProtoPcieDevIntro *di)
+{
+  struct SimbricksBaseIf *netif = &nicif->net.base;
+  struct SimbricksBaseIf *pcieif = &nicif->pcie.base;
+
+  // first allocate pool
+  size_t shm_size = 0;
+  if (netParams) {
+    shm_size += netParams->in_num_entries * netParams->in_entries_size;
+    shm_size += netParams->out_num_entries * netParams->out_entries_size;
+  }
+  if (pcieParams) {
+    shm_size += pcieParams->in_num_entries * pcieParams->in_entries_size;
+    shm_size += pcieParams->out_num_entries * pcieParams->out_entries_size;
+  }
+  if (SimbricksBaseIfSHMPoolCreate(&nicif->pool, shm_path, shm_size)) {
+    perror("SimbricksNicIfInit: SimbricksBaseIfSHMPoolCreate failed");
     return -1;
   }
-  printf("eth intro sent\n");
-  return 0;
-}
-
-static int accept_conns(struct SimbricksNicIf *nicif,
-                        struct SimbricksProtoPcieDevIntro *di, int pci_lfd,
-                        int *sync_pci, int eth_lfd, int *sync_eth) {
-  struct pollfd pfds[2];
-  int await_pci = pci_lfd != -1;
-  int await_eth = eth_lfd != -1;
-  int ret;
-
-  while (await_pci || await_eth) {
-    if (await_pci && await_eth) {
-      /* we're waiting on both fds */
-      pfds[0].fd = pci_lfd;
-      pfds[1].fd = eth_lfd;
-      pfds[0].events = pfds[1].events = POLLIN;
-      pfds[0].revents = pfds[1].revents = 0;
-
-      ret = poll(pfds, 2, -1);
-      if (ret < 0) {
-        perror("poll failed");
-        return -1;
-      }
 
-      if (pfds[0].revents) {
-        if (accept_pci(nicif, di, pci_lfd, sync_pci) != 0)
-          return -1;
-        await_pci = 0;
-      }
-      if (pfds[1].revents) {
-        if (accept_eth(nicif, eth_lfd, sync_eth) != 0)
-          return -1;
-        await_eth = 0;
-      }
-    } else if (await_pci) {
-      /* waiting just on pci */
-      if (accept_pci(nicif, di, pci_lfd, sync_pci) != 0)
-        return -1;
-      await_pci = 0;
-    } else {
-      /* waiting just on ethernet */
-      if (accept_eth(nicif, eth_lfd, sync_eth) != 0)
-        return -1;
-      await_eth = 0;
+  struct SimbricksBaseIf *bifs[2];
+  unsigned n_bifs = 0;
+  if (netParams) {
+    if (SimbricksBaseIfInit(netif, netParams)) {
+      perror("SimbricksNicIfInit: SimbricksBaseIfInit net failed");
+      return -1;
     }
-  }
-
-  return 0;
-}
-
-int SimbricksNicIfInit(struct SimbricksNicIf *nicif,
-                       struct SimbricksNicIfParams *params,
-                       struct SimbricksProtoPcieDevIntro *di) {
-  int pci_lfd = -1, eth_lfd = -1;
-  void *shmptr;
-  size_t shm_size;
-
-  /* initialize nicif struct */
-  memset(nicif, 0, sizeof(*nicif));
-  nicif->params = *params;
-  nicif->pci_cfd = nicif->eth_cfd = -1;
 
-  /* ready in memory queues */
-  shm_size = (uint64_t)D2H_ELEN * D2H_ENUM + (uint64_t)H2D_ELEN * H2D_ENUM +
-             (uint64_t)D2N_ELEN * D2N_ENUM + (uint64_t)N2D_ELEN * N2D_ENUM;
-  if ((nicif->shm_fd = shm_create(params->shm_path, shm_size, &shmptr)) < 0) {
-    return -1;
+    if (SimbricksBaseIfListen(netif, &nicif->pool)) {
+      perror("SimbricksNicIfInit: SimbricksBaseIfListen net failed");
+      return -1;
+    }
+    bifs[n_bifs++] = netif;
   }
 
-  nicif->d2h_off = 0;
-  nicif->h2d_off = nicif->d2h_off + (uint64_t)D2H_ELEN * D2H_ENUM;
-  nicif->d2n_off = nicif->h2d_off + (uint64_t)H2D_ELEN * H2D_ENUM;
-  nicif->n2d_off = nicif->d2n_off + (uint64_t)D2N_ELEN * D2N_ENUM;
-
-  nicif->d2h_queue = (uint8_t *)shmptr + nicif->d2h_off;
-  nicif->h2d_queue = (uint8_t *)shmptr + nicif->h2d_off;
-  nicif->d2n_queue = (uint8_t *)shmptr + nicif->d2n_off;
-  nicif->n2d_queue = (uint8_t *)shmptr + nicif->n2d_off;
-
-  nicif->d2h_pos = nicif->h2d_pos = nicif->d2n_pos = nicif->n2d_pos = 0;
-
-  /* get listening sockets ready */
-  if (params->pci_socket_path != NULL) {
-    if ((pci_lfd = uxsocket_init(params->pci_socket_path)) < 0) {
+  if (pcieParams) {
+    if (SimbricksBaseIfInit(pcieif, pcieParams)) {
+      perror("SimbricksNicIfInit: SimbricksBaseIfInit pcie failed");
       return -1;
     }
-  }
-  if (params->eth_socket_path != NULL) {
-    if ((eth_lfd = uxsocket_init(params->eth_socket_path)) < 0) {
+
+    if (SimbricksBaseIfListen(pcieif, &nicif->pool)) {
+      perror("SimbricksNicIfInit: SimbricksBaseIfListen pcie failed");
       return -1;
     }
+    bifs[n_bifs++] = pcieif;
   }
 
-  /* accept connection fds */
-  if (accept_conns(nicif, di, pci_lfd, &params->sync_pci, eth_lfd,
-                   &params->sync_eth) != 0) {
+  if (SimbricksBaseIfConnsWait(bifs, n_bifs)) {
+    perror("SimbricksNicIfInit: SimbricksBaseIfConnsWait failed");
     return -1;
   }
 
-  /* receive introductions from other end */
-  if (params->pci_socket_path != NULL) {
-    struct SimbricksProtoPcieHostIntro hi;
-    if (recv(nicif->pci_cfd, &hi, sizeof(hi), 0) != sizeof(hi)) {
+  bool netRxDone = true;
+  if (netParams) {
+    struct SimbricksProtoNetIntro intro;
+    memset(&intro, 0, sizeof(intro));
+    if (SimbricksBaseIfIntroSend(netif, &intro, sizeof(intro))) {
+      perror("SimbricksNicIfInit: SimbricksBaseIfIntroSend net failed");
       return -1;
     }
-    if ((hi.flags & SIMBRICKS_PROTO_PCIE_FLAGS_HI_SYNC) == 0)
-      params->sync_pci = 0;
-    printf("pci host info received\n");
+
+    netRxDone = false;
   }
-  if (params->eth_socket_path != NULL) {
-    struct SimbricksProtoNetNetIntro ni;
-    if (recv(nicif->eth_cfd, &ni, sizeof(ni), 0) != sizeof(ni)) {
+  bool pcieRxDone = true;
+  if (pcieParams) {
+    if (SimbricksBaseIfIntroSend(pcieif, di, sizeof(*di))) {
+      perror("SimbricksNicIfInit: SimbricksBaseIfIntroSend pcie failed");
       return -1;
     }
-    if ((ni.flags & SIMBRICKS_PROTO_NET_FLAGS_NI_SYNC) == 0)
-      params->sync_eth = 0;
-    printf("eth net info received\n");
+    pcieRxDone = false;
   }
 
-  nicif->params.sync_pci = params->sync_pci;
-  nicif->params.sync_eth = params->sync_eth;
-  return 0;
-}
-
-void SimbricksNicIfCleanup(struct SimbricksNicIf *nicif) {
-  close(nicif->pci_cfd);
-  close(nicif->eth_cfd);
-}
-
-/******************************************************************************/
-/* Sync */
 
-int SimbricksNicIfSync(struct SimbricksNicIf *nicif,
-                       uint64_t timestamp) {
-  int ret = 0;
-  struct SimbricksNicIfParams *params = &nicif->params;
-  volatile union SimbricksProtoPcieD2H *d2h;
-  volatile union SimbricksProtoNetD2N *d2n;
+  while (!netRxDone || !pcieRxDone) {
+    struct pollfd pfd[2];
+    unsigned n_pfd = 0;
 
-  /* sync PCI if necessary */
-  if (params->sync_pci) {
-    int sync;
-    switch (params->sync_mode) {
-      case SIMBRICKS_PROTO_SYNC_SIMBRICKS:
-        sync = nicif->pci_last_tx_time == 0 ||
-               timestamp - nicif->pci_last_tx_time >= params->sync_delay;
-        break;
-      case SIMBRICKS_PROTO_SYNC_BARRIER:
-        sync = nicif->current_epoch == 0 ||
-               timestamp - nicif->current_epoch >= params->sync_delay;
-        break;
-      default:
-        fprintf(stderr, "unsupported sync mode=%u\n", params->sync_mode);
-        return ret;
+    if (!netRxDone) {
+      pfd[n_pfd].fd = SimbricksBaseIfIntroFd(netif);
+      pfd[n_pfd].events = POLLIN;
+      pfd[n_pfd].revents = 0;
+      n_pfd++;
     }
-
-    if (sync) {
-      d2h = SimbricksNicIfD2HAlloc(nicif, timestamp);
-      if (d2h == NULL) {
-        ret = -1;
-      } else {
-        d2h->sync.own_type = SIMBRICKS_PROTO_PCIE_D2H_MSG_SYNC |
-                             SIMBRICKS_PROTO_PCIE_D2H_OWN_HOST;
-      }
+    if (!pcieRxDone) {
+      pfd[n_pfd].fd = SimbricksBaseIfIntroFd(pcieif);
+      pfd[n_pfd].events = POLLIN;
+      pfd[n_pfd].revents = 0;
+      n_pfd++;
     }
-  }
-
-  /* sync Ethernet if necessary */
-  if (params->sync_eth) {
-    int sync;
-    switch (params->sync_mode) {
-      case SIMBRICKS_PROTO_SYNC_SIMBRICKS:
-        sync = nicif->eth_last_tx_time == 0 ||
-               timestamp - nicif->eth_last_tx_time >= params->sync_delay;
-        break;
-      case SIMBRICKS_PROTO_SYNC_BARRIER:
-        sync = nicif->current_epoch == 0 ||
-               timestamp - nicif->current_epoch >= params->sync_delay;
-        break;
-      default:
-        fprintf(stderr, "unsupported sync mode=%u\n", params->sync_mode);
-        return ret;
+    if (poll(pfd, n_pfd, -1) < 0) {
+      perror("SimbricksNicIfInit: poll failed");
+      return -1;
     }
 
-    if (sync) {
-      d2n = SimbricksNicIfD2NAlloc(nicif, timestamp);
-      if (d2n == NULL) {
-        ret = -1;
-      } else {
-        d2n->sync.own_type =
-            SIMBRICKS_PROTO_NET_D2N_MSG_SYNC | SIMBRICKS_PROTO_NET_D2N_OWN_NET;
+    if (!netRxDone) {
+      struct SimbricksProtoNetIntro intro;
+      size_t plen = sizeof(intro);
+      int ret = SimbricksBaseIfIntroRecv(netif, &intro, &plen);
+      if (ret == 0) {
+        netRxDone = true;
+      } else if (ret < 0) {
+        perror("SimbricksNicIfInit: SimbricksBaseIfIntroRecv net failed");
+        return -1;
       }
     }
-  }
-
-  return ret;
-}
-
-void SimbricksNicIfAdvanceEpoch(struct SimbricksNicIf *nicif,
-                                uint64_t timestamp) {
-  struct SimbricksNicIfParams *params = &nicif->params;
-  if (params->sync_mode == SIMBRICKS_PROTO_SYNC_BARRIER) {
-    if ((params->sync_pci || params->sync_eth) &&
-        timestamp - nicif->current_epoch >= params->sync_delay) {
-      nicif->current_epoch = timestamp;
+    if (!pcieRxDone) {
+      struct SimbricksProtoPcieHostIntro intro;
+      size_t plen = sizeof(intro);
+      int ret = SimbricksBaseIfIntroRecv(pcieif, &intro, &plen);
+      if (ret == 0) {
+        pcieRxDone = true;
+      } else if (ret < 0) {
+        perror("SimbricksNicIfInit: SimbricksBaseIfIntroRecv pcie failed");
+        return -1;
+      }
     }
   }
+  return 0;
 }
 
-uint64_t SimbricksNicIfAdvanceTime(struct SimbricksNicIf *nicif,
-                                   uint64_t timestamp) {
-  struct SimbricksNicIfParams *params = &nicif->params;
-  switch (params->sync_mode) {
-    case SIMBRICKS_PROTO_SYNC_SIMBRICKS:
-      return timestamp;
-    case SIMBRICKS_PROTO_SYNC_BARRIER:
-      return timestamp < nicif->current_epoch + params->sync_delay
-                 ? timestamp
-                 : nicif->current_epoch + params->sync_delay;
-    default:
-      fprintf(stderr, "unsupported sync mode=%u\n", params->sync_mode);
-      return timestamp;
-  }
-}
-
-uint64_t SimbricksNicIfNextTimestamp(struct SimbricksNicIf *nicif) {
-  struct SimbricksNicIfParams *params = &nicif->params;
-  if (params->sync_pci && params->sync_eth) {
-    return (nicif->pci_last_rx_time <= nicif->eth_last_rx_time ?
-              nicif->pci_last_rx_time :
-              nicif->eth_last_rx_time);
-  } else if (params->sync_pci) {
-    return nicif->pci_last_rx_time;
-  } else if (params->sync_eth) {
-    return nicif->eth_last_rx_time;
-  } else {
-    return 0;
-  }
-}
-
-/******************************************************************************/
-/* PCI */
-
-volatile union SimbricksProtoPcieH2D *SimbricksNicIfH2DPoll(
-    struct SimbricksNicIf *nicif, uint64_t timestamp) {
-  volatile union SimbricksProtoPcieH2D *msg =
-      (volatile union SimbricksProtoPcieH2D *)
-      (nicif->h2d_queue + nicif->h2d_pos * H2D_ELEN);
-
-  /* message not ready */
-  if ((msg->dummy.own_type & SIMBRICKS_PROTO_PCIE_H2D_OWN_MASK) !=
-      SIMBRICKS_PROTO_PCIE_H2D_OWN_DEV)
-    return NULL;
-
-  /* if in sync mode, wait till message is ready */
-  nicif->pci_last_rx_time = msg->dummy.timestamp;
-  if (nicif->params.sync_pci && nicif->pci_last_rx_time > timestamp)
-    return NULL;
-
-  return msg;
-}
-
-void SimbricksNicIfH2DDone(struct SimbricksNicIf *nicif,
-                           volatile union SimbricksProtoPcieH2D *msg) {
-  msg->dummy.own_type =
-      (msg->dummy.own_type & SIMBRICKS_PROTO_PCIE_H2D_MSG_MASK) |
-      SIMBRICKS_PROTO_PCIE_H2D_OWN_HOST;
-}
-
-void SimbricksNicIfH2DNext(struct SimbricksNicIf *nicif) {
-  nicif->h2d_pos = (nicif->h2d_pos + 1) % H2D_ENUM;
-}
-
-volatile union SimbricksProtoPcieD2H *SimbricksNicIfD2HAlloc(
-    struct SimbricksNicIf *nicif, uint64_t timestamp) {
-  volatile union SimbricksProtoPcieD2H *msg =
-      (volatile union SimbricksProtoPcieD2H *)
-      (nicif->d2h_queue + nicif->d2h_pos * D2H_ELEN);
-
-  if ((msg->dummy.own_type & SIMBRICKS_PROTO_PCIE_D2H_OWN_MASK) !=
-      SIMBRICKS_PROTO_PCIE_D2H_OWN_DEV) {
-    return NULL;
-  }
-
-  msg->dummy.timestamp = timestamp + nicif->params.pci_latency;
-  nicif->pci_last_tx_time = timestamp;
-
-  nicif->d2h_pos = (nicif->d2h_pos + 1) % D2H_ENUM;
-  return msg;
-}
-
-/******************************************************************************/
-/* Ethernet */
-
-volatile union SimbricksProtoNetN2D *SimbricksNicIfN2DPoll(
-    struct SimbricksNicIf *nicif, uint64_t timestamp) {
-  volatile union SimbricksProtoNetN2D *msg =
-      (volatile union SimbricksProtoNetN2D *)
-      (nicif->n2d_queue + nicif->n2d_pos * N2D_ELEN);
-
-  /* message not ready */
-  if ((msg->dummy.own_type & SIMBRICKS_PROTO_NET_N2D_OWN_MASK) !=
-      SIMBRICKS_PROTO_NET_N2D_OWN_DEV)
-    return NULL;
-
-  /* if in sync mode, wait till message is ready */
-  nicif->eth_last_rx_time = msg->dummy.timestamp;
-  if (nicif->params.sync_eth && nicif->eth_last_rx_time > timestamp)
-    return NULL;
-
-  return msg;
-}
-
-void SimbricksNicIfN2DDone(struct SimbricksNicIf *nicif,
-                           volatile union SimbricksProtoNetN2D *msg) {
-  msg->dummy.own_type =
-      (msg->dummy.own_type & SIMBRICKS_PROTO_NET_N2D_MSG_MASK) |
-      SIMBRICKS_PROTO_NET_N2D_OWN_NET;
-}
-
-void SimbricksNicIfN2DNext(struct SimbricksNicIf *nicif) {
-  nicif->n2d_pos = (nicif->n2d_pos + 1) % N2D_ENUM;
-}
-
-volatile union SimbricksProtoNetD2N *SimbricksNicIfD2NAlloc(
-    struct SimbricksNicIf *nicif, uint64_t timestamp) {
-  volatile union SimbricksProtoNetD2N *msg =
-      (volatile union SimbricksProtoNetD2N *)
-      (nicif->d2n_queue + nicif->d2n_pos * D2N_ELEN);
-
-  if ((msg->dummy.own_type & SIMBRICKS_PROTO_NET_D2N_OWN_MASK) !=
-      SIMBRICKS_PROTO_NET_D2N_OWN_DEV) {
-    return NULL;
-  }
-
-  msg->dummy.timestamp = timestamp + nicif->params.eth_latency;
-  nicif->eth_last_tx_time = timestamp;
-
-  nicif->d2n_pos = (nicif->d2n_pos + 1) % D2N_ENUM;
-  return msg;
+int SimbricksNicIfCleanup(struct SimbricksNicIf *nicif)
+{
+  SimbricksBaseIfClose(&nicif->pcie.base);
+  SimbricksBaseIfClose(&nicif->net.base);
+  /* TODO: unlink? */
+  return -1;
 }

lib/simbricks/nicif/nicif.h

 /*
- * Copyright 2021 Max Planck Institute for Software Systems, and
+ * Copyright 2022 Max Planck Institute for Software Systems, and
  * National University of Singapore
  *
  * Permission is hereby granted, free of charge, to any person obtaining
@@ -28,82 +28,37 @@
 #include <stddef.h>
 #include <stdint.h>
 
-#include <simbricks/proto/network.h>
-#include <simbricks/proto/pcie.h>
-
-struct SimbricksNicIfParams {
-  const char *pci_socket_path;
-  const char *eth_socket_path;
-  const char *shm_path;
-
-  uint64_t pci_latency;
-  uint64_t eth_latency;
-  uint64_t sync_delay;
-
-  int sync_pci;
-  int sync_eth;
-  int sync_mode;
-};
+#include <simbricks/network/if.h>
+#include <simbricks/pcie/if.h>
 
 struct SimbricksNicIf {
-  uint8_t *d2h_queue;
-  size_t d2h_pos;
-  size_t d2h_off; /* offset in shm region */
-
-  uint8_t *h2d_queue;
-  size_t h2d_pos;
-  size_t h2d_off; /* offset in shm region */
-
-  uint8_t *d2n_queue;
-  size_t d2n_pos;
-  size_t d2n_off; /* offset in shm region */
-
-  uint8_t *n2d_queue;
-  size_t n2d_pos;
-  size_t n2d_off; /* offset in shm region */
-
-  uint64_t pci_last_rx_time;
-  uint64_t pci_last_tx_time;
-  uint64_t eth_last_rx_time;
-  uint64_t eth_last_tx_time;
-  uint64_t current_epoch;
-
-  struct SimbricksNicIfParams params;
-
-  int shm_fd;
-  int pci_cfd;
-  int eth_cfd;
+  struct SimbricksBaseIfSHMPool pool;
+  struct SimbricksNetIf net;
+  struct SimbricksPcieIf pcie;
 };
 
+
 int SimbricksNicIfInit(struct SimbricksNicIf *nicif,
-                       struct SimbricksNicIfParams *params,
+                       const char *shmPath,
+                       struct SimbricksBaseIfParams *netParams,
+                       struct SimbricksBaseIfParams *pcieParams,
                        struct SimbricksProtoPcieDevIntro *di);
-void SimbricksNicIfCleanup(struct SimbricksNicIf *nicif);
-
-int SimbricksNicIfSync(struct SimbricksNicIf *nicif,
-                       uint64_t timestamp);
-void SimbricksNicIfAdvanceEpoch(struct SimbricksNicIf *nicif,
-                                uint64_t timestamp);
-uint64_t SimbricksNicIfAdvanceTime(struct SimbricksNicIf *nicif,
-                                   uint64_t timestamp);
-uint64_t SimbricksNicIfNextTimestamp(struct SimbricksNicIf *nicif);
 
-volatile union SimbricksProtoPcieH2D *SimbricksNicIfH2DPoll(
-    struct SimbricksNicIf *nicif, uint64_t timestamp);
-void SimbricksNicIfH2DDone(struct SimbricksNicIf *nicif,
-                           volatile union SimbricksProtoPcieH2D *msg);
-void SimbricksNicIfH2DNext(struct SimbricksNicIf *nicif);
+int SimbricksNicIfCleanup(struct SimbricksNicIf *nicif);
 
-volatile union SimbricksProtoPcieD2H *SimbricksNicIfD2HAlloc(
-    struct SimbricksNicIf *nicif, uint64_t timestamp);
+static inline int SimbricksNicIfSync(struct SimbricksNicIf *nicif,
+                                     uint64_t cur_ts)
+{
+  return ((SimbricksNetIfOutSync(&nicif->net, cur_ts) == 0 &&
+          SimbricksPcieIfD2HOutSync(&nicif->pcie, cur_ts) == 0) ? 0 : -1);
+}
 
-volatile union SimbricksProtoNetN2D *SimbricksNicIfN2DPoll(
-    struct SimbricksNicIf *nicif, uint64_t timestamp);
-void SimbricksNicIfN2DDone(struct SimbricksNicIf *nicif,
-                           volatile union SimbricksProtoNetN2D *msg);
-void SimbricksNicIfN2DNext(struct SimbricksNicIf *nicif);
+static inline uint64_t SimbricksNicIfNextTimestamp(struct SimbricksNicIf *nicif)
+{
+  uint64_t net = SimbricksNetIfInTimestamp(&nicif->net);
+  uint64_t pcie = SimbricksPcieIfH2DInTimestamp(&nicif->pcie);
 
-volatile union SimbricksProtoNetD2N *SimbricksNicIfD2NAlloc(
-    struct SimbricksNicIf *nicif, uint64_t timestamp);
+  return (net < pcie ? net : pcie);
+}
 
 #endif  // SIMBRICKS_NICIF_NICIF_H_

lib/simbricks/nicif/rules.mk

@@ -22,9 +22,9 @@
 
 include mk/subdir_pre.mk
 
-lib_nicif := $(d)libnicif_common.a
+lib_nicif := $(d)libnicif.a
 
-OBJS := $(addprefix $(d),nicif.o utils.o)
+OBJS := $(addprefix $(d),nicif.o)
 
 $(lib_nicif): $(OBJS)
 

lib/simbricks/nicif/utils.c

-/*
- * Copyright 2021 Max Planck Institute for Software Systems, and
- * National University of Singapore
- *
- * Permission is hereby granted, free of charge, to any person obtaining
- * a copy of this software and associated documentation files (the
- * "Software"), to deal in the Software without restriction, including
- * without limitation the rights to use, copy, modify, merge, publish,
- * distribute, sublicense, and/or sell copies of the Software, and to
- * permit persons to whom the Software is furnished to do so, subject to
- * the following conditions:
- *
- * The above copyright notice and this permission notice shall be
- * included in all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
- * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
- * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
- * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
- * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- */
-
-#include <errno.h>
-#include <fcntl.h>
-#include <stdbool.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <sys/mman.h>
-#include <sys/socket.h>
-#include <sys/un.h>
-#include <unistd.h>
-
-#include "lib/simbricks/nicif/internal.h"
-
-int uxsocket_init(const char *path) {
-  int fd;
-  struct sockaddr_un saun;
-
-  if ((fd = socket(AF_UNIX, SOCK_STREAM, 0)) == -1) {
-    perror("uxsocket_init: socket failed");
-    goto error_exit;
-  }
-
-  memset(&saun, 0, sizeof(saun));
-  saun.sun_family = AF_UNIX;
-  memcpy(saun.sun_path, path, strlen(path));
-  if (bind(fd, (struct sockaddr *)&saun, sizeof(saun))) {
-    perror("uxsocket_init: bind failed");
-    goto error_close;
-  }
-
-  if (listen(fd, 5)) {
-    perror("uxsocket_init: listen failed");
-    goto error_close;
-  }
-
-  return fd;
-
-error_close:
-  close(fd);
-error_exit:
-  return -1;
-}
-
-int uxsocket_send(int connfd, void *data, size_t len, int fd) {
-  ssize_t tx;
-  struct iovec iov = {
-      .iov_base = data,
-      .iov_len = len,
-  };
-  union {
-    char buf[CMSG_SPACE(sizeof(int))];
-    struct cmsghdr align;
-  } u;
-  struct msghdr msg = {
-      .msg_name = NULL,
-      .msg_namelen = 0,
-      .msg_iov = &iov,
-      .msg_iovlen = 1,
-      .msg_control = u.buf,
-      .msg_controllen = 0,
-      .msg_flags = 0,
-  };
-  struct cmsghdr *cmsg = &u.align;
-
-  if (fd >= 0) {
-    msg.msg_controllen = sizeof(u.buf);
-
-    cmsg->cmsg_level = SOL_SOCKET;
-    cmsg->cmsg_type = SCM_RIGHTS;
-    cmsg->cmsg_len = CMSG_LEN(sizeof(int));
-
-    *(int *)CMSG_DATA(cmsg) = fd;
-  }
-
-  if ((tx = sendmsg(connfd, &msg, 0)) != (ssize_t)len) {
-    fprintf(stderr, "tx == %zd\n", tx);
-    return -1;
-  }
-
-  return 0;
-}
-
-int shm_create(const char *path, size_t size, void **addr) {
-  int fd;
-  void *p;
-
-#ifdef SHM_ROUND_UP
-  if (size % SHM_ROUND_UP != 0)
-    size += SHM_ROUND_UP - (size % SHM_ROUND_UP);
-#endif
-
-  if ((fd = open(path, O_CREAT | O_RDWR, 0666)) == -1) {
-    perror("util_create_shmsiszed: open failed");
-    goto error_out;
-  }
-  if (ftruncate(fd, size) != 0) {
-    perror("util_create_shmsiszed: ftruncate failed");
-    goto error_remove;
-  }
-
-  if ((p = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_POPULATE,
-                fd, 0)) == (void *)-1) {
-    perror("util_create_shmsiszed: mmap failed");
-    goto error_remove;
-  }
-
-  memset(p, 0, size);
-
-  *addr = p;
-  return fd;
-
-error_remove:
-  close(fd);
-  unlink(path);
-error_out:
-  return -1;
-}

lib/simbricks/netif/internal.h → lib/simbricks/pcie/if.c

@@ -22,13 +22,11 @@
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#ifndef SIMBRICKS_NETIF_INTERNAL_H_
-#define SIMBRICKS_NETIF_INTERNAL_H_
+#include "lib/simbricks/pcie/if.h"
 
-#include <stddef.h>
-
-int uxsocket_connect(const char *path);
-int uxsocket_recv(int fd, void *data, size_t len, int *pfd);
-void *shm_map(int shm_fd);
-
-#endif  // SIMBRICKS_NETIF_INTERNAL_H_
+void SimbricksPcieIfDefaultParams(struct SimbricksBaseIfParams *params)
+{
+  SimbricksBaseIfDefaultParams(params);
+  params->upper_layer_proto = SIMBRICKS_PROTO_ID_PCIE;
+  params->in_entries_size = params->out_entries_size = 9024 + 64;
+}
\ No newline at end of file

lib/simbricks/nicif/internal.h → lib/simbricks/pcie/if.h

@@ -22,14 +22,26 @@
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#ifndef SIMBRICKS_NICIF_INTERNAL_H_
-#define SIMBRICKS_NICIF_INTERNAL_H_
+#ifndef SIMBRICKS_PCIE_IF_H_
+#define SIMBRICKS_PCIE_IF_H_
 
 #include <stddef.h>
-#include <stdio.h>
+#include <stdint.h>
 
-int uxsocket_init(const char *path);
-int uxsocket_send(int connfd, void *data, size_t len, int fd);
-int shm_create(const char *path, size_t size, void **addr);
+#include <simbricks/base/if.h>
+#include <simbricks/base/generic.h>
+#include <simbricks/pcie/proto.h>
 
-#endif  // SIMBRICKS_NICIF_INTERNAL_H_
+void SimbricksPcieIfDefaultParams(struct SimbricksBaseIfParams *params);
+
+struct SimbricksPcieIf {
+  struct SimbricksBaseIf base;
+};
+
+/** Generate queue access functions for both directions */
+SIMBRICKS_BASEIF_GENERIC(SimbricksPcieIfH2D, SimbricksProtoPcieH2D,
+    SimbricksPcieIf);
+SIMBRICKS_BASEIF_GENERIC(SimbricksPcieIfD2H, SimbricksProtoPcieD2H,
+    SimbricksPcieIf);
+
+#endif  // SIMBRICKS_PCIE_IF_H_

lib/simbricks/proto/pcie.h → lib/simbricks/pcie/proto.h

@@ -22,21 +22,17 @@
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#ifndef SIMBRICKS_PROTO_PCIE_H_
-#define SIMBRICKS_PROTO_PCIE_H_
+#ifndef SIMBRICKS_PCIE_PROTO_H_
+#define SIMBRICKS_PCIE_PROTO_H_
 
+#include <assert.h>
 #include <stdint.h>
 
-// #define SIMBRICKS_PROTO_PCIE_MSG_SZCHECK(s) static_assert(sizeof(s) == 64)
-// #define SIMBRICKS_PROTO_PCIE_MSG_SZCHECK(s) _Static_assert(sizeof(s) == 64)
-#define SIMBRICKS_PROTO_PCIE_MSG_SZCHECK(s)
+#include <simbricks/base/proto.h>
 
 /******************************************************************************/
 /* Initialization messages on Unix socket */
 
-/** in dev_intro.flags to indicate that sender supports issuing syncs. */
-#define SIMBRICKS_PROTO_PCIE_FLAGS_DI_SYNC (1 << 0)
-
 /** Number of PCI bars */
 #define SIMBRICKS_PROTO_PCIE_NBARS 6
 
@@ -55,23 +51,6 @@
  * memory file descriptor attached.
  */
 struct SimbricksProtoPcieDevIntro {
-  /** flags: see SIMBRICKS_PROTO_PCIE_FLAGS_DI_* */
-  uint64_t flags;
-
-  /** offset of the device-to-host queue in shared memory region */
-  uint64_t d2h_offset;
-  /** size of an entry in the device-to-host queue in bytes */
-  uint64_t d2h_elen;
-  /** total device-to-host queue length in #entries */
-  uint64_t d2h_nentries;
-
-  /** offset of the host-to-device queue in shared memory region */
-  uint64_t h2d_offset;
-  /** size of an entry in the host-to-device queue in bytes */
-  uint64_t h2d_elen;
-  /** total host-to-device queue length in #entries */
-  uint64_t h2d_nentries;
-
   /** information for each BAR exposed by the device */
   struct {
     /** length of the bar in bytes (len = 0 indicates unused bar) */
@@ -110,48 +89,20 @@ struct SimbricksProtoPcieDevIntro {
   uint16_t psi_msix_cap_offset;
 } __attribute__((packed));
 
-#define SIMBRICKS_PROTO_PCIE_FLAGS_HI_SYNC (1 << 0)
 
 /** welcome message sent by host to device */
 struct SimbricksProtoPcieHostIntro {
-  /** flags: see SIMBRICKS_PROTO_PCIE_FLAGS_HI_* */
-  uint64_t flags;
 } __attribute__((packed));
 
 /******************************************************************************/
 /* Messages on in-memory device to host channel */
 
-/** Mask for ownership bit in own_type field */
-#define SIMBRICKS_PROTO_PCIE_D2H_OWN_MASK 0x80
-/** Message is owned by device */
-#define SIMBRICKS_PROTO_PCIE_D2H_OWN_DEV 0x00
-/** Message is owned by host */
-#define SIMBRICKS_PROTO_PCIE_D2H_OWN_HOST 0x80
-
 /** Mask for type value in own_type field */
-#define SIMBRICKS_PROTO_PCIE_D2H_MSG_MASK 0x7f
-#define SIMBRICKS_PROTO_PCIE_D2H_MSG_SYNC 0x1
-#define SIMBRICKS_PROTO_PCIE_D2H_MSG_READ 0x2
-#define SIMBRICKS_PROTO_PCIE_D2H_MSG_WRITE 0x3
-#define SIMBRICKS_PROTO_PCIE_D2H_MSG_INTERRUPT 0x4
-#define SIMBRICKS_PROTO_PCIE_D2H_MSG_READCOMP 0x5
-#define SIMBRICKS_PROTO_PCIE_D2H_MSG_WRITECOMP 0x6
-
-struct SimbricksProtoPcieD2HDummy {
-  uint8_t pad[48];
-  uint64_t timestamp;
-  uint8_t pad_[7];
-  uint8_t own_type;
-} __attribute__((packed));
-SIMBRICKS_PROTO_PCIE_MSG_SZCHECK(struct SimbricksProtoPcieD2HDummy);
-
-struct SimbricksProtoPcieD2HSync {
-  uint8_t pad[48];
-  uint64_t timestamp;
-  uint8_t pad_[7];
-  uint8_t own_type;
-} __attribute__((packed));
-SIMBRICKS_PROTO_PCIE_MSG_SZCHECK(struct SimbricksProtoPcieD2HSync);
+#define SIMBRICKS_PROTO_PCIE_D2H_MSG_READ 0x40
+#define SIMBRICKS_PROTO_PCIE_D2H_MSG_WRITE 0x41
+#define SIMBRICKS_PROTO_PCIE_D2H_MSG_INTERRUPT 0x42
+#define SIMBRICKS_PROTO_PCIE_D2H_MSG_READCOMP 0x43
+#define SIMBRICKS_PROTO_PCIE_D2H_MSG_WRITECOMP 0x44
 
 struct SimbricksProtoPcieD2HRead {
   uint64_t req_id;
@@ -162,7 +113,7 @@ struct SimbricksProtoPcieD2HRead {
   uint8_t pad_[7];
   uint8_t own_type;
 } __attribute__((packed));
-SIMBRICKS_PROTO_PCIE_MSG_SZCHECK(struct SimbricksProtoPcieD2HRead);
+SIMBRICKS_PROTO_MSG_SZCHECK(struct SimbricksProtoPcieD2HRead);
 
 struct SimbricksProtoPcieD2HWrite {
   uint64_t req_id;
@@ -174,7 +125,7 @@ struct SimbricksProtoPcieD2HWrite {
   uint8_t own_type;
   uint8_t data[];
 } __attribute__((packed));
-SIMBRICKS_PROTO_PCIE_MSG_SZCHECK(struct SimbricksProtoPcieD2HWrite);
+SIMBRICKS_PROTO_MSG_SZCHECK(struct SimbricksProtoPcieD2HWrite);
 
 #define SIMBRICKS_PROTO_PCIE_INT_LEGACY_HI 0
 #define SIMBRICKS_PROTO_PCIE_INT_LEGACY_LO 1
@@ -189,7 +140,7 @@ struct SimbricksProtoPcieD2HInterrupt {
   uint8_t pad_[7];
   uint8_t own_type;
 } __attribute__((packed));
-SIMBRICKS_PROTO_PCIE_MSG_SZCHECK(struct SimbricksProtoPcieD2HInterrupt);
+SIMBRICKS_PROTO_MSG_SZCHECK(struct SimbricksProtoPcieD2HInterrupt);
 
 struct SimbricksProtoPcieD2HReadcomp {
   uint64_t req_id;
@@ -199,7 +150,7 @@ struct SimbricksProtoPcieD2HReadcomp {
   uint8_t own_type;
   uint8_t data[];
 } __attribute__((packed));
-SIMBRICKS_PROTO_PCIE_MSG_SZCHECK(struct SimbricksProtoPcieD2HReadcomp);
+SIMBRICKS_PROTO_MSG_SZCHECK(struct SimbricksProtoPcieD2HReadcomp);
 
 struct SimbricksProtoPcieD2HWritecomp {
   uint64_t req_id;
@@ -208,51 +159,26 @@ struct SimbricksProtoPcieD2HWritecomp {
   uint8_t pad_[7];
   uint8_t own_type;
 } __attribute__((packed));
-SIMBRICKS_PROTO_PCIE_MSG_SZCHECK(struct SimbricksProtoPcieD2HWritecomp);
+SIMBRICKS_PROTO_MSG_SZCHECK(struct SimbricksProtoPcieD2HWritecomp);
 
 union SimbricksProtoPcieD2H {
-  struct SimbricksProtoPcieD2HDummy dummy;
-  struct SimbricksProtoPcieD2HSync sync;
+  union SimbricksProtoBaseMsg base;
   struct SimbricksProtoPcieD2HRead read;
   struct SimbricksProtoPcieD2HWrite write;
   struct SimbricksProtoPcieD2HInterrupt interrupt;
   struct SimbricksProtoPcieD2HReadcomp readcomp;
   struct SimbricksProtoPcieD2HWritecomp writecomp;
 } __attribute__((packed));
-SIMBRICKS_PROTO_PCIE_MSG_SZCHECK(union SimbricksProtoPcieD2H);
+SIMBRICKS_PROTO_MSG_SZCHECK(union SimbricksProtoPcieD2H);
 
 /******************************************************************************/
 /* Messages on in-memory host to device channel */
 
-#define SIMBRICKS_PROTO_PCIE_H2D_OWN_MASK 0x80
-/** Message is owned by host */
-#define SIMBRICKS_PROTO_PCIE_H2D_OWN_HOST 0x00
-/** Message is owned by device */
-#define SIMBRICKS_PROTO_PCIE_H2D_OWN_DEV 0x80
-
-#define SIMBRICKS_PROTO_PCIE_H2D_MSG_MASK 0x7f
-#define SIMBRICKS_PROTO_PCIE_H2D_MSG_SYNC 0x1
-#define SIMBRICKS_PROTO_PCIE_H2D_MSG_READ 0x2
-#define SIMBRICKS_PROTO_PCIE_H2D_MSG_WRITE 0x3
-#define SIMBRICKS_PROTO_PCIE_H2D_MSG_READCOMP 0x4
-#define SIMBRICKS_PROTO_PCIE_H2D_MSG_WRITECOMP 0x5
-#define SIMBRICKS_PROTO_PCIE_H2D_MSG_DEVCTRL 0x7
-
-struct SimbricksProtoPcieH2DDummy {
-  uint8_t pad[48];
-  uint64_t timestamp;
-  uint8_t pad_[7];
-  uint8_t own_type;
-} __attribute__((packed));
-SIMBRICKS_PROTO_PCIE_MSG_SZCHECK(struct SimbricksProtoPcieH2DDummy);
-
-struct SimbricksProtoPcieH2DSync {
-  uint8_t pad[48];
-  uint64_t timestamp;
-  uint8_t pad_[7];
-  uint8_t own_type;
-} __attribute__((packed));
-SIMBRICKS_PROTO_PCIE_MSG_SZCHECK(struct SimbricksProtoPcieH2DSync);
+#define SIMBRICKS_PROTO_PCIE_H2D_MSG_READ 0x60
+#define SIMBRICKS_PROTO_PCIE_H2D_MSG_WRITE 0x61
+#define SIMBRICKS_PROTO_PCIE_H2D_MSG_READCOMP 0x62
+#define SIMBRICKS_PROTO_PCIE_H2D_MSG_WRITECOMP 0x63
+#define SIMBRICKS_PROTO_PCIE_H2D_MSG_DEVCTRL 0x64
 
 struct SimbricksProtoPcieH2DRead {
   uint64_t req_id;
@@ -264,7 +190,7 @@ struct SimbricksProtoPcieH2DRead {
   uint8_t pad_[7];
   uint8_t own_type;
 } __attribute__((packed));
-SIMBRICKS_PROTO_PCIE_MSG_SZCHECK(struct SimbricksProtoPcieH2DRead);
+SIMBRICKS_PROTO_MSG_SZCHECK(struct SimbricksProtoPcieH2DRead);
 
 struct SimbricksProtoPcieH2DWrite {
   uint64_t req_id;
@@ -277,7 +203,7 @@ struct SimbricksProtoPcieH2DWrite {
   uint8_t own_type;
   uint8_t data[];
 } __attribute__((packed));
-SIMBRICKS_PROTO_PCIE_MSG_SZCHECK(struct SimbricksProtoPcieH2DWrite);
+SIMBRICKS_PROTO_MSG_SZCHECK(struct SimbricksProtoPcieH2DWrite);
 
 struct SimbricksProtoPcieH2DReadcomp {
   uint64_t req_id;
@@ -287,7 +213,7 @@ struct SimbricksProtoPcieH2DReadcomp {
   uint8_t own_type;
   uint8_t data[];
 } __attribute__((packed));
-SIMBRICKS_PROTO_PCIE_MSG_SZCHECK(struct SimbricksProtoPcieH2DReadcomp);
+SIMBRICKS_PROTO_MSG_SZCHECK(struct SimbricksProtoPcieH2DReadcomp);
 
 struct SimbricksProtoPcieH2DWritecomp {
   uint64_t req_id;
@@ -296,7 +222,7 @@ struct SimbricksProtoPcieH2DWritecomp {
   uint8_t pad_[7];
   uint8_t own_type;
 } __attribute__((packed));
-SIMBRICKS_PROTO_PCIE_MSG_SZCHECK(struct SimbricksProtoPcieH2DWritecomp);
+SIMBRICKS_PROTO_MSG_SZCHECK(struct SimbricksProtoPcieH2DWritecomp);
 
 #define SIMBRICKS_PROTO_PCIE_CTRL_INTX_EN (1 << 0)
 #define SIMBRICKS_PROTO_PCIE_CTRL_MSI_EN (1 << 1)
@@ -308,17 +234,16 @@ struct SimbricksProtoPcieH2DDevctrl {
   uint8_t pad_[7];
   uint8_t own_type;
 } __attribute__((packed));
-SIMBRICKS_PROTO_PCIE_MSG_SZCHECK(struct SimbricksProtoPcieH2DDevctrl);
+SIMBRICKS_PROTO_MSG_SZCHECK(struct SimbricksProtoPcieH2DDevctrl);
 
 union SimbricksProtoPcieH2D {
-  struct SimbricksProtoPcieH2DDummy dummy;
-  struct SimbricksProtoPcieH2DSync sync;
+  union SimbricksProtoBaseMsg base;
   struct SimbricksProtoPcieH2DRead read;
   struct SimbricksProtoPcieH2DWrite write;
   struct SimbricksProtoPcieH2DReadcomp readcomp;
   struct SimbricksProtoPcieH2DWritecomp writecomp;
   struct SimbricksProtoPcieH2DDevctrl devctrl;
 } __attribute__((packed));
-SIMBRICKS_PROTO_PCIE_MSG_SZCHECK(union SimbricksProtoPcieH2D);
+SIMBRICKS_PROTO_MSG_SZCHECK(union SimbricksProtoPcieH2D);
 
-#endif  // SIMBRICKS_PROTO_PCIE_H_
+#endif  // SIMBRICKS_PCIE_PROTO_H_

lib/simbricks/pcie/rules.mk

+# Copyright 2021 Max Planck Institute for Software Systems, and
+# National University of Singapore
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+include mk/subdir_pre.mk
+
+lib_pcie := $(d)libpcie.a
+
+OBJS := $(addprefix $(d),if.o)
+
+libsimbricks_objs += $(OBJS)
+
+$(lib_pcie): $(OBJS)
+
+CLEAN := $(lib_pcie) $(OBJS)
+include mk/subdir_post.mk

lib/simbricks/rules.mk

@@ -22,7 +22,9 @@
 
 include mk/subdir_pre.mk
 
-$(eval $(call subdir,netif))
+$(eval $(call subdir,base))
+$(eval $(call subdir,network))
+$(eval $(call subdir,pcie))
 $(eval $(call subdir,nicif))
 $(eval $(call subdir,nicbm))