experiments: add TAS experiment

experiments/Makefile

@@ -52,6 +52,8 @@ GUESTS := \
     qemu-mtcp-client \
     gem5-pair-i40e-server \
     gem5-pair-i40e-client \
+    qemu-tas-server \
+    qemu-tas-client \
 
 # experiments we want for the paper
 EXPERIMENTS := \
@@ -82,12 +84,14 @@ EXPERIMENTS := \
 EXP_CP := \
     gem5-timing-corundum-verilator-pair-cp \
     gem5-i40e-pair \
+    gem5-i40e-mtcp \
 
 #experiments not using check points
 EXP_NCP := \
     qemu-corundum-bm-pair \
     qemu-corundum-bm-switched-1 \
     qemu-i40e-bm-mtcp \
+    qemu-i40e-bm-tas \
 
 
 # all experiments in repo

experiments/common-functions.sh

@@ -51,6 +51,7 @@ init_out() {
 #   - Cosim instance
 #   - secondary hard drive
 #   - [optional primary image name: default ubuntu1804-base]
+#   - [optional: additinoal qemu arguments]
 run_qemu() {
     img_a="$OUTDIR/qemu.hd.a.$1"
     img_b="$OUTDIR/qemu.hd.b.$1"
@@ -74,7 +75,8 @@ run_qemu() {
         -serial mon:stdio -m $((16 * 1024)) -smp 1 -display none -enable-kvm \
         -nic none \
         -chardev socket,path=$pcisock,id=cosimcd \
-        -device cosim-pci,chardev=cosimcd &>$OUTDIR/qemu.$1.log &
+        -device cosim-pci,chardev=cosimcd \
+        $5 &>$OUTDIR/qemu.$1.log &
     pid=$!
     ALL_PIDS="$ALL_PIDS $pid"
     return $pid

experiments/experiments/qemu-i40e-bm-tas.sh

+#!/bin/bash
+
+source common-functions.sh
+
+init_out qemu-i40e-bm-tas $1
+run_i40e_bm a
+run_i40e_bm b
+sleep 0.5
+run_wire ab a b
+run_qemu a a build/qemu-tas-server.tar tas "-smp 4"
+run_qemu b b build/qemu-tas-client.tar tas "-smp 4"
+client_pid=$!
+wait $client_pid
+cleanup

experiments/guests/qemu-tas-client/mtcp.conf

+############### mtcp configuration file ###############
+
+# The underlying I/O module you want to use. Please
+# enable only one out of the two.
+#io = psio
+#io = netmap
+io = dpdk
+
+# No. of cores setting (enabling this option will override
+# the `cpu' config for those applications that accept
+# num_cores as command line arguments)
+# 
+# e.g. in case ./epserver is executed with `-N 4', the
+# mtcp core will still invoke 8 mTCP threads if the
+# following line is uncommented.
+num_cores = 1
+
+# Number of memory channels per processor socket (dpdk-only)
+num_mem_ch = 4
+
+# Used port (please adjust accordingly)
+port = dpdk0
+
+# Maximum concurrency per core
+max_concurrency = 4096
+
+# Maximum number of socket buffers per core
+# Set this to small value if there are many idle connections
+max_num_buffers = 4096
+
+# Receive buffer size of sockets
+rcvbuf = 8192
+#rcvbuf = 1048576
+
+# Send buffer size of sockets
+sndbuf = 8192
+#sndbuf = 1048576
+
+# TCP timeout seconds
+# (tcp_timeout = -1 can disable the timeout check)
+tcp_timeout = 10
+
+# TCP timewait seconds
+tcp_timewait = 0
+
+# Interface to print stats (please adjust accordingly)
+# You can enable multiple ports in separate lines
+#------ PSIO ports -------#
+#stat_print = xge0
+#stat_print = xge1
+#------ DPDK ports -------#
+stat_print = dpdk0
+#stat_print = dpdk0:0
+#stat_print = dpdk0:1
+#stat_print = dpdk1
+
+#######################################################

experiments/guests/qemu-tas-client/run.sh

+#!/bin/bash
+export HOME=/root
+export LANG=en_US
+export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games"
+
+mount -t proc proc /proc
+mount -t sysfs sysfs /sys
+mkdir -p /dev/hugepages
+mount -t hugetlbfs nodev /dev/hugepages
+mkdir -p /dev/shm
+mount -t tmpfs tmpfs /dev/shm
+insmod /root/dpdk/lib/modules/5.4.46/extra/dpdk/igb_uio.ko
+/root/dpdk/sbin/dpdk-devbind -b igb_uio 0000:00:02.0
+echo 4096 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages
+
+cd /root/tas
+tas/tas --ip-addr=192.168.64.2/24 --fp-cores-max=1 --fp-no-ints &
+sleep 3
+
+cd /root/tasbench/micro_rpc
+LD_PRELOAD=/root/tas/lib/libtas_interpose.so ./testclient_linux 192.168.64.1 1234 1 /tmp/guest/mtcp.conf 1024 1 128 2 0 8 &
+sleep 25
+
+poweroff -f

experiments/guests/qemu-tas-server/mtcp.conf

+ ############## mtcp configuration file ###############
+
+# The underlying I/O module you want to use. Please
+# enable only one out of the two.
+#io = psio
+#io = netmap
+io = dpdk
+
+# No. of cores setting (enabling this option will override
+# the `cpu' config for those applications that accept
+# num_cores as command line arguments)
+# 
+# e.g. in case ./epserver is executed with `-N 4', the
+# mtcp core will still invoke 8 mTCP threads if the
+# following line is uncommented.
+num_cores = 1
+
+# Number of memory channels per processor socket (dpdk-only)
+num_mem_ch = 4
+
+# Used port (please adjust accordingly)
+port = dpdk0
+
+# Maximum concurrency per core
+max_concurrency = 4096
+
+# Maximum number of socket buffers per core
+# Set this to small value if there are many idle connections
+max_num_buffers = 4096
+
+# Receive buffer size of sockets
+rcvbuf = 8192
+#rcvbuf = 1048576
+
+# Send buffer size of sockets
+sndbuf = 8192
+#sndbuf = 1048576
+
+# TCP timeout seconds
+# (tcp_timeout = -1 can disable the timeout check)
+tcp_timeout = 10
+
+# TCP timewait seconds
+tcp_timewait = 0
+
+# Interface to print stats (please adjust accordingly)
+# You can enable multiple ports in separate lines
+#------ PSIO ports -------#
+#stat_print = xge0
+#stat_print = xge1
+#------ DPDK ports -------#
+stat_print = dpdk0
+#stat_print = dpdk0:0
+#stat_print = dpdk0:1
+#stat_print = dpdk1
+
+#######################################################

experiments/guests/qemu-tas-server/mtcp.patch

+diff -ur mtcp.old/dpdk/lib/librte_mempool/rte_mempool.c mtcp/dpdk/lib/librte_mempool/rte_mempool.c
+--- mtcp.old/dpdk/lib/librte_mempool/rte_mempool.c	2020-09-08 21:17:34.052248962 +0200
++++ mtcp/dpdk/lib/librte_mempool/rte_mempool.c	2020-09-08 23:41:38.374992953 +0200
+@@ -916,6 +916,8 @@
+ 
+ 	mempool_list = RTE_TAILQ_CAST(rte_mempool_tailq.head, rte_mempool_list);
+ 
++        fprintf(stderr, "rte_mempool_create_empty: tart\n");
++
+ 	/* asked cache too big */
+ 	if (cache_size > RTE_MEMPOOL_CACHE_MAX_SIZE ||
+ 	    CALC_CACHE_FLUSHTHRESH(cache_size) > n) {
+@@ -933,6 +935,7 @@
+ 		return NULL;
+ 	}
+ 
++        fprintf(stderr, "rte_mempool_create_empty: calculated\n");
+ 	rte_rwlock_write_lock(RTE_EAL_MEMPOOL_RWLOCK);
+ 
+ 	/*
+@@ -954,6 +957,8 @@
+ 	mempool_size += private_data_size;
+ 	mempool_size = RTE_ALIGN_CEIL(mempool_size, RTE_MEMPOOL_ALIGN);
+ 
++        fprintf(stderr, "rte_mempool_create_empty: size=%zu\n", mempool_size);
++
+ 	ret = snprintf(mz_name, sizeof(mz_name), RTE_MEMPOOL_MZ_FORMAT, name);
+ 	if (ret < 0 || ret >= (int)sizeof(mz_name)) {
+ 		rte_errno = ENAMETOOLONG;
+@@ -961,8 +966,11 @@
+ 	}
+ 
+ 	mz = rte_memzone_reserve(mz_name, mempool_size, socket_id, mz_flags);
+-	if (mz == NULL)
++	if (mz == NULL) {
++            fprintf(stderr, "rte_mempool_create_empty: reserve failed\n");
++		RTE_LOG(ERR, MEMPOOL, "memzone reserve failed %zu\n", mempool_size);
+ 		goto exit_unlock;
++        }
+ 
+ 	/* init the mempool structure */
+ 	mp = mz->addr;
+@@ -972,6 +980,7 @@
+ 		rte_errno = ENAMETOOLONG;
+ 		goto exit_unlock;
+ 	}
++            fprintf(stderr, "rte_mempool_create_empty: init done\n");
+ 	mp->mz = mz;
+ 	mp->size = n;
+ 	mp->flags = flags;
+@@ -1028,6 +1037,7 @@
+ 
+ 	mp = rte_mempool_create_empty(name, n, elt_size, cache_size,
+ 		private_data_size, socket_id, flags);
++        fprintf(stderr, "rte_mempool_create_empty=%p\n", mp);
+ 	if (mp == NULL)
+ 		return NULL;
+ 
+@@ -1044,16 +1054,20 @@
+ 	else
+ 		ret = rte_mempool_set_ops_byname(mp, "ring_mp_mc", NULL);
+ 
++        fprintf(stderr, "rte_mempool_create: ops byname=%d\n", ret);
+ 	if (ret)
+ 		goto fail;
+ 
++        fprintf(stderr, "rte_mempool_create: before init\n");
+ 	/* call the mempool priv initializer */
+ 	if (mp_init)
+ 		mp_init(mp, mp_init_arg);
+ 
++        fprintf(stderr, "rte_mempool_create: before populate\n");
+ 	if (rte_mempool_populate_default(mp) < 0)
+ 		goto fail;
+ 
++        fprintf(stderr, "rte_mempool_create: before objinit\n");
+ 	/* call the object initializers */
+ 	if (obj_init)
+ 		rte_mempool_obj_iter(mp, obj_init, obj_init_arg);
+diff -ur mtcp.old/dpdk/lib/librte_mempool/rte_mempool_ops.c mtcp/dpdk/lib/librte_mempool/rte_mempool_ops.c
+--- mtcp.old/dpdk/lib/librte_mempool/rte_mempool_ops.c	2020-09-08 21:17:34.052248962 +0200
++++ mtcp/dpdk/lib/librte_mempool/rte_mempool_ops.c	2020-09-08 23:55:00.777498002 +0200
+@@ -23,6 +23,7 @@
+ 	struct rte_mempool_ops *ops;
+ 	int16_t ops_index;
+ 
++        fprintf(stderr, "rte_mempool_register_ops: h->name=(%s)\n", h->name);
+ 	rte_spinlock_lock(&rte_mempool_ops_table.sl);
+ 
+ 	if (rte_mempool_ops_table.num_ops >=
+@@ -30,6 +31,7 @@
+ 		rte_spinlock_unlock(&rte_mempool_ops_table.sl);
+ 		RTE_LOG(ERR, MEMPOOL,
+ 			"Maximum number of mempool ops structs exceeded\n");
++                fprintf(stderr, "rte_mempool_register_ops: (nospace)\n");
+ 		return -ENOSPC;
+ 	}
+ 
+@@ -38,6 +40,7 @@
+ 		rte_spinlock_unlock(&rte_mempool_ops_table.sl);
+ 		RTE_LOG(ERR, MEMPOOL,
+ 			"Missing callback while registering mempool ops\n");
++                fprintf(stderr, "rte_mempool_register_ops: (invalid)\n");
+ 		return -EINVAL;
+ 	}
+ 
+@@ -45,6 +48,7 @@
+ 		rte_spinlock_unlock(&rte_mempool_ops_table.sl);
+ 		RTE_LOG(DEBUG, EAL, "%s(): mempool_ops <%s>: name too long\n",
+ 				__func__, h->name);
++                fprintf(stderr, "rte_mempool_register_ops: (nametoolong)\n");
+ 		rte_errno = EEXIST;
+ 		return -EEXIST;
+ 	}
+@@ -159,8 +163,10 @@
+ 	unsigned i;
+ 
+ 	/* too late, the mempool is already populated. */
+-	if (mp->flags & MEMPOOL_F_POOL_CREATED)
++	if (mp->flags & MEMPOOL_F_POOL_CREATED) {
++                fprintf(stderr, "rte_mempool_set_ops_byname: already created\n");
+ 		return -EEXIST;
++        }
+ 
+ 	for (i = 0; i < rte_mempool_ops_table.num_ops; i++) {
+ 		if (!strcmp(name,
+@@ -170,8 +176,10 @@
+ 		}
+ 	}
+ 
+-	if (ops == NULL)
++	if (ops == NULL) {
++                fprintf(stderr, "rte_mempool_set_ops_byname: not found (%s)\n", name);
+ 		return -EINVAL;
++        }
+ 
+ 	mp->ops_index = i;
+ 	mp->pool_config = pool_config;
+Binary files mtcp.old/.git/index and mtcp/.git/index differ
+Binary files mtcp.old/.git/modules/dpdk/index and mtcp/.git/modules/dpdk/index differ
+Binary files mtcp.old/.git/modules/mtcp/src/libccp/index and mtcp/.git/modules/mtcp/src/libccp/index differ
+diff -ur mtcp.old/mtcp/src/io_module.c mtcp/mtcp/src/io_module.c
+--- mtcp.old/mtcp/src/io_module.c	2020-09-08 21:18:23.463852529 +0200
++++ mtcp/mtcp/src/io_module.c	2020-09-09 14:58:50.360193608 +0200
+@@ -303,12 +303,14 @@
+ 				       RTE_CACHE_LINE_SIZE);
+ 		
+ 		/* initialize the rte env, what a waste of implementation effort! */
+-		int argc = 6;//8;
++		int argc = 8;//8;
+ 		char *argv[RTE_ARGC_MAX] = {"",
+ 					    "-c",
+ 					    cpumaskbuf,
+ 					    "-n",
+ 					    mem_channels,
++                                            "--log-level",
++                                            "*:debug",
+ #if 0
+ 					    "--socket-mem",
+ 					    socket_mem_str,

experiments/guests/qemu-tas-server/run.sh

+#!/bin/bash
+export HOME=/root
+export LANG=en_US
+export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games"
+
+mount -t proc proc /proc
+mount -t sysfs sysfs /sys
+mkdir -p /dev/hugepages
+mount -t hugetlbfs nodev /dev/hugepages
+mkdir -p /dev/shm
+mount -t tmpfs tmpfs /dev/shm
+insmod /root/dpdk/lib/modules/5.4.46/extra/dpdk/igb_uio.ko
+/root/dpdk/sbin/dpdk-devbind -b igb_uio 0000:00:02.0
+echo 4096 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages
+
+cd /root/tas
+tas/tas --ip-addr=192.168.64.1/24 --fp-cores-max=1 --fp-no-ints &
+sleep 3
+
+cd /root/tasbench/micro_rpc
+LD_PRELOAD=/root/tas/lib/libtas_interpose.so ./echoserver_linux 1234 1 /tmp/guest/mtcp.conf 1024 1024
+poweroff -f