add mtcp experiment script

experiments/Makefile

@@ -53,6 +53,8 @@ GUESTS := \
     gem5-pair-client-largewin-cp \
     gem5-pair-server-udp-cp \
     gem5-pair-client-udp-cp \
+    qemu-mtcp-server \
+    qemu-mtcp-client \
 
 # experiments we want for the paper
 EXPERIMENTS := \

experiments/experiments/qemu-i40e-bm-mtcp.sh

+#!/bin/bash
+
+source common-functions.sh
+
+init_out qemu-i40e-bm-mtcp $1
+run_i40e_bm a
+run_i40e_bm b
+sleep 0.5
+run_wire ab a b
+run_qemu a a build/qemu-mtcp-server.tar mtcp
+run_qemu b b build/qemu-mtcp-client.tar mtcp
+#client_pid=$!
+#wait $client_pid
+sleep 20
+cleanup

experiments/guests/qemu-mtcp-client/mtcp.conf

+############### mtcp configuration file ###############
+
+# The underlying I/O module you want to use. Please
+# enable only one out of the two.
+#io = psio
+#io = netmap
+io = dpdk
+
+# No. of cores setting (enabling this option will override
+# the `cpu' config for those applications that accept
+# num_cores as command line arguments)
+# 
+# e.g. in case ./epserver is executed with `-N 4', the
+# mtcp core will still invoke 8 mTCP threads if the
+# following line is uncommented.
+num_cores = 1
+
+# Number of memory channels per processor socket (dpdk-only)
+num_mem_ch = 4
+
+# Used port (please adjust accordingly)
+port = dpdk0
+
+# Maximum concurrency per core
+max_concurrency = 1024
+
+# Maximum number of socket buffers per core
+# Set this to small value if there are many idle connections
+max_num_buffers = 100
+
+# Receive buffer size of sockets
+rcvbuf = 8192
+#rcvbuf = 1048576
+
+# Send buffer size of sockets
+sndbuf = 8192
+#sndbuf = 1048576
+
+# TCP timeout seconds
+# (tcp_timeout = -1 can disable the timeout check)
+tcp_timeout = 1
+
+# TCP timewait seconds
+tcp_timewait = 0
+
+# Interface to print stats (please adjust accordingly)
+# You can enable multiple ports in separate lines
+#------ PSIO ports -------#
+#stat_print = xge0
+#stat_print = xge1
+#------ DPDK ports -------#
+stat_print = dpdk0
+#stat_print = dpdk0:0
+#stat_print = dpdk0:1
+#stat_print = dpdk1
+
+#######################################################

experiments/guests/qemu-mtcp-client/run.sh

+#!/bin/bash
+export HOME=/root
+export LANG=en_US
+export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games"
+
+mount -t proc proc /proc
+mount -t sysfs sysfs /sys
+mkdir -p /dev/hugepages
+mount -t hugetlbfs nodev /dev/hugepages
+mkdir -p /dev/shm
+mount -t tmpfs tmpfs /dev/shm
+insmod /root/mtcp/dpdk/x86_64-native-linuxapp-gcc/kmod/igb_uio.ko
+/root/mtcp/dpdk/usertools/dpdk-devbind.py -b igb_uio 0000:00:02.0
+echo 4096 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages
+insmod /root/mtcp/dpdk-iface-kmod/dpdk_iface.ko
+/root/mtcp/dpdk-iface-kmod/dpdk_iface_main
+ip link
+ip link set dev dpdk0 up
+ip addr add 192.168.64.2/24 dev dpdk0
+
+cd /root/tasbench/micro_rpc
+./testclient_mtcp 192.168.64.1 1234 1 /tmp/guest/mtcp.conf 1024 1 1
+poweroff

experiments/guests/qemu-mtcp-server/mtcp.conf

+############### mtcp configuration file ###############
+
+# The underlying I/O module you want to use. Please
+# enable only one out of the two.
+#io = psio
+#io = netmap
+io = dpdk
+
+# No. of cores setting (enabling this option will override
+# the `cpu' config for those applications that accept
+# num_cores as command line arguments)
+# 
+# e.g. in case ./epserver is executed with `-N 4', the
+# mtcp core will still invoke 8 mTCP threads if the
+# following line is uncommented.
+num_cores = 1
+
+# Number of memory channels per processor socket (dpdk-only)
+num_mem_ch = 4
+
+# Used port (please adjust accordingly)
+port = dpdk0
+
+# Maximum concurrency per core
+max_concurrency = 1024
+
+# Maximum number of socket buffers per core
+# Set this to small value if there are many idle connections
+max_num_buffers = 100
+
+# Receive buffer size of sockets
+rcvbuf = 8192
+#rcvbuf = 1048576
+
+# Send buffer size of sockets
+sndbuf = 8192
+#sndbuf = 1048576
+
+# TCP timeout seconds
+# (tcp_timeout = -1 can disable the timeout check)
+tcp_timeout = 1
+
+# TCP timewait seconds
+tcp_timewait = 0
+
+# Interface to print stats (please adjust accordingly)
+# You can enable multiple ports in separate lines
+#------ PSIO ports -------#
+#stat_print = xge0
+#stat_print = xge1
+#------ DPDK ports -------#
+stat_print = dpdk0
+#stat_print = dpdk0:0
+#stat_print = dpdk0:1
+#stat_print = dpdk1
+
+#######################################################

experiments/guests/qemu-mtcp-server/mtcp.patch

+diff -ur mtcp.old/dpdk/lib/librte_mempool/rte_mempool.c mtcp/dpdk/lib/librte_mempool/rte_mempool.c
+--- mtcp.old/dpdk/lib/librte_mempool/rte_mempool.c	2020-09-08 21:17:34.052248962 +0200
++++ mtcp/dpdk/lib/librte_mempool/rte_mempool.c	2020-09-08 23:41:38.374992953 +0200
+@@ -916,6 +916,8 @@
+ 
+ 	mempool_list = RTE_TAILQ_CAST(rte_mempool_tailq.head, rte_mempool_list);
+ 
++        fprintf(stderr, "rte_mempool_create_empty: tart\n");
++
+ 	/* asked cache too big */
+ 	if (cache_size > RTE_MEMPOOL_CACHE_MAX_SIZE ||
+ 	    CALC_CACHE_FLUSHTHRESH(cache_size) > n) {
+@@ -933,6 +935,7 @@
+ 		return NULL;
+ 	}
+ 
++        fprintf(stderr, "rte_mempool_create_empty: calculated\n");
+ 	rte_rwlock_write_lock(RTE_EAL_MEMPOOL_RWLOCK);
+ 
+ 	/*
+@@ -954,6 +957,8 @@
+ 	mempool_size += private_data_size;
+ 	mempool_size = RTE_ALIGN_CEIL(mempool_size, RTE_MEMPOOL_ALIGN);
+ 
++        fprintf(stderr, "rte_mempool_create_empty: size=%zu\n", mempool_size);
++
+ 	ret = snprintf(mz_name, sizeof(mz_name), RTE_MEMPOOL_MZ_FORMAT, name);
+ 	if (ret < 0 || ret >= (int)sizeof(mz_name)) {
+ 		rte_errno = ENAMETOOLONG;
+@@ -961,8 +966,11 @@
+ 	}
+ 
+ 	mz = rte_memzone_reserve(mz_name, mempool_size, socket_id, mz_flags);
+-	if (mz == NULL)
++	if (mz == NULL) {
++            fprintf(stderr, "rte_mempool_create_empty: reserve failed\n");
++		RTE_LOG(ERR, MEMPOOL, "memzone reserve failed %zu\n", mempool_size);
+ 		goto exit_unlock;
++        }
+ 
+ 	/* init the mempool structure */
+ 	mp = mz->addr;
+@@ -972,6 +980,7 @@
+ 		rte_errno = ENAMETOOLONG;
+ 		goto exit_unlock;
+ 	}
++            fprintf(stderr, "rte_mempool_create_empty: init done\n");
+ 	mp->mz = mz;
+ 	mp->size = n;
+ 	mp->flags = flags;
+@@ -1028,6 +1037,7 @@
+ 
+ 	mp = rte_mempool_create_empty(name, n, elt_size, cache_size,
+ 		private_data_size, socket_id, flags);
++        fprintf(stderr, "rte_mempool_create_empty=%p\n", mp);
+ 	if (mp == NULL)
+ 		return NULL;
+ 
+@@ -1044,16 +1054,20 @@
+ 	else
+ 		ret = rte_mempool_set_ops_byname(mp, "ring_mp_mc", NULL);
+ 
++        fprintf(stderr, "rte_mempool_create: ops byname=%d\n", ret);
+ 	if (ret)
+ 		goto fail;
+ 
++        fprintf(stderr, "rte_mempool_create: before init\n");
+ 	/* call the mempool priv initializer */
+ 	if (mp_init)
+ 		mp_init(mp, mp_init_arg);
+ 
++        fprintf(stderr, "rte_mempool_create: before populate\n");
+ 	if (rte_mempool_populate_default(mp) < 0)
+ 		goto fail;
+ 
++        fprintf(stderr, "rte_mempool_create: before objinit\n");
+ 	/* call the object initializers */
+ 	if (obj_init)
+ 		rte_mempool_obj_iter(mp, obj_init, obj_init_arg);
+diff -ur mtcp.old/dpdk/lib/librte_mempool/rte_mempool_ops.c mtcp/dpdk/lib/librte_mempool/rte_mempool_ops.c
+--- mtcp.old/dpdk/lib/librte_mempool/rte_mempool_ops.c	2020-09-08 21:17:34.052248962 +0200
++++ mtcp/dpdk/lib/librte_mempool/rte_mempool_ops.c	2020-09-08 23:55:00.777498002 +0200
+@@ -23,6 +23,7 @@
+ 	struct rte_mempool_ops *ops;
+ 	int16_t ops_index;
+ 
++        fprintf(stderr, "rte_mempool_register_ops: h->name=(%s)\n", h->name);
+ 	rte_spinlock_lock(&rte_mempool_ops_table.sl);
+ 
+ 	if (rte_mempool_ops_table.num_ops >=
+@@ -30,6 +31,7 @@
+ 		rte_spinlock_unlock(&rte_mempool_ops_table.sl);
+ 		RTE_LOG(ERR, MEMPOOL,
+ 			"Maximum number of mempool ops structs exceeded\n");
++                fprintf(stderr, "rte_mempool_register_ops: (nospace)\n");
+ 		return -ENOSPC;
+ 	}
+ 
+@@ -38,6 +40,7 @@
+ 		rte_spinlock_unlock(&rte_mempool_ops_table.sl);
+ 		RTE_LOG(ERR, MEMPOOL,
+ 			"Missing callback while registering mempool ops\n");
++                fprintf(stderr, "rte_mempool_register_ops: (invalid)\n");
+ 		return -EINVAL;
+ 	}
+ 
+@@ -45,6 +48,7 @@
+ 		rte_spinlock_unlock(&rte_mempool_ops_table.sl);
+ 		RTE_LOG(DEBUG, EAL, "%s(): mempool_ops <%s>: name too long\n",
+ 				__func__, h->name);
++                fprintf(stderr, "rte_mempool_register_ops: (nametoolong)\n");
+ 		rte_errno = EEXIST;
+ 		return -EEXIST;
+ 	}
+@@ -159,8 +163,10 @@
+ 	unsigned i;
+ 
+ 	/* too late, the mempool is already populated. */
+-	if (mp->flags & MEMPOOL_F_POOL_CREATED)
++	if (mp->flags & MEMPOOL_F_POOL_CREATED) {
++                fprintf(stderr, "rte_mempool_set_ops_byname: already created\n");
+ 		return -EEXIST;
++        }
+ 
+ 	for (i = 0; i < rte_mempool_ops_table.num_ops; i++) {
+ 		if (!strcmp(name,
+@@ -170,8 +176,10 @@
+ 		}
+ 	}
+ 
+-	if (ops == NULL)
++	if (ops == NULL) {
++                fprintf(stderr, "rte_mempool_set_ops_byname: not found (%s)\n", name);
+ 		return -EINVAL;
++        }
+ 
+ 	mp->ops_index = i;
+ 	mp->pool_config = pool_config;
+Binary files mtcp.old/.git/index and mtcp/.git/index differ
+Binary files mtcp.old/.git/modules/dpdk/index and mtcp/.git/modules/dpdk/index differ
+Binary files mtcp.old/.git/modules/mtcp/src/libccp/index and mtcp/.git/modules/mtcp/src/libccp/index differ
+diff -ur mtcp.old/mtcp/src/io_module.c mtcp/mtcp/src/io_module.c
+--- mtcp.old/mtcp/src/io_module.c	2020-09-08 21:18:23.463852529 +0200
++++ mtcp/mtcp/src/io_module.c	2020-09-09 14:58:50.360193608 +0200
+@@ -303,12 +303,14 @@
+ 				       RTE_CACHE_LINE_SIZE);
+ 		
+ 		/* initialize the rte env, what a waste of implementation effort! */
+-		int argc = 6;//8;
++		int argc = 8;//8;
+ 		char *argv[RTE_ARGC_MAX] = {"",
+ 					    "-c",
+ 					    cpumaskbuf,
+ 					    "-n",
+ 					    mem_channels,
++                                            "--log-level",
++                                            "*:debug",
+ #if 0
+ 					    "--socket-mem",
+ 					    socket_mem_str,

experiments/guests/qemu-mtcp-server/run.sh

+#!/bin/bash
+export HOME=/root
+export LANG=en_US
+export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games"
+
+mount -t proc proc /proc
+mount -t sysfs sysfs /sys
+mkdir -p /dev/hugepages
+mount -t hugetlbfs nodev /dev/hugepages
+#-o pagesize=1G
+mkdir -p /dev/shm
+mount -t tmpfs tmpfs /dev/shm
+#find /sys/
+#modprobe vfio-pci
+#echo 1 >/sys/module/vfio/parameters/enable_unsafe_noiommu_mode
+insmod /root/mtcp/dpdk/x86_64-native-linuxapp-gcc/kmod/igb_uio.ko
+/root/mtcp/dpdk/usertools/dpdk-devbind.py -b igb_uio 0000:00:02.0
+find /sys/devices/system/node/node0/hugepages/
+#echo 4> /sys/devices/system/node/node0/hugepages/hugepages-1048576kB/nr_hugepages
+echo 4096 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages
+#echo 4096 > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages
+insmod /root/mtcp/dpdk-iface-kmod/dpdk_iface.ko
+/root/mtcp/dpdk-iface-kmod/dpdk_iface_main
+ip link
+ip link set dev dpdk0 up
+ip addr add 192.168.64.1/24 dev dpdk0
+
+
+
+#cd /root/mtcp
+#patch -p1 </tmp/guest/mtcp.patch
+#export RTE_SDK=/root/mtcp/dpdk
+#export RTE_TARGET=x86_64-native-linuxapp-gcc
+#
+#make -C dpdk install T=$RTE_TARGET
+#make
+
+cd /root/tasbench/micro_rpc
+#make clean
+#make echoserver_mtcp MTCP_BASE="/root/mtcp" TAS_CODE="/root/tas"
+#rm -rf /var/run/dpdk /dev/hugepages/*
+./echoserver_mtcp 1234 1 /tmp/guest/mtcp.conf 128 1024
+poweroff