doc: add documentation skeleton

Makefile

@@ -13,6 +13,7 @@ VFLAGS = +1364-2005ext+v \
 
 $(eval $(call subdir,lib))
 $(eval $(call subdir,sims))
+$(eval $(call subdir,doc))
 $(eval $(call subdir,images))
 
 
@@ -21,6 +22,7 @@ help:
 	@echo "  all: builds all the tools directly in this repo"
 	@echo "  clean: cleans all the tool folders in this repo"
 	@echo "  build-images: prepare prereqs for VMs (images directory)"
+	@echo "  documentation: build documentation in doc/build_"
 	@echo "  external: clone and build our tools in external repos "
 	@echo "            (qemu, gem5, ns-3)"
 

doc/api.rst

+API
+###################################
+
+The full API documentation extracted by doxygen can be found
+`here <https://tas.readthedocs.io/en/latest/_static/doxygen/>`_.
+
+
+TAS Low-Level Application API
+******************************
+
+.. doxygenfunction:: flextcp_init
+
+Contexts
+=========================
+
+.. doxygenstruct:: flextcp_context
+.. doxygenfunction:: flextcp_context_create
+.. doxygenfunction:: flextcp_context_poll
+.. doxygenfunction:: flextcp_block
+
+
+Connections
+=========================
+
+.. doxygenstruct:: flextcp_connection
+.. doxygenfunction:: flextcp_connection_open
+.. doxygenfunction:: flextcp_connection_close
+.. doxygenfunction:: flextcp_connection_rx_done
+.. doxygenfunction:: flextcp_connection_tx_alloc
+.. doxygenfunction:: flextcp_connection_tx_alloc2
+.. doxygenfunction:: flextcp_connection_tx_send
+.. doxygenfunction:: flextcp_connection_tx_close
+.. doxygenfunction:: flextcp_connection_tx_possible
+.. doxygenfunction:: flextcp_connection_move
+
+Listeners
+=========================
+
+.. doxygenstruct:: flextcp_listener
+.. doxygendefine:: FLEXTCP_LISTEN_REUSEPORT
+.. doxygenfunction:: flextcp_listen_open
+.. doxygenfunction:: flextcp_listen_accept
+
+Events
+=========================
+
+.. doxygenenum:: flextcp_event_type
+
+.. doxygenstruct:: flextcp_event
+  :members:
+
+TAS Sockets API
+******************************
+

doc/conf.py

+# -*- coding: utf-8 -*-
+#
+# Configuration file for the Sphinx documentation builder.
+#
+# This file does only contain a selection of the most common options. For a
+# full list see the documentation:
+# http://www.sphinx-doc.org/en/master/config
+
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- Project information -----------------------------------------------------
+
+project = 'SimBricks'
+copyright = '2021 Max Planck Institute for Software Systems, ' + \
+        'and National University of Singapore'
+
+author = 'Antoine Kaufmann'
+
+# The short X.Y version
+version = ''
+# The full version, including alpha/beta/rc tags
+release = ''
+
+
+# -- General configuration ---------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'breathe',
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = None
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'sphinx_rtd_theme'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# Custom sidebar templates, must be a dictionary that maps document names
+# to template names.
+#
+# The default sidebars (for documents that don't match any pattern) are
+# defined by theme itself.  Builtin themes are using these templates by
+# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
+# 'searchbox.html']``.
+#
+# html_sidebars = {}
+
+
+# -- Options for HTMLHelp output ---------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'SimBricksdoc'
+
+
+# -- Options for LaTeX output ------------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, 'simbricks.tex', 'SimBricks Documentation',
+     'Antoine Kaufmann', 'manual'),
+]
+
+
+# -- Options for manual page output ------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (master_doc, 'simbricks', 'SimBricks Documentation',
+     [author], 1)
+]
+
+
+# -- Options for Texinfo output ----------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'simbricks', 'SimBricks Documentation',
+     author, 'SimBricks', 'One line description of project.',
+     'Miscellaneous'),
+]
+
+
+# -- Options for Epub output -------------------------------------------------
+
+# Bibliographic Dublin Core info.
+epub_title = project
+
+# The unique identifier of the text. This can be a ISBN number
+# or the project homepage.
+#
+# epub_identifier = ''
+
+# A unique identification for the text.
+#
+# epub_uid = ''
+
+# A list of files that should not be packed into the epub file.
+epub_exclude_files = ['search.html']
+
+breathe_projects = { 'simbricks': 'doxygen/xml/' }
+breathe_default_project = 'simbricks'
+
+
+#################################################################################
+# For RTD
+import subprocess, os
+
+# Check if we're running on Read the Docs' servers
+read_the_docs_build = os.environ.get('READTHEDOCS', None) == 'True'
+
+if read_the_docs_build:
+    subprocess.call('cd .. && doxygen doc/Doxyfile', shell=True)
+    subprocess.call('cp -r doxygen/html _static/doxygen', shell=True)

doc/devel.rst

+###################################
+Developer Guide
+###################################
+
+
+******************************
+Code Structure
+******************************
+
+  * ``tas/``: service implementation
+
+    + ``tas/fast``: TAS fast path
+
+    + ``tas/slow``: TAS slow path
+
+  * ``lib/``: client libraries
+
+    + ``lib/tas``: lowlevel TAS client library (interface:
+      ``lib/tas/include/tas_ll.h``)
+
+    + ``lib/sockets``: socket emulation layer
+
+  * ``tools/``: debugging tools

doc/index.h

+/**
+ * @mainpage SimBricks Overview
+ *
+ * Components:
+ *    - TAS. (@ref tas)
+ *      + Slow Path: the control plane. (@ref tas-sp)
+ */

doc/index.rst

+SimBricks
+=========
+
+.. toctree::
+   :numbered:
+   :maxdepth: 2
+   :caption: Contents:
+
+   user
+   devel
+   api
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`

doc/mk

+../mk
\ No newline at end of file

doc/requirements.txt

+breathe

doc/rules.mk

+include mk/subdir_pre.mk
+
+doxygen_outdir := $(d)doxygen
+doxygen_srcs := $(wildcard $(d)/*.h)
+
+sphinx_outdir := $(d)_build
+sphinx_srcs := $(wildcard $(d)/*.rst $(d)/*/*.rst)
+
+documentation: $(doxygen_outdir) $(sphinx_outdir)
+.PHONY: documentation
+
+$(doxygen_outdir): $(d)Doxyfile $(doxygen_srcs)
+	cd $(base_dir). && doxygen doc/Doxyfile
+
+$(sphinx_outdir): $(d)conf.py $(sphinx_srcs)
+	cd $(base_dir). && sphinx-build doc/ doc/_build
+
+CLEAN := $(doxygen_outdir) $(sphinx_outdir)
+
+include mk/subdir_post.mk

doc/user.rst

+###################################
+User Guide
+###################################
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+   user/quickstart
+   user/parameters
+   user/troubleshooting

doc/user/parameters.rst

+###################################
+TAS Command-Line Parameters
+###################################
+
+******************************
+IP Configuration
+******************************
+
+   *  ``--ip-addr=ADDR[/PREFIXLEN]``
+
+      Set local IP address. Currently only exactly one IP address is supported.
+
+   *  ``--ip-route=DEST[/PREFIX],NEXTHOP``
+
+      Add an IP route for the destination subnet ``DEST/PREFIX`` via ``NEXTHOP``.
+      Can be specified more than once.
+      For example, a default route could be ``--ip-route=0.0.0.0/0,192.168.1.1``.
+
+
+******************************
+Fast Path Configuration
+******************************
+
+   *  ``--fp-cores-max=CORES``
+
+      Maximum number of cores to use for fast-path. (default: 1)
+
+   *  ``--fp-no-ints``
+
+      Disable receive interrupts in the NIC driver, switches over to just
+      polling.
+
+   *  ``--fp-no-xsumoffload``
+
+      Disable transmit checksum offloads, primarily useful to run TAS with NICs
+      that do not support checksum offload, but comes at a slight performance
+      cost.
+
+   *  ``--fp-no-autoscale``
+
+      Disable auto scaling, instead fix the number of cores used by the fast
+      path to the maximum.
+
+   *  ``--fp-no-hugepages``
+
+      Do not use huge pages for the shared memory region between TAS and
+      applications. (DPDK still uses huge pages for it's buffers unless
+      explicitly disabled through ``--dpdk-extra``)
+
+   *  ``--dpdk-extra=ARG``
+
+      Pass ``ARG`` through as a parameter to the dpdk EAL. (see
+      https://doc.dpdk.org/guides/linux_gsg/linux_eal_parameters.html)
+
+
+******************************
+TCP Protocol Parameters
+******************************
+
+   *  ``--tcp-rtt-init=RTT``
+
+      Initial RTT used for congestion control. Is updated with actual
+      measurements when they arrive.
+
+   *  ``--tcp-link-bw=BANDWIDH``
+
+      Link bandwidth in GBPS. TODO: what is this used for? (default: 10).
+
+   *  ``--tcp-rxbuf-len=LEN``
+
+      Connection receive buffer len in bytes (default: 8,192).
+
+   *  ``--tcp-txbuf-len=LEN``
+
+      Connection transmit buffer len in bytes (default: 8,192).
+
+   *  ``--tcp-handshake-timeout=TIMEOUT``
+
+      TCP handshake timeout in microseconds (default 10,000us).
+
+   *  ``--tcp-handshake-retries=RETRIES``
+
+      Maximum retries for timeouts during handshake.  (default: 10).
+
+
+******************************
+Congestion Control Parameters
+******************************
+
+   *  ``--cc=ALGORITHM``
+
+      Choose which congestion control algorithm to use. The supported options
+      are:
+
+         +  ``dctcp-rate``: dctcp algorithm adapted to directly operate on the
+            connection rate.
+
+         +  ``dctcp-win``: original dctcp algorithm with the window converted to
+            a rate for enforcement.
+
+         + ``timely``: latency-based TIMELY control law.
+
+         + ``const-rate``: set all connections to a constant rate (effectively
+           disables congestion control, useful for debugging).
+
+   *  ``--cc-control-interval=INT``
+
+      Control interval length as multiples of the connection's RTT. (default: 2)
+
+   *  ``--cc-control-granularity=G``
+
+      Minimal control loop granularity. Control loop is only executed at most
+      once every ``G`` microseconds. (default: 50)
+
+   *  ``--cc-rexmit-ints=INTERVALS``
+
+      Number of connection cnotrol intervals before TAS triggers a re-transmit.
+      (default: 4).
+
+DCTCP
+=========================
+For the ``dctcp-rate`` and ``dctcp-win`` algorithm:
+
+   *  ``--cc-dctcp-weight=WEIGHT``
+
+      EWMA weight for dctcp's ECN rate (alpha, default: 0.0625).
+
+   *  ``--cc-dctcp-mimd=INC_FACT``
+
+      Enable mutliplicative increase by ``INC_FACT`` (disabled by default, only
+      used for tests).
+
+   *  ``--cc-dctcp-min=RATE``
+
+      Minimum rate to set for flows (kbps, default: 10000).
+
+Timely
+=========================
+Parameters for the ``timely`` algorithm:
+
+   *  ``--cc-timely-tlow=TIME``
+
+      Tlow threshold in microseconds. (default: 30)
+
+   *  ``--cc-timely-thigh=TIME``
+
+      Thigh threshold in microseconds. (default: 150)
+
+   *  ``--cc-timely-step=STEP``
+
+      Additive increase step size in kbps (default: 10000)
+
+   *  ``--cc-timely-init=RATE``
+
+      Initial connection rate in kbps (default: 10000)
+
+   *  ``--cc-timely-alpha=FRAC``
+
+      EWMA weight for rtt diff. (default: 0.02)
+
+   *  ``--cc-timely-beta=FRAC``
+
+      Multiplicative decrease factor. (default: 0.8)
+
+   *  ``--cc-timely-minrtt=RTT``
+
+      Minimal RTT without queueing in microseconds. (default: 11)
+
+   *  ``--cc-timely-minrate=RTT``
+
+      Minimal connection rate to use in kbps (default: 10000)
+
+Constant Rate
+=========================
+For the ``const-rate`` "algorithm" the following configuration options apply:
+
+   *  ``--cc-const-rate=RATE``
+
+      Sets the rate to use in kbps.
+
+
+******************************
+ARP Protocol Parameters
+******************************
+
+   *  ``--arp-timeout=TIMEOUT``
+
+      Initial ARP request timetout in microseconds. This doubles with every
+      retry (default: 500).
+
+   *  ``--arp-timeout-max=TIMEOUT``
+
+      Maximal ARP timeout in microseconds. If the retry-timeout grows larger
+      than this, the request fails. (default: 10,000,000 us)
+
+
+******************************
+Slowpath Queues
+******************************
+
+   *  ``--nic-rx-len=LEN``
+
+      Number of entries in TAS slowpath receive queue. (default: 16,384).
+
+   *  ``--nic-tx-len=LEN``
+
+      Number of entries in TAS slowpath transmit queue. (default: 16,384).
+
+   *  ``--app-kin-len=LEN``
+
+      Application slow path receive queue length in bytes. (default: 1,048,576).
+
+   *  ``--app-kout-len=LEN``
+
+      Application slow path transmit queue length in bytes. (default: 1,048,576).
+
+
+******************************
+Host Kernel Interface
+******************************
+
+   *  ``--kni-name=NAME``
+
+      Enables the DPDK kernel network interface, by creating a dummy network
+      interface with the name ``NAME``. (default: disabled)
+
+
+******************************
+Miscellaneous
+******************************
+
+   *  ``--quiet``
+
+       Disable non-essential logging.
+
+   *  ``--ready-fd=FD``
+
+      Causes TAS to write to file descriptor ``FD`` when ready. Can be used by
+      supervisor processes to detect when TAS is ready, e.g. used in full system
+      tests.

doc/user/quickstart.rst

+###################################
+Quick Start
+###################################
+
+******************************
+Building
+******************************
+
+Requirements:
+
+  * TAS is built on top of Intel DPDK for direct access to the NIC. We have
+    tested this version with dpdk versions 17.11.9, 18.11.5, 19.11.
+
+Assuming that dpdk is installed in ``~/dpdk-inst`` TAS can be built as follows
+(for a system installation of dpdk the ``RTE_SDK`` variable does not need to be
+passed explicitly):
+
+.. code-block:: bash
+
+   make RTE_SDK=~/dpdk-inst
+
+
+This will build the TAS service (binary ``tas/tas``), client libraries (in
+``lib/``), and a few debugging tools (in ``tools/``).
+
+
+******************************
+Running
+******************************
+
+Before running TAS the following steps are necessary:
+
+   * Make sure ``hugetlbfs`` is mounted on ``/dev/hugepages`` and enough huge
+     pages are allocated for TAS and dpdk.
+
+   * Binding the NIC to the dpdk driver, as with any other dpdk application (for
+     Intel NICs use ``vfio`` because ``uio`` does not support multiple
+     interrupts).
+
+.. code-block:: bash
+
+   sudo modprobe vfio-pci
+   sudo mount -t hugetlbfs nodev /dev/hugepages
+   echo 1024 | sudo tee /sys/devices/system/node/node*/hugepages/hugepages-2048kB/nr_hugepages
+   sudo ~/dpdk-inst/sbin/dpdk-devbind  -b vfio-pci 0000:08:00.0
+
+To run (``--ip-addr`` and ``--fp-cores-max`` are the minimum arguments typically
+needed to run tas):
+
+.. code-block:: bash
+
+   sudo code/tas/tas --ip-addr=10.0.0.1/24 --fp-cores-max=2
+
+Once tas is running, applications that directly link to ``libtas`` or
+``libtas_sockets`` can be run directly. To run an unmodified application with
+sockets interposition run as follows (for example):
+
+.. code-block:: bash
+
+   sudo LD_PRELOAD=lib/libtas_interpose.so ../benchmarks/micro_rpc/echoserver_linux 1234 1 foo 8192 1
+
+
+******************************
+In Qemu/KVM
+******************************
+
+For functional testing and development TAS can run in Qemu (with or without
+acceleration through KVM). We have tested this with the ``virtio`` dpdk driver.
+By default, the qemu virtio device only provides a single queue, and thus only
+allows TAS to run on a single core. To run a virtual machine with support for
+multiple queue, qemu requires a tap device with multi-queue support enabled.
+
+Here is an example sequence of commands to create a tap device with multi queue
+support and then start a qemu instance that binds this tap device to a
+multi-queue virtio device:
+
+.. code-block:: bash
+
+   sudo ip link add tastap0 type tuntap
+   sudo ip tuntap add mode tap multi_queue name tastap0
+   sudo ip link set dev tastap0 up
+   qemu-system-x86_64 \
+       -machine q35 -cpu host \
+       -drive file=vm1.qcow2,if=virtio \
+       -netdev tap,ifname=tastap0,script=no,downscript=no,vhost=on,queues=8,id=nInt\
+       -device virtio-net-pci,mac=52:54:00:12:34:56,vectors=18,mq=on,netdev=nInt \
+       -serial mon:stdio -m 8192 -smp 16 -display none -enable-kvm
+
+
+Inside the virtual machine, the following sequence of commands first takes the
+linux network interface down, binds it to the ``uio_pci_generic`` driver that
+the dpdk virtio PMD supports, and then reserves huge pages:
+
+.. code-block:: bash
+
+   sudo ifconfig enp0s2 down
+   sudo modprobe uio
+   sudo modprobe uio_pci_generic
+   sudo dpdk-devbind.py -b uio_pci_generic 0000:00:02.0
+   echo 1024 | sudo tee /sys/devices/system/node/node*/hugepages/hugepages-2048kB/nr_hugepages
+
+Virtio does not support all the NIC features that we depend on in physical NICs.
+In particular virtio does not support transmit checksum offload or the RSS
+redirection table TAS uses for scaling up and down. The dpdk virtio PMD also
+does not support multiple MSI-X interrupts.  To run TAS given these constraints,
+the following command line parameters disable the use of these features (note
+that this implies busy polling and no autoscaling):
+
+.. code-block:: bash
+
+   sudo code/tas/tas --ip-addr=10.0.0.1/24 --fp-cores-max=8 \
+       --fp-no-xsumoffload --fp-no-ints --fp-no-autoscale
+
+
+******************************
+Kernel NIC Interface
+******************************
+
+TAS supports the DPDK kernel NIC interface (KNI) to pass packets to the Linux
+kernel network stack. With KNI enabled, TAS becomes an opt-in fastpath where
+TAS-enabled applications operate through TAS, and other applications can use the
+Linux network stack as before, sharing the same physical NIC.
+
+To run TAS with KNI the first step is to load the ``rte_kni`` kernel module.
+Next, when run with the ``--kni-name=`` option, TAS will create a KNI dummy
+network interface with the specified name. After assigning an IP address to this
+network interface, the Linux network stack can send and receive packets through
+this interface as long as TAS is running. Here is the complete sequence of
+commands:
+
+.. code-block:: bash
+
+   sudo modprobe rte_kni
+   sudo code/tas/tas --ip-addr=10.0.0.1/24 --kni-name=tas0
+   # in separate terminal
+   sudo ifconfig tas0 10.0.0.1/24 up

doc/user/troubleshooting.rst

+###################################
+TAS Troubleshooting
+###################################