veros

doc/_static/veros-logo-bright-400px.png

+../_images/veros-logo-bright-400px.png
\ No newline at end of file

doc/conf.py

+# -*- coding: utf-8 -*-
+#
+# Veros documentation build configuration file, created by
+# sphinx-quickstart on Tue Mar  7 23:56:46 2017.
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+
+import os
+import sys
+sys.path.insert(0, os.path.abspath('..'))
+sys.path.insert(0, os.path.abspath('_3rdparty'))
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+import sphinx_fontawesome
+
+extensions = [
+    'sphinx.ext.autodoc',
+    'sphinx.ext.autosummary',
+    'sphinx.ext.coverage',
+    'sphinx.ext.viewcode',
+    'sphinx.ext.mathjax',
+    'sphinx.ext.napoleon',
+    'sphinx.ext.intersphinx',
+    'sphinx_fontawesome',
+    'IPython.sphinxext.ipython_console_highlighting',
+    'IPython.sphinxext.ipython_directive',
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = u'Veros'
+copyright = u'2017-2023, The Veros Team, NBI Copenhagen'
+author = u'The Veros Team, NBI Copenhagen'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+from veros import __version__ as veros_version
+
+if "0+untagged" in veros_version:
+    veros_version = "main"
+
+# The short X.Y version.
+version = veros_version
+# The full version, including alpha/beta/rc tags.
+release = veros_version
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = 'en'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This patterns also effect to html_static_path and html_extra_path
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', '_generated']
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = False
+
+# Redefine the default output path of ipython directive @savefig
+ipython_savefig_dir = '_generated'
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = "furo"
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+html_theme_options = {
+    "light_logo": "veros-logo-400px.png",
+    "dark_logo": "veros-logo-bright-400px.png",
+    "sidebar_hide_name": True,
+}
+
+html_css_files = [
+    "https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.1.1/css/all.min.css"
+]
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ["_static"]
+
+
+# -- Options for HTMLHelp output ------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'veros_doc'
+
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, 'veros.tex', u'Veros Documentation',
+     u'The Veros Team', 'manual'),
+]
+
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (master_doc, 'Veros', u'Veros Documentation',
+     [author], 1)
+]
+
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'Veros', u'Veros Documentation',
+     author, 'Veros', 'The versatile ocean simulator in pure Python',
+     'Miscellaneous'),
+]
+
+# -- Options for autodoc --------------------------------------------------
+autodoc_member_order = "bysource"
+autodoc_default_options = {"show-inheritance": None}
+autodoc_mock_imports = ["loguru", "numpy", "h5netcdf", "scipy"]
+
+# -- Options for intersphinx ----------------------------------------------
+intersphinx_mapping = {'python': ('https://docs.python.org/3', None)}
+
+# -- Custom directives ----------------------------------------------------
+
+from os.path import basename
+
+try:
+    from StringIO import StringIO
+except ImportError:
+    from io import StringIO
+
+from docutils import nodes, statemachine
+from docutils.parsers.rst import Directive, directives
+
+
+class ExecDirective(Directive):
+    """Execute the specified python code and insert the output into the document"""
+    has_content = True
+
+    def run(self):
+        old_stdout, sys.stdout = sys.stdout, StringIO()
+
+        tab_width = self.options.get('tab-width', self.state.document.settings.tab_width)
+        source = self.state_machine.input_lines.source(self.lineno - self.state_machine.input_offset - 1)
+
+        try:
+            exec('\n'.join(self.content))
+            text = sys.stdout.getvalue()
+            lines = statemachine.string2lines(text, tab_width, convert_whitespace=True)
+            self.state_machine.insert_input(lines, source)
+            return []
+        except Exception:
+            warn_text = "Unable to execute python code at %s:%d" % (basename(source), self.lineno)
+            warning = self.state_machine.reporter.warning(warn_text)
+            return [warning, nodes.error(None, nodes.paragraph(text=warn_text), nodes.paragraph(text=str(sys.exc_info()[1])))]
+        finally:
+            sys.stdout = old_stdout
+
+
+class ClickDirective(Directive):
+    """Execute the specified click command and insert the output into the document"""
+    required_arguments = 1
+    optional_arguments = 0
+    option_spec = {
+        'args': directives.unchanged
+    }
+    has_content = False
+
+    def run(self):
+        import importlib
+        import shlex
+        from click.testing import CliRunner
+
+        arg = self.arguments[0]
+        options = shlex.split(self.options.get('args', ''))
+
+        try:
+            modname, funcname = arg.split(':')
+        except ValueError:
+            raise self.error('run-click argument must be "module:function"')
+
+        try:
+            mod = importlib.import_module(modname)
+            func = getattr(mod, funcname)
+
+            runner = CliRunner()
+            with runner.isolated_filesystem():
+                result = runner.invoke(func, options)
+
+            text = result.output
+
+            if result.exit_code != 0:
+                raise RuntimeError('Command exited with non-zero exit code; output: "%s"' % text)
+
+            node = nodes.literal_block(text=text)
+            node['language'] = 'text'
+            return [node]
+
+        except Exception:
+            warn_text = "Error while running command %s %s" % (arg, ' '.join(map(shlex.quote, options)))
+            warning = self.state_machine.reporter.warning(warn_text)
+            return [warning, nodes.error(None, nodes.paragraph(text=warn_text), nodes.paragraph(text=str(sys.exc_info()[1])))]
+
+
+def setup(app):
+    app.add_directive('exec', ExecDirective)
+    app.add_directive('run-click', ClickDirective)

doc/docutils.conf

+[general]
+footnote_backlinks=false

doc/index.rst

+:tocdepth: 5
+
+Versatile Ocean Simulation in Pure Python
+=========================================
+
+Veros, *the versatile ocean simulator*, aims to be the swiss army knife of ocean modeling. It is a full-fledged primitive equation ocean model that supports anything between idealized toy models and `realistic, high-resolution, global ocean simulations <https://agupubs.onlinelibrary.wiley.com/doi/10.1029/2021MS002717>`_. And because Veros is written in pure Python, the days of struggling with complicated model setup workflows, ancient programming environments, and obscure legacy code are finally over.
+
+*In a nutshell, we want to enable high-performance ocean modelling with a clear focus on flexibility and usability.*
+
+Veros supports a NumPy backend for small-scale problems, and a
+high-performance `JAX <https://github.com/google/jax>`_ backend
+with CPU and GPU support. It is fully parallelized via MPI and supports
+distributed execution on any number of nodes, including multi-GPU architectures (see also :doc:`our benchmarks </more/benchmarks>`).
+
+The dynamical core of Veros is based on `pyOM2 <https://wiki.cen.uni-hamburg.de/ifm/TO/pyOM2>`_, an ocean model with a Fortran backend and Fortran and Python frontends.
+
+If you want to learn more about the background and capabilities of Veros, you should check out :doc:`introduction/introduction`. If you are already convinced, you can jump right into action, and :doc:`learn how to get started <introduction/get-started>` instead!
+
+.. image:: /_images/tagline.png
+   :scale: 50%
+   :class: no-scaled-link
+   :alt: ... because the Baroque is over.
+
+.. seealso::
+
+   We outline some of our design philosophy and current direction in `this blog post <https://dionhaefner.github.io/2021/04/higher-level-geophysical-modelling/>`__.
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Start here
+
+   introduction/introduction
+   introduction/get-started
+   introduction/advanced-installation
+
+.. toctree::
+   :maxdepth: 1
+   :caption: Tutorials
+
+   tutorial/analysis
+   tutorial/cluster
+   tutorial/dev
+   tutorial/erda
+
+.. toctree::
+   :maxdepth: 1
+   :caption: Reference
+
+   reference/setup-gallery
+   reference/settings
+   reference/variables
+   reference/diagnostics
+   reference/cli
+   reference/public-api
+
+.. toctree::
+   :maxdepth: 1
+   :caption: Plug-ins
+
+   plugins/user-guide
+   Biogeochemistry plugin (external) <https://veros-bgc.readthedocs.io>
+   Extra setups plugin (external) <https://veros-extra-setups.readthedocs.io>
+   Sea ice plugin (external) <https://veris.readthedocs.io>
+   plugins/developer-guide
+
+.. toctree::
+   :maxdepth: 2
+   :caption: More Information
+
+   more/benchmarks
+   more/external_tools
+   more/howtocite
+   more/publications
+   Visit us on GitHub <https://github.com/team-ocean/veros>

doc/introduction/advanced-installation.rst

+Advanced installation
+=====================
+
+Because Veros should be usable by both beginners and experts, it has a number of *optional* dependencies that are not strictly required to run Veros, but highly recommended in high-performance contexts.
+
+.. note::
+
+    Veros notifies you when a slower code path has to be taken due to a missing dependency. So unless you are getting a warning, there is usually no need to install optional dependencies (except JAX and MPI).
+
+
+.. list-table:: Summary of optional dependencies
+    :widths: auto
+    :width: 100%
+    :header-rows: 1
+
+    * - Dependency
+      - Supported platforms
+      - When to use
+    * - JAX
+      - Linux, OSX, Windows
+      - To run 4x faster on CPU, and for GPU support
+    * - Veros Cython extensions
+      - Linux, OSX
+      - 20% speedup when using JAX
+    * - MPI + mpi4py
+      - Linux, OSX
+      - To run in parallel
+    * - mpi4jax
+      - Linux, OSX
+      - To run in parallel with JAX
+    * - PETSc + petsc4py
+      - Linux, OSX
+      - | Faster linear solver when using
+        | more than 10 processes (or GPUs)
+
+.. note::
+
+    On this page, we give all installation instructions via ``pip install``. If you used conda to install Veros, consider replacing them with ``conda install``.
+
+Using JAX
+---------
+
+Using the JAX backend, Veros is typically about 4x faster than with NumPy, so this should be the first thing to try if you want to get more performance.
+
+JAX is available on all major platforms and can be installed via::
+
+   $ pip install jax jaxlib
+
+To use JAX on GPU, you have to install a CUDA-enabled version of jaxlib, e.g.::
+
+   $ pip install jax jaxlib==0.1.67+cuda111 -f https://storage.googleapis.com/jax-releases/jax_releases.html
+
+(see also `the JAX installation guide <https://github.com/google/jax#installation>`__).
+
+Veros also supplies Cython extensions that optimize certain bottlenecks in JAX. You can make sure they are installed by running::
+
+   $ pip install cython
+   $ python setup.py build_ext --inplace
+
+in the Veros repository root.
+
+Using MPI
+---------
+
+To run Veros on more than one process you need to use MPI. This requires that you install an MPI implementation (such as OpenMPI) on your system. Additionally, you have to install ``mpi4py`` to interface with it::
+
+   $ pip install mpi4py
+
+Then, you can :ref:`run Veros in parallel via MPI <mpi-exec>`.
+
+For optimal performance on many processes, Veros supports using PETSc as a linear solver. To use it, you will have to install the PETSc library and ``petsc4py`` Python package::
+
+   $ PETSC_DIR=/path/to/petsc3.12 pip install petsc4py==3.12
+
+Note that the versions of PETSc and ``petsc4py`` have to match.
+
+
+Using JAX + MPI
+---------------
+
+To use JAX together with MPI, you need to install ``mpi4jax`` after installing ``mpi4py``::
+
+   $ pip install mpi4jax

doc/introduction/get-started.rst

+Getting started
+===============
+
+Installation
+------------
+
+Quick installation via pip
+++++++++++++++++++++++++++
+
+.. warning::
+
+  You should only install Veros this way if you want to get going as quickly as possible, and do not plan to access or modify the model source code. The recommended way to install Veros is by checking out the repository (see below).
+
+If you already have Python installed, the quickest way to get a working Veros installation is to run ::
+
+  $ pip install veros
+
+or, optionally::
+
+  $ pip install veros[jax]
+
+to use Veros with JAX.
+
+
+Using Conda (multi-platform)
+++++++++++++++++++++++++++++
+
+1. `Download and install Miniconda <https://docs.conda.io/en/latest/miniconda.html>`__. If you are using Windows, you may use the Anaconda prompt to execute the following steps.
+
+2. Clone the Veros repository:
+
+   .. exec::
+
+      from veros import __version__ as veros_version
+      if "0+untagged" in veros_version:
+          veros_version = "main"
+      else:
+          veros_version = f"v{veros_version}"
+      if "+" in veros_version:
+          veros_version, _ = veros_version.split("+")
+      print(".. code-block::\n")
+      print(f"   $ git clone https://github.com/team-ocean/veros.git -b {veros_version}")
+
+   (or `any other version of Veros <https://github.com/team-ocean/veros/releases>`__).
+
+   If you do not have git installed, you can do so via ``conda install git``.
+
+3. Create a new conda environment for Veros, and install all relevant dependencies by running ::
+
+       $ cd veros/  # if not already in the veros directory
+       $ conda env create -f conda-environment.yml
+
+4. To use Veros, just activate your new conda environment via ::
+
+       $ conda activate veros
+
+
+Using pip (Linux / OSX)
++++++++++++++++++++++++
+
+1. Ensure you have a working Python 3.x installation.
+
+2. Clone our repository:
+
+   .. exec::
+
+      from veros import __version__ as veros_version
+      if "0+untagged" in veros_version:
+          veros_version = "main"
+      else:
+          veros_version = f"v{veros_version}"
+      if "+" in veros_version:
+          veros_version, _ = veros_version.split("+")
+      print(".. code-block::\n")
+      print(f"   $ git clone https://github.com/team-ocean/veros.git -b {veros_version}")
+
+   (or `any other version of Veros <https://github.com/team-ocean/veros/releases>`__), or use ::
+
+      $ pip download veros
+
+   to download a tarball of the latest version (needs to be unpacked).
+
+3. Install Veros (preferably in a virtual environment) via ::
+
+      $ pip install -e ./veros
+
+   You might have to add the ``--user`` flag to ``pip install`` if you are using your system interpreter. The ``-e`` flag ensures that changes to the code are immediately reflected without reinstalling.
+
+4. Optionally, install JAX via ::
+
+      $ pip install -e ./veros[jax]
+
+
+Setting up a model
+------------------
+
+To run Veros, you need to set up a model - i.e., specify which settings and model domain you want to use. This is done by subclassing the :class:`Veros setup base class <veros.VerosSetup>` in a *setup script* that is written in Python. You should have a look at the pre-implemented model setups in the repository's :file:`setup` folder, or use the :command:`veros copy-setup` command to copy one into your current folder. A good place to start is the :class:`ACC model <acc.ACCSetup>`::
+
+    $ veros copy-setup acc
+
+By working through the existing models, you should quickly be able to figure out how to write your own simulation. Just keep in mind this general advice:
+
+- You can (and should) use any (external) Python tools you want in your model setup. Before implementing a certain functionality, you should check whether it is already provided by a common library. Especially `the SciPy module family <https://www.scipy.org/>`_ provides countless implementations of common scientific functions (and SciPy is installed along with Veros).
+
+- You have to decorate your methods with :func:`@veros_routine <veros.veros_routine>`. Only Veros routines are able to modify the :class:`model state object <veros.VerosState>`, which is passed as the first argument. The current numerical backend is available from the :mod:`veros.core.operators` module::
+
+      from veros import VerosSetup, veros_routine
+      from veros.core.operators import numpy as npx
+
+      class MyVerosSetup(VerosSetup):
+          ...
+          @veros_routine
+          def my_function(self, state):
+              arr = npx.array([1, 2, 3, 4]) # "npx" uses either NumPy or JAX
+
+- If you are curious about the general process how a model is set up and ran, you should read the source code of :class:`veros.VerosSetup` (especially the :meth:`setup` and :meth:`run` methods). This is also the best way to find out about the order in which routines are called.
+
+- Out of all functions that need to be implemented by your subclass of :class:`veros.VerosSetup`, the only one that is called in every time step is :meth:`set_forcing` (at the beginning of each iteration). This implies that, to achieve optimal performance, you should consider moving calculations that are constant in time to other functions.
+
+- There is another type of decorator called :func:`@veros_kernel <veros.veros_kernel>`. A kernel is a pure function that may be compiled to machine code by JAX. Kernels typically execute much faster, but are more restrictive to implement, as they cannot interact with the model state directly.
+
+  A common pattern in large setups is to implement :meth:`set_forcing` as a kernel for optimal performance (see e.g. :class:`the global_1deg setup file <veros.setups.global_1deg.GlobalOneDegreeSetup>`).
+
+
+Running Veros
+-------------
+
+After adapting your setup script, you are ready to run your first simulation. Just execute the following::
+
+   $ veros run my_setup.py
+
+.. seealso::
+
+   The Veros command line interface accepts a large number of options to configure your run; see :doc:`/reference/cli`.
+
+.. note::
+
+   You are not required to use the command line, and you are welcome to include your simulation class into other Python files and call it dynamically or interactively (e.g. in an IPython session). All you need to do is to call the ``setup()`` and ``run()`` methods of your :class:`veros.VerosSetup` object.
+
+
+Reading Veros output
+++++++++++++++++++++
+
+All output is handled by :doc:`the available diagnostics </reference/diagnostics>`. The most basic diagnostic, :class:`snapshot <veros.diagnostics.Snapshot>`, writes some model variables to netCDF files in regular intervals (and puts them into your current working directory).
+
+NetCDF is a binary format that is widely adopted in the geophysical modeling community. There are various packages for reading, visualizing and processing netCDF files (such as `ncview <http://meteora.ucsd.edu/~pierce/ncview_home_page.html>`_ and `ferret <http://ferret.pmel.noaa.gov/Ferret/>`_), and bindings for many programming languages (such as C, Fortran, MATLAB, and Python).
+
+For post-processing in Python, we recommend that you use `xarray <http://xarray.pydata.org/en/stable/>`__::
+
+   import xarray as xr
+
+   ds = xr.open_dataset("acc.snapshot.nc", engine="h5netcdf")
+
+   # plot surface velocity at the last time step included in the file
+   u_surface = ds.u.isel(Time=-1, zt=-1)
+   u_surface.plot.contourf()
+
+
+Re-starting from a previous run
++++++++++++++++++++++++++++++++
+
+Restart data (in HDF5 format) is written at the end of each simulation or after a regular time interval if the setting :ref:`restart_frequency <setting-restart_frequency>` is set to a finite value. To use this restart file as initial conditions for another simulation, you will have to point :ref:`restart_input_filename <setting-restart_input_filename>` of the new simulation to the corresponding restart file. This can also be given via the command line (as all settings)::
+
+   $ veros run my_setup.py -s restart_input_filename /path/to/restart_file.h5
+
+.. _mpi-exec:
+
+Running Veros on multiple processes via MPI
++++++++++++++++++++++++++++++++++++++++++++
+
+.. note::
+
+  This assumes that you are familiar with running applications through MPI, and is most useful on large architectures like a compute cluster. For smaller architectures, it is usually easier to stick to the thread-based parallelism of JAX.
+
+Running Veros through MPI requires some additional dependencies. For optimal performance, you will need to install ``mpi4py``, ``h5py``, ``petsc4py``, and ``mpi4jax``, linked to your MPI library.
+
+.. seealso::
+
+   :doc:`advanced-installation`
+
+After you have installed everything, you can start Veros on multiple processes like so:::
+
+   $ mpirun -np 4 veros run my_setup.py -n 2 2
+
+In this case, Veros would run on 4 processes, each process computing one-quarter of the domain. The arguments of the `-n` flag specify the number of domain partitions in x and y-direction, respectively.
+
+.. seealso::
+
+   For more information, see :doc:`/tutorial/cluster`.
+
+Enhancing Veros
+---------------
+
+Veros was written with extensibility in mind. If you already know some Python and have worked with NumPy, you are pretty much ready to write your own extension. The model code is located in the :file:`veros` subfolder, while all of the numerical routines are located in :file:`veros/core`.
+
+We believe that the best way to learn how Veros works is to read its source code. Starting from the :py:class:`Veros base class <veros.VerosSetup>`, you should be able to work your way through the flow of the program, and figure out where to add your modifications. If you installed Veros through :command:`pip -e` or :command:`setup.py develop`, all changes you make will immediately be reflected when running the code.
+
+In case you want to add additional output capabilities or compute additional quantities without changing the main solution of the simulation, you should consider :doc:`adding a custom diagnostic </reference/diagnostics>`.
+
+A convenient way to implement your modifications is to create your own fork of Veros on GitHub, and submit a `pull request <https://github.com/team-ocean/veros/pulls>`_ if you think your modifications could be useful for the Veros community.
+
+.. seealso::
+
+   More information is available in :doc:`our developer guide </tutorial/dev>`.

doc/introduction/introduction.rst

+A short introduction to Veros
+=============================
+
+The vision
+----------
+
+  Veros is an adaptation of `pyOM2 <https://wiki.cen.uni-hamburg.de/ifm/TO/pyOM2>`_ (v2.1.0), developed at Institut für Meereskunde, Hamburg University. In contrast to pyOM2, however, this implementation does not rely on a Fortran backend for computations - everything runs in pure Python, down to the last parameterization. We believe that using this approach it is possible to create an open source ocean model that is:
+
+1. **Easy to access**: Python modules are simple to install, and projects like `Anaconda <https://www.continuum.io/anaconda-overview>`_ are doing a great job in creating platform-independent environments.
+2. **Easy to use**: Anyone with some experience can use their favorite Python tools to set up, control, and post-process Veros.
+3. **Easy to modify**: Due to Python's popularity, available abstractions, and dynamic nature, Veros can be extended and modified with relatively little effort.
+
+However, choosing Python over a compiled language like Fortran usually comes at a high computational cost. We overcome this gap by using `JAX <https://github.com/google/jax>`_, a framework that can act as a high-performance replacement for NumPy. JAX takes care of all performance optimizations in the background, and runs on both CPUs and GPUs.
+
+Features
+--------
+
+.. note::
+
+  This section provides a quick overview of the capabilities and limitations of Veros. For a comprehensive description of the physics and numerics behind Veros, please refer to `the documentation of pyOM2 <https://wiki.zmaw.de/ifm/TO/pyOM2>`_. You can also obtain a copy of the PDF documentation :download:`here </_downloads/pyOM2.pdf>`.
+
+The model domain
+++++++++++++++++
+
+The numerical solution is calculated using finite differences on an *Arakawa C-grid*, which is staggered in every dimension. *Tracers* (like temperature and salinity) are calculated at different positions than zonal, meridional, and vertical *fluxes* (like the velocities u, v, and w). The following figure shows the relative positions of the so-called T, U, V, and ζ grid points (W not shown):
+
+.. figure:: /_images/introduction/c-grid.svg
+   :width: 80%
+   :align: center
+
+   The structure of the Arakawa C-grid.
+
+Veros supports both Cartesian and pseudo-spherical (i.e., including additional metric terms) coordinate systems. Islands or holes in the domain are fully supported by the streamfunction solver. Zonal boundaries can either be cyclic or regraded as walls (with free-slip boundary conditions).
+
+Available parameterizations
++++++++++++++++++++++++++++
+
+At its core, Veros currently offers the following solvers, numerical schemes, parameterizations, and closures:
+
+**Surface pressure**:
+ - a high-performance streamfunction solver via an iterative Poisson solver
+
+**Equation of state**:
+ - the full 48-term TEOS equation of state
+ - various linear and nonlinear model equations from [Vallis2006]_
+
+**Friction**:
+ - harmonic or biharmonic lateral friction
+ - linear or quadratic bottom friction
+ - interior Rayleigh friction
+ - explicit or fully implicit harmonic vertical friction
+
+**Advection**:
+ - a classical second-order central difference scheme
+ - a second-order scheme with a superbee flux-limiter
+
+**Diffusion**:
+ - harmonic or biharmonic lateral diffusion
+ - explicit or implicit harmonic vertical diffusion
+
+**Isoneutral mixing**:
+ - lateral mixing of tracers along neutral surfaces following [Griffies1998]_ (optional)
+
+**Internal wave breaking**:
+ - IDEMIX as in [OlbersEden2013]_ (optional)
+
+**EKE model** (eddy kinetic energy):
+ - meso-scale eddy mixing closure after [Gent1995]_, either with constant coefficients or calculated using the prognostic EKE closure by [EdenGreatbatch2008]_ (optional)
+
+**TKE model** (turbulent kinetic energy):
+ - prognostic TKE model for vertical mixing as introduced in [Gaspar1990]_ (optional)
+
+Diagnostics
++++++++++++
+
+Diagnostics are responsible for handling all model output, runtime checks of the solution, and restart file handling. They are implemented in a modular fashion, so additional diagnostics can be implemented easily. Already implemented diagnostics handle snapshot output, time-averaging of variables, monitoring of energy fluxes, and calculation of the overturning streamfunction.
+
+For more information, see :doc:`/reference/diagnostics`.
+
+Pre-configured model setups
++++++++++++++++++++++++++++
+
+Veros supports a wide range of model configurations. Several setups are already implemented that highlight some of the capabilities of Veros, and that serve as a basis for users to set up their own configuration: :doc:`/reference/setup-gallery`.
+
+Current limitations
++++++++++++++++++++
+
+Veros is still in development. There are many open issues that we would like to fix later on:
+
+**Physics**:
+ - Veros does not yet implement any of the more recent pyOM2.2 features such as the ROSSMIX parameterization, IDEMIX v3.0, open boundary conditions, or cyclic meridional boundaries. It neither implements all of pyOM2.1's features - missing are e.g. the non-hydrostatic solver, IDEMIX v2.0, and the surface pressure solver.
+ - Since the grid is required to be rectilinear, there is currently no natural way to handle the singularity at the North Pole. The northern and southern boundaries of the domain are always "walls".
+ - There is currently no ice sheet model in Veros. Some realistic setups employ a simple ice mask that cut off atmospheric forcing for water that gets too cold instead.
+
+**Technical issues**:
+ - For the time being, Veros' dynamical core is still more or less a direct port of pyOM2. This means that numerics and physics are still tightly coupled, which makes for a far from optimal user experience. In a future version of Veros, we would like to introduce additional abstraction to make the core routines a lot more readable than they are now.
+
+References
+++++++++++
+
+.. [EdenGreatbatch2008] Eden, Carsten, and Richard J. Greatbatch. "Towards a mesoscale eddy closure." Ocean Modelling 20.3 (2008): 223-239.
+
+.. [OlbersEden2013] Olbers, Dirk, and Carsten Eden. "A global model for the diapycnal diffusivity induced by internal gravity waves." Journal of Physical Oceanography 43.8 (2013): 1759-1779.
+
+.. [Gent1995] Gent, Peter R., et al. "Parameterizing eddy-induced tracer transports in ocean circulation models." Journal of Physical Oceanography 25.4 (1995): 463-474.
+
+.. [Griffies1998] Griffies, Stephen M. "The Gent–McWilliams skew flux." Journal of Physical Oceanography 28.5 (1998): 831-841.
+
+.. [Vallis2006] Vallis, Geoffrey K. "Atmospheric and oceanic fluid dynamics: fundamentals and large-scale circulation." Cambridge University Press, 2006.
+
+.. [Gaspar1990] Gaspar, Philippe, Yves Grégoris, and Jean‐Michel Lefevre. "A simple eddy kinetic energy model for simulations of the oceanic vertical mixing: Tests at station Papa and Long‐Term Upper Ocean Study site." Journal of Geophysical Research: Oceans 95.C9 (1990): 16179-16193.

doc/more/benchmarks.rst

+Benchmarks
+==========
+
+.. warning::
+
+   The following benchmarks are for general orientation only. Benchmark results are highly platform dependent; your mileage may vary.
+
+.. seealso::
+
+   See our recent paper `Fast, Cheap, and Turbulent—Global Ocean Modeling With GPU Acceleration in Python <https://agupubs.onlinelibrary.wiley.com/doi/10.1029/2021MS002717>`_ for more in-depth benchmarks, results, and interpretations.
+
+Varying problem size
+--------------------
+
+This benchmark varies the size of the computational domain and records the runtime per iteration. It is executed on a single machine with 24 CPU cores and an NVidia Tesla P100 GPU.
+
+We run the same model code with all Veros backends (``numpy``, ``numpy-mpi``, ``jax``, ``jax-mpi``, ``jax-gpu``) and PyOM2 (``fortran``, ``fortran-mpi``).
+
+
+.. figure:: /_images/benchmarks/acc_scaling_size.png
+   :width: 500px
+   :align: center
+
+
+.. figure:: /_images/benchmarks/acc_scaling_size_speedup.png
+   :width: 500px
+   :align: center
+
+
+As a rule of thumb, we find that JAX is about as fast as Fortran on CPU, with NumPy being about 4x slower. GPUs are a competitive alternative to CPUs, as long as the problem fits into GPU memory.
+
+
+Varying number of MPI processes
+-------------------------------
+
+In this benchmark, Veros and PyOM2 run for a fixed problem size, but varying number of processes. This allows us to check how both models scale with increased CPU count. The problem size corresponds to :doc:`1 degree global resolution </reference/setups/1deg>`.
+
+It is executed on the `DC³ <https://www.nbi.ku.dk/english/research/pice/dc3/>`__ cluster. Each cluster node contains 16 CPUs.
+
+.. figure:: /_images/benchmarks/acc_scaling_nproc.png
+   :width: 500px
+   :align: center
+
+
+We see that both Veros and PyOM2 scale well with increasing number of processes, even for this moderate problem size. JAX performance is comparable to Fortran.

doc/more/external_tools.rst

+External tools
+==============
+
+Veropt
+------
+
+.. warning::
+
+   The following package is not yet compatible with Veros v1.x.x. The last compatible version is `v0.2.3 <https://veros.readthedocs.io/en/v0.2.3/>`_.
+
+`Veropt <https://github.com/idax4325/veropt>`_ (Bayesian Optimisation for the Versatile Ocean Simulator) is a Python package that aims to make Bayesian Optimisation easy to approach, inspect and adjust. It was developed for `Veros <https://veros.readthedocs.io/en/v0.2.3/>`_ with the aim of providing a user-friendly optimisation tool to tune ocean simulations to real world data. Veropt can be used with any optimisation problem but has been developed for expensive optimisation problems with a small amount of evaluations (~100) and the default set-up will probably be most relevant in such a context.
+
+For more information about the package and the methods implemented in it, take a look at `veropt GitHub <https://github.com/idax4325/veropt>`_ repository and `Ida Stoustrup's thesis <https://nbi.ku.dk/english/theses/masters-theses/ida_lei_stoustrup/Ida_Stoustrup_MSc_Thesis.pdf>`_.

doc/more/howtocite.rst

+How to cite
+===========
+
+If you use Veros in scientific work, please consider citing `the following publication <https://gmd.copernicus.org/articles/11/3299/2018/>`_:
+
+::
+
+	@article{hafner_veros_2018,
+		title = {Veros v0.1 – a fast and versatile ocean simulator in pure {Python}},
+		volume = {11},
+		issn = {1991-959X},
+		url = {https://gmd.copernicus.org/articles/11/3299/2018/},
+		doi = {10.5194/gmd-11-3299-2018},
+		number = {8},
+		journal = {Geoscientific Model Development},
+		author = {Häfner, Dion and Jacobsen, René Løwe and Eden, Carsten and Kristensen, Mads R. B. and Jochum, Markus and Nuterman, Roman and Vinter, Brian},
+		month = aug,
+		year = {2018},
+		pages = {3299--3312},
+	}

doc/more/publications.rst

+Publications involving Veros
+============================
+
+Peer-reviewed articles
+----------------------
+
+About Veros
++++++++++++
+
+- Häfner, D., Nuterman, R., & Jochum, M. (2021). Fast, cheap, and turbulent—Global ocean modeling with GPU acceleration in Python. Journal of Advances in Modeling Earth Systems, 13, e2021MS002717.
+  `https://doi.org/10.1029/2021MS002717 <https://doi.org/10.1029/2021MS002717>`__
+- Häfner, D., Jacobsen, R. L., Eden, C., Kristensen, M. R., Jochum, M., Nuterman, R., & Vinter, B. (2018). Veros v0. 1–a fast and versatile ocean simulator in pure Python. Geoscientific Model Development, 11(8), 3299-3312.
+  `https://doi.org/10.5194/gmd-11-3299-2018 <https://doi.org/10.5194/gmd-11-3299-2018>`__
+
+Using Veros
++++++++++++
+
+- Lohmann, J., & Ditlevsen, P. D. (2021). Risk of tipping the overturning circulation due to increasing rates of ice melt. Proceedings of the National Academy of Sciences, 118(9).
+  `https://doi.org/10.1073/pnas.2017989118 <https://doi.org/10.1073/pnas.2017989118>`__
+
+Talks
+-----
+
+- Fast multi-platform ocean modelling in Python with JAX.
+  A talk held at the AGU fall meeting 2021.
+  `Abstract <https://agu.confex.com/agu/fm21/meetingapp.cgi/Paper/792922>`__
+- Higher-level geophysical modelling.
+  A talk held at the European Geosciences Union General Assembly 2021.
+  `Abstract <https://meetingorganizer.copernicus.org/EGU21/EGU21-2127.html>`__
+- Veros — A High-Performance Ocean Simulator Written in Pure Python.
+  A talk held at the 98th Annual Meeting of the American Meteorological Society (AMS).
+  `Abstract <https://ams.confex.com/ams/98Annual/webprogram/Paper324397.html>`__ and
+  `slides <http://slides.com/dionhaefner/veros-ams/>`_ available.
+- A Fast Versatile Ocean Simulator (Veros) in Pure Python.
+  A talk held at the European Geosciences Union General Assembly 2018, Vienna, Austria.
+  `Abstract <https://meetingorganizer.copernicus.org/EGU2018/EGU2018-7122-2.pdf>`__
+
+Theses
+------
+
+- Magnus Quaade Oddershede. (2022). Modelling of ocean turbulence.
+  Effects of resolution on predictive power of ocean models, BSc Thesis, 27p.
+  `Link to PDF <https://sid.erda.dk/share_redirect/CVvcrowL22/Thesis/Magnus_Quaade_Oddershede_BSc_thesis.pdf>`__.
+- Rasmus Ranum Hansen. (2021). Solving the turbulence closure problem I. The tropical ocean mixed layer,
+  BSc Thesis, 55p.
+  `Link to PDF <https://sid.erda.dk/share_redirect/CVvcrowL22/Thesis/Rasmus_Ranum_Hansen_BSc_thesis.pdf>`__.
+- Ida Lei Stoustrup. (2021). Automated parameter tuning for the Versatile Ocean Simulator (Veros),
+  MSc Thesis, 111p.
+  `Link to PDF <https://sid.erda.dk/share_redirect/CVvcrowL22/Thesis/Ida_Stoustrup_MSc_thesis.pdf>`__.
+- Marte Voorneveld. (2020). The effect of bathymetry changes on major oceanic currents and
+  overturning circulations, BSc Thesis, 31p.
+  `Link to PDF <https://sid.erda.dk/share_redirect/CVvcrowL22/Thesis/Marte_Voorneveld_BSc_thesis.pdf>`__.
+- Till Grenzdörffer. (2020). Accelerating ocean modelling: Adressing performance bottlenecks of
+  the ocean modelling framework Veros, MSc Thesis, 16p.
+  `Link to PDF <https://sid.erda.dk/share_redirect/CVvcrowL22/Thesis/Till_Grenzdorffer_MSc_thesis.pdf>`__.
+- Franka Jesse. (2020). Verification of various Island Rules using simulations in application
+  to the Indonesian Throughflow, BSc Thesis, 30p.
+  `Link to PDF <https://sid.erda.dk/share_redirect/CVvcrowL22/Thesis/Franka_Jesse_BSc_thesis.pdf>`__.
+- Denise Ruijsch. (2020). Multiple equilibria in the Eocene ocean circulation leading to the
+  Eocene-Oligocene transition, BSc Thesis, 48p.
+  `Link to PDF <https://sid.erda.dk/share_redirect/CVvcrowL22/Thesis/Denise_Ruijsch_BSc_thesis.pdf>`__.
+- Marie Cecilie Boysen. (2020). Godfrey's island rule and the Indonesian Through Flow,
+  BSc Thesis, 30p.
+  `Link to PDF <https://sid.erda.dk/share_redirect/CVvcrowL22/Thesis/Marie_Cecilie_Boysen_BSc_thesis.pdf>`__.
+- Laurits S. Andreasen. (2019). Time scales of the Bipolar seesaw:
+  The role of oceanic cross-hemisphere signals,
+  Southern Ocean eddies and wind changes, MSc Thesis, 42p.
+  `Link to PDF <https://sid.erda.dk/share_redirect/CVvcrowL22/Thesis/Laurits_Andreasen_MSc_thesis.pdf>`__.
+- Steffen Ole Randrup Kristensen. (2019). Extending the Veros climate simulator with biochemistry.
+  Model design and AMOC collapse, MSc Thesis, 67p.
+  `Link to PDF <https://sid.erda.dk/share_redirect/CVvcrowL22/Thesis/SteffenRandrup_MSc_thesis.pdf>`__.

doc/plot_benchmarks.py

+import json
+import click
+import numpy as np
+
+import matplotlib as mpl
+import seaborn as sns
+
+mpl.use("agg")
+
+import matplotlib.pyplot as plt  # noqa: E402
+
+sns.set_style("ticks")
+
+COMPONENT_COLORS = {
+    "numpy": "orangered",
+    "numpy-mpi": "coral",
+    "jax": "dodgerblue",
+    "jax-mpi": "steelblue",
+    "jax-gpu": "teal",
+    "fortran": "0.4",
+    "fortran-mpi": "0.2",
+}
+
+
+@click.argument("INFILES", nargs=-1, type=click.Path(dir_okay=False, exists=True))
+@click.option("--xaxis", type=click.Choice(["nproc", "size"]), required=True)
+@click.option("--norm-component", default=None)
+@click.command()
+def plot_benchmarks(infiles, xaxis, norm_component):
+    benchmarks = set()
+    components = set()
+    sizes = set()
+    nprocs = set()
+
+    for infile in infiles:
+        with open(infile) as f:
+            data = json.load(f)
+
+        meta = data["settings"]
+        benchmarks |= set(meta["only"])
+        components |= set(meta["components"])
+        sizes |= set(meta["sizes"])
+        nprocs.add(meta["nproc"])
+
+    if xaxis == "nproc":
+        assert len(sizes) == 1
+        xvals = np.array(sorted(nprocs))
+    elif xaxis == "size":
+        assert len(nprocs) == 1
+        xvals = np.array(sorted(sizes))
+    else:
+        assert False
+
+    if norm_component is not None and norm_component not in components:
+        raise ValueError(f"Did not find norm component {norm_component} in data")
+
+    component_data = {benchmark: {comp: np.full(len(xvals), np.nan) for comp in components} for benchmark in benchmarks}
+
+    for infile in infiles:
+        with open(infile) as f:
+            data = json.load(f)
+
+        for benchmark, bench_res in data["benchmarks"].items():
+            for res in bench_res:
+                if xaxis == "size":
+                    # sizes are approximate, take the closest one
+                    x_idx = np.argmin(np.abs(np.array(xvals) - res["size"]))
+                else:
+                    x_idx = xvals.tolist().index(data["settings"]["nproc"])
+
+                time = float(res["per_iteration"]["mean"])
+                component_data[benchmark][res["component"]][x_idx] = time
+
+    for benchmark in benchmarks:
+        fig, ax = plt.subplots(1, 1, figsize=(5.5, 4), dpi=150)
+
+        last_coords = {}
+        for component in components:
+            if norm_component:
+                # compute rel. speedup
+                yvals = component_data[benchmark][norm_component] / component_data[benchmark][component]
+            else:
+                yvals = component_data[benchmark][component]
+
+            plt.plot(xvals, yvals, ".--", color=COMPONENT_COLORS[component], lw=1)
+
+            finite_mask = np.isfinite(yvals)
+            if finite_mask.any():
+                last_coords[component] = (xvals[finite_mask][-1], yvals[finite_mask][-1])
+            else:
+                last_coords[component] = (xvals[0], 1)
+
+        ax.spines["right"].set_visible(False)
+        ax.spines["top"].set_visible(False)
+
+        title_kwargs = dict(fontdict=dict(weight="bold", size=11), ha="left", x=0.05, y=1.05)
+        if xaxis == "nproc":
+            plt.xlabel("Number of MPI processes")
+            mantissa, exponent = f"{list(sizes)[0]:.1e}".split("e")
+            exponent = exponent.lstrip("+0")
+            plt.title(f"Benchmark '{benchmark}' for size {mantissa} $\\times$ 10$^{{{exponent}}}$", **title_kwargs)
+
+        elif xaxis == "size":
+            nproc = list(nprocs)[0]
+            plt.xlabel("Problem size (# elements)")
+            plt.title(f"Benchmark '{benchmark}' on {nproc} processes", **title_kwargs)
+
+            if norm_component:
+                plt.axhline(nproc, linestyle="dashed", alpha=0.5, lw=1, color="C0")
+                plt.annotate(
+                    "Perfect CPU scaling",
+                    (min(xvals), nproc),
+                    xytext=(0, -2),
+                    textcoords="offset points",
+                    alpha=0.5,
+                    color="C0",
+                    va="top",
+                    fontsize=8,
+                )
+
+        if norm_component:
+            plt.ylabel("Relative speedup")
+            plt.text(0.05, 1.05, "Speedup (higher is better)", transform=ax.transAxes, va="top", color="0.4")
+        else:
+            plt.ylabel("Time per iteration (s)")
+            plt.text(0.05, 1.05, "Wall time (lower is better)", transform=ax.transAxes, va="top", color="0.4")
+
+        plt.xscale("log")
+        plt.yscale("log")
+
+        fig.canvas.draw()
+
+        # add annotations, make sure they don"t overlap
+        last_text_pos = 0
+        for component, (x, y) in sorted(last_coords.items(), key=lambda k: k[1][1]):
+            trans = ax.transData
+            _, tp = trans.transform((0, y))
+            tp = max(tp, last_text_pos + 20)
+            _, y = trans.inverted().transform((0, tp))
+
+            plt.annotate(
+                component,
+                (x, y),
+                xytext=(10, 0),
+                textcoords="offset points",
+                annotation_clip=False,
+                color=COMPONENT_COLORS[component],
+                va="center",
+                weight="bold",
+            )
+
+            last_text_pos = tp
+
+        fig.tight_layout()
+
+        suffix = ""
+        if norm_component:
+            suffix = f"-norm_{norm_component}"
+
+        fig.savefig(f"{benchmark}-{xaxis}{suffix}.png")
+        plt.close(fig)
+
+
+if __name__ == "__main__":
+    plot_benchmarks()

doc/plugins/developer-guide.rst

+How to write a Veros plug-in
+============================
+
+Writing a plug-in for Veros is relatively simple.
+A plug-in can be any Python package that accepts a :class:`VerosState <veros.state.VerosState>` object.
+The plug-in is then free to modify the model state in any way it pleases.
+
+The only requirement for the plug-in to be usable in Veros setups is that it declares a special object in its top :file:`__init__.py` file:
+
+::
+
+   __VEROS_INTERFACE__ = dict(
+       name='my-plugin',
+       setup_entrypoint=my_setup_function,
+       run_entrypoint=my_main_function
+   )
+
+The functions passed in :obj:`setup_entrypoint` and :obj:`run_entrypoint` are then called by Veros during model set-up and after each time step, respectively, with the model state as the sole argument.
+
+An example for these functions could be:
+
+::
+
+   from veros import veros_routine
+
+
+   @veros_routine
+   def my_setup_function(state):
+       pass
+
+
+   @veros_routine
+   def my_main_function(state):
+       from veros.core.operators import numpy as npx
+       vs = state.variables
+
+       # apply simple ice mask
+       mask = np.logical_and(vs.temp[:, :, -1, vs.tau] * vs.maskT[:, :, -1] < -1.8,
+                             vs.forc_temp_surface < 0.)
+       vs.forc_temp_surface = npx.where(mask, 0.0, vs.forc_temp_surface)
+       vs.forc_salt_surface = npx.where(mask, 0.0, vs.forc_salt_surface)
+
+
+In this case, the setup function does nothing, while the main function sets temperature and salinity forcing to 0 where the surface temperature is smaller than -1.8 degrees (a very crude sea ice model).
+
+Custom settings, variables, and diagnostics
+-------------------------------------------
+
+In real-world applications, you probably want to use custom settings, variables, and/or diagnostics in your plug-in.
+You can specify those as additional arguments to :obj:`__VEROS_INTERFACE__`:
+
+::
+
+   __VEROS_INTERFACE__ = dict(
+       name='my-plugin',
+       setup_entrypoint=my_setup_function,
+       run_entrypoint=my_main_function,
+       settings=my_settings,
+       variables=my_variables,
+       diagnostics=[MyDiagnostic]
+   )
+
+In this case, :obj:`my_settings` is a :class:`dict` mapping the name of the setting to a :class:`Setting <veros.settings.Setting>` object:
+
+::
+
+   from veros.settings import Setting
+
+   my_settings = {
+       'enable_my_plugin'; Setting(False, bool, 'Enable my plugin'),
+       'temperature_cutoff': Setting(-1.8, float, 'Cut-off surface temperature'),
+   }
+
+
+Similarly, for variables:
+
+::
+
+   from veros.variables import Variable, T_GRID
+
+   my_variables = {
+       'my_variable': Variable('Description', T_GRID, 'unit', 'Long description'),
+   }
+
+The so-defined settings and variables are then available as attributes of the Veros state object, as usual:
+
+::
+
+   @veros_routine
+   def my_function(state):
+       from veros.core.operators import update, at
+
+       if state.settings.enable_my_plugin:
+           state.variables.my_variable = update(state.variables.my_variable, at[...], 0.)
+
+.. seealso::
+
+   For more inspiration on how to specify settings and variables, have a look at the built-in :file:`settings.py` and :file:`variables.py` files.
+
+Diagnostics are defined similarly, but they have to be a subclass of :class:`VerosDiagnostic <veros.diagnostics.base.VerosDiagnostic>`.
+
+
+Shipping custom model setups
+----------------------------
+
+You can use a special entrypoint in the :file:`setup.py` file of your plug-in to inform the Veros command-line interface of your custom setups:
+
+::
+
+   from setuptools import setup
+
+   setup(
+      name='my-plugin',
+      packages='my_plugin',
+      entry_points={
+        'veros.setup_dirs': [
+            'my_plugin = my_plugin.setups'
+        ]
+      }
+   )
+
+This assumes, that your custom setups are located in the folder :file:`my_plugin/setups`.
+Then, `veros copy-setup` will automatically find your custom setups if the plug-in is installed:
+
+::
+
+   $ veros copy-setup --help
+   Usage: veros copy-setup [OPTIONS] SETUP
+
+   Copy a standard setup to another directory.
+
+   Available setups:
+
+      acc, acc_basic, acc_sector, eady, fjord, global_1deg, global_4deg,
+      global_flexible, my_setup, north_atlantic, wave_propagation
+
+   Example:
+
+      $ veros copy-setup global_4deg --to ~/veros-setups/4deg-lowfric
+
+   Further directories containing setup templates can be added to this
+   command via the VEROS_SETUP_DIR environment variable.
+
+   Options:
+   --to PATH  Target directory, must not exist (default: copy to current
+              working directory)
+   --help     Show this message and exit.
+
+In this case, the custom setup is located in the folder :file:`my_plugin/setups/my_setup`, and thus shows up as :obj:`my_setup`.

doc/plugins/user-guide.rst

+How to use plug-ins
+===================
+
+A plug-in is an optional extension for Veros that you can use to enable additional physics, outputs, or diagnostics.
+
+.. note::
+
+   Plug-ins do not necessarily meet the same quality standards as the Veros core, and not all plug-ins are directly affiliated with Veros or the Veros developers. You should carefully check whether a given plug-in fits your needs.
+
+As a first step, install the plug-in you want to use, e.g. :obj:`veros-bgc` via
+
+::
+
+   $ pip install veros-bgc
+
+
+You can then immediately use any custom setups included in the plug-in:
+
+::
+
+   $ veros copy-setup bgc_global_4deg
+
+
+To use a plug-in in a Veros setup, all you need to do is to import it and add it to the :obj:`__veros_plugins__` attribute of your setup class:
+
+::
+
+   import veros_bgc
+
+   class MySetup(VerosSetup):
+      __veros_plugins__ = (veros_bgc,)
+
+      # - rest of the setup definition -
+
+This step is probably not necessary if you use a setup that was shipped with the plug-in, but in doubt you should double-check that the plug-in is activated properly.
+
+.. seealso::
+
+   For more information, refer to the documentation of the plug-in in question. You can find some suggestions in the contents of this section.

doc/reference/api/distributed.rst

+Distributed communication primitives
+====================================
+
+.. automodule:: veros.distributed
+   :members: gather, scatter, global_sum, global_max, barrier, abort
+   :undoc-members:

doc/reference/api/operators.rst

+Numerical operators
+===================
+
+.. automodule:: veros.core.operators
+   :members: update, update_add, update_multiply, at, flush, solve_tridiagonal, for_loop, scan
+   :undoc-members:

doc/reference/api/routines.rst

+veros_routine and veros_kernel
+==============================
+
+veros_routine
+-------------
+
+.. autofunction:: veros.veros_routine
+
+veros_kernel
+------------
+
+.. autofunction:: veros.veros_kernel
+
+KernelOutput
+------------
+
+.. autofunction:: veros.KernelOutput

doc/reference/api/runtime.rst

+Runtime settings & state
+========================
+
+
+Runtime settings
+----------------
+
+.. autoclass:: veros.runtime.RuntimeSettings
+   :members:
+   :undoc-members:
+
+Runtime state
+-------------
+
+.. autoclass:: veros.runtime.RuntimeState
+   :members:
+   :undoc-members:

doc/reference/api/tools.rst

+Tools & utilities
+=================
+
+Assets
+++++++
+
+.. autofunction:: veros.tools.assets.get_assets
+
+
+Setup tools
++++++++++++
+
+.. automodule:: veros.tools.setup
+   :members:

doc/reference/api/veros-setup.rst

+Veros setup class
+=================
+
+.. autoclass:: veros.VerosSetup
+   :members: