veros

benchmarks/acc_benchmark.py

+from benchmark_base import benchmark_cli
+
+from veros.pyom_compat import load_pyom, pyom_from_state, run_pyom
+
+
+@benchmark_cli
+def main(pyom2_lib, timesteps, size):
+    from veros import veros_routine
+    from veros.setups.acc import ACCSetup
+    from veros.variables import Variable, allocate
+    from veros.distributed import global_min, global_max
+    from veros.core.operators import update, at, numpy as npx
+
+    class ACC2Benchmark(ACCSetup):
+        @veros_routine
+        def set_parameter(self, state):
+            settings = state.settings
+            settings.identifier = "acc"
+            settings.restart_output_filename = None
+
+            settings.nx, settings.ny, settings.nz = size
+            settings.dt_mom = 600
+            settings.dt_tracer = 600
+            settings.runlen = settings.dt_tracer * timesteps
+
+            settings.x_origin = 0.0
+            settings.y_origin = -40.0
+
+            settings.coord_degree = True
+            settings.enable_cyclic_x = True
+
+            settings.enable_neutral_diffusion = True
+            settings.K_iso_0 = 1000.0
+            settings.K_iso_steep = 500.0
+            settings.iso_dslope = 0.005
+            settings.iso_slopec = 0.01
+            settings.enable_skew_diffusion = True
+
+            settings.enable_hor_friction = True
+            settings.A_h = (2 * settings.degtom) ** 3 * 2e-11
+            settings.enable_hor_friction_cos_scaling = True
+            settings.hor_friction_cosPower = 1
+
+            settings.enable_bottom_friction = True
+            settings.r_bot = 1e-5
+
+            settings.enable_implicit_vert_friction = True
+
+            settings.enable_tke = True
+            settings.c_k = 0.1
+            settings.c_eps = 0.7
+            settings.alpha_tke = 30.0
+            settings.mxl_min = 1e-8
+            settings.tke_mxl_choice = 2
+            settings.kappaM_min = 2e-4
+            settings.kappaH_min = 0.0
+            settings.enable_kappaH_profile = False
+            settings.enable_Prandtl_tke = False
+
+            settings.K_gm_0 = 1000.0
+            settings.enable_eke = True
+            settings.eke_k_max = 1e4
+            settings.eke_c_k = 0.4
+            settings.eke_c_eps = 0.5
+            settings.eke_cross = 2.0
+            settings.eke_crhin = 1.0
+            settings.eke_lmin = 100.0
+            settings.enable_eke_superbee_advection = True
+            settings.enable_eke_isopycnal_diffusion = True
+            settings.enable_streamfunction = False
+
+            settings.enable_idemix = True
+            settings.enable_idemix_hor_diffusion = True
+            settings.enable_eke_diss_surfbot = True
+            settings.eke_diss_surfbot_frac = 0.2
+            settings.enable_idemix_superbee_advection = True
+
+            settings.eq_of_state_type = 5
+
+            var_meta = state.var_meta
+            var_meta.update(
+                t_star=Variable("t_star", ("yt",), "deg C", "Reference surface temperature"),
+                t_rest=Variable("t_rest", ("xt", "yt"), "1/s", "Surface temperature restoring time scale"),
+            )
+
+        @veros_routine
+        def set_grid(self, state):
+            vs = state.variables
+            settings = state.settings
+            vs.dxt = update(vs.dxt, at[...], 120 / settings.nx)
+            vs.dyt = update(vs.dyt, at[...], 80 / settings.ny)
+            vs.dzt = update(vs.dzt, at[...], 4000 / settings.nz)
+
+        @veros_routine
+        def set_coriolis(self, state):
+            vs = state.variables
+            settings = state.settings
+            vs.coriolis_t = update(
+                vs.coriolis_t, at[...], 2 * settings.omega * npx.sin(vs.yt[None, :] / 180.0 * settings.pi)
+            )
+
+        @veros_routine
+        def set_topography(self, state):
+            vs = state.variables
+            x, y = npx.meshgrid(vs.xt, vs.yt, indexing="ij")
+            vs.kbot = npx.logical_or(x > 1.0, y < -20).astype("int")
+
+        @veros_routine
+        def set_initial_conditions(self, state):
+            vs = state.variables
+            settings = state.settings
+
+            # initial conditions
+            vs.temp = update(vs.temp, at[...], ((1 - vs.zt[None, None, :] / vs.zw[0]) * 15 * vs.maskT)[..., None])
+            vs.salt = update(vs.salt, at[...], 35.0 * vs.maskT[..., None])
+
+            # wind stress forcing
+            yt_min = global_min(vs.yt.min())
+            yu_min = global_min(vs.yu.min())
+            yt_max = global_max(vs.yt.max())
+            yu_max = global_max(vs.yu.max())
+
+            taux = allocate(state.dimensions, ("yt",))
+            taux = npx.where(vs.yt < -20, 0.1 * npx.sin(settings.pi * (vs.yu - yu_min) / (-20.0 - yt_min)), taux)
+            taux = npx.where(vs.yt > 10, 0.1 * (1 - npx.cos(2 * settings.pi * (vs.yu - 10.0) / (yu_max - 10.0))), taux)
+            vs.surface_taux = taux * vs.maskU[:, :, -1]
+
+            # surface heatflux forcing
+            vs.t_star = allocate(state.dimensions, ("yt",), fill=15)
+            vs.t_star = npx.where(vs.yt < -20, 15 * (vs.yt - yt_min) / (-20 - yt_min), vs.t_star)
+            vs.t_star = npx.where(vs.yt > 20, 15 * (1 - (vs.yt - 20) / (yt_max - 20)), vs.t_star)
+            vs.t_rest = vs.dzt[npx.newaxis, -1] / (30.0 * 86400.0) * vs.maskT[:, :, -1]
+
+            if settings.enable_tke:
+                vs.forc_tke_surface = update(
+                    vs.forc_tke_surface,
+                    at[2:-2, 2:-2],
+                    npx.sqrt(
+                        (0.5 * (vs.surface_taux[2:-2, 2:-2] + vs.surface_taux[1:-3, 2:-2]) / settings.rho_0) ** 2
+                        + (0.5 * (vs.surface_tauy[2:-2, 2:-2] + vs.surface_tauy[2:-2, 1:-3]) / settings.rho_0) ** 2
+                    )
+                    ** (1.5),
+                )
+
+            if settings.enable_idemix:
+                vs.forc_iw_bottom = 1e-6 * vs.maskW[:, :, -1]
+                vs.forc_iw_surface = 1e-7 * vs.maskW[:, :, -1]
+
+        @veros_routine
+        def set_forcing(self, state):
+            vs = state.variables
+            vs.forc_temp_surface = vs.t_rest * (vs.t_star - vs.temp[:, :, -1, vs.tau])
+
+        @veros_routine
+        def set_diagnostics(self, state):
+            state.diagnostics.clear()
+
+    sim = ACC2Benchmark()
+    sim.setup()
+
+    if not pyom2_lib:
+        sim.run()
+        return
+
+    pyom_obj = load_pyom(pyom2_lib)
+    pyom_obj = pyom_from_state(sim.state, pyom_obj, ignore_attrs=("t_star", "t_rest"))
+
+    # different units in pyom
+    pyom_obj.main_module.surface_taux /= 1000
+
+    t_rest = sim.state.variables.t_rest
+    t_star = sim.state.variables.t_star
+
+    def set_forcing_pyom(pyom_obj):
+        m = pyom_obj.main_module
+        m.forc_temp_surface[:] = t_rest * (t_star - m.temp[:, :, -1, m.tau - 1])
+
+    run_pyom(pyom_obj, set_forcing_pyom)
+
+
+if __name__ == "__main__":
+    main()

benchmarks/bench-external-assets.json

+{
+    "4deg-press": {
+        "url": "https://sid.erda.dk/share_redirect/CD8UzHCj2Q/testdata/external/matrix-4deg-press.h5",
+        "md5": "0e6b9b4119133a56d78bdb1079a0d22e"
+    },
+    "4deg-stream": {
+        "url": "https://sid.erda.dk/share_redirect/CD8UzHCj2Q/testdata/external/matrix-4deg-stream.h5",
+        "md5": "6ea76c736f2d58a1848b66407402f797"
+    },
+    "1deg-press": {
+        "url": "https://sid.erda.dk/share_redirect/CD8UzHCj2Q/testdata/external/matrix-1deg-press.h5",
+        "md5": "5ea1bfea8f2118df4c407cbd244f32f0"
+    },
+    "1deg-stream": {
+        "url": "https://sid.erda.dk/share_redirect/CD8UzHCj2Q/testdata/external/matrix-1deg-stream.h5",
+        "md5": "0161722c7e36f505e1a227ee5a70bd75"
+    },
+    "01deg-stream": {
+        "url": "https://sid.erda.dk/share_redirect/CD8UzHCj2Q/testdata/external/matrix-01deg-stream.h5",
+        "md5": "7995b3a6c9b199a4182fa5c38f7b37f8"
+    },
+    "01deg-press": {
+        "url": "https://sid.erda.dk/share_redirect/CD8UzHCj2Q/testdata/external/matrix-01deg-press.h5",
+        "md5": "0eb93209b5c759b362b46c53a836b523"
+    }
+}

benchmarks/benchmark_base.py

+import os
+import click
+
+
+def benchmark_cli(func):
+    @click.option("--size", type=int, nargs=3, required=True)
+    @click.option("--timesteps", type=int, required=True)
+    @click.option("-f", "--pyom2-lib", type=click.Path(readable=True, dir_okay=False), default=None)
+    @click.option("-b", "--backend", type=click.Choice(["numpy", "jax"]), default="numpy")
+    @click.option("-d", "--device", type=click.Choice(["cpu", "gpu"]), default="cpu")
+    @click.option("-n", "--nproc", type=int, nargs=2, default=(1, 1))
+    @click.option("--float-type", type=click.Choice(["float64", "float32"]), default="float64")
+    @click.option("-v", "--loglevel", type=click.Choice(["debug", "trace"]), default="debug")
+    @click.option("--profile-mode", is_flag=True)
+    @click.command()
+    def inner(backend, device, nproc, float_type, loglevel, profile_mode, **kwargs):
+        from veros import runtime_settings, runtime_state
+
+        runtime_settings.update(
+            backend=backend,
+            device=device,
+            float_type=float_type,
+            num_proc=nproc,
+            loglevel=loglevel,
+            profile_mode=profile_mode,
+        )
+
+        if device == "gpu" and runtime_state.proc_num > 1:
+            os.environ["CUDA_VISIBLE_DEVICES"] = str(runtime_state.proc_rank)
+
+        return func(**kwargs)
+
+    return inner

benchmarks/isoneutral_benchmark.py

+from benchmark_base import benchmark_cli
+
+from time import perf_counter
+
+from veros import logger
+from veros.pyom_compat import load_pyom, pyom_from_state
+
+
+@benchmark_cli
+def main(pyom2_lib, timesteps, size):
+    from veros.state import get_default_state
+    from veros.distributed import barrier
+    from veros.core.isoneutral import isoneutral_diffusion_pre
+    from veros.core.operators import flush
+
+    state = get_default_state()
+
+    with state.settings.unlock():
+        state.settings.update(
+            nx=size[0],
+            ny=size[1],
+            nz=size[2],
+            enable_neutral_diffusion=True,
+        )
+
+    state.initialize_variables()
+    state.variables.__locked__ = False
+
+    if not pyom2_lib:
+
+        def run():
+            return isoneutral_diffusion_pre(state)
+
+    else:
+        pyom_obj = load_pyom(pyom2_lib)
+        pyom_obj = pyom_from_state(state, pyom_obj, init_streamfunction=False)
+
+        def run():
+            return pyom_obj.isoneutral_diffusion_pre()
+
+    for _ in range(timesteps):
+        start = perf_counter()
+
+        run()
+        flush()
+        barrier()
+
+        end = perf_counter()
+
+        logger.debug(f"Time step took {end - start}s")
+
+
+if __name__ == "__main__":
+    main()

benchmarks/pressure_solver_benchmark.py

+import os
+from time import perf_counter
+
+from benchmark_base import benchmark_cli
+
+from veros import logger, runtime_settings
+from veros.pyom_compat import load_pyom, pyom_from_state
+
+VARIABLES_USED = ["cost", "cosu", "dxt", "dxu", "dyt", "dyu", "hu", "hv", "maskT"]
+
+VARIABLES_ALLOCATED = VARIABLES_USED + ["hur", "hvr", "isle_boundary_mask"]
+
+SETTINGS_USED = [
+    "nx",
+    "ny",
+    "nz",
+    "dt_tracer",
+    "dt_mom",
+    "enable_streamfunction",
+    "enable_cyclic_x",
+]
+
+
+def get_dummy_state(infile):
+    import h5py
+    from veros.state import VerosState
+    from veros.distributed import get_chunk_slices, exchange_overlap
+    from veros.variables import VARIABLES, DIM_TO_SHAPE_VAR, get_shape
+    from veros.settings import SETTINGS
+    from veros.core.operators import numpy as npx, update, at
+
+    variables_subset = {key: VARIABLES[key] for key in VARIABLES_ALLOCATED}
+
+    state = VerosState(var_meta=variables_subset, setting_meta=SETTINGS, dimensions=DIM_TO_SHAPE_VAR)
+
+    with h5py.File(infile, "r") as f:
+        input_settings = dict(f["settings"].attrs.items())
+
+        with state.settings.unlock():
+            state.settings.update({sett: input_settings[sett] for sett in SETTINGS_USED})
+
+        state.initialize_variables()
+
+        dimensions = state.dimensions
+        var_meta = state.var_meta
+
+        input_data = {}
+        for v in f.keys():
+            if v == "settings":
+                continue
+
+            if v in var_meta:
+                dims = var_meta[v].dims
+            else:
+                dims = ("xt", "yt")
+
+            local_shape = get_shape(dimensions, dims, local=True, include_ghosts=True)
+            gidx, lidx = get_chunk_slices(dimensions["xt"], dimensions["yt"], dims, include_overlap=True)
+
+            if npx.issubdtype(f[v].dtype, npx.floating):
+                dtype = runtime_settings.float_type
+            else:
+                dtype = str(f[v].dtype)
+
+            var = npx.empty(local_shape, dtype=dtype)
+            var = update(var, at[lidx], f[v][gidx])
+            var = exchange_overlap(var, dims, state.settings.enable_cyclic_x)
+            input_data[v] = var
+
+    with state.variables.unlock():
+        for key in VARIABLES_USED:
+            setattr(state.variables, key, input_data[key])
+
+    return state, input_data
+
+
+@benchmark_cli
+def main(pyom2_lib, timesteps, size):
+    from veros.tools import get_assets
+    from veros.distributed import barrier
+    from veros.core.operators import flush, numpy as npx
+    from veros.core.external.solve_pressure import get_linear_solver
+
+    here = os.path.dirname(__file__)
+    assets = get_assets("bench-external", os.path.join(here, "bench-external-assets.json"))
+
+    total_size = size[0] * size[1] * size[2]
+
+    if 1e8 <= total_size <= 1e9:
+        infile = assets["01deg-press"]
+    elif 1e6 <= total_size <= 1e7:
+        infile = assets["1deg-press"]
+    elif 1e4 <= total_size <= 1e5:
+        infile = assets["4deg-press"]
+    else:
+        raise ValueError(
+            "Pressure solver benchmark only support 4deg, 1deg, and 0.1deg resolution"
+            " (n = 5e4, 5e6, 5e8, respectively)."
+        )
+
+    state, input_data = get_dummy_state(infile)
+    solver = get_linear_solver(state)
+
+    if not pyom2_lib:
+
+        def reset():
+            pass
+
+        def run():
+            return solver.solve(state, input_data["rhs"], input_data["x0"])
+
+    else:
+        pyom_obj = load_pyom(pyom2_lib)
+        pyom_obj = pyom_from_state(state, pyom_obj, init_streamfunction=False)
+        m = pyom_obj.main_module
+        m.taum1, m.tau, m.taup1 = 1, 2, 3
+
+        def reset():
+            m.psi[..., m.taup1 - 1] = input_data["x0"]
+
+        def run():
+            pyom_obj.congrad_surf_press(
+                m.is_pe - m.onx, m.ie_pe + m.onx, m.js_pe - m.onx, m.je_pe + m.onx, input_data["rhs"], 0
+            )
+            return m.psi[..., m.taup1 - 1]
+
+    for _ in range(timesteps):
+        reset()
+        flush()
+        barrier()
+
+        start = perf_counter()
+        res = run()
+        flush()
+        barrier()
+        end = perf_counter()
+
+        logger.debug(f"Time step took {end - start}s")
+
+    if not pyom2_lib:
+        # monitor residual to expected solution, with generous margin
+        def rms(arr):
+            return npx.sqrt(npx.mean(arr**2))
+
+        rms_err = rms(res[2:-2, 2:-2] - input_data["res"][2:-2, 2:-2]) / rms(input_data["res"][2:-2, 2:-2])
+        assert rms_err < 1e-3, rms_err
+
+
+if __name__ == "__main__":
+    main()

benchmarks/streamfunction_solver_benchmark.py

+import os
+from time import perf_counter
+
+from benchmark_base import benchmark_cli
+
+from veros import logger, runtime_settings
+from veros.pyom_compat import load_pyom, pyom_from_state
+
+
+VARIABLES_USED = [
+    "isle_boundary_mask",
+    "cost",
+    "cosu",
+    "dxt",
+    "dxu",
+    "dyt",
+    "dyu",
+    "hur",
+    "hvr",
+    "kbot",
+]
+VARIABLES_ALLOCATED = VARIABLES_USED + ["hu", "hv", "maskT"]
+
+SETTINGS_USED = [
+    "nx",
+    "ny",
+    "nz",
+    "dt_tracer",
+    "dt_mom",
+    "enable_streamfunction",
+    "enable_cyclic_x",
+]
+
+
+def get_dummy_state(infile):
+    import h5py
+    from veros.state import VerosState
+    from veros.distributed import get_chunk_slices, exchange_overlap
+    from veros.variables import VARIABLES, DIM_TO_SHAPE_VAR, get_shape
+    from veros.settings import SETTINGS
+    from veros.core.operators import numpy as npx, update, at
+
+    variables_subset = {key: VARIABLES[key] for key in VARIABLES_ALLOCATED}
+
+    state = VerosState(var_meta=variables_subset, setting_meta=SETTINGS, dimensions=DIM_TO_SHAPE_VAR)
+
+    with h5py.File(infile, "r") as f:
+        input_settings = dict(f["settings"].attrs.items())
+
+        with state.settings.unlock():
+            state.settings.update({sett: input_settings[sett] for sett in SETTINGS_USED})
+
+        state.initialize_variables()
+
+        dimensions = state.dimensions
+        var_meta = state.var_meta
+
+        input_data = {}
+        for v in f.keys():
+            if v == "settings":
+                continue
+
+            if v in var_meta:
+                dims = var_meta[v].dims
+            else:
+                dims = ("xt", "yt")
+
+            local_shape = get_shape(dimensions, dims, local=True, include_ghosts=True)
+            gidx, lidx = get_chunk_slices(dimensions["xt"], dimensions["yt"], dims, include_overlap=True)
+
+            if npx.issubdtype(f[v].dtype, npx.floating):
+                dtype = runtime_settings.float_type
+            else:
+                dtype = str(f[v].dtype)
+
+            var = npx.empty(local_shape, dtype=dtype)
+            var = update(var, at[lidx], f[v][gidx])
+            var = exchange_overlap(var, dims, state.settings.enable_cyclic_x)
+            input_data[v] = var
+
+    with state.variables.unlock():
+        for key in VARIABLES_USED:
+            if key == "kbot":
+                continue
+
+            setattr(state.variables, key, input_data[key])
+
+        # needed for PyOM init
+        state.variables.kbot = npx.where(state.variables.hur > 0, 1, 0)
+
+    return state, input_data
+
+
+@benchmark_cli
+def main(pyom2_lib, timesteps, size):
+    from veros.tools import get_assets
+    from veros.distributed import barrier
+    from veros.core.operators import flush, numpy as npx
+    from veros.core.utilities import enforce_boundaries
+    from veros.core.external.solve_stream import get_linear_solver
+
+    here = os.path.dirname(__file__)
+    assets = get_assets("bench-external", os.path.join(here, "bench-external-assets.json"))
+
+    total_size = size[0] * size[1] * size[2]
+
+    if 1e8 <= total_size <= 1e9:
+        infile = assets["01deg-stream"]
+    elif 1e6 <= total_size <= 1e7:
+        infile = assets["1deg-stream"]
+    elif 1e4 <= total_size <= 1e5:
+        infile = assets["4deg-stream"]
+    else:
+        raise ValueError(
+            "Streamfunction solver benchmark only support 4deg, 1deg, and 0.1deg resolution"
+            " (n = 5e4, 5e6, 5e8, respectively)."
+        )
+
+    state, input_data = get_dummy_state(infile)
+    solver = get_linear_solver(state)
+
+    if not pyom2_lib:
+
+        def reset():
+            pass
+
+        def run():
+            return solver.solve(state, input_data["rhs"], input_data["x0"])
+
+    else:
+        pyom_obj = load_pyom(pyom2_lib)
+        pyom_obj = pyom_from_state(state, pyom_obj, init_streamfunction=True)
+
+        m = pyom_obj.main_module
+        m.taum1, m.tau, m.taup1 = 1, 2, 3
+
+        def reset():
+            m.dpsi[..., m.taup1 - 1] = enforce_boundaries(input_data["x0"], True)
+
+        def run():
+            pyom_obj.congrad_streamfunction(
+                m.is_pe - m.onx,
+                m.ie_pe + m.onx,
+                m.js_pe - m.onx,
+                m.je_pe + m.onx,
+                input_data["rhs"],
+                0,
+                m.dpsi[..., m.taup1 - 1],
+                False,
+            )
+            return m.dpsi[..., m.taup1 - 1]
+
+    for _ in range(timesteps):
+        reset()
+        flush()
+        barrier()
+
+        start = perf_counter()
+        res = run()
+        flush()
+        barrier()
+        end = perf_counter()
+
+        logger.debug(f"Time step took {end - start}s")
+
+    if not pyom2_lib:
+        # monitor residual to expected solution, with generous margin
+        def rms(arr):
+            return npx.sqrt(npx.mean(arr**2))
+
+        rms_err = rms(res[2:-2, 2:-2] - input_data["res"][2:-2, 2:-2]) / rms(input_data["res"][2:-2, 2:-2])
+        assert rms_err < 1e-3, rms_err
+
+
+if __name__ == "__main__":
+    main()

benchmarks/tdma_benchmark.py

+from benchmark_base import benchmark_cli
+
+from time import perf_counter
+
+import numpy as np
+from veros import logger
+from veros.pyom_compat import load_pyom, pyom_from_state
+
+
+@benchmark_cli
+def main(pyom2_lib, timesteps, size):
+    from veros.state import get_default_state
+    from veros.distributed import barrier
+    from veros.core.utilities import create_water_masks
+    from veros.core.operators import flush, solve_tridiagonal
+
+    state = get_default_state()
+
+    with state.settings.unlock():
+        state.settings.update(
+            nx=size[0],
+            ny=size[1],
+            nz=size[2],
+            enable_neutral_diffusion=True,
+        )
+
+    state.initialize_variables()
+    state.variables.__locked__ = False
+
+    nx, ny, nz = 70, 60, 50
+    a, b, c, d = (np.random.randn(nx, ny, nz) for _ in range(4))
+    kbot = np.random.randint(0, nz, size=(nx, ny))
+
+    if not pyom2_lib:
+        _, water_mask, edge_mask = create_water_masks(kbot, nz)
+
+        def run():
+            out_vs = solve_tridiagonal(a, b, c, d, water_mask, edge_mask)
+            return out_vs
+
+    else:
+        pyom_obj = load_pyom(pyom2_lib)
+        pyom_obj = pyom_from_state(state, pyom_obj, init_streamfunction=False)
+
+        def run():
+            out_pyom = np.zeros((nx, ny, nz))
+            for i in range(nx):
+                for j in range(ny):
+                    ks = kbot[i, j] - 1
+                    ke = nz
+
+                    if ks < 0:
+                        continue
+
+                    out_pyom[i, j, ks:ke] = pyom_obj.solve_tridiag(
+                        a=a[i, j, ks:ke], b=b[i, j, ks:ke], c=c[i, j, ks:ke], d=d[i, j, ks:ke], n=ke - ks
+                    )
+            return out_pyom
+
+    for _ in range(timesteps):
+        start = perf_counter()
+
+        run()
+        flush()
+        barrier()
+
+        end = perf_counter()
+
+        logger.debug(f"Time step took {end - start}s")
+
+
+if __name__ == "__main__":
+    main()

conda-environment.yml

+name: veros
+
+channels:
+    - conda-forge
+
+dependencies:
+    - python=3.8
+    - pip
+    - click
+    - entrypoints
+    - requests
+    - numpy
+    - scipy
+    - h5netcdf
+    - h5py
+    - pillow
+    - loguru
+    - tqdm
+    - cython
+    - pip:
+        - -e .

cuda_ext.py

+import os
+
+from setuptools.command.build_ext import build_ext
+from distutils.unixccompiler import UnixCCompiler
+
+# This is based on
+# https://github.com/rmcgibbo/npcuda-example/blob/dd2768d8ccb5688c0f08678dd8f1ad5afe3e4332/cython/setup.py
+# published under BSD 2-Clause "Simplified" License
+
+
+def find_in_path(name, path):
+    """Find a file in a search path"""
+
+    # Adapted fom http://code.activestate.com/recipes/52224
+    for dir in path.split(os.pathsep):
+        binpath = os.path.join(dir, name)
+        if os.path.exists(binpath):
+            return os.path.abspath(binpath)
+
+    return None
+
+
+def locate_cuda():
+    """Locate the CUDA environment on the system
+
+    Returns a dict with keys 'cuda_root', 'nvcc', 'include', and 'lib64'
+    and values giving the absolute path to each directory.
+
+    Starts by looking for the CUDAHOME and CUDA_ROOT env variables.
+    If not found, everything is based on finding 'nvcc' in the PATH.
+    """
+
+    # First check if any common env variable is in use
+    if "CUDAHOME" in os.environ:
+        cuda_root = os.environ["CUDAHOME"]
+        nvcc = os.path.join(cuda_root, "bin", "nvcc")
+    elif "CUDA_ROOT" in os.environ:
+        cuda_root = os.environ["CUDA_ROOT"]
+        nvcc = os.path.join(cuda_root, "bin", "nvcc")
+    else:
+        # Otherwise, search the PATH for NVCC
+        nvcc = find_in_path("nvcc", os.environ["PATH"])
+        if nvcc is not None:
+            cuda_root = os.path.dirname(os.path.dirname(nvcc))
+        else:
+            cuda_root = None
+
+    if cuda_root is None:
+        return {
+            "cuda_root": "",
+            "nvcc": "nvcc",
+            "include": [],
+            "lib64": [],
+            "cflags": [],
+        }
+
+    cflags = ["-c", "--compiler-options", "'-fPIC'", "-std=c++11"]
+
+    cm = os.environ.get("CUDA_COMPUTE_CAPABILITY")
+    if cm is not None:
+        cflags.append("-gencode=arch=compute_{cm},code=compute_{cm}".format(cm=cm))
+    else:
+        print(
+            "Warning: Consider settings the CUDA_COMPUTE_CAPABILITY environment "
+            "variable to your GPU's compute capability."
+        )
+
+    return {
+        "cuda_root": cuda_root,
+        "nvcc": nvcc,
+        "include": [os.path.join(cuda_root, "include")],
+        "lib64": [os.path.join(cuda_root, "lib64")],
+        "cflags": cflags,
+    }
+
+
+def customize_compiler_for_nvcc(self):
+    if not isinstance(self, UnixCCompiler):
+        # Just give up
+        default_compile = self._compile
+
+        def _compile(obj, src, ext, cc_args, extra_postargs, pp_opts):
+            postargs = extra_postargs["gcc"]
+            return default_compile(obj, src, ext, cc_args, postargs, pp_opts)
+
+        self._compile = _compile
+        return
+
+    # Tell the compiler it can process .cu
+    self.src_extensions.append(".cu")
+
+    # Save references to the default compiler_so and _compile methods
+    default_compiler_so = self.compiler_so
+    default_compile = self._compile
+
+    # Now redefine the _compile method. This gets executed for each
+    # object but distutils doesn't have the ability to change compilers
+    # based on source extension: we add it.
+    def _compile(obj, src, ext, cc_args, extra_postargs, pp_opts):
+        if os.path.splitext(src)[1] == ".cu":
+            # use the cuda for .cu files
+            self.set_executable("compiler_so", cuda_info["nvcc"])
+            # use only a subset of the extra_postargs, which are 1-1
+            # translated from the extra_compile_args in the Extension class
+            postargs = extra_postargs["nvcc"]
+        else:
+            postargs = extra_postargs["gcc"]
+
+        default_compile(obj, src, ext, cc_args, postargs, pp_opts)
+        # Reset the default compiler_so, which we might have changed for cuda
+        self.compiler_so = default_compiler_so
+
+    # Inject our redefined _compile method into the class
+    self._compile = _compile
+
+
+# Run the customize_compiler
+class custom_build_ext(build_ext):
+    def build_extensions(self):
+        customize_compiler_for_nvcc(self.compiler)
+        super().build_extensions()
+
+
+cuda_info = locate_cuda()

doc/Makefile

+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    = 
+SPHINXBUILD   = sphinx-build
+SPHINXPROJ    = Veros
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

doc/_3rdparty/sphinx_fontawesome/LICENSE

+GNU GENERAL PUBLIC LICENSE
+   Version 2, June 1991
+
+Copyright (C) 1989, 1991 Free Software Foundation, Inc., <http://fsf.org/>
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+Everyone is permitted to copy and distribute verbatim copies
+of this license document, but changing it is not allowed.
+
+        Preamble
+
+The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it.  (Some other Free Software Foundation software is covered by
+the GNU Lesser General Public License instead.)  You can apply it to
+your programs, too.
+
+When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must show them these terms so they know their
+rights.
+
+We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+Finally, any free program is threatened constantly by software
+patents.  We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary.  To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+The precise terms and conditions for copying, distribution and
+modification follow.
+
+GNU GENERAL PUBLIC LICENSE
+TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+a) You must cause the modified files to carry prominent notices
+stating that you changed the files and the date of any change.
+
+b) You must cause any work that you distribute or publish, that in
+whole or in part contains or is derived from the Program or any
+part thereof, to be licensed as a whole at no charge to all third
+parties under the terms of this License.
+
+c) If the modified program normally reads commands interactively
+when run, you must cause it, when started running for such
+interactive use in the most ordinary way, to print or display an
+announcement including an appropriate copyright notice and a
+notice that there is no warranty (or else, saying that you provide
+a warranty) and that users may redistribute the program under
+these conditions, and telling the user how to view a copy of this
+License.  (Exception: if the Program itself is interactive but
+does not normally print such an announcement, your work based on
+the Program is not required to print an announcement.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+a) Accompany it with the complete corresponding machine-readable
+source code, which must be distributed under the terms of Sections
+1 and 2 above on a medium customarily used for software interchange; or,
+
+b) Accompany it with a written offer, valid for at least three
+years, to give any third party, for a charge no more than your
+cost of physically performing source distribution, a complete
+machine-readable copy of the corresponding source code, to be
+distributed under the terms of Sections 1 and 2 above on a medium
+customarily used for software interchange; or,
+
+c) Accompany it with the information you received as to the offer
+to distribute corresponding source code.  (This alternative is
+allowed only for noncommercial distribution and only if you
+received the program in object code or executable form with such
+an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+        NO WARRANTY
+
+11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+ END OF TERMS AND CONDITIONS
+
+How to Apply These Terms to Your New Programs
+
+If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+{description}
+Copyright (C) {year}  {fullname}
+
+This program is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+Gnomovision version 69, Copyright (C) year name of author
+Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+This is free software, and you are welcome to redistribute it
+under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary.  Here is a sample; alter the names:
+
+Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+`Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+{signature of Ty Coon}, 1 April 1989
+Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs.  If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.
+

doc/_3rdparty/sphinx_fontawesome/__init__.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+    Module sphinx_fontawesome
+"""
+
+import os
+from sphinx import addnodes
+from docutils.nodes import strong, emphasis, reference, Text
+from docutils.parsers.rst.roles import set_classes
+from docutils.parsers.rst import Directive
+import docutils.parsers.rst.directives as directives
+
+import sphinx_fontawesome.constant
+
+__version_info__ = (0, 0, 2)
+__version__ = '.'.join([str(val) for val in  __version_info__])
+
+sub_special = [
+                {'id' : 'o', 'key' : 'square-o'},
+                {'id' : 'x', 'key' : 'check-square-o'},
+                {'id' : 'smile', 'key' : 'smile-o'},
+                {'id' : 'mail', 'key' : 'envelope'},
+                {'id' : 'note', 'key' : 'info-circle'},
+              ]
+# add role
+def fa_global(key=''):
+    def fa(role, rawtext, text, lineno, inliner, options={}, content=[]):
+        options.update({'classes': []})
+        options['classes'].append('fa')
+        if key:
+            options['classes'].append('fa-%s' % key)
+        else:
+             for x in text.split(","):
+                options['classes'].append('fa-%s' % x)
+        set_classes(options)
+        node = emphasis(**options)
+        return [node], []
+    return fa
+
+#add directive
+class Fa(Directive):
+
+    has_content = True
+
+    def run(self):
+        options= {'classes' : []}
+        options['classes'].append('fa')
+        for x in self.content[0].split(' '):
+            options['classes'].append('fa-%s' % x)
+        set_classes(options)
+        node = emphasis(**options)
+        return [node]
+ 
+prolog = '\n'.join(['.. |%s| fa:: %s' % (icon, icon) for icon in sphinx_fontawesome.constant.icons])
+prolog += '\n'
+prolog += '\n'.join(['.. |%s| fa:: %s' % (icon['id'], icon['key']) for icon in sub_special])
+
+
+
+def setup(app):
+    app.add_role('fa', fa_global())
+    app.add_directive('fa', Fa)
+    app.config.rst_prolog = prolog
+    return {'version': __version__}
+

doc/_3rdparty/sphinx_fontawesome/constant.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+    list of icons Font-Awesome
+"""
+
+icons = [
+	'glass',
+	'music',
+	'search',
+	'envelope-o',
+	'heart',
+	'star',
+	'star-o',
+	'user',
+	'film',
+	'th-large',
+	'th',
+	'th-list',
+	'check',
+	'times',
+	'search-plus',
+	'search-minus',
+	'power-off',
+	'signal',
+	'cog',
+	'trash-o',
+	'home',
+	'file-o',
+	'clock-o',
+	'road',
+	'download',
+	'arrow-circle-o-down',
+	'arrow-circle-o-up',
+	'inbox',
+	'play-circle-o',
+	'repeat',
+	'refresh',
+	'list-alt',
+	'lock',
+	'flag',
+	'headphones',
+	'volume-off',
+	'volume-down',
+	'volume-up',
+	'qrcode',
+	'barcode',
+	'tag',
+	'tags',
+	'book',
+	'bookmark',
+	'print',
+	'camera',
+	'font',
+	'bold',
+	'italic',
+	'text-height',
+	'text-width',
+	'align-left',
+	'align-center',
+	'align-right',
+	'align-justify',
+	'list',
+	'outdent',
+	'indent',
+	'video-camera',
+	'picture-o',
+	'pencil',
+	'map-marker',
+	'adjust',
+	'tint',
+	'pencil-square-o',
+	'share-square-o',
+	'check-square-o',
+	'arrows',
+	'step-backward',
+	'fast-backward',
+	'backward',
+	'play',
+	'pause',
+	'stop',
+	'forward',
+	'fast-forward',
+	'step-forward',
+	'eject',
+	'chevron-left',
+	'chevron-right',
+	'plus-circle',
+	'minus-circle',
+	'times-circle',
+	'check-circle',
+	'question-circle',
+	'info-circle',
+	'crosshairs',
+	'times-circle-o',
+	'check-circle-o',
+	'ban',
+	'arrow-left',
+	'arrow-right',
+	'arrow-up',
+	'arrow-down',
+	'share',
+	'expand',
+	'compress',
+	'plus',
+	'minus',
+	'asterisk',
+	'exclamation-circle',
+	'gift',
+	'leaf',
+	'fire',
+	'eye',
+	'eye-slash',
+	'exclamation-triangle',
+	'plane',
+	'calendar',
+	'random',
+	'comment',
+	'magnet',
+	'chevron-up',
+	'chevron-down',
+	'retweet',
+	'shopping-cart',
+	'folder',
+	'folder-open',
+	'arrows-v',
+	'arrows-h',
+	'bar-chart',
+	'twitter-square',
+	'facebook-square',
+	'camera-retro',
+	'key',
+	'cogs',
+	'comments',
+	'thumbs-o-up',
+	'thumbs-o-down',
+	'star-half',
+	'heart-o',
+	'sign-out',
+	'linkedin-square',
+	'thumb-tack',
+	'external-link',
+	'sign-in',
+	'trophy',
+	'github-square',
+	'upload',
+	'lemon-o',
+	'phone',
+	'square-o',
+	'bookmark-o',
+	'phone-square',
+	'twitter',
+	'facebook',
+	'github',
+	'unlock',
+	'credit-card',
+	'rss',
+	'hdd-o',
+	'bullhorn',
+	'bell',
+	'certificate',
+	'hand-o-right',
+	'hand-o-left',
+	'hand-o-up',
+	'hand-o-down',
+	'arrow-circle-left',
+	'arrow-circle-right',
+	'arrow-circle-up',
+	'arrow-circle-down',
+	'globe',
+	'wrench',
+	'tasks',
+	'filter',
+	'briefcase',
+	'arrows-alt',
+	'users',
+	'link',
+	'cloud',
+	'flask',
+	'scissors',
+	'files-o',
+	'paperclip',
+	'floppy-o',
+	'square',
+	'bars',
+	'list-ul',
+	'list-ol',
+	'strikethrough',
+	'underline',
+	'table',
+	'magic',
+	'truck',
+	'pinterest',
+	'pinterest-square',
+	'google-plus-square',
+	'google-plus',
+	'money',
+	'caret-down',
+	'caret-up',
+	'caret-left',
+	'caret-right',
+	'columns',
+	'sort',
+	'sort-desc',
+	'sort-asc',
+	'envelope',
+	'linkedin',
+	'undo',
+	'gavel',
+	'tachometer',
+	'comment-o',
+	'comments-o',
+	'bolt',
+	'sitemap',
+	'umbrella',
+	'clipboard',
+	'lightbulb-o',
+	'exchange',
+	'cloud-download',
+	'cloud-upload',
+	'user-md',
+	'stethoscope',
+	'suitcase',
+	'bell-o',
+	'coffee',
+	'cutlery',
+	'file-text-o',
+	'building-o',
+	'hospital-o',
+	'ambulance',
+	'medkit',
+	'fighter-jet',
+	'beer',
+	'h-square',
+	'plus-square',
+	'angle-double-left',
+	'angle-double-right',
+	'angle-double-up',
+	'angle-double-down',
+	'angle-left',
+	'angle-right',
+	'angle-up',
+	'angle-down',
+	'desktop',
+	'laptop',
+	'tablet',
+	'mobile',
+	'circle-o',
+	'quote-left',
+	'quote-right',
+	'spinner',
+	'circle',
+	'reply',
+	'github-alt',
+	'folder-o',
+	'folder-open-o',
+	'smile-o',
+	'frown-o',
+	'meh-o',
+	'gamepad',
+	'keyboard-o',
+	'flag-o',
+	'flag-checkered',
+	'terminal',
+	'code',
+	'reply-all',
+	'star-half-o',
+	'location-arrow',
+	'crop',
+	'code-fork',
+	'chain-broken',
+	'question',
+	'info',
+	'exclamation',
+	'superscript',
+	'subscript',
+	'eraser',
+	'puzzle-piece',
+	'microphone',
+	'microphone-slash',
+	'shield',
+	'calendar-o',
+	'fire-extinguisher',
+	'rocket',
+	'maxcdn',
+	'chevron-circle-left',
+	'chevron-circle-right',
+	'chevron-circle-up',
+	'chevron-circle-down',
+	'html5',
+	'css3',
+	'anchor',
+	'unlock-alt',
+	'bullseye',
+	'ellipsis-h',
+	'ellipsis-v',
+	'rss-square',
+	'play-circle',
+	'ticket',
+	'minus-square',
+	'minus-square-o',
+	'level-up',
+	'level-down',
+	'check-square',
+	'pencil-square',
+	'external-link-square',
+	'share-square',
+	'compass',
+	'caret-square-o-down',
+	'caret-square-o-up',
+	'caret-square-o-right',
+	'eur',
+	'gbp',
+	'usd',
+	'inr',
+	'jpy',
+	'rub',
+	'krw',
+	'btc',
+	'file',
+	'file-text',
+	'sort-alpha-asc',
+	'sort-alpha-desc',
+	'sort-amount-asc',
+	'sort-amount-desc',
+	'sort-numeric-asc',
+	'sort-numeric-desc',
+	'thumbs-up',
+	'thumbs-down',
+	'youtube-square',
+	'youtube',
+	'xing',
+	'xing-square',
+	'youtube-play',
+	'dropbox',
+	'stack-overflow',
+	'instagram',
+	'flickr',
+	'adn',
+	'bitbucket',
+	'bitbucket-square',
+	'tumblr',
+	'tumblr-square',
+	'long-arrow-down',
+	'long-arrow-up',
+	'long-arrow-left',
+	'long-arrow-right',
+	'apple',
+	'windows',
+	'android',
+	'linux',
+	'dribbble',
+	'skype',
+	'foursquare',
+	'trello',
+	'female',
+	'male',
+	'gratipay',
+	'sun-o',
+	'moon-o',
+	'archive',
+	'bug',
+	'vk',
+	'weibo',
+	'renren',
+	'pagelines',
+	'stack-exchange',
+	'arrow-circle-o-right',
+	'arrow-circle-o-left',
+	'caret-square-o-left',
+	'dot-circle-o',
+	'wheelchair',
+	'vimeo-square',
+	'try',
+	'plus-square-o',
+	'space-shuttle',
+	'slack',
+	'envelope-square',
+	'wordpress',
+	'openid',
+	'university',
+	'graduation-cap',
+	'yahoo',
+	'google',
+	'reddit',
+	'reddit-square',
+	'stumbleupon-circle',
+	'stumbleupon',
+	'delicious',
+	'digg',
+	'pied-piper',
+	'pied-piper-alt',
+	'drupal',
+	'joomla',
+	'language',
+	'fax',
+	'building',
+	'child',
+	'paw',
+	'spoon',
+	'cube',
+	'cubes',
+	'behance',
+	'behance-square',
+	'steam',
+	'steam-square',
+	'recycle',
+	'car',
+	'taxi',
+	'tree',
+	'spotify',
+	'deviantart',
+	'soundcloud',
+	'database',
+	'file-pdf-o',
+	'file-word-o',
+	'file-excel-o',
+	'file-powerpoint-o',
+	'file-image-o',
+	'file-archive-o',
+	'file-audio-o',
+	'file-video-o',
+	'file-code-o',
+	'vine',
+	'codepen',
+	'jsfiddle',
+	'life-ring',
+	'circle-o-notch',
+	'rebel',
+	'empire',
+	'git-square',
+	'git',
+	'hacker-news',
+	'tencent-weibo',
+	'qq',
+	'weixin',
+	'paper-plane',
+	'paper-plane-o',
+	'history',
+	'circle-thin',
+	'header',
+	'paragraph',
+	'sliders',
+	'share-alt',
+	'share-alt-square',
+	'bomb',
+	'futbol-o',
+	'tty',
+	'binoculars',
+	'plug',
+	'slideshare',
+	'twitch',
+	'yelp',
+	'newspaper-o',
+	'wifi',
+	'calculator',
+	'paypal',
+	'google-wallet',
+	'cc-visa',
+	'cc-mastercard',
+	'cc-discover',
+	'cc-amex',
+	'cc-paypal',
+	'cc-stripe',
+	'bell-slash',
+	'bell-slash-o',
+	'trash',
+	'copyright',
+	'at',
+	'eyedropper',
+	'paint-brush',
+	'birthday-cake',
+	'area-chart',
+	'pie-chart',
+	'line-chart',
+	'lastfm',
+	'lastfm-square',
+	'toggle-off',
+	'toggle-on',
+	'bicycle',
+	'bus',
+	'ioxhost',
+	'angellist',
+	'cc',
+	'ils',
+	'meanpath',
+	'buysellads',
+	'connectdevelop',
+	'dashcube',
+	'forumbee',
+	'leanpub',
+	'sellsy',
+	'shirtsinbulk',
+	'simplybuilt',
+	'skyatlas',
+	'cart-plus',
+	'cart-arrow-down',
+	'diamond',
+	'ship',
+	'user-secret',
+	'motorcycle',
+	'street-view',
+	'heartbeat',
+	'venus',
+	'mars',
+	'mercury',
+	'transgender',
+	'transgender-alt',
+	'venus-double',
+	'mars-double',
+	'venus-mars',
+	'mars-stroke',
+	'mars-stroke-v',
+	'mars-stroke-h',
+	'neuter',
+	'genderless',
+	'facebook-official',
+	'pinterest-p',
+	'whatsapp',
+	'server',
+	'user-plus',
+	'user-times',
+	'bed',
+	'viacoin',
+	'train',
+	'subway',
+	'medium',
+	'y-combinator',
+	'optin-monster',
+	'opencart',
+	'expeditedssl',
+	'battery-full',
+	'battery-three-quarters',
+	'battery-half',
+	'battery-quarter',
+	'battery-empty',
+	'mouse-pointer',
+	'i-cursor',
+	'object-group',
+	'object-ungroup',
+	'sticky-note',
+	'sticky-note-o',
+	'cc-jcb',
+	'cc-diners-club',
+	'clone',
+	'balance-scale',
+	'hourglass-o',
+	'hourglass-start',
+	'hourglass-half',
+	'hourglass-end',
+	'hourglass',
+	'hand-rock-o',
+	'hand-paper-o',
+	'hand-scissors-o',
+	'hand-lizard-o',
+	'hand-spock-o',
+	'hand-pointer-o',
+	'hand-peace-o',
+	'trademark',
+	'registered',
+	'creative-commons',
+	'gg',
+	'gg-circle',
+	'tripadvisor',
+	'odnoklassniki',
+	'odnoklassniki-square',
+	'get-pocket',
+	'wikipedia-w',
+	'safari',
+	'chrome',
+	'firefox',
+	'opera',
+	'internet-explorer',
+	'television',
+	'contao',
+	'500px',
+	'amazon',
+	'calendar-plus-o',
+	'calendar-minus-o',
+	'calendar-times-o',
+	'calendar-check-o',
+	'industry',
+	'map-pin',
+	'map-signs',
+	'map-o',
+	'map',
+	'commenting',
+	'commenting-o',
+	'houzz',
+	'vimeo',
+	'black-tie',
+	'fonticons',
+]
+# generate by tools/list_icons
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+    list of icons Font-Awesome
+"""
+
+icons = [
+	'glass',
+	'music',
+	'search',
+	'envelope-o',
+	'heart',
+	'star',
+	'star-o',
+	'user',
+	'film',
+	'th-large',
+	'th',
+	'th-list',
+	'check',
+	'times',
+	'remove',
+	'close',
+	'search-plus',
+	'search-minus',
+	'power-off',
+	'signal',
+	'cog',
+	'gear',
+	'trash-o',
+	'home',
+	'file-o',
+	'clock-o',
+	'road',
+	'download',
+	'arrow-circle-o-down',
+	'arrow-circle-o-up',
+	'inbox',
+	'play-circle-o',
+	'repeat',
+	'rotate-right',
+	'refresh',
+	'list-alt',
+	'lock',
+	'flag',
+	'headphones',
+	'volume-off',
+	'volume-down',
+	'volume-up',
+	'qrcode',
+	'barcode',
+	'tag',
+	'tags',
+	'book',
+	'bookmark',
+	'print',
+	'camera',
+	'font',
+	'bold',
+	'italic',
+	'text-height',
+	'text-width',
+	'align-left',
+	'align-center',
+	'align-right',
+	'align-justify',
+	'list',
+	'outdent',
+	'dedent',
+	'indent',
+	'video-camera',
+	'picture-o',
+	'photo',
+	'image',
+	'pencil',
+	'map-marker',
+	'adjust',
+	'tint',
+	'pencil-square-o',
+	'edit',
+	'share-square-o',
+	'check-square-o',
+	'arrows',
+	'step-backward',
+	'fast-backward',
+	'backward',
+	'play',
+	'pause',
+	'stop',
+	'forward',
+	'fast-forward',
+	'step-forward',
+	'eject',
+	'chevron-left',
+	'chevron-right',
+	'plus-circle',
+	'minus-circle',
+	'times-circle',
+	'check-circle',
+	'question-circle',
+	'info-circle',
+	'crosshairs',
+	'times-circle-o',
+	'check-circle-o',
+	'ban',
+	'arrow-left',
+	'arrow-right',
+	'arrow-up',
+	'arrow-down',
+	'share',
+	'mail-forward',
+	'expand',
+	'compress',
+	'plus',
+	'minus',
+	'asterisk',
+	'exclamation-circle',
+	'gift',
+	'leaf',
+	'fire',
+	'eye',
+	'eye-slash',
+	'exclamation-triangle',
+	'warning',
+	'plane',
+	'calendar',
+	'random',
+	'comment',
+	'magnet',
+	'chevron-up',
+	'chevron-down',
+	'retweet',
+	'shopping-cart',
+	'folder',
+	'folder-open',
+	'arrows-v',
+	'arrows-h',
+	'bar-chart',
+	'bar-chart-o',
+	'twitter-square',
+	'facebook-square',
+	'camera-retro',
+	'key',
+	'cogs',
+	'gears',
+	'comments',
+	'thumbs-o-up',
+	'thumbs-o-down',
+	'star-half',
+	'heart-o',
+	'sign-out',
+	'linkedin-square',
+	'thumb-tack',
+	'external-link',
+	'sign-in',
+	'trophy',
+	'github-square',
+	'upload',
+	'lemon-o',
+	'phone',
+	'square-o',
+	'bookmark-o',
+	'phone-square',
+	'twitter',
+	'facebook',
+	'facebook-f',
+	'github',
+	'unlock',
+	'credit-card',
+	'rss',
+	'feed',
+	'hdd-o',
+	'bullhorn',
+	'bell',
+	'certificate',
+	'hand-o-right',
+	'hand-o-left',
+	'hand-o-up',
+	'hand-o-down',
+	'arrow-circle-left',
+	'arrow-circle-right',
+	'arrow-circle-up',
+	'arrow-circle-down',
+	'globe',
+	'wrench',
+	'tasks',
+	'filter',
+	'briefcase',
+	'arrows-alt',
+	'users',
+	'group',
+	'link',
+	'chain',
+	'cloud',
+	'flask',
+	'scissors',
+	'cut',
+	'files-o',
+	'copy',
+	'paperclip',
+	'floppy-o',
+	'save',
+	'square',
+	'bars',
+	'navicon',
+	'reorder',
+	'list-ul',
+	'list-ol',
+	'strikethrough',
+	'underline',
+	'table',
+	'magic',
+	'truck',
+	'pinterest',
+	'pinterest-square',
+	'google-plus-square',
+	'google-plus',
+	'money',
+	'caret-down',
+	'caret-up',
+	'caret-left',
+	'caret-right',
+	'columns',
+	'sort',
+	'unsorted',
+	'sort-desc',
+	'sort-down',
+	'sort-asc',
+	'sort-up',
+	'envelope',
+	'linkedin',
+	'undo',
+	'rotate-left',
+	'gavel',
+	'legal',
+	'tachometer',
+	'dashboard',
+	'comment-o',
+	'comments-o',
+	'bolt',
+	'flash',
+	'sitemap',
+	'umbrella',
+	'clipboard',
+	'paste',
+	'lightbulb-o',
+	'exchange',
+	'cloud-download',
+	'cloud-upload',
+	'user-md',
+	'stethoscope',
+	'suitcase',
+	'bell-o',
+	'coffee',
+	'cutlery',
+	'file-text-o',
+	'building-o',
+	'hospital-o',
+	'ambulance',
+	'medkit',
+	'fighter-jet',
+	'beer',
+	'h-square',
+	'plus-square',
+	'angle-double-left',
+	'angle-double-right',
+	'angle-double-up',
+	'angle-double-down',
+	'angle-left',
+	'angle-right',
+	'angle-up',
+	'angle-down',
+	'desktop',
+	'laptop',
+	'tablet',
+	'mobile',
+	'mobile-phone',
+	'circle-o',
+	'quote-left',
+	'quote-right',
+	'spinner',
+	'circle',
+	'reply',
+	'mail-reply',
+	'github-alt',
+	'folder-o',
+	'folder-open-o',
+	'smile-o',
+	'frown-o',
+	'meh-o',
+	'gamepad',
+	'keyboard-o',
+	'flag-o',
+	'flag-checkered',
+	'terminal',
+	'code',
+	'reply-all',
+	'mail-reply-all',
+	'star-half-o',
+	'star-half-empty',
+	'star-half-full',
+	'location-arrow',
+	'crop',
+	'code-fork',
+	'chain-broken',
+	'unlink',
+	'question',
+	'info',
+	'exclamation',
+	'superscript',
+	'subscript',
+	'eraser',
+	'puzzle-piece',
+	'microphone',
+	'microphone-slash',
+	'shield',
+	'calendar-o',
+	'fire-extinguisher',
+	'rocket',
+	'maxcdn',
+	'chevron-circle-left',
+	'chevron-circle-right',
+	'chevron-circle-up',
+	'chevron-circle-down',
+	'html5',
+	'css3',
+	'anchor',
+	'unlock-alt',
+	'bullseye',
+	'ellipsis-h',
+	'ellipsis-v',
+	'rss-square',
+	'play-circle',
+	'ticket',
+	'minus-square',
+	'minus-square-o',
+	'level-up',
+	'level-down',
+	'check-square',
+	'pencil-square',
+	'external-link-square',
+	'share-square',
+	'compass',
+	'caret-square-o-down',
+	'toggle-down',
+	'caret-square-o-up',
+	'toggle-up',
+	'caret-square-o-right',
+	'toggle-right',
+	'eur',
+	'euro',
+	'gbp',
+	'usd',
+	'dollar',
+	'inr',
+	'rupee',
+	'jpy',
+	'cny',
+	'rmb',
+	'yen',
+	'rub',
+	'ruble',
+	'rouble',
+	'krw',
+	'won',
+	'btc',
+	'bitcoin',
+	'file',
+	'file-text',
+	'sort-alpha-asc',
+	'sort-alpha-desc',
+	'sort-amount-asc',
+	'sort-amount-desc',
+	'sort-numeric-asc',
+	'sort-numeric-desc',
+	'thumbs-up',
+	'thumbs-down',
+	'youtube-square',
+	'youtube',
+	'xing',
+	'xing-square',
+	'youtube-play',
+	'dropbox',
+	'stack-overflow',
+	'instagram',
+	'flickr',
+	'adn',
+	'bitbucket',
+	'bitbucket-square',
+	'tumblr',
+	'tumblr-square',
+	'long-arrow-down',
+	'long-arrow-up',
+	'long-arrow-left',
+	'long-arrow-right',
+	'apple',
+	'windows',
+	'android',
+	'linux',
+	'dribbble',
+	'skype',
+	'foursquare',
+	'trello',
+	'female',
+	'male',
+	'gratipay',
+	'gittip',
+	'sun-o',
+	'moon-o',
+	'archive',
+	'bug',
+	'vk',
+	'weibo',
+	'renren',
+	'pagelines',
+	'stack-exchange',
+	'arrow-circle-o-right',
+	'arrow-circle-o-left',
+	'caret-square-o-left',
+	'toggle-left',
+	'dot-circle-o',
+	'wheelchair',
+	'vimeo-square',
+	'try',
+	'turkish-lira',
+	'plus-square-o',
+	'space-shuttle',
+	'slack',
+	'envelope-square',
+	'wordpress',
+	'openid',
+	'university',
+	'institution',
+	'bank',
+	'graduation-cap',
+	'mortar-board',
+	'yahoo',
+	'google',
+	'reddit',
+	'reddit-square',
+	'stumbleupon-circle',
+	'stumbleupon',
+	'delicious',
+	'digg',
+	'pied-piper',
+	'pied-piper-alt',
+	'drupal',
+	'joomla',
+	'language',
+	'fax',
+	'building',
+	'child',
+	'paw',
+	'spoon',
+	'cube',
+	'cubes',
+	'behance',
+	'behance-square',
+	'steam',
+	'steam-square',
+	'recycle',
+	'car',
+	'automobile',
+	'taxi',
+	'cab',
+	'tree',
+	'spotify',
+	'deviantart',
+	'soundcloud',
+	'database',
+	'file-pdf-o',
+	'file-word-o',
+	'file-excel-o',
+	'file-powerpoint-o',
+	'file-image-o',
+	'file-photo-o',
+	'file-picture-o',
+	'file-archive-o',
+	'file-zip-o',
+	'file-audio-o',
+	'file-sound-o',
+	'file-video-o',
+	'file-movie-o',
+	'file-code-o',
+	'vine',
+	'codepen',
+	'jsfiddle',
+	'life-ring',
+	'life-bouy',
+	'life-buoy',
+	'life-saver',
+	'support',
+	'circle-o-notch',
+	'rebel',
+	'ra',
+	'empire',
+	'ge',
+	'git-square',
+	'git',
+	'hacker-news',
+	'y-combinator-square',
+	'yc-square',
+	'tencent-weibo',
+	'qq',
+	'weixin',
+	'wechat',
+	'paper-plane',
+	'send',
+	'paper-plane-o',
+	'send-o',
+	'history',
+	'circle-thin',
+	'header',
+	'paragraph',
+	'sliders',
+	'share-alt',
+	'share-alt-square',
+	'bomb',
+	'futbol-o',
+	'soccer-ball-o',
+	'tty',
+	'binoculars',
+	'plug',
+	'slideshare',
+	'twitch',
+	'yelp',
+	'newspaper-o',
+	'wifi',
+	'calculator',
+	'paypal',
+	'google-wallet',
+	'cc-visa',
+	'cc-mastercard',
+	'cc-discover',
+	'cc-amex',
+	'cc-paypal',
+	'cc-stripe',
+	'bell-slash',
+	'bell-slash-o',
+	'trash',
+	'copyright',
+	'at',
+	'eyedropper',
+	'paint-brush',
+	'birthday-cake',
+	'area-chart',
+	'pie-chart',
+	'line-chart',
+	'lastfm',
+	'lastfm-square',
+	'toggle-off',
+	'toggle-on',
+	'bicycle',
+	'bus',
+	'ioxhost',
+	'angellist',
+	'cc',
+	'ils',
+	'shekel',
+	'sheqel',
+	'meanpath',
+	'buysellads',
+	'connectdevelop',
+	'dashcube',
+	'forumbee',
+	'leanpub',
+	'sellsy',
+	'shirtsinbulk',
+	'simplybuilt',
+	'skyatlas',
+	'cart-plus',
+	'cart-arrow-down',
+	'diamond',
+	'ship',
+	'user-secret',
+	'motorcycle',
+	'street-view',
+	'heartbeat',
+	'venus',
+	'mars',
+	'mercury',
+	'transgender',
+	'intersex',
+	'transgender-alt',
+	'venus-double',
+	'mars-double',
+	'venus-mars',
+	'mars-stroke',
+	'mars-stroke-v',
+	'mars-stroke-h',
+	'neuter',
+	'genderless',
+	'facebook-official',
+	'pinterest-p',
+	'whatsapp',
+	'server',
+	'user-plus',
+	'user-times',
+	'bed',
+	'hotel',
+	'viacoin',
+	'train',
+	'subway',
+	'medium',
+	'y-combinator',
+	'yc',
+	'optin-monster',
+	'opencart',
+	'expeditedssl',
+	'battery-full',
+	'battery-4',
+	'battery-three-quarters',
+	'battery-3',
+	'battery-half',
+	'battery-2',
+	'battery-quarter',
+	'battery-1',
+	'battery-empty',
+	'battery-0',
+	'mouse-pointer',
+	'i-cursor',
+	'object-group',
+	'object-ungroup',
+	'sticky-note',
+	'sticky-note-o',
+	'cc-jcb',
+	'cc-diners-club',
+	'clone',
+	'balance-scale',
+	'hourglass-o',
+	'hourglass-start',
+	'hourglass-1',
+	'hourglass-half',
+	'hourglass-2',
+	'hourglass-end',
+	'hourglass-3',
+	'hourglass',
+	'hand-rock-o',
+	'hand-grab-o',
+	'hand-paper-o',
+	'hand-stop-o',
+	'hand-scissors-o',
+	'hand-lizard-o',
+	'hand-spock-o',
+	'hand-pointer-o',
+	'hand-peace-o',
+	'trademark',
+	'registered',
+	'creative-commons',
+	'gg',
+	'gg-circle',
+	'tripadvisor',
+	'odnoklassniki',
+	'odnoklassniki-square',
+	'get-pocket',
+	'wikipedia-w',
+	'safari',
+	'chrome',
+	'firefox',
+	'opera',
+	'internet-explorer',
+	'television',
+	'tv',
+	'contao',
+	'500px',
+	'amazon',
+	'calendar-plus-o',
+	'calendar-minus-o',
+	'calendar-times-o',
+	'calendar-check-o',
+	'industry',
+	'map-pin',
+	'map-signs',
+	'map-o',
+	'map',
+	'commenting',
+	'commenting-o',
+	'houzz',
+	'vimeo',
+	'black-tie',
+	'fonticons',
+]
+# generate by tools/list_icons
+

doc/_downloads/modi_veros_batch.sh

+#!/bin/bash -l
+#
+#SBATCH -p modi_short
+#SBATCH --nodes=1
+#SBATCH --ntasks=1
+#SBATCH --job-name=verosbgc
+
+echo started at `date`
+
+singularity exec ~/modi_images/hpc-ocean-notebook_latest.sif bash -l ~/modi_mount/modi_veros_run.sh $1
+
+echo finished at `date`
+

doc/_downloads/modi_veros_run.sh

+#!/bin/bash -l
+#
+# Activate conda in your $PATH
+# This ensures that we discover every conda environment
+# before we try to activate it.
+source $CONDA_DIR/etc/profile.d/conda.sh
+
+# Activate the existing veros environment
+conda activate ~/modi_mount/conda-env-veros
+
+# Change your current directory to available Veros-BGC setup
+cd `dirname "$1"`
+
+# Run your setup in the current directory (~/modi_mount/bgc_global_4deg)
+python3 $1
+

doc/_downloads/veros_batch.sh

+#!/bin/bash -l
+#
+#SBATCH -p mycluster
+#SBATCH -A myaccount
+#SBATCH --job-name=veros_mysetup
+#SBATCH --nodes=2
+#SBATCH --ntasks=64
+#SBATCH --cpus-per-task=1
+#SBATCH --exclusive
+#SBATCH --mail-type=ALL
+#SBATCH --mail-user=your@email.xyz
+
+# load module dependencies
+# module load petsc4py mpi4py h5py ...
+
+export OMP_NUM_THREADS=1
+
+# adapt srun command to your available scheduler / MPI implementation
+veros resubmit -i my_run -n 8 -l 7776000 \
+    -c "srun --mpi=pmi2 -- veros run my_setup.py -b jax -n 4 4" \
+    --callback "sbatch veros_batch.sh"