main.rs 4.94 KB
Newer Older
1
2
3
4
5
// SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

use std::env;

6
use clap::{CommandFactory as _, Parser};
7

8
9
use dynamo_llm::entrypoint::input::Input;
use dynamo_run::Output;
Neelay Shah's avatar
Neelay Shah committed
10
use dynamo_runtime::logging;
11
12

const HELP: &str = r#"
Neelay Shah's avatar
Neelay Shah committed
13
dynamo-run is a single binary that wires together the various inputs (http, text, network) and workers (network, engine), that runs the services. It is the simplest way to use dynamo locally.
14

15
16
17
18
19
Verbosity:
- -v enables debug logs
- -vv enables full trace logs
- Default is info level logging

20
Example:
21
22
- cargo build --features cuda -p dynamo-run
- cd target/debug
23
- ./dynamo-run Qwen/Qwen3-0.6B (OR ./dynamo-run /data/hf-checkouts/Qwen3-0.6B)
24
25

See `docs/guides/dynamo_run.md` in the repo for full details.
26
27
"#;

28
const USAGE: &str = "USAGE: dynamo-run in=[http|grpc|text|dyn://<path>|batch:<folder>] out=ENGINE_LIST|auto|dyn://<path> [--http-port 8080] [--model-path <path>] [--model-name <served-model-name>] [--context-length=N] [--kv-cache-block-size=16] [--extra-engine-args=args.json] [--router-mode random|round-robin|kv] [--kv-overlap-score-weight=2.0] [--router-temperature=0.0] [--use-kv-events] [--max-num-batched-tokens=1.0] [--migration-limit=0] [--verbosity (-v|-vv)]";
29
30

fn main() -> anyhow::Result<()> {
31
32
33
34
35
36
37
38
39
    // Set log level based on verbosity flag
    let log_level = match dynamo_run::Flags::try_parse() {
        Ok(flags) => match flags.verbosity {
            0 => "info",
            1 => "debug",
            2 => "trace",
            _ => {
                return Err(anyhow::anyhow!(
                    "Invalid verbosity level. Valid values are v (debug) or vv (trace)"
40
                ));
41
42
43
44
45
46
            }
        },
        Err(_) => "info",
    };

    if log_level != "info" {
47
        unsafe { std::env::set_var("DYN_LOG", log_level) };
48
49
    }

50
51
52
    logging::init();

    // max_worker_threads and max_blocking_threads from env vars or config file.
Neelay Shah's avatar
Neelay Shah committed
53
    let rt_config = dynamo_runtime::RuntimeConfig::from_settings()?;
54
    tracing::debug!("Runtime config: {rt_config}");
55

56
    // One per process. Wraps a Runtime with holds one or two tokio runtimes.
Neelay Shah's avatar
Neelay Shah committed
57
    let worker = dynamo_runtime::Worker::from_config(rt_config)?;
58

59
    worker.execute(wrapper)
60
61
}

Neelay Shah's avatar
Neelay Shah committed
62
async fn wrapper(runtime: dynamo_runtime::Runtime) -> anyhow::Result<()> {
63
64
65
    let mut in_opt = None;
    let mut out_opt = None;
    let args: Vec<String> = env::args().skip(1).collect();
66
67
68
69
70
    if args.is_empty()
        || args[0] == "-h"
        || args[0] == "--help"
        || (args.iter().all(|arg| arg == "-v" || arg == "-vv"))
    {
71
72
73
        let engine_list = Output::available_engines().join("|");
        let usage = USAGE.replace("ENGINE_LIST", &engine_list);
        println!("{usage}");
74
        println!("{HELP}");
75
        dynamo_run::Flags::command().print_long_help().unwrap();
76
        return Ok(());
77
78
79
80
81
82
83
    } else if args[0] == "--version" {
        if let Some(describe) = option_env!("VERGEN_GIT_DESCRIBE") {
            println!("dynamo-run {}", describe);
        } else {
            println!("Version not available (git describe not available)");
        }
        return Ok(());
84
85
86
    }
    for arg in env::args().skip(1).take(2) {
        let Some((in_out, val)) = arg.split_once('=') else {
87
88
            // Probably we're defaulting in and/or out, and this is a flag
            continue;
89
90
91
92
93
94
        };
        match in_out {
            "in" => {
                in_opt = Some(val.try_into()?);
            }
            "out" => {
95
                if val == "sglang" || val == "trtllm" || val == "vllm" {
96
                    tracing::error!(
97
                        "To run the {val} engine please use the Python interface, see root README or look in directory `examples/backends/`."
98
                    );
99
100
101
                    std::process::exit(1);
                }

102
103
104
105
106
107
108
                out_opt = Some(val.try_into()?);
            }
            _ => {
                anyhow::bail!("Invalid argument, must start with 'in' or 'out. {USAGE}");
            }
        }
    }
109
110
111
112
113
114
    let mut non_flag_params = 1; // binary name
    let in_opt = match in_opt {
        Some(x) => {
            non_flag_params += 1;
            x
        }
115
        None => Input::default(),
116
    };
117
118
119
    if out_opt.is_some() {
        non_flag_params += 1;
    }
120
121

    // Clap skips the first argument expecting it to be the binary name, so add it back
122
    // Note `--model-path` has index=1 (in lib.rs) so that doesn't need a flag.
Neelay Shah's avatar
Neelay Shah committed
123
124
    let flags = dynamo_run::Flags::try_parse_from(
        ["dynamo-run".to_string()]
125
126
127
            .into_iter()
            .chain(env::args().skip(non_flag_params)),
    )?;
128

129
130
131
132
    if is_in_dynamic(&in_opt) && is_out_dynamic(&out_opt) {
        anyhow::bail!("Cannot use endpoint for both in and out");
    }

133
    dynamo_run::run(runtime, in_opt, out_opt, flags).await
134
}
135
136
137
138
139
140

fn is_in_dynamic(in_opt: &Input) -> bool {
    matches!(in_opt, Input::Endpoint(_))
}

fn is_out_dynamic(out_opt: &Option<Output>) -> bool {
141
    matches!(out_opt, Some(Output::Auto))
142
}