"lib/llm/vscode:/vscode.git/clone" did not exist on "0844f8ee0365472bd38612b3e692e0b8625fee27"
main.rs 4.2 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
// SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

use std::env;

use clap::Parser;

Neelay Shah's avatar
Neelay Shah committed
20
21
use dynamo_run::{Input, Output};
use dynamo_runtime::logging;
22
23

const HELP: &str = r#"
Neelay Shah's avatar
Neelay Shah committed
24
dynamo-run is a single binary that wires together the various inputs (http, text, network) and workers (network, engine), that runs the services. It is the simplest way to use dynamo locally.
25
26

Example:
27
28
- cargo build --features cuda -p dynamo-run
- cd target/debug
29
- ./dynamo-run Qwen/Qwen3-0.6B
30
- OR: ./dynamo-run /data/models/Llama-3.2-1B-Instruct-Q4_K_M.gguf
31
32
"#;

33
const USAGE: &str = "USAGE: dynamo-run in=[http|text|dyn://<path>|batch:<folder>] out=ENGINE_LIST|dyn [--http-port 8080] [--model-path <path>] [--model-name <served-model-name>] [--model-config <hf-repo>] [--tensor-parallel-size=1] [--context-length=N] [--kv-cache-block-size=16] [--num-nodes=1] [--node-rank=0] [--leader-addr=127.0.0.1:9876] [--base-gpu-id=0] [--extra-engine-args=args.json] [--router-mode random|round-robin|kv]";
34
35

fn main() -> anyhow::Result<()> {
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
    // Set log level based on verbosity flag
    let log_level = match dynamo_run::Flags::try_parse() {
        Ok(flags) => match flags.verbosity {
            0 => "info",
            1 => "debug",
            2 => "trace",
            _ => {
                return Err(anyhow::anyhow!(
                    "Invalid verbosity level. Valid values are v (debug) or vv (trace)"
                ))
            }
        },
        Err(_) => "info",
    };

    if log_level != "info" {
        std::env::set_var("DYN_LOG", log_level);
    }

55
56
57
    logging::init();

    // max_worker_threads and max_blocking_threads from env vars or config file.
Neelay Shah's avatar
Neelay Shah committed
58
    let rt_config = dynamo_runtime::RuntimeConfig::from_settings()?;
59
60

    // One per process. Wraps a Runtime with holds two tokio runtimes.
Neelay Shah's avatar
Neelay Shah committed
61
    let worker = dynamo_runtime::Worker::from_config(rt_config)?;
62

63
    worker.execute(wrapper)
64
65
}

Neelay Shah's avatar
Neelay Shah committed
66
async fn wrapper(runtime: dynamo_runtime::Runtime) -> anyhow::Result<()> {
67
68
69
    let mut in_opt = None;
    let mut out_opt = None;
    let args: Vec<String> = env::args().skip(1).collect();
70
71
72
73
74
    if args.is_empty()
        || args[0] == "-h"
        || args[0] == "--help"
        || (args.iter().all(|arg| arg == "-v" || arg == "-vv"))
    {
75
76
77
        let engine_list = Output::available_engines().join("|");
        let usage = USAGE.replace("ENGINE_LIST", &engine_list);
        println!("{usage}");
78
79
80
81
82
        println!("{HELP}");
        return Ok(());
    }
    for arg in env::args().skip(1).take(2) {
        let Some((in_out, val)) = arg.split_once('=') else {
83
84
            // Probably we're defaulting in and/or out, and this is a flag
            continue;
85
86
87
88
89
90
91
92
93
94
95
96
97
        };
        match in_out {
            "in" => {
                in_opt = Some(val.try_into()?);
            }
            "out" => {
                out_opt = Some(val.try_into()?);
            }
            _ => {
                anyhow::bail!("Invalid argument, must start with 'in' or 'out. {USAGE}");
            }
        }
    }
98
99
100
101
102
103
    let mut non_flag_params = 1; // binary name
    let in_opt = match in_opt {
        Some(x) => {
            non_flag_params += 1;
            x
        }
104
        None => Input::default(),
105
    };
106
107
108
    if out_opt.is_some() {
        non_flag_params += 1;
    }
109
110

    // Clap skips the first argument expecting it to be the binary name, so add it back
111
    // Note `--model-path` has index=1 (in lib.rs) so that doesn't need a flag.
Neelay Shah's avatar
Neelay Shah committed
112
113
    let flags = dynamo_run::Flags::try_parse_from(
        ["dynamo-run".to_string()]
114
115
116
            .into_iter()
            .chain(env::args().skip(non_flag_params)),
    )?;
117

118
    dynamo_run::run(runtime, in_opt, out_opt, flags).await
119
}