Unverified Commit 877b2ec3 authored by Abrar Shivani's avatar Abrar Shivani Committed by GitHub
Browse files

feat: Support hf:// URLs in dynamo run (#917)

Allow `hf://` prefix on command line. 

Closes GitHub issue: https://github.com/ai-dynamo/dynamo/issues/829
parent 2be5e8f5
......@@ -15,7 +15,7 @@
#[cfg(any(feature = "vllm", feature = "sglang"))]
use std::{future::Future, pin::Pin};
use std::{io::Read, sync::Arc};
use std::{io::Read, path::PathBuf, sync::Arc};
use dynamo_llm::{
backend::ExecutionContext, engines::StreamingEngine, kv_router::publisher::KvMetricsPublisher,
......@@ -53,6 +53,9 @@ const PYTHON_STR_SCHEME: &str = "pystr:";
#[cfg(feature = "python")]
const PYTHON_TOK_SCHEME: &str = "pytok:";
/// Prefix for Hugging Face model repository
const HF_SCHEME: &str = "hf://";
pub enum EngineConfig {
/// An remote networked engine we don't know about yet
Dynamic(Endpoint),
......@@ -91,12 +94,16 @@ pub async fn run(
) -> anyhow::Result<()> {
let cancel_token = runtime.primary_token();
// Turn relative paths into absolute paths
// Turn relative paths into absolute paths and canonicalize them
let mut model_path = flags
.model_path_pos
.clone()
.or(flags.model_path_flag.clone())
.and_then(|p| {
// Check for hf:// prefix first
if let Some(hf_path) = p.to_string_lossy().strip_prefix(HF_SCHEME) {
return Some(PathBuf::from(hf_path));
}
if p.exists() {
p.canonicalize().ok()
} else {
......@@ -124,7 +131,7 @@ pub async fn run(
// If it's an HF repo download it
if let Some(inner_model_path) = model_path.as_ref() {
if !inner_model_path.exists() {
if !inner_model_path.exists() && !inner_model_path.is_absolute() {
model_name = Some(inner_model_path.display().to_string());
model_path = Some(hub::from_hf(inner_model_path).await?);
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment