You need to sign in or sign up before continuing.
Unverified Commit 877b2ec3 authored by Abrar Shivani's avatar Abrar Shivani Committed by GitHub
Browse files

feat: Support hf:// URLs in dynamo run (#917)

Allow `hf://` prefix on command line. 

Closes GitHub issue: https://github.com/ai-dynamo/dynamo/issues/829
parent 2be5e8f5
...@@ -15,7 +15,7 @@ ...@@ -15,7 +15,7 @@
#[cfg(any(feature = "vllm", feature = "sglang"))] #[cfg(any(feature = "vllm", feature = "sglang"))]
use std::{future::Future, pin::Pin}; use std::{future::Future, pin::Pin};
use std::{io::Read, sync::Arc}; use std::{io::Read, path::PathBuf, sync::Arc};
use dynamo_llm::{ use dynamo_llm::{
backend::ExecutionContext, engines::StreamingEngine, kv_router::publisher::KvMetricsPublisher, backend::ExecutionContext, engines::StreamingEngine, kv_router::publisher::KvMetricsPublisher,
...@@ -53,6 +53,9 @@ const PYTHON_STR_SCHEME: &str = "pystr:"; ...@@ -53,6 +53,9 @@ const PYTHON_STR_SCHEME: &str = "pystr:";
#[cfg(feature = "python")] #[cfg(feature = "python")]
const PYTHON_TOK_SCHEME: &str = "pytok:"; const PYTHON_TOK_SCHEME: &str = "pytok:";
/// Prefix for Hugging Face model repository
const HF_SCHEME: &str = "hf://";
pub enum EngineConfig { pub enum EngineConfig {
/// An remote networked engine we don't know about yet /// An remote networked engine we don't know about yet
Dynamic(Endpoint), Dynamic(Endpoint),
...@@ -91,12 +94,16 @@ pub async fn run( ...@@ -91,12 +94,16 @@ pub async fn run(
) -> anyhow::Result<()> { ) -> anyhow::Result<()> {
let cancel_token = runtime.primary_token(); let cancel_token = runtime.primary_token();
// Turn relative paths into absolute paths // Turn relative paths into absolute paths and canonicalize them
let mut model_path = flags let mut model_path = flags
.model_path_pos .model_path_pos
.clone() .clone()
.or(flags.model_path_flag.clone()) .or(flags.model_path_flag.clone())
.and_then(|p| { .and_then(|p| {
// Check for hf:// prefix first
if let Some(hf_path) = p.to_string_lossy().strip_prefix(HF_SCHEME) {
return Some(PathBuf::from(hf_path));
}
if p.exists() { if p.exists() {
p.canonicalize().ok() p.canonicalize().ok()
} else { } else {
...@@ -124,7 +131,7 @@ pub async fn run( ...@@ -124,7 +131,7 @@ pub async fn run(
// If it's an HF repo download it // If it's an HF repo download it
if let Some(inner_model_path) = model_path.as_ref() { if let Some(inner_model_path) = model_path.as_ref() {
if !inner_model_path.exists() { if !inner_model_path.exists() && !inner_model_path.is_absolute() {
model_name = Some(inner_model_path.display().to_string()); model_name = Some(inner_model_path.display().to_string());
model_path = Some(hub::from_hf(inner_model_path).await?); model_path = Some(hub::from_hf(inner_model_path).await?);
} }
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment