Skip to content

GitLab

Commit 43f3f3f2 authored by wangkx1's avatar wangkx1
Browse files

init

parent 932303a0
Hide whitespace changes
Inline Side-by-side
Showing with 111 additions and 87 deletions
new-moon/config.yaml
View file @ 43f3f3f2
auto_delete_after_upload: false
auto_upload_after_download: true
csghub: csghub:
base_url: http://10.17.27.227:4997 base_url: http://10.17.27.227:4997
repo_type: model repo_type: model
......
new-moon/model_download_manager.py
View file @ 43f3f3f2
...@@ -211,6 +211,9 @@ def estimate_model_size(model_id): ...@@ -211,6 +211,9 @@ def estimate_model_size(model_id):
return size_str return size_str
except Exception as e: except Exception as e:
print(f"预估大小失败: {e}") print(f"预估大小失败: {e}")
# 缓存失败结果，避免重复请求不存在的模型
global_state.state["remote_cache"][cache_key] = {"size": "未知", "ts": time.time()}
global_state.save_state()
return "未知" return "未知"
def get_remote_file_count(model_id): def get_remote_file_count(model_id):
...@@ -244,6 +247,9 @@ def get_remote_file_count(model_id): ...@@ -244,6 +247,9 @@ def get_remote_file_count(model_id):
return file_count return file_count
except Exception as e: except Exception as e:
print(f"获取文件数量失败: {e}") print(f"获取文件数量失败: {e}")
# 缓存失败结果，避免重复请求不存在的模型
global_state.state["remote_cache"][cache_key] = {"count": 0, "ts": time.time()}
global_state.save_state()
return 0 return 0
def scan_local_models(): def scan_local_models():
...@@ -798,35 +804,44 @@ def get_combined_tasks_data(): ...@@ -798,35 +804,44 @@ def get_combined_tasks_data():
"model_id": task[1], "model_id": task[1],
"status": task[2], "status": task[2],
"progress": task[3], "progress": task[3],
"size": task[4], "size": task[4], # 这个已经包含了远程模型大小
"local_size": "", # 本地大小待下载完成后更新
"file_count": task[5].split("/")[-1] if task[5] else "0", "file_count": task[5].split("/")[-1] if task[5] else "0",
"message": task[6] "message": task[6]
}) })
# 添加上传任务 # 添加上传任务
for task in uploads: for task in uploads:
model_id = task[0]
# 获取远程模型大小用于比较
remote_size = estimate_model_size(model_id)
combined.append({ combined.append({
"type": "upload", "type": "upload",
"id": task[0], # 模型ID "id": model_id, # 模型ID
"model_id": task[0], "model_id": model_id,
"status": task[1], "status": task[1],
"progress": task[2], "progress": task[2],
"size": "", "size": remote_size, # 远程模型大小
"local_size": "", # 本地大小
"file_count": f"{task[3]}/{task[4]}", "file_count": f"{task[3]}/{task[4]}",
"message": task[5] "message": task[5]
}) })
# 添加本地模型 # 添加本地模型
for model in local_models: for model in local_models:
model_id = model[0]
# 获取远程模型大小用于比较
remote_size = estimate_model_size(model_id)
combined.append({ combined.append({
"type": "local", "type": "local",
"id": model[0], # 模型ID "id": model_id, # 模型ID
"model_id": model[0], "model_id": model_id,
"status": model[1], "status": model[1],
"progress": "", "progress": "",
"size": model[2], "size": remote_size, # 远程模型大小
"local_size": model[2], # 本地模型大小
"file_count": model[3], "file_count": model[3],
"message": "已上传" if model[4] == "是" else "未上传" "message": f"已上传，本地大小: {model[2]}, 远程大小: {remote_size}"
}) })
# 转换为表格数据格式 # 转换为表格数据格式
...@@ -837,7 +852,8 @@ def get_combined_tasks_data(): ...@@ -837,7 +852,8 @@ def get_combined_tasks_data():
item["type"], # 任务类型 item["type"], # 任务类型
item["status"], item["status"],
item["progress"], item["progress"],
item["size"], item["size"], # 远程大小
item["local_size"], # 本地大小
item["file_count"], item["file_count"],
item["message"] item["message"]
]) ])
...@@ -903,7 +919,9 @@ def create_interface(): ...@@ -903,7 +919,9 @@ def create_interface():
value=global_state.config.get("local", {}).get("default_model_path", ""), value=global_state.config.get("local", {}).get("default_model_path", ""),
placeholder="请输入本地模型存储目录" placeholder="请输入本地模型存储目录"
) )
set_dir_btn = gr.Button("保存目录", variant="primary") with gr.Row():
config_path_btn = gr.Button("配置路径", variant="primary")
clear_tasks_btn = gr.Button("删除之前任务列表", variant="secondary")
config_result = gr.Textbox(label="配置结果", interactive=False) config_result = gr.Textbox(label="配置结果", interactive=False)
gr.Markdown("## 高级配置") gr.Markdown("## 高级配置")
...@@ -954,8 +972,8 @@ def create_interface(): ...@@ -954,8 +972,8 @@ def create_interface():
# 综合任务管理表格 # 综合任务管理表格
combined_tasks_table = gr.Dataframe( combined_tasks_table = gr.Dataframe(
headers=["模型ID", "任务类型", "状态", "进度", "大小", "文件数", "消息"], headers=["模型ID", "任务类型", "状态", "进度", "远程大小", "本地大小", "文件数", "消息"],
datatype=["str", "str", "str", "str", "str", "str", "str"], datatype=["str", "str", "str", "str", "str", "str", "str", "str"],
value=get_combined_tasks_data(), value=get_combined_tasks_data(),
interactive=False interactive=False
) )
...@@ -997,20 +1015,54 @@ def create_interface(): ...@@ -997,20 +1015,54 @@ def create_interface():
delete_completed_tasks_btn = gr.Button("删除已完成任务", variant="secondary") delete_completed_tasks_btn = gr.Button("删除已完成任务", variant="secondary")
delete_tasks_result = gr.Textbox(label="批量操作结果", interactive=False, container=False) delete_tasks_result = gr.Textbox(label="批量操作结果", interactive=False, container=False)
# 配置保存 # 配置保存 - 仅保存路径，不清空任务
def save_config_dir(d): def save_config_path(d):
if d: if d:
# 更新本地模型目录配置
global_state.config.get("local", {}).update({"default_model_path": d}) global_state.config.get("local", {}).update({"default_model_path": d})
global_state.save_config() global_state.save_config()
return "目录已设置" return "目录已设置"
return "目录无效" return "目录无效"
set_dir_btn.click( # 清空任务列表
fn=save_config_dir, def clear_tasks():
with global_state.operation_lock:
# 清空下载任务
global_state.state["download_tasks"].clear()
# 清空上传任务
global_state.state["upload_tasks"].clear()
# 重置队列
global_state.download_queue = queue.PriorityQueue()
global_state.upload_queue = queue.Queue()
# 清空活动任务
global_state.active_downloads.clear()
global_state.active_uploads.clear()
# 终止所有正在运行的下载进程
for task_id, process in list(global_state.download_processes.items()):
try:
process.terminate()
process.join(timeout=5)
if process.is_alive():
process.kill()
except Exception as e:
print(f"终止进程失败: {e}")
global_state.download_processes.clear()
return "任务列表已清空"
# 配置路径按钮点击事件
config_path_btn.click(
fn=save_config_path,
inputs=[local_dir_input], inputs=[local_dir_input],
outputs=[config_result] outputs=[config_result]
).then(fn=refresh_all, outputs=[downloads_table, combined_tasks_table]) ).then(fn=refresh_all, outputs=[downloads_table, combined_tasks_table])
# 删除任务列表按钮点击事件
clear_tasks_btn.click(
fn=clear_tasks,
inputs=None,
outputs=[config_result]
).then(fn=refresh_all, outputs=[downloads_table, combined_tasks_table])
# 高级配置 # 高级配置
update_config_btn.click( update_config_btn.click(
fn=lambda au, ad: (global_state.config.update({ fn=lambda au, ad: (global_state.config.update({
...@@ -1438,6 +1490,8 @@ def create_interface(): ...@@ -1438,6 +1490,8 @@ def create_interface():
# 切换标签页的函数 # 切换标签页的函数
def switch_to_config_tab(): def switch_to_config_tab():
# 当切换到配置标签时，刷新本地目录输入框的值
local_dir = global_state.config.get("local", {}).get("default_model_path", "")
return ( return (
"config_tab", "config_tab",
gr.Column(visible=True), # config_tab_content gr.Column(visible=True), # config_tab_content
...@@ -1445,10 +1499,12 @@ def create_interface(): ...@@ -1445,10 +1499,12 @@ def create_interface():
gr.Column(visible=False), # manage_tab_content gr.Column(visible=False), # manage_tab_content
gr.Button(variant="primary"), # config_btn gr.Button(variant="primary"), # config_btn
gr.Button(variant="secondary"), # download_btn gr.Button(variant="secondary"), # download_btn
gr.Button(variant="secondary") # manage_btn gr.Button(variant="secondary"), # manage_btn
local_dir # 更新本地目录输入框
) )
def switch_to_download_tab(): def switch_to_download_tab():
# 保持与config_tab相同的返回值数量
return ( return (
"download_tab", "download_tab",
gr.Column(visible=False), # config_tab_content gr.Column(visible=False), # config_tab_content
...@@ -1456,10 +1512,12 @@ def create_interface(): ...@@ -1456,10 +1512,12 @@ def create_interface():
gr.Column(visible=False), # manage_tab_content gr.Column(visible=False), # manage_tab_content
gr.Button(variant="secondary"), # config_btn gr.Button(variant="secondary"), # config_btn
gr.Button(variant="primary"), # download_btn gr.Button(variant="primary"), # download_btn
gr.Button(variant="secondary") # manage_btn gr.Button(variant="secondary"), # manage_btn
global_state.config.get("local", {}).get("default_model_path", "") # 保持数量一致
) )
def switch_to_manage_tab(): def switch_to_manage_tab():
# 保持与config_tab相同的返回值数量
return ( return (
"manage_tab", "manage_tab",
gr.Column(visible=False), # config_tab_content gr.Column(visible=False), # config_tab_content
...@@ -1467,29 +1525,37 @@ def create_interface(): ...@@ -1467,29 +1525,37 @@ def create_interface():
gr.Column(visible=True), # manage_tab_content gr.Column(visible=True), # manage_tab_content
gr.Button(variant="secondary"), # config_btn gr.Button(variant="secondary"), # config_btn
gr.Button(variant="secondary"), # download_btn gr.Button(variant="secondary"), # download_btn
gr.Button(variant="primary") # manage_btn gr.Button(variant="primary"), # manage_btn
global_state.config.get("local", {}).get("default_model_path", "") # 保持数量一致
) )
# 左侧按钮点击事件 # 左侧按钮点击事件
config_btn.click( config_btn.click(
fn=switch_to_config_tab, fn=switch_to_config_tab,
outputs=[current_tab, config_tab_content, download_tab_content, manage_tab_content, config_btn, download_btn, manage_btn] outputs=[current_tab, config_tab_content, download_tab_content, manage_tab_content, config_btn, download_btn, manage_btn, local_dir_input]
) )
download_btn.click( download_btn.click(
fn=switch_to_download_tab, fn=switch_to_download_tab,
outputs=[current_tab, config_tab_content, download_tab_content, manage_tab_content, config_btn, download_btn, manage_btn] outputs=[current_tab, config_tab_content, download_tab_content, manage_tab_content, config_btn, download_btn, manage_btn, local_dir_input]
) )
manage_btn.click( manage_btn.click(
fn=switch_to_manage_tab, fn=switch_to_manage_tab,
outputs=[current_tab, config_tab_content, download_tab_content, manage_tab_content, config_btn, download_btn, manage_btn] outputs=[current_tab, config_tab_content, download_tab_content, manage_tab_content, config_btn, download_btn, manage_btn, local_dir_input]
) )
# 移除了主题切换功能，因为在Gradio 6.0中不支持将Blocks作为输出组件 # 移除了主题切换功能，因为在Gradio 6.0中不支持将Blocks作为输出组件
# 初始化完成 # 初始化完成
# 应用加载时刷新数据 # 应用加载时刷新数据和配置
app.load(fn=refresh_all, outputs=[downloads_table, combined_tasks_table]) def load_all():
# 刷新任务数据
downloads_data, combined_data = refresh_all()
# 获取最新的本地目录配置
local_dir = global_state.config.get("local", {}).get("default_model_path", "")
return downloads_data, combined_data, local_dir
app.load(fn=load_all, outputs=[downloads_table, combined_tasks_table, local_dir_input])
return app return app
......
new-moon/model_manager_config.json deleted 100644 → 0
View file @ 932303a0
{
"local_dir": "/data/DataStore/models/exp-net/new-moon/models",
"max_concurrent_downloads": 1,
"max_concurrent_uploads": 1,
"auto_upload_after_download": false,
"auto_delete_after_upload": false,
"auto_delete_after_download": false
}
\ No newline at end of file
new-moon/model_manager_state.json
View file @ 43f3f3f2
{ {
"download_tasks": { "download_tasks": {},
"dl_1776058653752_62": {
"task_id": "dl_1776058653752_62",
"model_id": "ZhipuAI/GLM-5.1",
"priority": 11,
"status": "paused",
"progress": 0,
"total_files": 283,
"completed_files": 0,
"estimated_size": "1404.21 GB",
"message": "任务已暂停",
"retry_count": 0,
"auto_upload": false,
"auto_delete": false,
"start_time": 1776063865.4479935,
"end_time": null
},
"dl_1776060948371_3712": {
"task_id": "dl_1776060948371_3712",
"model_id": "MiniMax/MiniMax-M2.7",
"priority": 12,
"status": "paused",
"progress": 0,
"total_files": 126,
"completed_files": 0,
"estimated_size": "214.36 GB",
"message": "任务已暂停",
"retry_count": 0,
"auto_upload": false,
"auto_delete": false,
"start_time": 1776065919.5642004,
"end_time": null
},
"dl_1776066712087_6690": {
"task_id": "dl_1776066712087_6690",
"model_id": "FunAudioLLM/Fun-CineForge",
"priority": 0,
"status": "paused",
"progress": 0,
"total_files": 1,
"completed_files": 0,
"estimated_size": "12.68 GB",
"message": "任务已暂停",
"retry_count": 0,
"auto_upload": false,
"auto_delete": false,
"start_time": 1776066712.2827172,
"end_time": null
}
},
"upload_tasks": {}, "upload_tasks": {},
"local_models": [], "local_models": [],
"remote_cache": { "remote_cache": {
...@@ -98,7 +49,23 @@ ...@@ -98,7 +49,23 @@
}, },
"size_Qwen/Qwen3.5-0.8B": { "size_Qwen/Qwen3.5-0.8B": {
"size": "1.65 GB", "size": "1.65 GB",
"ts": 1776067001.8658347 "ts": 1776403923.5235713
},
"size_Qwen/Qwen3___5-0___8B": {
"size": "未知",
"ts": 1776403923.3104131
},
"file_count_Qwen/Qwen3-0.6B": {
"count": 1,
"ts": 1776398267.5521998
},
"size_Qwen/Qwen3-0.6B": {
"size": "1.41 GB",
"ts": 1776403923.8460574
},
"size_Qwen/Qwen3-0___6B": {
"size": "未知",
"ts": 1776403924.0711582
} }
} }
} }
\ No newline at end of file
new-moon/start_server.sh
View file @ 43f3f3f2
...@@ -6,7 +6,4 @@ docker run -d --name model_download_manager \ ...@@ -6,7 +6,4 @@ docker run -d --name model_download_manager \
-v /data/DataStore/models:/data/DataStore/models \ -v /data/DataStore/models:/data/DataStore/models \
-p 7865:7865 -w /workspace/new-moon -e PYTHONUNBUFFERED=1 \ -p 7865:7865 -w /workspace/new-moon -e PYTHONUNBUFFERED=1 \
--restart unless-stopped 10.17.27.227/dev/man-model:v2.0 \ --restart unless-stopped 10.17.27.227/dev/man-model:v2.0 \
bash -c "python3 model_download_manager.py" bash -c "python3 model_download_manager.py"
\ No newline at end of file
# 容器启动后，即可在浏览器输入 <服务器IP:7865> 访问网站, 输入当前 之前 -v 映射到的容器内的目录, 即可将模型自动下载到服务器;
# 下载失败会自动尝试10次，直到下载成功
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment