chore: fix the hf2ms ci (#429)

* chore: fix the script paths

* fix the repo dir

* install hf_xet

* fix the paths

* fix the repo names

* add ignored patterns

* update again

* upload git attribute

.github/workflows/sync-hf-to-ms.yaml

@@ -17,7 +17,7 @@ jobs:
           - nunchaku-flux.1-fill-dev
           - nunchaku-flux.1-depth-dev
           - nunchaku-flux.1-canny-dev
-          - nunchaku-flux.1-shuttle-jaguar
+          - nunchaku-shuttle-jaguar
           - nunchaku-sana
           - svdq-fp4-flux.1-schnell
           - svdq-int4-flux.1-schnell
@@ -29,8 +29,8 @@ jobs:
           - svdq-int4-flux.1-depth-dev
           - svdq-fp4-flux.1-canny-dev
           - svdq-int4-flux.1-canny-dev
-          - svdq-fp4-flux.1-shuttle-jaguar
-          - svdq-int4-flux.1-shuttle-jaguar
+          - svdq-fp4-shuttle-jaguar
+          - svdq-int4-shuttle-jaguar
           - svdq-int4-sana-1600m
           - svdq-flux.1-schnell-pix2pix-turbo
     steps:
@@ -43,7 +43,8 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install huggingface_hub modelscope
+          pip install huggingface_hub modelscope hf_xet
       - name: Sync HF Repo ${{ matrix.repo }}
         run: |
-          MODELSCOPE_TOKEN=${{ secrets.MODELSCOPE_TOKEN }} python sync_hf_to_ms.py --repo ${{ matrix.repo }}
+          cd scripts
+          HF_TOKEN=${{ secrets.HF_TOKEN }} MODELSCOPE_TOKEN=${{ secrets.MODELSCOPE_TOKEN }} python sync_hf_to_ms.py --repo ${{ matrix.repo }}

scripts/sync_hf_to_ms.py

@@ -6,7 +6,7 @@ from modelscope.hub.api import HubApi
 from modelscope.hub.constants import Licenses, ModelVisibility
 
 
-def sync_model(hf_repo, ms_repo):
+def sync_model(repo_name: str, hf_repo: str, ms_repo: str):
     print(f"\n🔄 Syncing {hf_repo} -> {ms_repo}")
 
     # Login to ModelScope
@@ -17,7 +17,7 @@ def sync_model(hf_repo, ms_repo):
     api.login(MODELSCOPE_TOKEN)
 
     # Download the model snapshot from Hugging Face
-    local_dir = snapshot_download(repo_id=hf_repo)
+    local_dir = snapshot_download(repo_id=hf_repo, cache_dir=repo_name, local_dir=repo_name)
     print(f"📥 Downloaded to: {local_dir}")
 
     # Check if the ModelScope repo already exists
@@ -45,6 +45,7 @@ def sync_model(hf_repo, ms_repo):
         repo_id=ms_repo,
         folder_path=local_dir,
         commit_message=f"Sync from Hugging Face {hf_repo}",
+        ignore_patterns=["*mit-han-lab*"],
     )
     print(f"✅ Sync complete: {hf_repo} -> {ms_repo}")
 
@@ -59,4 +60,4 @@ if __name__ == "__main__":
         help="Name of the HuggingFace repository under mit-han-lab to sync to (e.g., `nunchaku`)",
     )
     args = parser.parse_args()
-    sync_model(f"mit-han-lab/{args.repo_name}", f"Lmxyy1999/{args.repo_name}")
+    sync_model(args.repo_name, f"mit-han-lab/{args.repo_name}", f"Lmxyy1999/{args.repo_name}")