chore: add a ci to sync hf repos to ms (#428)

* add a script to sync hf to ms

* add a script to sync hf to ms

.github/workflows/sync-hf-ms.yaml

+name: Auto Sync HuggingFace Repos to ModelScope
+on:
+  schedule:
+    - cron: '0 3 * * *' # Run every day at 03:00 UTC
+  workflow_dispatch: # Allow manual trigger from GitHub UI
+jobs:
+  sync:
+    runs-on: ubuntu-latest
+    if: github.repository == 'mit-han-lab/nunchaku'
+    strategy:
+      matrix:
+        repo:
+          - nunchaku
+          - nunchaku-t5
+          - nunchaku-flux.1-schnell
+          - nunchaku-flux.1-dev
+          - nunchaku-flux.1-fill-dev
+          - nunchaku-flux.1-depth-dev
+          - nunchaku-flux.1-canny-dev
+          - nunchaku-flux.1-shuttle-jaguar
+          - nunchaku-sana
+          - svdq-fp4-flux.1-schnell
+          - svdq-int4-flux.1-schnell
+          - svdq-fp4-flux.1-dev
+          - svdq-int4-flux.1-dev
+          - svdq-fp4-flux.1-fill-dev
+          - svdq-int4-flux.1-fill-dev
+          - svdq-fp4-flux.1-depth-dev
+          - svdq-int4-flux.1-depth-dev
+          - svdq-fp4-flux.1-canny-dev
+          - svdq-int4-flux.1-canny-dev
+          - svdq-fp4-flux.1-shuttle-jaguar
+          - svdq-int4-flux.1-shuttle-jaguar
+          - svdq-int4-sana-1600m
+          - svdq-flux.1-schnell-pix2pix-turbo
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install huggingface_hub modelscope
+      - name: Sync HF Repo ${{ matrix.repo }}
+        run: |
+          MODELSCOPE_TOKEN=${{ secrets.MODELSCOPE_TOKEN }} python sync_hf_to_ms.py --repo ${{ matrix.repo }}

scripts/sync_hf_to_ms.py

+import argparse
+import os
+
+from huggingface_hub import snapshot_download
+from modelscope.hub.api import HubApi
+from modelscope.hub.constants import Licenses, ModelVisibility
+
+
+def sync_model(hf_repo, ms_repo):
+    print(f"\n🔄 Syncing {hf_repo} -> {ms_repo}")
+
+    # Login to ModelScope
+    MODELSCOPE_TOKEN = os.getenv("MODELSCOPE_TOKEN")
+    assert MODELSCOPE_TOKEN, "Please set the MODELSCOPE_TOKEN environment variable or hardcode the token."
+
+    api = HubApi()
+    api.login(MODELSCOPE_TOKEN)
+
+    # Download the model snapshot from Hugging Face
+    local_dir = snapshot_download(repo_id=hf_repo)
+    print(f"📥 Downloaded to: {local_dir}")
+
+    # Check if the ModelScope repo already exists
+    exists = False
+    try:
+        api.get_model(ms_repo)
+        exists = True
+        print(f"✅ Model already exists on ModelScope: {ms_repo}")
+    except Exception:
+        print(f"ℹ️ Model not found on ModelScope: {ms_repo}, creating...")
+
+    # Create repo if it doesn't exist
+    if not exists:
+        api.create_model(
+            model_id=ms_repo,
+            visibility=ModelVisibility.PUBLIC,  # Change to "Private" if needed
+            license=Licenses.APACHE_V2,
+        )
+        print(f"✅ Created ModelScope repo: {ms_repo}")
+
+    # Upload model files to ModelScope
+    print(f"⏫ Uploading to ModelScope...")
+    # api.upload_model(model_dir=local_dir, model_id=ms_repo)
+    api.upload_folder(
+        repo_id=ms_repo,
+        folder_path=local_dir,
+        commit_message=f"Sync from Hugging Face {hf_repo}",
+    )
+    print(f"✅ Sync complete: {hf_repo} -> {ms_repo}")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-n",
+        "--repo-name",
+        type=str,
+        required=True,
+        help="Name of the HuggingFace repository under mit-han-lab to sync to (e.g., `nunchaku`)",
+    )
+    args = parser.parse_args()
+    sync_model(f"mit-han-lab/{args.repo_name}", f"Lmxyy1999/{args.repo_name}")