updata to v0.9.1_stable

.dockerignore

@@ -7,6 +7,8 @@ data
 docker
 saves
 hf_cache
+ms_cache
+om_cache
 output
 .dockerignore
 .gitattributes

.env.local

 # Note: actually we do not support .env, just for reference
 # api
-API_HOST=0.0.0.0
-API_PORT=8000
+API_HOST=
+API_PORT=
 API_KEY=
-API_MODEL_NAME=gpt-3.5-turbo
+API_MODEL_NAME=
 FASTAPI_ROOT_PATH=
+MAX_CONCURRENT=
 # general
 DISABLE_VERSION_CHECK=
 FORCE_CHECK_IMPORTS=
 LLAMAFACTORY_VERBOSITY=
 USE_MODELSCOPE_HUB=
+USE_OPENMIND_HUB=
 RECORD_VRAM=
 # torchrun
 FORCE_TORCHRUN=
 MASTER_ADDR=
 MASTER_PORT=
 NNODES=
-RANK=
+NODE_RANK=
 NPROC_PER_NODE=
 # wandb
 WANDB_DISABLED=
-WANDB_PROJECT=huggingface
+WANDB_PROJECT=
 WANDB_API_KEY=
 # gradio ui
-GRADIO_SHARE=False
-GRADIO_SERVER_NAME=0.0.0.0
+GRADIO_SHARE=
+GRADIO_SERVER_NAME=
 GRADIO_SERVER_PORT=
 GRADIO_ROOT_PATH=
+GRADIO_IPV6=
 # setup
 ENABLE_SHORT_CONSOLE=1
 # reserved (do not use)

.gitignore

@@ -159,9 +159,13 @@ cython_debug/
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 .idea/
 
+# vscode
+.vscode/
+
 # custom .gitignore
 ms_cache/
 hf_cache/
+om_cache/
 cache/
 config/
 saves/

.pre-commit-config.yaml

+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+    -   id: check-ast
+    -   id: check-added-large-files
+        args: ['--maxkb=25000']
+    -   id: check-merge-conflict
+    -   id: check-yaml
+    -   id: debug-statements
+    -   id: end-of-file-fixer
+    -   id: trailing-whitespace
+        args: [--markdown-linebreak-ext=md]
+    -   id: no-commit-to-branch
+        args: ['--branch', 'main']
+
+-   repo: https://github.com/asottile/pyupgrade
+    rev: v3.17.0
+    hooks:
+    -   id: pyupgrade
+        args: [--py38-plus]
+
+-   repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.6.9
+    hooks:
+    -   id: ruff
+        args: [--fix]
+    -   id: ruff-format

Makefile

-.PHONY: quality style test
+.PHONY: build commit quality style test
 
 check_dirs := scripts src tests setup.py
 
+build:
+	pip install build && python -m build
+
+commit:
+	pre-commit install
+	pre-commit run --all-files
+
 quality:
 	ruff check $(check_dirs)
 	ruff format --check $(check_dirs)
@@ -11,4 +18,4 @@ style:
 	ruff format $(check_dirs)
 
 test:
-	CUDA_VISIBLE_DEVICES= pytest tests/
+	CUDA_VISIBLE_DEVICES= WANDB_DISABLED=true pytest -vv tests/

data/alpaca_en_demo.json

@@ -4999,4 +4999,4 @@
     "input": "Time waits for no one.",
     "output": "No one can stop time from moving forward."
   }
-]
+]
\ No newline at end of file

data/alpaca_zh_demo.json

@@ -4999,4 +4999,4 @@
     "input": "",
     "output": "安第斯山脉位于南美洲，横跨七个国家，包括委内瑞拉，哥伦比亚，厄瓜多尔，秘鲁，玻利维亚，智利和阿根廷。安第斯山脉是世界上最长的山脉之一，全长约7,000千米（4,350英里），其山脉沿着南美洲西海岸蜿蜒延伸，平均海拔约为4,000米（13,000英尺）。在其南部，安第斯山脉宽度达到700千米（430英里），在其北部宽度约为500千米（310英里）。"
   }
-]
+]
\ No newline at end of file

data/belle_multiturn/belle_multiturn.py

@@ -17,9 +17,9 @@ _CITATION = """\
 }
 """
 
-_HOMEPAGE = "{}/datasets/BelleGroup/multiturn_chat_0.8M".format(_HF_ENDPOINT)
+_HOMEPAGE = f"{_HF_ENDPOINT}/datasets/BelleGroup/multiturn_chat_0.8M"
 _LICENSE = "gpl-3.0"
-_URL = "{}/datasets/BelleGroup/multiturn_chat_0.8M/resolve/main/multiturn_chat_0.8M.json".format(_HF_ENDPOINT)
+_URL = f"{_HF_ENDPOINT}/datasets/BelleGroup/multiturn_chat_0.8M/resolve/main/multiturn_chat_0.8M.json"
 
 
 class BelleMultiturn(datasets.GeneratorBasedBuilder):
@@ -38,7 +38,7 @@ class BelleMultiturn(datasets.GeneratorBasedBuilder):
         return [datasets.SplitGenerator(name=datasets.Split.TRAIN, gen_kwargs={"filepath": file_path})]
 
     def _generate_examples(self, filepath: str):
-        with open(filepath, "r", encoding="utf-8") as f:
+        with open(filepath, encoding="utf-8") as f:
             for key, row in enumerate(f):
                 data = json.loads(row)
                 conversations = []

data/c4_demo.json

@@ -899,4 +899,4 @@
   {
     "text": "Having a DisplayPort with you at your house will always keep you entertained if you love gaming and live to stream. The cables fit perfectly in a monitor, projector, laptop or tablet. There are many products in the market you can choose from, but you have to do proper research to avoid purchasing a wrong product.\nThis is the perfect cable for your laptop or desktop that is equipped with a display port. You can also use it to enjoy your movies where you can also expand your laptop for better view or when you are presenting at school. It provides you with quality and high-definition audio and videos from your laptop.\nThis cable is gold plated, and this helps to resist corrosion or resist any damage and enhance durability. This kind of plating also ensures a perfect signal transmission. This display port connector provides a secure connection where it has a release button that is always lowered before unplugging. The display port is also portable where you can carry it anywhere with you.\nThis mini display port cable can give you quality transmission of videos and audios. You can connect it to your computer or tablet to get HD videos. Has gold plated connectors, copper conductors, and foil-and-braid shielding where all these provide enough energy and convenient connection.\nThis cable measures a maximum of 10 feet and gives an easy time when connecting it to your laptop. The cable has a sleek design that gives you an easy time to carry and store the cable. It is secure and reliable where it will provide you with a good connection when you need it.\nIf you are looking for a perfect cable for video streaming and gaming, this is the perfect fit. It has a construction design that prevents the cable from cracking or any damage. It contains gold-plated connectors and multiple shielding where this prevents any damage and enhance the durability of the cable.\nThe cable provides you with free non-interfered data transmission from any time and position. Provides a secure and reliable connection where you only plug and play without struggling. You can directly connect the display port that is equipped to your desktop for high definition videos and clear audios.\nThis is a convenient cable that connects a display port cable directly to a projector or HD monitor with display port unit. This cable is portable and mobile where you can use it anywhere. It features gold-plated connectors that with copper connectors and braid shielding where they all combine to provide powerful cable performance reliable connections.\nIt transmits high definition videos and clear music from your computer to a monitor for gaming or video streaming. It provides a secure connection where it has a button that is always lowered before unplugging. This cable is 10 foot long, and this gives you an easy time when using it from any position.\nThis cable has gold-plated connectors that resist corrosion and any form of damage. The gold connectors also enhance reliable and increased connectivity. It features inner braided foil that reduces interference and increases video and audio quality. The 6 feet cable connects a display port enabled computer to an HD monitor or projector with DP port unit for audio and video streaming.\nThe cable gives you the best and quality video resolution you need. The DP has a latch that provides a secure and reliable connection with the port where the latch must be pressed down before unplugging the connector. This is perfect for extended display or mirrored displays.\nThe cable supports resolutions of up to 3840 by 2160 and high-definition videos. This display port provides you with a secure and reliable connection any time you need it. It has pure copper conductors and triple metal shielding that enhance a perfect performance.\nThis cable transmits both high definition video from your computer or tablet to an HD display. It has gold-plated and copper conductors that protect the cable from damage and improve its performance.\nHas foil and braid shielding that helps in reducing electromagnetic interference.\nIt has an ergonomic design where it has secured grips for easy plugging and unplugging. The cable measures 6 foot and supports a high video resolution. This cable provides you with a high-quality connection no matter where you are.\nThis is a perfect choice when you need quality and clear videos and pictures. The cable provides you with a high video resolution. The cable is 10 foot long to provide you with enough and secure connection from any position.\nThe cable is gold-plated for preventing corrosion and providing a secure connection. Has a release button that is always lowered before unplugging. Has a slim design where you can fold it easily for easy carrying.\nThis cable has latching gold-plated connectors for providing a secure and reliable connection. The gold plating also helps in resisting damage, and this enhances the durability of the cable. Features braided copper conductors for providing a reliable signal from source to display design with foil design for an uninterrupted connection.\nThe display port provides you with a secure connection where it has a release button that must be lowered before unplugging. It transmits HD audio, and video from you’re your computer to a monitor for video streaming and gaming. You can also connect and configure your monitor for an extended monitor display.\nThis cable is easily connected to a display port that is equipped with a desktop to an HD monitor or projector for easy streaming and gaming. The cable transmits high definition video and audio from your computer to a monitor where it also convenient for gaming and live streaming. It supports it provides you with a high video resolution.\nIt can work with various display port modes for providing you with deep color depths and sharp images. It is gold plated for increasing providing quality connection where the plating also prevents damage. The cable has an available price that will favor your pocket.\nFlexibility: Before buying any cable, you ask yourself the questions like what type of install is the cable going into? Is everything fixed? You need to find a cable that is flexible enough such that it is mobile to be used everywhere and in different scenarios. A flexible will always give you the freedom of using the cable everywhere provided the cable has a slim design for easy carrying and storage.\nDurability: Buying a durable display port cable means you will keep yourself out of the shops buying new products time by time. Consider buying a cable that is plated with strong material that prevents corrosion or any form of damage. There are other cables that are made from strong materials that are strong enough to enhance the durability of the cable. You have to do better research on the product you want to buy because there are many products in the market and buying a durable cable can be a tiring job.\nCost: Consider buying a port that is very well priced and a cable that will suit your pocket. Mostly, the high-quality cable is costly than the low-quality cables. You can still buy the low priced cables where they perform their function efficiently. Buying an expensive or low costing product only depends on the strength of your pocket. Avoid buying a product that will affect your budget, but if you are financially stable, you can spend on the expensive cables.\nDesign: A good cable should always have a perfect design that will make its use simple. It should have a slim design for easy carrying and storage. A foldable cable is always portable, and this makes it best to use in any position. A cable that is molded with good design will always provide you with the best results you want.\nQuality: Consider buying a cable that is of high quality and a cable that will suit your needs very well. If you purchase a low-quality cable, you will be forced to return it to the shops or buy a new one, and this will be costly. To avoid all these problems, do proper research on the product you want to buy.\nChoosing the best Display port on the market can give you stress because the products are numerous in the market and you have to do proper research before buying any cable. You have to consider many things to avoid buying a product that will not suit your functions. Consider the products in the guide above because they are of high quality and well researched to suit your functions."
   }
-]
+]
\ No newline at end of file

data/dataset_info.json

@@ -54,7 +54,8 @@
   },
   "alpaca_en": {
     "hf_hub_url": "llamafactory/alpaca_en",
-    "ms_hub_url": "llamafactory/alpaca_en"
+    "ms_hub_url": "llamafactory/alpaca_en",
+    "om_hub_url": "HaM/alpaca_en"
   },
   "alpaca_zh": {
     "hf_hub_url": "llamafactory/alpaca_zh",
@@ -66,7 +67,8 @@
   },
   "alpaca_gpt4_zh": {
     "hf_hub_url": "llamafactory/alpaca_gpt4_zh",
-    "ms_hub_url": "llamafactory/alpaca_gpt4_zh"
+    "ms_hub_url": "llamafactory/alpaca_gpt4_zh",
+    "om_hub_url": "State_Cloud/alpaca-gpt4-data-zh"
   },
   "glaive_toolcall_en": {
     "hf_hub_url": "llamafactory/glaive_toolcall_en",
@@ -623,4 +625,4 @@
     },
     "folder": "python"
   }
-}
+}
\ No newline at end of file

data/dpo_en_demo.json

@@ -7223,4 +7223,4 @@
       "value": "Abstraction is a principle in object-oriented programming that refers to the process of focusing on the essential features of an object or concept without emphasizing the details that are not important for its functionality. It enables programmers to create classes that represent the objects they want to work with, and specifies only the behavior of those objects, leaving out unnecessary details. Abstraction helps to make the code more maintainable, modular, and scalable. It also improves the productivity of developers by reducing the amount of code they need to write."
     }
   }
-]
+]
\ No newline at end of file

data/dpo_zh_demo.json

@@ -5055,4 +5055,4 @@
       "value": "C. 参与讨论"
     }
   }
-]
+]
\ No newline at end of file

data/glaive_toolcall_en_demo.json

@@ -9155,4 +9155,4 @@
     ],
     "tools": "[]"
   }
-]
+]
\ No newline at end of file

data/glaive_toolcall_zh_demo.json

@@ -9019,4 +9019,4 @@
     ],
     "tools": "[]"
   }
-]
+]
\ No newline at end of file

data/hh_rlhf_en/hh_rlhf_en.py

@@ -8,9 +8,9 @@ import datasets
 _HF_ENDPOINT = os.getenv("HF_ENDPOINT", "https://huggingface.co")
 _DESCRIPTION = "Human preference data about helpfulness and harmlessness."
 _CITATION = ""
-_HOMEPAGE = "{}/datasets/Anthropic/hh-rlhf".format(_HF_ENDPOINT)
+_HOMEPAGE = f"{_HF_ENDPOINT}/datasets/Anthropic/hh-rlhf"
 _LICENSE = "mit"
-_URL = "{}/datasets/Anthropic/hh-rlhf/resolve/main/".format(_HF_ENDPOINT)
+_URL = f"{_HF_ENDPOINT}/datasets/Anthropic/hh-rlhf/resolve/main/"
 _URLS = {
     "train": [
         _URL + "harmless-base/train.jsonl.gz",
@@ -53,7 +53,7 @@ class HhRlhfEn(datasets.GeneratorBasedBuilder):
     def _generate_examples(self, filepaths: List[str]):
         key = 0
         for filepath in filepaths:
-            with open(filepath, "r", encoding="utf-8") as f:
+            with open(filepath, encoding="utf-8") as f:
                 for row in f:
                     data = json.loads(row)
                     chosen = data["chosen"]

data/identity.json

@@ -454,4 +454,4 @@
     "input": "",
     "output": "抱歉，我不是 OpenAI 开发的 ChatGPT，我是 {{author}} 开发的 {{name}}，旨在为用户提供智能化的回答和帮助。"
   }
-]
+]
\ No newline at end of file

data/kto_en_demo.json

@@ -5395,4 +5395,4 @@
     ],
     "label": false
   }
-]
+]
\ No newline at end of file