"example/ck_tile/18_flexattn/utils.hpp" did not exist on "8107ee627035d927e0bdc7a4d857c6199e74f964"
accessory_mixtral_8x7b.py 1.22 KB
Newer Older
1
2
3
4
5
6
7
8
from opencompass.models import LLaMA2AccessoryModel

# Please follow the LLaMA2-Accessory installation document
# https://llama2-accessory.readthedocs.io/en/latest/install.html
# to install LLaMA2-Accessory

models = [
    dict(
9
        abbr='Accessory_mixtral_8x7b',
10
11
12
13
14
15
16
17
        type=LLaMA2AccessoryModel,

        # additional_stop_symbols=["###"],  # for models tuned with chat template  # noqa
        additional_stop_symbols=[],

        # <begin> kwargs for accessory.MetaModel.from_pretrained
        # download from https://huggingface.co/Alpha-VLLM/MoE-Mixtral-7B-8Expert/tree/main/converted_sparse  # noqa
        # see https://llama2-accessory.readthedocs.io/en/latest/projects/mixtral-8x7b.html for more details  # noqa
18
        pretrained_path='path/to/MoE-Mixtral-7B-8Expert/converted_sparse',
19
20
21
22
23
24
25
26
27
28
29
30
31
        llama_type=None,  # None for automatic probe from pretrained_path
        llama_config=None,  # None for automatic probe from pretrained_path
        tokenizer_path=None,  # None for automatic probe from pretrained_path
        with_visual=False,
        max_seq_len=4096,
        quant=False,
        # <end>

        batch_size=2,
        # LLaMA2-Accessory needs num_gpus==num_procs
        run_cfg=dict(num_gpus=2, num_procs=2),
    ),
]