filesystem_resolver.py 1.94 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
5
6
7
8
9
10
11
12
13
14
15
import json
import os
from typing import Optional

import vllm.envs as envs
from vllm.lora.request import LoRARequest
from vllm.lora.resolver import LoRAResolver, LoRAResolverRegistry


class FilesystemResolver(LoRAResolver):
    def __init__(self, lora_cache_dir: str):
        self.lora_cache_dir = lora_cache_dir

16
17
18
    async def resolve_lora(
        self, base_model_name: str, lora_name: str
    ) -> Optional[LoRARequest]:
19
20
        lora_path = os.path.join(self.lora_cache_dir, lora_name)
        if os.path.exists(lora_path):
21
22
23
            adapter_config_path = os.path.join(
                self.lora_cache_dir, lora_name, "adapter_config.json"
            )
24
25
26
            if os.path.exists(adapter_config_path):
                with open(adapter_config_path) as file:
                    adapter_config = json.load(file)
27
28
29
30
31
32
33
34
35
                if (
                    adapter_config["peft_type"] == "LORA"
                    and adapter_config["base_model_name_or_path"] == base_model_name
                ):
                    lora_request = LoRARequest(
                        lora_name=lora_name,
                        lora_int_id=abs(hash(lora_name)),
                        lora_path=lora_path,
                    )
36
37
38
39
40
41
42
43
44
                    return lora_request
        return None


def register_filesystem_resolver():
    """Register the filesystem LoRA Resolver with vLLM"""

    lora_cache_dir = envs.VLLM_LORA_RESOLVER_CACHE_DIR
    if lora_cache_dir:
45
        if not os.path.exists(lora_cache_dir) or not os.path.isdir(lora_cache_dir):
46
47
            raise ValueError(
                "VLLM_LORA_RESOLVER_CACHE_DIR must be set to a valid directory \
48
49
                for Filesystem Resolver plugin to function"
            )
50
        fs_resolver = FilesystemResolver(lora_cache_dir)
51
        LoRAResolverRegistry.register_resolver("Filesystem Resolver", fs_resolver)
52
53

    return