openai_completions.py 9.59 KB
Newer Older
Jason Phang's avatar
gpt3  
Jason Phang committed
1
import os
Baber Abbasi's avatar
Baber Abbasi committed
2
from functools import cached_property
3
from operator import itemgetter
Baber Abbasi's avatar
Baber Abbasi committed
4
from typing import Any, Dict, List, Optional, Tuple, Union
5

6
from lm_eval.api.registry import register_model
Baber Abbasi's avatar
Baber Abbasi committed
7
from lm_eval.models.api_models import TemplateAPI
8
from lm_eval.models.utils import handle_stop_sequences
9
from lm_eval.utils import eval_logger
Leo Gao's avatar
Leo Gao committed
10

lintangsutawika's avatar
update  
lintangsutawika committed
11

Baber Abbasi's avatar
Baber Abbasi committed
12
13
@register_model("local-completions")
class LocalCompletionsAPI(TemplateAPI):
lintangsutawika's avatar
lintangsutawika committed
14
15
    def __init__(
        self,
Baber Abbasi's avatar
Baber Abbasi committed
16
17
18
19
20
21
22
        base_url=None,
        tokenizer_backend="huggingface",
        **kwargs,
    ):
        super().__init__(
            base_url=base_url, tokenizer_backend=tokenizer_backend, **kwargs
        )
lintangsutawika's avatar
lintangsutawika committed
23

Baber Abbasi's avatar
Baber Abbasi committed
24
25
26
27
28
    def _create_payload(
        self,
        messages: Union[List[List[int]], List[dict], List[str], str],
        generate=False,
        gen_kwargs: Optional[dict] = None,
29
        seed: int = 1234,
30
        eos=None,
Baber Abbasi's avatar
Baber Abbasi committed
31
32
33
34
        **kwargs,
    ) -> dict:
        if generate:
            gen_kwargs.pop("do_sample", False)
35
36
37
38
            if "max_tokens" in gen_kwargs:
                max_tokens = gen_kwargs.pop("max_tokens")
            else:
                max_tokens = gen_kwargs.pop("max_gen_toks", self._max_gen_toks)
Baber Abbasi's avatar
Baber Abbasi committed
39
            temperature = gen_kwargs.pop("temperature", 0)
40
            stop = handle_stop_sequences(gen_kwargs.pop("until", None), eos)
Baber Abbasi's avatar
Baber Abbasi committed
41
42
43
44
45
46
            return {
                "prompt": messages,
                "model": self.model,
                "max_tokens": max_tokens,
                "temperature": temperature,
                "stop": stop,
47
                "seed": seed,
Baber Abbasi's avatar
Baber Abbasi committed
48
49
                **gen_kwargs,
            }
Baber Abbasi's avatar
Baber Abbasi committed
50
        else:
Baber Abbasi's avatar
Baber Abbasi committed
51
52
53
            return {
                "model": self.model,
                "prompt": messages,
54
                "temperature": 0,
Baber Abbasi's avatar
Baber Abbasi committed
55
56
                "max_tokens": 1,
                "logprobs": 1,
57
                "seed": seed,
Baber Abbasi's avatar
Baber Abbasi committed
58
59
60
61
62
63
64
65
66
                "echo": True,
            }

    @staticmethod
    def parse_logprobs(
        outputs: Union[Dict, List[Dict]],
        tokens: List[List[int]] = None,
        ctxlens: List[int] = None,
        **kwargs,
lintangsutawika's avatar
lintangsutawika committed
67
68
    ) -> List[Tuple[float, bool]]:
        res = []
Baber Abbasi's avatar
Baber Abbasi committed
69
70
71
        if not isinstance(outputs, list):
            outputs = [outputs]
        for out in outputs:
72
73
74
            for choice, ctxlen in zip(
                sorted(out["choices"], key=itemgetter("index")), ctxlens
            ):
Baber Abbasi's avatar
Baber Abbasi committed
75
76
                assert ctxlen > 0, "Context length must be greater than 0"
                logprobs = sum(choice["logprobs"]["token_logprobs"][ctxlen:-1])
77
                tokens_logprobs = choice["logprobs"]["token_logprobs"][ctxlen:-1]
Baber Abbasi's avatar
Baber Abbasi committed
78
79
                top_logprobs = choice["logprobs"]["top_logprobs"][ctxlen:-1]
                is_greedy = True
80
81
                for tok, top in zip(tokens_logprobs, top_logprobs):
                    if tok != max(top.values()):
Baber Abbasi's avatar
Baber Abbasi committed
82
83
84
85
86
87
88
                        is_greedy = False
                        break
                res.append((logprobs, is_greedy))
        return res

    @staticmethod
    def parse_generations(outputs: Union[Dict, List[Dict]], **kwargs) -> List[str]:
lintangsutawika's avatar
lintangsutawika committed
89
        res = []
Baber Abbasi's avatar
Baber Abbasi committed
90
91
92
        if not isinstance(outputs, list):
            outputs = [outputs]
        for out in outputs:
93
            tmp = [None] * len(out["choices"])
Baber Abbasi's avatar
Baber Abbasi committed
94
            for choices in out["choices"]:
95
96
                tmp[choices["index"]] = choices["text"]
            res = res + tmp
Baber Abbasi's avatar
Baber Abbasi committed
97
        return res
lintangsutawika's avatar
lintangsutawika committed
98

Baber Abbasi's avatar
Baber Abbasi committed
99
100
101
    @property
    def api_key(self):
        return os.environ.get("OPENAI_API_KEY", "")
lintangsutawika's avatar
lintangsutawika committed
102
103


Baber Abbasi's avatar
Baber Abbasi committed
104
105
106
107
108
109
110
111
112
@register_model("local-chat-completions")
class LocalChatCompletion(LocalCompletionsAPI):
    def __init__(
        self,
        base_url=None,
        tokenizer_backend=None,
        tokenized_requests=False,
        **kwargs,
    ):
113
114
115
        eval_logger.warning(
            "chat-completions endpoint requires the `--apply_chat_template` flag."
        )
Baber Abbasi's avatar
Baber Abbasi committed
116
117
118
119
120
121
122
123
124
        super().__init__(
            base_url=base_url,
            tokenizer_backend=tokenizer_backend,
            tokenized_requests=tokenized_requests,
            **kwargs,
        )
        if self._batch_size > 1:
            eval_logger.warning(
                "Chat completions does not support batching. Defaulting to batch size 1."
lintangsutawika's avatar
lintangsutawika committed
125
            )
Baber Abbasi's avatar
Baber Abbasi committed
126
127
128
            self._batch_size = 1

    def _create_payload(
129
130
131
132
133
        self,
        messages: List[Dict],
        generate=False,
        gen_kwargs: dict = None,
        seed=1234,
134
        eos=None,
135
        **kwargs,
Baber Abbasi's avatar
Baber Abbasi committed
136
    ) -> dict:
137
138
139
        assert (
            type(messages) is not str
        ), "chat-completions require the --apply_chat_template flag."
Baber Abbasi's avatar
Baber Abbasi committed
140
        gen_kwargs.pop("do_sample", False)
141
142
143
144
        if "max_tokens" in gen_kwargs:
            max_tokens = gen_kwargs.pop("max_tokens")
        else:
            max_tokens = gen_kwargs.pop("max_gen_toks", self._max_gen_toks)
Baber Abbasi's avatar
Baber Abbasi committed
145
        temperature = gen_kwargs.pop("temperature", 0)
146
        stop = handle_stop_sequences(gen_kwargs.pop("until", None), eos)
Baber Abbasi's avatar
Baber Abbasi committed
147
148
149
150
151
152
153
154
        if not isinstance(stop, (list, tuple)):
            stop = [stop]
        return {
            "messages": messages,
            "model": self.model,
            "max_tokens": max_tokens,
            "temperature": temperature,
            "stop": stop[:4],
155
            "seed": seed,
Baber Abbasi's avatar
Baber Abbasi committed
156
157
158
159
160
161
162
163
164
            **gen_kwargs,
        }

    @staticmethod
    def parse_generations(outputs: Union[Dict, List[Dict]], **kwargs) -> List[str]:
        res = []
        if not isinstance(outputs, list):
            outputs = [outputs]
        for out in outputs:
165
            tmp = [None] * len(out["choices"])
Baber Abbasi's avatar
Baber Abbasi committed
166
            for choices in out["choices"]:
167
168
                tmp[choices["index"]] = choices["message"]["content"]
            res = res + tmp
Baber Abbasi's avatar
Baber Abbasi committed
169
170
171
172
173
174
175
176
177
178
        return res

    def tok_encode(
        self,
        string: Union[str, Any],
        left_truncate_len=None,
        add_special_tokens=None,
        **kwargs,
    ) -> Union[List[str], List[int], Any]:
        return string
lintangsutawika's avatar
lintangsutawika committed
179

Baber Abbasi's avatar
Baber Abbasi committed
180
    def loglikelihood(self, requests, **kwargs):
Baber Abbasi's avatar
Baber Abbasi committed
181
182
183
        raise NotImplementedError(
            "Loglikelihood is not supported for chat completions. Consider using the completions API instead."
        )
lintangsutawika's avatar
lintangsutawika committed
184
185


Baber Abbasi's avatar
Baber Abbasi committed
186
187
188
189
@register_model(
    "openai-completions",
)
class OpenAICompletionsAPI(LocalCompletionsAPI):
190
    def __init__(
191
        self,
Baber Abbasi's avatar
Baber Abbasi committed
192
193
        base_url="https://api.openai.com/v1/completions",
        tokenizer_backend="tiktoken",
194
        **kwargs,
Baber Abbasi's avatar
Baber Abbasi committed
195
196
197
198
    ):
        super().__init__(
            base_url=base_url, tokenizer_backend=tokenizer_backend, **kwargs
        )
199

Baber Abbasi's avatar
Baber Abbasi committed
200
201
202
203
204
205
    @cached_property
    def api_key(self):
        """Override this property to return the API key for the API request."""
        key = os.environ.get("OPENAI_API_KEY", None)
        if key is None:
            raise ValueError(
206
                "API key not found. Please set the `OPENAI_API_KEY` environment variable."
207
            )
Baber Abbasi's avatar
Baber Abbasi committed
208
        return key
209

Baber Abbasi's avatar
Baber Abbasi committed
210
    def loglikelihood(self, requests, **kwargs):
Baber Abbasi's avatar
Baber Abbasi committed
211
        assert (
212
213
214
215
216
217
            self.model
            in [
                "babbage-002",
                "davinci-002",
            ]
        ), f"Prompt loglikelihoods are only supported by OpenAI's API for {['babbage-002', 'davinci-002']}."
Baber Abbasi's avatar
Baber Abbasi committed
218
        return super().loglikelihood(requests, **kwargs)
219

220
221
222
    def chat_template(self, chat_template: Union[bool, str] = False) -> Optional[str]:
        return ""

223

Baber Abbasi's avatar
Baber Abbasi committed
224
@register_model("openai-chat-completions")
Baber Abbasi's avatar
Baber Abbasi committed
225
226
227
228
229
230
231
232
class OpenAIChatCompletion(LocalChatCompletion):
    def __init__(
        self,
        base_url="https://api.openai.com/v1/chat/completions",
        tokenizer_backend=None,
        tokenized_requests=False,
        **kwargs,
    ):
233
234
235
236
        if "o1" in kwargs.get("model", ""):
            eval_logger.warning(
                "o1 models do not support `stop` and only support temperature=1"
            )
Baber Abbasi's avatar
Baber Abbasi committed
237
238
239
240
241
242
        super().__init__(
            base_url=base_url,
            tokenizer_backend=tokenizer_backend,
            tokenized_requests=tokenized_requests,
            **kwargs,
        )
243

Baber Abbasi's avatar
Baber Abbasi committed
244
245
246
247
248
249
    @cached_property
    def api_key(self):
        """Override this property to return the API key for the API request."""
        key = os.environ.get("OPENAI_API_KEY", None)
        if key is None:
            raise ValueError(
250
                "API key not found. Please set the `OPENAI_API_KEY` environment variable."
251
            )
Baber Abbasi's avatar
Baber Abbasi committed
252
        return key
253
254
255
256
257

    def loglikelihood(self, requests, **kwargs):
        raise NotImplementedError(
            "Loglikelihood (and therefore `multiple_choice`-type tasks) is not supported for chat completions as OpenAI does not provide prompt logprobs. See https://github.com/EleutherAI/lm-evaluation-harness/issues/942#issuecomment-1777836312 or https://github.com/EleutherAI/lm-evaluation-harness/issues/1196 for more background on this limitation."
        )
258
259
260
261
262
263
264

    def _create_payload(
        self,
        messages: List[Dict],
        generate=False,
        gen_kwargs: dict = None,
        seed=1234,
265
        eos="<|endoftext|>",
266
267
268
269
270
271
272
273
274
275
276
        **kwargs,
    ) -> dict:
        assert (
            type(messages) is not str
        ), "chat-completions require the --apply_chat_template flag."
        gen_kwargs.pop("do_sample", False)
        if "max_tokens" in gen_kwargs:
            max_tokens = gen_kwargs.pop("max_tokens")
        else:
            max_tokens = gen_kwargs.pop("max_gen_toks", self._max_gen_toks)
        temperature = gen_kwargs.pop("temperature", 0)
277
        stop = handle_stop_sequences(gen_kwargs.pop("until", ["<|endoftext|>"]), eos)
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
        if not isinstance(stop, (list, tuple)):
            stop = [stop]
        output = {
            "messages": messages,
            "model": self.model,
            "max_completion_tokens": max_tokens,
            "temperature": temperature,
            "stop": stop[:4],
            "seed": seed,
            **gen_kwargs,
        }
        if "o1" in self.model:
            output.pop("stop")
            output["temperature"] = 1
        return output