__main__.py 9.55 KB
Newer Older
1
2
3
import argparse
import json
import logging
lintangsutawika's avatar
lintangsutawika committed
4
import os
lintangsutawika's avatar
lintangsutawika committed
5
import re
6
import sys
7
from pathlib import Path
haileyschoelkopf's avatar
haileyschoelkopf committed
8
from typing import Union
Leo Gao's avatar
Leo Gao committed
9

10
11
import numpy as np

12
from lm_eval import evaluator, utils
lintangsutawika's avatar
lintangsutawika committed
13
from lm_eval.tasks import initialize_tasks
14
from lm_eval.utils import make_table
lintangsutawika's avatar
format  
lintangsutawika committed
15

16

17
def _handle_non_serializable(o):
18
    if isinstance(o, np.int64) or isinstance(o, np.int32):
19
20
21
        return int(o)
    elif isinstance(o, set):
        return list(o)
22
23
    else:
        return str(o)
Fabrizio Milo's avatar
Fabrizio Milo committed
24

25

haileyschoelkopf's avatar
haileyschoelkopf committed
26
def parse_eval_args() -> argparse.Namespace:
lintangsutawika's avatar
lintangsutawika committed
27
    parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter)
Baber Abbasi's avatar
Baber Abbasi committed
28
    parser.add_argument("--model", "-m", default="hf", help="Name of model e.g. `hf`")
lintangsutawika's avatar
lintangsutawika committed
29
30
    parser.add_argument(
        "--tasks",
Baber Abbasi's avatar
Baber Abbasi committed
31
        "-t",
lintangsutawika's avatar
lintangsutawika committed
32
        default=None,
33
        metavar="task1,task2",
lintangsutawika's avatar
lintangsutawika committed
34
        help="To get full list of tasks, use the command lm-eval --tasks list",
lintangsutawika's avatar
lintangsutawika committed
35
    )
36
37
    parser.add_argument(
        "--model_args",
Baber Abbasi's avatar
Baber Abbasi committed
38
        "-a",
39
        default="",
40
        help="Comma separated string arguments for model, e.g. `pretrained=EleutherAI/pythia-160m,dtype=float32`",
41
    )
lintangsutawika's avatar
lintangsutawika committed
42
    parser.add_argument(
43
        "--num_fewshot",
Baber Abbasi's avatar
Baber Abbasi committed
44
        "-f",
45
        type=int,
46
        default=None,
47
        metavar="N",
48
49
        help="Number of examples in few-shot context",
    )
50
51
    parser.add_argument(
        "--batch_size",
Baber Abbasi's avatar
Baber Abbasi committed
52
        "-b",
53
54
55
56
57
        type=str,
        default=1,
        metavar="auto|auto:N|N",
        help="Acceptable values are 'auto', 'auto:N' or N, where N is an integer. Default 1.",
    )
lintangsutawika's avatar
lintangsutawika committed
58
59
60
61
    parser.add_argument(
        "--max_batch_size",
        type=int,
        default=None,
62
63
        metavar="N",
        help="Maximal batch size to try with --batch_size auto.",
lintangsutawika's avatar
lintangsutawika committed
64
    )
65
66
67
68
    parser.add_argument(
        "--device",
        type=str,
        default=None,
69
        help="Device to use (e.g. cuda, cuda:0, cpu).",
70
71
72
    )
    parser.add_argument(
        "--output_path",
Baber Abbasi's avatar
Baber Abbasi committed
73
        "-o",
74
75
        default=None,
        type=str,
76
        metavar="DIR|DIR/file.json",
77
        help="The path to the output file where the result metrics will be saved. If the path is a directory and log_samples is true, the results will be saved in the directory. Else the parent directory will be used.",
78
    )
lintangsutawika's avatar
lintangsutawika committed
79
80
    parser.add_argument(
        "--limit",
Baber Abbasi's avatar
Baber Abbasi committed
81
        "-L",
lintangsutawika's avatar
lintangsutawika committed
82
83
        type=float,
        default=None,
84
        metavar="N|0<N<1",
lintangsutawika's avatar
lintangsutawika committed
85
86
87
        help="Limit the number of examples per task. "
        "If <1, limit is a percentage of the total number of examples.",
    )
88
89
    parser.add_argument(
        "--use_cache",
Baber Abbasi's avatar
Baber Abbasi committed
90
        "-c",
91
92
        type=str,
        default=None,
93
        metavar="DIR",
94
95
96
97
98
99
        help="A path to a sqlite db file for caching model responses. `None` if not caching.",
    )
    parser.add_argument("--decontamination_ngrams_path", default=None)  # TODO: not used
    parser.add_argument(
        "--check_integrity",
        action="store_true",
100
        help="Whether to run the relevant part of the test suite for the tasks.",
101
102
103
    )
    parser.add_argument(
        "--write_out",
Baber Abbasi's avatar
Baber Abbasi committed
104
        "-w",
105
106
        action="store_true",
        default=False,
107
        help="Prints the prompt for the first few documents.",
108
109
110
    )
    parser.add_argument(
        "--log_samples",
Baber Abbasi's avatar
Baber Abbasi committed
111
        "-s",
112
113
        action="store_true",
        default=False,
114
        help="If True, write out all model outputs and documents for per-sample measurement and post-hoc analysis. Use with --output_path.",
115
    )
116
117
118
119
120
121
    parser.add_argument(
        "--show_config",
        action="store_true",
        default=False,
        help="If True, shows the the full config of all tasks at the end of the evaluation.",
    )
122
123
124
125
    parser.add_argument(
        "--include_path",
        type=str,
        default=None,
126
        metavar="DIR",
127
128
        help="Additional path to include if there are external tasks to include.",
    )
129
130
    parser.add_argument(
        "--gen_kwargs",
131
        default=None,
USVSN Sai Prashanth's avatar
USVSN Sai Prashanth committed
132
133
        help=(
            "String arguments for model generation on greedy_until tasks,"
134
            " e.g. `temperature=0,top_k=0,top_p=0`."
lintangsutawika's avatar
lintangsutawika committed
135
136
137
        ),
    )
    parser.add_argument(
lintangsutawika's avatar
lintangsutawika committed
138
        "--verbosity",
Baber Abbasi's avatar
Baber Abbasi committed
139
140
        "-v",
        type=str.upper,
lintangsutawika's avatar
lintangsutawika committed
141
        default="INFO",
142
143
        metavar="CRITICAL|ERROR|WARNING|INFO|DEBUG",
        help="Controls the reported logging error level. Set to DEBUG when testing + adding new task configurations for comprehensive log output.",
144
    )
Jason Phang's avatar
Jason Phang committed
145
146
    return parser.parse_args()

Fabrizio Milo's avatar
Fabrizio Milo committed
147

haileyschoelkopf's avatar
haileyschoelkopf committed
148
149
150
151
152
def cli_evaluate(args: Union[argparse.Namespace, None] = None) -> None:
    if not args:
        # we allow for args to be passed externally, else we parse them ourselves
        args = parse_eval_args()

153
    eval_logger = utils.eval_logger
lintangsutawika's avatar
lintangsutawika committed
154
    eval_logger.setLevel(getattr(logging, f"{args.verbosity}"))
155
    eval_logger.info(f"Verbosity set to {args.verbosity}")
haileyschoelkopf's avatar
haileyschoelkopf committed
156
    os.environ["TOKENIZERS_PARALLELISM"] = "false"
Fabrizio Milo's avatar
Fabrizio Milo committed
157

lintangsutawika's avatar
lintangsutawika committed
158
159
    # initialize_tasks(args.verbosity)
    ALL_TASKS = initialize_tasks(args.verbosity, include_path=args.include_path)
Fabrizio Milo's avatar
Fabrizio Milo committed
160

Leo Gao's avatar
Leo Gao committed
161
    if args.limit:
lintangsutawika's avatar
lintangsutawika committed
162
163
164
        eval_logger.warning(
            " --limit SHOULD ONLY BE USED FOR TESTING."
            "REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT."
Fabrizio Milo's avatar
Fabrizio Milo committed
165
        )
lintangsutawika's avatar
lintangsutawika committed
166

167
    if args.tasks is None:
lintangsutawika's avatar
lintangsutawika committed
168
169
170
        eval_logger.error(
            "Need to specify task to evaluate."
        )
171
    elif args.tasks == "list":
lintangsutawika's avatar
lintangsutawika committed
172
        eval_logger.info(
173
            "Available Tasks:\n - {}".format("\n - ".join(sorted(ALL_TASKS)))
lintangsutawika's avatar
lintangsutawika committed
174
        )
Jason Phang's avatar
Jason Phang committed
175
    else:
176
177
        if os.path.isdir(args.tasks):
            import glob
178
179

            task_names = []
180
181
            yaml_path = os.path.join(args.tasks, "*.yaml")
            for yaml_file in glob.glob(yaml_path):
lintangsutawika's avatar
lintangsutawika committed
182
                config = utils.load_yaml_config(yaml_file)
183
184
                task_names.append(config)
        else:
185
            tasks_list = args.tasks.split(",")
186
            task_names = utils.pattern_match(tasks_list, ALL_TASKS)
187
188
            for task in [task for task in tasks_list if task not in task_names]:
                if os.path.isfile(task):
lintangsutawika's avatar
lintangsutawika committed
189
                    config = utils.load_yaml_config(task)
190
                    task_names.append(config)
191
192
193
194
            task_missing = [
                task
                for task in tasks_list
                if task not in task_names and "*" not in task
195
            ]  # we don't want errors if a wildcard ("*") task name was used
lintangsutawika's avatar
lintangsutawika committed
196

baberabb's avatar
baberabb committed
197
198
199
200
            if task_missing:
                missing = ", ".join(task_missing)
                eval_logger.error(
                    f"Tasks were not found: {missing}\n"
lintangsutawika's avatar
lintangsutawika committed
201
                    f"{utils.SPACING}Try `lm-eval --tasks list` for list of available tasks",
baberabb's avatar
baberabb committed
202
203
                )
                raise ValueError(
204
                    f"Tasks not found: {missing}. Try `lm-eval --tasks list` for list of available tasks, or '--verbosity DEBUG' to troubleshoot task registration issues."
baberabb's avatar
baberabb committed
205
                )
lintangsutawika's avatar
lintangsutawika committed
206

207
208
    if args.output_path:
        path = Path(args.output_path)
Lintang Sutawika's avatar
Lintang Sutawika committed
209
        # check if file or 'dir/results.json' exists
baberabb's avatar
baberabb committed
210
        if path.is_file() or Path(args.output_path).joinpath("results.json").is_file():
211
212
213
            eval_logger.warning(
                f"File already exists at {path}. Results will be overwritten."
            )
lintangsutawika's avatar
lintangsutawika committed
214
            output_path_file = path.joinpath("results.json")
215
216
217
218
219
220
221
222
223
            assert not path.is_file(), "File already exists"
        # if path json then get parent dir
        elif path.suffix in (".json", ".jsonl"):
            output_path_file = path
            path.parent.mkdir(parents=True, exist_ok=True)
            path = path.parent
        else:
            path.mkdir(parents=True, exist_ok=True)
            output_path_file = path.joinpath("results.json")
224
225
    elif args.log_samples and not args.output_path:
        assert args.output_path, "Specify --output_path"
226

lintangsutawika's avatar
lintangsutawika committed
227
    eval_logger.info(f"Selected Tasks: {task_names}")
228

229
230
231
232
233
234
    results = evaluator.simple_evaluate(
        model=args.model,
        model_args=args.model_args,
        tasks=task_names,
        num_fewshot=args.num_fewshot,
        batch_size=args.batch_size,
235
        max_batch_size=args.max_batch_size,
236
        device=args.device,
haileyschoelkopf's avatar
haileyschoelkopf committed
237
        use_cache=args.use_cache,
238
239
240
        limit=args.limit,
        decontamination_ngrams_path=args.decontamination_ngrams_path,
        check_integrity=args.check_integrity,
241
        write_out=args.write_out,
242
        log_samples=args.log_samples,
lintangsutawika's avatar
lintangsutawika committed
243
        gen_kwargs=args.gen_kwargs,
244
    )
245

246
    if results is not None:
247
248
        if args.log_samples:
            samples = results.pop("samples")
249
250
251
        dumped = json.dumps(
            results, indent=2, default=_handle_non_serializable, ensure_ascii=False
        )
252
253
        if args.show_config:
            print(dumped)
254

255
256
        batch_sizes = ",".join(map(str, results["config"]["batch_sizes"]))

257
        if args.output_path:
258
            output_path_file.open("w").write(dumped)
259

260
261
262
            if args.log_samples:
                for task_name, config in results["configs"].items():
                    output_name = "{}_{}".format(
lintangsutawika's avatar
lintangsutawika committed
263
                        re.sub("/|=", "__", args.model_args), task_name
lintangsutawika's avatar
lintangsutawika committed
264
                    )
265
                    filename = path.joinpath(f"{output_name}.jsonl")
266
                    samples_dumped = json.dumps(
267
268
269
270
                        samples[task_name],
                        indent=2,
                        default=_handle_non_serializable,
                        ensure_ascii=False,
271
                    )
272
                    filename.write_text(samples_dumped, encoding="utf-8")
lintangsutawika's avatar
lintangsutawika committed
273

274
        print(
275
            f"{args.model} ({args.model_args}), gen_kwargs: ({args.gen_kwargs}), limit: {args.limit}, num_fewshot: {args.num_fewshot}, "
276
            f"batch_size: {args.batch_size}{f' ({batch_sizes})' if batch_sizes else ''}"
277
        )
278
        print(make_table(results))
lintangsutawika's avatar
lintangsutawika committed
279
        if "groups" in results:
280
            print(make_table(results, "groups"))
Jason Phang's avatar
lib  
Jason Phang committed
281

282

Jason Phang's avatar
Jason Phang committed
283
if __name__ == "__main__":
haileyschoelkopf's avatar
haileyschoelkopf committed
284
    cli_evaluate()