diff --git a/.github/workflows/new_tasks.yml b/.github/workflows/new_tasks.yml index 0df7111cfb0025f56b5f78f2dda068e9886d206b..b748aab5c06533fd3f8d41cfd519841a9af93f75 100644 --- a/.github/workflows/new_tasks.yml +++ b/.github/workflows/new_tasks.yml @@ -56,7 +56,7 @@ jobs: if: steps.changed-tasks.outputs.tasks_any_modified == 'true' || steps.changed-tasks.outputs.api_any_modified == 'true' run: | python -m pip install --upgrade pip - pip install -e '.[dev]' --extra-index-url https://download.pytorch.org/whl/cpu + pip install -e '.[dev,ifeval]' --extra-index-url https://download.pytorch.org/whl/cpu # Install optional git dependencies # pip install bleurt@https://github.com/google-research/bleurt/archive/b610120347ef22b494b6d69b4316e303f5932516.zip#egg=bleurt # if [ -f requirements.txt ]; then pip install -r requirements.txt; fi diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index f9dcaf4c95b0a76d0bc884bdc110e11cfe85f7b5..49b85fb9a4541f6c6dfecd4395a4544dc4ec5aac 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -56,7 +56,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -e '.[dev,anthropic,sentencepiece]' --extra-index-url https://download.pytorch.org/whl/cpu + pip install -e '.[dev,sentencepiece,api]' --extra-index-url https://download.pytorch.org/whl/cpu # Install optional git dependencies # pip install bleurt@https://github.com/google-research/bleurt/archive/b610120347ef22b494b6d69b4316e303f5932516.zip#egg=bleurt # if [ -f requirements.txt ]; then pip install -r requirements.txt; fi @@ -84,7 +84,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -e '.[dev,optimum,deepsparse,sparseml]' --extra-index-url https://download.pytorch.org/whl/cpu + pip install -e '.[dev,optimum,deepsparse,sparseml,api]' --extra-index-url https://download.pytorch.org/whl/cpu - name: Test with pytest run: python -m pytest tests/models --showlocals -s -vv - name: Archive artifacts diff --git a/.gitignore b/.gitignore index 020622dfdc82dd22630c522c0f9841754aefa638..56bb8038644e6e3f745cf99da87282922882c239 100644 --- a/.gitignore +++ b/.gitignore @@ -8,11 +8,13 @@ build dist *.egg-info venv +.venv/ .vscode/ temp __pycache__ .ipynb_checkpoints temp +test_logs/ # IPython profile_default/ ipython_config.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d066cd9e6b4b174a267155a2b8e558a3b946dac1..5df94412a72a6bf06c58c7e2c61626fd84b0373c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ exclude: ^tests/testdata/ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v4.6.0 hooks: - id: check-added-large-files - id: check-ast @@ -29,7 +29,7 @@ repos: - id: mixed-line-ending args: [--fix=lf] - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.8 + rev: v0.6.8 hooks: # Run the linter. - id: ruff diff --git a/CODEOWNERS b/CODEOWNERS index 9e6375466f1419af1e4c3450c5fbf7934f89b82f..874c877153c67909cdd514b48faa0c8c6f3f0329 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1 +1 @@ -* @haileyschoelkopf @lintangsutawika +* @baberabb @lintangsutawika diff --git a/README.md b/README.md index ef00b8a68aa969221e920c1a3c44c7f9efc3ee7c..c453de30c2906340606d897c5314e0068a3436b6 100644 --- a/README.md +++ b/README.md @@ -2,11 +2,22 @@ [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.10256836.svg)](https://doi.org/10.5281/zenodo.10256836) +--- + +*Latest News 📣* + +- [2024/09] We are prototyping allowing users of LM Evaluation Harness to create and evaluate on text+image multimodal input, text output tasks, and have just added the `hf-multimodal` and `vllm-vlm` model types and `mmmu` task as a prototype feature. We welcome users to try out this in-progress feature and stress-test it for themselves, and suggest they check out [`lmms-eval`](https://github.com/EvolvingLMMs-Lab/lmms-eval), a wonderful project originally forking off of the lm-evaluation-harness, for a broader range of multimodal tasks, models, and features. +- [2024/07] [API model](docs/API_guide.md) support has been updated and refactored, introducing support for batched and async requests, and making it significantly easier to customize and use for your own purposes. **To run Llama 405B, we recommend using VLLM's OpenAI-compliant API to host the model, and use the `local-completions` model type to evaluate the model.** +- [2024/07] New Open LLM Leaderboard tasks have been added ! You can find them under the [leaderboard](lm_eval/tasks/leaderboard/README.md) task group. + +--- + ## Announcement **A new v0.4.0 release of lm-evaluation-harness is available** ! New updates and features include: +- **New Open LLM Leaderboard tasks have been added ! You can find them under the [leaderboard](lm_eval/tasks/leaderboard/README.md) task group.** - Internal refactoring - Config-based task creation and configuration - Easier import and sharing of externally-defined task config YAMLs @@ -20,13 +31,15 @@ Please see our updated documentation pages in `docs/` for more details. Development will be continuing on the `main` branch, and we encourage you to give us feedback on what features are desired and how to improve the library further, or ask questions, either in issues or PRs on GitHub, or in the [EleutherAI discord](https://discord.gg/eleutherai)! +--- + ## Overview This project provides a unified framework to test generative language models on a large number of different evaluation tasks. **Features:** - Over 60 standard academic benchmarks for LLMs, with hundreds of subtasks and variants implemented. -- Support for models loaded via [transformers](https://github.com/huggingface/transformers/) (including quantization via [AutoGPTQ](https://github.com/PanQiWei/AutoGPTQ)), [GPT-NeoX](https://github.com/EleutherAI/gpt-neox), and [Megatron-DeepSpeed](https://github.com/microsoft/Megatron-DeepSpeed/), with a flexible tokenization-agnostic interface. +- Support for models loaded via [transformers](https://github.com/huggingface/transformers/) (including quantization via [GPTQModel](https://github.com/ModelCloud/GPTQModel) and [AutoGPTQ](https://github.com/PanQiWei/AutoGPTQ)), [GPT-NeoX](https://github.com/EleutherAI/gpt-neox), and [Megatron-DeepSpeed](https://github.com/microsoft/Megatron-DeepSpeed/), with a flexible tokenization-agnostic interface. - Support for fast and memory-efficient inference with [vLLM](https://github.com/vllm-project/vllm). - Support for commercial APIs including [OpenAI](https://openai.com), and [TextSynth](https://textsynth.com/). - Support for evaluation on adapters (e.g. LoRA) supported in [HuggingFace's PEFT library](https://github.com/huggingface/peft). @@ -41,7 +54,7 @@ The Language Model Evaluation Harness is the backend for 🤗 Hugging Face's pop To install the `lm-eval` package from the github repository, run: ```bash -git clone https://github.com/EleutherAI/lm-evaluation-harness +git clone --depth 1 https://github.com/EleutherAI/lm-evaluation-harness cd lm-evaluation-harness pip install -e . ``` @@ -94,7 +107,7 @@ lm_eval --model hf \ #### Multi-GPU Evaluation with Hugging Face `accelerate` -We support two main ways of using Hugging Face's [accelerate 🚀](https://github.com/huggingface/accelerate) library for multi-GPU evaluation. +We support three main ways of using Hugging Face's [accelerate 🚀](https://github.com/huggingface/accelerate) library for multi-GPU evaluation. To perform *data-parallel evaluation* (where each GPU loads a **separate full copy** of the model), we leverage the `accelerate` launcher as follows: @@ -111,7 +124,7 @@ For cases where your model can fit on a single GPU, this allows you to evaluate The second way of using `accelerate` for multi-GPU evaluation is when your model is *too large to fit on a single GPU.* -In this setting, run the library *outside of the `accelerate` launcher*, but passing `parallelize=True` to `--model_args` as follows: +In this setting, run the library *outside the `accelerate` launcher*, but passing `parallelize=True` to `--model_args` as follows: ``` lm_eval --model hf \ @@ -128,7 +141,19 @@ For more advanced users or even larger models, we allow for the following argume - `max_cpu_memory`: the max amount of CPU memory to use when offloading the model weights to RAM. - `offload_folder`: a folder where model weights will be offloaded to disk if needed. -These two options (`accelerate launch` and `parallelize=True`) are mutually exclusive. +The third option is to use both at the same time. This will allow you to take advantage of both data parallelism and model sharding, and is especially useful for models that are too large to fit on a single GPU. + +``` +accelerate launch --multi_gpu --num_processes {nb_of_copies_of_your_model} \ + -m lm_eval --model hf \ + --tasks lambada_openai,arc_easy \ + --model_args parallelize=True \ + --batch_size 16 +``` + +To learn more about model parallelism and how to use it with the `accelerate` library, see the [accelerate documentation](https://huggingface.co/docs/transformers/v4.15.0/en/parallelism) + +**Warning: We do not natively support multi-node evaluation using the `hf` model type! Please reference [our GPT-NeoX library integration](https://github.com/EleutherAI/gpt-neox/blob/main/eval.py) for an example of code in which a custom multi-machine evaluation script is written.** **Note: we do not currently support multi-node evaluations natively, and advise using either an externally hosted server to run inference requests against, or creating a custom integration with your distributed framework [as is done for the GPT-NeoX library](https://github.com/EleutherAI/gpt-neox/blob/main/eval_tasks/eval_adapter.py).** @@ -216,26 +241,26 @@ lm_eval --model openai-completions \ We also support using your own local inference server with servers that mirror the OpenAI Completions and ChatCompletions APIs. ```bash -lm_eval --model local-chat-completions --tasks gsm8k --model_args model=facebook/opt-125m,base_url=http://{yourip}:8000/v1 +lm_eval --model local-completions --tasks gsm8k --model_args model=facebook/opt-125m,base_url=http://{yourip}:8000/v1/completions,num_concurrent=1,max_retries=3,tokenized_requests=False,batch_size=16 ``` -Note that for externally hosted models, configs such as `--device` and `--batch_size` should not be used and do not function. Just like you can use `--model_args` to pass arbitrary arguments to the model constructor for local models, you can use it to pass arbitrary arguments to the model API for hosted models. See the documentation of the hosting service for information on what arguments they support. - -| API or Inference Server | Implemented? | `--model ` name | Models supported: | Request Types: | -|---------------------------------------------------------------------------------------------------------------------------|---------------------------------|---------------------------------------------------------------------|-----------------------------------------------------------------------------------------------|------------------------------------------------------------| -| OpenAI Completions | :heavy_check_mark: | `openai-completions`, `local-completions` | All OpenAI Completions API models | `generate_until`, `loglikelihood`, `loglikelihood_rolling` | -| OpenAI ChatCompletions | :heavy_check_mark: | `openai-chat-completions`, `local-chat-completions` | [All ChatCompletions API models](https://platform.openai.com/docs/guides/gpt) | `generate_until` (no logprobs) | -| Anthropic | :heavy_check_mark: | `anthropic` | [Supported Anthropic Engines](https://docs.anthropic.com/claude/reference/selecting-a-model) | `generate_until` (no logprobs) | -| Anthropic Chat | :heavy_check_mark: | `anthropic-chat`, `anthropic-chat-completions` | [Supported Anthropic Engines](https://docs.anthropic.com/claude/docs/models-overview) | `generate_until` (no logprobs) | -| Textsynth | :heavy_check_mark: | `textsynth` | [All supported engines](https://textsynth.com/documentation.html#engines) | `generate_until`, `loglikelihood`, `loglikelihood_rolling` | -| Cohere | [:hourglass: - blocked on Cohere API bug](https://github.com/EleutherAI/lm-evaluation-harness/pull/395) | N/A | [All `cohere.generate()` engines](https://docs.cohere.com/docs/models) | `generate_until`, `loglikelihood`, `loglikelihood_rolling` | -| [Llama.cpp](https://github.com/ggerganov/llama.cpp) (via [llama-cpp-python](https://github.com/abetlen/llama-cpp-python)) | :heavy_check_mark: | `gguf`, `ggml` | [All models supported by llama.cpp](https://github.com/ggerganov/llama.cpp) | `generate_until`, `loglikelihood`, (perplexity evaluation not yet implemented) | -| vLLM | :heavy_check_mark: | `vllm` | [Most HF Causal Language Models](https://docs.vllm.ai/en/latest/models/supported_models.html) | `generate_until`, `loglikelihood`, `loglikelihood_rolling` | -| Mamba | :heavy_check_mark: | `mamba_ssm` | [Mamba architecture Language Models via the `mamba_ssm` package](https://huggingface.co/state-spaces) | `generate_until`, `loglikelihood`, `loglikelihood_rolling` | -| Huggingface Optimum (Causal LMs) | ✔️ | `openvino` | Any decoder-only AutoModelForCausalLM converted with Huggingface Optimum into OpenVINO™ Intermediate Representation (IR) format | `generate_until`, `loglikelihood`, `loglikelihood_rolling` | ... | -| Neuron via AWS Inf2 (Causal LMs) | ✔️ | `neuronx` | Any decoder-only AutoModelForCausalLM supported to run on [huggingface-ami image for inferentia2](https://aws.amazon.com/marketplace/pp/prodview-gr3e6yiscria2) | `generate_until`, `loglikelihood`, `loglikelihood_rolling` | ... | -| [Neural Magic DeepSparse](https://github.com/neuralmagic/deepsparse) | ✔️ | `deepsparse` | Any LM from [SparseZoo](https://sparsezoo.neuralmagic.com/) or on [HF Hub with the "deepsparse" tag](https://huggingface.co/models?other=deepsparse) | `generate_until`, `loglikelihood` | ... | -| [Neural Magic SparseML](https://github.com/neuralmagic/sparseml) | ✔️ | `sparseml` | Any decoder-only AutoModelForCausalLM from [SparseZoo](https://sparsezoo.neuralmagic.com/) or on [HF Hub](https://huggingface.co/neuralmagic). Especially useful for models with quantization like [`zoo:llama2-7b-gsm8k_llama2_pretrain-pruned60_quantized`](https://sparsezoo.neuralmagic.com/models/llama2-7b-gsm8k_llama2_pretrain-pruned60_quantized) | `generate_until`, `loglikelihood`, `loglikelihood_rolling` | ... | -| Your local inference server! | :heavy_check_mark: | `local-completions` or `local-chat-completions` (using `openai-chat-completions` model type) | Any server address that accepts GET requests using HF models and mirror's OpenAI's Completions or ChatCompletions interface | `generate_until` | | ... | +Note that for externally hosted models, configs such as `--device` which relate to where to place a local model should not be used and do not function. Just like you can use `--model_args` to pass arbitrary arguments to the model constructor for local models, you can use it to pass arbitrary arguments to the model API for hosted models. See the documentation of the hosting service for information on what arguments they support. + +| API or Inference Server | Implemented? | `--model ` name | Models supported: | Request Types: | +|---------------------------------------------------------------------------------------------------------------------------|---------------------------------|-----------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------| +| OpenAI Completions | :heavy_check_mark: | `openai-completions`, `local-completions` | All OpenAI Completions API models | `generate_until`, `loglikelihood`, `loglikelihood_rolling` | +| OpenAI ChatCompletions | :heavy_check_mark: | `openai-chat-completions`, `local-chat-completions` | [All ChatCompletions API models](https://platform.openai.com/docs/guides/gpt) | `generate_until` (no logprobs) | +| Anthropic | :heavy_check_mark: | `anthropic` | [Supported Anthropic Engines](https://docs.anthropic.com/claude/reference/selecting-a-model) | `generate_until` (no logprobs) | +| Anthropic Chat | :heavy_check_mark: | `anthropic-chat`, `anthropic-chat-completions` | [Supported Anthropic Engines](https://docs.anthropic.com/claude/docs/models-overview) | `generate_until` (no logprobs) | +| Textsynth | :heavy_check_mark: | `textsynth` | [All supported engines](https://textsynth.com/documentation.html#engines) | `generate_until`, `loglikelihood`, `loglikelihood_rolling` | +| Cohere | [:hourglass: - blocked on Cohere API bug](https://github.com/EleutherAI/lm-evaluation-harness/pull/395) | N/A | [All `cohere.generate()` engines](https://docs.cohere.com/docs/models) | `generate_until`, `loglikelihood`, `loglikelihood_rolling` | +| [Llama.cpp](https://github.com/ggerganov/llama.cpp) (via [llama-cpp-python](https://github.com/abetlen/llama-cpp-python)) | :heavy_check_mark: | `gguf`, `ggml` | [All models supported by llama.cpp](https://github.com/ggerganov/llama.cpp) | `generate_until`, `loglikelihood`, (perplexity evaluation not yet implemented) | +| vLLM | :heavy_check_mark: | `vllm` | [Most HF Causal Language Models](https://docs.vllm.ai/en/latest/models/supported_models.html) | `generate_until`, `loglikelihood`, `loglikelihood_rolling` | +| Mamba | :heavy_check_mark: | `mamba_ssm` | [Mamba architecture Language Models via the `mamba_ssm` package](https://huggingface.co/state-spaces) | `generate_until`, `loglikelihood`, `loglikelihood_rolling` | +| Huggingface Optimum (Causal LMs) | ✔️ | `openvino` | Any decoder-only AutoModelForCausalLM converted with Huggingface Optimum into OpenVINO™ Intermediate Representation (IR) format | `generate_until`, `loglikelihood`, `loglikelihood_rolling` | ... | +| Neuron via AWS Inf2 (Causal LMs) | ✔️ | `neuronx` | Any decoder-only AutoModelForCausalLM supported to run on [huggingface-ami image for inferentia2](https://aws.amazon.com/marketplace/pp/prodview-gr3e6yiscria2) | `generate_until`, `loglikelihood`, `loglikelihood_rolling` | ... | +| [Neural Magic DeepSparse](https://github.com/neuralmagic/deepsparse) | ✔️ | `deepsparse` | Any LM from [SparseZoo](https://sparsezoo.neuralmagic.com/) or on [HF Hub with the "deepsparse" tag](https://huggingface.co/models?other=deepsparse) | `generate_until`, `loglikelihood` | ... | +| [Neural Magic SparseML](https://github.com/neuralmagic/sparseml) | ✔️ | `sparseml` | Any decoder-only AutoModelForCausalLM from [SparseZoo](https://sparsezoo.neuralmagic.com/) or on [HF Hub](https://huggingface.co/neuralmagic). Especially useful for models with quantization like [`zoo:llama2-7b-gsm8k_llama2_pretrain-pruned60_quantized`](https://sparsezoo.neuralmagic.com/models/llama2-7b-gsm8k_llama2_pretrain-pruned60_quantized) | `generate_until`, `loglikelihood`, `loglikelihood_rolling` | ... | +| Your local inference server! | :heavy_check_mark: | `local-completions` or `local-chat-completions` | Support for OpenAI API-compatible servers, with easy customization for other APIs. | `generate_until`, `loglikelihood`, `loglikelihood_rolling` | | ... | Models which do not supply logits or logprobs can be used with tasks of type `generate_until` only, while local models, or APIs that supply logprobs/logits of their prompts, can be run on all task types: `generate_until`, `loglikelihood`, `loglikelihood_rolling`, and `multiple_choice`. @@ -294,8 +319,16 @@ lm_eval --model hf \ --tasks hellaswag ``` -[GPTQ](https://github.com/PanQiWei/AutoGPTQ) quantized models can be loaded by specifying their file names in `,autogptq=NAME` (or `,autogptq=True` for default names) in the `model_args` argument: +GPTQ quantized models can be loaded using [GPTQModel](https://github.com/ModelCloud/GPTQModel) (faster) or [AutoGPTQ](https://github.com/PanQiWei/AutoGPTQ) + +GPTQModel: add `,gptqmodel=True` to `model_args` +```bash +lm_eval --model hf \ + --model_args pretrained=model-name-or-path,gptqmodel=True \ + --tasks hellaswag +``` +AutoGPTQ: add `,autogptq=True` to `model_args`: ```bash lm_eval --model hf \ --model_args pretrained=model-name-or-path,autogptq=model.safetensors,gptq_use_triton=True \ @@ -435,29 +468,27 @@ The best way to get support is to open an issue on this repo or join the [Eleuth ## Optional Extras Extras dependencies can be installed via `pip install -e ".[NAME]"` -| Name | Use | -|---------------|---------------------------------------| -| anthropic | For using Anthropic's models | -| deepsparse | For running NM's DeepSparse models | -| dev | For linting PRs and contributions | -| gptq | For loading models with GPTQ | -| hf_transfer | For speeding up HF Hub file downloads | -| ifeval | For running the IFEval task | -| neuronx | For running on AWS inf2 instances | -| mamba | For loading Mamba SSM models | -| math | For running math task answer checking | -| multilingual | For multilingual tokenizers | -| openai | For using OpenAI's models | -| optimum | For running Intel OpenVINO models | -| promptsource | For using PromptSource prompts | -| sentencepiece | For using the sentencepiece tokenizer | -| sparseml | For using NM's SparseML models | -| testing | For running library test suite | -| unitxt | For IBM's unitxt dataset tasks | -| vllm | For loading models with vLLM | -| zeno | For visualizing results with Zeno | -|---------------|---------------------------------------| -| all | Loads all extras (not recommended) | +| Name | Use | +|-----------------|----------------------------------------------| +| api | For using api models (Anthropic, OpenAI API) | +| deepsparse | For running NM's DeepSparse models | +| dev | For linting PRs and contributions | +| gptq | For loading models with GPTQ | +| hf_transfer | For speeding up HF Hub file downloads | +| ifeval | For running the IFEval task | +| neuronx | For running on AWS inf2 instances | +| mamba | For loading Mamba SSM models | +| math | For running math task answer checking | +| multilingual | For multilingual tokenizers | +| optimum | For running Intel OpenVINO models | +| promptsource | For using PromptSource prompts | +| sentencepiece | For using the sentencepiece tokenizer | +| sparseml | For using NM's SparseML models | +| testing | For running library test suite | +| vllm | For loading models with vLLM | +| zeno | For visualizing results with Zeno | +| --------------- | --------------------------------------- | +| all | Loads all extras (not recommended) | ## Cite as @@ -465,11 +496,11 @@ Extras dependencies can be installed via `pip install -e ".[NAME]"` @misc{eval-harness, author = {Gao, Leo and Tow, Jonathan and Abbasi, Baber and Biderman, Stella and Black, Sid and DiPofi, Anthony and Foster, Charles and Golding, Laurence and Hsu, Jeffrey and Le Noac'h, Alain and Li, Haonan and McDonell, Kyle and Muennighoff, Niklas and Ociepa, Chris and Phang, Jason and Reynolds, Laria and Schoelkopf, Hailey and Skowron, Aviya and Sutawika, Lintang and Tang, Eric and Thite, Anish and Wang, Ben and Wang, Kevin and Zou, Andy}, title = {A framework for few-shot language model evaluation}, - month = 12, - year = 2023, + month = 07, + year = 2024, publisher = {Zenodo}, - version = {v0.4.0}, - doi = {10.5281/zenodo.10256836}, - url = {https://zenodo.org/records/10256836} + version = {v0.4.3}, + doi = {10.5281/zenodo.12608602}, + url = {https://zenodo.org/records/12608602} } ``` diff --git a/docs/API_guide.md b/docs/API_guide.md new file mode 100644 index 0000000000000000000000000000000000000000..bb8ca2abd510b7e4b812be1837debecfcaf7fced --- /dev/null +++ b/docs/API_guide.md @@ -0,0 +1,202 @@ +# TemplateAPI Usage Guide + +The `TemplateAPI` class is a versatile superclass designed to facilitate the integration of various API-based language models into the lm-evaluation-harness framework. This guide will explain how to use and extend the `TemplateAPI` class to implement your own API models. If your API implements the OpenAI API you can use the `local-completions` or the `local-chat-completions` (defined [here](https://github.com/EleutherAI/lm-evaluation-harness/blob/main/lm_eval/models/openai_completions.py)) model types, which can also serve as examples of how to effectively subclass this template. + +## Overview + +The `TemplateAPI` class provides a template for creating API-based model implementations. It handles common functionalities such as: + +- Tokenization (optional) +- Batch processing +- Caching +- Retrying failed requests +- Parsing API responses + +To use this class, you typically need to subclass it and implement specific methods for your API. + +## Key Methods to Implement + +When subclassing `TemplateAPI`, you need to implement the following methods: + +1. `_create_payload`: Creates the JSON payload for API requests. +2. `parse_logprobs`: Parses log probabilities from API responses. +3. `parse_generations`: Parses generated text from API responses. +4. `headers`: Returns the headers for the API request. + +You may also need to override other methods or properties depending on your API's specific requirements. + +> [!NOTE] +> Currently loglikelihood and MCQ based tasks (such as MMLU) are only supported for completion endpoints. Not for chat-completion — those that expect a list of dicts — endpoints! Completion APIs which support instruct tuned models can be evaluated with the `--apply_chat_template` option in order to simultaneously evaluate models using a chat template format while still being able to access the model logits needed for loglikelihood-based tasks. + +# TemplateAPI Usage Guide + +## TemplateAPI Arguments + +When initializing a `TemplateAPI` instance or a subclass, you can provide several arguments to customize its behavior. Here's a detailed explanation of some important arguments: + +- `model` or `pretrained` (str): + - The name or identifier of the model to use. + - `model` takes precedence over `pretrained` when both are provided. + +- `base_url` (str): + - The base URL for the API endpoint. + +- `tokenizer` (str, optional): + - The name or path of the tokenizer to use. + - If not provided, it defaults to using the same tokenizer name as the model. + +- `num_concurrent` (int): + - Number of concurrent requests to make to the API. + - Useful for APIs that support parallel processing. + - Default is 1 (sequential processing). + +- `tokenized_requests` (bool): + - Determines whether the input is pre-tokenized. Defaults to `True`. + - Requests can be sent in either tokenized form (`list[list[int]]`) or as text (`list[str]`, or `str` for batch_size=1). + - For loglikelihood-based tasks, prompts require tokenization to calculate the context length. If `False` prompts are decoded back to text before being sent to the API. + - Not as important for `generate_until` tasks. + - Ignored for chat formatted inputs (list[dict...]) or if tokenizer_backend is None. + +- `tokenizer_backend` (str, optional): + - Required for loglikelihood-based or MCQ tasks. + - Specifies the tokenizer library to use. Options are "tiktoken", "huggingface", or None. + - Default is "huggingface". + +- `max_length` (int, optional): + - Maximum length of input + output. + - Default is 2048. + +- `max_retries` (int, optional): + - Maximum number of retries for failed API requests. + - Default is 3. + +- `max_gen_toks` (int, optional): + - Maximum number of tokens to generate in completion tasks. + - Default is 256 or set in task yaml. + +- `batch_size` (int or str, optional): + - Number of requests to batch together (if the API supports batching). + - Can be an integer or "auto" (which defaults to 1 for API models). + - Default is 1. + +- `seed` (int, optional): + - Random seed for reproducibility. + - Default is 1234. + +- `add_bos_token` (bool, optional): + - Whether to add the beginning-of-sequence token to inputs (when tokenizing). + - Default is False. + +- `custom_prefix_token_id` (int, optional): + - Custom token ID to use as a prefix for inputs. + - If not provided, uses the model's default BOS or EOS token (if `add_bos_token` is True). + +- `verify_certificate` (bool, optional): + - Whether to validate the certificate of the API endpoint (if HTTPS). + - Default is True. + + +Example usage: + +```python +class MyAPIModel(TemplateAPI): + def __init__(self, **kwargs): + super().__init__( + model="my-model", + base_url="https://api.mymodel.com/v1/completions", + tokenizer_backend="huggingface", + num_concurrent=5, + max_retries=5, + batch_size=10, + **kwargs + ) + + # Implement other required methods... +``` + +When subclassing `TemplateAPI`, you can override these arguments in your `__init__` method to set default values specific to your API. You can also add additional (potentially user-specified) arguments as needed for your specific implementation. + +## Example Implementation: OpenAI API + +The `OpenAICompletionsAPI` and `OpenAIChatCompletion` ([here](https://github.com/EleutherAI/lm-evaluation-harness/blob/main/lm_eval/models/openai_completions.py) classes demonstrate how to implement API models using the `TemplateAPI` class. Here's a breakdown of the key components: + +### 1. Subclassing and Initialization + +```python +@register_model("openai-completions") +class OpenAICompletionsAPI(LocalCompletionsAPI): + def __init__( + self, + base_url="https://api.openai.com/v1/completions", + tokenizer_backend="tiktoken", + **kwargs, + ): + super().__init__( + base_url=base_url, tokenizer_backend=tokenizer_backend, **kwargs + ) +``` + +### 2. Implementing API Key Retrieval + +```python +@cached_property +def api_key(self): + key = os.environ.get("OPENAI_API_KEY", None) + if key is None: + raise ValueError( + "API key not found. Please set the OPENAI_API_KEY environment variable." + ) + return key +``` + +### 3. Creating the Payload + +```python +def _create_payload( + self, + messages: Union[List[List[int]], List[dict], List[str], str], + generate=False, + gen_kwargs: Optional[dict] = None, + **kwargs, +) -> dict: + if generate: + # ... (implementation for generation) + else: + # ... (implementation for log likelihood) +``` + +### 4. Parsing API Responses + +```python +@staticmethod +def parse_logprobs( + outputs: Union[Dict, List[Dict]], + tokens: List[List[int]] = None, + ctxlens: List[int] = None, + **kwargs, +) -> List[Tuple[float, bool]]: + # ... (implementation) + +@staticmethod +def parse_generations(outputs: Union[Dict, List[Dict]], **kwargs) -> List[str]: + # ... (implementation) +``` + +The requests are initiated in the `model_call` or the `amodel_call` methods. + +## Implementing Your Own API Model + +To implement your own API model: + +1. Subclass `TemplateAPI` or one of its subclasses (e.g., `LocalCompletionsAPI`). +2. Override the `__init__` method if you need to set specific parameters. +3. Implement the `_create_payload` and `header` methods to create the appropriate payload for your API. +4. Implement the `parse_logprobs` and `parse_generations` methods to parse your API's responses. +5. Override the `api_key` property if your API requires authentication. +6. Override any other methods as necessary to match your API's behavior. + +## Best Practices + +1. Use the `@register_model` decorator to register your model with the framework (and import it in `lm_eval/models/__init__.py`!). +3. Use environment variables for sensitive information like API keys. +4. Properly handle batching and concurrent requests if supported by your API. diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 8b9ef616dc0a0a9503292248999fb9c32515bde9..48b5c332c22c50306f00f36bd3004db4393a49df 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -2,8 +2,6 @@ Welcome and thank you for your interest in the LM Evaluation Harness! We welcome contributions and feedback and appreciate your time spent with our library, and hope you find it useful! -We intend LM Evaluation Harness to be a broadly useful and - ## Important Resources There are several places information about LM Evaluation Harness is located: @@ -11,7 +9,7 @@ There are several places information about LM Evaluation Harness is located: - Our [documentation pages](https://github.com/EleutherAI/lm-evaluation-harness/tree/main/docs) - We occasionally use [GitHub Milestones](https://github.com/EleutherAI/lm-evaluation-harness/milestones) to track progress toward specific near-term version releases. - We maintain a [Project Board](https://github.com/orgs/EleutherAI/projects/25) for tracking current work items and PRs, and for future roadmap items or feature requests. -- Further discussion and support conversations are located in the #lm-thunderdome channel of the [EleutherAI discord](discord.gg/eleutherai). +- Further discussion and support conversations are located in the #lm-thunderdome channel of the [EleutherAI discord](https://discord.gg/eleutherai). ## Code Style @@ -32,7 +30,7 @@ in order to ensure linters and other checks will be run upon committing. We use [pytest](https://docs.pytest.org/en/latest/) for running unit tests. All library unit tests can be run via: ``` -python -m pytest --ignore=tests/tests_master --ignore=tests/extra +python -m pytest --showlocals -s -vv -n=auto --ignore=tests/models/test_neuralmagic.py --ignore=tests/models/test_openvino.py ``` ## Contributor License Agreement diff --git a/docs/README.md b/docs/README.md index 974e2e2c1c3f05b802d2a095d60cc36f0cde564b..f040eabaef41b55e9bd8297f3f4653fbc16bf0bc 100644 --- a/docs/README.md +++ b/docs/README.md @@ -4,7 +4,8 @@ Welcome to the docs for the LM Evaluation Harness! ## Table of Contents -* To learn about the public interface of the library, as well as how to evaluate via the commandline or as integrated into an external library, see the [Interface](./interface.md) +* To learn about the public interface of the library, as well as how to evaluate via the command line or as integrated into an external library, see the [Interface](./interface.md). * To learn how to add a new library, API, or model type to the library, as well as a quick explainer on the types of ways to evaluate an LM, see the [Model Guide](./model_guide.md). + * For an extended description of how to extend the library to new model classes served over an API, see the [API Guide](./API_guide.md). * For a crash course on adding new tasks to the library, see our [New Task Guide](./new_task_guide.md). * To learn more about pushing the limits of task configuration that the Eval Harness supports, see the [Task Configuration Guide](./task_guide.md). diff --git a/docs/interface.md b/docs/interface.md index 89e0397b9ffffb740c8ebb4925727872de4d47f7..47cf00b49694bdbdd86a431c318f0497a2cb4f5a 100644 --- a/docs/interface.md +++ b/docs/interface.md @@ -46,7 +46,11 @@ This mode supports a number of command-line arguments, the details of which can - `--system_instruction`: Specifies a system instruction string to prepend to the prompt. -- `--apply_chat_template` : If this flag is on, a chat template will be applied to the prompt. For Hugging Face models, the chat template is taken from the tokenizer, if the tokenizer does not have a chat template, a default one will be applied. For other models, chat templating is not currently implemented. +- `--apply_chat_template` : This flag specifies whether to apply a chat template to the prompt. It can be used in the following ways: + - `--apply_chat_template` : When used without an argument, applies the only available chat template to the prompt. For Hugging Face models, if no dedicated chat template exists, the default chat template will be applied. + - `--apply_chat_template template_name` : If the model has multiple chat templates, apply the specified template to the prompt. + + For Hugging Face models, the default chat template can be found in the [`default_chat_template`](https://github.com/huggingface/transformers/blob/fc35907f95459d7a6c5281dfadd680b6f7b620e3/src/transformers/tokenization_utils_base.py#L1912) property of the Transformers Tokenizer. - `--fewshot_as_multiturn` : If this flag is on, the Fewshot examples are treated as a multi-turn conversation. Questions are provided as user content and answers are provided as assistant responses. Requires `--num_fewshot` to be set to be greater than 0, and `--apply_chat_template` to be on. @@ -58,12 +62,15 @@ This mode supports a number of command-line arguments, the details of which can * `--hf_hub_log_args` : Logs evaluation results to Hugging Face Hub. Accepts a string with the arguments separated by commas. Available arguments: * `hub_results_org` - organization name on Hugging Face Hub, e.g., `EleutherAI`. If not provided, the results will be pushed to the owner of the Hugging Face token, - * `hub_repo_name` - repository name on Hugging Face Hub, e.g., `lm-eval-results`, + * `hub_repo_name` - repository name on Hugging Face Hub (deprecated, `details_repo_name` and `results_repo_name` should be used instead), e.g., `lm-eval-results`, + * `details_repo_name` - repository name on Hugging Face Hub to store details, e.g., `lm-eval-results`, + * `results_repo_name` - repository name on Hugging Face Hub to store results, e.g., `lm-eval-results`, * `push_results_to_hub` - whether to push results to Hugging Face Hub, can be `True` or `False`, * `push_samples_to_hub` - whether to push samples results to Hugging Face Hub, can be `True` or `False`. Requires `--log_samples` to be set, * `public_repo` - whether the repository is public, can be `True` or `False`, * `leaderboard_url` - URL to the leaderboard, e.g., `https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard`. * `point_of_contact` - Point of contact for the results dataset, e.g., `yourname@example.com`. + * `gated` - whether to gate the details dataset, can be `True` or `False`. ## External Library Usage diff --git a/docs/model_guide.md b/docs/model_guide.md index b14935cb4f5690c769c338788a66cbbbeeb3edd2..810801cbfa47294a5c02ca0d0c38da02975dd713 100644 --- a/docs/model_guide.md +++ b/docs/model_guide.md @@ -118,17 +118,45 @@ class MyCustomLM(LM): #... @property def tokenizer_name(self) -> str: - # should return a string denoting the name of the model's tokenizer and/or the accompanying chat template. - - @property - def chat_template(self) -> str: - # should return a chat template formatting string that is used to build prompt from a user/assistant chat history. - # this will be saved in the evaluation results for reproducibility. + """ + Return the name of the model's tokenizer and/or the accompanying chat template. + The returned string is used to cache requests. + + Returns: + str: The name of the model's tokenizer and/or chat template. + """ + + def chat_template(self, chat_template: Union[bool, str] = False) -> str: + """ + Get the appropriate chat template for the model based on the `chat_template` argument. + + This method returns the chat template string to build the prompt from a chat history. + The chat template is saved in the evaluation results for reproducibility. + Boolean arguments should be used with models that have only one chat template, + while string arguments are used with models that have multiple chat templates. + For the reference implementation, see HFLM class in `lm_eval.models.huggingface`. + + Args: + chat_template (Union[bool, str]): Specifies whether to apply a chat template: + - If False: Do not apply any chat template. + - If True: Apply the default chat template. + - If str: Apply the specified chat template by name. + + Returns: + str: The selected chat template in Jinja format. + """ def apply_chat_template(self, chat_history: List[Dict[str, str]]) -> str: - # responsible for taking as input a chat history that would be fed into the model, and - # rendering it as a string that can be then tokenized and input into the model. - #... + """ + Process a chat history to create a string that can be tokenized and input into the model. + + Args: + chat_history (List[Dict[str, str]]): A list of dictionaries representing the chat history, + where each dictionary has "role" and "content" keys. + + Returns: + str: A string representing the chat history that can be tokenized and fed into the model. + """ ``` - `apply_chat_template` diff --git a/docs/new_task_guide.md b/docs/new_task_guide.md index 2f6d32177036407da973e7a4afa5f21574f193ea..dac8541e82e5175ac602c440f2be4b7eb8abd18a 100644 --- a/docs/new_task_guide.md +++ b/docs/new_task_guide.md @@ -86,20 +86,20 @@ Let's create a python file in the directory where we're writing our YAML file: ```bash touch lm_eval/tasks//utils.py ``` -Now, in `utils.py` we'll write a function to process each split of our dataset: - -TODO: Change the example to one that's in the tasks/ +Now, in `utils.py` we'll write a function to process each split of our dataset (the following example is drawn from [the `hellaswag` task](../lm_eval/tasks/hellaswag/utils.py)): ```python -def process_docs(dataset: datasets.Dataset): - def _helper(doc): - # modifies the contents of a single - # document in our dataset. - doc["choices"] = [doc["choice1"], doc["choice2"], doc["wrong_answer"]] - doc["gold"] = doc["label"] - return doc +def process_docs(dataset: datasets.Dataset) -> datasets.Dataset: + def _process_doc(doc): + ctx = doc["ctx_a"] + " " + doc["ctx_b"].capitalize() + out_doc = { + "query": preprocess(doc["activity_label"] + ": " + ctx), + "choices": [preprocess(ending) for ending in doc["endings"]], + "gold": int(doc["label"]), + } + return out_doc - return dataset.map(_helper) # returns back a datasets.Dataset object + return dataset.map(_process_doc) ``` Now, in our YAML config file we'll use the `!function` constructor, and tell the config where our imported Python function will come from. At runtime, before doing anything else we will preprocess our dataset according to this function! @@ -285,7 +285,7 @@ As a heuristic check: For more detail on the task system and advanced features, see [`docs/task_guide.md`](https://github.com/EleutherAI/lm-evaluation-harness/blob/main/docs/task_guide.md) . If none of the above sound like they apply to your task, it's time to continue onto checking your task performance! -### Task name + groups (registering a task) +### Task name + tags (registering a task) To test a task conveniently, it helps to *register* the task--that is, to give it a name and make the `lm-eval` library aware it exists! @@ -296,14 +296,14 @@ task: ``` Including a task name is mandatory. -It is often also convenient to label your task with several `groups`, or tags, though this field is optional: +It is often also convenient to label your task with several `tag` values, though this field is optional: ```yaml -group: - - group1 - - group2 +tag: + - tag1 + - tag2 ``` -This will add your task to the `group1` and `group2` groups, enabling people to know how to categorize your task, and if desired run all tasks in one of these groups at once, your task along with them. +This will add your task to the `tag1` and `tag2` tags, enabling people to know how to categorize your task, and if desired run all tasks in one of these groups at once, your task along with them. If your task is not in the `lm_eval/tasks` folder, you'll need to tell the Eval Harness where to look for YAML files. @@ -319,7 +319,48 @@ Passing `--tasks /path/to/yaml/file` is also accepted. ### Advanced Group Configs -You can make more complete group config while also tailoring parameters for individual tasks. +While `tag` values are helpful when you want to be able to quickly and conveniently run a set of related tasks via `--tasks my_tag_name`, often, we wish to implement more complex logic. For example, the MMLU benchmark contains 57 *subtasks* that must all be *averaged* together in order to report a final 'MMLU score'. + +Groupings of tasks might also use particular variants of a task--for example, we might want to default to evaluating a task as 5-shot when called as part of a given grouping, but not have a preference for number of shots when evaluating it as a standalone. + +We implement this via **groups**, which are distinct from tags. Groups can be implemented via *group config* YAML files, which are laid out similarly but slightly differently to tasks' YAML configs. + +The most basic form of group can be defined via a YAML config similar to the following: + +```yaml +group: nli_tasks +task: + - cb + - anli_r1 + - rte +metadata: + version: 1.0 +``` + +This will behave almost identically to a `tag` that includes these 3 tasks, but with one key distinction: we'll print the `nli_tasks` group as a row (with no associated metrics) in our table of outputs, and visually show that these 3 tasks appear under its subheader. + + +Now, let's assume we actually want to report an aggregate score for `nli_tasks`. We would instead use a YAML config like the following: + +```yaml +group: nli_tasks +task: + - cb + - anli_r1 + - rte +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true # defaults to `true`. Set this to `false` to do a "macro" average (taking each subtask's average accuracy, and summing those accuracies and dividing by 3)--by default we do a "micro" average (retain all subtasks' per-document accuracies, and take the mean over all documents' accuracies to get our aggregate mean). +metadata: + version: 1.0 +``` + +Similar to our `metric_list` for listing out the metrics we want to calculate for a given task, we use an `aggregate_metric_list` field to specify which metric name to aggregate across subtasks, what aggregation function to use, and whether we should micro- or macro- average these metrics. See [./task_guide.md](./task_guide.md) for a full list of related sub-keys. + +**[!Tip]: currently, we predominantly only support the aggregation of group metrics that use `mean` (either micro- or macro- averaged) over their subtasks. If you require even more complex aggregation rules, you may want to perform aggregation offline.** + +Group configs can be fairly complex! We can do various operations, such as defining new subtask(s) inline in our group YAML, overriding an existing task's specific config value, or nesting existing groups within our For example, let's build a config for evaluating MMLU and a few natural language inference tasks. For MMLU, we can write the name for the benchmark as a subtask written under `task`. You can configure the parameters such as `num_fewshot`. If the task being configured is a group such as `mmlu` or `super_glue`, the parameter set will be applied to all of the subtasks. @@ -331,33 +372,13 @@ task: - cb - anli_r1 - rte + aggregate_metric_list: + - metric: acc + aggregation: mean + higher_is_better: true - task: mmlu num_fewshot: 2 ``` -It's also important to note how you can basically insert a group config as a task. Here, to make a group of natural language inference tasks, you simply write like how you would normally write a group config but this time place that as part of a task list under the main group being built. - -### Duplicate Tasks in Group Configs - -There might be cases where you might want to evaluate prompts and how models perform over prompt variations. You can list an existing task (In the example below, `anli_r1`) which varying `doc_to_text` implementation. To differentiate from each variation, we can utilize `task_alias`. LM-Eval will recognize that there are multiple variations of the same tasks and differentiate them. -```yaml -group: flan_held_in -group_alias: Flan (Held-In) -task: - # ANLI R1 - - group: anli_r1_flan - group_alias: ANLI R1 - task: - - task: anli_r1 - task_alias: prompt-0 - include: _held_in_template_yaml - doc_to_text: "{{premise}}\n\nChoose your answer ..." - ... - - task: anli_r1 - task_alias: prompt-1 - include: _held_in_template_yaml - doc_to_text: "{{premise}}\n\nBased on ..." - ... -``` ### Configuring python classes @@ -382,21 +403,29 @@ task: ... ``` +You can also pass a custom argument to your class by accepting `config` in the custom class constructor. +Here's how to do it: + +```yaml +task: 20_newsgroups +class: !function task.Unitxt +recipe: card=cards.20_newsgroups,template=templates.classification.multi_class.title +``` + +In this example, `recipe` is the custom argument for the `Unitxt` class. + ## Beautifying Table Display -To avoid conflict, each task needs to be registered with a unique name. Because of this, slight variations of task are still counted as unique tasks and need to be named uniquely. This could be done by appending an additional naming that may refer to the variation such as in MMLU where the template used to evaluated for flan are differentiated from the default by the prefix `mmlu_flan_*`. Printing the full task names can easily clutter the results table at the end of the evaluation especially when you have a long list of tasks or are using a benchmark that comprises of many tasks. To make it more legible, you can use `task_alias` and `group_alias` to provide an alternative task name and group name that will be printed. For example in `mmlu_abstract_algebra.yaml` we set `group_alias` to `stem` and `task_alias` to `abstract_algebra`. +To avoid conflict, each task needs to be registered with a unique name. Because of this, slight variations of task are still counted as unique tasks and need to be named uniquely. This could be done by appending an additional naming that may refer to the variation such as in MMLU where the template used to evaluated for flan are differentiated from the default by the prefix `mmlu_flan_*`. Printing the full task names can easily clutter the results table at the end of the evaluation especially when you have a long list of tasks or are using a benchmark that comprises of many tasks. To make it more legible, you can use `task_alias` and `group_alias` to provide an alternative task name and group name that will be printed. For example in `mmlu_abstract_algebra.yaml` we set `task_alias` to `abstract_algebra`. In group configs, a `group_alias` for a group can also be set. ``` "dataset_name": "abstract_algebra" "description": "The following are multiple choice questions (with answers) about abstract\ \ algebra.\n\n" -"group": "mmlu_stem" -"group_alias": "stem" "include": "_default_template_yaml" "task": "mmlu_abstract_algebra" "task_alias": "abstract_algebra" ``` -Note: Even though `group` can be a list, for now, `group_alias` can only be a single string. ## Checking validity @@ -416,9 +445,9 @@ a simple eye test. ## Versioning -One key feature in LM Evaluation Harness is the ability to version tasks--that is, mark them with a specific version number that can be bumped whenever a breaking change is made. +One key feature in LM Evaluation Harness is the ability to version tasks and groups--that is, mark them with a specific version number that can be bumped whenever a breaking change is made. -This version info can be provided by adding the following to your new task config file: +This version info can be provided by adding the following to your new task or group config file: ``` metadata: diff --git a/docs/task_guide.md b/docs/task_guide.md index b10ca7d9a96cfbb5a2204c69531f56c16ac427f4..34e47c413694eeb8da2d3dc5c743eaba2740e0b0 100644 --- a/docs/task_guide.md +++ b/docs/task_guide.md @@ -16,7 +16,8 @@ Tasks are configured via the `TaskConfig` object. Below, we describe all fields Task naming + registration: - **task** (`str`, defaults to None) — name of the task. -- **group** (`str`, *optional*) — name of the task group(s) a task belongs to. Enables one to run all tasks with a specified tag or group name at once. +- **task_alias** (`str`, defaults to None) - Alias of the task name that will be printed in the final table results. +- **tag** (`str`, *optional*) — name of the task tags(s) a task belongs to. Enables one to run all tasks with a specified tag name at once. Dataset configuration options: - **dataset_path** (`str`) — The name of the dataset as listed by HF in the datasets Hub. @@ -55,8 +56,6 @@ Other: ## Filters -Explain: What are filters? What is their place in the pipeline? - A key component of the `lm-evaluation-harness` library is the `Filter` object. In a typical evaluation run of the harness, we take the formatted inputs and run them through our LM, with the appropriate output type (greedy or free-form generation, or loglikelihood-based comparative scoring). After getting scores or output text from our LM on each `Instance` or document in the dataset, we then need to feed these responses into a metric or scoring function to return scores to a user. @@ -295,105 +294,24 @@ Generative tasks: Tasks using complex filtering: - GSM8k with CoT (+ with Self-Consistency): (`lm_eval/tasks/gsm8k/gsm8k-cot.yaml` ; `lm_eval/tasks/gsm8k/gsm8k-cot-self-consistency.yaml`) - -## Benchmarks +# Group Configuration When evaluating a language model, it's is not unusual to test across a number of tasks that may not be related to one another in order to assess a variety of capabilities. To this end, it may be combursome to have to list the set of tasks or add a new group name to each yaml of each individual task. -To solve this, we can create a benchmark yaml config. This is a config that contains the names of the tasks that should be included in a particular benchmark. The config consists of two main keys `group` which denotes the name of the benchmark and `task` which is where we can list the tasks. The tasks listed in `task` are the task names that have been registered. A good example would be the list of tasks used to evaluate the Pythia Suite. - -```yaml -group: pythia -task: - - lambada_openai - - wikitext - - piqa - - sciq - - wsc - - winogrande - - arc - - logiqa - - blimp - - hendrycksTest* -``` - -It is also possible to list an existing task in your benchmark configuration with some adjustments. For example, a few tasks from mmlu is included `multimedqa`. There, the `task_alias` and `group_alias` (See [here](https://github.com/EleutherAI/lm-evaluation-harness/blob/main/docs/new_task_guide.md#beautifying-table-display) for more details) are modified to suit the benchmark. - -```yaml -group: multimedqa -task: - - pubmedqa - - medmcqa - - medqa_4options - - task: mmlu_anatomy - task_alias: "anatomy (mmlu)" - group_alias: null - - task: mmlu_clinical_knowledge - task_alias: "clinical_knowledge (mmlu)" - group_alias: null - ... -``` +To solve this, we can create a **group** yaml config. This is a config that contains the names of the tasks that should be included in a particular group. The config consists of two main keys: a `group` key which denotes the name of the group (as it would be called from the command line, e.g. `mmlu`) and a `task` key which is where we can list the tasks. The tasks listed in `task` are the task names that have been registered. A good example of a group yaml config can be found at [../lm_eval/tasks/mmlu/default/_mmlu.yaml]. See also the [New Task Guide](./new_task_guide.md) for a more in-depth and tutorial-esque explanation of how to write complex GroupConfigs. -Alternatively, benchmarks can have tasks that are customizable for each task. They can be defined like how a yaml task is usually set. +## Configurations -```yaml -group: t0_eval -task: - # Coreference Resolution - - dataset_path: super_glue - dataset_name: wsc.fixed - use_prompt: promptsource:* - training_split: train - validation_split: validation - metric_list: - - metric: exact_match - aggregation: mean - higher_is_better: true - ignore_case: true - ignore_punctuation: true - # Coreference Resolution - - dataset_path: winogrande - dataset_name: winogrande_xl - use_prompt: promptsource:* - training_split: train - validation_split: validation - metric_list: - - metric: exact_match - aggregation: mean - higher_is_better: true - ignore_case: true - ignore_punctuation: true - ... -``` +Groups are configured via the `GroupConfig` object. Below, we describe all fields usable within the object, and their role in defining a task. -If the benchmark contains the same dataset but with different configurations, use `task` to differentiate between them. For example, T0-Eval evaluates on 3 versions of ANLI but the huggingface dataset collects them in one dataset. - -```YAML -group: t0_eval -task: - ... - - task: anli_r1 - dataset_path: anli - use_prompt: promptsource:* - training_split: train_r1 - validation_split: dev_r1 - metric_list: - - metric: exact_match - aggregation: mean - higher_is_better: true - ignore_case: true - ignore_punctuation: true - - task: anli_r2 - dataset_path: anli - use_prompt: promptsource:* - training_split: train_r2 - validation_split: dev_r2 - metric_list: - - metric: exact_match - aggregation: mean - higher_is_better: true - ignore_case: true - ignore_punctuation: true -``` +### Parameters -Calling the benchmark is done the same way we would call any task with `--tasks`. Benchmarks can be added in `lm_eval/tasks/benchmarks/` +- **group** (`str`, defaults to `None`) — name of the group. Used to invoke it from the command line. +- **group_alias** (`str`, defaults to `None`) - Alternative name for the group that will be printed in the table output. +- **task** (`Union[str, list]`, defaults to `None`) - List of tasks that constitute the group. +- **aggregate_metric_list** (`list`, defaults to `None`) - similar to `metric_list` in TaskConfigs, provide a list of configurations for metrics that should be aggregated across subtasks. Leaving empty will result in no aggregation being performed for this group. Keys for each list entry are: + - `metric: str` - the name of the metric to aggregate over (all subtasks must report a metric holding this name.) + - `aggregation: str` - what aggregation function to apply to aggregate these per-subtask metrics. **currently, only `mean` is supported.** + - `weight_by_size: bool = True` whether to perform micro- averaging (`True`) or macro- (`False`) averaging of subtasks' accuracy scores when reporting the group's metric. MMLU, for example, averages over per-document accuracies (the *micro average*), resulting in the same accuracy as if one simply concatenated all 57 subjects into a single dataset and evaluated accuracy on that dataset. + - `filter_list: Union[str, List[str]] = "none"` - what filter keys one should match on to aggregate results. For example, if trying to aggregate over the `exact_match` metric using `strict-match` filter for `bbh_cot_zeroshot`, then set this to be `filter_list: "strict-match"`. +- **metadata** (`dict`, *optional*) - As with TaskConfigs, a field where extra config metadata can be passed. set the `num_fewshot` key within this to override the printed n_shot value in a results table for your group, for example. diff --git a/examples/lm-eval-overview.ipynb b/examples/lm-eval-overview.ipynb index 898192ce9f8e5963a3c9bb3e00910fa898fdcbdb..9cb1f5e26a798074a300457e135940cf2dbc9172 100644 --- a/examples/lm-eval-overview.ipynb +++ b/examples/lm-eval-overview.ipynb @@ -1,1231 +1,1232 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "Qw83KAePAhaS" - }, - "source": [ - "# Releasing LM-Evaluation-Harness v0.4.0" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Z7k2vq1iAdqr" - }, - "source": [ - "With the vast amount of work done in the field today, it helps to have a tool that people can use easily to share their results and use to check others to ensure reported numbers are valid. The LM Evaluation Harness is one such tool the community has used extensively. We want to continue to support the community and with that in mind, we’re excited to announce a major update on the LM Evaluation Harness to further our goal for open and accessible AI research." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "0gDoM0AJAvEc" - }, - "source": [ - "Our refactor stems from our desires to make the following believed best practices easier to carry out. \n", - "\n", - "1. Never copy results from other papers\n", - "2. Always share your exact prompts\n", - "3. Always provide model outputs\n", - "4. Qualitatively review a small batch of outputs before running evaluation jobs at scale\n", - "\n", - "We also wanted to make the library a better experience to use and to contribute or design evaluations within. New features in the new release that serve this purpose include:\n", - "\n", - "1. Faster Evaluation Runtimes (accelerated data-parallel inference with HF Transformers + Accelerate, and commonly used or faster inference libraries such as vLLM and Llama-CPP)\n", - "2. Easier addition and sharing of new tasks (YAML-based task config formats, allowing single-file sharing of custom tasks)\n", - "3. More configurability, for more advanced workflows and easier operation with modifying prompts\n", - "4. Better logging of data at runtime and post-hoc" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "nnwsOpjda_YW" - }, - "source": [ - "In this notebook we will be going through a short tutorial on how things work." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "zAov81vTbL2K" - }, - "source": [ - "## Install LM-Eval" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "8hiosGzq_qZg", - "outputId": "6ab73e5e-1f54-417e-a388-07e0d870b132" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Collecting git+https://github.com/EleutherAI/lm-evaluation-harness.git@big-refactor\n", - " Cloning https://github.com/EleutherAI/lm-evaluation-harness.git (to revision big-refactor) to /tmp/pip-req-build-tnssql5s\n", - " Running command git clone --filter=blob:none --quiet https://github.com/EleutherAI/lm-evaluation-harness.git /tmp/pip-req-build-tnssql5s\n", - " Running command git checkout -b big-refactor --track origin/big-refactor\n", - " Switched to a new branch 'big-refactor'\n", - " Branch 'big-refactor' set up to track remote branch 'big-refactor' from 'origin'.\n", - " Resolved https://github.com/EleutherAI/lm-evaluation-harness.git to commit 42f486ee49b65926a444cb0620870a39a5b4b0a8\n", - " Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n", - " Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n", - " Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", - "Collecting accelerate>=0.21.0 (from lm-eval==1.0.0)\n", - " Downloading accelerate-0.24.1-py3-none-any.whl (261 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m261.4/261.4 kB\u001b[0m \u001b[31m4.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hCollecting evaluate (from lm-eval==1.0.0)\n", - " Downloading evaluate-0.4.1-py3-none-any.whl (84 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m84.1/84.1 kB\u001b[0m \u001b[31m5.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hCollecting datasets>=2.0.0 (from lm-eval==1.0.0)\n", - " Downloading datasets-2.15.0-py3-none-any.whl (521 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m521.2/521.2 kB\u001b[0m \u001b[31m9.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hCollecting jsonlines (from lm-eval==1.0.0)\n", - " Downloading jsonlines-4.0.0-py3-none-any.whl (8.7 kB)\n", - "Requirement already satisfied: numexpr in /usr/local/lib/python3.10/dist-packages (from lm-eval==1.0.0) (2.8.7)\n", - "Collecting peft>=0.2.0 (from lm-eval==1.0.0)\n", - " Downloading peft-0.6.2-py3-none-any.whl (174 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m174.7/174.7 kB\u001b[0m \u001b[31m7.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hCollecting pybind11>=2.6.2 (from lm-eval==1.0.0)\n", - " Downloading pybind11-2.11.1-py3-none-any.whl (227 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m227.7/227.7 kB\u001b[0m \u001b[31m12.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hCollecting pytablewriter (from lm-eval==1.0.0)\n", - " Downloading pytablewriter-1.2.0-py3-none-any.whl (111 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m111.1/111.1 kB\u001b[0m \u001b[31m8.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hCollecting rouge-score>=0.0.4 (from lm-eval==1.0.0)\n", - " Downloading rouge_score-0.1.2.tar.gz (17 kB)\n", - " Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n", - "Collecting sacrebleu>=1.5.0 (from lm-eval==1.0.0)\n", - " Downloading sacrebleu-2.3.2-py3-none-any.whl (119 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m119.7/119.7 kB\u001b[0m \u001b[31m8.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hRequirement already satisfied: scikit-learn>=0.24.1 in /usr/local/lib/python3.10/dist-packages (from lm-eval==1.0.0) (1.2.2)\n", - "Collecting sqlitedict (from lm-eval==1.0.0)\n", - " Downloading sqlitedict-2.1.0.tar.gz (21 kB)\n", - " Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n", - "Requirement already satisfied: torch>=1.8 in /usr/local/lib/python3.10/dist-packages (from lm-eval==1.0.0) (2.1.0+cu118)\n", - "Collecting tqdm-multiprocess (from lm-eval==1.0.0)\n", - " Downloading tqdm_multiprocess-0.0.11-py3-none-any.whl (9.8 kB)\n", - "Requirement already satisfied: transformers>=4.1 in /usr/local/lib/python3.10/dist-packages (from lm-eval==1.0.0) (4.35.2)\n", - "Collecting zstandard (from lm-eval==1.0.0)\n", - " Downloading zstandard-0.22.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (5.4 MB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.4/5.4 MB\u001b[0m \u001b[31m29.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from accelerate>=0.21.0->lm-eval==1.0.0) (1.23.5)\n", - "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from accelerate>=0.21.0->lm-eval==1.0.0) (23.2)\n", - "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate>=0.21.0->lm-eval==1.0.0) (5.9.5)\n", - "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from accelerate>=0.21.0->lm-eval==1.0.0) (6.0.1)\n", - "Requirement already satisfied: huggingface-hub in /usr/local/lib/python3.10/dist-packages (from accelerate>=0.21.0->lm-eval==1.0.0) (0.19.4)\n", - "Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.0.0->lm-eval==1.0.0) (9.0.0)\n", - "Collecting pyarrow-hotfix (from datasets>=2.0.0->lm-eval==1.0.0)\n", - " Downloading pyarrow_hotfix-0.6-py3-none-any.whl (7.9 kB)\n", - "Collecting dill<0.3.8,>=0.3.0 (from datasets>=2.0.0->lm-eval==1.0.0)\n", - " Downloading dill-0.3.7-py3-none-any.whl (115 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m115.3/115.3 kB\u001b[0m \u001b[31m14.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets>=2.0.0->lm-eval==1.0.0) (1.5.3)\n", - "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.0.0->lm-eval==1.0.0) (2.31.0)\n", - "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.0.0->lm-eval==1.0.0) (4.66.1)\n", - "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from datasets>=2.0.0->lm-eval==1.0.0) (3.4.1)\n", - "Collecting multiprocess (from datasets>=2.0.0->lm-eval==1.0.0)\n", - " Downloading multiprocess-0.70.15-py310-none-any.whl (134 kB)\n", - "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m19.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", - "\u001b[?25hRequirement already satisfied: fsspec[http]<=2023.10.0,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.0.0->lm-eval==1.0.0) (2023.6.0)\n", - "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets>=2.0.0->lm-eval==1.0.0) (3.8.6)\n", - "Collecting responses<0.19 (from evaluate->lm-eval==1.0.0)\n", - " Downloading responses-0.18.0-py3-none-any.whl (38 kB)\n", - "Requirement already satisfied: safetensors in /usr/local/lib/python3.10/dist-packages (from peft>=0.2.0->lm-eval==1.0.0) (0.4.0)\n", - "Requirement already satisfied: absl-py in /usr/local/lib/python3.10/dist-packages (from rouge-score>=0.0.4->lm-eval==1.0.0) (1.4.0)\n", - "Requirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (from rouge-score>=0.0.4->lm-eval==1.0.0) (3.8.1)\n", - "Requirement already satisfied: six>=1.14.0 in /usr/local/lib/python3.10/dist-packages (from rouge-score>=0.0.4->lm-eval==1.0.0) (1.16.0)\n", - "Collecting portalocker (from sacrebleu>=1.5.0->lm-eval==1.0.0)\n", - " Downloading portalocker-2.8.2-py3-none-any.whl (17 kB)\n", - "Requirement already satisfied: regex in /usr/local/lib/python3.10/dist-packages (from sacrebleu>=1.5.0->lm-eval==1.0.0) (2023.6.3)\n", - "Requirement already satisfied: tabulate>=0.8.9 in /usr/local/lib/python3.10/dist-packages (from sacrebleu>=1.5.0->lm-eval==1.0.0) (0.9.0)\n", - "Collecting colorama (from sacrebleu>=1.5.0->lm-eval==1.0.0)\n", - " Downloading colorama-0.4.6-py2.py3-none-any.whl (25 kB)\n", - "Requirement already satisfied: lxml in /usr/local/lib/python3.10/dist-packages (from sacrebleu>=1.5.0->lm-eval==1.0.0) (4.9.3)\n", - "Requirement already satisfied: scipy>=1.3.2 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.24.1->lm-eval==1.0.0) (1.11.3)\n", - "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.24.1->lm-eval==1.0.0) (1.3.2)\n", - "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.24.1->lm-eval==1.0.0) (3.2.0)\n", - "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch>=1.8->lm-eval==1.0.0) (3.13.1)\n", - "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from torch>=1.8->lm-eval==1.0.0) (4.5.0)\n", - "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.8->lm-eval==1.0.0) (1.12)\n", - "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.8->lm-eval==1.0.0) (3.2.1)\n", - "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.8->lm-eval==1.0.0) (3.1.2)\n", - "Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.8->lm-eval==1.0.0) (2.1.0)\n", - "Requirement already satisfied: tokenizers<0.19,>=0.14 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.1->lm-eval==1.0.0) (0.15.0)\n", - "Requirement already satisfied: attrs>=19.2.0 in /usr/local/lib/python3.10/dist-packages (from jsonlines->lm-eval==1.0.0) (23.1.0)\n", - "Requirement already satisfied: setuptools>=38.3.0 in /usr/local/lib/python3.10/dist-packages (from pytablewriter->lm-eval==1.0.0) (67.7.2)\n", - "Collecting DataProperty<2,>=1.0.1 (from pytablewriter->lm-eval==1.0.0)\n", - " Downloading DataProperty-1.0.1-py3-none-any.whl (27 kB)\n", - "Collecting mbstrdecoder<2,>=1.0.0 (from pytablewriter->lm-eval==1.0.0)\n", - " Downloading mbstrdecoder-1.1.3-py3-none-any.whl (7.8 kB)\n", - "Collecting pathvalidate<4,>=2.3.0 (from pytablewriter->lm-eval==1.0.0)\n", - " Downloading pathvalidate-3.2.0-py3-none-any.whl (23 kB)\n", - "Collecting tabledata<2,>=1.3.1 (from pytablewriter->lm-eval==1.0.0)\n", - " Downloading tabledata-1.3.3-py3-none-any.whl (11 kB)\n", - "Collecting tcolorpy<1,>=0.0.5 (from pytablewriter->lm-eval==1.0.0)\n", - " Downloading tcolorpy-0.1.4-py3-none-any.whl (7.9 kB)\n", - "Collecting typepy[datetime]<2,>=1.3.2 (from pytablewriter->lm-eval==1.0.0)\n", - " Downloading typepy-1.3.2-py3-none-any.whl (31 kB)\n", - "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.0.0->lm-eval==1.0.0) (3.3.2)\n", - "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.0.0->lm-eval==1.0.0) (6.0.4)\n", - "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.0.0->lm-eval==1.0.0) (4.0.3)\n", - "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.0.0->lm-eval==1.0.0) (1.9.2)\n", - "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.0.0->lm-eval==1.0.0) (1.4.0)\n", - "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.0.0->lm-eval==1.0.0) (1.3.1)\n", - "Requirement already satisfied: chardet<6,>=3.0.4 in /usr/local/lib/python3.10/dist-packages (from mbstrdecoder<2,>=1.0.0->pytablewriter->lm-eval==1.0.0) (5.2.0)\n", - "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets>=2.0.0->lm-eval==1.0.0) (3.4)\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets>=2.0.0->lm-eval==1.0.0) (2.0.7)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets>=2.0.0->lm-eval==1.0.0) (2023.7.22)\n", - "Requirement already satisfied: python-dateutil<3.0.0,>=2.8.0 in /usr/local/lib/python3.10/dist-packages (from typepy[datetime]<2,>=1.3.2->pytablewriter->lm-eval==1.0.0) (2.8.2)\n", - "Requirement already satisfied: pytz>=2018.9 in /usr/local/lib/python3.10/dist-packages (from typepy[datetime]<2,>=1.3.2->pytablewriter->lm-eval==1.0.0) (2023.3.post1)\n", - "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.8->lm-eval==1.0.0) (2.1.3)\n", - "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk->rouge-score>=0.0.4->lm-eval==1.0.0) (8.1.7)\n", - "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.8->lm-eval==1.0.0) (1.3.0)\n", - "Building wheels for collected packages: lm-eval, rouge-score, sqlitedict\n", - " Building wheel for lm-eval (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", - " Created wheel for lm-eval: filename=lm_eval-1.0.0-py3-none-any.whl size=994254 sha256=88356155b19f2891981ecef948326ad6ce8ca40a6009378410ec20d0e225995a\n", - " Stored in directory: /tmp/pip-ephem-wheel-cache-9v6ye7h3/wheels/17/01/26/599c0779e9858a70a73fa8a306699b5b9a868f820c225457b0\n", - " Building wheel for rouge-score (setup.py) ... \u001b[?25l\u001b[?25hdone\n", - " Created wheel for rouge-score: filename=rouge_score-0.1.2-py3-none-any.whl size=24933 sha256=6bb0d44e4881972c43ce194e7cb65233d309758cb15f0dec54590d3d2efcfc36\n", - " Stored in directory: /root/.cache/pip/wheels/5f/dd/89/461065a73be61a532ff8599a28e9beef17985c9e9c31e541b4\n", - " Building wheel for sqlitedict (setup.py) ... \u001b[?25l\u001b[?25hdone\n", - " Created wheel for sqlitedict: filename=sqlitedict-2.1.0-py3-none-any.whl size=16863 sha256=5747f7dd73ddf3d8fbcebf51b5e4f718fabe1e94bccdf16d2f22a2e65ee7fdf4\n", - " Stored in directory: /root/.cache/pip/wheels/79/d6/e7/304e0e6cb2221022c26d8161f7c23cd4f259a9e41e8bbcfabd\n", - "Successfully built lm-eval rouge-score sqlitedict\n", - "Installing collected packages: sqlitedict, zstandard, tcolorpy, pybind11, pyarrow-hotfix, portalocker, pathvalidate, mbstrdecoder, jsonlines, dill, colorama, typepy, tqdm-multiprocess, sacrebleu, rouge-score, responses, multiprocess, accelerate, datasets, DataProperty, tabledata, peft, evaluate, pytablewriter, lm-eval\n", - "Successfully installed DataProperty-1.0.1 accelerate-0.24.1 colorama-0.4.6 datasets-2.15.0 dill-0.3.7 evaluate-0.4.1 jsonlines-4.0.0 lm-eval-1.0.0 mbstrdecoder-1.1.3 multiprocess-0.70.15 pathvalidate-3.2.0 peft-0.6.2 portalocker-2.8.2 pyarrow-hotfix-0.6 pybind11-2.11.1 pytablewriter-1.2.0 responses-0.18.0 rouge-score-0.1.2 sacrebleu-2.3.2 sqlitedict-2.1.0 tabledata-1.3.3 tcolorpy-0.1.4 tqdm-multiprocess-0.0.11 typepy-1.3.2 zstandard-0.22.0\n" - ] - } - ], - "source": [ - "# Install LM-Eval\n", - "!pip install git+https://github.com/EleutherAI/lm-evaluation-harness.git@big-refactor" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 0, - "referenced_widgets": [ - "a1d3a8aa016544a78e8821c8f6199e06", - "f61ed33fad754146bdd2ac9db1ba1c48", - "bfa0af6aeff344c6845e1080a878e92e", - "fd1ad9e0367d4004aae853b91c3a7617", - "6b2d90209ec14230b3d58a74ac9b83bf", - "a73f357065d34d7baf0453ae4a8d75e2", - "46f521b73fd943c081c648fd873ebc0a", - "7c5689bc13684db8a22681f41863dddd", - "48763b6233374554ae76035c0483066f", - "4986a21eb560448fa79f4b25cde48951", - "aed3acd2f2d74003b44079c333a0698e" - ] - }, - "id": "uyO5MaKkZyah", - "outputId": "d46e8096-5086-4e49-967e-ea33d4a2a335" - }, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "a1d3a8aa016544a78e8821c8f6199e06", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Downloading builder script: 0%| | 0.00/5.67k [00:00\n", - "\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "id": "fthNg3ywO-kA" - }, - "outputs": [], - "source": [ - "YAML_cola_string = '''\n", - "group: yes_or_no_tasks\n", - "task: demo_cola\n", - "dataset_path: glue\n", - "dataset_name: cola\n", - "output_type: multiple_choice\n", - "training_split: train\n", - "validation_split: validation\n", - "doc_to_text: \"{{sentence}}\\nQuestion: Does this sentence make sense?\\nAnswer:\"\n", - "doc_to_target: label\n", - "doc_to_choice: [\"no\", \"yes\"]\n", - "should_decontaminate: true\n", - "doc_to_decontamination_query: sentence\n", - "metric_list:\n", - " - metric: acc\n", - "'''\n", - "with open('cola.yaml', 'w') as f:\n", - " f.write(YAML_cola_string)" - ] + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "Qw83KAePAhaS" + }, + "source": [ + "# Releasing LM-Evaluation-Harness v0.4.0" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Z7k2vq1iAdqr" + }, + "source": [ + "With the vast amount of work done in the field today, it helps to have a tool that people can use easily to share their results and use to check others to ensure reported numbers are valid. The LM Evaluation Harness is one such tool the community has used extensively. We want to continue to support the community and with that in mind, we’re excited to announce a major update on the LM Evaluation Harness to further our goal for open and accessible AI research." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "0gDoM0AJAvEc" + }, + "source": [ + "Our refactor stems from our desires to make the following believed best practices easier to carry out. \n", + "\n", + "1. Never copy results from other papers\n", + "2. Always share your exact prompts\n", + "3. Always provide model outputs\n", + "4. Qualitatively review a small batch of outputs before running evaluation jobs at scale\n", + "\n", + "We also wanted to make the library a better experience to use and to contribute or design evaluations within. New features in the new release that serve this purpose include:\n", + "\n", + "1. Faster Evaluation Runtimes (accelerated data-parallel inference with HF Transformers + Accelerate, and commonly used or faster inference libraries such as vLLM and Llama-CPP)\n", + "2. Easier addition and sharing of new tasks (YAML-based task config formats, allowing single-file sharing of custom tasks)\n", + "3. More configurability, for more advanced workflows and easier operation with modifying prompts\n", + "4. Better logging of data at runtime and post-hoc" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "nnwsOpjda_YW" + }, + "source": [ + "In this notebook we will be going through a short tutorial on how things work." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "zAov81vTbL2K" + }, + "source": [ + "## Install LM-Eval" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "8hiosGzq_qZg", + "outputId": "6ab73e5e-1f54-417e-a388-07e0d870b132" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "id": "XceRKCuuDtbn" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2023-11-29:11:56:33,016 INFO [utils.py:160] NumExpr defaulting to 2 threads.\n", - "2023-11-29 11:56:33.852995: E tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:9342] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", - "2023-11-29 11:56:33.853050: E tensorflow/compiler/xla/stream_executor/cuda/cuda_fft.cc:609] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", - "2023-11-29 11:56:33.853087: E tensorflow/compiler/xla/stream_executor/cuda/cuda_blas.cc:1518] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", - "2023-11-29 11:56:35.129047: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n", - "2023-11-29:11:56:38,546 INFO [__main__.py:132] Verbosity set to INFO\n", - "2023-11-29:11:56:47,509 WARNING [__main__.py:138] --limit SHOULD ONLY BE USED FOR TESTING.REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.\n", - "2023-11-29:11:56:47,509 INFO [__main__.py:143] Including path: ./\n", - "2023-11-29:11:56:47,517 INFO [__main__.py:205] Selected Tasks: ['yes_or_no_tasks']\n", - "2023-11-29:11:56:47,520 WARNING [evaluator.py:93] generation_kwargs specified through cli, these settings will be used over set parameters in yaml tasks.\n", - "2023-11-29:11:56:47,550 INFO [huggingface.py:120] Using device 'cuda'\n", - "2023-11-29:11:57:08,743 WARNING [task.py:614] [Task: demo_cola] metric acc is defined, but aggregation is not. using default aggregation=mean\n", - "2023-11-29:11:57:08,743 WARNING [task.py:626] [Task: demo_cola] metric acc is defined, but higher_is_better is not. using default higher_is_better=True\n", - "Downloading builder script: 100% 28.8k/28.8k [00:00<00:00, 52.7MB/s]\n", - "Downloading metadata: 100% 28.7k/28.7k [00:00<00:00, 51.9MB/s]\n", - "Downloading readme: 100% 27.9k/27.9k [00:00<00:00, 48.0MB/s]\n", - "Downloading data: 100% 377k/377k [00:00<00:00, 12.0MB/s]\n", - "Generating train split: 100% 8551/8551 [00:00<00:00, 19744.58 examples/s]\n", - "Generating validation split: 100% 1043/1043 [00:00<00:00, 27057.01 examples/s]\n", - "Generating test split: 100% 1063/1063 [00:00<00:00, 22705.17 examples/s]\n", - "2023-11-29:11:57:11,698 INFO [task.py:355] Building contexts for task on rank 0...\n", - "2023-11-29:11:57:11,704 INFO [evaluator.py:319] Running loglikelihood requests\n", - "100% 20/20 [00:03<00:00, 5.15it/s]\n", - "fatal: not a git repository (or any of the parent directories): .git\n", - "hf (pretrained=EleutherAI/pythia-2.8b), gen_kwargs: (), limit: 10.0, num_fewshot: None, batch_size: 1\n", - "| Tasks |Version|Filter|n-shot|Metric|Value| |Stderr|\n", - "|---------------|-------|------|-----:|------|----:|---|-----:|\n", - "|yes_or_no_tasks|N/A |none | 0|acc | 0.7|± |0.1528|\n", - "| - demo_cola |Yaml |none | 0|acc | 0.7|± |0.1528|\n", - "\n", - "| Groups |Version|Filter|n-shot|Metric|Value| |Stderr|\n", - "|---------------|-------|------|-----:|------|----:|---|-----:|\n", - "|yes_or_no_tasks|N/A |none | 0|acc | 0.7|± |0.1528|\n", - "\n" - ] - } - ], - "source": [ - "# !accelerate launch --no_python\n", - "!lm_eval \\\n", - " --model hf \\\n", - " --model_args pretrained=EleutherAI/pythia-2.8b \\\n", - " --include_path ./ \\\n", - " --tasks yes_or_no_tasks \\\n", - " --limit 10 \\\n", - " --output output/yes_or_no_tasks/ \\\n", - " --log_samples\n" - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "Collecting git+https://github.com/EleutherAI/lm-evaluation-harness.git@big-refactor\n", + " Cloning https://github.com/EleutherAI/lm-evaluation-harness.git (to revision big-refactor) to /tmp/pip-req-build-tnssql5s\n", + " Running command git clone --filter=blob:none --quiet https://github.com/EleutherAI/lm-evaluation-harness.git /tmp/pip-req-build-tnssql5s\n", + " Running command git checkout -b big-refactor --track origin/big-refactor\n", + " Switched to a new branch 'big-refactor'\n", + " Branch 'big-refactor' set up to track remote branch 'big-refactor' from 'origin'.\n", + " Resolved https://github.com/EleutherAI/lm-evaluation-harness.git to commit 42f486ee49b65926a444cb0620870a39a5b4b0a8\n", + " Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n", + " Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n", + " Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", + "Collecting accelerate>=0.21.0 (from lm-eval==1.0.0)\n", + " Downloading accelerate-0.24.1-py3-none-any.whl (261 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m261.4/261.4 kB\u001b[0m \u001b[31m4.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting evaluate (from lm-eval==1.0.0)\n", + " Downloading evaluate-0.4.1-py3-none-any.whl (84 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m84.1/84.1 kB\u001b[0m \u001b[31m5.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting datasets>=2.0.0 (from lm-eval==1.0.0)\n", + " Downloading datasets-2.15.0-py3-none-any.whl (521 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m521.2/521.2 kB\u001b[0m \u001b[31m9.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting jsonlines (from lm-eval==1.0.0)\n", + " Downloading jsonlines-4.0.0-py3-none-any.whl (8.7 kB)\n", + "Requirement already satisfied: numexpr in /usr/local/lib/python3.10/dist-packages (from lm-eval==1.0.0) (2.8.7)\n", + "Collecting peft>=0.2.0 (from lm-eval==1.0.0)\n", + " Downloading peft-0.6.2-py3-none-any.whl (174 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m174.7/174.7 kB\u001b[0m \u001b[31m7.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting pybind11>=2.6.2 (from lm-eval==1.0.0)\n", + " Downloading pybind11-2.11.1-py3-none-any.whl (227 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m227.7/227.7 kB\u001b[0m \u001b[31m12.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting pytablewriter (from lm-eval==1.0.0)\n", + " Downloading pytablewriter-1.2.0-py3-none-any.whl (111 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m111.1/111.1 kB\u001b[0m \u001b[31m8.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hCollecting rouge-score>=0.0.4 (from lm-eval==1.0.0)\n", + " Downloading rouge_score-0.1.2.tar.gz (17 kB)\n", + " Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + "Collecting sacrebleu>=1.5.0 (from lm-eval==1.0.0)\n", + " Downloading sacrebleu-2.3.2-py3-none-any.whl (119 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m119.7/119.7 kB\u001b[0m \u001b[31m8.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: scikit-learn>=0.24.1 in /usr/local/lib/python3.10/dist-packages (from lm-eval==1.0.0) (1.2.2)\n", + "Collecting sqlitedict (from lm-eval==1.0.0)\n", + " Downloading sqlitedict-2.1.0.tar.gz (21 kB)\n", + " Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + "Requirement already satisfied: torch>=1.8 in /usr/local/lib/python3.10/dist-packages (from lm-eval==1.0.0) (2.1.0+cu118)\n", + "Collecting tqdm-multiprocess (from lm-eval==1.0.0)\n", + " Downloading tqdm_multiprocess-0.0.11-py3-none-any.whl (9.8 kB)\n", + "Requirement already satisfied: transformers>=4.1 in /usr/local/lib/python3.10/dist-packages (from lm-eval==1.0.0) (4.35.2)\n", + "Collecting zstandard (from lm-eval==1.0.0)\n", + " Downloading zstandard-0.22.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (5.4 MB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.4/5.4 MB\u001b[0m \u001b[31m29.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from accelerate>=0.21.0->lm-eval==1.0.0) (1.23.5)\n", + "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from accelerate>=0.21.0->lm-eval==1.0.0) (23.2)\n", + "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate>=0.21.0->lm-eval==1.0.0) (5.9.5)\n", + "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from accelerate>=0.21.0->lm-eval==1.0.0) (6.0.1)\n", + "Requirement already satisfied: huggingface-hub in /usr/local/lib/python3.10/dist-packages (from accelerate>=0.21.0->lm-eval==1.0.0) (0.19.4)\n", + "Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.0.0->lm-eval==1.0.0) (9.0.0)\n", + "Collecting pyarrow-hotfix (from datasets>=2.0.0->lm-eval==1.0.0)\n", + " Downloading pyarrow_hotfix-0.6-py3-none-any.whl (7.9 kB)\n", + "Collecting dill<0.3.8,>=0.3.0 (from datasets>=2.0.0->lm-eval==1.0.0)\n", + " Downloading dill-0.3.7-py3-none-any.whl (115 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m115.3/115.3 kB\u001b[0m \u001b[31m14.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets>=2.0.0->lm-eval==1.0.0) (1.5.3)\n", + "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.0.0->lm-eval==1.0.0) (2.31.0)\n", + "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.0.0->lm-eval==1.0.0) (4.66.1)\n", + "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from datasets>=2.0.0->lm-eval==1.0.0) (3.4.1)\n", + "Collecting multiprocess (from datasets>=2.0.0->lm-eval==1.0.0)\n", + " Downloading multiprocess-0.70.15-py310-none-any.whl (134 kB)\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m19.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25hRequirement already satisfied: fsspec[http]<=2023.10.0,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.0.0->lm-eval==1.0.0) (2023.6.0)\n", + "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets>=2.0.0->lm-eval==1.0.0) (3.8.6)\n", + "Collecting responses<0.19 (from evaluate->lm-eval==1.0.0)\n", + " Downloading responses-0.18.0-py3-none-any.whl (38 kB)\n", + "Requirement already satisfied: safetensors in /usr/local/lib/python3.10/dist-packages (from peft>=0.2.0->lm-eval==1.0.0) (0.4.0)\n", + "Requirement already satisfied: absl-py in /usr/local/lib/python3.10/dist-packages (from rouge-score>=0.0.4->lm-eval==1.0.0) (1.4.0)\n", + "Requirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (from rouge-score>=0.0.4->lm-eval==1.0.0) (3.8.1)\n", + "Requirement already satisfied: six>=1.14.0 in /usr/local/lib/python3.10/dist-packages (from rouge-score>=0.0.4->lm-eval==1.0.0) (1.16.0)\n", + "Collecting portalocker (from sacrebleu>=1.5.0->lm-eval==1.0.0)\n", + " Downloading portalocker-2.8.2-py3-none-any.whl (17 kB)\n", + "Requirement already satisfied: regex in /usr/local/lib/python3.10/dist-packages (from sacrebleu>=1.5.0->lm-eval==1.0.0) (2023.6.3)\n", + "Requirement already satisfied: tabulate>=0.8.9 in /usr/local/lib/python3.10/dist-packages (from sacrebleu>=1.5.0->lm-eval==1.0.0) (0.9.0)\n", + "Collecting colorama (from sacrebleu>=1.5.0->lm-eval==1.0.0)\n", + " Downloading colorama-0.4.6-py2.py3-none-any.whl (25 kB)\n", + "Requirement already satisfied: lxml in /usr/local/lib/python3.10/dist-packages (from sacrebleu>=1.5.0->lm-eval==1.0.0) (4.9.3)\n", + "Requirement already satisfied: scipy>=1.3.2 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.24.1->lm-eval==1.0.0) (1.11.3)\n", + "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.24.1->lm-eval==1.0.0) (1.3.2)\n", + "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.24.1->lm-eval==1.0.0) (3.2.0)\n", + "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch>=1.8->lm-eval==1.0.0) (3.13.1)\n", + "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from torch>=1.8->lm-eval==1.0.0) (4.5.0)\n", + "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.8->lm-eval==1.0.0) (1.12)\n", + "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.8->lm-eval==1.0.0) (3.2.1)\n", + "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.8->lm-eval==1.0.0) (3.1.2)\n", + "Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.8->lm-eval==1.0.0) (2.1.0)\n", + "Requirement already satisfied: tokenizers<0.19,>=0.14 in /usr/local/lib/python3.10/dist-packages (from transformers>=4.1->lm-eval==1.0.0) (0.15.0)\n", + "Requirement already satisfied: attrs>=19.2.0 in /usr/local/lib/python3.10/dist-packages (from jsonlines->lm-eval==1.0.0) (23.1.0)\n", + "Requirement already satisfied: setuptools>=38.3.0 in /usr/local/lib/python3.10/dist-packages (from pytablewriter->lm-eval==1.0.0) (67.7.2)\n", + "Collecting DataProperty<2,>=1.0.1 (from pytablewriter->lm-eval==1.0.0)\n", + " Downloading DataProperty-1.0.1-py3-none-any.whl (27 kB)\n", + "Collecting mbstrdecoder<2,>=1.0.0 (from pytablewriter->lm-eval==1.0.0)\n", + " Downloading mbstrdecoder-1.1.3-py3-none-any.whl (7.8 kB)\n", + "Collecting pathvalidate<4,>=2.3.0 (from pytablewriter->lm-eval==1.0.0)\n", + " Downloading pathvalidate-3.2.0-py3-none-any.whl (23 kB)\n", + "Collecting tabledata<2,>=1.3.1 (from pytablewriter->lm-eval==1.0.0)\n", + " Downloading tabledata-1.3.3-py3-none-any.whl (11 kB)\n", + "Collecting tcolorpy<1,>=0.0.5 (from pytablewriter->lm-eval==1.0.0)\n", + " Downloading tcolorpy-0.1.4-py3-none-any.whl (7.9 kB)\n", + "Collecting typepy[datetime]<2,>=1.3.2 (from pytablewriter->lm-eval==1.0.0)\n", + " Downloading typepy-1.3.2-py3-none-any.whl (31 kB)\n", + "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.0.0->lm-eval==1.0.0) (3.3.2)\n", + "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.0.0->lm-eval==1.0.0) (6.0.4)\n", + "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.0.0->lm-eval==1.0.0) (4.0.3)\n", + "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.0.0->lm-eval==1.0.0) (1.9.2)\n", + "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.0.0->lm-eval==1.0.0) (1.4.0)\n", + "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.0.0->lm-eval==1.0.0) (1.3.1)\n", + "Requirement already satisfied: chardet<6,>=3.0.4 in /usr/local/lib/python3.10/dist-packages (from mbstrdecoder<2,>=1.0.0->pytablewriter->lm-eval==1.0.0) (5.2.0)\n", + "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets>=2.0.0->lm-eval==1.0.0) (3.4)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets>=2.0.0->lm-eval==1.0.0) (2.0.7)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets>=2.0.0->lm-eval==1.0.0) (2023.7.22)\n", + "Requirement already satisfied: python-dateutil<3.0.0,>=2.8.0 in /usr/local/lib/python3.10/dist-packages (from typepy[datetime]<2,>=1.3.2->pytablewriter->lm-eval==1.0.0) (2.8.2)\n", + "Requirement already satisfied: pytz>=2018.9 in /usr/local/lib/python3.10/dist-packages (from typepy[datetime]<2,>=1.3.2->pytablewriter->lm-eval==1.0.0) (2023.3.post1)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.8->lm-eval==1.0.0) (2.1.3)\n", + "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk->rouge-score>=0.0.4->lm-eval==1.0.0) (8.1.7)\n", + "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.8->lm-eval==1.0.0) (1.3.0)\n", + "Building wheels for collected packages: lm-eval, rouge-score, sqlitedict\n", + " Building wheel for lm-eval (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for lm-eval: filename=lm_eval-1.0.0-py3-none-any.whl size=994254 sha256=88356155b19f2891981ecef948326ad6ce8ca40a6009378410ec20d0e225995a\n", + " Stored in directory: /tmp/pip-ephem-wheel-cache-9v6ye7h3/wheels/17/01/26/599c0779e9858a70a73fa8a306699b5b9a868f820c225457b0\n", + " Building wheel for rouge-score (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for rouge-score: filename=rouge_score-0.1.2-py3-none-any.whl size=24933 sha256=6bb0d44e4881972c43ce194e7cb65233d309758cb15f0dec54590d3d2efcfc36\n", + " Stored in directory: /root/.cache/pip/wheels/5f/dd/89/461065a73be61a532ff8599a28e9beef17985c9e9c31e541b4\n", + " Building wheel for sqlitedict (setup.py) ... \u001b[?25l\u001b[?25hdone\n", + " Created wheel for sqlitedict: filename=sqlitedict-2.1.0-py3-none-any.whl size=16863 sha256=5747f7dd73ddf3d8fbcebf51b5e4f718fabe1e94bccdf16d2f22a2e65ee7fdf4\n", + " Stored in directory: /root/.cache/pip/wheels/79/d6/e7/304e0e6cb2221022c26d8161f7c23cd4f259a9e41e8bbcfabd\n", + "Successfully built lm-eval rouge-score sqlitedict\n", + "Installing collected packages: sqlitedict, zstandard, tcolorpy, pybind11, pyarrow-hotfix, portalocker, pathvalidate, mbstrdecoder, jsonlines, dill, colorama, typepy, tqdm-multiprocess, sacrebleu, rouge-score, responses, multiprocess, accelerate, datasets, DataProperty, tabledata, peft, evaluate, pytablewriter, lm-eval\n", + "Successfully installed DataProperty-1.0.1 accelerate-0.24.1 colorama-0.4.6 datasets-2.15.0 dill-0.3.7 evaluate-0.4.1 jsonlines-4.0.0 lm-eval-1.0.0 mbstrdecoder-1.1.3 multiprocess-0.70.15 pathvalidate-3.2.0 peft-0.6.2 portalocker-2.8.2 pyarrow-hotfix-0.6 pybind11-2.11.1 pytablewriter-1.2.0 responses-0.18.0 rouge-score-0.1.2 sacrebleu-2.3.2 sqlitedict-2.1.0 tabledata-1.3.3 tcolorpy-0.1.4 tqdm-multiprocess-0.0.11 typepy-1.3.2 zstandard-0.22.0\n" + ] + } + ], + "source": [ + "# Install LM-Eval\n", + "!pip install git+https://github.com/EleutherAI/lm-evaluation-harness.git" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 0, + "referenced_widgets": [ + "a1d3a8aa016544a78e8821c8f6199e06", + "f61ed33fad754146bdd2ac9db1ba1c48", + "bfa0af6aeff344c6845e1080a878e92e", + "fd1ad9e0367d4004aae853b91c3a7617", + "6b2d90209ec14230b3d58a74ac9b83bf", + "a73f357065d34d7baf0453ae4a8d75e2", + "46f521b73fd943c081c648fd873ebc0a", + "7c5689bc13684db8a22681f41863dddd", + "48763b6233374554ae76035c0483066f", + "4986a21eb560448fa79f4b25cde48951", + "aed3acd2f2d74003b44079c333a0698e" + ] }, + "id": "uyO5MaKkZyah", + "outputId": "d46e8096-5086-4e49-967e-ea33d4a2a335" + }, + "outputs": [ { - "cell_type": "markdown", - "metadata": { - "id": "XceRKCuuDtbn" + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "a1d3a8aa016544a78e8821c8f6199e06", + "version_major": 2, + "version_minor": 0 }, - "source": [ - "## Edit Prompt Templates Quickly\n", - "\n", - "The following is a yaml made to evaluate the specific subtask of `high_school_geography` from MMLU. It uses the standard prompt where the we choose the letters from the options with most likelihood as the model's prediction." + "text/plain": [ + "Downloading builder script: 0%| | 0.00/5.67k [00:00\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "id": "fthNg3ywO-kA" + }, + "outputs": [], + "source": [ + "YAML_cola_string = \"\"\"\n", + "tag: yes_or_no_tasks\n", + "task: demo_cola\n", + "dataset_path: glue\n", + "dataset_name: cola\n", + "output_type: multiple_choice\n", + "training_split: train\n", + "validation_split: validation\n", + "doc_to_text: \"{{sentence}}\\nQuestion: Does this sentence make sense?\\nAnswer:\"\n", + "doc_to_target: label\n", + "doc_to_choice: [\"no\", \"yes\"]\n", + "should_decontaminate: true\n", + "doc_to_decontamination_query: sentence\n", + "metric_list:\n", + " - metric: acc\n", + "\"\"\"\n", + "with open(\"cola.yaml\", \"w\") as f:\n", + " f.write(YAML_cola_string)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "id": "XceRKCuuDtbn" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "id": "jyKOfCsKb-xy" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2023-11-29:11:57:23,598 INFO [utils.py:160] NumExpr defaulting to 2 threads.\n", - "2023-11-29 11:57:24.719750: E tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:9342] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", - "2023-11-29 11:57:24.719806: E tensorflow/compiler/xla/stream_executor/cuda/cuda_fft.cc:609] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", - "2023-11-29 11:57:24.719847: E tensorflow/compiler/xla/stream_executor/cuda/cuda_blas.cc:1518] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", - "2023-11-29 11:57:26.656125: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n", - "2023-11-29:11:57:31,563 INFO [__main__.py:132] Verbosity set to INFO\n", - "2023-11-29:11:57:40,541 WARNING [__main__.py:138] --limit SHOULD ONLY BE USED FOR TESTING.REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.\n", - "2023-11-29:11:57:40,541 INFO [__main__.py:143] Including path: ./\n", - "2023-11-29:11:57:40,558 INFO [__main__.py:205] Selected Tasks: ['demo_mmlu_high_school_geography']\n", - "2023-11-29:11:57:40,559 WARNING [evaluator.py:93] generation_kwargs specified through cli, these settings will be used over set parameters in yaml tasks.\n", - "2023-11-29:11:57:40,589 INFO [huggingface.py:120] Using device 'cuda'\n", - "Downloading builder script: 100% 5.84k/5.84k [00:00<00:00, 17.7MB/s]\n", - "Downloading metadata: 100% 106k/106k [00:00<00:00, 892kB/s] \n", - "Downloading readme: 100% 39.7k/39.7k [00:00<00:00, 631kB/s]\n", - "Downloading data: 100% 166M/166M [00:01<00:00, 89.0MB/s]\n", - "Generating auxiliary_train split: 100% 99842/99842 [00:07<00:00, 12536.83 examples/s]\n", - "Generating test split: 100% 198/198 [00:00<00:00, 1439.20 examples/s]\n", - "Generating validation split: 100% 22/22 [00:00<00:00, 4181.76 examples/s]\n", - "Generating dev split: 100% 5/5 [00:00<00:00, 36.25 examples/s]\n", - "2023-11-29:11:58:09,798 INFO [task.py:355] Building contexts for task on rank 0...\n", - "2023-11-29:11:58:09,822 INFO [evaluator.py:319] Running loglikelihood requests\n", - "100% 40/40 [00:05<00:00, 7.86it/s]\n", - "fatal: not a git repository (or any of the parent directories): .git\n", - "hf (pretrained=EleutherAI/pythia-2.8b), gen_kwargs: (), limit: 10.0, num_fewshot: None, batch_size: 1\n", - "| Tasks |Version|Filter|n-shot| Metric |Value| |Stderr|\n", - "|-------------------------------|-------|------|-----:|--------|----:|---|-----:|\n", - "|demo_mmlu_high_school_geography|Yaml |none | 0|acc | 0.3|± |0.1528|\n", - "| | |none | 0|acc_norm| 0.3|± |0.1528|\n", - "\n" - ] - } - ], - "source": [ - "# !accelerate launch --no_python\n", - "!lm_eval \\\n", - " --model hf \\\n", - " --model_args pretrained=EleutherAI/pythia-2.8b \\\n", - " --include_path ./ \\\n", - " --tasks demo_mmlu_high_school_geography \\\n", - " --limit 10 \\\n", - " --output output/mmlu_high_school_geography/ \\\n", - " --log_samples" - ] - }, + "name": "stdout", + "output_type": "stream", + "text": [ + "2023-11-29:11:56:33,016 INFO [utils.py:160] NumExpr defaulting to 2 threads.\n", + "2023-11-29 11:56:33.852995: E tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:9342] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2023-11-29 11:56:33.853050: E tensorflow/compiler/xla/stream_executor/cuda/cuda_fft.cc:609] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2023-11-29 11:56:33.853087: E tensorflow/compiler/xla/stream_executor/cuda/cuda_blas.cc:1518] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", + "2023-11-29 11:56:35.129047: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n", + "2023-11-29:11:56:38,546 INFO [__main__.py:132] Verbosity set to INFO\n", + "2023-11-29:11:56:47,509 WARNING [__main__.py:138] --limit SHOULD ONLY BE USED FOR TESTING.REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.\n", + "2023-11-29:11:56:47,509 INFO [__main__.py:143] Including path: ./\n", + "2023-11-29:11:56:47,517 INFO [__main__.py:205] Selected Tasks: ['yes_or_no_tasks']\n", + "2023-11-29:11:56:47,520 WARNING [evaluator.py:93] generation_kwargs specified through cli, these settings will be used over set parameters in yaml tasks.\n", + "2023-11-29:11:56:47,550 INFO [huggingface.py:120] Using device 'cuda'\n", + "2023-11-29:11:57:08,743 WARNING [task.py:614] [Task: demo_cola] metric acc is defined, but aggregation is not. using default aggregation=mean\n", + "2023-11-29:11:57:08,743 WARNING [task.py:626] [Task: demo_cola] metric acc is defined, but higher_is_better is not. using default higher_is_better=True\n", + "Downloading builder script: 100% 28.8k/28.8k [00:00<00:00, 52.7MB/s]\n", + "Downloading metadata: 100% 28.7k/28.7k [00:00<00:00, 51.9MB/s]\n", + "Downloading readme: 100% 27.9k/27.9k [00:00<00:00, 48.0MB/s]\n", + "Downloading data: 100% 377k/377k [00:00<00:00, 12.0MB/s]\n", + "Generating train split: 100% 8551/8551 [00:00<00:00, 19744.58 examples/s]\n", + "Generating validation split: 100% 1043/1043 [00:00<00:00, 27057.01 examples/s]\n", + "Generating test split: 100% 1063/1063 [00:00<00:00, 22705.17 examples/s]\n", + "2023-11-29:11:57:11,698 INFO [task.py:355] Building contexts for task on rank 0...\n", + "2023-11-29:11:57:11,704 INFO [evaluator.py:319] Running loglikelihood requests\n", + "100% 20/20 [00:03<00:00, 5.15it/s]\n", + "fatal: not a git repository (or any of the parent directories): .git\n", + "hf (pretrained=EleutherAI/pythia-2.8b), gen_kwargs: (), limit: 10.0, num_fewshot: None, batch_size: 1\n", + "| Tasks |Version|Filter|n-shot|Metric|Value| |Stderr|\n", + "|---------------|-------|------|-----:|------|----:|---|-----:|\n", + "|yes_or_no_tasks|N/A |none | 0|acc | 0.7|± |0.1528|\n", + "| - demo_cola |Yaml |none | 0|acc | 0.7|± |0.1528|\n", + "\n", + "| Groups |Version|Filter|n-shot|Metric|Value| |Stderr|\n", + "|---------------|-------|------|-----:|------|----:|---|-----:|\n", + "|yes_or_no_tasks|N/A |none | 0|acc | 0.7|± |0.1528|\n", + "\n" + ] + } + ], + "source": [ + "# !accelerate launch --no_python\n", + "!lm_eval \\\n", + " --model hf \\\n", + " --model_args pretrained=EleutherAI/pythia-2.8b \\\n", + " --include_path ./ \\\n", + " --tasks yes_or_no_tasks \\\n", + " --limit 10 \\\n", + " --output output/yes_or_no_tasks/ \\\n", + " --log_samples" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "XceRKCuuDtbn" + }, + "source": [ + "## Edit Prompt Templates Quickly\n", + "\n", + "The following is a yaml made to evaluate the specific subtask of `high_school_geography` from MMLU. It uses the standard prompt where the we choose the letters from the options with most likelihood as the model's prediction." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "id": "GTFvdt9kSlBG" + }, + "outputs": [], + "source": [ + "YAML_mmlu_geo_string = \"\"\"\n", + "task: demo_mmlu_high_school_geography\n", + "dataset_path: cais/mmlu\n", + "dataset_name: high_school_geography\n", + "description: \"The following are multiple choice questions (with answers) about high school geography.\\n\\n\"\n", + "test_split: test\n", + "fewshot_split: dev\n", + "fewshot_config:\n", + " sampler: first_n\n", + "output_type: multiple_choice\n", + "doc_to_text: \"{{question.strip()}}\\nA. {{choices[0]}}\\nB. {{choices[1]}}\\nC. {{choices[2]}}\\nD. {{choices[3]}}\\nAnswer:\"\n", + "doc_to_choice: [\"A\", \"B\", \"C\", \"D\"]\n", + "doc_to_target: answer\n", + "metric_list:\n", + " - metric: acc\n", + " aggregation: mean\n", + " higher_is_better: true\n", + " - metric: acc_norm\n", + " aggregation: mean\n", + " higher_is_better: true\n", + "\"\"\"\n", + "with open(\"mmlu_high_school_geography.yaml\", \"w\") as f:\n", + " f.write(YAML_mmlu_geo_string)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "id": "jyKOfCsKb-xy" + }, + "outputs": [ { - "cell_type": "markdown", - "metadata": { - "id": "jyKOfCsKb-xy" - }, - "source": [ - "We could also evaluate this task in a different way. For example, instead of observing the loglikelihood of the letters, we can instead evaluate on the choices themselves as the continuation. This is done by simply changing `doc_to_choice` from a list of letters to the corresponding `choices` field from the HF dataset. We write `\"{{choices}}\"` so that the string field is interpreted as jinja string that acquires the list from the HF dataset directly.\n", - "\n", - "Another convenient feature here is since we're only modifying the `doc_to_choice` and the rest of config is the same as the task above, we can use the above configuration as a template by using `include: mmlu_high_school_geography.yaml` to load the config from that file. We'll need to add a unique task name as to not colide with the existing yaml config we're including. For this case we'll simply name this one `mmlu_high_school_geography_continuation`. `doc_to_text` is added here just for sake of clarity." - ] - }, + "name": "stdout", + "output_type": "stream", + "text": [ + "2023-11-29:11:57:23,598 INFO [utils.py:160] NumExpr defaulting to 2 threads.\n", + "2023-11-29 11:57:24.719750: E tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:9342] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2023-11-29 11:57:24.719806: E tensorflow/compiler/xla/stream_executor/cuda/cuda_fft.cc:609] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2023-11-29 11:57:24.719847: E tensorflow/compiler/xla/stream_executor/cuda/cuda_blas.cc:1518] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", + "2023-11-29 11:57:26.656125: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n", + "2023-11-29:11:57:31,563 INFO [__main__.py:132] Verbosity set to INFO\n", + "2023-11-29:11:57:40,541 WARNING [__main__.py:138] --limit SHOULD ONLY BE USED FOR TESTING.REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.\n", + "2023-11-29:11:57:40,541 INFO [__main__.py:143] Including path: ./\n", + "2023-11-29:11:57:40,558 INFO [__main__.py:205] Selected Tasks: ['demo_mmlu_high_school_geography']\n", + "2023-11-29:11:57:40,559 WARNING [evaluator.py:93] generation_kwargs specified through cli, these settings will be used over set parameters in yaml tasks.\n", + "2023-11-29:11:57:40,589 INFO [huggingface.py:120] Using device 'cuda'\n", + "Downloading builder script: 100% 5.84k/5.84k [00:00<00:00, 17.7MB/s]\n", + "Downloading metadata: 100% 106k/106k [00:00<00:00, 892kB/s] \n", + "Downloading readme: 100% 39.7k/39.7k [00:00<00:00, 631kB/s]\n", + "Downloading data: 100% 166M/166M [00:01<00:00, 89.0MB/s]\n", + "Generating auxiliary_train split: 100% 99842/99842 [00:07<00:00, 12536.83 examples/s]\n", + "Generating test split: 100% 198/198 [00:00<00:00, 1439.20 examples/s]\n", + "Generating validation split: 100% 22/22 [00:00<00:00, 4181.76 examples/s]\n", + "Generating dev split: 100% 5/5 [00:00<00:00, 36.25 examples/s]\n", + "2023-11-29:11:58:09,798 INFO [task.py:355] Building contexts for task on rank 0...\n", + "2023-11-29:11:58:09,822 INFO [evaluator.py:319] Running loglikelihood requests\n", + "100% 40/40 [00:05<00:00, 7.86it/s]\n", + "fatal: not a git repository (or any of the parent directories): .git\n", + "hf (pretrained=EleutherAI/pythia-2.8b), gen_kwargs: (), limit: 10.0, num_fewshot: None, batch_size: 1\n", + "| Tasks |Version|Filter|n-shot| Metric |Value| |Stderr|\n", + "|-------------------------------|-------|------|-----:|--------|----:|---|-----:|\n", + "|demo_mmlu_high_school_geography|Yaml |none | 0|acc | 0.3|± |0.1528|\n", + "| | |none | 0|acc_norm| 0.3|± |0.1528|\n", + "\n" + ] + } + ], + "source": [ + "# !accelerate launch --no_python\n", + "!lm_eval \\\n", + " --model hf \\\n", + " --model_args pretrained=EleutherAI/pythia-2.8b \\\n", + " --include_path ./ \\\n", + " --tasks demo_mmlu_high_school_geography \\\n", + " --limit 10 \\\n", + " --output output/mmlu_high_school_geography/ \\\n", + " --log_samples" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "jyKOfCsKb-xy" + }, + "source": [ + "We could also evaluate this task in a different way. For example, instead of observing the loglikelihood of the letters, we can instead evaluate on the choices themselves as the continuation. This is done by simply changing `doc_to_choice` from a list of letters to the corresponding `choices` field from the HF dataset. We write `\"{{choices}}\"` so that the string field is interpreted as jinja string that acquires the list from the HF dataset directly.\n", + "\n", + "Another convenient feature here is since we're only modifying the `doc_to_choice` and the rest of config is the same as the task above, we can use the above configuration as a template by using `include: mmlu_high_school_geography.yaml` to load the config from that file. We'll need to add a unique task name as to not colide with the existing yaml config we're including. For this case we'll simply name this one `mmlu_high_school_geography_continuation`. `doc_to_text` is added here just for sake of clarity." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "id": "lqElwU54TaK-" + }, + "outputs": [], + "source": [ + "YAML_mmlu_geo_string = \"\"\"\n", + "include: mmlu_high_school_geography.yaml\n", + "task: demo_mmlu_high_school_geography_continuation\n", + "doc_to_text: \"{{question.strip()}}\\nA. {{choices[0]}}\\nB. {{choices[1]}}\\nC. {{choices[2]}}\\nD. {{choices[3]}}\\nAnswer:\"\n", + "doc_to_choice: \"{{choices}}\"\n", + "\"\"\"\n", + "with open(\"mmlu_high_school_geography_continuation.yaml\", \"w\") as f:\n", + " f.write(YAML_mmlu_geo_string)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "id": "-_CVnDirdy7j" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "id": "lqElwU54TaK-" - }, - "outputs": [], - "source": [ - "YAML_mmlu_geo_string = '''\n", - "include: mmlu_high_school_geography.yaml\n", - "task: demo_mmlu_high_school_geography_continuation\n", - "doc_to_text: \"{{question.strip()}}\\nA. {{choices[0]}}\\nB. {{choices[1]}}\\nC. {{choices[2]}}\\nD. {{choices[3]}}\\nAnswer:\"\n", - "doc_to_choice: \"{{choices}}\"\n", - "'''\n", - "with open('mmlu_high_school_geography_continuation.yaml', 'w') as f:\n", - " f.write(YAML_mmlu_geo_string)\n" - ] - }, + "name": "stdout", + "output_type": "stream", + "text": [ + "2023-11-29:11:58:21,284 INFO [utils.py:160] NumExpr defaulting to 2 threads.\n", + "2023-11-29 11:58:22.850159: E tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:9342] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2023-11-29 11:58:22.850219: E tensorflow/compiler/xla/stream_executor/cuda/cuda_fft.cc:609] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2023-11-29 11:58:22.850254: E tensorflow/compiler/xla/stream_executor/cuda/cuda_blas.cc:1518] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", + "2023-11-29 11:58:24.948103: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n", + "2023-11-29:11:58:28,460 INFO [__main__.py:132] Verbosity set to INFO\n", + "2023-11-29:11:58:37,935 WARNING [__main__.py:138] --limit SHOULD ONLY BE USED FOR TESTING.REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.\n", + "2023-11-29:11:58:37,935 INFO [__main__.py:143] Including path: ./\n", + "2023-11-29:11:58:37,969 INFO [__main__.py:205] Selected Tasks: ['demo_mmlu_high_school_geography_continuation']\n", + "2023-11-29:11:58:37,972 WARNING [evaluator.py:93] generation_kwargs specified through cli, these settings will be used over set parameters in yaml tasks.\n", + "2023-11-29:11:58:38,008 INFO [huggingface.py:120] Using device 'cuda'\n", + "2023-11-29:11:58:59,758 INFO [task.py:355] Building contexts for task on rank 0...\n", + "2023-11-29:11:58:59,777 INFO [evaluator.py:319] Running loglikelihood requests\n", + "100% 40/40 [00:02<00:00, 16.23it/s]\n", + "fatal: not a git repository (or any of the parent directories): .git\n", + "hf (pretrained=EleutherAI/pythia-2.8b), gen_kwargs: (), limit: 10.0, num_fewshot: None, batch_size: 1\n", + "| Tasks |Version|Filter|n-shot| Metric |Value| |Stderr|\n", + "|--------------------------------------------|-------|------|-----:|--------|----:|---|-----:|\n", + "|demo_mmlu_high_school_geography_continuation|Yaml |none | 0|acc | 0.1|± |0.1000|\n", + "| | |none | 0|acc_norm| 0.2|± |0.1333|\n", + "\n" + ] + } + ], + "source": [ + "# !accelerate launch --no_python\n", + "!lm_eval \\\n", + " --model hf \\\n", + " --model_args pretrained=EleutherAI/pythia-2.8b \\\n", + " --include_path ./ \\\n", + " --tasks demo_mmlu_high_school_geography_continuation \\\n", + " --limit 10 \\\n", + " --output output/mmlu_high_school_geography_continuation/ \\\n", + " --log_samples" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "-_CVnDirdy7j" + }, + "source": [ + "If we take a look at the samples, we can see that it is in fact evaluating the continuation based on the choices rather than the letters." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "id": "duBDqC6PAdjL" + }, + "outputs": [ { - "cell_type": "code", - "execution_count": 10, - "metadata": { - "id": "-_CVnDirdy7j" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2023-11-29:11:58:21,284 INFO [utils.py:160] NumExpr defaulting to 2 threads.\n", - "2023-11-29 11:58:22.850159: E tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:9342] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", - "2023-11-29 11:58:22.850219: E tensorflow/compiler/xla/stream_executor/cuda/cuda_fft.cc:609] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", - "2023-11-29 11:58:22.850254: E tensorflow/compiler/xla/stream_executor/cuda/cuda_blas.cc:1518] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", - "2023-11-29 11:58:24.948103: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n", - "2023-11-29:11:58:28,460 INFO [__main__.py:132] Verbosity set to INFO\n", - "2023-11-29:11:58:37,935 WARNING [__main__.py:138] --limit SHOULD ONLY BE USED FOR TESTING.REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.\n", - "2023-11-29:11:58:37,935 INFO [__main__.py:143] Including path: ./\n", - "2023-11-29:11:58:37,969 INFO [__main__.py:205] Selected Tasks: ['demo_mmlu_high_school_geography_continuation']\n", - "2023-11-29:11:58:37,972 WARNING [evaluator.py:93] generation_kwargs specified through cli, these settings will be used over set parameters in yaml tasks.\n", - "2023-11-29:11:58:38,008 INFO [huggingface.py:120] Using device 'cuda'\n", - "2023-11-29:11:58:59,758 INFO [task.py:355] Building contexts for task on rank 0...\n", - "2023-11-29:11:58:59,777 INFO [evaluator.py:319] Running loglikelihood requests\n", - "100% 40/40 [00:02<00:00, 16.23it/s]\n", - "fatal: not a git repository (or any of the parent directories): .git\n", - "hf (pretrained=EleutherAI/pythia-2.8b), gen_kwargs: (), limit: 10.0, num_fewshot: None, batch_size: 1\n", - "| Tasks |Version|Filter|n-shot| Metric |Value| |Stderr|\n", - "|--------------------------------------------|-------|------|-----:|--------|----:|---|-----:|\n", - "|demo_mmlu_high_school_geography_continuation|Yaml |none | 0|acc | 0.1|± |0.1000|\n", - "| | |none | 0|acc_norm| 0.2|± |0.1333|\n", - "\n" - ] - } - ], - "source": [ - "# !accelerate launch --no_python\n", - "!lm_eval \\\n", - " --model hf \\\n", - " --model_args pretrained=EleutherAI/pythia-2.8b \\\n", - " --include_path ./ \\\n", - " --tasks demo_mmlu_high_school_geography_continuation \\\n", - " --limit 10 \\\n", - " --output output/mmlu_high_school_geography_continuation/ \\\n", - " --log_samples\n" + "data": { + "application/javascript": "\n ((filepath) => {{\n if (!google.colab.kernel.accessAllowed) {{\n return;\n }}\n google.colab.files.view(filepath);\n }})(\"/content/output/mmlu_high_school_geography_continuation/pretrained__EleutherAI__pythia-2.8b_demo_mmlu_high_school_geography_continuation.jsonl\")", + "text/plain": [ + "" ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from google.colab import files\n", + "\n", + "\n", + "files.view(\n", + " \"output/mmlu_high_school_geography_continuation/pretrained__EleutherAI__pythia-2.8b_demo_mmlu_high_school_geography_continuation.jsonl\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "6p0-KPwAgK5j" + }, + "source": [ + "## Closer Look at YAML Fields\n", + "\n", + "To prepare a task we can simply fill in a YAML config with the relevant information.\n", + "\n", + "`output_type`\n", + "The current provided evaluation types comprise of the following:\n", + "1. `loglikelihood`: Evaluates the loglikelihood of a continuation, conditioned on some input string.\n", + "2. `loglikelihood_rolling`: evaluate the loglikelihood of producing a string, conditioned on the empty string. (Used for perplexity evaluations)\n", + "3. `multiple_choice`: Evaluates loglikelihood among the a number of choices predicted by the model.\n", + "4. `greedy_until`: Model outputs greedy generation (can be configured to to use beam search and other generation-related parameters)\n", + "\n", + "The core prompt revolves around 3 fields.\n", + "1. `doc_to_text`: Denotes the prompt template that will be used as input to the model.\n", + "2. `doc_to_choice`: Available choices that will be used as continuation for the model. This is used when the `output_type` is `multiple_choice`, and otherwise can be left as `None`.\n", + "3. `doc_to_target`: When `output_type` is `multiple_choice`, this can be an index that corresponds to the correct answer, or the answer string itself (must be a subset of `doc_to_choice`). For other tasks, this is expected to be a string. You can fill this field with a feature name from the HF dataset so long as the resulting feature follows the conditioned described.\n", + "\n", + "These three fields can be expressed as strings, column names from the source dataset, or as Jinja2 templates that can use fields from the source dataset as variables.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "6p0-KPwAgK5j" + }, + "source": [ + "## What if Jinja is not Sufficient?\n", + "\n", + "There can be times where the Jinja2 templating language is not enough to make the prompt we had in mind. There are a few ways to circumvent this limitation:\n", + "\n", + "1. Use `!function` operator for the prompt-related fields to pass a python function that takes as input the dataset row, and will output the prompt template component.\n", + "2. Perform a transformation on the dataset beforehand." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Below, we show an example of using `!function` to create `doc_to_text` from a python function:" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" }, + "id": "DYZ5c0JhR1lJ", + "outputId": "ca945235-fb9e-4f17-8bfa-78e7d6ec1490" + }, + "outputs": [ { - "cell_type": "markdown", - "metadata": { - "id": "-_CVnDirdy7j" - }, - "source": [ - "If we take a look at the samples, we can see that it is in fact evaluating the continuation based on the choices rather than the letters." - ] + "name": "stdout", + "output_type": "stream", + "text": [ + "2023-11-29:11:59:08,312 INFO [utils.py:160] NumExpr defaulting to 2 threads.\n", + "2023-11-29 11:59:09.348327: E tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:9342] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2023-11-29 11:59:09.348387: E tensorflow/compiler/xla/stream_executor/cuda/cuda_fft.cc:609] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2023-11-29 11:59:09.348421: E tensorflow/compiler/xla/stream_executor/cuda/cuda_blas.cc:1518] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", + "2023-11-29 11:59:10.573752: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n", + "2023-11-29:11:59:14,044 INFO [__main__.py:132] Verbosity set to INFO\n", + "2023-11-29:11:59:23,654 WARNING [__main__.py:138] --limit SHOULD ONLY BE USED FOR TESTING.REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.\n", + "2023-11-29:11:59:23,654 INFO [__main__.py:143] Including path: ./\n", + "2023-11-29:11:59:23,678 INFO [__main__.py:205] Selected Tasks: ['demo_mmlu_high_school_geography_function_prompt']\n", + "2023-11-29:11:59:23,679 WARNING [evaluator.py:93] generation_kwargs specified through cli, these settings will be used over set parameters in yaml tasks.\n", + "2023-11-29:11:59:23,708 INFO [huggingface.py:120] Using device 'cuda'\n", + "2023-11-29:11:59:44,516 INFO [task.py:355] Building contexts for task on rank 0...\n", + "2023-11-29:11:59:44,524 INFO [evaluator.py:319] Running loglikelihood requests\n", + "100% 40/40 [00:02<00:00, 15.41it/s]\n", + "fatal: not a git repository (or any of the parent directories): .git\n", + "hf (pretrained=EleutherAI/pythia-2.8b), gen_kwargs: (), limit: 10.0, num_fewshot: None, batch_size: 1\n", + "| Tasks |Version|Filter|n-shot| Metric |Value| |Stderr|\n", + "|-----------------------------------------------|-------|------|-----:|--------|----:|---|-----:|\n", + "|demo_mmlu_high_school_geography_function_prompt|Yaml |none | 0|acc | 0.1|± |0.1000|\n", + "| | |none | 0|acc_norm| 0.2|± |0.1333|\n", + "\n" + ] + } + ], + "source": [ + "YAML_mmlu_geo_string = \"\"\"\n", + "include: mmlu_high_school_geography.yaml\n", + "task: demo_mmlu_high_school_geography_function_prompt\n", + "doc_to_text: !function utils.doc_to_text\n", + "doc_to_choice: \"{{choices}}\"\n", + "\"\"\"\n", + "with open(\"demo_mmlu_high_school_geography_function_prompt.yaml\", \"w\") as f:\n", + " f.write(YAML_mmlu_geo_string)\n", + "\n", + "DOC_TO_TEXT = \"\"\"\n", + "def doc_to_text(x):\n", + " question = x[\"question\"].strip()\n", + " choices = x[\"choices\"]\n", + " option_a = choices[0]\n", + " option_b = choices[1]\n", + " option_c = choices[2]\n", + " option_d = choices[3]\n", + " return f\"{question}\\\\nA. {option_a}\\\\nB. {option_b}\\\\nC. {option_c}\\\\nD. {option_d}\\\\nAnswer:\"\n", + "\"\"\"\n", + "with open(\"utils.py\", \"w\") as f:\n", + " f.write(DOC_TO_TEXT)\n", + "\n", + "!lm_eval \\\n", + " --model hf \\\n", + " --model_args pretrained=EleutherAI/pythia-2.8b \\\n", + " --include_path ./ \\\n", + " --tasks demo_mmlu_high_school_geography_function_prompt \\\n", + " --limit 10 \\\n", + " --output output/demo_mmlu_high_school_geography_function_prompt/ \\\n", + " --log_samples" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, we'll also show how to do this via preprocessing the dataset as necessary using the `process_docs` config field:\n", + "\n", + "We will write a function that will modify each document in our evaluation dataset's split to add a field that is suitable for us to use in `doc_to_text`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "YAML_mmlu_geo_string = \"\"\"\n", + "include: mmlu_high_school_geography.yaml\n", + "task: demo_mmlu_high_school_geography_function_prompt_2\n", + "process_docs: !function utils_process_docs.process_docs\n", + "doc_to_text: \"{{input}}\"\n", + "doc_to_choice: \"{{choices}}\"\n", + "\"\"\"\n", + "with open(\"demo_mmlu_high_school_geography_process_docs.yaml\", \"w\") as f:\n", + " f.write(YAML_mmlu_geo_string)\n", + "\n", + "DOC_TO_TEXT = \"\"\"\n", + "def process_docs(dataset):\n", + " def _process_doc(x):\n", + " question = x[\"question\"].strip()\n", + " choices = x[\"choices\"]\n", + " option_a = choices[0]\n", + " option_b = choices[1]\n", + " option_c = choices[2]\n", + " option_d = choices[3]\n", + " doc[\"input\"] = f\"{question}\\\\nA. {option_a}\\\\nB. {option_b}\\\\nC. {option_c}\\\\nD. {option_d}\\\\nAnswer:\"\n", + " return out_doc\n", + "\n", + " return dataset.map(_process_doc)\n", + "\"\"\"\n", + "\n", + "with open(\"utils_process_docs.py\", \"w\") as f:\n", + " f.write(DOC_TO_TEXT)\n", + "\n", + "!lm_eval \\\n", + " --model hf \\\n", + " --model_args pretrained=EleutherAI/pythia-2.8b \\\n", + " --include_path ./ \\\n", + " --tasks demo_mmlu_high_school_geography_function_prompt_2 \\\n", + " --limit 10 \\\n", + " --output output/demo_mmlu_high_school_geography_function_prompt_2/ \\\n", + " --log_samples" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We hope that this explainer gives you a sense of what can be done with and how to work with LM-Evaluation-Harnes v0.4.0 ! \n", + "\n", + "For more information, check out our documentation pages in the `docs/` folder, and if you have questions, please raise them in GitHub issues, or in #lm-thunderdome or #release-discussion on the EleutherAI discord server." + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "collapsed_sections": [ + "zAov81vTbL2K" + ], + "gpuType": "T4", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "46f521b73fd943c081c648fd873ebc0a": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": { - "id": "duBDqC6PAdjL" - }, - "outputs": [ - { - "data": { - "application/javascript": "\n ((filepath) => {{\n if (!google.colab.kernel.accessAllowed) {{\n return;\n }}\n google.colab.files.view(filepath);\n }})(\"/content/output/mmlu_high_school_geography_continuation/pretrained__EleutherAI__pythia-2.8b_demo_mmlu_high_school_geography_continuation.jsonl\")", - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "from google.colab import files\n", - "files.view(\"output/mmlu_high_school_geography_continuation/pretrained__EleutherAI__pythia-2.8b_demo_mmlu_high_school_geography_continuation.jsonl\")\n" - ] + "48763b6233374554ae76035c0483066f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } }, - { - "cell_type": "markdown", - "metadata": { - "id": "6p0-KPwAgK5j" - }, - "source": [ - "## Closer Look at YAML Fields\n", - "\n", - "To prepare a task we can simply fill in a YAML config with the relevant information.\n", - "\n", - "`output_type`\n", - "The current provided evaluation types comprise of the following:\n", - "1. `loglikelihood`: Evaluates the loglikelihood of a continuation, conditioned on some input string.\n", - "2. `loglikelihood_rolling`: evaluate the loglikelihood of producing a string, conditioned on the empty string. (Used for perplexity evaluations)\n", - "3. `multiple_choice`: Evaluates loglikelihood among the a number of choices predicted by the model.\n", - "4. `greedy_until`: Model outputs greedy generation (can be configured to to use beam search and other generation-related parameters)\n", - "\n", - "The core prompt revolves around 3 fields.\n", - "1. `doc_to_text`: Denotes the prompt template that will be used as input to the model.\n", - "2. `doc_to_choice`: Available choices that will be used as continuation for the model. This is used when the `output_type` is `multiple_choice`, and otherwise can be left as `None`.\n", - "3. `doc_to_target`: When `output_type` is `multiple_choice`, this can be an index that corresponds to the correct answer, or the answer string itself (must be a subset of `doc_to_choice`). For other tasks, this is expected to be a string. You can fill this field with a feature name from the HF dataset so long as the resulting feature follows the conditioned described.\n", - "\n", - "These three fields can be expressed as strings, column names from the source dataset, or as Jinja2 templates that can use fields from the source dataset as variables.\n" - ] + "4986a21eb560448fa79f4b25cde48951": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } }, - { - "cell_type": "markdown", - "metadata": { - "id": "6p0-KPwAgK5j" - }, - "source": [ - "## What if Jinja is not Sufficient?\n", - "\n", - "There can be times where the Jinja2 templating language is not enough to make the prompt we had in mind. There are a few ways to circumvent this limitation:\n", - "\n", - "1. Use `!function` operator for the prompt-related fields to pass a python function that takes as input the dataset row, and will output the prompt template component.\n", - "2. Perform a transformation on the dataset beforehand." - ] + "6b2d90209ec14230b3d58a74ac9b83bf": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Below, we show an example of using `!function` to create `doc_to_text` from a python function:" - ] + "7c5689bc13684db8a22681f41863dddd": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "DYZ5c0JhR1lJ", - "outputId": "ca945235-fb9e-4f17-8bfa-78e7d6ec1490" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2023-11-29:11:59:08,312 INFO [utils.py:160] NumExpr defaulting to 2 threads.\n", - "2023-11-29 11:59:09.348327: E tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:9342] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", - "2023-11-29 11:59:09.348387: E tensorflow/compiler/xla/stream_executor/cuda/cuda_fft.cc:609] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", - "2023-11-29 11:59:09.348421: E tensorflow/compiler/xla/stream_executor/cuda/cuda_blas.cc:1518] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", - "2023-11-29 11:59:10.573752: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n", - "2023-11-29:11:59:14,044 INFO [__main__.py:132] Verbosity set to INFO\n", - "2023-11-29:11:59:23,654 WARNING [__main__.py:138] --limit SHOULD ONLY BE USED FOR TESTING.REAL METRICS SHOULD NOT BE COMPUTED USING LIMIT.\n", - "2023-11-29:11:59:23,654 INFO [__main__.py:143] Including path: ./\n", - "2023-11-29:11:59:23,678 INFO [__main__.py:205] Selected Tasks: ['demo_mmlu_high_school_geography_function_prompt']\n", - "2023-11-29:11:59:23,679 WARNING [evaluator.py:93] generation_kwargs specified through cli, these settings will be used over set parameters in yaml tasks.\n", - "2023-11-29:11:59:23,708 INFO [huggingface.py:120] Using device 'cuda'\n", - "2023-11-29:11:59:44,516 INFO [task.py:355] Building contexts for task on rank 0...\n", - "2023-11-29:11:59:44,524 INFO [evaluator.py:319] Running loglikelihood requests\n", - "100% 40/40 [00:02<00:00, 15.41it/s]\n", - "fatal: not a git repository (or any of the parent directories): .git\n", - "hf (pretrained=EleutherAI/pythia-2.8b), gen_kwargs: (), limit: 10.0, num_fewshot: None, batch_size: 1\n", - "| Tasks |Version|Filter|n-shot| Metric |Value| |Stderr|\n", - "|-----------------------------------------------|-------|------|-----:|--------|----:|---|-----:|\n", - "|demo_mmlu_high_school_geography_function_prompt|Yaml |none | 0|acc | 0.1|± |0.1000|\n", - "| | |none | 0|acc_norm| 0.2|± |0.1333|\n", - "\n" - ] - } + "a1d3a8aa016544a78e8821c8f6199e06": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_f61ed33fad754146bdd2ac9db1ba1c48", + "IPY_MODEL_bfa0af6aeff344c6845e1080a878e92e", + "IPY_MODEL_fd1ad9e0367d4004aae853b91c3a7617" ], - "source": [ - "YAML_mmlu_geo_string = '''\n", - "include: mmlu_high_school_geography.yaml\n", - "task: demo_mmlu_high_school_geography_function_prompt\n", - "doc_to_text: !function utils.doc_to_text\n", - "doc_to_choice: \"{{choices}}\"\n", - "'''\n", - "with open('demo_mmlu_high_school_geography_function_prompt.yaml', 'w') as f:\n", - " f.write(YAML_mmlu_geo_string)\n", - "\n", - "DOC_TO_TEXT = '''\n", - "def doc_to_text(x):\n", - " question = x[\"question\"].strip()\n", - " choices = x[\"choices\"]\n", - " option_a = choices[0]\n", - " option_b = choices[1]\n", - " option_c = choices[2]\n", - " option_d = choices[3]\n", - " return f\"{question}\\\\nA. {option_a}\\\\nB. {option_b}\\\\nC. {option_c}\\\\nD. {option_d}\\\\nAnswer:\"\n", - "'''\n", - "with open('utils.py', 'w') as f:\n", - " f.write(DOC_TO_TEXT)\n", - "\n", - "!lm_eval \\\n", - " --model hf \\\n", - " --model_args pretrained=EleutherAI/pythia-2.8b \\\n", - " --include_path ./ \\\n", - " --tasks demo_mmlu_high_school_geography_function_prompt \\\n", - " --limit 10 \\\n", - " --output output/demo_mmlu_high_school_geography_function_prompt/ \\\n", - " --log_samples\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Next, we'll also show how to do this via preprocessing the dataset as necessary using the `process_docs` config field:\n", - "\n", - "We will write a function that will modify each document in our evaluation dataset's split to add a field that is suitable for us to use in `doc_to_text`." - ] + "layout": "IPY_MODEL_6b2d90209ec14230b3d58a74ac9b83bf" + } }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "YAML_mmlu_geo_string = '''\n", - "include: mmlu_high_school_geography.yaml\n", - "task: demo_mmlu_high_school_geography_function_prompt_2\n", - "process_docs: !function utils_process_docs.process_docs\n", - "doc_to_text: \"{{input}}\"\n", - "doc_to_choice: \"{{choices}}\"\n", - "'''\n", - "with open('demo_mmlu_high_school_geography_process_docs.yaml', 'w') as f:\n", - " f.write(YAML_mmlu_geo_string)\n", - "\n", - "DOC_TO_TEXT = '''\n", - "def process_docs(dataset):\n", - " def _process_doc(x):\n", - " question = x[\"question\"].strip()\n", - " choices = x[\"choices\"]\n", - " option_a = choices[0]\n", - " option_b = choices[1]\n", - " option_c = choices[2]\n", - " option_d = choices[3]\n", - " doc[\"input\"] = f\"{question}\\\\nA. {option_a}\\\\nB. {option_b}\\\\nC. {option_c}\\\\nD. {option_d}\\\\nAnswer:\"\n", - " return out_doc\n", - "\n", - " return dataset.map(_process_doc)\n", - "'''\n", - "\n", - "with open('utils_process_docs.py', 'w') as f:\n", - " f.write(DOC_TO_TEXT)\n", - "\n", - "!lm_eval \\\n", - " --model hf \\\n", - " --model_args pretrained=EleutherAI/pythia-2.8b \\\n", - " --include_path ./ \\\n", - " --tasks demo_mmlu_high_school_geography_function_prompt_2 \\\n", - " --limit 10 \\\n", - " --output output/demo_mmlu_high_school_geography_function_prompt_2/ \\\n", - " --log_samples\n" - ] + "a73f357065d34d7baf0453ae4a8d75e2": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "1.2.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "1.2.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "overflow_x": null, + "overflow_y": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We hope that this explainer gives you a sense of what can be done with and how to work with LM-Evaluation-Harnes v0.4.0 ! \n", - "\n", - "For more information, check out our documentation pages in the `docs/` folder, and if you have questions, please raise them in GitHub issues, or in #lm-thunderdome or #release-discussion on the EleutherAI discord server." - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "collapsed_sections": [ - "zAov81vTbL2K" - ], - "gpuType": "T4", - "provenance": [] + "aed3acd2f2d74003b44079c333a0698e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "DescriptionStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "DescriptionStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "1.2.0", + "_view_name": "StyleView", + "description_width": "" + } }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" + "bfa0af6aeff344c6845e1080a878e92e": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_7c5689bc13684db8a22681f41863dddd", + "max": 5669, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_48763b6233374554ae76035c0483066f", + "value": 5669 + } }, - "language_info": { - "name": "python" + "f61ed33fad754146bdd2ac9db1ba1c48": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_a73f357065d34d7baf0453ae4a8d75e2", + "placeholder": "​", + "style": "IPY_MODEL_46f521b73fd943c081c648fd873ebc0a", + "value": "Downloading builder script: 100%" + } }, - "widgets": { - "application/vnd.jupyter.widget-state+json": { - "46f521b73fd943c081c648fd873ebc0a": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "48763b6233374554ae76035c0483066f": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "ProgressStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "ProgressStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "bar_color": null, - "description_width": "" - } - }, - "4986a21eb560448fa79f4b25cde48951": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "6b2d90209ec14230b3d58a74ac9b83bf": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "7c5689bc13684db8a22681f41863dddd": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "a1d3a8aa016544a78e8821c8f6199e06": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HBoxModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HBoxModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HBoxView", - "box_style": "", - "children": [ - "IPY_MODEL_f61ed33fad754146bdd2ac9db1ba1c48", - "IPY_MODEL_bfa0af6aeff344c6845e1080a878e92e", - "IPY_MODEL_fd1ad9e0367d4004aae853b91c3a7617" - ], - "layout": "IPY_MODEL_6b2d90209ec14230b3d58a74ac9b83bf" - } - }, - "a73f357065d34d7baf0453ae4a8d75e2": { - "model_module": "@jupyter-widgets/base", - "model_module_version": "1.2.0", - "model_name": "LayoutModel", - "state": { - "_model_module": "@jupyter-widgets/base", - "_model_module_version": "1.2.0", - "_model_name": "LayoutModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "LayoutView", - "align_content": null, - "align_items": null, - "align_self": null, - "border": null, - "bottom": null, - "display": null, - "flex": null, - "flex_flow": null, - "grid_area": null, - "grid_auto_columns": null, - "grid_auto_flow": null, - "grid_auto_rows": null, - "grid_column": null, - "grid_gap": null, - "grid_row": null, - "grid_template_areas": null, - "grid_template_columns": null, - "grid_template_rows": null, - "height": null, - "justify_content": null, - "justify_items": null, - "left": null, - "margin": null, - "max_height": null, - "max_width": null, - "min_height": null, - "min_width": null, - "object_fit": null, - "object_position": null, - "order": null, - "overflow": null, - "overflow_x": null, - "overflow_y": null, - "padding": null, - "right": null, - "top": null, - "visibility": null, - "width": null - } - }, - "aed3acd2f2d74003b44079c333a0698e": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "DescriptionStyleModel", - "state": { - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "DescriptionStyleModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/base", - "_view_module_version": "1.2.0", - "_view_name": "StyleView", - "description_width": "" - } - }, - "bfa0af6aeff344c6845e1080a878e92e": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "FloatProgressModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "FloatProgressModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "ProgressView", - "bar_style": "success", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_7c5689bc13684db8a22681f41863dddd", - "max": 5669, - "min": 0, - "orientation": "horizontal", - "style": "IPY_MODEL_48763b6233374554ae76035c0483066f", - "value": 5669 - } - }, - "f61ed33fad754146bdd2ac9db1ba1c48": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_a73f357065d34d7baf0453ae4a8d75e2", - "placeholder": "​", - "style": "IPY_MODEL_46f521b73fd943c081c648fd873ebc0a", - "value": "Downloading builder script: 100%" - } - }, - "fd1ad9e0367d4004aae853b91c3a7617": { - "model_module": "@jupyter-widgets/controls", - "model_module_version": "1.5.0", - "model_name": "HTMLModel", - "state": { - "_dom_classes": [], - "_model_module": "@jupyter-widgets/controls", - "_model_module_version": "1.5.0", - "_model_name": "HTMLModel", - "_view_count": null, - "_view_module": "@jupyter-widgets/controls", - "_view_module_version": "1.5.0", - "_view_name": "HTMLView", - "description": "", - "description_tooltip": null, - "layout": "IPY_MODEL_4986a21eb560448fa79f4b25cde48951", - "placeholder": "​", - "style": "IPY_MODEL_aed3acd2f2d74003b44079c333a0698e", - "value": " 5.67k/5.67k [00:00<00:00, 205kB/s]" - } - } - } + "fd1ad9e0367d4004aae853b91c3a7617": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "1.5.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "1.5.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "1.5.0", + "_view_name": "HTMLView", + "description": "", + "description_tooltip": null, + "layout": "IPY_MODEL_4986a21eb560448fa79f4b25cde48951", + "placeholder": "​", + "style": "IPY_MODEL_aed3acd2f2d74003b44079c333a0698e", + "value": " 5.67k/5.67k [00:00<00:00, 205kB/s]" + } } - }, - "nbformat": 4, - "nbformat_minor": 0 + } + } + }, + "nbformat": 4, + "nbformat_minor": 0 } diff --git a/examples/visualize-wandb.ipynb b/examples/visualize-wandb.ipynb index b6ac47ce62d1791a9210b82bf70b34490ae1adf6..70d25fe608bba91199d12737b55054079d007c2d 100644 --- a/examples/visualize-wandb.ipynb +++ b/examples/visualize-wandb.ipynb @@ -68,6 +68,7 @@ "source": [ "import wandb\n", "\n", + "\n", "wandb.login()" ] }, @@ -110,13 +111,15 @@ "cell_type": "markdown", "id": "e974cabdbe70b667", "metadata": {}, - "source": "" + "source": [] }, { "cell_type": "markdown", "id": "5178ca9445b844e4", "metadata": {}, - "source": "W&B can also be initialized programmatically for use outside the CLI to parse and log the results." + "source": [ + "W&B can also be initialized programmatically for use outside the CLI to parse and log the results." + ] }, { "cell_type": "code", @@ -126,7 +129,8 @@ "outputs": [], "source": [ "import lm_eval\n", - "from lm_eval.logging_utils import WandbLogger\n", + "from lm_eval.loggers import WandbLogger\n", + "\n", "\n", "results = lm_eval.simple_evaluate(\n", " model=\"hf\",\n", diff --git a/lm_eval/__main__.py b/lm_eval/__main__.py index 711df07a47f259796c0d259f490ef9ea3c80a90c..ab68781939599c9fe959c5b642ff385db1067510 100644 --- a/lm_eval/__main__.py +++ b/lm_eval/__main__.py @@ -73,7 +73,7 @@ def setup_parser() -> argparse.ArgumentParser: default=None, type=str, metavar="task1,task2", - help="To get full list of tasks, use the command lm-eval --tasks list", + help="Comma-separated list of task names or task groupings to evaluate on.\nTo get full list of tasks, use one of the commands `lm-eval --tasks {{list_groups,list_subtasks,list_tags,list}}` to list out all available names for task groupings; only (sub)tasks; tags; or all of the above", ) parser.add_argument( "--model_args", @@ -170,9 +170,16 @@ def setup_parser() -> argparse.ArgumentParser: ) parser.add_argument( "--apply_chat_template", - action="store_true", + type=str, + nargs="?", + const=True, default=False, - help="If True, applies the chat template to the prompt", + help=( + "If True, apply chat template to the prompt. " + "Providing `--apply_chat_template` without an argument will apply the default chat template to the prompt. " + "To apply a specific template from the available list of templates, provide the template name as an argument. " + "E.g. `--apply_chat_template template_name`" + ), ) parser.add_argument( "--fewshot_as_multiturn", @@ -289,14 +296,7 @@ def cli_evaluate(args: Union[argparse.Namespace, None] = None) -> None: if args.fewshot_as_multiturn and args.apply_chat_template is False: raise ValueError( - "If fewshot_as_multiturn is set, apply_chat_template must be set to True." - ) - - if ( - args.num_fewshot is None or args.num_fewshot == 0 - ) and args.fewshot_as_multiturn: - raise ValueError( - "If fewshot_as_multiturn is set, num_fewshot must be greater than 0." + "When `fewshot_as_multiturn` is selected, `apply_chat_template` must be set (either to `True` or to the chosen template name)." ) if args.include_path is not None: @@ -318,9 +318,16 @@ def cli_evaluate(args: Union[argparse.Namespace, None] = None) -> None: eval_logger.error("Need to specify task to evaluate.") sys.exit() elif args.tasks == "list": - eval_logger.info( - "Available Tasks:\n - {}".format("\n - ".join(task_manager.all_tasks)) - ) + print(task_manager.list_all_tasks()) + sys.exit() + elif args.tasks == "list_groups": + print(task_manager.list_all_tasks(list_subtasks=False, list_tags=False)) + sys.exit() + elif args.tasks == "list_tags": + print(task_manager.list_all_tasks(list_groups=False, list_subtasks=False)) + sys.exit() + elif args.tasks == "list_subtasks": + print(task_manager.list_all_tasks(list_groups=False, list_tags=False)) sys.exit() else: if os.path.isdir(args.tasks): @@ -349,7 +356,7 @@ def cli_evaluate(args: Union[argparse.Namespace, None] = None) -> None: f"{utils.SPACING}Try `lm-eval --tasks list` for list of available tasks", ) raise ValueError( - f"Tasks not found: {missing}. Try `lm-eval --tasks list` for list of available tasks, or '--verbosity DEBUG' to troubleshoot task registration issues." + f"Tasks not found: {missing}. Try `lm-eval --tasks {{list_groups,list_subtasks,list_tags,list}}` to list out all available names for task groupings; only (sub)tasks; tags; or all of the above, or pass '--verbosity DEBUG' to troubleshoot task registration issues." ) # Respect user's value passed in via CLI, otherwise default to True and add to comma-separated model args diff --git a/lm_eval/api/group.py b/lm_eval/api/group.py new file mode 100644 index 0000000000000000000000000000000000000000..e258692b9fad1cf570a2423c05d25d1604885d7e --- /dev/null +++ b/lm_eval/api/group.py @@ -0,0 +1,117 @@ +import abc +from dataclasses import asdict, dataclass +from inspect import getsource +from typing import Any, Callable, List, Optional, Union + + +@dataclass +class AggMetricConfig(dict): + metric: Optional[str] = None + aggregation: Optional[str] = "mean" + weight_by_size: Optional[str] = False + # list of filter names which should be incorporated into the aggregated metric. + filter_list: Optional[Union[str, list]] = "none" + + def __post_init__(self): + if self.aggregation != "mean" and not callable(self.aggregation): + raise ValueError( + f"Currently, 'mean' is the only pre-defined aggregation across groups' subtasks. Got '{self.aggregation}'." + ) + + if isinstance(self.filter_list, str): + self.filter_list = [self.filter_list] + + +@dataclass +class GroupConfig(dict): + group: Optional[str] = None + group_alias: Optional[str] = None + task: Optional[Union[str, list]] = None + aggregate_metric_list: Optional[ + Union[List[AggMetricConfig], AggMetricConfig, dict] + ] = None + metadata: Optional[dict] = ( + None # by default, not used in the code. allows for users to pass arbitrary info to tasks + ) + + def __getitem__(self, item): + return getattr(self, item) + + def __setitem__(self, item, value): + return setattr(self, item, value) + + def __post_init__(self): + if self.aggregate_metric_list is not None: + if isinstance(self.aggregate_metric_list, dict): + self.aggregate_metric_list = [self.aggregate_metric_list] + + self.aggregate_metric_list = [ + AggMetricConfig(**item) if isinstance(item, dict) else item + for item in self.aggregate_metric_list + ] + + def to_dict(self, keep_callable: bool = False) -> dict: + """dumps the current config as a dictionary object, as a printable format. + null fields will not be printed. + Used for dumping results alongside full task configuration + + :return: dict + A printable dictionary version of the TaskConfig object. + + # TODO: should any default value in the TaskConfig not be printed? + """ + cfg_dict = asdict(self) + # remove values that are `None` + for k, v in list(cfg_dict.items()): + if callable(v): + cfg_dict[k] = self.serialize_function(v, keep_callable=keep_callable) + return cfg_dict + + def serialize_function( + self, value: Union[Callable, str], keep_callable=False + ) -> Union[Callable, str]: + """Serializes a given function or string. + + If 'keep_callable' is True, the original callable is returned. + Otherwise, attempts to return the source code of the callable using 'getsource'. + """ + if keep_callable: + return value + else: + try: + return getsource(value) + except (TypeError, OSError): + return str(value) + + +class ConfigurableGroup(abc.ABC): + def __init__( + self, + config: Optional[dict] = None, + ) -> None: + self._config = GroupConfig(**config) + + @property + def group(self): + return self._config.group + + @property + def group_alias(self): + return self._config.group_alias + + @property + def version(self): + return self._config.version + + @property + def config(self): + return self._config.to_dict() + + @property + def group_name(self) -> Any: + return self._config.group + + def __repr__(self): + return ( + f"ConfigurableGroup(group={self.group}," f"group_alias={self.group_alias})" + ) diff --git a/lm_eval/api/metrics.py b/lm_eval/api/metrics.py index 27b60ed1712d4736255a76575b42c4c59318a0f9..a8459aa7397fd02947917dad616520bb4cb777bd 100644 --- a/lm_eval/api/metrics.py +++ b/lm_eval/api/metrics.py @@ -1,13 +1,13 @@ import logging import math import random +import re +import string from collections.abc import Iterable from typing import List -import evaluate as hf_evaluate import numpy as np import sacrebleu -import sklearn.metrics from lm_eval.api.registry import register_aggregation, register_metric @@ -50,21 +50,24 @@ def bits_per_byte(items): @register_aggregation("f1") def f1_score(items): + from sklearn.metrics import f1_score + unzipped_list = list(zip(*items)) golds = unzipped_list[0] preds = unzipped_list[1] - fscore = sklearn.metrics.f1_score(golds, preds) + fscore = f1_score(golds, preds) return np.max(fscore) @register_aggregation("matthews_corrcoef") def matthews_corrcoef(items): + from sklearn.metrics import matthews_corrcoef + unzipped_list = list(zip(*items)) golds = unzipped_list[0] preds = unzipped_list[1] - # print(preds) - return sklearn.metrics.matthews_corrcoef(golds, preds) + return matthews_corrcoef(golds, preds) @register_aggregation("bleu") @@ -166,7 +169,60 @@ def acc_mutual_info_fn(items): # This is a passthrough function return items -exact_match = hf_evaluate.load("exact_match") +### the code used in the `exact_match_hf_evaluate` function is ported from +### https://github.com/huggingface/evaluate/blob/main/metrics/exact_match/exact_match.py +### which is under the apache license. + +# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +def exact_match_hf_evaluate( + predictions, + references, + regexes_to_ignore=None, + ignore_case=False, + ignore_punctuation=False, + ignore_numbers=False, +): + if regexes_to_ignore is not None: + for s in regexes_to_ignore: + predictions = np.array([re.sub(s, "", x) for x in predictions]) + references = np.array([re.sub(s, "", x) for x in references]) + else: + predictions = np.asarray(predictions) + references = np.asarray(references) + + if ignore_case: + predictions = np.char.lower(predictions) + references = np.char.lower(references) + + if ignore_punctuation: + repl_table = string.punctuation.maketrans("", "", string.punctuation) + predictions = np.char.translate(predictions, table=repl_table) + references = np.char.translate(references, table=repl_table) + + if ignore_numbers: + repl_table = string.digits.maketrans("", "", string.digits) + predictions = np.char.translate(predictions, table=repl_table) + references = np.char.translate(references, table=repl_table) + + score_list = predictions == references + + return {"exact_match": np.mean(score_list)} + + +### @register_metric( @@ -176,7 +232,7 @@ exact_match = hf_evaluate.load("exact_match") aggregation="mean", ) def exact_match_fn(**kwargs): - return exact_match.compute(**kwargs) + return exact_match_hf_evaluate(**kwargs) @register_metric( diff --git a/lm_eval/api/model.py b/lm_eval/api/model.py index 7152441bd443a2b9fb081df84821bf0a66a5eb78..b5c2999336471107dc7e4aac7cce6b725e33506e 100644 --- a/lm_eval/api/model.py +++ b/lm_eval/api/model.py @@ -3,7 +3,7 @@ import hashlib import json import logging import os -from typing import Dict, List, Optional, Tuple, Type, TypeVar +from typing import Dict, List, Optional, Tuple, Type, TypeVar, Union import transformers from sqlitedict import SqliteDict @@ -55,7 +55,7 @@ class LM(abc.ABC): pass @abc.abstractmethod - def loglikelihood_rolling(self, requests) -> List[Tuple[float]]: + def loglikelihood_rolling(self, requests) -> List[float]: """Compute full log-likelihood of a string, with no truncation, for perplexity computation - We will use the full max context length of the model. - For inputs that exceed the max context length, we divide the tokenized string into chunks of up to @@ -101,14 +101,13 @@ class LM(abc.ABC): """Generate greedily until a stopping sequence :param requests: list[Instance] - A list of Instance objects with property `args` which returns a tuple (context, until). + A list of Instance objects with property `args` which returns a tuple (context, gen_kwargs). context: str Context string - until: [str] - The string sequences to generate until. These string sequences - may each span across multiple tokens, or may be part of one token. + gen_kwargs: dict + A dictionary of keyword arguments to pass to the generation function e.g. top_k, until, etc. :return: list[str] - A list of strings continuation + A list of model generated continuations. continuation: str The generated continuation. """ @@ -193,15 +192,13 @@ class LM(abc.ABC): "To use this model with chat templates, please implement the 'tokenizer_name' property." ) - @property - def chat_template(self) -> str: - """Must be defined for LM subclasses that implement Chat Templating. - Should return the structure of the chat template applied to user/assistant messages. - This is used only to save in the experiment results for reproducibility. + def chat_template(self, chat_template: Union[bool, str] = False) -> Optional[str]: + """Returns the chat template structure for user/assistant messages if a template is provided. + This method is intended to be overridden in a subclass to define a specific chat template format. + For models that do not support chat templates, this method returns None by default. """ - raise NotImplementedError( - "To use this model with chat templates, please implement the 'chat_template' property." - ) + + return "" def set_cache_hook(self, cache_hook) -> None: self.cache_hook = cache_hook @@ -246,9 +243,10 @@ class CachingLM: # add hook to lm lm.set_cache_hook(self.get_cache_hook()) - def __getattr__(self, attr): + def __getattr__(self, attr: str): lm_attr = getattr(self.lm, attr) - if not callable(lm_attr): + if attr not in ["loglikelihood", "loglikelihood_rolling", "generate_until"]: + eval_logger.debug(f"Passing through attribute '{attr}' to underlying LM") return lm_attr def fn(requests): @@ -283,8 +281,11 @@ class CachingLM: eval_logger.info( f"Cached requests: {len(requests) - len(remaining_reqs)}, Requests remaining: {len(remaining_reqs)}" ) - # actually run the LM on the requests that do not have cached results - rem_res = getattr(self.lm, attr)(remaining_reqs) + if remaining_reqs: + # actually run the LM on the requests that do not have cached results + rem_res = getattr(self.lm, attr)(remaining_reqs) + else: + rem_res = [] # stick the new ones back into the list and also cache any of the new ones resptr = 0 @@ -313,6 +314,8 @@ class TemplateLM(LM): and boilerplate often included in other LM subclasses. """ + tokenizer = None + @property @abc.abstractmethod def eot_token_id(self): @@ -324,14 +327,19 @@ class TemplateLM(LM): return self.eot_token_id @abc.abstractmethod - def tok_encode(self, string: str, **kwargs): + def tok_encode(self, string: str, **kwargs) -> List[int]: + """ + Tokenize a string using the model's tokenizer and return a list of token IDs. + """ pass @abc.abstractmethod - def _loglikelihood_tokens(self, requests, **kwargs): + def _loglikelihood_tokens(self, requests, **kwargs) -> List[Tuple[float, bool]]: pass - def _encode_pair(self, context, continuation): + def _encode_pair( + self, context: str, continuation: str + ) -> Tuple[List[int], List[int]]: n_spaces = len(context) - len(context.rstrip()) if n_spaces > 0: continuation = context[-n_spaces:] + continuation @@ -372,9 +380,110 @@ class TemplateLM(LM): @abc.abstractmethod def loglikelihood_rolling( self, requests, disable_tqdm: bool = False - ) -> List[Tuple[float, bool]]: + ) -> List[float]: pass @abc.abstractmethod def generate_until(self, requests, disable_tqdm: bool = False) -> List[str]: pass + + def chat_template(self, chat_template: Union[bool, str] = False) -> Optional[str]: + """ + Set and get the appropriate chat template for the model. + This method sets the tokenizer's chat_template and returns the template string for reproducibility. + + The template selection logic is adapted from the Transformers library's `apply_chat_template` + method in the Tokenizer class. The original implementation can be found at: + https://github.com/huggingface/transformers/blob/fc35907f95459d7a6c5281dfadd680b6f7b620e3/src/transformers/tokenization_utils_base.py#L1687 + + This method ensures that the right template is chosen based on the following: + 0. If the model has no 'tokenizer' attribute: assumes that there is only a single possible chat template, handled on the model provider side internally. Returns the empty string. + 1. If the model's tokenizer has multiple templates: + a. Use the specified template if it exists in the dictionary. + b. Use the default template from the list if no specific template is provided. + c. Raise an error if no default template exists and no specific template is provided. + 2. If the model's tokenizer has a single template or no template: + a. Use the tokenizer's chat template if available. + b. Fall back to the default chat template if no tokenizer chat template exists. + + Args: + chat_template (Union[bool, str]): Specifies the chat template to use. + - If False or None, no template is applied. + - If True, the default or only available template is used. + - If a string, the template with the matching name is used. + + Returns: + Optional[str]: The selected chat template, or None if no template is applied. + """ + if self.tokenizer is None: + return "" + + if chat_template is False or chat_template is None: + eval_logger.warning( + "model.chat_template was called with the chat_template set to False or None. " + "Therefore no chat template will be applied. Make sure this is an intended behavior." + ) + return None + + # Convert boolean chat_template to None to ensure compatibility with the adapted logic + if isinstance(chat_template, bool): + chat_template = None + using_default_template = False + + # First, handle the cases when the model has a dict of multiple templates + try: + template = ( + self.tokenizer.chat_template or self.tokenizer.default_chat_template + ) + except AttributeError: + return None + + if isinstance(template, dict): + using_default_dict = self.tokenizer.chat_template is None + + if chat_template is not None: + if chat_template in template: + selected_template = template[chat_template] + if using_default_dict: + using_default_template = True + else: + raise ValueError( + f"The specified chat template '{chat_template}' is not available. " + f"Available template names are {sorted(template.keys())}." + ) + else: + # If user didn't pass a chat template, use the default template from the dict + if "default" in template: + selected_template = template["default"] + using_default_template = True + else: + raise ValueError( + "This model has multiple chat templates with no default specified! Please either pass a chat " + "template or the name of the template you wish to use to the `chat_template` argument. Available " + f"template names are {sorted(template.keys())}." + ) + + # Cases when the model has a single template or no template + else: + # priority: `chat_template` argument > `tokenizer.chat_template` > `tokenizer.default_chat_template + if isinstance(chat_template, str): + eval_logger.warning( + "Chat template name provided, but the tokenizer's chat template is not a dictionary. " + "Using the tokenizer's chat template or the default template instead." + ) + if self.tokenizer.chat_template is not None: + selected_template = self.tokenizer.chat_template + else: + selected_template = self.tokenizer.default_chat_template + using_default_template = True + + if using_default_template: + eval_logger.warning( + "No chat template is set for this tokenizer, falling back to a default class-level template. This is " + "very error-prone, because models are often trained with templates different from the class default! " + "Default chat templates are a legacy feature and will be removed in Transformers v4.43, at which " + "point any code depending on them will stop working. We recommend setting a valid chat template before " + "then to ensure that this model continues working without issues." + ) + + return selected_template diff --git a/lm_eval/api/samplers.py b/lm_eval/api/samplers.py index 74513bbc9775258906cb267f618823f595330620..2cdc4e43e7f73065b1df554f729d7bd92c4398b5 100644 --- a/lm_eval/api/samplers.py +++ b/lm_eval/api/samplers.py @@ -1,3 +1,5 @@ +from functools import partial + import datasets @@ -15,9 +17,38 @@ class ContextSampler: self.target_delimiter = self.config.target_delimiter self.fewshot_delimiter = self.config.fewshot_delimiter - self.doc_to_text = self.task.doc_to_text - self.doc_to_target = self.task.doc_to_target - self.doc_to_choice = self.task.doc_to_choice + if ( + self.config.fewshot_config is not None + and self.config.fewshot_config.get("doc_to_text", None) is not None + ): + self.doc_to_text = partial( + self.task.doc_to_text, + doc_to_text=self.config.fewshot_config.get("doc_to_text", None), + ) + else: + self.doc_to_text = self.task.doc_to_text + + if ( + self.config.fewshot_config is not None + and self.config.fewshot_config.get("doc_to_target", None) is not None + ): + self.doc_to_target = partial( + self.task.doc_to_target, + doc_to_target=self.config.fewshot_config.get("doc_to_target", None), + ) + else: + self.doc_to_target = self.task.doc_to_target + + if ( + self.config.fewshot_config is not None + and self.config.fewshot_config.get("doc_to_choice", None) is not None + ): + self.doc_to_choice = partial( + self.task.doc_to_choice, + doc_to_choice=self.config.fewshot_config.get("doc_to_choice", None), + ) + else: + self.doc_to_choice = self.task.doc_to_choice self.docs = docs # HF dataset split, provided by task._fewshot_docs() if fewshot_indices: # subset few-shot docs from @@ -51,15 +82,17 @@ class ContextSampler: if self.config.doc_to_choice is None or isinstance(doc_content, str) else self.doc_to_choice(doc)[doc_content] ) - labeled_examples += self.target_delimiter - labeled_examples += ( - str(doc_target[0]) - if isinstance(doc_target, list) - else doc_target - if self.config.doc_to_choice is None or isinstance(doc_target, str) - else str(self.doc_to_choice(doc)[doc_target]) - ) - labeled_examples += self.fewshot_delimiter + + if doc_target != "": + labeled_examples += self.target_delimiter + labeled_examples += ( + str(doc_target[0]) + if isinstance(doc_target, list) + else doc_target + if self.config.doc_to_choice is None or isinstance(doc_target, str) + else str(self.doc_to_choice(doc)[doc_target]) + ) + labeled_examples += self.fewshot_delimiter return labeled_examples diff --git a/lm_eval/api/task.py b/lm_eval/api/task.py index ccfda5090641c1b58dbaad8e6ff06bdbad7df8f9..f0be8db99acbc9e51a6aa5b99e298c201a1ab4a4 100644 --- a/lm_eval/api/task.py +++ b/lm_eval/api/task.py @@ -56,8 +56,7 @@ class TaskConfig(dict): # task naming/registry task: Optional[str] = None task_alias: Optional[str] = None - group: Optional[Union[str, list]] = None - group_alias: Optional[Union[str, list]] = None + tag: Optional[Union[str, list]] = None # HF dataset options. # which dataset to use, # and what splits for what purpose @@ -68,13 +67,14 @@ class TaskConfig(dict): validation_split: Optional[str] = None test_split: Optional[str] = None fewshot_split: Optional[str] = ( - None # TODO: assert that this not None if num_fewshot > 0. (?) assert if this is same split as one evaling (?) + None # TODO: assert that this not None if num_fewshot > 0. (?) assert if this is same split as one evaluating (?) ) # formatting / prompting options. # see docs/advanced_task_guide.md for more info process_docs: Optional[Callable] = None doc_to_text: Optional[Union[Callable, str]] = None doc_to_target: Optional[Union[Callable, str]] = None + doc_to_image: Union[Callable, str] = None doc_to_choice: Optional[Union[Callable, str, dict, list]] = None process_results: Optional[Union[Callable, str]] = None use_prompt: Optional[str] = None @@ -365,6 +365,10 @@ class Task(abc.ABC): def doc_to_target(self, doc): pass + # not an abstractmethod because not every language-only task has to implement this + def doc_to_image(self, doc): + raise NotImplementedError + def build_all_requests( self, *, @@ -723,6 +727,10 @@ class ConfigurableTask(Task): ) self.OUTPUT_TYPE = self.config.output_type + if self.config.doc_to_image is not None: + # mark the task as requiring multimodality. + self.MULTIMODAL = True + if self.config.dataset_path is not None: self.DATASET_PATH = self.config.dataset_path @@ -980,7 +988,7 @@ class ConfigurableTask(Task): else: if (self.config.num_fewshot is not None) and (self.config.num_fewshot > 0): eval_logger.warning( - f"Task '{self.config.task}': " + f"[Task: {self.config.task}] " "num_fewshot > 0 but fewshot_split is None. " "using preconfigured rule." ) @@ -1030,8 +1038,8 @@ class ConfigurableTask(Task): Whether to apply the chat template to the fewshot context. :param fewshot_as_multiturn: bool Whether to provide the fewshot examples as a multiturn conversation or a single user turn. - :param chat_template: Callable - Chat template to be applied to the fewshot context. + :param chat_template: + callable (from lm.apply_chat_template) that takes in a list[Dict] chat transcript and renders it into a string. :returns: str The fewshot context. """ @@ -1159,9 +1167,11 @@ class ConfigurableTask(Task): """ return doc - def doc_to_text(self, doc): + def doc_to_text(self, doc, doc_to_text=None): if self.prompt is not None: doc_to_text = self.prompt + elif doc_to_text is not None: + doc_to_text = doc_to_text else: doc_to_text = self.config.doc_to_text @@ -1193,9 +1203,11 @@ class ConfigurableTask(Task): print(type(doc_to_text)) raise TypeError - def doc_to_target(self, doc: Mapping) -> Union[int, str, list]: + def doc_to_target(self, doc: Mapping, doc_to_target=None) -> Union[int, str, list]: if self.prompt is not None: doc_to_target = self.prompt + elif doc_to_target is not None: + doc_to_target = doc_to_target else: doc_to_target = self.config.doc_to_target @@ -1237,9 +1249,11 @@ class ConfigurableTask(Task): else: raise TypeError - def doc_to_choice(self, doc: Any) -> List[str]: + def doc_to_choice(self, doc: Any, doc_to_choice=None) -> List[str]: if self.prompt is not None: doc_to_choice = self.prompt + elif doc_to_choice is not None: + doc_to_choice = doc_to_choice elif self.config.doc_to_choice is None: eval_logger.error("doc_to_choice was called but not set in config") else: @@ -1261,9 +1275,34 @@ class ConfigurableTask(Task): else: raise TypeError + def doc_to_image(self, doc: Any, doc_to_image=None) -> Union[int, str, list]: + if doc_to_image is not None: + doc_to_image = doc_to_image + elif self.config.doc_to_image is not None: + doc_to_image = self.config.doc_to_image + else: + return None + + if isinstance(doc_to_image, list): + image_feature = [ + self.doc_to_image(doc, feature) for feature in doc_to_image + ] + return [feature for feature in image_feature if feature is not None] + elif isinstance(doc_to_image, str): + if doc_to_image in self.features: + return doc[doc_to_image] + else: + return ast.literal_eval(utils.apply_template(doc_to_image, doc)) + elif callable(doc_to_image): + return doc_to_image(doc) + else: + return None + def construct_requests( self, doc: dict, ctx: str, **kwargs ) -> Union[List[Instance], Instance]: + aux_arguments = None + if self.OUTPUT_TYPE == "loglikelihood": arguments = (ctx, self.doc_to_target(doc)) elif self.OUTPUT_TYPE == "loglikelihood_rolling": @@ -1281,6 +1320,37 @@ class ConfigurableTask(Task): # Otherwise they are placed in the continuation arguments = [(ctx, f"{target_delimiter}{cont}") for cont in choices] + # TODO: we should raise a warning telling users this will at most ~2x runtime. + if "acc_mutual_info" in self._metric_fn_list.keys(): + # if we are calculating multiple choice accuracy + # using mutual information instead of raw loglikelihood as metric, need unconditional lls. + + # here mutual info refers to calculating + # log(P(choice|ctx) / P(choice)) = log(P(choice|ctx)) - log(P(choice)) + # in other words normalizing by subtracting the unconditional logprob of each choice. + aux_arguments = [("", f"{choice}") for choice in choices] + + arguments.extend(aux_arguments) + + elif self.OUTPUT_TYPE == "generate_until": + arguments = (ctx, deepcopy(self.config.generation_kwargs)) + + multimodal_arg = {} + if ( + self.config.doc_to_image + ): # TODO: ensure that non-multimodal tasks aren't getting visual args + multimodal_arg = { + **multimodal_arg, + **{"visual": self.doc_to_image(doc)}, + } + + if bool(multimodal_arg): + if isinstance(arguments, list): + arguments = [arg + (multimodal_arg,) for arg in arguments] + else: + arguments = arguments + (multimodal_arg,) + + if self.OUTPUT_TYPE == "multiple_choice": request_list = [ Instance( request_type="loglikelihood", @@ -1291,33 +1361,15 @@ class ConfigurableTask(Task): ) for i, arg in enumerate(arguments) ] - # TODO: we should raise a warning telling users this will at most ~2x runtime. - if "acc_mutual_info" in self._metric_fn_list.keys(): - # if we are calculating multiple choice accuracy - # using mutual information instead of raw loglikelihood as metric, need unconditional lls. - # here mutual info refers to calculating - # log(P(choice|ctx) / P(choice)) = log(P(choice|ctx)) - log(P(choice)) - # in other words normalizing by subtracting the unconditional logprob of each choice. - request_list.extend( - [ - Instance( - request_type="loglikelihood", - doc=doc, - arguments=("", "{}".format(choice)), - idx=i, - **kwargs, - ) - for i, choice in enumerate(choices) - ] - ) return request_list - elif self.OUTPUT_TYPE == "generate_until": - arguments = (ctx, deepcopy(self.config.generation_kwargs)) - return Instance( - request_type=self.OUTPUT_TYPE, doc=doc, arguments=arguments, idx=0, **kwargs + request_type=self.OUTPUT_TYPE, + doc=doc, + arguments=arguments, + idx=0, + **kwargs, ) def process_results(self, doc, results): @@ -1446,7 +1498,7 @@ class ConfigurableTask(Task): # we expect multiple_targets to be a list. elif self.multiple_target: gold = list(gold) - elif type(gold) != type(result): + elif type(gold) is not type(result): # cast gold to the same type as result gold = type(result)(gold) @@ -1520,10 +1572,13 @@ class ConfigurableTask(Task): def get_config(self, key: str) -> Any: return getattr(self._config, key, None) + @property + def task_name(self) -> Any: + return getattr(self.config, "task", None) + def __repr__(self): return ( f"ConfigurableTask(task_name={getattr(self.config, 'task', None)}," - f"group_name={getattr(self.config, 'group', None)}," f"output_type={self.OUTPUT_TYPE}," f"num_fewshot={getattr(self.config, 'num_fewshot', None)}," f"num_samples={len(self.eval_docs)})" diff --git a/lm_eval/caching/__init__.py b/lm_eval/caching/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lm_eval/evaluator.py b/lm_eval/evaluator.py index 73226b0514f1b556e4e2d4b06c9e03362180b335..d0c1a19a659216c854c7fc37167926b4db861184 100644 --- a/lm_eval/evaluator.py +++ b/lm_eval/evaluator.py @@ -11,11 +11,14 @@ import torch import lm_eval.api.metrics import lm_eval.api.registry +import lm_eval.api.task import lm_eval.models from lm_eval.caching.cache import delete_cache from lm_eval.evaluator_utils import ( + consolidate_group_results, consolidate_results, get_sample_size, + get_subtask_list, get_task_list, prepare_print_tasks, print_writeout, @@ -23,7 +26,10 @@ from lm_eval.evaluator_utils import ( ) from lm_eval.loggers import EvaluationTracker from lm_eval.loggers.utils import add_env_info, add_tokenizer_info, get_git_commit_hash -from lm_eval.tasks import TaskManager, get_task_dict +from lm_eval.tasks import ( + TaskManager, + get_task_dict, +) from lm_eval.utils import ( eval_logger, handle_non_serializable, @@ -35,7 +41,7 @@ from lm_eval.utils import ( if TYPE_CHECKING: from lm_eval.api.model import LM - from lm_eval.tasks import Task + from lm_eval.api.task import Task @positional_deprecated @@ -44,7 +50,7 @@ def simple_evaluate( model_args: Optional[Union[str, dict]] = None, tasks: Optional[List[Union[str, dict, object]]] = None, num_fewshot: Optional[int] = None, - batch_size: Optional[int] = None, + batch_size: Optional[Union[int, str]] = None, max_batch_size: Optional[int] = None, device: Optional[str] = None, use_cache: Optional[str] = None, @@ -58,7 +64,7 @@ def simple_evaluate( log_samples: bool = True, evaluation_tracker: Optional[EvaluationTracker] = None, system_instruction: Optional[str] = None, - apply_chat_template: bool = False, + apply_chat_template: Union[bool, str] = False, fewshot_as_multiturn: bool = False, gen_kwargs: Optional[str] = None, task_manager: Optional[TaskManager] = None, @@ -106,8 +112,11 @@ def simple_evaluate( If True, write out all model outputs and documents for per-sample measurement and post-hoc analysis :param system_instruction: str System instruction to be applied to the prompt - :param apply_chat_template: bool - If True, apply chat template to the prompt + :param apply_chat_template: Union[bool, str] + Specifies whether to apply a chat template to the prompt. + - If set to True, the default chat template is applied. + - If set to a string, applies the specified chat template by name. + Defaults to False (no chat template applied). :param fewshot_as_multiturn: bool Whether to provide the fewshot examples as a multiturn conversation or a single user turn. :param gen_kwargs: str @@ -148,6 +157,9 @@ def simple_evaluate( seed_message.append(f"Setting torch manual seed to {torch_random_seed}") torch.manual_seed(torch_random_seed) + if fewshot_random_seed is not None: + seed_message.append(f"Setting fewshot manual seed to {fewshot_random_seed}") + if seed_message: eval_logger.info(" | ".join(seed_message)) @@ -199,7 +211,9 @@ def simple_evaluate( ) else: if not isinstance(model, lm_eval.api.model.LM): - raise TypeError + raise TypeError( + f"The value of `model` passed to simple_evaluate() was of type {type(model)}, but is required to be a subclass of lm_eval.api.model.LM . This may be because you are passing an initialized Hugging Face PreTrainedModel without having wrapped it in `lm_eval.models.huggingface.HFLM(pretrained=my_model)` first." + ) eval_logger.info("Using pre-initialized model") lm = model @@ -219,48 +233,58 @@ def simple_evaluate( task_manager = TaskManager(verbosity) task_dict = get_task_dict(tasks, task_manager) - for task_name in task_dict.keys(): - task_obj = task_dict[task_name] - if isinstance(task_obj, tuple): - _, task_obj = task_obj - if task_obj is None: - continue - - if task_obj.get_config("output_type") == "generate_until": - if gen_kwargs is not None: - task_obj.set_config( - key="generation_kwargs", value=gen_kwargs, update=True - ) - if predict_only: - log_samples = True - eval_logger.info( - f"Processing {task_name} in output-only mode. Metrics will not be calculated!" - ) - # we have to change the class properties post-hoc. This is pretty hacky. - task_obj.override_metric(metric_name="bypass") - - # override tasks' fewshot values to the provided num_fewshot arg value - # except if tasks have it set to 0 manually in their configs--then we should never overwrite that - if num_fewshot is not None: - if (default_num_fewshot := task_obj.get_config("num_fewshot")) == 0: - eval_logger.info( - f"num_fewshot has been set to 0 for {task_name} in its config. Manual configuration will be ignored." - ) + # helper function to recursively apply config overrides to leaf subtasks, skipping their constituent groups. + # (setting of num_fewshot ; bypassing metric calculation ; setting fewshot seed) + def _adjust_config(task_dict): + adjusted_task_dict = {} + for task_name, task_obj in task_dict.items(): + if isinstance(task_obj, dict): + adjusted_task_dict = { + **adjusted_task_dict, + **{task_name: _adjust_config(task_obj)}, + } + else: - eval_logger.warning( - f"Overwriting default num_fewshot of {task_name} from {default_num_fewshot} to {num_fewshot}" - ) - task_obj.set_config(key="num_fewshot", value=num_fewshot) - else: - # if num_fewshot not provided, and the task does not define a default one, default to 0 - if (default_num_fewshot := task_obj.get_config("num_fewshot")) is None: - task_obj.set_config(key="num_fewshot", value=0) - # fewshot_random_seed set for tasks, even with a default num_fewshot (e.g. in the YAML file) - task_obj.set_fewshot_seed(seed=fewshot_random_seed) - eval_logger.info( - f"Setting fewshot random generator seed to {fewshot_random_seed}" - ) + if task_obj.get_config("output_type") == "generate_until": + if gen_kwargs is not None: + task_obj.set_config( + key="generation_kwargs", value=gen_kwargs, update=True + ) + + if predict_only: + eval_logger.info( + f"Processing {task_name} in output-only mode. Metrics will not be calculated!" + ) + # we have to change the class properties post-hoc. This is pretty hacky. + task_obj.override_metric(metric_name="bypass") + + # override tasks' fewshot values to the provided num_fewshot arg value + # except if tasks have it set to 0 manually in their configs--then we should never overwrite that + if num_fewshot is not None: + if (default_num_fewshot := task_obj.get_config("num_fewshot")) == 0: + eval_logger.info( + f"num_fewshot has been set to 0 for {task_name} in its config. Manual configuration will be ignored." + ) + else: + eval_logger.warning( + f"Overwriting default num_fewshot of {task_name} from {default_num_fewshot} to {num_fewshot}" + ) + task_obj.set_config(key="num_fewshot", value=num_fewshot) + else: + # if num_fewshot not provided, and the task does not define a default one, default to 0 + if ( + default_num_fewshot := task_obj.get_config("num_fewshot") + ) is None: + task_obj.set_config(key="num_fewshot", value=0) + # fewshot_random_seed set for tasks, even with a default num_fewshot (e.g. in the YAML file) + task_obj.set_fewshot_seed(seed=fewshot_random_seed) + + adjusted_task_dict[task_name] = task_obj + + return adjusted_task_dict + + task_dict = _adjust_config(task_dict) if check_integrity: run_task_tests(task_list=tasks) @@ -270,7 +294,7 @@ def simple_evaluate( model_source=model, model_args=model_args, system_instruction=system_instruction, - chat_template=lm.chat_template if apply_chat_template else None, + chat_template=lm.chat_template(apply_chat_template), fewshot_as_multiturn=fewshot_as_multiturn, ) @@ -282,7 +306,7 @@ def simple_evaluate( rewrite_requests_cache=rewrite_requests_cache, bootstrap_iters=bootstrap_iters, write_out=write_out, - log_samples=log_samples, + log_samples=True if predict_only else log_samples, system_instruction=system_instruction, apply_chat_template=apply_chat_template, fewshot_as_multiturn=fewshot_as_multiturn, @@ -343,7 +367,7 @@ def evaluate( write_out: bool = False, log_samples: bool = True, system_instruction: Optional[str] = None, - apply_chat_template: bool = False, + apply_chat_template: Union[bool, str] = False, fewshot_as_multiturn: bool = False, verbosity: str = "INFO", ): @@ -363,8 +387,11 @@ def evaluate( If True, write out all model outputs and documents for per-sample measurement and post-hoc analysis :param system_instruction: str System instruction to be applied to the prompt - :param apply_chat_template: bool - If True, apply chat template to the prompt + :param apply_chat_template: Union[bool, str] + Specifies whether to apply a chat template to the prompt. + - If set to True, the default chat template is applied. + - If set to a string, applies the specified chat template by name. + Defaults to False (no chat template applied). :param fewshot_as_multiturn: bool Whether to provide the fewshot examples as a multiturn conversation or a single user turn. :return @@ -380,16 +407,40 @@ def evaluate( padding_requests = defaultdict(int) # get lists of group hierarchy and each type of request - task_hierarchy, eval_tasks = get_task_list(task_dict) + eval_tasks = get_task_list(task_dict) if not log_samples: if not all( "bypass" not in getattr(task_output.task, "_metric_fn_list", {}).keys() for task_output in eval_tasks ): raise ValueError("log_samples must be True for 'bypass' metric-only tasks") + + # validation check: are we running multimodal task <-> non-multimodal model class, or vice-versa. + incompatible_tasks = [] for task_output in eval_tasks: task: Task = task_output.task - limit = get_sample_size(task, limit) + + if getattr(lm, "MULTIMODAL", False) != getattr(task, "MULTIMODAL", False): + incompatible_tasks.append(task_output.task_name) + if len(incompatible_tasks) > 0: + if not getattr(lm, "MULTIMODAL", False): + raise ValueError( + f"Attempted to run tasks: {incompatible_tasks} which require multimodal input, but the selected model type does not currently implement this. Multimodal support is currently restricted to the ['hf-multimodal', 'vllm-vlm'] model type." + ) + else: + raise ValueError( + f"Attempted to run tasks: {incompatible_tasks} which are text-only, but used a model type which only currently supports multimodal tasks." + ) + # end multimodality validation check + + # Cache the limit arg. + limit_arg = limit + limits = [] + for task_output in eval_tasks: + task: Task = task_output.task + + limit = get_sample_size(task, limit_arg) + limits.append(limit) task.build_all_requests( limit=limit, rank=lm.rank, @@ -397,7 +448,7 @@ def evaluate( cache_requests=cache_requests, rewrite_requests_cache=rewrite_requests_cache, system_instruction=system_instruction, - apply_chat_template=apply_chat_template, + apply_chat_template=bool(apply_chat_template), fewshot_as_multiturn=fewshot_as_multiturn, chat_template=getattr(lm, "apply_chat_template") if apply_chat_template @@ -459,7 +510,7 @@ def evaluate( WORLD_SIZE = lm.world_size ### Postprocess outputs ### # TODO: del model here, maybe (idea: allow user to specify device of e.g. reward model separately) - for task_output in eval_tasks: + for task_output, limit in zip(eval_tasks, limits): task = task_output.task task.apply_filters() @@ -557,106 +608,45 @@ def evaluate( ### Calculate group metrics ### if bool(results): - for group, task_list in reversed(task_hierarchy.items()): - if len(task_list) == 0: - # task_hierarchy entries are either - # `group_name: [subtask1, subtask2, ...]` - # or `task_name: []`. - # we only want to operate on groups here. - continue - - # collect all higher_is_better values for metrics - # in the group's subtasks. - # TODO: clean this up ; unify with the below metric_list loop? - _higher_is_better = {} + results, versions, show_group_table, *_ = consolidate_group_results( + results, versions, task_dict + ) + + results_agg, group_agg = prepare_print_tasks(task_dict, results) + subtask_list = get_subtask_list(task_dict) + + # collect all higher_is_better values for metrics + # in the group's subtasks. + # TODO: clean this up ; unify with the below metric_list loop? + _higher_is_better = {} + for group, task_list in subtask_list.items(): + if ( + len(task_list) != 0 + ): # subtask list will list "task_name": [] for solo tasks for task in task_list: for m, h in higher_is_better[task].items(): if m not in _higher_is_better.keys(): _higher_is_better[m] = h - if ( - m in _higher_is_better - and _higher_is_better[m] is not None - and _higher_is_better[m] != h - ): - eval_logger.warning( - f"Higher_is_better values for metric {m} in group {group} are not consistent. Defaulting to None." - ) - _higher_is_better[m] = None - higher_is_better[group] = _higher_is_better - - # collect all metric keys used by a subtask in the group. - metric_list = list( - { - key - for task in task_list - for key in results[task].keys() - if "_stderr" not in key and key not in ["alias", "samples"] - } - ) - for metric in metric_list: - stderr = "_stderr,".join(metric.split(",")) - - # gather metrics, sizes, and stderrs from subtasks - metrics = [ - results[task][metric] - for task in task_list - if metric in results[task] - ] # TODO: copy? - stderrs = [ - results[task][stderr] - for task in task_list - if stderr in results[task] - ] - sizes = [ - results[task]["samples"] - for task in task_list - if metric in results[task] - ] - - # compute group's pooled metric and stderr - results[group][metric] = ( - lm_eval.api.metrics.aggregate_subtask_metrics(metrics, sizes) - ) - # TODO: calculate grouped metric using aggregation fn - if "N/A" in stderrs: - results[group][stderr] = "N/A" - else: - results[group][stderr] = ( - lm_eval.api.metrics.pooled_sample_stderr(stderrs, sizes) - ) - # TODO: allow GroupConfigs to choose which variance formula is used, for back-compatibility - # To use the old (likely incorrect) variance formula, comment out the above and uncomment this line: - # results[group][stderr] = lm_eval.api.metrics.combined_sample_stderr(stderrs, sizes, metrics=metrics) - - results[group]["samples"] = sum(sizes) - - results_agg = defaultdict(dict) - groups_agg = defaultdict(dict) - all_tasks_list = list(task_hierarchy.keys()) - while True: - add_tasks_list = list(k for k in results_agg.keys()) - left_tasks_list = sorted(list(set(all_tasks_list) - set(add_tasks_list))) - if len(left_tasks_list) == 0: - break - - _task_hierarchy = { - k: v for k, v in task_hierarchy.items() if k in left_tasks_list - } - _results_agg, _groups_agg = prepare_print_tasks(_task_hierarchy, results) - results_agg = {**results_agg, **_results_agg} - groups_agg = {**groups_agg, **_groups_agg} - - for group_name, task_list in task_hierarchy.items(): - if task_list: - num_fewshot[group_name] = num_fewshot[ - task_list[0] - ] # TODO: validate this + if ( + m in _higher_is_better + and _higher_is_better[m] is not None + and _higher_is_better[m] != h + ): + eval_logger.warning( + f"Higher_is_better values for metric {m} in group {group} are not consistent. Defaulting to None." + ) + _higher_is_better[m] = None + higher_is_better[group] = _higher_is_better results_dict = { "results": dict(results_agg.items()), - **({"groups": dict(groups_agg.items())} if bool(groups_agg) else {}), - "group_subtasks": dict(reversed(task_hierarchy.items())), + **( + {"groups": dict(group_agg.items())} + if (bool(group_agg) & show_group_table) + else {} + ), + "group_subtasks": dict(reversed(subtask_list.items())), "configs": dict(sorted(configs.items())), "versions": dict(sorted(versions.items())), "n-shot": dict(sorted(num_fewshot.items())), @@ -669,7 +659,7 @@ def evaluate( len(task_output.task.eval_docs), ), } - for task_output in eval_tasks + for task_output, limit in zip(eval_tasks, limits) }, } if log_samples: diff --git a/lm_eval/evaluator_utils.py b/lm_eval/evaluator_utils.py index a773c92ea7e1ca6cb456db8d7d63b41e07a69d9b..d5a08326014279335521dcd1f5f70c1fe12c5003 100644 --- a/lm_eval/evaluator_utils.py +++ b/lm_eval/evaluator_utils.py @@ -2,9 +2,15 @@ import collections import math import pathlib import sys -from typing import Dict, List, Optional, Tuple, Union - -from lm_eval.api import metrics +from typing import List, Optional, Tuple, Union + +from lm_eval.api.group import ConfigurableGroup +from lm_eval.api.metrics import ( + aggregate_subtask_metrics, + pooled_sample_stderr, + stderr_for_metric, +) +from lm_eval.api.task import Task from lm_eval.utils import eval_logger, positional_deprecated @@ -98,7 +104,7 @@ class TaskOutput: self.agg_metrics[metric_key] = agg_fn(items) self.sample_len = len(items) # TODO: same sample size for each metric? if isinstance(bootstrap_iters, int): - stderr_fn = metrics.stderr_for_metric( + stderr_fn = stderr_for_metric( metric=agg_fn, bootstrap_iters=min(bootstrap_iters, 100) if metric in ["bleu", "chrf", "ter"] @@ -116,23 +122,71 @@ class TaskOutput: return ( f"TaskOutput(task_name={self.task_name}, " f"group_name={self.group_name}, " - f"version={self.version}," - f"n_shot={self.n_shot}" - f"task_alias={self.task_alias}, group_alias={self.group_alias})" + f"version={self.version}, " + f"n_shot={self.n_shot}, " + f"task_alias={self.task_alias}, " + f"group_alias={self.group_alias})" ) -def get_task_list(task_dict: dict) -> Tuple[Dict[str, list], List[TaskOutput]]: - task_hierarchy = collections.defaultdict(list) - outputs = list(TaskOutput.from_taskdict(x, y) for x, y in task_dict.items()) - for task_output in outputs: - if group_name := task_output.group_name: - task_hierarchy[group_name].append(task_output.task_name) +def get_task_list(task_dict: dict) -> List[TaskOutput]: + outputs = [] + for task_name, task_obj in task_dict.items(): + if isinstance(task_obj, dict): + _outputs = get_task_list(task_obj) + outputs.extend(_outputs) else: - task_hierarchy[task_output.task_name] = [] - # returns task_hierarchy tracking which groups contain which subtasks, - # and a list of TaskOutput classes for each non-group subtask - return task_hierarchy, [x for x in outputs if x.task] + task_output = TaskOutput.from_taskdict(task_name, task_obj) + outputs.append(task_output) + + return outputs + + +def get_subtask_list(task_dict, task_root=None, depth=0): + subtask_list = {} + for group_obj, task_obj in task_dict.items(): + if isinstance(group_obj, ConfigurableGroup): + # group_name = group_obj.group_name + group_name = group_obj.group_name + else: + group_name = group_obj + if isinstance(task_obj, dict): + _subtask_list = get_subtask_list( + task_obj, task_root=group_name, depth=depth + 1 + ) + if task_root: + subtask_list.setdefault((task_root, depth), []).extend( + [ + _task + for (_task, _depth) in _subtask_list.keys() + if (_depth - 1) == depth + ] + ) + + subtask_list = {**subtask_list, **_subtask_list} + else: + if isinstance(task_obj, ConfigurableGroup): + # group_or_task_name = task_obj.group_name + group_or_task_name = task_obj.group_name + elif isinstance(task_obj, Task): + # group_or_task_name = task_obj.task_name + group_or_task_name = task_obj.task_name + + if task_root is None: + subtask_list.setdefault((group_or_task_name, depth), []) + else: + subtask_list.setdefault((task_root, depth), []).append( + group_or_task_name + ) + + if depth == 0: + _subtask_list = {} + for group_key, task_list in subtask_list.items(): + group_name, depth = group_key + _subtask_list[group_name] = task_list + subtask_list = _subtask_list + + return subtask_list def print_writeout(task) -> None: @@ -155,70 +209,95 @@ def get_sample_size(task, limit: Optional[int]) -> Union[int, None]: def prepare_print_tasks( - task_hierarchy: dict, results: dict, tab=0 + task_dict: dict, + results: dict, + task_depth=0, + group_depth=0, ) -> Tuple[dict, dict]: """ - @param task_hierarchy: Dictionary representing the group hierarchy of tasks. Each key is a group name and its + @param task_dict: Dictionary representing the group hierarchy of tasks. Each key is a group name and its value is a list of task names. @param results: Dictionary containing the results of each task. Each key is a group name and its value is a dictionary of task results. - @param tab: The indentation level for printing the task + @param task_depth: The indentation level for printing the task + hierarchy. Default is 0. + @param group_depth: The indentation level for printing the group hierarchy. Default is 0. @return: A tuple of two dictionaries: results_agg and groups_agg. results_agg contains aggregated results for each task, and groups_agg contains aggregated results for each group. Prepares the task hierarchy and aggregates the results for each task and group recursively for printing. """ - results_agg = collections.defaultdict(dict) - groups_agg = collections.defaultdict(dict) - - (group_name, task_list), *_ = task_hierarchy.items() - task_list = sorted(task_list) - - results_agg[group_name] = results[group_name].copy() - # results_agg[group_name]["tab"] = tab - if "samples" in results_agg[group_name]: - results_agg[group_name].pop("samples") - - tab_string = " " * tab + "- " if tab > 0 else "" - if "alias" in results_agg[group_name]: - results_agg[group_name]["alias"] = tab_string + results_agg[group_name]["alias"] - else: - results_agg[group_name]["alias"] = tab_string + group_name - - if len(task_list) > 0: - groups_agg[group_name] = results[group_name].copy() - # groups_agg[group_name]["tab"] = tab - if "samples" in groups_agg[group_name]: - groups_agg[group_name].pop("samples") - - if "alias" in groups_agg[group_name]: - groups_agg[group_name]["alias"] = ( - tab_string + groups_agg[group_name]["alias"] + def _sort_task_dict(task_dict): + """ + Helper utility. Sorts the task dict at the current level of the hierarchy based on alphabetized task name. + Required so that we end up sorting within each sub-header correctly. + """ + + return dict( + sorted( + task_dict.items(), + key=lambda item: item[0].group_name + if isinstance(item[0], ConfigurableGroup) + else item[0], ) - else: - groups_agg[group_name]["alias"] = tab_string + group_name + ) - for task_name in task_list: - if task_name in task_hierarchy: - _task_hierarchy = { - **{task_name: task_hierarchy[task_name]}, - **task_hierarchy, - } + task_agg = collections.defaultdict(dict) + group_agg = collections.defaultdict(dict) + task_dict = _sort_task_dict(task_dict) + for task_or_group_name, task_or_group_obj in task_dict.items(): + tab_string = " " * task_depth + "- " if task_depth > 0 else "" + if isinstance(task_or_group_name, ConfigurableGroup): + # string_name = task_or_group_name.group_name + name = task_or_group_name.group_name + from_configurable_group = True + task_or_group_obj = _sort_task_dict(task_or_group_obj) + elif isinstance(task_or_group_name, str): + name = task_or_group_name + if isinstance(task_or_group_obj, Task): + # string_name = task_or_group_obj.task_name + name = task_or_group_obj.task_name + from_configurable_group = False + + task_agg[name] = results[name].copy() + if from_configurable_group: + if task_or_group_name.group_alias is not None: + alias = task_or_group_name.group_alias else: - _task_hierarchy = { - **{task_name: []}, - **task_hierarchy, - } - - _results_agg, _groups_agg = prepare_print_tasks( - _task_hierarchy, results, tab + 1 + alias = task_or_group_name.group + else: + if "alias" in task_agg[name]: + alias = task_agg[name]["alias"] + else: + alias = name + + task_agg[name]["alias"] = tab_string + alias + if "samples" in task_agg[name]: + task_agg[name].pop("samples") + + if from_configurable_group and (" " not in results[name]): + group_tab_string = " " * group_depth + "- " if group_depth > 0 else "" + group_agg[name] = results[name].copy() + group_agg[name]["alias"] = group_tab_string + alias + if "samples" in group_agg[name]: + group_agg[name].pop("samples") + + if isinstance(task_or_group_obj, dict): + task_depth += 1 + group_depth += 1 + _task_agg, _group_agg = prepare_print_tasks( + task_or_group_obj, results, task_depth, group_depth ) - results_agg = {**results_agg, **_results_agg} - groups_agg = {**groups_agg, **_groups_agg} - - return results_agg, groups_agg + task_agg = { + **task_agg, + **_task_agg, + } + group_agg = {**group_agg, **_group_agg} + task_depth -= 1 + group_depth -= 1 + return task_agg, group_agg def consolidate_results( @@ -261,6 +340,8 @@ def consolidate_results( for task_output in eval_tasks: if "task_alias" in (task_config := task_output.task_config): results[task_output.task_name]["alias"] = task_config["task_alias"] + else: + results[task_output.task_name]["alias"] = task_output.task_name if group_alias := task_output.group_alias: if group_alias not in results and (group_name := task_output.group_name): results[group_name]["alias"] = group_alias @@ -281,6 +362,147 @@ def consolidate_results( return results, samples, configs, versions, num_fewshot, higher_is_better +def consolidate_group_results( + results, + versions, + task_dict, + task_root=None, + show_group_table=False, + task_aggregation_list=None, +) -> Tuple[dict, dict, bool, Union[None,]]: + """ + (Recursively) calculates groups' aggregated metrics and updates the results and versions dictionaries with this info. + + @return: a tuple [results, versions, show_group_table, task_aggregation_list] with formats described below: + + - results: A defaultdict with task names (and, after this function is called, group names of + groups that perform aggregation) as keys, and dictionaries with "alias" and metric,filter_name pairs as keys. + - versions: A defaultdict with task names (and, after this function is called, group names of + groups that perform aggregation) as keys, and float values representing the task or group's version if a version is specified. (defaulting to None). + - show_group_table: a boolean which is true if there exists a group that requires printing of its aggregated scores in a group table. + - task_aggregation_list: a defaultdict listing the subtasks to average over to produce a given group's end metric. + + The method then returns the updated results, versions, show_group_table, and task_aggregation_list as a tuple. + In the top-level invocation of this function, task_aggregation_list is ignored. + """ + if task_root is None: + task_root = {} + + if task_aggregation_list is None: + task_aggregation_list = {} + + for group_or_task, group_or_task_info in task_dict.items(): + # Convert to string + if isinstance(group_or_task, ConfigurableGroup): + group_config = group_or_task.config + group_or_task = group_or_task.group_name + else: + group_config = None + + if isinstance(group_or_task_info, Task): + if task_root: + task_aggregation_list.setdefault(task_root, []).append( + group_or_task_info.task_name + ) + else: + ( + results, + versions, + show_group_table, + _task_aggregation_list, + ) = consolidate_group_results( + results, + versions, + group_or_task_info, + group_or_task, + show_group_table, + task_aggregation_list, + ) + if task_root: + task_aggregation_list.setdefault(task_root, []).extend( + task_aggregation_list.get(group_or_task, []) + ) + + if (group_config is None) or ( + group_config["aggregate_metric_list"] is None + ): + results[group_or_task][" "] = " " + continue + + if "aggregate_metric_list" in group_config: + agg_metric_list = group_config["aggregate_metric_list"] + + show_group_table = show_group_table | bool( + group_config["aggregate_metric_list"] + ) + + task_list = _task_aggregation_list[group_or_task] + + metric_list = list( + { + key + for task in task_list + for key in results[task].keys() + if "_stderr" not in key and key not in ["task", "alias", "samples"] + } + ) + for metric in metric_list: + stderr = "_stderr,".join(metric.split(",")) + + # gather metrics, sizes, and stderrs from subtasks + metrics = [ + results[task][metric] + for task in task_list + if metric in results[task] + ] # TODO: copy? + stderrs = [ + results[task][stderr] + for task in task_list + if stderr in results[task] + ] + sizes = [ + results[task]["samples"] + for task in task_list + if metric in results[task] + ] + + for metric_config in agg_metric_list: + for filter_name in metric_config["filter_list"]: + if metric != ",".join([metric_config["metric"], filter_name]): + continue + + # compute group's pooled metric and stderr + if metric_config["aggregation"] == "mean": + aggregate_fn = aggregate_subtask_metrics + elif callable(metric_config["aggregation"]): + aggregate_fn = metric_config["aggregation"] + else: + raise ValueError( + f"Currently, only 'mean' is supported for automatically aggregating scores across groups' subtasks. Got '{metric_config['aggregation']}' for group '{group_or_task}'" + ) + + results[group_or_task][metric] = aggregate_fn( + metrics, + sizes, + metric_config["weight_by_size"], + ) + # TODO: calculate groups' metrics using arbitrary agg fns + if "N/A" in stderrs: + results[group_or_task][stderr] = "N/A" + else: + # NOTE: this assumes we are using the mean to aggregate. There are warnings about this elsewhere + results[group_or_task][stderr] = pooled_sample_stderr( + stderrs, sizes + ) + + results[group_or_task]["samples"] = sum(sizes) + group_metadata = group_config.get("metadata", None) + if group_metadata is not None: + versions[group_or_task] = group_metadata.get("version", None) + # print(results) + return results, versions, show_group_table, task_aggregation_list + + @positional_deprecated def find_test_root(start_path: pathlib.Path) -> pathlib.Path: """ diff --git a/lm_eval/filters/extraction.py b/lm_eval/filters/extraction.py index 9230808a6678e652821889f8060af5a441c221f4..41dc6208ce67ce36d69b2d91dcb6815a3fefb5a9 100644 --- a/lm_eval/filters/extraction.py +++ b/lm_eval/filters/extraction.py @@ -62,11 +62,8 @@ class WhitespaceFilter(Filter): def filter_set(inst): filtered_resp = [] for resp in inst: - if resp.startswith(" "): - resp = resp[1:] - + resp = resp.lstrip() filtered_resp.append(resp) - return filtered_resp filtered_resps = [filter_set(resp) for resp in resps] diff --git a/lm_eval/loggers/evaluation_tracker.py b/lm_eval/loggers/evaluation_tracker.py index 6aa7f25bef9454534855d75ed584f025bb4e6401..067b047b599fac2a0045f3a32e42b6ecec0afcaf 100644 --- a/lm_eval/loggers/evaluation_tracker.py +++ b/lm_eval/loggers/evaluation_tracker.py @@ -1,4 +1,5 @@ import json +import os import re import time from collections import defaultdict @@ -14,6 +15,7 @@ from huggingface_hub import ( HfApi, hf_hub_url, ) +from huggingface_hub.utils import build_hf_headers, get_session, hf_raise_for_status from lm_eval.utils import ( eval_logger, @@ -112,12 +114,15 @@ class EvaluationTracker: output_path: str = None, hub_results_org: str = "", hub_repo_name: str = "", + details_repo_name: str = "", + results_repo_name: str = "", push_results_to_hub: bool = False, push_samples_to_hub: bool = False, public_repo: bool = False, token: str = "", leaderboard_url: str = "", point_of_contact: str = "", + gated: bool = False, ) -> None: """ Creates all the necessary loggers for evaluation tracking. @@ -126,12 +131,15 @@ class EvaluationTracker: output_path (str): Path to save the results. If not provided, the results won't be saved. hub_results_org (str): The Hugging Face organization to push the results to. If not provided, the results will be pushed to the owner of the Hugging Face token. hub_repo_name (str): The name of the Hugging Face repository to push the results to. If not provided, the results will be pushed to `lm-eval-results`. + details_repo_name (str): The name of the Hugging Face repository to push the details to. If not provided, the results will be pushed to `lm-eval-results`. + result_repo_name (str): The name of the Hugging Face repository to push the results to. If not provided, the results will not be pushed and will be found in the details_hub_repo. push_results_to_hub (bool): Whether to push the results to the Hugging Face hub. push_samples_to_hub (bool): Whether to push the samples to the Hugging Face hub. public_repo (bool): Whether to push the results to a public or private repository. token (str): Token to use when pushing to the Hugging Face hub. This token should have write access to `hub_results_org`. leaderboard_url (str): URL to the leaderboard on the Hugging Face hub on the dataset card. point_of_contact (str): Contact information on the Hugging Face hub dataset card. + gated (bool): Whether to gate the repository. """ self.general_config_tracker = GeneralConfigTracker() @@ -142,6 +150,7 @@ class EvaluationTracker: self.leaderboard_url = leaderboard_url self.point_of_contact = point_of_contact self.api = HfApi(token=token) if token else None + self.gated_repo = gated if not self.api and (push_results_to_hub or push_samples_to_hub): raise ValueError( @@ -159,9 +168,24 @@ class EvaluationTracker: f"hub_results_org was not specified. Results will be pushed to '{hub_results_org}'." ) - hub_repo_name = hub_repo_name if hub_repo_name else "lm-eval-results" - self.hub_results_repo = f"{hub_results_org}/{hub_repo_name}" - self.hub_results_repo_private = f"{hub_results_org}/{hub_repo_name}-private" + if hub_repo_name == "": + details_repo_name = ( + details_repo_name if details_repo_name != "" else "lm-eval-results" + ) + results_repo_name = ( + results_repo_name if results_repo_name != "" else details_repo_name + ) + else: + details_repo_name = hub_repo_name + results_repo_name = hub_repo_name + eval_logger.warning( + "hub_repo_name was specified. Both details and results will be pushed to the same repository. Using hub_repo_name is no longer recommended, details_repo_name and results_repo_name should be used instead." + ) + + self.details_repo = f"{hub_results_org}/{details_repo_name}" + self.details_repo_private = f"{hub_results_org}/{details_repo_name}-private" + self.results_repo = f"{hub_results_org}/{results_repo_name}" + self.results_repo_private = f"{hub_results_org}/{results_repo_name}-private" def save_results_aggregated( self, @@ -211,9 +235,9 @@ class EvaluationTracker: if self.api and self.push_results_to_hub: repo_id = ( - self.hub_results_repo + self.results_repo if self.public_repo - else self.hub_results_repo_private + else self.results_repo_private ) self.api.create_repo( repo_id=repo_id, @@ -221,10 +245,15 @@ class EvaluationTracker: private=not self.public_repo, exist_ok=True, ) - self.api.upload_folder( + self.api.upload_file( repo_id=repo_id, - folder_path=str(path), - path_in_repo=self.general_config_tracker.model_name_sanitized, + path_or_fileobj=str( + path.joinpath(f"results_{self.date_id}.json") + ), + path_in_repo=os.path.join( + self.general_config_tracker.model_name, + f"results_{self.date_id}.json", + ), repo_type="dataset", commit_message=f"Adding aggregated results for {self.general_config_tracker.model_name}", ) @@ -278,6 +307,7 @@ class EvaluationTracker: sample["resps"] = sanitize_list(sample["resps"]) sample["filtered_resps"] = sanitize_list(sample["filtered_resps"]) sample["arguments"] = arguments + sample["target"] = str(sample["target"]) sample_dump = ( json.dumps( @@ -288,14 +318,14 @@ class EvaluationTracker: + "\n" ) - with open(file_results_samples, "a") as f: + with open(file_results_samples, "a", encoding="utf-8") as f: f.write(sample_dump) if self.api and self.push_samples_to_hub: repo_id = ( - self.hub_results_repo + self.details_repo if self.public_repo - else self.hub_results_repo_private + else self.details_repo_private ) self.api.create_repo( repo_id=repo_id, @@ -303,6 +333,18 @@ class EvaluationTracker: private=not self.public_repo, exist_ok=True, ) + try: + if self.gated_repo: + headers = build_hf_headers() + r = get_session().put( + url=f"https://huggingface.co/api/datasets/{repo_id}/settings", + headers=headers, + json={"gated": "auto"}, + ) + hf_raise_for_status(r) + except Exception as e: + eval_logger.warning("Could not gate the repository") + eval_logger.info(repr(e)) self.api.upload_folder( repo_id=repo_id, folder_path=str(path), @@ -327,9 +369,7 @@ class EvaluationTracker: """ eval_logger.info("Recreating metadata card") - repo_id = ( - self.hub_results_repo if self.public_repo else self.hub_results_repo_private - ) + repo_id = self.details_repo if self.public_repo else self.details_repo_private files_in_repo = self.api.list_repo_files(repo_id=repo_id, repo_type="dataset") results_files = get_results_filenames(files_in_repo) @@ -360,7 +400,10 @@ class EvaluationTracker: results_datetime, ) latest_task_results_datetime[samples_key] = latest_datetime - latest_task_results_datetime[results_key] = latest_datetime + latest_task_results_datetime[results_key] = max( + latest_task_results_datetime[results_key], + latest_datetime, + ) # Create metadata card card_metadata = MetadataConfigs() @@ -377,14 +420,15 @@ class EvaluationTracker: sanitized_last_eval_date_results = re.sub( r"[^\w\.]", "_", latest_task_results_datetime[config_name] ) - # Ensure that all results files are listed in the metadata card - current_results = card_metadata.get(config_name, {"data_files": []}) - current_results["data_files"].append( - {"split": eval_date_sanitized, "path": [str(results_filename)]} - ) - card_metadata[config_name] = current_results - # If the results file is the newest, update the "latest" field in the metadata card + if eval_date_sanitized == sanitized_last_eval_date_results: + # Ensure that all results files are listed in the metadata card + current_results = card_metadata.get(config_name, {"data_files": []}) + current_results["data_files"].append( + {"split": eval_date_sanitized, "path": [str(results_filename)]} + ) + card_metadata[config_name] = current_results + # If the results file is the newest, update the "latest" field in the metadata card card_metadata[config_name]["data_files"].append( {"split": "latest", "path": [str(results_filename)]} ) @@ -403,65 +447,20 @@ class EvaluationTracker: sanitized_last_eval_date_results = re.sub( r"[^\w\.]", "_", latest_task_results_datetime[config_name] ) - # Ensure that all sample results files are listed in the metadata card - current_details_for_task = card_metadata.get( - config_name, {"data_files": []} - ) - current_details_for_task["data_files"].append( - {"split": eval_date_sanitized, "path": [str(results_filename)]} - ) - card_metadata[config_name] = current_details_for_task - # If the samples results file is the newest, update the "latest" field in the metadata card if eval_date_sanitized == sanitized_last_eval_date_results: + # Ensure that all sample results files are listed in the metadata card + current_details_for_task = card_metadata.get( + config_name, {"data_files": []} + ) + current_details_for_task["data_files"].append( + {"split": eval_date_sanitized, "path": [str(results_filename)]} + ) + card_metadata[config_name] = current_details_for_task + # If the samples results file is the newest, update the "latest" field in the metadata card card_metadata[config_name]["data_files"].append( {"split": "latest", "path": [str(results_filename)]} ) - # Special case for MMLU with a single split covering it all - # We add another config with all MMLU splits results together for easy inspection - SPECIAL_TASKS = ["mmlu", "gpqa", "minerva_math"] - for special_task in SPECIAL_TASKS: - if special_task in config_name: - special_task = f"{model_name}__{special_task}" - former_entry = card_metadata.get(special_task, {"data_files": []}) - - former_split = [ - (i, entry) - for i, entry in enumerate(former_entry["data_files"]) - if entry.get("split", None) == eval_date_sanitized - ] - - if len(former_split) == 0: - former_entry["data_files"].append( - { - "split": eval_date_sanitized, - "path": [str(results_filename)], - } - ) - else: - split_index, _ = former_split[0] - former_entry["data_files"][split_index]["path"].append( - str(results_filename) - ) - - if eval_date_sanitized == sanitized_last_eval_date_results: - latest_split = [ - (i, entry) - for i, entry in enumerate(former_entry["data_files"]) - if entry.get("split", None) == "latest" - ] - if len(latest_split) == 0: - former_entry["data_files"].append( - {"split": "latest", "path": [str(results_filename)]} - ) - else: - latest_index, _ = latest_split[0] - former_entry["data_files"][latest_index]["path"].append( - str(results_filename) - ) - - card_metadata[special_task] = former_entry - # Get latest results and extract info to update metadata card examples latest_datetime = max(latest_task_results_datetime.values()) latest_model_name = max( diff --git a/lm_eval/loggers/utils.py b/lm_eval/loggers/utils.py index 670cdbff1002e90d4453dab3bcca1ea722fcad66..ded8f820ec8c8658becbcd5e18304158c294e91e 100644 --- a/lm_eval/loggers/utils.py +++ b/lm_eval/loggers/utils.py @@ -114,15 +114,29 @@ def add_env_info(storage: Dict[str, Any]): def add_tokenizer_info(storage: Dict[str, Any], lm): if getattr(lm, "tokenizer", False): - tokenizer_info = { - "tokenizer_pad_token": [lm.tokenizer.pad_token, lm.tokenizer.pad_token_id], - "tokenizer_eos_token": [lm.tokenizer.eos_token, lm.tokenizer.eos_token_id], - "tokenizer_bos_token": [lm.tokenizer.bos_token, lm.tokenizer.bos_token_id], - "eot_token_id": getattr(lm, "eot_token_id", None), - "max_length": getattr(lm, "max_length", None), - } - storage.update(tokenizer_info) - # seems gguf and textsynth do not have tokenizer + try: + tokenizer_info = { + "tokenizer_pad_token": [ + lm.tokenizer.pad_token, + str(lm.tokenizer.pad_token_id), + ], + "tokenizer_eos_token": [ + lm.tokenizer.eos_token, + str(lm.tokenizer.eos_token_id), + ], + "tokenizer_bos_token": [ + lm.tokenizer.bos_token, + str(lm.tokenizer.bos_token_id), + ], + "eot_token_id": getattr(lm, "eot_token_id", None), + "max_length": getattr(lm, "max_length", None), + } + storage.update(tokenizer_info) + except Exception as err: + logger.debug( + f"Logging detailed tokenizer info failed with {err}, skipping..." + ) + # seems gguf and textsynth do not have tokenizer else: logger.debug( "LM does not have a 'tokenizer' attribute, not logging tokenizer metadata to results." diff --git a/lm_eval/models/__init__.py b/lm_eval/models/__init__.py index 698c912f277fb1de6cca0ab4068e399bcbd29607..cde586ec9fbaaf37826a1925e1e105549fb554ff 100644 --- a/lm_eval/models/__init__.py +++ b/lm_eval/models/__init__.py @@ -1,8 +1,11 @@ from . import ( anthropic_llms, + api_models, dummy, gguf, + hf_vlms, huggingface, + ibm_watsonx_ai, mamba_lm, nemo_lm, neuralmagic, @@ -11,6 +14,7 @@ from . import ( optimum_lm, textsynth, vllm_causallms, + vllm_vlms, ) diff --git a/lm_eval/models/anthropic_llms.py b/lm_eval/models/anthropic_llms.py index d011fe074f931fb9e83b7b50baaef67ae47f9c5e..daba6734d2a362c1799f0aff2362ae97a249f7e5 100644 --- a/lm_eval/models/anthropic_llms.py +++ b/lm_eval/models/anthropic_llms.py @@ -1,10 +1,13 @@ -from typing import Any, List, Tuple +import os +from functools import cached_property +from typing import Any, Dict, List, Tuple, Union from tqdm import tqdm from lm_eval import utils from lm_eval.api.model import LM from lm_eval.api.registry import register_model +from lm_eval.models.openai_completions import LocalCompletionsAPI from lm_eval.models.utils import retry_on_specific_exceptions @@ -42,8 +45,8 @@ def anthropic_completion( try: import anthropic - except ModuleNotFoundError: - raise Exception( + except ModuleNotFoundError as exception: + raise type(exception)( "attempted to use 'anthropic' LM type, but package `anthropic` is not installed. \ please install anthropic via `pip install 'lm-eval[anthropic]'` or `pip install -e '.[anthropic]'`", ) @@ -105,8 +108,8 @@ def anthropic_chat( try: import anthropic - except ModuleNotFoundError: - raise Exception( + except ModuleNotFoundError as exception: + raise type(exception)( "attempted to use 'anthropic' LM type, but package `anthropic` is not installed. \ please install anthropic via `pip install 'lm-eval[anthropic]'` or `pip install -e '.[anthropic]'`", ) @@ -138,7 +141,7 @@ please install anthropic via `pip install 'lm-eval[anthropic]'` or `pip install return messages() -@register_model("anthropic") +@register_model("anthropic-completions") class AnthropicLM(LM): REQ_CHUNK_SIZE = 20 # TODO: not used @@ -165,8 +168,8 @@ class AnthropicLM(LM): try: import anthropic - except ModuleNotFoundError: - raise Exception( + except ModuleNotFoundError as exception: + raise type(exception)( "attempted to use 'anthropic' LM type, but package `anthropic` is not installed. \ please install anthropic via `pip install 'lm-eval[anthropic]'` or `pip install -e '.[anthropic]'`", ) @@ -214,8 +217,8 @@ please install anthropic via `pip install 'lm-eval[anthropic]'` or `pip install def generate_until(self, requests, disable_tqdm: bool = False) -> List[str]: try: import anthropic - except ModuleNotFoundError: - raise Exception( + except ModuleNotFoundError as exception: + raise type(exception)( "attempted to use 'anthropic' LM type, but package `anthropic` is not installed. \ please install anthropic via `pip install 'lm-eval[anthropic]'` or `pip install -e '.[anthropic]'`", ) @@ -271,90 +274,89 @@ please install anthropic via `pip install 'lm-eval[anthropic]'` or `pip install @register_model("anthropic-chat", "anthropic-chat-completions") -class AnthropicChatLM(AnthropicLM): - REQ_CHUNK_SIZE = 20 # TODO: not used - +class AnthropicChat(LocalCompletionsAPI): def __init__( self, - model: str, - batch_size: int = 1, - max_tokens: int = 256, - temperature: float = 0, # defaults to 1 - **kwargs, # top_p, top_k, etc. - ) -> None: - """Anthropic API wrapper. - - :param model: str - Anthropic model e.g. 'claude-3-opus-20240229', 'claude-3-sonnet-20240229' - :param max_tokens: int - Maximum number of tokens to sample from the model - :param temperature: float - Sampling temperature - :param kwargs: Any - Additional model_args to pass to the API client - """ - super().__init__() + base_url="https://api.anthropic.com/v1/messages", + tokenizer_backend=None, + **kwargs, + ): + super().__init__( + base_url=base_url, tokenizer_backend=tokenizer_backend, **kwargs + ) + eval_logger.warning( + "Chat completions does not support batching. Defaulting to batch size 1." + ) + self._batch_size = 1 + self.anthropic_version = "2023-06-01" + eval_logger.warning( + f"Using Anthropic Version: {self.anthropic_version}. Confirm the current version here: https://docs.anthropic.com/en/api/versioning" + ) - try: - import anthropic - except ModuleNotFoundError: - raise Exception( - "attempted to use 'anthropic' LM type, but package `anthropic` is not installed. \ -please install anthropic via `pip install 'lm-eval[anthropic]'` or `pip install -e '.[anthropic]'`", + @cached_property + def api_key(self): + """Override this property to return the API key for the API request.""" + key = os.environ.get("ANTHROPIC_API_KEY", None) + if key is None: + raise ValueError( + "API key not found. Please set the ANTHROPIC_API_KEY environment variable." ) - - self.model = model - # defaults to os.environ.get("ANTHROPIC_API_KEY") - self.client = anthropic.Anthropic() - self.temperature = temperature - self.max_tokens = max_tokens - self.tokenizer = self.client.get_tokenizer() - self.kwargs = kwargs - - @property - def max_gen_toks(self) -> int: - return self.max_tokens - - def generate_until(self, requests) -> List[str]: - try: - import anthropic - except ModuleNotFoundError: - raise Exception( - "attempted to use 'anthropic' LM type, but package `anthropic` is not installed. \ -please install anthropic via `pip install 'lm-eval[anthropic]'` or `pip install -e '.[anthropic]'`", - ) - - if not requests: - return [] - - _requests: List[Tuple[str, dict]] = [req.args for req in requests] - + return key + + @cached_property + def header(self): + return { + "x-api-key": f"{self.api_key}", + "anthropic-version": self.anthropic_version, + } + + def _create_payload( + self, messages: List[Dict], generate=True, gen_kwargs: dict = None, **kwargs + ) -> dict: + system = ( + messages[0].get("content") if messages[0].get("role") == "system" else None + ) + if system: + messages = messages[1:] + gen_kwargs.pop("do_sample", False) + max_tokens = gen_kwargs.pop("max_gen_toks", self._max_gen_toks) + temperature = gen_kwargs.pop("temperature", 0) + stop = gen_kwargs.pop("until", ["\n\nHuman:"]) + if not isinstance(stop, list): + stop = [stop] + out = { + "messages": messages, + "model": self.model, + "max_tokens": max_tokens, + "temperature": temperature, + "stop_sequences": stop, + **gen_kwargs, + } + if system: + out["system"] = system + return out + + def parse_generations( + self, outputs: Union[Dict, List[Dict]], **kwargs + ) -> List[str]: res = [] - for request in tqdm(_requests): - try: - inp = request[0] - request_args = request[1] - # generation_kwargs - until = request_args.get("until") - max_tokens = request_args.get("max_gen_toks", self.max_length) - temperature = request_args.get("temperature", self.temperature) - response = anthropic_chat( - client=self.client, - model=self.model, - prompt=inp, - max_tokens=max_tokens, - temperature=temperature, # TODO: implement non-greedy sampling for Anthropic - stop=until, # type: ignore - **self.kwargs, - ) - res.append(response) - - self.cache_hook.add_partial("generate_until", request, response) - except anthropic.APIConnectionError as e: # type: ignore # noqa: F821 - eval_logger.critical(f"Server unreachable: {e.__cause__}") - break - except anthropic.APIStatusError as e: # type: ignore # noqa: F821 - eval_logger.critical(f"API error {e.status_code}: {e.message}") - break - + if not isinstance(outputs, list): + outputs = [outputs] + for out in outputs: + for choices in out["content"]: + res.append(choices["text"]) return res + + def tok_encode( + self, + string: str, + left_truncate_len=None, + add_special_tokens=None, + **kwargs, + ) -> List[str]: + return [string] + + def loglikelihood(self, requests, **kwargs): + raise NotImplementedError( + "Anthropic Chat Completions API does not support the return of loglikelihood" + ) diff --git a/lm_eval/models/api_models.py b/lm_eval/models/api_models.py new file mode 100644 index 0000000000000000000000000000000000000000..e23d3e6f6bb0e48ce89c5e1d2a81da035357e27a --- /dev/null +++ b/lm_eval/models/api_models.py @@ -0,0 +1,652 @@ +import abc +import asyncio +import copy +import itertools +import json +from functools import cached_property +from typing import ( + Any, + Awaitable, + Callable, + Dict, + Iterable, + List, + Literal, + NamedTuple, + Optional, + Tuple, + Union, +) + + +try: + import requests + from aiohttp import ClientSession, TCPConnector + from tenacity import RetryError, retry, stop_after_attempt, wait_exponential + from tqdm import tqdm + from tqdm.asyncio import tqdm_asyncio +except ModuleNotFoundError: + pass + + +from importlib.util import find_spec + +from lm_eval import utils +from lm_eval.api.instance import Instance +from lm_eval.api.model import TemplateLM +from lm_eval.models.utils import Collator, chunks, configure_pad_token + + +LogLikelihoodInputs = Tuple[Tuple[str, str], List[int], List[int]] + + +# utility class to keep track of json encoded chats +class JsonChatStr(NamedTuple): + prompt: str + + def encode(self, encoding): + return self.prompt.encode(encoding) + + +eval_logger = utils.eval_logger + + +class TemplateAPI(TemplateLM): + def __init__( + self, + model: str = None, + pretrained: str = None, # `model` takes precedence over `pretrained` when passed. + base_url: str = None, + tokenizer: Optional[str] = None, + # Loglikelihood tasks require a tokenizer to calculate context lengths, + # however the requests can be sent as a string if the API doesn't support token inputs. + # use tokenized_requests=False + tokenizer_backend: Optional[ + Literal["tiktoken", "huggingface", None] + ] = "huggingface", + truncate: bool = False, + # number of concurrent requests. More useful if not batching + num_concurrent: int = 1, + max_retries: int = 3, + max_gen_toks: int = 256, + batch_size: Union[str, int] = 1, + seed: int = 1234, + max_length: Optional[int] = 2048, + add_bos_token: bool = False, + custom_prefix_token_id: int = None, + # send the requests as tokens or strings + tokenized_requests: bool = True, + trust_remote_code: bool = False, + revision: Optional[str] = "main", + use_fast_tokenizer: bool = True, + verify_certificate: bool = True, + **kwargs, + ) -> None: + super().__init__() + missing_packages = [ + pkg + for pkg in ["aiohttp", "tqdm", "tenacity", "requests"] + if find_spec(pkg) is None + ] + if missing_packages: + raise ModuleNotFoundError( + f"Attempted to use an API model, but the required packages {missing_packages} are not installed. " + 'Please install these via `pip install lm-eval[api]` or `pip install -e ."[api]"`' + ) + self.model = model or pretrained + self.base_url = base_url + self.tokenizer = tokenizer + if not isinstance(batch_size, int) and "auto" in batch_size: + eval_logger.warning( + "Automatic batch size is not supported for API models. Defaulting to batch size 1." + ) + elif int(batch_size) > 1: + eval_logger.warning( + "Batch size > 1 detected. Ensure your API supports batched requests with varying total sequence lengths." + ) + self._batch_size = int(batch_size) if batch_size != "auto" else 1 + self._truncate = truncate + self._max_gen_toks = int(max_gen_toks) + self._seed = int(seed) + # max_length - 1 as we always have 1 token for generation + eval_logger.info(f"Using max length {max_length} - 1") + self.max_length = max_length - 1 + if int(num_concurrent) <= 1: + eval_logger.info( + "Concurrent requests are disabled. To enable concurrent requests, set `num_concurrent` > 1." + ) + self._concurrent = int(num_concurrent) + self.tokenizer_backend = tokenizer_backend + self.add_bos_token = add_bos_token + self.custom_prefix_token_id = custom_prefix_token_id + self.tokenized_requests = tokenized_requests + self.max_retries = int(max_retries) + self.verify_certificate = verify_certificate + + eval_logger.info(f"Using tokenizer {self.tokenizer_backend}") + if self.tokenizer_backend is None: + self.tokenizer = None + self.tokenized_requests = False + else: + if self.tokenizer is None: + if self.tokenizer_backend == "huggingface": + import transformers + + self.tokenizer = transformers.AutoTokenizer.from_pretrained( + self.tokenizer if self.tokenizer else self.model, + trust_remote_code=trust_remote_code, + revision=revision, + use_fast=use_fast_tokenizer, + ) + # Not used as the API will handle padding but to mirror the behavior of the HFLM + self.tokenizer = configure_pad_token(self.tokenizer) + elif self.tokenizer_backend == "tiktoken": + try: + import tiktoken + + self.tokenizer = tiktoken.encoding_for_model(self.model) + except ModuleNotFoundError as e: + raise ModuleNotFoundError( + "Attempted to use 'openai' LM type, but the package `tiktoken` is not installed. " + "Please install it via `pip install lm-eval[api]` or `pip install -e .[api]`." + ) from e + if "openai" not in self.base_url: + eval_logger.warning( + f"Passed `base_url={self.base_url}` but using (OpenAI) Tiktoken tokenizer backend. " + "Pass `tokenizer_backend=huggingface` and provide the HF tokenizer name if your model does not use Tiktoken." + ) + else: + import transformers + + assert isinstance(tokenizer, str), "tokenizer must be a string" + self.tokenizer = transformers.AutoTokenizer.from_pretrained( + tokenizer, + trust_remote_code=trust_remote_code, + revision=revision, + use_fast=use_fast_tokenizer, + ) + + @abc.abstractmethod + def _create_payload( + self, + messages: Union[List[List[int]], List[dict], List[str], str], + *, + generate: bool = True, + gen_kwargs: Optional[dict] = None, + seed: int = 1234, + **kwargs, + ) -> dict: + """This method is responsible for creating the json payload that will be sent to the API.""" + raise NotImplementedError + + def create_message( + self, + messages: Union[List[List[int]], List[str], List[JsonChatStr]], + generate=False, + ) -> Union[List[List[int]], List[dict], List[str], str]: + """Helper method to transform the prompt into the expected API input format. messages consist of batched requests""" + if isinstance(messages[0], JsonChatStr): + # for chat completions we need to decode the json string to list[dict,...] + assert ( + self._batch_size == 1 + ), "non-tokenized chat requests are only supported with batch_size=1" + # list[dict["role":..., "content":...],...] + return json.loads(messages[0].prompt) + + if not self.tokenized_requests: + # if messages are tokenized: + if isinstance(messages[0][0], int): + # assuming decoding is lossless. However, this is only for loglikelihood requests + # as we need to compute the context length. For generations, we don't need to tokenize. + messages = self.decode_batch(messages) + if self._batch_size <= 1: + # if batch is 1 return str + return messages[0] + else: + # list[str,...] + return messages + + # list[list[int], ...] + return messages + + @staticmethod + @abc.abstractmethod + def parse_logprobs( + outputs: Union[Any, List[Any]], + tokens: List[List[int]] = None, + ctxlen: List[int] = None, + **kwargs, + ) -> List[Tuple[float, bool]]: + """Method used to parse the logprobs from the (batched) API response. This method should return a list of tuples""" + raise NotImplementedError + + @staticmethod + @abc.abstractmethod + def parse_generations(outputs: Union[Any, List[Any]], **kwargs) -> List[str]: + """Method used to parse the generations from the (batched) API response. This method should return a list of str""" + raise NotImplementedError + + @cached_property + def api_key(self) -> str: + """Override this property to return the API key for the API request.""" + return "" + + @cached_property + def header(self) -> dict: + """Override this property to return the headers for the API request.""" + return {"Authorization": f"Bearer {self.api_key}"} + + @property + def tokenizer_name(self) -> str: + """Must be defined for LM subclasses which implement Chat Templating. + Should return the name of the tokenizer or chat template used. + Used only to properly fingerprint caches when requests are being cached with `--cache_requests`, otherwise not used. + """ + return "" + + def apply_chat_template( + self, chat_history: List[Dict[str, str]] + ) -> Union[str, JsonChatStr]: + """Applies a chat template to a list of chat history between user and model.""" + if self.tokenizer_backend == "huggingface" and self.tokenized_requests: + return self.tokenizer.apply_chat_template( + chat_history, tokenize=False, add_generation_prompt=True + ) + else: + # bit of a hack. We'll load back before sending to the API + return JsonChatStr(json.dumps(chat_history)) + + @cached_property + def eot_token_id(self) -> Optional[int]: + if self.tokenizer is None: + return None + else: + if self.tokenizer_backend == "huggingface": + return self.tokenizer.eos_token_id + elif self.tokenizer_backend == "tiktoken": + return self.tokenizer.eot_token + + @cached_property + def prefix_token_id(self) -> Optional[int]: + if self.tokenizer is None: + return None + else: + if self.custom_prefix_token_id is not None: + return self.custom_prefix_token_id + if self.tokenizer_backend == "huggingface": + if self.tokenizer.bos_token_id is not None: + return self.tokenizer.bos_token_id + return self.tokenizer.eos_token_id + else: + return self.tokenizer.eot_token + + def tok_encode( + self, + string: str, + left_truncate_len: int = None, + add_special_tokens: bool = False, + truncation: bool = False, + **kwargs, + ) -> Union[List[List[int]], List[int], List[str]]: + if self.tokenizer_backend is None: + return [string] + elif self.tokenizer_backend == "huggingface": + # by default for CausalLM - false or self.add_bos_token is set + if not add_special_tokens: + add_special_tokens = False or self.add_bos_token + encoding: Union[List[List[int]], List[int]] = self.tokenizer( + string, + add_special_tokens=add_special_tokens, + truncation=truncation, + return_attention_mask=False, + ).input_ids + + # left-truncate the encoded context to be at most `left_truncate_len` tokens long + if left_truncate_len: + if not isinstance(string, str): + encoding = [enc[-left_truncate_len:] for enc in encoding] + else: + encoding = encoding[-left_truncate_len:] + + return encoding + + else: + try: + encoding = self.tokenizer.encode(string) + except Exception: + encoding = self.tokenizer.encode_batch(string) + return encoding + + def decode_batch(self, tokens: List[List[int]]) -> List[str]: + if self.tokenizer_backend == "huggingface": + return self.tokenizer.batch_decode(tokens) + elif self.tokenizer_backend == "tiktoken": + return self.tokenizer.decode_batch(tokens) + + def model_call( + self, + messages: Union[List[List[int]], List[str], List[JsonChatStr]], + *, + generate: bool = True, + gen_kwargs: Optional[Dict] = None, + **kwargs, + ) -> Optional[dict]: + # !!! Copy: shared dict for each request, need new object !!! + gen_kwargs = copy.deepcopy(gen_kwargs) + try: + response = requests.post( + self.base_url, + json=self._create_payload( + self.create_message(messages), + generate=generate, + gen_kwargs=gen_kwargs, + seed=self._seed, + **kwargs, + ), + headers=self.header, + verify=self.verify_certificate, + ) + if not response.ok: + eval_logger.warning( + f"API request failed with error message: {response.text}. Retrying..." + ) + response.raise_for_status() + return response.json() + except RetryError: + eval_logger.error( + "API request failed after multiple retries. Please check the API status." + ) + return None + + async def amodel_call( + self, + session: ClientSession, + messages: Union[List[List[int]], List[str], List[JsonChatStr]], + *, + generate: bool = True, + cache_keys: list = None, + ctxlens: Optional[List[int]] = None, + gen_kwargs: Optional[Dict] = None, + **kwargs, + ) -> Union[List[str], List[Tuple[float, bool]], None]: + # !!! Copy: shared dict for each request, need new object !!! + gen_kwargs = copy.deepcopy(gen_kwargs) + payload = self._create_payload( + self.create_message(messages), + generate=generate, + gen_kwargs=gen_kwargs, + seed=self._seed, + **kwargs, + ) + cache_method = "generate_until" if generate else "loglikelihood" + try: + async with session.post( + self.base_url, + json=payload, + headers=self.header, + ) as response: + if not response.ok: + error_text = await response.text() + eval_logger.warning( + f"API request failed with error message: {error_text}. Retrying..." + ) + # raising exception will retry the request + response.raise_for_status() + outputs = await response.json() + answers = ( + self.parse_generations( + outputs=outputs, + ) + if generate + else self.parse_logprobs( + outputs=outputs, + tokens=messages, + ctxlens=ctxlens, + ) + ) + if cache_keys: + for res, cache in zip(answers, cache_keys): + self.cache_hook.add_partial(cache_method, cache, res) + return answers + # If the retries also fail + except RetryError: + eval_logger.error( + "API request failed after multiple retries. Please check the API status." + ) + return None + + def batch_loglikelihood_requests( + self, chunks: Iterable[List[LogLikelihoodInputs]] + ) -> Tuple[List[List[int]], List[int], List[Tuple[str, str]]]: + inputs = [] + ctxlens = [] + cache_keys = [] + for chunk in chunks: + for cache_key, context_enc, continuation_enc in chunk: + # max_length - 1 as we always have 1 token for generation + inp = (context_enc + continuation_enc)[-(self.max_length) :] + ctxlen = len(context_enc) - max( + 0, len(context_enc) + len(continuation_enc) - (self.max_length) + ) + + inputs.append(inp) + ctxlens.append(ctxlen) + cache_keys.append(cache_key) + return inputs, ctxlens, cache_keys + + async def get_batched_requests( + self, + requests: list, + cache_keys: list, + *, + generate: bool = True, + ctxlens: List[int] = None, + **kwargs, + ) -> Union[List[List[str]], List[List[Tuple[float, bool]]]]: + ctxlens = ctxlens if ctxlens else [None] * len(requests) + conn = TCPConnector(limit=self._concurrent) + async with ClientSession(connector=conn) as session: + retry_: Callable[..., Awaitable[Any]] = retry( + stop=stop_after_attempt(self.max_retries), + wait=wait_exponential(multiplier=0.5, min=1, max=10), + reraise=True, + )(self.amodel_call) + # Create tasks for each batch of request + tasks = [ + asyncio.create_task( + retry_( + session=session, + messages=message, + cache_keys=cache_key, + generate=generate, + ctxlens=ctxlen, + **kwargs, + ) + ) + for message, cache_key, ctxlen in zip( + chunks(requests, n=self._batch_size), + chunks(cache_keys, n=self._batch_size), + chunks(ctxlens, n=self._batch_size), + ) + ] + + return await tqdm_asyncio.gather(*tasks, desc="Requesting API") + + def _loglikelihood_tokens(self, requests, **kwargs) -> List[Tuple[float, bool]]: + assert ( + self.tokenizer is not None + ), "Tokenizer is required for loglikelihood tasks to compute context lengths." + res = [] + + def _collate(req: LogLikelihoodInputs): + """Defines the key for the sorted method""" + # the negative sign on len(toks) sorts descending - this has a few advantages: + # - time estimates will always be over not underestimates, which is more useful for planning + # - to know the size of a batch when going through the list, you know the first one is always the batch + # padded context length. this is useful to simplify the batching logic and more importantly to make + # automatic adaptive batches much much easier to implement + # - any OOMs will happen right away rather than near the end + + toks = req[1] + req[2] + return -len(toks), tuple(toks) + + re_ord = Collator( + requests, + sort_fn=_collate, + group_by=None, + ) + # if concurrent then we'll batch in the async context + chunked = re_ord.get_batched(n=self._batch_size if self._concurrent <= 1 else 0) + if self._concurrent <= 1: + pbar = tqdm(desc="Requesting API", total=len(requests)) + for chunk in chunked: + inputs, ctxlens, cache_keys = self.batch_loglikelihood_requests([chunk]) + + outputs = retry( + stop=stop_after_attempt(self.max_retries), + wait=wait_exponential(multiplier=0.5, min=1, max=10), + reraise=True, + )(self.model_call)(messages=inputs, generate=False) + if isinstance(outputs, dict): + outputs = [outputs] + for answer_, cache_key in zip( + self.parse_logprobs( + outputs=outputs, tokens=inputs, ctxlens=ctxlens + ), + cache_keys, + ): + if answer_ is not None: + res.append(answer_) + # cache requests that aren't from a loglikelihood_rolling request + if cache_key is not None: + self.cache_hook.add_partial( + "loglikelihood", cache_key, answer_ + ) + pbar.update(1) + else: + inputs, ctxlens, cache_keys = self.batch_loglikelihood_requests(chunked) + res = itertools.chain.from_iterable( + asyncio.run( + self.get_batched_requests( + inputs, cache_keys, generate=False, ctxlens=ctxlens + ) + ) + ) + + return re_ord.get_original(res) + + def generate_until( + self, requests: List[Instance], disable_tqdm: bool = False + ) -> List[str]: + res = [] + + def _collate_gen(_requests): + # sort by the length of the non-tokenized contexts + return -len(_requests[0]) + + # Let the API deal with tokenization + requests, all_gen_kwargs = zip(*(req.args for req in requests)) + if self.tokenized_requests: + encodings_list = self.tok_encode( + requests, add_special_tokens=self.add_bos_token + ) + else: + encodings_list = [None] * len(requests) + requests = [ + (a, b, c) for a, b, c in zip(requests, all_gen_kwargs, encodings_list) + ] + + re_ord = Collator( + requests, + sort_fn=_collate_gen, + group_by="gen_kwargs", + ) + chunked = re_ord.get_batched( + n=self._batch_size if self._concurrent <= 1 else 0, batch_fn=None + ) + if self._concurrent <= 1: + pbar = tqdm(desc="Requesting API", total=len(requests)) + for chunk in chunked: + contexts, all_gen_kwargs, encodings_list = zip(*chunk) + req = encodings_list if self.tokenized_requests else contexts + outputs = retry( + stop=stop_after_attempt(self.max_retries), + wait=wait_exponential(multiplier=0.5, min=1, max=10), + reraise=True, + )(self.model_call)( + messages=req, + generate=True, + gen_kwargs=copy.deepcopy(all_gen_kwargs[0]), + ) + for generated_text, context in zip( + self.parse_generations( + outputs=outputs, + contexts=contexts, + ), + contexts, + ): + if generated_text is not None: + res.append(generated_text) + + # partial caching + if context is not None: + self.cache_hook.add_partial( + "generate_until", + (context, all_gen_kwargs[0]), + generated_text, + ) + pbar.update(1) + else: + for chunk in chunked: + contexts, all_gen_kwargs, encodings_list = zip(*chunk) + req = encodings_list if self.tokenized_requests else contexts + results = itertools.chain.from_iterable( + asyncio.run( + self.get_batched_requests( + req, + cache_keys=[(ctx, all_gen_kwargs[0]) for ctx in contexts], + generate=True, + gen_kwargs=copy.deepcopy(all_gen_kwargs[0]), + ) + ) + ) + res.extend(results) + + return re_ord.get_original(res) + + def loglikelihood_rolling( + self, requests: List[Instance], disable_tqdm: bool = False + ) -> List[float]: + loglikelihoods = [] + + for (string,) in tqdm([req.args for req in requests], disable=disable_tqdm): + rolling_token_windows = list( + map( + utils.make_disjoint_window, + utils.get_rolling_token_windows( + token_list=self.tok_encode(string), + prefix_token=self.prefix_token_id, + # max_seq_len - (1 for context) + max_seq_len=self.max_length - 1, + context_len=1, + ), + ) + ) + + # TODO: Right now, we pass single EOT token to the Encoder and the full context to the decoder, in seq2seq case + rolling_token_windows = [(None,) + x for x in rolling_token_windows] + + string_nll = self._loglikelihood_tokens( + rolling_token_windows, + disable_tqdm=True, + ) + + # discard is_greedy + string_nll = [x[0] for x in string_nll] + + string_nll = sum(string_nll) + loglikelihoods.append(string_nll) + + # cache this loglikelihood_rolling request + self.cache_hook.add_partial("loglikelihood_rolling", (string,), string_nll) + return loglikelihoods diff --git a/lm_eval/models/dummy.py b/lm_eval/models/dummy.py index 83737739672724f5fd6581ad59955e555b770ec4..014ad49ee36f756acd0428340f945312d79590e8 100644 --- a/lm_eval/models/dummy.py +++ b/lm_eval/models/dummy.py @@ -26,9 +26,9 @@ class DummyLM(LM): def generate_until(self, requests, disable_tqdm: bool = False): res = [] - for ctx, _ in tqdm(requests, disable=disable_tqdm): + for request in tqdm(requests, disable=disable_tqdm): res.append("lol") - assert ctx.strip() != "" + assert request.arguments[0].strip() != "" return res diff --git a/lm_eval/models/gguf.py b/lm_eval/models/gguf.py index ee1362c6b0bedd8f831a1a4f93821b8c661f25e3..52aef0dee62edb2eb390fe695e939f8e06d0555f 100644 --- a/lm_eval/models/gguf.py +++ b/lm_eval/models/gguf.py @@ -68,7 +68,9 @@ class GGUFLM(LM): logger.error(f"RequestException: {e}") time.sleep(delay) # wait before retrying else: - raise Exception(f"Failed to get a valid response after {retries} retries.") + raise RuntimeError( + f"Failed to get a valid response after {retries} retries." + ) def loglikelihood(self, requests, disable_tqdm: bool = False): if not requests: diff --git a/lm_eval/models/hf_vlms.py b/lm_eval/models/hf_vlms.py new file mode 100644 index 0000000000000000000000000000000000000000..f2fcdd7027640dcfd1778d64737eb911781d7312 --- /dev/null +++ b/lm_eval/models/hf_vlms.py @@ -0,0 +1,724 @@ +import copy +from typing import Dict, List, Optional, Tuple, Union + +import torch +import torch.nn.functional as F +import transformers +from tqdm import tqdm +from transformers import BatchEncoding + +from lm_eval import utils +from lm_eval.api.instance import Instance +from lm_eval.api.registry import register_model +from lm_eval.models.huggingface import HFLM +from lm_eval.models.utils import ( + Collator, + flatten_image_list, + pad_and_concat, + replace_placeholders, + stop_sequences_criteria, +) + + +DEFAULT_IMAGE_PLACEHOLDER = "" + + +eval_logger = utils.eval_logger + + +@register_model("hf-multimodal") +class HFMultimodalLM(HFLM): + """ + An abstracted Hugging Face model class for multimodal LMs like Llava and Idefics. + """ + + AUTO_MODEL_CLASS = transformers.AutoModelForVision2Seq + MULTIMODAL = True # flag to indicate, for now, that this model type can run multimodal requests + + def __init__( + self, + pretrained: Union[str, transformers.PreTrainedModel], + image_token_id: Optional[int] = None, + image_string: Optional[str] = None, + interleave: bool = True, + # TODO: handle whitespace in image placeholder (replacement) + max_images: Optional[int] = 999, + convert_img_format=False, + **kwargs, + ): + # We initialize using HFLM's init. Sub-methods like _create_model and _create_tokenizer + # modify init behavior. + super().__init__(pretrained, **kwargs) + + assert ( + self.batch_size != "auto" + ), "Batch size 'auto' is not yet supported for hf-multimodal models." + self.chat_applied: bool = False + # TODO: phi-3.5 "image placeholders" are , , ... in order. how to handle this case + + # HF AutoModelForVision2Seq models have an `image_token_id` value in their configs + # denoting the token which indicates a location where an image will be substituted in. + # This can take different string values across models, e.g. for Idefics2 and <|image_pad|> for Qwen2-VL + self.interleave = interleave + self.max_images = max_images + self.rgb = convert_img_format + # WARNING: improperly set image_token_id can lead to ignored image input or other (potentially silent) errors! + if not image_string: + self.image_token_id = ( + int(image_token_id) + if image_token_id + else ( + getattr(self.config, "image_token_id", None) + or getattr(self.config, "image_token_index", None) + ) + ) + assert ( + self.image_token_id is not None + ), "Must have a non-None image_token_id to evaluate a Hugging Face AutoModelForVision2Seq model. Please pass `image_token_id` in `--model_args` if model's config does not already specify one." + # get the string this token ID corresponds to + self.image_token = self.tok_decode( + [self.image_token_id], skip_special_tokens=False + ) + if image_token_id is not None: + eval_logger.info( + f"A non-default image_token_id with image_token_id={self.image_token_id} and string value '{self.image_token}' was specified manually. Note that using an improper image_token placeholder may lead to ignored image input or errors!" + ) + else: + eval_logger.info( + f"A non-default image_token string with string value image_string='{image_string}' was specified manually. Note that using an improper image_token placeholder may lead to ignored image input or errors!" + ) + self.image_token = image_string + + def _create_tokenizer( + self, + pretrained: Union[str, transformers.PreTrainedModel], + tokenizer: Optional[ + Union[ + str, + transformers.ProcessorMixin, + ] + ], + revision: Optional[str] = "main", + trust_remote_code: Optional[bool] = False, + **kwargs, + ) -> None: + """ + Helper method during initialization. + + For the multimodal variant, we initialize not just + `self.tokenizer` but also `self.processor`. + """ + + if tokenizer: + if isinstance(tokenizer, str): + return transformers.AutoProcessor.from_pretrained( + tokenizer, + revision=revision, + trust_remote_code=trust_remote_code, + # use_fast=use_fast_tokenizer, + ) + else: + assert isinstance( + tokenizer, transformers.ProcessorMixin + ) # TODO: check this condition + return tokenizer + + # Get tokenizer based on 'pretrained' + if isinstance(pretrained, str): + model_name = pretrained + else: + # get the HF hub name via accessor on model + model_name = self.model.name_or_path + + self.processor = transformers.AutoProcessor.from_pretrained( + model_name, + revision=revision, + trust_remote_code=trust_remote_code, + # use_fast=use_fast_tokenizer, + ) + + self.tokenizer = self.processor.tokenizer + + def tok_multimodal_encode( + self, string, images, left_truncate_len=None, add_special_tokens=None + ): + """Helper function which encodes an image + string combo using AutoProcessor""" + # We inherit special token kwarg setup from HFLM.tok_encode + # special_tokens_kwargs = {} + + # by default for CausalLM - false or self.add_bos_token is set + # if add_special_tokens is None: + # special_tokens_kwargs = {"add_special_tokens": False or self.add_bos_token} + # otherwise the method explicitly defines the value + # else: + # special_tokens_kwargs = {"add_special_tokens": add_special_tokens} + + # encode text+images + # TODO: why does (Qwen2-VL) processor error when attempting to add special tokens to text? + encoding = self.processor( + text=string, images=images, return_tensors=None + ) # , **special_tokens_kwargs) + + # remove (and store) our tokenized text + text_encoding = encoding.pop("input_ids") + encoding.pop("attention_mask") + + # left-truncate the encoded context to be at most `left_truncate_len` tokens long + if left_truncate_len: + text_encoding = text_encoding[-left_truncate_len:] + + return text_encoding, encoding # image_encoding is a dict + + def _encode_multimodal_pair(self, context, continuation, images): + """Helper function to perform the role of TemplateLM._encode_pair + Except allowing for image input to also be processed alongside `context`. + + This method is a bit messy due to the need to defer conversion of image and text token input + into PyTorch tensors until the main inference loop. + """ + + n_spaces = len(context) - len(context.rstrip()) + if n_spaces > 0: + continuation = context[-n_spaces:] + continuation + context = context[:-n_spaces] + + # TODO: replace default placeholder with self.image_token, for contexts + + whole_enc, image_enc = self.tok_multimodal_encode( + context + continuation, images + ) + context_enc, _ = self.tok_multimodal_encode(context, images) + + # tok_multimodal_encode returns List[List[int]] for tokenized text. Get rid of the batch dim + # since we only are encoding a single string. + # TODO: this is a bit hacky, it'd be nice to make this generally cleaner + whole_enc, context_enc = whole_enc[0], context_enc[0] + + context_enc_len = len(context_enc) + continuation_enc = whole_enc[context_enc_len:] + + return context_enc, continuation_enc, image_enc + + def apply_chat_template(self, chat_history: List[Dict[str, str]]) -> str: + self.chat_applied = True + if not self.interleave: + for content in chat_history: + c = [] + text = content["content"] + + # Count and remove image placeholders + image_count = min( + self.max_images, text.count(DEFAULT_IMAGE_PLACEHOLDER) + ) + text = text.replace(DEFAULT_IMAGE_PLACEHOLDER, "") + + # Add image entries + for _ in range(image_count): + c.append({"type": "image", "image": None}) + + # Add single text entry at the end + c.append({"type": "text", "text": text}) + + content["content"] = c + else: + for content in chat_history: + c = [] + text = content["content"] + expected_image_count = min( + self.max_images, text.count(DEFAULT_IMAGE_PLACEHOLDER) + ) + actual_image_count = 0 + + text_parts = text.split(DEFAULT_IMAGE_PLACEHOLDER) + + for i, part in enumerate(text_parts): + # TODO: concatenate text parts (esp. if skipping images)? + if part: # Add non-empty text parts + c.append({"type": "text", "text": part}) + if ( + (i < len(text_parts) - 1) and i < self.max_images + ): # Add image placeholder after each split except the last + c.append({"type": "image"}) + actual_image_count += 1 + + content["content"] = c + + if actual_image_count != expected_image_count: + raise ValueError( + f"Mismatch in image placeholder count. Expected: {expected_image_count}, Actual: {actual_image_count}" + ) + + return self.processor.apply_chat_template( + chat_history, add_generation_prompt=True + ) + + def chat_template(self, chat_template: Union[bool, str] = False) -> Optional[str]: + if hasattr(self.processor, "apply_chat_template"): + _tokenizer = self.tokenizer + self.tokenizer = self.processor + + selected_template = super().chat_template(chat_template) + + self.tokenizer = _tokenizer + return selected_template + else: + return super().chat_template(chat_template) + + def tok_batch_multimodal_encode( + self, + strings: List[str], # note that input signature of this fn is different + images: List[List], # TODO: images are pil.Image at the moment, update typehint + padding_side: str = "left", + left_truncate_len: int = None, + truncation: bool = False, + ) -> Union[ + BatchEncoding, Dict[str, torch.Tensor] + ]: # note that this return signature differs from HFLM tok_batch_encode. + # NOTE: here, we replace tags with our model's corresponding image_token string value. + if not self.chat_applied: + # TODO: This still keeps the whitespace in the image placeholder, which is not ideal. + strings = [ + replace_placeholders( + string, DEFAULT_IMAGE_PLACEHOLDER, self.image_token, self.max_images + ) + for string in strings + ] + + # encode a batch of strings. converts to tensors and pads automatically, unlike tok_encode. + old_padding_side = self.tokenizer.padding_side + self.tokenizer.padding_side = padding_side + + # add_special_tokens = {"add_special_tokens": False or self.add_bos_token} + + images = [img[: self.max_images] for img in images] + if self.rgb: + images = [[img.convert("RGB") for img in sublist] for sublist in images] + + # certain models like llava expect a single-level image list even for bs>1, multi-image. TODO: port this over to loglikelihoods + if getattr(self.config, "model_type", "") == "llava": + images = flatten_image_list(images) + + encoding = self.processor( + images=images, + text=strings, + truncation=truncation, + padding="longest", + return_tensors="pt", + # **add_special_tokens, # TODO: at least some Processors error out when passing this. How do we control whether text gets BOS added? + ) + + encoding.to( # TODO: our other tokenization methods in HFLM don't typically move to device. this breaks convention + self.device, self.model.dtype + ) # TODO: This only casts the pixel values. Should they always be float16? + if left_truncate_len: + encoding["input_ids"] = encoding["input_ids"][:, -left_truncate_len:] + encoding["attention_mask"] = encoding["attention_mask"][ + :, -left_truncate_len: + ] + self.tokenizer.padding_side = old_padding_side + + return encoding + + def _model_multimodal_call(self, inps, imgs, attn_mask=None, labels=None): + """ + TODO: update docstring + """ + # note: imgs is a dict. + with torch.no_grad(): + return self.model(inps, **imgs).logits + + def _model_multimodal_generate(self, inputs, max_length, stop, **generation_kwargs): + generation_kwargs["temperature"] = generation_kwargs.get("temperature", 0.0) + do_sample = generation_kwargs.get("do_sample", None) + + # The temperature has to be a strictly positive float -- if it is 0.0, use greedy decoding strategies + if generation_kwargs.get("temperature") == 0.0 and do_sample is None: + generation_kwargs["do_sample"] = do_sample = False + + if do_sample is False and generation_kwargs.get("temperature") == 0.0: + generation_kwargs.pop("temperature") + + stopping_criteria = stop_sequences_criteria( + self.tokenizer, + stop, + inputs["input_ids"].shape[1], + inputs["input_ids"].shape[0], + ) + return self.model.generate( + **inputs, + max_length=max_length, + stopping_criteria=stopping_criteria, + pad_token_id=self.tokenizer.pad_token_id, + use_cache=True, + **generation_kwargs, + ) + + def _batch_images(self, image_encs): + """ + Helper function: batch together image encodings across examples in a batch. + # TODO: for variable-sized images, this may break down. + """ + batched_imgs = {} + for key in image_encs[0].keys(): + batched_imgs[key] = torch.cat( + [ + torch.tensor( + image_enc[key], device=self.device, dtype=self.model.dtype + ) + for image_enc in image_encs + ], + dim=0, + ) + return batched_imgs + + def loglikelihood_rolling(self, requests: List[Instance]) -> List[float]: + raise NotImplementedError( + "model type `hf-multimodal` does not support loglikelihood_rolling. Use 'hf' model type for text-only loglikelihood_rolling tasks ", + "this is because we do not support measuring the loglikelihood a model assigns to an image.", + ) + + def loglikelihood( + self, requests: List[Instance], disable_tqdm: bool = False + ) -> List[Tuple[float, bool]]: + raise NotImplementedError( + "'loglikelihood' requests for model type `hf-multimodal` are not yet tested. This feature will be enabled when a loglikelihood-based multiple-choice VQA dataset is added!" + ) + + new_reqs = [] + for context, continuation, aux_arguments in [req.args for req in requests]: + if context == "": + raise ValueError( + "Must get non-empty context for multimodal requests! You might be trying to run 'loglikelihood_rolling', which is not supported in the multimodal case." + ) + else: + visuals = aux_arguments["visual"] + + context_enc, continuation_enc, image_enc = self._encode_multimodal_pair( + context, continuation, visuals + ) + # TODO: key to pick for caching images + new_reqs.append( + ( + (context, continuation, visuals), + context_enc, + continuation_enc, + image_enc, + ) + ) + + return self._loglikelihood_tokens(new_reqs, disable_tqdm=disable_tqdm) + + def _loglikelihood_tokens( + self, + requests: List[ + Tuple[Tuple[None, str, str], List[int], List[int], List[int]] + ], # TODO: update typehint to be correct + disable_tqdm: bool = False, + override_bs: int = None, + ) -> List[Tuple[float, bool]]: + res = [] + + # TODO: **improve multimodal collation.** We currently ignore image size when ordering docs. ideally we'd take them into account + def _collate(req: Tuple[Tuple[str, str], List[int], List[int]]): + """Defines the key for the sorted method""" + # the negative sign on len(toks) sorts descending - this has a few advantages: + # - time estimates will always be over not underestimates, which is more useful for planning + # - to know the size of a batch when going through the list, you know the first one is always the batch + # padded context length. this is useful to simplify the batching logic and more importantly to make + # automatic adaptive batches much much easier to implement + # - any OOMs will happen right away rather than near the end + toks = req[1] + req[2] + return -len(toks), tuple(toks) + + def _lookup_one_token_cont(req: Tuple[Tuple[str, str], List[int], List[int]]): + """Defines the key to group and lookup one-token continuations""" + # Use with group_by="contexts" (optional)" + # allows for the creation of a lookup, so we can reuse logits in case of one-token continuations. + # speeds up some multiple-choice tasks proportionally to the number of choices. + # groups requests by context+continuation[:-1] and infer on one request/group. + return req[-1] + req[-3] + req[-2][:-1] + + re_ord = Collator( + requests, + sort_fn=_collate, + group_by="contexts" # TODO: can't group-by just "contexts" any more, need to incorporate imgs + if self.AUTO_MODEL_CLASS == transformers.AutoModelForCausalLM + and self.logits_cache + else None, + group_fn=_lookup_one_token_cont, + ) + + # automatic (variable) batch size detection for vectorization + # pull longest context sample from request + n_reordered_requests = len(re_ord) + batch_size = ( + self.batch_size + if self.batch_size != "auto" + else override_bs + if override_bs is not None + else 0 + ) + batch_fn = ( + self._batch_scheduler + if self.batch_size == "auto" + and n_reordered_requests > 0 + and not override_bs + else None + ) + + chunks = re_ord.get_batched(n=batch_size, batch_fn=batch_fn) + pbar = tqdm( + total=len(requests), + disable=(disable_tqdm or (self.rank != 0)), + desc="Running loglikelihood requests with text+image input", + ) + for chunk in chunks: + imgs = [] + inps = [] + cont_toks_list = [] + inplens = [] + + padding_len_inp = None + # because vectorizing is annoying, we first convert each (context, continuation) pair to padded + # tensors, then we pack them together into a batch, call the model, and then pick it all apart + # again because vectorizing is annoying + + for _, context_enc, continuation_enc, image_enc in chunk: + # sanity check + assert len(image_enc) > 0 + assert len(context_enc) > 0 + assert len(continuation_enc) > 0 + assert len(continuation_enc) <= self.max_length + + # how this all works (illustrated on a causal decoder-only setup): + # CTX CONT + # inp 0 1 2 3|4 5 6 7 8 9 <- last token is deleted by inp[:, :-1] + # model \ \ + # logits 1 2 3|4 5 6 7 8 9 <- the ctx half gets tossed out by the + # cont_toks 4 5 6 7 8 9 [:, -len(continuation_enc):, :self.vocab_size] slice + + # when too long to fit in context, truncate from the left + # TODO: assuming that we won't handle enc-dec Vision2Seq models. Is that a safe assumption? + inp = torch.tensor( + (context_enc + continuation_enc)[-(self.max_length + 1) :][:-1], + dtype=torch.long, + device=self.device, + ) + (inplen,) = inp.shape + + padding_len_inp = ( + max(padding_len_inp, inplen) + if padding_len_inp is not None + else inplen + ) + + inps.append(inp) # [1, inp_length] + cont_toks_list.append(continuation_enc) + inplens.append(inplen) + + imgs.append(image_enc) + + # create encoder attn mask and batched conts, if seq2seq + call_kwargs = {} + batched_inps = pad_and_concat( + padding_len_inp, inps, padding_side="right" + ) # [batch, padding_len_inp] + # batch our examples' image inputs together + batched_imgs = self._batch_images( + imgs + ) # TODO: fix/test for bs>1 case with differently-sized imgs! + + multi_logits = F.log_softmax( + self._model_multimodal_call(batched_inps, batched_imgs, **call_kwargs), + dim=-1, + ) # [batch, padding_length (inp or cont), vocab] + + for ( + request_str, + ctx_tokens, + _, + image_encs, + ), logits, inplen, cont_toks in zip( + chunk, multi_logits, inplens, cont_toks_list + ): + # Slice to original seq length + contlen = len(cont_toks) + # take only logits in the continuation + # (discard context toks if decoder-only ; discard right-padding) + # also discards + checks for "virtual tokens" in the causal LM's input window + # from prompt/prefix tuning tokens, if applicable + ctx_len = ( + inplen + (logits.shape[0] - padding_len_inp) + if self.AUTO_MODEL_CLASS == transformers.AutoModelForCausalLM + else None + ) + logits = self._select_cont_toks(logits, contlen=contlen, inplen=ctx_len) + logits = logits.unsqueeze(0) # [1, seq, vocab] + + # Check if per-token argmax is exactly equal to continuation + greedy_tokens = logits.argmax(dim=-1) + + # check for one-token continuation cache hits. + # noop in case group_by != "contexts" or no cache hit and returns the + # original args. Otherwise, expands the logits batch dimension and yields each + # batch along with matching continuation tokens and prompt strings. + # logits -> [1, seq, vocab] + for request_str, cont_toks, logits in re_ord.get_cache( + req_str=request_str, + cxt_toks=ctx_tokens, + cont_toks=cont_toks, + logits=logits, + ): + cont_toks = torch.tensor( + cont_toks, dtype=torch.long, device=self.device + ).unsqueeze(0) # [1, seq] + max_equal = (greedy_tokens == cont_toks).all() + + # Obtain log-probs at the corresponding continuation token indices + # last_token_slice = logits[:, -1, :].squeeze(0).tolist() + logits = torch.gather(logits, 2, cont_toks.unsqueeze(-1)).squeeze( + -1 + ) # [1, seq] + + # Answer: (log prob, is-exact-match) + answer = (float(logits.sum()), bool(max_equal)) + + res.append(answer) + + self.cache_hook.add_partial( + "loglikelihood", request_str, answer + ) # TODO: choose convention for adding images into the cache key + pbar.update(1) + + pbar.close() + + return re_ord.get_original(res) + + def generate_until( + self, requests: List[Instance], disable_tqdm: bool = False + ) -> List[str]: + # TODO: back out to HFLM.generate_until() for all requests without aux_arguments (text-only reqs) + res = [] + + def _collate(x): + # the negative sign on len(toks) sorts descending - this has a few advantages: + # - time estimates will always be over not underestimates, which is more useful for planning + # - to know the size of a batch when going through the list, you know the first one is always the batch + # padded context length. this is useful to simplify the batching logic and more importantly to make + # automatic adaptive batches much much easier to implement + # - any OOMs will happen right away rather than near the end + toks = self.tok_encode(x[0]) + return -len(toks), x[0] + + pbar = tqdm( + total=len(requests), + disable=(disable_tqdm or (self.rank != 0)), + desc="Running generate_until requests with text+image input", + ) + # TODO: port auto-batch sizing into this. + + # we group requests by their generation_kwargs, + # so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling + # in the same batch. + re_ords = Collator( + [reg.args for reg in requests], + _collate, + group_by="gen_kwargs", + group_fn=lambda x: x[1], + ) + chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None) + + ### Up to here: was identical to non-multimodal HFLM generate_until ### + + for chunk in chunks: + contexts, all_gen_kwargs, aux_arguments = zip(*chunk) + + visuals = [arg["visual"] for arg in aux_arguments] + + if not isinstance(contexts, list): + contexts = list( + contexts + ) # for Qwen2-VL, processor is unhappy accepting a tuple of strings instead of a list. + # TODO: could we upstream this workaround to HF? + ### this part onward: same as HFLM ### + + # we assume all gen kwargs in the batch are the same + # this is safe to assume because the `grouper` object ensures it. + gen_kwargs = all_gen_kwargs[0] + # unpack our keyword arguments. + until = None + if isinstance(gen_kwargs, dict): + kwargs = copy.deepcopy(gen_kwargs) # edge case for repeats > 1 + if "until" in kwargs.keys(): + until = kwargs.pop("until") + if isinstance(until, str): + until = [until] + elif not isinstance(until, list): + raise ValueError( + f"Expected `kwargs['until']` to be of type Union[str,list] but got {until}" + ) + else: + raise ValueError( + f"Expected `kwargs` to be of type `dict` but got {type(gen_kwargs)}" + ) + # add EOS token to stop sequences + eos = self.tok_decode(self.eot_token_id, skip_special_tokens=False) + if not until: + until = [eos] + else: + until.append(eos) + if "max_gen_toks" in kwargs.keys(): + max_gen_toks = kwargs.pop("max_gen_toks") + else: + max_gen_toks = self.max_gen_toks + + ### end stuff that's entirely copied verbatim from HFLM ### + + max_ctx_len = self.max_length - max_gen_toks + + inputs = self.tok_batch_multimodal_encode( + contexts, + visuals, + left_truncate_len=max_ctx_len, + truncation=self.truncation, + ) + + context_enc = inputs["input_ids"] + + if "max_length" not in kwargs: + kwargs["max_length"] = context_enc.shape[1] + max_gen_toks + + cont = self._model_multimodal_generate(inputs, stop=until, **kwargs) + + del inputs + torch.cuda.empty_cache() + import gc + + gc.collect() + + ### essentially same as HFLM beyond this line! + + cont_toks_list = cont.tolist() + for cont_toks, context in zip(cont_toks_list, contexts): + # discard context + left-padding toks if using causal decoder-only VLM + cont_toks = cont_toks[context_enc.shape[1] :] + + s = self.tok_decode(cont_toks) + + # use secondary stop seqs to cut off should-have-been-stopped content post-hoc + for term in until: + if len(term) > 0: + # ignore '' separator, + # for seq2seq case where self.tok_decode(self.eot_token_id) = '' + s = s.split(term)[0] + + res.append(s) + self.cache_hook.add_partial( + "generate_until", (context, gen_kwargs), s + ) # TODO: cache key for multimodal input should be what? + pbar.update(1) + # reorder this group of results back to original unsorted form + res = re_ords.get_original(res) + + pbar.close() + return res diff --git a/lm_eval/models/huggingface.py b/lm_eval/models/huggingface.py index 28c394cca35c77b34accd03528eab9c53dfd4fc0..cdaa76d6bbb4afbb72f6263330df9fff6918a6d5 100644 --- a/lm_eval/models/huggingface.py +++ b/lm_eval/models/huggingface.py @@ -9,10 +9,10 @@ import torch.nn.functional as F import transformers from accelerate import ( Accelerator, - DistributedType, InitProcessGroupKwargs, find_executable_batch_size, ) +from accelerate.utils import get_max_memory from huggingface_hub import HfApi from packaging import version from peft import PeftModel @@ -30,6 +30,7 @@ from lm_eval.api.registry import register_model from lm_eval.models.utils import ( Collator, clear_torch_cache, + configure_pad_token, get_dtype, pad_and_concat, stop_sequences_criteria, @@ -39,31 +40,6 @@ from lm_eval.models.utils import ( eval_logger = utils.eval_logger -def _get_accelerate_args( - device_map_option: Optional[str] = "auto", - max_memory_per_gpu: Optional[Union[int, str]] = None, - max_cpu_memory: Optional[Union[int, str]] = None, - offload_folder: Optional[str] = "./offload", - gpus: Optional[int] = None, -) -> dict: - """Returns the kwargs needed to apply `accelerate` in `AutoModel.from_pretrained`.""" - max_memory = {} - if max_memory_per_gpu is not None: - max_memory_per_gpu_map = { - device_idx: max_memory_per_gpu for device_idx in range(gpus) - } - max_memory.update(max_memory_per_gpu_map) - if max_cpu_memory is not None: - max_memory["cpu"] = max_cpu_memory - - args = {} - if max_memory: - args["max_memory"] = max_memory - args["device_map"] = device_map_option - args["offload_folder"] = offload_folder - return args - - @register_model("hf-auto", "hf", "huggingface") class HFLM(TemplateLM): """ @@ -79,7 +55,7 @@ class HFLM(TemplateLM): def __init__( self, pretrained: Union[str, transformers.PreTrainedModel], - backend: Optional[Literal["default", "causal", "seq2seq"]] = "default", + backend: Literal["default", "causal", "seq2seq"] = "default", # override whether the model should be treated as decoder-only (causal) or encoder-decoder (seq2seq) revision: Optional[str] = "main", subfolder: Optional[str] = None, @@ -104,7 +80,6 @@ class HFLM(TemplateLM): # arguments used for splitting a model across GPUs naively. # only used if `parallelize=True`. parallelize: Optional[bool] = False, - device_map_option: Optional[str] = "auto", max_memory_per_gpu: Optional[Union[int, str]] = None, max_cpu_memory: Optional[Union[int, str]] = None, offload_folder: Optional[Union[str, os.PathLike]] = "./offload", @@ -112,10 +87,10 @@ class HFLM(TemplateLM): peft: Optional[str] = None, delta: Optional[str] = None, autogptq: Optional[Union[bool, str]] = False, + gptqmodel: Optional[bool] = False, **kwargs, ) -> None: super().__init__() - # optionally: take in an already-initialized transformers.PreTrainedModel if not isinstance(pretrained, str): eval_logger.warning( @@ -127,21 +102,6 @@ class HFLM(TemplateLM): self._config = self._model.config gpus = 0 - if tokenizer: - assert isinstance( - tokenizer, transformers.PreTrainedTokenizer - ) or isinstance(tokenizer, transformers.PreTrainedTokenizerFast) - self.tokenizer = tokenizer - else: - # Get tokenizer - model_name = self._model.name_or_path - self.tokenizer = transformers.AutoTokenizer.from_pretrained( - model_name, - revision=revision, - trust_remote_code=trust_remote_code, - use_fast=use_fast_tokenizer, - ) - else: assert isinstance(device, str) assert isinstance(pretrained, str) @@ -156,6 +116,7 @@ class HFLM(TemplateLM): if "npu" in accelerator.device.type: gpus = torch.npu.device_count() + # using one process with no model parallelism if not (parallelize or accelerator.num_processes > 1): # use user-passed device device_list = set( @@ -181,14 +142,19 @@ class HFLM(TemplateLM): if torch.cuda.is_available() else torch.device("cpu") ) - else: + else: # Parallelism managed by accelerate if device != "cuda": eval_logger.info( f"Using `accelerate launch` or `parallelize=True`, device '{device}' will be overridden when placing model." ) # TODO: include in warning that `load_in_8bit` etc. affect this too - self._device = torch.device(device) + self._device = ( + self.accelerator.device + if hasattr(self, "accelerator") + else torch.device(device) + ) + revision = str(revision) # cast to string if not already one # TODO: update this to be less of a hack once subfolder is fixed in HF revision = revision + ("/" + subfolder if subfolder is not None else "") @@ -198,7 +164,7 @@ class HFLM(TemplateLM): trust_remote_code=trust_remote_code, ) - # determine which of 'causal' and 'seq2seq' backends to use + # determine which of 'causal' and 'seq2seq' backends to use for HF models self._get_backend( config=self.config, backend=backend, trust_remote_code=trust_remote_code ) @@ -221,13 +187,13 @@ class HFLM(TemplateLM): trust_remote_code=trust_remote_code, parallelize=parallelize, gpus=gpus, - device_map_option=device_map_option, max_memory_per_gpu=max_memory_per_gpu, max_cpu_memory=max_cpu_memory, offload_folder=offload_folder, peft=peft, delta=delta, autogptq=autogptq, + gptqmodel=gptqmodel, **kwargs, ) @@ -236,52 +202,17 @@ class HFLM(TemplateLM): self.model.eval() self.model.tie_weights() - if isinstance(pretrained, str) and (gpus >= 1 or str(self.device) == "mps"): - # TODO: can remove this whole snippet except in the mps case, perhaps? - if not (parallelize or autogptq or hasattr(self, "accelerator")): - # place model onto device requested manually, - # if not using HF Accelerate or device_map - # or any other option that preloads model onto device - try: - self.model.to(self.device) - except ValueError: - eval_logger.debug( - "Failed to place model onto specified device. This may be because the model is quantized via `bitsandbytes` or `device_map` is provided. If the desired GPU is being used, this message is safe to ignore." - ) - self.truncation = truncation self.logits_cache = logits_cache self.vocab_size = self.tokenizer.vocab_size # select (or create) a pad token to use - if self.tokenizer.pad_token: - pass - elif self.tokenizer.unk_token: - self.tokenizer.pad_token_id = self.tokenizer.unk_token_id - elif self.tokenizer.eos_token: - self.tokenizer.pad_token_id = self.tokenizer.eos_token_id - else: - if getattr(self.config, "model_type", None) == "qwen": - # Qwen's trust_remote_code tokenizer does not allow for adding special tokens - self.tokenizer.pad_token = "<|endoftext|>" - elif ( - self.tokenizer.__class__.__name__ == "RWKVWorldTokenizer" - or self.tokenizer.__class__.__name__ == "Rwkv5Tokenizer" - ): - # The RWKV world tokenizer, does not allow for adding special tokens / setting the pad token (which is set as 0) - # The additional tokenizer name check is needed, as there exists rwkv4 models with neox tokenizer - # --- - # Note that the world tokenizer class name, might change in the future for the final huggingface merge - # https://github.com/huggingface/transformers/pull/26963 - assert self.tokenizer.pad_token_id == 0 - else: - self.tokenizer.add_special_tokens({"pad_token": "<|pad|>"}) + self.tokenizer = configure_pad_token(self.tokenizer, model_config=self.config) - # TODO: override this for Gemma self.add_bos_token = add_bos_token - if getattr(self.config, "model_type", None) == "gemma": + if "gemma" in getattr(self.config, "model_type", ""): self.add_bos_token = True eval_logger.info( - f"Model type is '{self.config.model_type}', a BOS token will be used as Gemma underperforms without it." + f"Model type is '{self.config.model_type}', part of the Gemma family--a BOS token will be used as Gemma underperforms without it." ) self._max_length = max_length @@ -302,49 +233,46 @@ class HFLM(TemplateLM): self.batch_size_per_gpu = int(batch_size) if isinstance(pretrained, str): + if gpus >= 1 or str(self.device) == "mps": + # TODO: can remove this whole snippet except in the mps case, perhaps? + if not (parallelize or autogptq or hasattr(self, "accelerator")): + # place model onto device requested manually, + # if not using HF Accelerate or device_map + # or any other option that preloads model onto device + try: + self.model.to(self.device) + except ValueError: + eval_logger.debug( + "Failed to place model onto specified device. This may be because the model is quantized via `bitsandbytes` or `device_map` is provided. If the desired GPU is being used, this message is safe to ignore." + ) # multigpu data-parallel support when launched with accelerate if gpus > 1: - if parallelize: - if accelerator.num_processes > 1: - raise RuntimeError( - "Attempted to use both a HF Accelerate `device_map` and to launch via `accelerate launch`. If this is the case, please either remove `parallelize=True` from --model_args or launch outside of the Accelerate launcher." + if accelerator.num_processes > 1: + if parallelize: + eval_logger.warning( + "You are both using a HF Accelerate `device_map` (`--model_args parallelize=True`) and launching via `accelerate launch`. This will attempt to do model and data parallelism depending on the resources available." ) - else: - pass - elif accelerator.num_processes == 1: - # if we aren't launching via accelerate, ditch - self._rank = 0 - self._world_size = 1 - else: - if gpus > accelerator.num_processes: + elif gpus > accelerator.num_processes: eval_logger.warning( "WARNING: The number of total system GPUs does not match the number of spawned processes. " "If you would like to use data parallelism, please launch the script " "with 'accelerate launch *script*'. " f"Current run will proceed with {accelerator.num_processes} devices." ) - assert ( - accelerator.distributed_type - in [ - DistributedType.FSDP, - DistributedType.MULTI_GPU, - DistributedType.MULTI_NPU, - ] - ), "Unsupported distributed type provided. Only DDP and FSDP are supported." - if accelerator.distributed_type == DistributedType.FSDP: - self._model = accelerator.prepare(self.model) - else: - self._model = accelerator.prepare_model( - self.model, evaluation_mode=True - ) + if self.accelerator.is_local_main_process: + eval_logger.info( + f"Using {gpus} devices with data parallelism" + ) + self._device = torch.device(f"{accelerator.device}") self.accelerator = accelerator - if self.accelerator.is_local_main_process: - eval_logger.info(f"Using {gpus} devices with data parallelism") - self._rank = self.accelerator.local_process_index self._world_size = self.accelerator.num_processes + else: + # if we aren't launching via accelerate, ditch + self._rank = 0 + self._world_size = 1 else: # if a PreTrainedModel was passed into HFLM, we forgo distributed setup. eval_logger.warning( @@ -359,6 +287,94 @@ class HFLM(TemplateLM): f"Loglikelihood prefix token id used in evaluation: {self.prefix_token_id}" ) + def _get_accelerate_args( + self, + parallelize: Optional[bool] = None, + device_map: Optional[str] = "auto", + max_memory_per_gpu: Optional[Union[int, str]] = None, + max_cpu_memory: Optional[Union[int, str]] = None, + offload_folder: Optional[str] = "./offload", + gpus: Optional[int] = None, + ) -> dict: + """Returns the kwargs needed to apply `accelerate` in `AutoModel.from_pretrained`.""" + num_local_processes = int(os.environ.get("LOCAL_WORLD_SIZE", 1)) + num_machines = int(os.environ.get("WORLD_SIZE", 0)) // num_local_processes + if ( + num_machines == 0 + and hasattr(self, "accelerator") + and self.accelerator is not None + ): + eval_logger.info( + "We are not in a distributed setting for accelerate. Setting model_parallel to False." + ) + parallelize = False + + if parallelize is None: + # If parallelism is unset by the user, we automatically assign model parallelism + # if enough extra GPUs are available + max_memory_all_gpus = get_max_memory() + # We just want gpu, not cpu, max memory + if "cpu" in max_memory_all_gpus: + del max_memory_all_gpus["cpu"] + parallelize = bool(num_local_processes < len(max_memory_all_gpus)) + eval_logger.info( + f"Setting model parallel to {parallelize} since " + f"the number of local processes is {num_local_processes} " + f"and the number of GPUs is {len(max_memory_all_gpus)}" + ) + + args = {} + if parallelize: # Model parallelism will be used + max_memory = {} + if max_memory_per_gpu is not None: # Using the provided memory requirements + max_memory_per_gpu_map = { + device_idx: max_memory_per_gpu for device_idx in range(gpus) + } + else: # Estimating the possible memory requirements + max_memory_all_gpus = get_max_memory() + if "cpu" in max_memory_all_gpus: + del max_memory_all_gpus["cpu"] + if not hasattr(self, "accelerator"): + max_memory_per_gpu_map = { + k: v for k, v in max_memory_all_gpus.items() + } + else: + # use only 1 / num_processes of the GPUs if we are running under accelerate launch + max_memory_per_gpu_map = { + k: v + for k, v in max_memory_all_gpus.items() + if k % num_local_processes + == (self.accelerator.process_index % num_local_processes) + } + args["max_memory"] = max_memory_per_gpu_map + args["device_map"] = "auto" if device_map is None else device_map + eval_logger.info( + f"Model parallel was set to True, setting max memory per GPU to {max_memory_per_gpu_map} and device map to {args.get('device_map')}" + ) + + if max_cpu_memory is not None: + max_memory["cpu"] = max_cpu_memory + + args["offload_folder"] = offload_folder + elif ( + device_map is None + ): # No model parallelism, we use the default provided device for our model + if hasattr(self, "accelerator"): + device_map = {"": f"{self.accelerator.device}"} + else: + device_map = {"": str(self.device)} + args["max_memory"] = None + args["device_map"] = device_map + eval_logger.info( + f"Model parallel was set to False, max memory was not set, and device map was set to {device_map}" + ) + else: + args["max_memory"] = None + args["device_map"] = None + eval_logger.info("Model parallel was set to False.") + + return args + @property def config(self): # return the associated transformers.AutoConfig for the given pretrained model. @@ -424,31 +440,29 @@ class HFLM(TemplateLM): def tokenizer_name(self) -> str: return self.tokenizer.name_or_path.replace("/", "__") - @property - def chat_template(self) -> str: - if self.tokenizer.chat_template is not None: - return self.tokenizer.chat_template - return self.tokenizer.default_chat_template - def _get_backend( self, config: Union[transformers.PretrainedConfig, transformers.AutoConfig], - backend: Optional[Literal["default", "causal", "seq2seq"]] = "default", + backend: Literal["default", "causal", "seq2seq"] = "default", trust_remote_code: Optional[bool] = False, ) -> None: """ Helper method during initialization. - Determines the backend ("causal" (decoder-only) or "seq2seq" (encoder-decoder)) - model type to be used. + Determines the backend ("causal" (decoder-only) or "seq2seq" (encoder-decoder)) model type to be used. + sets `self.AUTO_MODEL_CLASS` appropriately if not already set. + + **If not calling HFLM.__init__() or HFLM._get_backend() within a subclass of HFLM, + user must set `self.backend` to be either "causal" or "seq2seq" manually!** """ + assert backend in ["default", "causal", "seq2seq"] if backend != "default": # if we've settled on non-default backend, use that manually if backend == "causal": - self.AUTO_MODEL_CLASS = transformers.AutoModelForCausalLM + self.backend = backend elif backend == "seq2seq": - self.AUTO_MODEL_CLASS = transformers.AutoModelForSeq2SeqLM + self.backend = backend eval_logger.info( f"Overrode HF model backend type, and using type '{backend}'" ) @@ -461,26 +475,32 @@ class HFLM(TemplateLM): # first check if model type is listed under seq2seq models, since some # models like MBart are listed in both seq2seq and causal mistakenly in HF transformers. # these special cases should be treated as seq2seq models. - self.AUTO_MODEL_CLASS = transformers.AutoModelForSeq2SeqLM + self.backend = "seq2seq" + eval_logger.info(f"Using model type '{backend}'") elif ( getattr(self.config, "model_type") in MODEL_FOR_CAUSAL_LM_MAPPING_NAMES ): - self.AUTO_MODEL_CLASS = transformers.AutoModelForCausalLM + self.backend = "causal" + eval_logger.info(f"Using model type '{backend}'") else: if not trust_remote_code: eval_logger.warning( "HF model type is neither marked as CausalLM or Seq2SeqLM. \ This is expected if your model requires `trust_remote_code=True` but may be an error otherwise." + "Setting backend to causal" ) # if model type is neither in HF transformers causal or seq2seq model registries - # then we default to AutoModelForCausalLM - self.AUTO_MODEL_CLASS = transformers.AutoModelForCausalLM + # then we default to assuming AutoModelForCausalLM + self.backend = "causal" + eval_logger.info( + f"Model type cannot be determined. Using default model type '{backend}'" + ) - assert self.AUTO_MODEL_CLASS in [ - transformers.AutoModelForCausalLM, - transformers.AutoModelForSeq2SeqLM, - ] - return None + if self.AUTO_MODEL_CLASS is None: + if self.backend == "causal": + self.AUTO_MODEL_CLASS = transformers.AutoModelForCausalLM + elif self.backend == "seq2seq": + self.AUTO_MODEL_CLASS = transformers.AutoModelForSeq2SeqLM def _get_config( self, @@ -488,6 +508,7 @@ class HFLM(TemplateLM): revision: str = "main", trust_remote_code: bool = False, ) -> None: + """Return the model config for HuggingFace models""" self._config = transformers.AutoConfig.from_pretrained( pretrained, revision=revision, @@ -505,7 +526,6 @@ class HFLM(TemplateLM): # (accelerate naive PP (device_map) options) parallelize: Optional[bool] = False, gpus: Optional[int] = None, - device_map_option: Optional[str] = "auto", max_memory_per_gpu: Optional[Union[int, str]] = None, max_cpu_memory: Optional[Union[int, str]] = None, offload_folder: Optional[str] = "./offload", @@ -513,6 +533,7 @@ class HFLM(TemplateLM): peft: Optional[str] = None, delta: Optional[str] = None, autogptq: Optional[Union[bool, str]] = False, + gptqmodel: Optional[bool] = False, **kwargs, ) -> None: """ @@ -529,27 +550,18 @@ class HFLM(TemplateLM): model_kwargs = kwargs if kwargs else {} - if parallelize: - model_kwargs.update( - _get_accelerate_args( - device_map_option, # TODO: phase out device_map_option? - max_memory_per_gpu, - max_cpu_memory, - offload_folder, - gpus, - ) + model_kwargs.update( + self._get_accelerate_args( + parallelize=parallelize, + device_map=kwargs.get("device_map", None), + max_memory_per_gpu=max_memory_per_gpu, + max_cpu_memory=max_cpu_memory, + offload_folder=offload_folder, + gpus=gpus, ) - elif "device_map" not in model_kwargs: - # set a device_map to initialize model on the right GPU. - # this is needed because it seems that the default behavior - # for quantized models now seems to be device_map="auto" - # which breaks data-parallel mode. - if hasattr(self, "accelerator"): - model_kwargs.update({"device_map": {"": f"{self.accelerator.device}"}}) - else: - model_kwargs.update({"device_map": {"": str(self.device)}}) + ) - if not autogptq: + if not autogptq and not gptqmodel: if model_kwargs.get("load_in_4bit", None): assert ( transformers.__version__ >= "4.30.0" @@ -560,6 +572,7 @@ class HFLM(TemplateLM): model_kwargs["bnb_4bit_compute_dtype"] = get_dtype( model_kwargs["bnb_4bit_compute_dtype"] ) + self._model = self.AUTO_MODEL_CLASS.from_pretrained( pretrained, revision=revision, @@ -568,23 +581,42 @@ class HFLM(TemplateLM): **model_kwargs, ) else: - try: - from auto_gptq import AutoGPTQForCausalLM - except ModuleNotFoundError: - raise Exception( - "Tried to load auto_gptq, but auto-gptq is not installed ", - "please install auto-gptq via pip install lm-eval[gptq] or pip install -e .[gptq]", + if autogptq and gptqmodel: + raise ValueError( + "Cannot use both 'autogptq' and 'gptqmodel' options at the same time." ) - self._model = AutoGPTQForCausalLM.from_quantized( - pretrained, - trust_remote_code=trust_remote_code, - model_basename=None if autogptq is True else Path(autogptq).stem, - use_safetensors=True - if autogptq is True - else autogptq.endswith(".safetensors"), - **model_kwargs, - ) + if autogptq: + try: + from auto_gptq import AutoGPTQForCausalLM + except ModuleNotFoundError as exception: + raise type(exception)( + "Tried to load auto_gptq, but auto-gptq is not installed ", + "please install auto-gptq via pip install lm-eval[gptq] or pip install -e .[gptq]", + ) + + self._model = AutoGPTQForCausalLM.from_quantized( + pretrained, + trust_remote_code=trust_remote_code, + model_basename=None if autogptq is True else Path(autogptq).stem, + use_safetensors=True + if autogptq is True + else autogptq.endswith(".safetensors"), + **model_kwargs, + ) + + if gptqmodel: + try: + from gptqmodel import GPTQModel + except ModuleNotFoundError as exception: + raise type(exception)( + "Tried to load gptqmodel, but gptqmodel is not installed ", + "please install gptqmodel via `pip install gptqmodel --no-build-isolation` or `pip install lm-eval[gptqmodel] --no-build-isolation`", + ) + + self._model = GPTQModel.from_quantized( + pretrained, trust_remote_code=trust_remote_code, **model_kwargs + ) if peft and delta: raise ValueError( @@ -597,10 +629,10 @@ class HFLM(TemplateLM): raise AssertionError("load_in_4bit requires peft >= 0.4.0") if self._model.config.vocab_size != len(self.tokenizer): # resize model for LoRAs with added tokens - self._model.resize_token_embeddings(len(self.tokenizer)) eval_logger.info( f"Model config indicates vocab_size='{self._model.config.vocab_size}', but found tokenizer with vocab size '{len(self.tokenizer)}'. Resizing model embedding layer..." ) + self._model.resize_token_embeddings(len(self.tokenizer)) self._model = PeftModel.from_pretrained( self._model, peft, revision=revision ) @@ -718,7 +750,7 @@ class HFLM(TemplateLM): @find_executable_batch_size(starting_batch_size=self.max_batch_size) def forward_batch(batch_size): security_margin = int(0.05 * security_margin_factor * batch_size) - if self.AUTO_MODEL_CLASS == transformers.AutoModelForSeq2SeqLM: + if self.backend == "seq2seq": length = max(max_context_enc, max_cont_enc) batched_conts = torch.ones( (batch_size + security_margin, length), device=self.device @@ -773,7 +805,7 @@ class HFLM(TemplateLM): # by default for CausalLM - false or self.add_bos_token is set if add_special_tokens is None: - if self.AUTO_MODEL_CLASS == transformers.AutoModelForCausalLM: + if self.backend == "causal": special_tokens_kwargs = { "add_special_tokens": False or self.add_bos_token } @@ -801,7 +833,7 @@ class HFLM(TemplateLM): self.tokenizer.padding_side = padding_side add_special_tokens = {} - if self.AUTO_MODEL_CLASS == transformers.AutoModelForCausalLM: + if self.backend == "causal": add_special_tokens = {"add_special_tokens": False or self.add_bos_token} encoding = self.tokenizer( @@ -879,14 +911,14 @@ class HFLM(TemplateLM): def _select_cont_toks( self, logits: torch.Tensor, contlen: int = None, inplen: int = None ) -> torch.Tensor: - if self.AUTO_MODEL_CLASS == transformers.AutoModelForCausalLM: + if self.backend == "causal": assert ( contlen and inplen ), "Must pass input len and cont. len to select scored logits for causal LM" # discard right-padding. # also discard the input/context tokens. we'll only score continuations. logits = logits[inplen - contlen : inplen] - elif self.AUTO_MODEL_CLASS == transformers.AutoModelForSeq2SeqLM: + elif self.backend == "seq2seq": assert ( contlen and not inplen ), "Selecting scored logits for Seq2SeqLM requires only cont. len" @@ -944,6 +976,9 @@ class HFLM(TemplateLM): string_nll = sum(string_nll) loglikelihoods.append(string_nll) + # cache this loglikelihood_rolling request + self.cache_hook.add_partial("loglikelihood_rolling", (string,), string_nll) + return loglikelihoods def _batch_scheduler(self, pos, n_reordered_requests): @@ -1000,8 +1035,7 @@ class HFLM(TemplateLM): requests, sort_fn=_collate, group_by="contexts" - if self.AUTO_MODEL_CLASS == transformers.AutoModelForCausalLM - and self.logits_cache + if self.backend == "causal" and self.logits_cache else None, group_fn=_lookup_one_token_cont, ) @@ -1058,14 +1092,14 @@ class HFLM(TemplateLM): # cont_toks 4 5 6 7 8 9 [:, -len(continuation_enc):, :self.vocab_size] slice # when too long to fit in context, truncate from the left - if self.AUTO_MODEL_CLASS == transformers.AutoModelForCausalLM: + if self.backend == "causal": inp = torch.tensor( (context_enc + continuation_enc)[-(self.max_length + 1) :][:-1], dtype=torch.long, device=self.device, ) (inplen,) = inp.shape - elif self.AUTO_MODEL_CLASS == transformers.AutoModelForSeq2SeqLM: + elif self.backend == "seq2seq": inp = torch.tensor( (context_enc)[-self.max_length :], dtype=torch.long, @@ -1105,11 +1139,11 @@ class HFLM(TemplateLM): # create encoder attn mask and batched conts, if seq2seq call_kwargs = {} - if self.AUTO_MODEL_CLASS == transformers.AutoModelForCausalLM: + if self.backend == "causal": batched_inps = pad_and_concat( padding_len_inp, inps, padding_side="right" ) # [batch, padding_len_inp] - elif self.AUTO_MODEL_CLASS == transformers.AutoModelForSeq2SeqLM: + elif self.backend == "seq2seq": # TODO: left-pad encoder inps and mask? batched_inps = pad_and_concat( padding_len_inp, inps @@ -1142,7 +1176,7 @@ class HFLM(TemplateLM): # from prompt/prefix tuning tokens, if applicable ctx_len = ( inplen + (logits.shape[0] - padding_len_inp) - if self.AUTO_MODEL_CLASS == transformers.AutoModelForCausalLM + if self.backend == "causal" else None ) logits = self._select_cont_toks(logits, contlen=contlen, inplen=ctx_len) @@ -1178,7 +1212,13 @@ class HFLM(TemplateLM): res.append(answer) - self.cache_hook.add_partial("loglikelihood", request_str, answer) + if request_str is not None: + # special case: loglikelihood_rolling produces a number of loglikelihood requests + # all with cache key None. instead do add_partial on the per-example level + # in the loglikelihood_rolling() function for those. + self.cache_hook.add_partial( + "loglikelihood", request_str, answer + ) pbar.update(1) pbar.close() @@ -1260,7 +1300,7 @@ class HFLM(TemplateLM): max_gen_toks = self.max_gen_toks # set the max length in tokens of inputs ("context_enc") - if self.AUTO_MODEL_CLASS == transformers.AutoModelForCausalLM: + if self.backend == "causal": # max len for inputs = max length, minus room to generate the max new tokens # if the max new tokens is too large, halve it until it fits as we cannot change # the max model length @@ -1268,7 +1308,7 @@ class HFLM(TemplateLM): while max_ctx_len <= 0: max_gen_toks = max_gen_toks // 2 max_ctx_len = self.max_length - max_gen_toks - elif self.AUTO_MODEL_CLASS == transformers.AutoModelForSeq2SeqLM: + elif self.backend == "seq2seq": # max len for inputs = encoder's whole max_length max_ctx_len = self.max_length @@ -1295,7 +1335,7 @@ class HFLM(TemplateLM): cont_toks_list = cont.tolist() for cont_toks, context in zip(cont_toks_list, contexts): # discard context + left-padding toks if using causal decoder-only LM - if self.AUTO_MODEL_CLASS == transformers.AutoModelForCausalLM: + if self.backend == "causal": cont_toks = cont_toks[context_enc.shape[1] :] s = self.tok_decode(cont_toks) diff --git a/lm_eval/models/ibm_watsonx_ai.py b/lm_eval/models/ibm_watsonx_ai.py new file mode 100644 index 0000000000000000000000000000000000000000..6f6a09c1707db948b43034ad7679f672afad91e0 --- /dev/null +++ b/lm_eval/models/ibm_watsonx_ai.py @@ -0,0 +1,409 @@ +import json +import os +from configparser import ConfigParser +from functools import lru_cache +from pathlib import Path +from typing import Any, Dict, List, NamedTuple, Optional, Tuple, Type, cast + +from tqdm import tqdm + +from lm_eval.api.instance import Instance +from lm_eval.api.model import LM +from lm_eval.api.registry import register_model +from lm_eval.utils import eval_logger, simple_parse_args_string + + +class LogLikelihoodResult(NamedTuple): + log_likelihood: float + is_greedy: bool + + +@lru_cache(maxsize=None) +def get_watsonx_credentials( + env_name: str = "YP_QA", + config_path: str = "config.ini", +) -> Dict[str, str]: + """ + Retrieves Watsonx API credentials from environmental variables or from a configuration file. + Args: + env_name (str, optional): The name of the environment from which to retrieve credentials. Defaults to "YP_QA". + config_path (str, optional): The file path to the `config.ini` configuration file. Defaults to "config.ini". + Returns: + dict[str, str]: A dictionary containing the credentials necessary for authentication, including + keys such as `apikey`, `url`, and `project_id`. + Raises: + FileNotFoundError: If the specified configuration file does not exist. + AssertionError: If the credentials format is invalid. + """ + + def _verify_credentials(creds: Any) -> None: + assert isinstance(creds, dict) and all( + key in creds.keys() for key in ["apikey", "url", "project_id"] + ), "Wrong configuration for credentials." + + credentials = { + "apikey": os.getenv("WATSONX_API_KEY", None), + "url": os.getenv("WATSONX_URL", None), + "project_id": os.getenv("WATSONX_PROJECT_ID", None), + } + + if any(credentials.get(key) is None for key in ["apikey", "url", "project_id"]): + eval_logger.warning( + "One or more required environment variables are missing, trying to load config.ini file." + ) + + config_path = "config.ini" if not config_path else config_path + + if not Path(config_path).is_absolute(): + config_path = os.path.join( + Path(__file__).parent.parent.absolute(), config_path + ) + + if not os.path.exists(config_path): + raise FileNotFoundError( + f"Provided config file path {config_path} does not exist. " + "You need to specify credentials in config.ini file under specified location." + ) + + config = ConfigParser() + config.read(config_path) + credentials = json.loads(config.get(env_name)) + + _verify_credentials(credentials) + return credentials + + +@register_model("watsonx_llm") +class WatsonxLLM(LM): + """ + Implementation of LM model interface for evaluating Watsonx model with the lm_eval framework. + See https://github.com/EleutherAI/lm-evaluation-harness/blob/main/docs/model_guide.md for reference. + """ + + @classmethod + def create_from_arg_string( + cls: Type["WatsonxLLM"], + arg_string: str, + config_path: Optional[str] = None, + ) -> "WatsonxLLM": + """ + Allow the user to specify model parameters (TextGenerationParameters) in CLI arguments. + """ + try: + from ibm_watsonx_ai.metanames import GenTextParamsMetaNames as GenParams + except ImportError: + raise ImportError( + "Could not import ibm_watsonx_ai: Please install lm_eval[ibm_watsonx_ai] package." + ) + + args = simple_parse_args_string(arg_string) + model_id = args.pop("model_id", None) + if model_id is None: + raise ValueError("'model_id' is required, please pass it in 'model_args'") + + if not args.get("do_sample", None): + args["temperature"] = None + args["top_p"] = None + args["top_k"] = None + args["seed"] = None + + cls.generate_params = { + GenParams.DECODING_METHOD: ( + "greedy" if not args.get("do_sample", None) else "sample" + ), + GenParams.LENGTH_PENALTY: args.get("length_penalty", None), + GenParams.TEMPERATURE: args.get("temperature", None), + GenParams.TOP_P: args.get("top_p", None), + GenParams.TOP_K: args.get("top_k", None), + GenParams.RANDOM_SEED: args.get("seed", None), + GenParams.REPETITION_PENALTY: args.get("repetition_penalty", None), + GenParams.MIN_NEW_TOKENS: args.get("min_new_tokens", None), + GenParams.MAX_NEW_TOKENS: args.get("max_new_tokens", 256), + GenParams.STOP_SEQUENCES: args.get("stop_sequences", None), + GenParams.TIME_LIMIT: args.get("time_limit", None), + GenParams.TRUNCATE_INPUT_TOKENS: args.get("truncate_input_tokens", None), + GenParams.RETURN_OPTIONS: { + "generated_tokens": True, + "input_tokens": True, + "token_logprobs": True, + "token_ranks": True, + }, + } + + generate_params = { + k: v for k, v in cls.generate_params.items() if v is not None + } + + return cls( + watsonx_credentials=get_watsonx_credentials(config_path), + model_id=model_id, + generate_params=generate_params, + ) + + def __init__( + self, + watsonx_credentials: Dict, + model_id, + generate_params: Optional[Dict[Any, Any]] = None, + ) -> None: + try: + from ibm_watsonx_ai import APIClient + from ibm_watsonx_ai.foundation_models import ModelInference + except ImportError: + raise ImportError( + "Could not import ibm_watsonx_ai: Please install lm_eval[ibm_watsonx_ai] package." + ) + super().__init__() + client = APIClient(watsonx_credentials) + project_id = watsonx_credentials.get("project_id", None) + deployment_id = watsonx_credentials.get("deployment_id", None) + client.set.default_project(project_id) + self.generate_params = generate_params or {} + self.model = ModelInference( + model_id=model_id, + deployment_id=deployment_id, + api_client=client, + project_id=project_id, + ) + self._model_id = model_id + + @staticmethod + def _has_stop_token(response_tokens: List[str], context_tokens: List[str]) -> bool: + """ + Determines whether a stop token has been generated in the `response_tokens` compared to the `context_tokens`. + If the tokens do not match as expected, the function raises a RuntimeError, indicating a possible + misalignment between the tokens generated by the tokenizer and the model. + Args: + response_tokens (List[str]): The List of tokens generated as a response by the model. + context_tokens (List[str]): The List of tokens representing the input context. + Returns: + bool: True if the `response_tokens` likely contain a stop token that terminates the sequence, + otherwise raises an exception. + Raises: + RuntimeError: If there is an unexpected mismatch between the `response_tokens` and the `context_tokens`. + """ + context_length = len(context_tokens) + if response_tokens[: context_length - 1] == context_tokens[:-1]: + return ( + response_tokens[-1] != context_tokens[-1] + ) # only last token differs, probably stop sequence () + raise RuntimeError( + f"There is an unexpected difference between tokenizer and model tokens:\n" + f"context_tokens={context_tokens}\n" + f"response_tokens={response_tokens[:context_length]}" + ) + + def _check_model_logprobs_support(self): + """ + Verifies if the model supports returning log probabilities for input tokens. + This function sends a prompt to the model and checks whether the model's response + includes log probabilities for the input tokens. If log probabilities are not present, + it raises a `RuntimeError`, indicating that the model is not supported. + Raises: + RuntimeError: If the model does not return log probabilities for input tokens. + """ + tokens = self.model.generate_text( + prompt=["The best ice cream flavor is:"], + params=self.generate_params, + raw_response=True, + )[0]["results"][0] + if all(token.get("logprob", None) is None for token in tokens["input_tokens"]): + raise RuntimeError( + f"Model {self._model_id} is not supported: does not return logprobs for input tokens" + ) + + def _get_log_likelihood( + self, + input_tokens: List[Dict[str, float]], + context_tokens: List[Dict[str, float]], + ) -> LogLikelihoodResult: + """ + Calculates the log likelihood of the generated tokens compared to the context tokens. + Args: + input_tokens (List[dict[str, float]]): A List of token dictionaries, each containing + token information like `text` and `logprob`. + context_tokens (List[dict[str, float]]): A List of token dictionaries representing + the input context. + Returns: + LogLikelihoodResult: An object containing the calculated log likelihood and a boolean + flag indicating if the tokens were generated greedily. + """ + + response_tokens = [token["text"] for token in input_tokens] + context_length = len(context_tokens) + + if self._has_stop_token(response_tokens, context_tokens): + context_length -= 1 + + return LogLikelihoodResult( + log_likelihood=sum( + token.get("logprob", 0) for token in input_tokens[context_length:] + ), + is_greedy=all( + token["rank"] == 1 for token in input_tokens[context_length:] + ), + ) + + def generate_until(self, requests: List[Instance]) -> List[str]: + """ + Generates text responses for a List of requests, with progress tracking and caching. + Args: + requests (List[Instance]): A List of instances, each containing a text input to be processed. + Returns: + List[str]: A List of generated responses. + """ + requests = [request.args[0] for request in requests] + results = [] + batch_size = 5 + + for i in tqdm( + range(0, len(requests), batch_size), + desc=f"Running generate_until function with batch size {batch_size}", + ): + batch = requests[i : i + batch_size] + try: + responses = self.model.generate_text(batch, self.generate_params) + + except Exception as exp: + eval_logger.error(f"Error while generating text {exp}") + continue + + for response, context in zip(responses, batch): + results.append(response) + self.cache_hook.add_partial("generated_text", context, response) + + eval_logger.info("Cached responses") + + return results + + def loglikelihood(self, requests: List[Instance]) -> List[Tuple[float, bool]]: + """ + Args: + requests: Each request contains Instance.args : Tuple[str, str] containing: + 1. an input string to the LM and + 2. a target string on which the loglikelihood of the LM producing this target, + conditioned on the input, will be returned. + Returns: + tuple (loglikelihood, is_greedy) for each request according to the input order: + loglikelihood: probability of generating the target string conditioned on the input + is_greedy: True if and only if the target string would be generated by greedy sampling from the LM + """ + try: + from ibm_watsonx_ai.metanames import GenTextParamsMetaNames as GenParams + except ImportError: + raise ImportError( + "Could not import ibm_watsonx_ai: Please install lm_eval[ibm_watsonx_ai] package." + ) + self._check_model_logprobs_support() + self.generate_params[GenParams.MAX_NEW_TOKENS] = 1 + + requests = [request.args for request in requests] + results: List[LogLikelihoodResult] = [] + batch_size = 5 + + for i in tqdm( + range(0, len(requests), batch_size), + desc=f"Running loglikelihood function with batch size {batch_size}", + ): + batch = requests[i : i + batch_size] + try: + tokenized_contexts = [ + self.model.tokenize(prompt=context, return_tokens=True)["result"][ + "tokens" + ] + for context, _ in batch + ] + except Exception as exp: + eval_logger.error(f"Error while model tokenize:\n {exp}") + continue + + input_prompts = [context + continuation for context, continuation in batch] + + try: + responses = self.model.generate_text( + prompt=input_prompts, params=self.generate_params, raw_response=True + ) + except Exception as exp: + eval_logger.error(f"Error while model generate text:\n {exp}") + continue + + for response, tokenized_context, (context, continuation) in zip( + responses, tokenized_contexts, batch + ): + log_likelihood_response = self._get_log_likelihood( + response["results"][0]["input_tokens"], tokenized_context + ) + results.append(log_likelihood_response) + self.cache_hook.add_partial( + "loglikelihood", + (context, continuation), + ( + log_likelihood_response.log_likelihood, + log_likelihood_response.is_greedy, + ), + ) + eval_logger.info("Cached batch") + + return cast(List[Tuple[float, bool]], results) + + def loglikelihood_rolling(self, requests) -> List[Tuple[float, bool]]: + """ + Used to evaluate perplexity on a data distribution. + Args: + requests: Each request contains Instance.args : tuple[str] containing an input string to the model whose + entire loglikelihood, conditioned on purely the EOT token, will be calculated. + Returns: + tuple (loglikelihood,) for each request according to the input order: + loglikelihood: solely the probability of producing each piece of text given no starting input. + """ + try: + from ibm_watsonx_ai.metanames import GenTextParamsMetaNames as GenParams + except ImportError: + raise ImportError( + "Could not import ibm_watsonx_ai: Please install lm_eval[ibm_watsonx_ai] package." + ) + self._check_model_logprobs_support() + self.generate_params[GenParams.MAX_NEW_TOKENS] = 1 + + requests = [request.args[0] for request in requests] + results: List[LogLikelihoodResult] = [] + batch_size = 5 + + for i in tqdm( + range(0, len(requests), batch_size), + desc=f"Running loglikelihood_rolling function with batch size {batch_size}", + ): + batch = requests[i : i + batch_size] + + try: + responses = self.model.generate_text( + prompt=batch, params=self.generate_params, raw_response=True + ) + except Exception as exp: + eval_logger.error(f"Error while model generate text:\n {exp}") + continue + + for response, context in zip(responses, batch): + try: + log_likelihood_response = self._get_log_likelihood( + response["results"][0]["input_tokens"], [] + ) + results.append(log_likelihood_response) + + self.cache_hook.add_partial( + "loglikelihood_rolling", + context, + ( + log_likelihood_response.log_likelihood, + log_likelihood_response.is_greedy, + ), + ) + except Exception as exp: + eval_logger.error( + f"Error during log likelihood calculation:\n {exp}" + ) + continue + + eval_logger.info("Cached batch") + + return cast(List[Tuple[float, bool]], results) diff --git a/lm_eval/models/mamba_lm.py b/lm_eval/models/mamba_lm.py index cd9049836838a1dabb2baf383f8e8ce5a02e7391..47a241f0a503e286359f6c2cdecc63cd1f8095c2 100644 --- a/lm_eval/models/mamba_lm.py +++ b/lm_eval/models/mamba_lm.py @@ -69,8 +69,8 @@ class MambaLMWrapper(HFLM): ) -> None: try: from mamba_ssm.utils.hf import load_config_hf # noqa: F811 - except ModuleNotFoundError: - raise Exception( + except ModuleNotFoundError as exception: + raise type(exception)( "attempted to use 'mamba_ssm' LM type, but package `mamba_ssm` is not installed. \ please install mamba via `pip install lm-eval[mamba]` or `pip install -e .[mamba]`", ) @@ -88,8 +88,8 @@ please install mamba via `pip install lm-eval[mamba]` or `pip install -e .[mamba ) -> None: try: from mamba_ssm.models.mixer_seq_simple import MambaLMHeadModel # noqa: F811 - except ModuleNotFoundError: - raise Exception( + except ModuleNotFoundError as exception: + raise type(exception)( "attempted to use 'mamba_ssm' LM type, but package `mamba_ssm` is not installed. \ please install mamba via `pip install lm-eval[mamba]` or `pip install -e .[mamba]`", ) diff --git a/lm_eval/models/nemo_lm.py b/lm_eval/models/nemo_lm.py index cb1aca1eec18a05725ffb29e15f633078cab699b..cf56019715a9d65c446f9b7285458cd00fc499ea 100644 --- a/lm_eval/models/nemo_lm.py +++ b/lm_eval/models/nemo_lm.py @@ -39,8 +39,8 @@ def _patch_pretrained_cfg( ): try: import omegaconf - except ModuleNotFoundError: - raise Exception( + except ModuleNotFoundError as exception: + raise type(exception)( "Attempted to use 'nemo_lm' model type, but package `nemo` is not installed" "Please install nemo following the instructions in the README: either with a NVIDIA PyTorch or NeMo container, " "or installing nemo following https://github.com/NVIDIA/NeMo.", @@ -79,8 +79,8 @@ def load_model( MegatronGPTModel, ) from nemo.collections.nlp.parts.nlp_overrides import NLPSaveRestoreConnector - except ModuleNotFoundError: - raise Exception( + except ModuleNotFoundError as exception: + raise type(exception)( "Attempted to use 'nemo_lm' model type, but package `nemo` is not installed" "Please install nemo following the instructions in the README: either with a NVIDIA PyTorch or NeMo container, " "or installing nemo following https://github.com/NVIDIA/NeMo.", @@ -140,8 +140,8 @@ def load_model( def setup_distributed_environment(trainer): try: from nemo.utils.app_state import AppState - except ModuleNotFoundError: - raise Exception( + except ModuleNotFoundError as exception: + raise type(exception)( "Attempted to use 'nemo_lm' model type, but package `nemo` is not installed" "Please install nemo following the instructions in the README: either with a NVIDIA PyTorch or NeMo container, " "or installing nemo following https://github.com/NVIDIA/NeMo.", @@ -194,8 +194,8 @@ class NeMoLM(LM): from pytorch_lightning.trainer.trainer import Trainer self.generate = generate - except ModuleNotFoundError: - raise Exception( + except ModuleNotFoundError as exception: + raise type(exception)( "Attempted to use 'nemo_lm' model type, but package `nemo` is not installed" "Please install nemo following the instructions in the README: either with a NVIDIA PyTorch or NeMo container, " "or installing nemo following https://github.com/NVIDIA/NeMo.", @@ -386,6 +386,9 @@ class NeMoLM(LM): string_nll = sum(string_nll) loglikelihoods.append(string_nll) + + # cache this loglikelihood_rolling request + self.cache_hook.add_partial("loglikelihood_rolling", (string,), string_nll) return loglikelihoods def _loglikelihood_tokens(self, requests, disable_tqdm=False): @@ -468,6 +471,9 @@ class NeMoLM(LM): answer = (logprob, is_greedy) if cache_key is not None: + # special case: loglikelihood_rolling produces a number of loglikelihood requests + # all with cache key None. instead do add_partial on the per-example level + # in the loglikelihood_rolling() function for those. self.cache_hook.add_partial("loglikelihood", cache_key, answer) res.append(answer) diff --git a/lm_eval/models/neuralmagic.py b/lm_eval/models/neuralmagic.py index 7c16b06d50b2b8117cf0b6d6b33d9d4a2b681923..0d66f599e671c60d89716498199224335d8929c7 100644 --- a/lm_eval/models/neuralmagic.py +++ b/lm_eval/models/neuralmagic.py @@ -38,8 +38,8 @@ class SparseMLLM(HFLM): ) -> None: try: from sparseml.transformers import SparseAutoModelForCausalLM - except ModuleNotFoundError: - raise Exception( + except ModuleNotFoundError as exception: + raise type(exception)( "Package `sparseml` is not installed. " "Please install it via `pip install sparseml[transformers]`" ) @@ -88,8 +88,8 @@ class SparseMLLM(HFLM): def _get_config(self, pretrained: str, **kwargs) -> None: try: from sparseml.transformers import SparseAutoConfig - except ModuleNotFoundError: - raise Exception( + except ModuleNotFoundError as exception: + raise type(exception)( "Package `sparseml` is not installed. " "Please install it via `pip install sparseml[transformers]`" ) @@ -112,8 +112,8 @@ class SparseMLLM(HFLM): ) -> None: try: from sparseml.transformers import SparseAutoTokenizer - except ModuleNotFoundError: - raise Exception( + except ModuleNotFoundError as exception: + raise type(exception)( "Package `sparseml` is not installed. " "Please install it via `pip install sparseml[transformers]`" ) @@ -171,8 +171,8 @@ class DeepSparseLM(LM): try: import deepsparse - except ModuleNotFoundError: - raise Exception( + except ModuleNotFoundError as exception: + raise type(exception)( "Package `deepsparse` is not installed. " "Please install it via `pip install deepsparse[transformers]`" ) @@ -321,6 +321,9 @@ class DeepSparseLM(LM): res.append(answer) if cache_key is not None: + # special case: loglikelihood_rolling produces a number of loglikelihood requests + # all with cache key None. instead do add_partial on the per-example level + # in the loglikelihood_rolling() function for those. self.cache_hook.add_partial("loglikelihood", cache_key, answer) return re_ord.get_original(res) diff --git a/lm_eval/models/neuron_optimum.py b/lm_eval/models/neuron_optimum.py index e3f53bcf5fd8c19a7dcf517edf23ab0394064efc..ca2aaf657eeba309e116ef4b99db98686e7c1376 100644 --- a/lm_eval/models/neuron_optimum.py +++ b/lm_eval/models/neuron_optimum.py @@ -1,7 +1,5 @@ import copy -import json import logging -import subprocess from collections import defaultdict from typing import List, Optional, Union @@ -33,54 +31,6 @@ except ImportError: logger = logging.getLogger(__name__) -def get_nc_count() -> Union[int, None]: - """Returns the number of neuron cores on the current instance.""" - try: - cmd = "neuron-ls --json-output" - result = subprocess.run(cmd, shell=True, capture_output=True) - print(f"inferring nc_count from `neuron-ls` {result.stdout}") - json_output = json.loads(result.stdout) - count = sum([x["nc_count"] for x in json_output]) - print(f"nc_count={count}") - return count - except Exception: - return None - - -def wrap_constant_batch_size(func): - def _decorator(self, input_ids): - """input_ids a 2D array with batch_size on dim=0 - - makes sure the func runs with self.batch_size - """ - # access a from TestSample - batch_size = input_ids.shape[0] - - if batch_size < self.batch_size: - # handle the event of input_ids.shape[0] != batch_size - # Neuron cores expect constant batch_size - input_ids = torch.concat( - ( - input_ids, - # add missing_batch_size dummy - torch.zeros( - [self.batch_size - batch_size, *input_ids.size()[1:]], - dtype=input_ids.dtype, - device=input_ids.device, - ), - ), - dim=0, - ) - elif batch_size > self.batch_size: - raise ValueError( - f"The specified batch_size ({batch_size}) exceeds the model static batch size ({self.batch_size})" - ) - # return the forward pass that requires constant batch size - return func(self, input_ids)[:batch_size] - - return _decorator - - class CustomNeuronModelForCausalLM(NeuronModelForCausalLM): """NeuronModelForCausalLM with `stopping_criteria` in `generate`""" @@ -146,7 +96,7 @@ class CustomNeuronModelForCausalLM(NeuronModelForCausalLM): raise ValueError( f"The specified batch_size ({batch_size}) exceeds the model static batch size ({self.batch_size})" ) - elif batch_size < self.batch_size: + elif batch_size < self.batch_size and not self.continuous_batching: logger.warning( "Inputs will be padded to match the model static batch size. This will increase latency." ) @@ -158,8 +108,6 @@ class CustomNeuronModelForCausalLM(NeuronModelForCausalLM): if attention_mask is not None: padding = torch.zeros(padding_shape, dtype=torch.int64) padded_attention_mask = torch.cat([attention_mask, padding]) - # Drop the current generation context and clear the Key/Value cache - self.reset_generation() output_ids = self.generate_tokens( padded_input_ids, @@ -179,8 +127,6 @@ class NEURON_HF(TemplateLM): Tested with neuron 2.17.0 """ - _DEFAULT_MAX_LENGTH = 2048 - def __init__( self, pretrained: Optional[str] = "TinyLlama/TinyLlama-1.1B-Chat-v1.0", @@ -198,12 +144,12 @@ class NEURON_HF(TemplateLM): add_bos_token: Optional[bool] = False, ) -> None: if not NEURON_AVAILABLE: - raise Exception( + raise ImportError( "Tried to load neuron model, but neuron is not installed ", "please install neuron via pip install transformers-neuron ", "also make sure you are running on an AWS inf2 instance", ) - if version.parse(optimum_neuron_version) != version.parse("0.0.17"): + if version.parse(optimum_neuron_version) != version.parse("0.0.24"): logger.warning( '`optimum-neuron` model requires `pip install "optimum[neuronx]>=0.0.17" ' "preferably using the Hugging Face Neuron Deep Learning AMI (Ubuntu 22.04) " @@ -217,34 +163,16 @@ class NEURON_HF(TemplateLM): self.batch_size_per_gpu = int(batch_size) batch_size = int(batch_size) - if tp_degree is None: - # execute `neuron-ls --json-output | jq '.[0].nc_count'`` - # to get the number of neuron cores on your instance - tp_degree = get_nc_count() - - assert isinstance(tp_degree, int), ( - f"model_args must include tp_degree. tp_degree must be set to an integer," - f" but is tp_degree=`{tp_degree}` with type=`{type(tp_degree)}`." - "Set it to number of neuron cores on your instance." - " For inf2.xlarge and inf2.8xlarge, set it to `2`." - " For inf2.24xlarge, set it to `12`." - " For inf2.48xlarge, set it to `24`." - ) - - # TODO: update this to be less of a hack once subfolder is fixed in HF - revision = revision + ("/" + subfolder if subfolder is not None else "") self._config = transformers.AutoConfig.from_pretrained( pretrained, revision=revision, trust_remote_code=trust_remote_code, ) - torch_dtype = lm_eval.models.utils.get_dtype(dtype) - assert torch_dtype in [ - torch.float16, - torch.bfloat16, - ], "Only float16 and bfloat16 are supported" + revision = str(revision) # cast to string if not already one + # TODO: update this to be less of a hack once subfolder is fixed in HF + revision = revision + ("/" + subfolder if subfolder is not None else "") self.tokenizer = transformers.AutoTokenizer.from_pretrained( pretrained if tokenizer is None else tokenizer, @@ -253,36 +181,58 @@ class NEURON_HF(TemplateLM): use_fast=use_fast_tokenizer, ) - # Neuron specific code - if torch_dtype == torch.float16: - self.amp_dtype = "f16" - elif torch_dtype == torch.bfloat16: - self.amp_dtype = "bf16" - elif torch_dtype == torch.float32: - self.amp_dtype = "f32" - else: - raise NotImplementedError("Only float16 and bfloat16 are implemented.") - - compiler_args = {"num_cores": tp_degree, "auto_cast_type": self.amp_dtype} - input_shapes = { - "batch_size": batch_size, - "sequence_length": self._DEFAULT_MAX_LENGTH, - } + neuron_config = getattr(self._config, "neuron", None) + if neuron_config is None: + # Check export parameters + if tp_degree is not None: + assert isinstance(tp_degree, int), ( + f"tp_degree must be set to an integer," + f" but is tp_degree=`{tp_degree}` with type=`{type(tp_degree)}`." + "Set it to a number lower than the number of neuron cores on your instance." + " For inf2.xlarge and inf2.8xlarge, set it to `2`." + " For inf2.24xlarge, set it <= `12`." + " For inf2.48xlarge, set it <= `24`." + ) + torch_dtype = lm_eval.models.utils.get_dtype(dtype) + + if torch_dtype == torch.float16: + self.amp_dtype = "f16" + elif torch_dtype == torch.bfloat16: + self.amp_dtype = "bf16" + elif torch_dtype == torch.float32: + self.amp_dtype = "f32" + else: + raise NotImplementedError( + "Only float16/bfloat16/float32 are supported." + ) - print( - f"{'='*20} \n loading model to neuron with" - f" {compiler_args}, {input_shapes}..." - ) - self.model = CustomNeuronModelForCausalLM.from_pretrained( - pretrained, - revision=revision, - trust_remote_code=trust_remote_code, - low_cpu_mem_usage=low_cpu_mem_usage, - export=True, - **compiler_args, - **input_shapes, - ) - print(f"SUCCESS: neuron model compiled. \n {'='*20}") + print(f"{'='*20} \n exporting model to neuron") + self.model = CustomNeuronModelForCausalLM.from_pretrained( + pretrained, + revision=revision, + trust_remote_code=trust_remote_code, + low_cpu_mem_usage=low_cpu_mem_usage, + export=True, + batch_size=batch_size, + num_cores=tp_degree, + auto_cast_type=self.amp_dtype, + sequence_length=max_length, + ) + neuron_config = self.model.config.neuron + print( + f"SUCCESS: neuron model exported with config {neuron_config}. \n {'='*20}" + ) + else: + print( + f"{'='*20} \n loading neuron model with config" f" {neuron_config}..." + ) + self.model = CustomNeuronModelForCausalLM.from_pretrained( + pretrained, + revision=revision, + trust_remote_code=trust_remote_code, + low_cpu_mem_usage=low_cpu_mem_usage, + ) + print(f"SUCCESS: neuron model loaded. \n {'='*20}") self.truncation = truncation @@ -290,8 +240,6 @@ class NEURON_HF(TemplateLM): self.tokenizer.pad_token_id = self.tokenizer.eos_token_id self.add_bos_token = add_bos_token - self._max_length = max_length - self.batch_schedule = 1 self.batch_sizes = {} @@ -312,17 +260,7 @@ class NEURON_HF(TemplateLM): @property def max_length(self): - if self._max_length: # if max length manually set, return it - return self._max_length - seqlen_config_attrs = ("n_positions", "max_position_embeddings", "n_ctx") - for attr in seqlen_config_attrs: - if hasattr(self.model.config, attr): - return getattr(self.model.config, attr) - if hasattr(self.tokenizer, "model_max_length"): - if self.tokenizer.model_max_length == 1000000000000000019884624838656: - return self._DEFAULT_MAX_LENGTH - return self.tokenizer.model_max_length - return self._DEFAULT_MAX_LENGTH + return self.model.max_length @property def max_gen_toks(self) -> int: @@ -390,34 +328,6 @@ class NEURON_HF(TemplateLM): def tok_decode(self, tokens): return self.tokenizer.decode(tokens) - @wrap_constant_batch_size - def _model_call(self, input_ids: torch.Tensor): - """ - get logits for the entire sequence - - :param input_ids: torch.Tensor - A torch tensor of shape [batch, sequence_cont] - the size of sequence may vary from call to call - :return - A torch tensor of shape [batch, sequence, vocab] with the - logits returned from the model's decoder-lm head - """ - _, sequence_length = input_ids.shape - - with torch.inference_mode(): - cache_ids = torch.arange(0, sequence_length, dtype=torch.int32).split(1) - input_ids_split = input_ids.split(1, dim=1) - - return torch.concat( - [ - self.model.forward( - input_ids=input_id, cache_ids=cache_id, return_dict=False - )[0] - for input_id, cache_id in zip(input_ids_split, cache_ids) - ], - dim=1, - ) - def _model_generate(self, context, max_length, stop, **generation_kwargs): # we require users to pass do_sample=True explicitly # for non-greedy gen. This should be reevaluated when considering beam search. @@ -501,7 +411,8 @@ class NEURON_HF(TemplateLM): string_nll = sum(string_nll) loglikelihoods.append(string_nll) - + # cache this loglikelihood_rolling request + self.cache_hook.add_partial("loglikelihood_rolling", (string,), string_nll) return loglikelihoods def _loglikelihood_tokens( @@ -578,15 +489,41 @@ class NEURON_HF(TemplateLM): cont_toks_list.append(continuation_enc) inplens.append(inplen) - # create encoder attn mask and batched conts, if seq2seq - call_kwargs = {} + # Add dummy inputs up to the model static batch size + if len(inps) < self.batch_size: + inps = inps + [ + torch.zeros_like(inps[0]), + ] * (self.batch_size - len(inps)) + + masks = [torch.ones_like(inp) for inp in inps] batched_inps = lm_eval.models.utils.pad_and_concat( padding_len_inp, inps, padding_side="right" ) # [batch, padding_len_inp] - multi_logits = F.log_softmax( - self._model_call(batched_inps, **call_kwargs), dim=-1 - ) # [batch, padding_length (inp or cont), vocab] + batched_masks = lm_eval.models.utils.pad_and_concat( + padding_len_inp, masks, padding_side="right" + ) + if self.model.model.neuron_config.output_all_logits: + inputs = self.model.prepare_inputs_for_prefill( + batched_inps, batched_masks + ) + multi_logits = F.log_softmax( + self.model.forward(**inputs).logits, dim=-1 + ) # [batch, padding_length (inp or cont), vocab] + else: + # The model will only return the logits for the last input token, so we need + # to iterate over inputs to accumulate logits. + # To speed things up we use the KV cache as we would do when generating. + inputs = self.model.prepare_inputs_for_prefill( + batched_inps[:, :1], batched_masks[:, :1] + ) + outputs = [self.model.forward(**inputs).logits] + for i in range(1, padding_len_inp): + inputs = self.model.prepare_inputs_for_decode( + batched_inps[:, : i + 1], batched_masks[:, : i + 1] + ) + outputs.append(self.model.forward(**inputs).logits) + multi_logits = F.log_softmax(torch.concat(outputs, dim=1), dim=-1) for (cache_key, _, _), logits, inplen, cont_toks in zip( chunk, multi_logits, inplens, cont_toks_list @@ -619,7 +556,11 @@ class NEURON_HF(TemplateLM): res.append(answer) - self.cache_hook.add_partial("loglikelihood", cache_key, answer) + if cache_key is not None: + # special case: loglikelihood_rolling produces a number of loglikelihood requests + # all with cache key None. instead do add_partial on the per-example level + # in the loglikelihood_rolling() function for those. + self.cache_hook.add_partial("loglikelihood", cache_key, answer) return re_ord.get_original(res) diff --git a/lm_eval/models/openai_completions.py b/lm_eval/models/openai_completions.py index 492c41fb3144c9ca0df2f0d503e2ac2a851ffc52..2f10316f754bdff488f9edfa04190d03a109c619 100644 --- a/lm_eval/models/openai_completions.py +++ b/lm_eval/models/openai_completions.py @@ -1,478 +1,281 @@ -import copy import os -from collections import defaultdict -from importlib.util import find_spec -from typing import List, Literal, Optional, Tuple +from functools import cached_property +from typing import Any, Dict, List, Optional, Tuple, Union -from tqdm import tqdm - -import lm_eval.models.utils -from lm_eval import utils -from lm_eval.api.model import LM, TemplateLM from lm_eval.api.registry import register_model -from lm_eval.models.utils import retry_on_specific_exceptions +from lm_eval.models.api_models import TemplateAPI from lm_eval.utils import eval_logger -def get_result(response) -> Tuple[float, bool]: - """Process results from OpenAI API response. - - :param response: dict - OpenAI API Response - :return: - continuation_logprobs: np.array - Log probabilities of continuation tokens - is_greedy: bool - whether argmax matches given continuation exactly - """ - is_greedy = True - logprobs = response.logprobs.token_logprobs - continuation_logprobs = sum(logprobs) - - for i in range(len(response.logprobs.token_logprobs)): - token = response.logprobs.token_logprobs[i] - top_tokens = response.logprobs.top_logprobs[i] - top_token = max(top_tokens.keys(), key=lambda x: top_tokens[x]) - if top_token != token: - is_greedy = False - break - - return continuation_logprobs, is_greedy - - -def oa_completion(client, chat: bool = False, **kwargs): - """Query OpenAI API for completion. - - Retry with back-off until they respond - """ - if not find_spec("openai") or not find_spec("tiktoken"): - raise Exception( - "attempted to use 'openai' LM type, but package `openai` or `tiktoken` are not installed. " - "Please install these via `pip install lm-eval[openai]` or `pip install -e .[openai]`" +@register_model("local-completions") +class LocalCompletionsAPI(TemplateAPI): + def __init__( + self, + base_url=None, + tokenizer_backend="huggingface", + **kwargs, + ): + super().__init__( + base_url=base_url, tokenizer_backend=tokenizer_backend, **kwargs ) - else: - import openai - - def _exception_callback(e: Exception, sleep_time: float) -> None: - import traceback - - traceback.print_exc() - - @retry_on_specific_exceptions( - on_exceptions=[openai.OpenAIError], - max_retries=None, # retry forever, consider changing - on_exception_callback=_exception_callback, - ) - def completion(): - if chat: - return client.chat.completions.create(**kwargs) - else: - return client.completions.create(**kwargs) - return completion() - - -@register_model("openai-completions", "local-completions") -class OpenaiCompletionsLM(TemplateLM): - _DEFAULT_MAX_LENGTH = 2048 - - def __init__( + def _create_payload( self, - model: str, - base_url: str = None, - tokenizer: Optional[str] = None, - tokenizer_backend: Literal["tiktoken", "huggingface"] = "tiktoken", - truncate: bool = False, - max_gen_toks: int = 256, - batch_size: int = 1, + messages: Union[List[List[int]], List[dict], List[str], str], + generate=False, + gen_kwargs: Optional[dict] = None, seed: int = 1234, - max_length: Optional[int] = None, - ) -> None: - """ - - :param engine: str - OpenAI API engine (e.g. gpt-3.5-turbo-instruct) - :param truncate: bool - Truncate input if too long (if False and input is too long, throw error) - """ - super().__init__() - self.seed = seed - try: - import openai # noqa: E401 - import tiktoken - except ModuleNotFoundError: - raise Exception( - "attempted to use 'openai' LM type, but package `openai` or `tiktoken` are not installed. \ - please install these via `pip install lm-eval[openai]` or `pip install -e .\"[openai]\"`", - ) - self.model = model - self.base_url = base_url - self.tokenizer_backend = tokenizer_backend - self.truncate = truncate - self._batch_size = int(batch_size) - self._max_gen_toks = max_gen_toks - self._max_length = max_length - - # if we have a local model, use HF tokenizer over tiktoken - if self.tokenizer_backend == "huggingface": - import transformers # noqa: E401 - - self.tokenizer = transformers.AutoTokenizer.from_pretrained( - tokenizer if tokenizer else self.model - ) - self.vocab_size = self.tokenizer.vocab - self.end_of_text_token_id = self.tokenizer.eos_token - elif self.tokenizer_backend == "tiktoken": - if self.base_url: - eval_logger.warning( - f"Passed `base_url={self.base_url}` but using Tiktoken tokenizer backend. " - "Pass `tokenizer_backend=huggingface` and provide the HF tokenizer name if your model does not use Tiktoken." - ) - - self.tokenizer = tiktoken.encoding_for_model(self.model) - self.vocab_size = self.tokenizer.n_vocab - self.end_of_text_token_id = self.tokenizer.eot_token - else: - raise ValueError( - f"Expected tokenizer_backend to be one of ['tiktoken', 'huggingface'] but got {self.tokenizer_backend}" - ) - - # Read from environment variable OPENAI_API_KEY - # Set to EMPTY for local - openai.api_key = os.environ["OPENAI_API_KEY"] - if self.base_url: - self.client = openai.OpenAI(base_url=self.base_url) - else: - self.client = openai.OpenAI() - - @property - def eot_token_id(self): - return self.end_of_text_token_id - - @property - def max_length(self) -> int: - if self._max_length: - return self._max_length + **kwargs, + ) -> dict: + if generate: + gen_kwargs.pop("do_sample", False) + if "max_tokens" in gen_kwargs: + max_tokens = gen_kwargs.pop("max_tokens") + else: + max_tokens = gen_kwargs.pop("max_gen_toks", self._max_gen_toks) + temperature = gen_kwargs.pop("temperature", 0) + stop = gen_kwargs.pop("until", ["<|endoftext|>"]) + return { + "prompt": messages, + "model": self.model, + "max_tokens": max_tokens, + "temperature": temperature, + "stop": stop, + "seed": seed, + **gen_kwargs, + } else: - return self._DEFAULT_MAX_LENGTH - - @property - def max_gen_toks(self) -> int: - return self._max_gen_toks - - @property - def batch_size(self) -> int: - return self._batch_size - - @property - def device(self): - # Isn't used because we override _loglikelihood_tokens - raise NotImplementedError() - - def tok_encode(self, string: str, **kwargs) -> List[int]: - return self.tokenizer.encode(string) - - def tok_decode(self, tokens: List[int]) -> str: - return self.tokenizer.decode(tokens) - - def _loglikelihood_tokens( - self, requests, disable_tqdm: bool = False + return { + "model": self.model, + "prompt": messages, + "temperature": 0, + "max_tokens": 1, + "logprobs": 1, + "seed": seed, + "echo": True, + } + + @staticmethod + def parse_logprobs( + outputs: Union[Dict, List[Dict]], + tokens: List[List[int]] = None, + ctxlens: List[int] = None, + **kwargs, ) -> List[Tuple[float, bool]]: res = [] - - def _collate(x): - # this doesn't efficiently handle last-token differences yet, but those are kinda annoying because - # it's not guaranteed that the 100 or so logprobs we get to see actually contain all the continuations - # we care about, and so we need some kind of backup for when it isn't - toks = x[1] + x[2] - return -len(toks), tuple(toks) - - re_ord = utils.Reorderer(requests, _collate) - - for chunk in tqdm( - list(lm_eval.models.utils.chunks(re_ord.get_reordered(), self.batch_size)), - disable=disable_tqdm, - ): - inps = [] - ctxlens = [] - for cache_key, context_enc, continuation_enc in chunk: - # max_length+1 because the API takes up to 2049 tokens, including the first context token - inp = (context_enc + continuation_enc)[-(self.max_length + 1) :] - # TODO: the logic is much simpler if we just look at the length of continuation tokens - ctxlen = len(context_enc) - max( - 0, len(context_enc) + len(continuation_enc) - (self.max_length + 1) - ) - - inps.append(inp) - ctxlens.append(ctxlen) - - response = oa_completion( - client=self.client, - model=self.model, - prompt=inps, - max_tokens=0, - temperature=0.0, - logprobs=10, - seed=self.seed, - ) - - for resp, ctxlen, (cache_key, context_enc, continuation_enc) in zip( - response.choices, ctxlens, chunk - ): - answer = get_result(resp) - - res.append(answer) - - # partial caching - if cache_key is not None: - self.cache_hook.add_partial("loglikelihood", cache_key, answer) - return re_ord.get_original(res) - - def generate_until(self, requests, disable_tqdm: bool = False) -> List[str]: - if not requests: - return [] + if not isinstance(outputs, list): + outputs = [outputs] + for out in outputs: + for choice, ctxlen in zip(out["choices"], ctxlens): + assert ctxlen > 0, "Context length must be greater than 0" + logprobs = sum(choice["logprobs"]["token_logprobs"][ctxlen:-1]) + tokens_logprobs = choice["logprobs"]["token_logprobs"][ctxlen:-1] + top_logprobs = choice["logprobs"]["top_logprobs"][ctxlen:-1] + is_greedy = True + for tok, top in zip(tokens_logprobs, top_logprobs): + if tok != max(top.values()): + is_greedy = False + break + res.append((logprobs, is_greedy)) + return res + + @staticmethod + def parse_generations(outputs: Union[Dict, List[Dict]], **kwargs) -> List[str]: res = [] - requests = [req.args for req in requests] - - def _collate(x): - toks = self.tok_encode(x[0]) - return len(toks), x[0] + if not isinstance(outputs, list): + outputs = [outputs] + for out in outputs: + for choices in out["choices"]: + res.append(choices["text"]) + return res - re_ord = utils.Reorderer(requests, _collate) - - def sameuntil_chunks(xs, size): - ret = [] - lastuntil = xs[0][1] - for x in xs: - if len(ret) >= size or x[1] != lastuntil: - yield ret, lastuntil - ret = [] - lastuntil = x[1] - ret.append(x) - - if ret: - yield ret, lastuntil - - # todo: more intelligent batching for heterogeneous `until` - for chunk, request_args in tqdm( - list(sameuntil_chunks(re_ord.get_reordered(), self.batch_size)), - disable=disable_tqdm, - ): - inps = [] - self._max_gen_toks = request_args.get("max_gen_toks", self.max_gen_toks) - for context, _ in chunk: - context_enc = self.tok_encode(context) - inp = context_enc[-(self.max_length - self.max_gen_toks) :] - inps.append(inp) + @property + def api_key(self): + return os.environ.get("OPENAI_API_KEY", "") - until = request_args.get("until", ["<|endoftext|>"]) - request_args["temperature"] = request_args.get("temperature", 0) - response = oa_completion( - client=self.client, - model=self.model, - prompt=inps, - max_tokens=self.max_gen_toks, - stop=until, - seed=self.seed, - **{ - k: v - for k, v in request_args.items() - if k not in {"do_sample", "max_gen_toks", "until"} - }, +@register_model("local-chat-completions") +class LocalChatCompletion(LocalCompletionsAPI): + def __init__( + self, + base_url=None, + tokenizer_backend=None, + tokenized_requests=False, + **kwargs, + ): + eval_logger.warning( + "chat-completions endpoint requires the `--apply_chat_template` flag." + ) + super().__init__( + base_url=base_url, + tokenizer_backend=tokenizer_backend, + tokenized_requests=tokenized_requests, + **kwargs, + ) + if self._batch_size > 1: + eval_logger.warning( + "Chat completions does not support batching. Defaulting to batch size 1." ) - for resp, (context, args_) in zip(response.choices, chunk): - s = getattr(resp, "text") - - until_ = until - - for term in until_: - if len(term) > 0: - s = s.split(term)[0] - - # partial caching - self.cache_hook.add_partial( - "generate_until", (context, {"until": until_}), s - ) - - res.append(s) - return re_ord.get_original(res) - - def _model_call(self, inps): - # Isn't used because we override _loglikelihood_tokens - raise NotImplementedError() + self._batch_size = 1 - def _model_generate(self, context, max_length, eos_token_id): - # Isn't used because we override generate_until - raise NotImplementedError() + def _create_payload( + self, + messages: List[Dict], + generate=False, + gen_kwargs: dict = None, + seed=1234, + **kwargs, + ) -> dict: + assert ( + type(messages) is not str + ), "chat-completions require the --apply_chat_template flag." + gen_kwargs.pop("do_sample", False) + if "max_tokens" in gen_kwargs: + max_tokens = gen_kwargs.pop("max_tokens") + else: + max_tokens = gen_kwargs.pop("max_gen_toks", self._max_gen_toks) + temperature = gen_kwargs.pop("temperature", 0) + stop = gen_kwargs.pop("until", ["<|endoftext|>"]) + if not isinstance(stop, (list, tuple)): + stop = [stop] + return { + "messages": messages, + "model": self.model, + "max_tokens": max_tokens, + "temperature": temperature, + "stop": stop[:4], + "seed": seed, + **gen_kwargs, + } + + @staticmethod + def parse_generations(outputs: Union[Dict, List[Dict]], **kwargs) -> List[str]: + res = [] + if not isinstance(outputs, list): + outputs = [outputs] + for out in outputs: + for choices in out["choices"]: + res.append(choices["message"]["content"]) + return res + + def tok_encode( + self, + string: Union[str, Any], + left_truncate_len=None, + add_special_tokens=None, + **kwargs, + ) -> Union[List[str], List[int], Any]: + return string - def loglikelihood_rolling( - self, requests, disable_tqdm: bool = False - ) -> List[float]: - loglikelihoods = [] + def loglikelihood(self, requests, **kwargs): + raise NotImplementedError( + "Loglikelihood is not supported for chat completions. Consider using the completions API instead." + ) - for (string,) in tqdm([req.args for req in requests], disable=disable_tqdm): - rolling_token_windows = list( - map( - utils.make_disjoint_window, - utils.get_rolling_token_windows( - token_list=self.tok_encode(string), - prefix_token=self.eot_token_id, - max_seq_len=self.max_length, - context_len=1, - ), - ) - ) - # TODO: Right now, we pass single EOT token to the Encoder and the full context to the decoder, in seq2seq case - rolling_token_windows = [(None,) + x for x in rolling_token_windows] +@register_model( + "openai-completions", +) +class OpenAICompletionsAPI(LocalCompletionsAPI): + def __init__( + self, + base_url="https://api.openai.com/v1/completions", + tokenizer_backend="tiktoken", + **kwargs, + ): + super().__init__( + base_url=base_url, tokenizer_backend=tokenizer_backend, **kwargs + ) - string_nll = self._loglikelihood_tokens( - rolling_token_windows, - disable_tqdm=True, + @cached_property + def api_key(self): + """Override this property to return the API key for the API request.""" + key = os.environ.get("OPENAI_API_KEY", None) + if key is None: + raise ValueError( + "API key not found. Please set the `OPENAI_API_KEY` environment variable." ) + return key - # discard is_greedy - string_nll = [x[0] for x in string_nll] + def loglikelihood(self, requests, **kwargs): + assert ( + self.model + in [ + "babbage-002", + "davinci-002", + ] + ), f"Prompt loglikelihoods are only supported by OpenAI's API for {['babbage-002', 'davinci-002']}." + return super().loglikelihood(requests, **kwargs) - string_nll = sum(string_nll) - loglikelihoods.append(string_nll) - return loglikelihoods + def chat_template(self, chat_template: Union[bool, str] = False) -> Optional[str]: + return "" -@register_model("openai-chat-completions", "local-chat-completions") -class OpenaiChatCompletionsLM(LM): +@register_model("openai-chat-completions") +class OpenAIChatCompletion(LocalChatCompletion): def __init__( self, - model: str = "gpt-3.5-turbo", # GPT model or Local model using HuggingFace model paths - base_url: str = None, - truncate: bool = False, + base_url="https://api.openai.com/v1/chat/completions", + tokenizer_backend=None, + tokenized_requests=False, **kwargs, - ) -> None: - """ - - :param model: str - Implements an OpenAI-style chat completion API for - accessing both OpenAI OR locally-hosted models using - HuggingFace Tokenizer - OpenAI API model (e.g. gpt-3.5-turbo) - using the **gen_kwargs passed on init - :param truncate: bool - Truncate input if too long (if False and input is too long, throw error) - """ - super().__init__() - try: - import openai # noqa: E401 - except ModuleNotFoundError: - raise Exception( - "attempted to use 'openai' LM type, but package `openai` or `tiktoken` are not installed. \ - please install these via `pip install lm-eval[openai]` or `pip install -e .[openai]`", + ): + if "o1" in kwargs.get("model", ""): + eval_logger.warning( + "o1 models do not support `stop` and only support temperature=1" ) - self.model = model - self.base_url = base_url - self.truncate = truncate - - # Read from environment variable OPENAI_API_KEY - # Set to EMPTY for local - if self.base_url: - self.client = openai.OpenAI(base_url=self.base_url) - else: - self.client = openai.OpenAI() # openai.AsyncOpenAI() - - @property - def max_length(self) -> int: - # Note: the OpenAI API supports up to 2049 tokens, with the first token being the first input token - return 2048 - - @property - def max_gen_toks(self) -> int: - return 256 - - @property - def batch_size(self): - # Isn't used because we override _loglikelihood_tokens - raise NotImplementedError() - - @property - def device(self): - # Isn't used because we override _loglikelihood_tokens - raise NotImplementedError() - - def generate_until(self, requests, disable_tqdm: bool = False) -> List[str]: - res = defaultdict(list) - re_ords = {} + super().__init__( + base_url=base_url, + tokenizer_backend=tokenizer_backend, + tokenized_requests=tokenized_requests, + **kwargs, + ) - # we group requests by their generation_kwargs, - # so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling - # in the same batch. - grouper = lm_eval.models.utils.Grouper(requests, lambda x: str(x.args[1])) - for key, reqs in grouper.get_grouped().items(): - # within each set of reqs for given kwargs, we reorder by token length, descending. - re_ords[key] = utils.Reorderer( - [req.args for req in reqs], lambda x: (-len(x[0]), x[0]) + @cached_property + def api_key(self): + """Override this property to return the API key for the API request.""" + key = os.environ.get("OPENAI_API_KEY", None) + if key is None: + raise ValueError( + "API key not found. Please set the `OPENAI_API_KEY` environment variable." ) + return key - pbar = tqdm(total=len(requests), disable=(disable_tqdm or (self.rank != 0))) - for key, re_ord in re_ords.items(): - # n needs to be 1 because messages in - # chat completion are not batch but - # is regarded as a single conversation. - chunks = lm_eval.models.utils.chunks(re_ord.get_reordered(), n=1) - for chunk in chunks: - contexts, all_gen_kwargs = zip(*chunk) - inps = [{"role": "user", "content": context} for context in contexts] - - gen_kwargs = all_gen_kwargs[0] - until = None - if isinstance(kwargs := copy.deepcopy(gen_kwargs), dict): - if "do_sample" in kwargs.keys(): - kwargs.pop("do_sample") - if "until" in kwargs.keys(): - until = kwargs.pop("until") - if isinstance(until, str): - until = [until] - elif not isinstance(until, list): - raise ValueError( - f"Expected repr(kwargs['until']) to be of type Union[str, list] but got {until}" - ) - kwargs["stop"] = until - kwargs["max_tokens"] = kwargs.pop("max_gen_toks", self.max_gen_toks) - else: - raise ValueError( - f"Expected repr(kwargs) to be of type repr(dict) but got {kwargs}" - ) - - response = oa_completion( - client=self.client, - chat=True, - messages=inps, - model=self.model, - **kwargs, - ) - - for resp, (context, args_) in zip(response.choices, chunk): - s = resp.message.content - - if until is not None: - for term in until: - if len(term) > 0: - s = s.split(term)[0] - - res[key].append(s) - - self.cache_hook.add_partial( - "generate_until", (context, {"until": until}), s - ) - pbar.update(1) - # reorder this group of results back to original unsorted form - res[key] = re_ord.get_original(res[key]) - - pbar.close() - - return grouper.get_original(res) - - def loglikelihood(self, requests, disable_tqdm: bool = False): - raise NotImplementedError("No support for logits.") + def loglikelihood(self, requests, **kwargs): + raise NotImplementedError( + "Loglikelihood (and therefore `multiple_choice`-type tasks) is not supported for chat completions as OpenAI does not provide prompt logprobs. See https://github.com/EleutherAI/lm-evaluation-harness/issues/942#issuecomment-1777836312 or https://github.com/EleutherAI/lm-evaluation-harness/issues/1196 for more background on this limitation." + ) - def loglikelihood_rolling(self, requests, disable_tqdm: bool = False): - raise NotImplementedError("No support for logits.") + def _create_payload( + self, + messages: List[Dict], + generate=False, + gen_kwargs: dict = None, + seed=1234, + **kwargs, + ) -> dict: + assert ( + type(messages) is not str + ), "chat-completions require the --apply_chat_template flag." + gen_kwargs.pop("do_sample", False) + if "max_tokens" in gen_kwargs: + max_tokens = gen_kwargs.pop("max_tokens") + else: + max_tokens = gen_kwargs.pop("max_gen_toks", self._max_gen_toks) + temperature = gen_kwargs.pop("temperature", 0) + stop = gen_kwargs.pop("until", ["<|endoftext|>"]) + if not isinstance(stop, (list, tuple)): + stop = [stop] + output = { + "messages": messages, + "model": self.model, + "max_completion_tokens": max_tokens, + "temperature": temperature, + "stop": stop[:4], + "seed": seed, + **gen_kwargs, + } + if "o1" in self.model: + output.pop("stop") + output["temperature"] = 1 + return output diff --git a/lm_eval/models/optimum_lm.py b/lm_eval/models/optimum_lm.py index 70d44abdaca859fa79bd1beed789c96ad2c22ca9..bd26239875e08e8153770ef43ac62558bf3acc12 100644 --- a/lm_eval/models/optimum_lm.py +++ b/lm_eval/models/optimum_lm.py @@ -50,7 +50,7 @@ class OptimumLM(HFLM): **kwargs, ) -> None: if not find_spec("optimum"): - raise Exception( + raise ModuleNotFoundError( "package `optimum` is not installed. Please install it via `pip install optimum[openvino]`" ) else: diff --git a/lm_eval/models/utils.py b/lm_eval/models/utils.py index 8bf553d1b585098289e734fb7c17bc80ff14fb2c..f4d6780f7cb891e2063ecf8d0b2496ee34eb48b2 100644 --- a/lm_eval/models/utils.py +++ b/lm_eval/models/utils.py @@ -5,6 +5,7 @@ import itertools import time from functools import wraps from typing import ( + TYPE_CHECKING, Any, Callable, Dict, @@ -24,6 +25,11 @@ import transformers from lm_eval.utils import eval_logger +if TYPE_CHECKING: + from transformers import PreTrainedTokenizerBase + from transformers.configuration_utils import PretrainedConfig + + def chunks(iter, n: int = 0, fn=None): """ Divides an iterable into chunks of specified size or based on a given function. @@ -621,3 +627,93 @@ class Collator: if arr: yield arr + + +def configure_pad_token( + tokenizer: "PreTrainedTokenizerBase", + model_config: Optional["PretrainedConfig"] = None, +) -> "PreTrainedTokenizerBase": + """ + This function checks if the (Hugging Face) tokenizer has a padding token and sets it if not present. + Some tokenizers require special handling. + + Args: + tokenizer: The tokenizer for which the padding token is to be handled. + model_config: The configuration of the model. Default is None. + + Returns: + The tokenizer after the padding token has been handled. + + Raises: + AssertionError: If the tokenizer is of type RWKVWorldTokenizer or Rwkv5Tokenizer and the padding token id is not 0. + """ + if tokenizer.pad_token: + pass + elif tokenizer.unk_token: + tokenizer.pad_token_id = tokenizer.unk_token_id + elif tokenizer.eos_token: + tokenizer.pad_token_id = tokenizer.eos_token_id + else: + # handle special cases + if model_config and getattr(model_config, "model_type", None) == "qwen": + # Qwen's trust_remote_code tokenizer does not allow for adding special tokens + tokenizer.pad_token = "<|endoftext|>" + elif ( + tokenizer.__class__.__name__ == "RWKVWorldTokenizer" + or tokenizer.__class__.__name__ == "Rwkv5Tokenizer" + ): + # The RWKV world tokenizer, does not allow for adding special tokens / setting the pad token (which is set as 0) + # The additional tokenizer name check is needed, as there exists rwkv4 models with neox tokenizer + # --- + # Note that the world tokenizer class name, might change in the future for the final huggingface merge + # https://github.com/huggingface/transformers/pull/26963 + assert tokenizer.pad_token_id == 0 + else: + tokenizer.add_special_tokens({"pad_token": "<|pad|>"}) + + return tokenizer + + +def replace_placeholders( + string: str, default_placeholder: str, image_token: str, max_images: int +): + """ + A utility function used for local multimodal models. It locates all `placeholder` string + occurrences in the given input `string_` and replaces the first `max_count` instances with + `replacement`, and all subsequent occurrences with the empty string. + + This is used to replace placeholder tags by model-specific image tokens like <|image_pad|> + and to allow for only the first `max_count` images to be passed to a model if desired. + + :param string: The original string containing placeholders. + :param default_placeholder: The placeholder text to be replaced. + :param image_token: The token to replace the placeholder with. + :param max_images: The maximum number of replacements to make. + :return: The string with placeholders replaced. + """ + count = 0 + result = [] + + parts = string.split(default_placeholder) + for part in parts[:-1]: # Iterate through all but the last part + result.append(part) + if count < max_images: + result.append(image_token) + count += 1 + elif default_placeholder != image_token: + result.append(default_placeholder) + + # Add the last part of the string + result.append(parts[-1]) + return "".join(result) + + +def flatten_image_list(images: List[List]): + """ + Takes in a list of lists of images, and returns a single list of all images in order. + Used for some multimodal models like Llava-1.5 which expects this flattened-list format for its image processor. + + :param images: A list of lists of PIL images. + :return: a list of PIL images, via concatenating all the sub-lists in order. + """ + return [image for image_list in images for image in image_list] diff --git a/lm_eval/models/vllm_causallms.py b/lm_eval/models/vllm_causallms.py index 34a63bf282efc0b55b5de1f60a016addcd22a829..0d8f3cea59787311fde213af009b1ff8f16c7423 100644 --- a/lm_eval/models/vllm_causallms.py +++ b/lm_eval/models/vllm_causallms.py @@ -1,7 +1,7 @@ import copy from importlib.metadata import version from importlib.util import find_spec -from typing import List, Literal, Optional, Tuple, Union +from typing import TYPE_CHECKING, Dict, List, Literal, Optional, Tuple, Union from more_itertools import distribute from packaging.version import parse as parse_version @@ -10,7 +10,7 @@ from tqdm import tqdm from lm_eval.api.instance import Instance from lm_eval.api.model import TemplateLM from lm_eval.api.registry import register_model -from lm_eval.models.utils import Collator, undistribute +from lm_eval.models.utils import Collator, configure_pad_token, undistribute from lm_eval.utils import ( eval_logger, get_rolling_token_windows, @@ -26,6 +26,8 @@ try: except ModuleNotFoundError: pass +if TYPE_CHECKING: + pass eval_logger = eval_logger @@ -63,7 +65,7 @@ class VLLM(TemplateLM): super().__init__() if not find_spec("vllm"): - raise Exception( + raise ModuleNotFoundError( "attempted to use 'vllm' LM type, but package `vllm` is not installed. " "Please install vllm via `pip install lm-eval[vllm]` or `pip install -e .[vllm]`" ) @@ -118,11 +120,12 @@ class VLLM(TemplateLM): trust_remote_code=trust_remote_code, tokenizer_revision=tokenizer_revision, ) + self.tokenizer = configure_pad_token(self.tokenizer) self.add_bos_token = add_bos_token if "gemma" in pretrained.lower(): self.add_bos_token = True eval_logger.info( - "Found 'gemma' in model name, a BOS token will be used as Gemma underperforms without it." + "Found 'gemma' in model name, a BOS token will be used as Gemma series models underperform without it." ) self.custom_prefix_token_id = prefix_token_id @@ -176,23 +179,40 @@ class VLLM(TemplateLM): def max_gen_toks(self): return self._max_gen_toks + def apply_chat_template(self, chat_history: List[Dict[str, str]]) -> str: + """ + Method to apply a chat template to a list of chat history between user and model. + """ + return self.tokenizer.apply_chat_template( + chat_history, tokenize=False, add_generation_prompt=True + ) + + @property + def tokenizer_name(self) -> str: + return self.tokenizer.name_or_path.replace("/", "__") + def tok_encode( self, - string: str, - left_truncate_len=None, - add_special_tokens=None, - truncation=False, - ): - """ """ + string: Union[str, List[str]], + left_truncate_len: int = None, + add_special_tokens: bool = False, + truncation: bool = False, + ) -> Union[List[int], List[List[int]]]: if not add_special_tokens: add_special_tokens = False or self.add_bos_token - encoding = self.tokenizer.encode( - string, add_special_tokens=add_special_tokens, truncation=truncation - ) + encoding: Union[List[List[int]], List[int]] = self.tokenizer( + string, + add_special_tokens=add_special_tokens, + truncation=truncation, + return_attention_mask=False, + ).input_ids # left-truncate the encoded context to be at most `left_truncate_len` tokens long if left_truncate_len: - encoding = encoding[-left_truncate_len:] + if not isinstance(string, str): + encoding = [enc[-left_truncate_len:] for enc in encoding] + else: + encoding = encoding[-left_truncate_len:] return encoding @@ -209,7 +229,7 @@ class VLLM(TemplateLM): sampling_params = SamplingParams(max_tokens=max_tokens, stop=stop, **kwargs) else: sampling_params = SamplingParams( - temperature=0, prompt_logprobs=1, max_tokens=1 + temperature=0, prompt_logprobs=1, max_tokens=1, detokenize=False ) if self.data_parallel_size > 1: # vLLM hangs if tensor_parallel > 1 and resources are set in ray.remote @@ -219,17 +239,25 @@ class VLLM(TemplateLM): # but then tensor_parallel breaks @ray.remote def run_inference_one_model( - model_args: dict, sampling_params, requests: List[List[int]] + model_args: dict, + sampling_params, + requests: List[List[int]], + lora_request: LoRARequest, ): llm = LLM(**model_args) return llm.generate( - prompt_token_ids=requests, sampling_params=sampling_params + prompt_token_ids=requests, + sampling_params=sampling_params, + lora_request=lora_request, ) # dispatch requests to all self.data_parallel_size workers, in interleaved fashion # interleaved important to balance context lengths across workers requests = [list(x) for x in distribute(self.data_parallel_size, requests)] - inputs = ((self.model_args, sampling_params, req) for req in requests) + inputs = ( + (self.model_args, sampling_params, req, self.lora_request) + for req in requests + ) object_refs = [run_inference_one_model.remote(*x) for x in inputs] results = ray.get(object_refs) # Invoke ray.shutdown() to prevent hang-ups if subsequent calls required. @@ -237,19 +265,12 @@ class VLLM(TemplateLM): # flatten results return undistribute(results) - if self.lora_request is not None: - outputs = self.model.generate( - prompt_token_ids=requests, - sampling_params=sampling_params, - use_tqdm=True if self.batch_size == "auto" else False, - lora_request=self.lora_request, - ) - else: - outputs = self.model.generate( - prompt_token_ids=requests, - sampling_params=sampling_params, - use_tqdm=True if self.batch_size == "auto" else False, - ) + outputs = self.model.generate( + prompt_token_ids=requests, + sampling_params=sampling_params, + use_tqdm=True if self.batch_size == "auto" else False, + lora_request=self.lora_request, + ) return outputs def loglikelihood_rolling( @@ -263,7 +284,8 @@ class VLLM(TemplateLM): make_disjoint_window, get_rolling_token_windows( token_list=self.tok_encode(string), - prefix_token=self.eot_token_id, + prefix_token=self.prefix_token_id, + # max_seq_len - (1 for context) max_seq_len=self.max_length - 1, context_len=1, ), @@ -281,6 +303,10 @@ class VLLM(TemplateLM): string_nll = sum(string_nll) loglikelihoods.append(string_nll) + + # cache this loglikelihood_rolling request + self.cache_hook.add_partial("loglikelihood_rolling", (string,), string_nll) + return loglikelihoods def generate_until( @@ -290,7 +316,9 @@ class VLLM(TemplateLM): # batch tokenize contexts context, all_gen_kwargs = zip(*(req.args for req in requests)) - context_encoding = self.tokenizer(context, add_special_tokens=False).input_ids + context_encoding: List[List[int]] = self.tok_encode( + context, add_special_tokens=self.add_bos_token + ) requests = [ ((a, b), c) for a, b, c in zip(context, context_encoding, all_gen_kwargs) ] @@ -425,8 +453,10 @@ class VLLM(TemplateLM): res.append(answer) - # partial caching if cache_key is not None: + # special case: loglikelihood_rolling produces a number of loglikelihood requests + # all with cache key None. instead do add_partial on the per-example level + # in the loglikelihood_rolling() function for those. self.cache_hook.add_partial("loglikelihood", cache_key, answer) pbar.update(1) pbar.close() diff --git a/lm_eval/models/vllm_vlms.py b/lm_eval/models/vllm_vlms.py new file mode 100644 index 0000000000000000000000000000000000000000..13a2a3ff4ca44429fee10d6511475581c1144c3c --- /dev/null +++ b/lm_eval/models/vllm_vlms.py @@ -0,0 +1,291 @@ +import copy +from typing import Dict, List, Optional + +import transformers +from more_itertools import distribute +from tqdm import tqdm + +from lm_eval.api.instance import Instance +from lm_eval.api.registry import register_model +from lm_eval.models.utils import Collator, replace_placeholders, undistribute +from lm_eval.models.vllm_causallms import VLLM +from lm_eval.utils import eval_logger + + +try: + import ray + from vllm import LLM, SamplingParams + from vllm.lora.request import LoRARequest # noqa: F401 + from vllm.transformers_utils.tokenizer import get_tokenizer # noqa: F401 +except ModuleNotFoundError: + pass + + +DEFAULT_IMAGE_PLACEHOLDER = "" + + +@register_model("vllm-vlm") +class VLLM_VLM(VLLM): + MULTIMODAL = True + + def __init__( + self, + pretrained: str, + trust_remote_code: Optional[bool] = False, + revision: Optional[str] = None, + interleave: bool = True, + # TODO: handle max_images and limit_mm_per_prompt better + max_images: int = 999, + **kwargs, + ): + if max_images != 999: + kwargs["limit_mm_per_prompt"] = {"image": max_images} + eval_logger.info(f"Setting limit_mm_per_prompt[image] to {max_images}") + super().__init__( + pretrained=pretrained, + trust_remote_code=trust_remote_code, + revision=revision, + **kwargs, + ) + self.interleave = interleave + self.max_images = max_images + self.processor = transformers.AutoProcessor.from_pretrained( + pretrained, + revision=revision, + trust_remote_code=trust_remote_code, + ) + self.chat_applied: bool = False + + def tok_batch_multimodal_encode( + self, + strings: List[str], # note that input signature of this fn is different + images, # TODO: typehint on this + left_truncate_len: int = None, + truncation: bool = False, + ): + images = [img[: self.max_images] for img in images] + # TODO: is the default placeholder always ? + if self.chat_applied is False: + strings = [ + replace_placeholders( + string, + DEFAULT_IMAGE_PLACEHOLDER, + DEFAULT_IMAGE_PLACEHOLDER, + self.max_images, + ) + for string in strings + ] + + outputs = [] + for x, i in zip(strings, images): + inputs = { + "prompt": x, + "multi_modal_data": {"image": i}, + } + outputs.append(inputs) + return outputs + + def _model_generate( + self, + requests: List[List[dict]] = None, + generate: bool = False, + max_tokens: int = None, + stop: Optional[List[str]] = None, + **kwargs, + ): + if generate: + kwargs = self.modify_gen_kwargs(kwargs) + sampling_params = SamplingParams(max_tokens=max_tokens, stop=stop, **kwargs) + else: + sampling_params = SamplingParams( + temperature=0, prompt_logprobs=1, max_tokens=1, detokenize=False + ) + if self.data_parallel_size > 1: + # vLLM hangs if tensor_parallel > 1 and resources are set in ray.remote + # also seems to only work with decorator and not with ray.remote() fn + # see https://github.com/vllm-project/vllm/issues/973 + # note: this has changed on 0.3.3, and it only works now if num_gpus are set. + # but then tensor_parallel breaks + @ray.remote + def run_inference_one_model( + model_args: dict, sampling_params, requests: List[List[dict]] + ): + llm = LLM(**model_args) + return llm.generate(requests, sampling_params=sampling_params) + + # dispatch requests to all self.data_parallel_size workers, in interleaved fashion + # interleaved important to balance context lengths across workers + requests = [list(x) for x in distribute(self.data_parallel_size, requests)] + inputs = ((self.model_args, sampling_params, req) for req in requests) + object_refs = [run_inference_one_model.remote(*x) for x in inputs] + results = ray.get(object_refs) + # Invoke ray.shutdown() to prevent hang-ups if subsequent calls required. + ray.shutdown() + # flatten results + return undistribute(results) + + if self.lora_request is not None: + outputs = self.model.generate( + requests, + sampling_params=sampling_params, + use_tqdm=True if self.batch_size == "auto" else False, + lora_request=self.lora_request, + ) + else: + outputs = self.model.generate( + requests, + sampling_params=sampling_params, + use_tqdm=True if self.batch_size == "auto" else False, + ) + return outputs + + def apply_chat_template(self, chat_history: List[Dict[str, str]]) -> str: + self.chat_applied = True + if not self.interleave: + for content in chat_history: + c = [] + text = content["content"] + + # Count and remove image placeholders + image_count = min( + self.max_images, text.count(DEFAULT_IMAGE_PLACEHOLDER) + ) + text = text.replace(DEFAULT_IMAGE_PLACEHOLDER, "") + + # Add image entries + for _ in range(image_count): + c.append({"type": "image", "image": None}) + + # Add single text entry at the end + c.append({"type": "text", "text": text}) + + content["content"] = c + else: + for content in chat_history: + c = [] + text = content["content"] + expected_image_count = min( + self.max_images, text.count(DEFAULT_IMAGE_PLACEHOLDER) + ) + actual_image_count = 0 + + text_parts = text.split(DEFAULT_IMAGE_PLACEHOLDER) + + for i, part in enumerate(text_parts): + # TODO: concatenate text parts (esp. if skipping images)? + if part: # Add non-empty text parts + c.append({"type": "text", "text": part}) + if ( + (i < len(text_parts) - 1) and i < self.max_images + ): # Add image placeholder after each split except the last + c.append({"type": "image"}) + actual_image_count += 1 + + content["content"] = c + + if actual_image_count != expected_image_count: + raise ValueError( + f"Mismatch in image placeholder count. Expected: {expected_image_count}, Actual: {actual_image_count}" + ) + + return self.processor.apply_chat_template( + chat_history, add_generation_prompt=True + ) + + def generate_until( + self, requests: List[Instance], disable_tqdm: bool = False + ) -> List[str]: + # TODO: support text-only reqs + res = [] + + def _collate(x): + # the negative sign on len(toks) sorts descending - this has a few advantages: + # - time estimates will always be over not underestimates, which is more useful for planning + # - to know the size of a batch when going through the list, you know the first one is always the batch + # padded context length. this is useful to simplify the batching logic and more importantly to make + # automatic adaptive batches much much easier to implement + # - any OOMs will happen right away rather than near the end + toks = self.tok_encode(x[0]) + return -len(toks), x[0] + + pbar = tqdm( + total=len(requests), + disable=(disable_tqdm or (self.rank != 0)), + desc="Running generate_until requests with text+image input", + ) + # TODO: port auto-batch sizing into this. + + # we group requests by their generation_kwargs, + # so that we don't try to execute e.g. greedy sampling and temp=0.8 sampling + # in the same batch. + re_ords = Collator( + [reg.args for reg in requests], + _collate, + group_by="gen_kwargs", + group_fn=lambda x: x[1], + ) + chunks = re_ords.get_batched(n=self.batch_size, batch_fn=None) + + for chunk in chunks: + contexts, all_gen_kwargs, aux_arguments = zip(*chunk) + + visuals = [arg["visual"] for arg in aux_arguments] + + if not isinstance(contexts, list): + contexts = list( + contexts + ) # for Qwen2-VL, processor is unhappy accepting a tuple of strings instead of a list. + # TODO: could we upstream this workaround to HF? + + # we assume all gen kwargs in the batch are the same + # this is safe to assume because the `grouper` object ensures it. + gen_kwargs = all_gen_kwargs[0] + # unpack our keyword arguments. + until = None + if isinstance(gen_kwargs, dict): + kwargs = copy.deepcopy(gen_kwargs) # edge case for repeats > 1 + if "until" in kwargs.keys(): + until = kwargs.pop("until") + if isinstance(until, str): + until = [until] + elif not isinstance(until, list): + raise ValueError( + f"Expected `kwargs['until']` to be of type Union[str,list] but got {until}" + ) + else: + raise ValueError( + f"Expected `kwargs` to be of type `dict` but got {type(gen_kwargs)}" + ) + # add EOS token to stop sequences + eos = self.tokenizer.decode(self.eot_token_id) + if not until: + until = [eos] + else: + until.append(eos) + if "max_gen_toks" in kwargs.keys(): + max_gen_toks = kwargs.pop("max_gen_toks") + else: + max_gen_toks = self.max_gen_toks + + max_ctx_len = self.max_length - max_gen_toks + + inputs = self.tok_batch_multimodal_encode( + contexts, + visuals, + left_truncate_len=max_ctx_len, + ) + + cont = self._model_generate(inputs, stop=until, generate=True, **kwargs) + + for output, context in zip(cont, contexts): + generated_text = output.outputs[0].text + res.append(generated_text) + self.cache_hook.add_partial( + "generate_until", (context, gen_kwargs), generated_text + ) + pbar.update(1) + # reorder this group of results back to original unsorted form + res = re_ords.get_original(res) + + pbar.close() + return res diff --git a/lm_eval/prompts/__init__.py b/lm_eval/prompts/__init__.py index 1f814214de4afaabd1367854c74dc2143c346744..c486223b4327cbcad573965fce5b86105c36e9d1 100644 --- a/lm_eval/prompts/__init__.py +++ b/lm_eval/prompts/__init__.py @@ -29,8 +29,8 @@ def get_prompt(prompt_id: str, dataset_name: str = None, subset_name: str = None if category_name == "promptsource": try: from promptsource.templates import DatasetTemplates - except ModuleNotFoundError: - raise Exception( + except ModuleNotFoundError as exception: + raise type(exception)( "Tried to load a Promptsource template, but promptsource is not installed ", "please install promptsource via pip install lm-eval[promptsource] or pip install -e .[promptsource]", ) @@ -118,7 +118,7 @@ class PromptString: # TODO need a way to process doc_to_choice if "doc_to_choice" in self.prompt_string: - raise Exception("Not yet implemented to accept doc_to_choice") + raise NotImplementedError("Not yet implemented to accept doc_to_choice") text_string = utils.apply_template(doc_to_text, doc) target_string = utils.apply_template(doc_to_target, doc) diff --git a/lm_eval/tasks/README.md b/lm_eval/tasks/README.md index bb04d4f279e4e63949f79dfe215a7edcc74be0d3..5c1fc627572858a814b26f09d32fbdf1c7b89b44 100644 --- a/lm_eval/tasks/README.md +++ b/lm_eval/tasks/README.md @@ -11,11 +11,14 @@ | [aexams](aexams/README.md) | Tasks in Arabic related to various academic exams covering a range of subjects. | Arabic | | [agieval](agieval/README.md) | Tasks involving historical data or questions related to history and historical texts. | English, Chinese | | [anli](anli/README.md) | Adversarial natural language inference tasks designed to test model robustness. | English | +| [arabic_leaderboard_complete](arabic_leaderboard_complete/README.md) | A full version of the tasks in the Open Arabic LLM Leaderboard, focusing on the evaluation of models that reflect the characteristics of Arabic language understanding and comprehension, culture, and heritage. Note that some of these tasks are machine-translated. | Arabic (Some MT) | +| [arabic_leaderboard_light](arabic_leaderboard_light/README.md) | A light version of the tasks in the Open Arabic LLM Leaderboard (i.e., 10% samples of the test set in the original benchmarks), focusing on the evaluation of models that reflect the characteristics of Arabic language understanding and comprehension, culture, and heritage. Note that some of these tasks are machine-translated. | Arabic (Some MT) | | [arabicmmlu](arabicmmlu/README.md) | Localized Arabic version of MMLU with multiple-choice questions from 40 subjects. | Arabic | | [arc](arc/README.md) | Tasks involving complex reasoning over a diverse set of questions. | English | | [arithmetic](arithmetic/README.md) | Tasks involving numerical computations and arithmetic reasoning. | English | | [asdiv](asdiv/README.md) | Tasks involving arithmetic and mathematical reasoning challenges. | English | | [babi](babi/README.md) | Tasks designed as question and answering challenges based on simulated stories. | English | +| [basque_bench](basque_bench/README.md) | Collection of tasks in Basque encompassing various evaluation areas. | Basque | | [basqueglue](basqueglue/README.md) | Tasks designed to evaluate language understanding in Basque language. | Basque | | [bbh](bbh/README.md) | Tasks focused on deep semantic understanding through hypothesization and reasoning. | English, German | | [belebele](belebele/README.md) | Language understanding tasks in a variety of languages and scripts. | Multiple (122 languages) | @@ -23,10 +26,11 @@ | [bertaqa](bertaqa/README.md) | Local Basque cultural trivia QA tests in English and Basque languages. | English, Basque, Basque (MT) | | [bigbench](bigbench/README.md) | Broad tasks from the BIG-bench benchmark designed to push the boundaries of large models. | Multiple | | [blimp](blimp/README.md) | Tasks testing grammatical phenomena to evaluate language model's linguistic capabilities. | English | +| [catalan_bench](catalan_bench/README.md) | Collection of tasks in Catalan encompassing various evaluation areas. | Catalan | | [ceval](ceval/README.md) | Tasks that evaluate language understanding and reasoning in an educational context. | Chinese | | [cmmlu](cmmlu/README.md) | Multi-subject multiple choice question tasks for comprehensive academic assessment. | Chinese | | code_x_glue | Tasks that involve understanding and generating code across multiple programming languages. | Go, Java, JS, PHP, Python, Ruby | -| [commonsense_qa](commmonsense_qa/README.md) | CommonsenseQA, a multiple-choice QA dataset for measuring commonsense knowledge. | English | +| [commonsense_qa](commonsense_qa/README.md) | CommonsenseQA, a multiple-choice QA dataset for measuring commonsense knowledge. | English | | [copal_id](copal_id/README.md) | Indonesian causal commonsense reasoning dataset that captures local nuances. | Indonesian | | [coqa](coqa/README.md) | Conversational question answering tasks to test dialog understanding. | English | | [crows_pairs](crows_pairs/README.md) | Tasks designed to test model biases in various sociodemographic groups. | English, French | @@ -40,6 +44,7 @@ | [fda](fda/README.md) | Tasks for extracting key-value pairs from FDA documents to test information extraction. | English | | [fld](fld/README.md) | Tasks involving free-form and directed dialogue understanding. | English | | [french_bench](french_bench/README.md) | Set of tasks designed to assess language model performance in French. | French| +| [galician_bench](galician_bench/README.md) | Collection of tasks in Galician encompassing various evaluation areas. | Galician | | [glue](glue/README.md) | General Language Understanding Evaluation benchmark to test broad language abilities. | English | | [gpqa](gpqa/README.md) | Tasks designed for general public question answering and knowledge verification. | English | | [gsm8k](gsm8k/README.md) | A benchmark of grade school math problems aimed at evaluating reasoning capabilities. | English | @@ -49,6 +54,8 @@ | [hendrycks_ethics](hendrycks_ethics/README.md) | Tasks designed to evaluate the ethical reasoning capabilities of models. | English | | [hendrycks_math](hendrycks_math/README.md) | Mathematical problem-solving tasks to test numerical reasoning and problem-solving. | English | | [ifeval](ifeval/README.md) | Interactive fiction evaluation tasks for narrative understanding and reasoning. | English | +| [inverse_scaling](inverse_scaling/README.md) | Multiple-choice tasks from the Inverse Scaling Prize, designed to find settings where larger language models perform worse. | English | +| [japanese_leaderboard](japanese_leaderboard/README.md) | Japanese language understanding tasks to benchmark model performance on various linguistic aspects. | Japanese | | [kmmlu](kmmlu/README.md) | Knowledge-based multi-subject multiple choice questions for academic evaluation. | Korean | | [kobest](kobest/README.md) | A collection of tasks designed to evaluate understanding in Korean language. | Korean | | [kormedmcqa](kormedmcqa/README.md) | Medical question answering tasks in Korean to test specialized domain knowledge. | Korean | @@ -56,22 +63,27 @@ | [lambada_cloze](lambada_cloze/README.md) | Cloze-style LAMBADA dataset. | English | | [lambada_multilingual](lambada_multilingual/README.md) | Multilingual LAMBADA dataset. This is a legacy version of the multilingual dataset, and users should instead use `lambada_multilingual_stablelm`. | German, English, Spanish, French, Italian | | [lambada_multilingual_stablelm](lambada_multilingual_stablelm/README.md) | Multilingual LAMBADA dataset. Users should prefer evaluating on this version of the multilingual dataset instead of on `lambada_multilingual`. | German, English, Spanish, French, Italian, Dutch, Portuguese | +| [leaderboard](leaderboard/README.md) | Task group used by Hugging Face's [Open LLM Leaderboard v2](https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard). Those tasks are static and will not change through time | English | +| [lingoly](lingoly/README.md) | Challenging logical reasoning benchmark in low-resource languages with controls for memorization | English, Multilingual | | [logiqa](logiqa/README.md) | Logical reasoning tasks requiring advanced inference and deduction. | English, Chinese | | [logiqa2](logiqa2/README.md) | Large-scale logical reasoning dataset adapted from the Chinese Civil Service Examination. | English, Chinese | | [mathqa](mathqa/README.md) | Question answering tasks involving mathematical reasoning and problem-solving. | English | | [mc_taco](mc_taco/README.md) | Question-answer pairs that require temporal commonsense comprehension. | English | +| [med_concepts_qa](med_concepts_qa/README.md) | Benchmark for evaluating LLMs on their abilities to interpret medical codes and distinguish between medical concept. | English | | medmcqa | Medical multiple choice questions assessing detailed medical knowledge. | English | | medqa | Multiple choice question answering based on the United States Medical License Exams. | | | [mgsm](mgsm/README.md) | Benchmark of multilingual grade-school math problems. | Spanish, French, German, Russian, Chinese, Japanese, Thai, Swahili, Bengali, Telugu | | [minerva_math](minerva_math/README.md) | Mathematics-focused tasks requiring numerical reasoning and problem-solving skills. | English | -| mmlu | Massive Multitask Language Understanding benchmark for broad domain language evaluation. Several variants are supported. | English | +| [mmlu](mmlu/README.md) | Massive Multitask Language Understanding benchmark for broad domain language evaluation. Several variants are supported. | English | +| [mmlu_pro](mmlu_pro/README.md) | A refined set of MMLU, integrating more challenging, reasoning-focused questions and expanding the choice set from four to ten options. | English | +| [mmlusr](mmlusr/README.md) | Variation of MMLU designed to be more rigorous. | English | | model_written_evals | Evaluation tasks auto-generated for evaluating a collection of AI Safety concerns. | | | [mutual](mutual/README.md) | A retrieval-based dataset for multi-turn dialogue reasoning. | English | | [nq_open](nq_open/README.md) | Open domain question answering tasks based on the Natural Questions dataset. | English | | [okapi/arc_multilingual](okapi/arc_multilingual/README.md) | Tasks that involve reading comprehension and information retrieval challenges. | Multiple (31 languages) **Machine Translated.** | -| [okapi/hellaswag_multilingual](okapi/hellaswag_multilingual/README.md) | Tasks that involve reading comprehension and information retrieval challenges. | Multiple (30 languages) | -| okapi/mmlu_multilingual | Tasks that involve reading comprehension and information retrieval challenges. | Multiple (34 languages) | -| [okapi/truthfulqa_multilingual](okapi/truthfulqa_multilingual/README.md) | Tasks that involve reading comprehension and information retrieval challenges. | Multiple (31 languages) | +| [okapi/hellaswag_multilingual](okapi/hellaswag_multilingual/README.md) | Tasks that involve reading comprehension and information retrieval challenges. | Multiple (30 languages) **Machine Translated.** | +| okapi/mmlu_multilingual | Tasks that involve reading comprehension and information retrieval challenges. | Multiple (34 languages) **Machine Translated.** | +| [okapi/truthfulqa_multilingual](okapi/truthfulqa_multilingual/README.md) | Tasks that involve reading comprehension and information retrieval challenges. | Multiple (31 languages) **Machine Translated.** | | [openbookqa](openbookqa/README.md) | Open-book question answering tasks that require external knowledge and reasoning. | English | | [paloma](paloma/README.md) | Paloma is a comprehensive benchmark designed to evaluate open language models across a wide range of domains, ranging from niche artist communities to mental health forums on Reddit. | English | | [paws-x](paws-x/README.md) | Paraphrase Adversaries from Word Scrambling, focusing on cross-lingual capabilities. | English, French, Spanish, German, Chinese, Japanese, Korean | @@ -79,6 +91,7 @@ | [pile_10k](pile_10k/README.md) | The first 10K elements of The Pile, useful for debugging models trained on it. | English | | [piqa](piqa/README.md) | Physical Interaction Question Answering tasks to test physical commonsense reasoning. | English | | [polemo2](polemo2/README.md) | Sentiment analysis and emotion detection tasks based on Polish language data. | Polish | +| [portuguese_bench](portuguese_bench/README.md) | Collection of tasks in European Portuguese encompassing various evaluation areas. | Portuguese | | [prost](prost/README.md) | Tasks requiring understanding of professional standards and ethics in various domains. | English | | [pubmedqa](pubmedqa/README.md) | Question answering tasks based on PubMed research articles for biomedical understanding. | English | | [qa4mre](qa4mre/README.md) | Question Answering for Machine Reading Evaluation, assessing comprehension and reasoning. | English | @@ -88,6 +101,7 @@ | [sciq](sciq/README.md) | Science Question Answering tasks to assess understanding of scientific concepts. | English | | [scrolls](scrolls/README.md) | Tasks that involve long-form reading comprehension across various domains. | English | | [siqa](siqa/README.md) | Social Interaction Question Answering to evaluate common sense and social reasoning. | English | +| [spanish_bench](spanish_bench/README.md) | Collection of tasks in Spanish encompassing various evaluation areas. | Spanish | | [squad_completion](squad_completion/README.md) | A variant of the SQuAD question answering task designed for zero-shot evaluation of small LMs. | English | | [squadv2](squadv2/README.md) | Stanford Question Answering Dataset version 2, a reading comprehension benchmark. | English | | [storycloze](storycloze/README.md) | Tasks to predict story endings, focusing on narrative logic and coherence. | English | @@ -100,6 +114,7 @@ | [translation](translation/README.md) | Tasks focused on evaluating the language translation capabilities of models. | Arabic, English, Spanish, Basque, Hindi, Indonesian, Burmese, Russian, Swahili, Telugu, Chinese | | [triviaqa](triviaqa/README.md) | A large-scale dataset for trivia question answering to test general knowledge. | English | | [truthfulqa](truthfulqa/README.md) | A QA task aimed at evaluating the truthfulness and factual accuracy of model responses. | English | +| [turkishmmlu](turkishmmlu/README.md) | A multiple-choice QA test modeled after MMLU, written in Turkish based on Turkish high-school level exams. | Turkish | | [unitxt](unitxt/README.md) | A number of tasks implemented using the unitxt library for flexible, shareable, and reusable data preparation and evaluation for generative AI. | English | | [unscramble](unscramble/README.md) | Tasks involving the rearrangement of scrambled sentences to test syntactic understanding. | English | | [webqs](webqs/README.md) | Web-based question answering tasks designed to evaluate internet search and retrieval. | English | @@ -109,7 +124,8 @@ | [wmt2016](wmt2016/README.md) | Tasks from the WMT 2016 shared task, focusing on translation between multiple languages. | English, Czech, German, Finnish, Russian, Romanian, Turkish | | [wsc273](wsc273/README.md) | The Winograd Schema Challenge, a test of commonsense reasoning and coreference resolution. | English | | [xcopa](xcopa/README.md) | Cross-lingual Choice of Plausible Alternatives, testing reasoning in multiple languages. | Estonian, Haitian, Indonesian, Italian, Quechua, Swahili, Tamil, Thai, Turkish, Vietnamese, Chinese | -| [xnli](xnli/README.md) | Cross-Lingual Natural Language Inference to test understanding across different languages. | Arabic, Bulgarian, German, Greekm English, Spanish, French, Hindi, Russian, Swahili, Thai, Turkish, Urdu, Vietnamese, Chinese | +| [xnli](xnli/README.md) | Cross-Lingual Natural Language Inference to test understanding across different languages. | Arabic, Bulgarian, German, Greek, English, Spanish, French, Hindi, Russian, Swahili, Thai, Turkish, Urdu, Vietnamese, Chinese | | [xnli_eu](xnli_eu/README.md) | Cross-lingual Natural Language Inference tasks in Basque. | Basque | +| [xquad](xquad/README.md) | Cross-lingual Question Answering Dataset in multiple languages. | Arabic, German, Greek, English, Spanish, Hindi, Romanian, Russian, Thai, Turkish, Vietnamese, Chinese | | [xstorycloze](xstorycloze/README.md) | Cross-lingual narrative understanding tasks to predict story endings in multiple languages. | Russian, Simplified Chinese, Spanish, Arabic, Hindi, Indonesian, Telugu, Swahili, Basque, Burmese | | [xwinograd](xwinograd/README.md) | Cross-lingual Winograd schema tasks for coreference resolution in multiple languages. | English, French, Japanese, Portuguese, Russian, Chinese | diff --git a/lm_eval/tasks/__init__.py b/lm_eval/tasks/__init__.py index 1fd63cdb8110c8c86ad0a3a8c8de4e9b3403c242..edcd104dfaa87d17939a4365e5b5e102cfc3083b 100644 --- a/lm_eval/tasks/__init__.py +++ b/lm_eval/tasks/__init__.py @@ -1,11 +1,17 @@ import collections +import inspect import logging import os from functools import partial from typing import Dict, List, Mapping, Optional, Union from lm_eval import utils +from lm_eval.api.group import ConfigurableGroup, GroupConfig from lm_eval.api.task import ConfigurableTask, Task +from lm_eval.evaluator_utils import get_subtask_list + + +GROUP_ONLY_KEYS = list(GroupConfig().to_dict().keys()) class TaskManager: @@ -30,6 +36,20 @@ class TaskManager: ) self._all_tasks = sorted(list(self._task_index.keys())) + self._all_groups = sorted( + [x for x in self._all_tasks if self._task_index[x]["type"] == "group"] + ) + self._all_subtasks = sorted( + [ + x + for x in self._all_tasks + if self._task_index[x]["type"] in ["task", "python_task"] + ] + ) + self._all_tags = sorted( + [x for x in self._all_tasks if self._task_index[x]["type"] == "tag"] + ) + self.task_group_map = collections.defaultdict(list) def initialize_tasks( @@ -67,10 +87,88 @@ class TaskManager: def all_tasks(self): return self._all_tasks + @property + def all_groups(self): + return self._all_groups + + @property + def all_subtasks(self): + return self._all_subtasks + + @property + def all_tags(self): + return self._all_tags + @property def task_index(self): return self._task_index + def list_all_tasks( + self, list_groups=True, list_tags=True, list_subtasks=True + ) -> str: + from pytablewriter import MarkdownTableWriter + + def sanitize_path(path): + # don't print full path if we are within the lm_eval/tasks dir ! + # if we aren't though, provide the full path. + if "lm_eval/tasks/" in path: + return "lm_eval/tasks/" + path.split("lm_eval/tasks/")[-1] + else: + return path + + group_table = MarkdownTableWriter() + group_table.headers = ["Group", "Config Location"] + gt_values = [] + for g in self.all_groups: + path = self.task_index[g]["yaml_path"] + if path == -1: + path = "---" + else: + path = sanitize_path(path) + gt_values.append([g, path]) + group_table.value_matrix = gt_values + + tag_table = MarkdownTableWriter() + tag_table.headers = ["Tag"] + tag_table.value_matrix = [[t] for t in self.all_tags] + + subtask_table = MarkdownTableWriter() + subtask_table.headers = ["Task", "Config Location", "Output Type"] + st_values = [] + for t in self.all_subtasks: + path = self.task_index[t]["yaml_path"] + + output_type = "" + + # read the yaml file to determine the output type + if path != -1: + config = utils.load_yaml_config(path, mode="simple") + if "output_type" in config: + output_type = config["output_type"] + elif ( + "include" in config + ): # if no output type, check if there is an include with an output type + include_path = path.split("/")[:-1] + config["include"] + include_config = utils.load_yaml_config(include_path, mode="simple") + if "output_type" in include_config: + output_type = include_config["output_type"] + + if path == -1: + path = "---" + else: + path = sanitize_path(path) + st_values.append([t, path, output_type]) + subtask_table.value_matrix = st_values + + result = "\n" + if list_groups: + result += group_table.dumps() + "\n\n" + if list_tags: + result += tag_table.dumps() + "\n\n" + if list_subtasks: + result += subtask_table.dumps() + "\n\n" + return result + def match_tasks(self, task_list): return utils.pattern_match(task_list, self.all_tasks) @@ -80,7 +178,12 @@ class TaskManager: return False def _name_is_task(self, name) -> bool: - if self._name_is_registered(name) and ("task" in self.task_index[name]["type"]): + if self._name_is_registered(name) and (self.task_index[name]["type"] == "task"): + return True + return False + + def _name_is_tag(self, name) -> bool: + if self._name_is_registered(name) and (self.task_index[name]["type"] == "tag"): return True return False @@ -141,89 +244,126 @@ class TaskManager: config["group_alias"] = None return config + def _class_has_config_in_constructor(self, cls): + constructor = getattr(cls, "__init__", None) + return ( + "config" in inspect.signature(constructor).parameters + if constructor + else False + ) + def _load_individual_task_or_group( self, name_or_config: Optional[Union[str, dict]] = None, parent_name: Optional[str] = None, update_config: Optional[dict] = None, - yaml_path: Optional[str] = None, ) -> Mapping: - def load_task(config, task, group=None, yaml_path=None): + def _load_task(config, task): if "include" in config: - if yaml_path is None: - raise ValueError config = { **utils.load_yaml_config( - yaml_path, + yaml_path=None, yaml_config={"include": config.pop("include")}, mode="full", ), **config, } if self._config_is_python_task(config): - task_object = config["class"]() + if self._class_has_config_in_constructor(config["class"]): + task_object = config["class"](config=config) + else: + task_object = config["class"]() + if isinstance(task_object, ConfigurableTask): + # very scuffed: set task name here. TODO: fixme? + task_object.config.task = task else: - config = self._process_alias(config, group=group) task_object = ConfigurableTask(config=config) - if group is not None: - task_object = (group, task_object) + return {task: task_object} + def _get_group_and_subtask_from_config(config): + group_name = ConfigurableGroup(config=config) + subtask_list = [] + for task in group_name.config["task"]: + if isinstance(task, str) and self._name_is_tag(task): + subtask_list.extend(self._get_tasklist(task)) + else: + subtask_list.append(task) + return group_name, subtask_list + + def _process_group_config(config, update_config=None): + if update_config is not None: + config = {**config, **update_config} + _update_config = { + k: v for k, v in config.items() if k not in GROUP_ONLY_KEYS + } + if not bool(_update_config): + _update_config = None + + group_config = {k: v for k, v in config.items() if k in GROUP_ONLY_KEYS} + return group_config, _update_config + if isinstance(name_or_config, str): if update_config is not None: # Process name_or_config as a dict instead name_or_config = {"task": name_or_config, **update_config} - elif self._name_is_task(name_or_config): + elif self._name_is_task(name_or_config) or self._name_is_python_task( + name_or_config + ): task_config = self._get_config(name_or_config) - return load_task(task_config, task=name_or_config, group=parent_name) + return _load_task(task_config, task=name_or_config) else: - group_name = name_or_config subtask_list = self._get_tasklist(name_or_config) if subtask_list == -1: group_config = self._get_config(name_or_config) - subtask_list = group_config["task"] - - # This checks if we're at the root. - if parent_name is None: - group_config = self._get_config(name_or_config) - if set(group_config.keys()) > {"task", "group"}: - update_config = { - k: v - for k, v in group_config.items() - if k not in ["task", "group"] - } - yaml_path = self._get_yaml_path(group_name) - - if (update_config is not None) and ("group_alias" in update_config): - group_name = update_config["group_alias"] - update_config.pop("group_alias") + group_config, update_config = _process_group_config(group_config) + group_name, subtask_list = _get_group_and_subtask_from_config( + group_config + ) + else: + if self._name_is_tag(name_or_config): + fn = partial( + self._load_individual_task_or_group, + update_config=name_or_config + if isinstance(name_or_config, dict) + else None, + ) + return dict( + collections.ChainMap(*map(fn, reversed(subtask_list))) + ) + else: + group_name = ConfigurableGroup( + config={"group": name_or_config, "task": subtask_list} + ) if isinstance(name_or_config, dict): - if update_config is not None: - name_or_config = { - **name_or_config, - **update_config, - } - if self._config_is_task(name_or_config): - name = name_or_config["task"] + name = name_or_config.pop("task") + if update_config is not None: + name_or_config = {**name_or_config, **update_config} # If the name is registered as a group - # if self._name_is_task(name) is False: if self._name_is_group(name): - group_name = name - update_config = { - k: v for k, v in name_or_config.items() if k != "task" - } + group_config = self._get_config(name) + + group_config, update_config = _process_group_config( + group_config, name_or_config + ) + group_name, subtask_list = _get_group_and_subtask_from_config( + group_config + ) + elif self._name_is_tag(name): subtask_list = self._get_tasklist(name) - if subtask_list == -1: - subtask_list = self._get_config(name)["task"] + fn = partial( + self._load_individual_task_or_group, + update_config=name_or_config, + ) + return dict(collections.ChainMap(*map(fn, reversed(subtask_list)))) else: if self._name_is_registered(name): base_task_config = self._get_config(name) # Check if this is a duplicate. if parent_name is not None: - name_or_config["group"] = parent_name num_duplicate = len( list( filter( @@ -242,34 +382,21 @@ class TaskManager: } else: task_config = name_or_config - return load_task( - task_config, task=name, group=parent_name, yaml_path=yaml_path - ) + return _load_task(task_config, task=name) else: - group_name = name_or_config["group"] - subtask_list = name_or_config["task"] - if set(name_or_config.keys()) > {"task", "group"}: - update_config = { - k: v - for k, v in name_or_config.items() - if k not in ["task", "group"] - } - - all_subtasks = {} - if parent_name is not None: - all_subtasks = {group_name: (parent_name, None)} + group_config, update_config = _process_group_config(name_or_config) + group_name, subtask_list = _get_group_and_subtask_from_config( + group_config + ) fn = partial( self._load_individual_task_or_group, parent_name=group_name, update_config=update_config, - yaml_path=yaml_path, ) - all_subtasks = { - **all_subtasks, - **dict(collections.ChainMap(*map(fn, subtask_list))), + return { + group_name: dict(collections.ChainMap(*map(fn, reversed(subtask_list)))) } - return all_subtasks def load_task_or_group(self, task_list: Optional[Union[str, list]] = None) -> dict: """Loads a dictionary of task objects from a list @@ -293,10 +420,11 @@ class TaskManager: def _get_task_and_group(self, task_dir: str): """Creates a dictionary of tasks index with the following metadata, - - `type`, that can be either `task`, `python_task`, or `group`. + - `type`, that can be either `task`, `python_task`, `group` or `tags`. `task` refer to regular task configs, `python_task` are special yaml files that only consists of `task` and `class` parameters. - `group` are group configs. + `group` are group configs. `tags` are labels that can be assigned + to tasks to assist in sorting and calling tasks of certain themes. - `yaml_path`, path to the yaml file. If the entry is a `group` that was configured through a task config, the yaml_path will be -1 and all subtasks will be listed in `task` (see below) @@ -312,6 +440,32 @@ class TaskManager: :return Dictionary of task names as key and task metadata """ + + def _populate_tags_and_groups(config, task, tasks_and_groups, print_info): + # TODO: remove group in next release + if "tag" in config: + attr_list = config["tag"] + if isinstance(attr_list, str): + attr_list = [attr_list] + + for tag in attr_list: + if tag not in tasks_and_groups: + tasks_and_groups[tag] = { + "type": "tag", + "task": [task], + "yaml_path": -1, + } + elif tasks_and_groups[tag]["type"] != "tag": + self.logger.info( + f"The tag '{tag}' is already registered as a group, this tag will not be registered. " + "This may affect tasks you want to call." + ) + break + else: + tasks_and_groups[tag]["task"].append(task) + + # TODO: remove group in next release + print_info = True ignore_dirs = [ "__pycache__", ".ipynb_checkpoints", @@ -325,10 +479,14 @@ class TaskManager: config = utils.load_yaml_config(yaml_path, mode="simple") if self._config_is_python_task(config): # This is a python class config - tasks_and_groups[config["task"]] = { + task = config["task"] + tasks_and_groups[task] = { "type": "python_task", "yaml_path": yaml_path, } + _populate_tags_and_groups( + config, task, tasks_and_groups, print_info + ) elif self._config_is_group(config): # This is a group config tasks_and_groups[config["group"]] = { @@ -357,21 +515,9 @@ class TaskManager: "type": "task", "yaml_path": yaml_path, } - - if "group" in config: - groups = config["group"] - if isinstance(config["group"], str): - groups = [groups] - - for group in groups: - if group not in tasks_and_groups: - tasks_and_groups[group] = { - "type": "group", - "task": [task], - "yaml_path": -1, - } - else: - tasks_and_groups[group]["task"].append(task) + _populate_tags_and_groups( + config, task, tasks_and_groups, print_info + ) else: self.logger.debug(f"File {f} in {root} could not be loaded") @@ -400,6 +546,33 @@ def get_task_name_from_object(task_object): ) +def _check_duplicates(task_dict: dict) -> List[str]: + """helper function solely used in validating get_task_dict output. + Takes the output of lm_eval.evaluator_utils.get_subtask_list and + returns a list of all leaf subtasks contained within, and errors if any such leaf subtasks are + "oversubscribed" to several disjoint groups. + """ + subtask_names = [] + for key, value in task_dict.items(): + subtask_names.extend(value) + + duplicate_tasks = { + task_name for task_name in subtask_names if subtask_names.count(task_name) > 1 + } + + # locate the potentially problematic groups that seem to 'compete' for constituent subtasks + competing_groups = [ + group + for group in task_dict.keys() + if len(set(task_dict[group]).intersection(duplicate_tasks)) > 0 + ] + + if len(duplicate_tasks) > 0: + raise ValueError( + f"Found 1 or more tasks while trying to call get_task_dict() that were members of more than 1 called group: {list(duplicate_tasks)}. Offending groups: {competing_groups}. Please call groups which overlap their constituent tasks in separate evaluation runs." + ) + + def get_task_dict( task_name_list: Union[str, List[Union[str, Dict, Task]]], task_manager: Optional[TaskManager] = None, @@ -417,6 +590,7 @@ def get_task_dict( :return Dictionary of task objects """ + task_name_from_string_dict = {} task_name_from_config_dict = {} task_name_from_object_dict = {} @@ -463,8 +637,16 @@ def get_task_dict( ): raise ValueError - return { + final_task_dict = { **task_name_from_string_dict, **task_name_from_config_dict, **task_name_from_object_dict, } + + # behavior can get odd if one tries to invoke several groups that "compete" for the same task. + # (notably, because one could request several num_fewshot values at once in GroupConfig overrides for the subtask + # and we'd be unsure which to use and report.) + # we explicitly check and error in this case. + _check_duplicates(get_subtask_list(final_task_dict)) + + return final_task_dict diff --git a/lm_eval/tasks/aclue/README.md b/lm_eval/tasks/aclue/README.md index d8707c01bf555d54f14d825501f54428c33cbe89..6323ef7e3ed1902493d536bd3e6670848227b922 100644 --- a/lm_eval/tasks/aclue/README.md +++ b/lm_eval/tasks/aclue/README.md @@ -14,7 +14,7 @@ Homepage: https://github.com/isen-zhang/ACLUE ```bibtex @inproceedings{zhang-li-2023-large, - title = "Can Large Langauge Model Comprehend {A}ncient {C}hinese? A Preliminary Test on {ACLUE}", + title = "Can Large Language Model Comprehend {A}ncient {C}hinese? A Preliminary Test on {ACLUE}", author = "Zhang, Yixuan and Li, Haonan", booktitle = "Proceedings of the Ancient Language Processing Workshop", month = sep, @@ -26,7 +26,7 @@ Homepage: https://github.com/isen-zhang/ACLUE } ``` -### Groups and Tasks +### Groups, Tags, and Tasks #### Groups diff --git a/lm_eval/tasks/aclue/_aclue.yaml b/lm_eval/tasks/aclue/_aclue.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a2ae37ef5a6794a0db58005ea14f3943e56c87e3 --- /dev/null +++ b/lm_eval/tasks/aclue/_aclue.yaml @@ -0,0 +1,26 @@ +group: aclue +task: + - aclue_ancient_chinese_culture + - aclue_ancient_literature + - aclue_ancient_medical + - aclue_ancient_phonetics + - aclue_basic_ancient_chinese + - aclue_couplet_prediction + - aclue_homographic_character_resolution + - aclue_named_entity_recognition + - aclue_poetry_appreciate + - aclue_poetry_context_prediction + - aclue_poetry_quality_assessment + - aclue_poetry_sentiment_analysis + - aclue_polysemy_resolution + - aclue_reading_comprehension + - aclue_sentence_segmentation +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/aclue/_default_template_yaml b/lm_eval/tasks/aclue/_default_template_yaml index d93ce382d57e8a449868deb79ab544551f7e605b..9505197a72cef39b25bd5ef39d65c13bd97a89ea 100644 --- a/lm_eval/tasks/aclue/_default_template_yaml +++ b/lm_eval/tasks/aclue/_default_template_yaml @@ -1,4 +1,3 @@ -group: aclue dataset_path: tyouisen/aclue test_split: test fewshot_split: dev @@ -16,4 +15,4 @@ metric_list: aggregation: mean higher_is_better: true metadata: - version: 0.0 + version: 1.0 diff --git a/lm_eval/tasks/aexams/README.md b/lm_eval/tasks/aexams/README.md index 799c6c1ff90b9b38d71c92a30e787b073e139073..0c0461920c1cd51b6c3a4deb2af68843558116e1 100644 --- a/lm_eval/tasks/aexams/README.md +++ b/lm_eval/tasks/aexams/README.md @@ -24,11 +24,11 @@ Homepage for Arabic EXAMS: [EXAMS Arabic Homepage](https://github.com/FreedomInt ### Citation -### Groups and Tasks +### Groups, Tags, and Tasks #### Groups -- `EXAMS Arabic`: include IslamicStudies, Biology, Science, Physics, Social. +- `aexams`: Arabic EXAMS dataset, including IslamicStudies, Biology, Science, Physics, Social subjects. #### Tasks diff --git a/lm_eval/tasks/aexams/_aexams.yaml b/lm_eval/tasks/aexams/_aexams.yaml new file mode 100644 index 0000000000000000000000000000000000000000..59099b9c38c11e5391e031a2e07808a83d645938 --- /dev/null +++ b/lm_eval/tasks/aexams/_aexams.yaml @@ -0,0 +1,16 @@ +group: aexams +task: + - aexams_Biology + - aexams_IslamicStudies + - aexams_Physics + - aexams_Science + - aexams_Social +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/aexams/_default_template_yaml b/lm_eval/tasks/aexams/_default_template_yaml index 12438deb7f3ec3ffddf44ec85a4dc5150b9c6411..3f7100ad70190a67bd86675ce7a15d88a5a5976a 100644 --- a/lm_eval/tasks/aexams/_default_template_yaml +++ b/lm_eval/tasks/aexams/_default_template_yaml @@ -1,4 +1,3 @@ -group: aexams dataset_path: Hennara/aexams test_split: test fewshot_split: dev @@ -16,4 +15,4 @@ metric_list: aggregation: mean higher_is_better: true metadata: - version: 0.0 + version: 1.0 diff --git a/lm_eval/tasks/afrimgsm/README.md b/lm_eval/tasks/afrimgsm/README.md new file mode 100644 index 0000000000000000000000000000000000000000..cca14d968d2d87312d48fdb031e4a3518c9f915a --- /dev/null +++ b/lm_eval/tasks/afrimgsm/README.md @@ -0,0 +1,52 @@ +# MathQA + +### Paper + +IrokoBench: A New Benchmark for African Languages in the Age of Large Language Models +https://arxiv.org/pdf/2406.03368 + +IrokoBench is a human-translated benchmark dataset for 16 typologically diverse +low-resource African languages covering three tasks: natural language inference (AfriXNLI), +mathematical reasoning (AfriMGSM), and multi-choice knowledge-based QA (AfriMMLU). + + +### Citation + +``` +@misc{adelani2024irokobenchnewbenchmarkafrican, + title={IrokoBench: A New Benchmark for African Languages in the Age of Large Language Models}, + author={David Ifeoluwa Adelani and Jessica Ojo and Israel Abebe Azime and Jian Yun Zhuang and Jesujoba O. Alabi and Xuanli He and Millicent Ochieng and Sara Hooker and Andiswa Bukula and En-Shiun Annie Lee and Chiamaka Chukwuneke and Happy Buzaaba and Blessing Sibanda and Godson Kalipe and Jonathan Mukiibi and Salomon Kabongo and Foutse Yuehgoh and Mmasibidi Setaka and Lolwethu Ndolela and Nkiruka Odu and Rooweither Mabuya and Shamsuddeen Hassan Muhammad and Salomey Osei and Sokhar Samb and Tadesse Kebede Guge and Pontus Stenetorp}, + year={2024}, + eprint={2406.03368}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2406.03368}, +} +``` + +### Groups and Tasks + +#### Groups + +* `afrimgsm`: All afrimgsm tasks +* `afrimgsm_direct`: afrimgsm_direct evaluates models performance on the curated dataset +* `afrimgsm_en_cot`: afrimgsm_en_cot includes 5-shot of exemplars for chain-of-thought approach +* `afrimgsm_translate`: afrimgsm_translate evaluates models in translate-test setting + +#### Tasks +* `afrimgsm_direct_{language_code}`: each task evaluates for one language +* `afrimgsm_en_cot_{language_code}`: each task evaluates for one language +* `afrimgsm_translate_{language_code}`: each task evaluates for one language + +### Checklist + +For adding novel benchmarks/datasets to the library: +* [x] Is the task an existing benchmark in the literature? + * [x] Have you referenced the original paper that introduced the task? + * [ ] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + +If other tasks on this dataset are already supported: +* [x] Is the "Main" variant of this task clearly denoted? +* [x] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [x] Have you noted which, if any, published evaluation setups are matched by this variant? + * [x] Checked for equivalence with v0.3.0 LM Evaluation Harness diff --git a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_amh.yaml b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_amh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..04d0bdd67114f3c0887979fdce210f0fa94616e7 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_amh.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: amh +doc_to_target: '{% if answer is not none %}{{answer[15:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: direct_yaml +task: afrimgsm_direct_amh diff --git a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_eng.yaml b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5804270d4d0072764ca3d1190a75d7629bc251e9 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_eng.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: eng +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: direct_yaml +task: afrimgsm_direct_eng diff --git a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_ewe.yaml b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_ewe.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4eae6fc4c790968040080aee824c345bd786db44 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_ewe.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: ewe +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: direct_yaml +task: afrimgsm_direct_ewe diff --git a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_fra.yaml b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_fra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..16aeacf2c54706a18165bd1230ee812bb080ceb8 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_fra.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: fra +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: direct_yaml +task: afrimgsm_direct_fra diff --git a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_hau.yaml b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3a6668e989af297b60b1aafd53a3cb44e3936a60 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_hau.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: hau +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: direct_yaml +task: afrimgsm_direct_hau diff --git a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_ibo.yaml b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ab79986a5dec2af92711a675b3a4d79b31b044a9 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_ibo.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: direct_yaml +task: afrimgsm_direct_ibo diff --git a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_kin.yaml b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_kin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d4c9c75af0ccfc6d2b0b18138dec074e10b6047e --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_kin.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: kin +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: direct_yaml +task: afrimgsm_direct_kin diff --git a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_lin.yaml b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_lin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7136d7370cfd8f9e35b4ebc5e0615330b84edddc --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_lin.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: lin +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: direct_yaml +task: afrimgsm_direct_lin diff --git a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_lug.yaml b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_lug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..03fc0c2884cf9d14cadcf583cce1e81c47938963 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_lug.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: lug +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: direct_yaml +task: afrimgsm_direct_lug diff --git a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_orm.yaml b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_orm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..49d7e93390dc5c63ce83364ea1ec8ede77537ea8 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_orm.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: orm +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: direct_yaml +task: afrimgsm_direct_orm diff --git a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_sna.yaml b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_sna.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a61de85a3ffbbd5c2f3e91d5f26eb63a6241d78c --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_sna.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: sna +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: direct_yaml +task: afrimgsm_direct_sna diff --git a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_sot.yaml b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_sot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..455c1adcc5b896ce2c2140c9f30e8fa1857e60a2 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_sot.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: sot +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: direct_yaml +task: afrimgsm_direct_sot diff --git a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_swa.yaml b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..462ddfd378f8c02a872780a8013f0f74378551e0 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_swa.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: swa +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: direct_yaml +task: afrimgsm_direct_swa diff --git a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_twi.yaml b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_twi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8c4673b7ba00668d5d3bdcacfd2e00f342362194 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_twi.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: twi +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: direct_yaml +task: afrimgsm_direct_twi diff --git a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_wol.yaml b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_wol.yaml new file mode 100644 index 0000000000000000000000000000000000000000..08a8e030a4c0c0d444ac464b974d9886e434ff43 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_wol.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: wol +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: direct_yaml +task: afrimgsm_direct_wol diff --git a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_xho.yaml b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_xho.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2103d182f3ca1703c43e03279a6d1aa9bcc9532d --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_xho.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: xho +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: direct_yaml +task: afrimgsm_direct_xho diff --git a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_yor.yaml b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aa084c32a645cab532b002565f3c8a324708d6ba --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_yor.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: yor +doc_to_target: '{% if answer is not none %}{{answer[16:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: direct_yaml +task: afrimgsm_direct_yor diff --git a/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_zul.yaml b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_zul.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dcffb6944658282d620f7dbcec9d6513bcaf36c5 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/afrimgsm_direct_zul.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: zul +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: direct_yaml +task: afrimgsm_direct_zul diff --git a/lm_eval/tasks/afrimgsm/direct/direct_yaml b/lm_eval/tasks/afrimgsm/direct/direct_yaml new file mode 100644 index 0000000000000000000000000000000000000000..be97482c9c08c309f511689a03e8e9635e1d583c --- /dev/null +++ b/lm_eval/tasks/afrimgsm/direct/direct_yaml @@ -0,0 +1,37 @@ +# This file will be included in the generated language-specific task configs. +# It doesn't have a yaml file extension as it is not meant to be imported directly +# by the harness. +group: + - afrimgsm + - afrimgsm_direct +dataset_path: masakhane/afrimgsm +dataset_name: null # Overridden by language-specific config. +output_type: generate_until +# training_split: train +test_split: test +target_delimiter: "" +generation_kwargs: + until: + - "\n\n" + - "\n" + do_sample: false + temperature: 0.0 +filter_list: + - name: remove_whitespace + filter: + - function: remove_whitespace + - function: take_first + - filter: + - function: regex + group_select: -1 + regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+) + - function: take_first + name: flexible-extract +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 2.0 diff --git a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_amh.yaml b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_amh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f00400d96d15547bb73acd53c84ad5d4ce6f024f --- /dev/null +++ b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_amh.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: amh +doc_to_target: '{% if answer is not none %}{{answer[15:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: cot_yaml +task: afrimgsm_en_cot_amh diff --git a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_eng.yaml b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c62bf206a3ff5644c5d213ef394f4f0cbe3667d0 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_eng.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: eng +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: cot_yaml +task: afrimgsm_en_cot_eng diff --git a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_ewe.yaml b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_ewe.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ea246f7c16cec59da6562b0e17b43da0268caa0e --- /dev/null +++ b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_ewe.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: ewe +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: cot_yaml +task: afrimgsm_en_cot_ewe diff --git a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_fra.yaml b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_fra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..16bf57b76e4d48384ee909854ce7ac4050215894 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_fra.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: fra +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: cot_yaml +task: afrimgsm_en_cot_fra diff --git a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_hau.yaml b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2a397baf1e40185883569b53ffc9bb82265b4257 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_hau.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: hau +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: cot_yaml +task: afrimgsm_en_cot_hau diff --git a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_ibo.yaml b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9bd7bf62b4c9fed96aa01280c9d157a08cc04efb --- /dev/null +++ b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_ibo.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: cot_yaml +task: afrimgsm_en_cot_ibo diff --git a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_kin.yaml b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_kin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..841913b7c689a30833282cd40fdbc6a6db4a3dac --- /dev/null +++ b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_kin.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: kin +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: cot_yaml +task: afrimgsm_en_cot_kin diff --git a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_lin.yaml b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_lin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..76d7fdb91fb8dd39b23d4c8c5a0513eaa6538a6d --- /dev/null +++ b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_lin.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: lin +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: cot_yaml +task: afrimgsm_en_cot_lin diff --git a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_lug.yaml b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_lug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..84c05bb292fdec783de75f708002ad5e53c3e3fc --- /dev/null +++ b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_lug.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: lug +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: cot_yaml +task: afrimgsm_en_cot_lug diff --git a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_orm.yaml b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_orm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e9e5600e99104054e169ef1d29da528ef5a9be39 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_orm.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: orm +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: cot_yaml +task: afrimgsm_en_cot_orm diff --git a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_sna.yaml b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_sna.yaml new file mode 100644 index 0000000000000000000000000000000000000000..058689623d3fa6147743052f840ab25f8ef0bb4f --- /dev/null +++ b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_sna.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: sna +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: cot_yaml +task: afrimgsm_en_cot_sna diff --git a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_sot.yaml b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_sot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ae443f1833c3b248941bd0cdbae2e0a058625d4a --- /dev/null +++ b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_sot.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: sot +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: cot_yaml +task: afrimgsm_en_cot_sot diff --git a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_swa.yaml b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1aa2d07d0e132e0cf2787d75ab6e7281b4302f97 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_swa.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: swa +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: cot_yaml +task: afrimgsm_en_cot_swa diff --git a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_twi.yaml b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_twi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2957cb378e5ec6b27f0911eeab048aa91bf40e43 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_twi.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: twi +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: cot_yaml +task: afrimgsm_en_cot_twi diff --git a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_wol.yaml b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_wol.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6ecf4c44eff8d04d081a15062272ba168bab7ded --- /dev/null +++ b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_wol.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: wol +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: cot_yaml +task: afrimgsm_en_cot_wol diff --git a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_xho.yaml b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_xho.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9dc6691bdee31264bcba551b0288980de24b6e7f --- /dev/null +++ b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_xho.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: xho +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: cot_yaml +task: afrimgsm_en_cot_xho diff --git a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_yor.yaml b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8ef29830fa23b3fa561276bf6472a453c7e80384 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_yor.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: yor +doc_to_target: '{% if answer is not none %}{{answer[16:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: cot_yaml +task: afrimgsm_en_cot_yor diff --git a/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_zul.yaml b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_zul.yaml new file mode 100644 index 0000000000000000000000000000000000000000..24f486e0af03eda4a290eee0881da5a3b07dd96c --- /dev/null +++ b/lm_eval/tasks/afrimgsm/en_cot/afrimgsm_en_cot_zul.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: zul +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nStep-by-Step Answer:"}}{% else %}{{"Question: "+question+"\nStep-by-Step Answer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: cot_yaml +task: afrimgsm_en_cot_zul diff --git a/lm_eval/tasks/afrimgsm/en_cot/cot_yaml b/lm_eval/tasks/afrimgsm/en_cot/cot_yaml new file mode 100644 index 0000000000000000000000000000000000000000..7b320046526f51725a1b01cc407793a82223439b --- /dev/null +++ b/lm_eval/tasks/afrimgsm/en_cot/cot_yaml @@ -0,0 +1,37 @@ +# This file will be included in the generated language-specific task configs. +# It doesn't have a yaml file extension as it is not meant to be imported directly by the harness. +group: + - afrimgsm + - afrimgsm_en_cot +dataset_path: masakhane/afrimgsm +dataset_name: null # Overridden by language-specific config. +output_type: generate_until +training_split: train +test_split: test +generation_kwargs: + until: + - "\n\n" + - "\n" + do_sample: false + temperature: 0.0 +target_delimiter: " " +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +filter_list: + - name: "strict-match" + filter: + - function: "regex" + regex_pattern: "The answer is (\\-?[0-9\\.\\,]+)" + - function: "take_first" + - filter: + - function: regex + group_select: -1 + regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+) + - function: take_first + name: flexible-extract +metadata: + version: 2.0 diff --git a/lm_eval/tasks/afrimgsm/gen_yaml.sh b/lm_eval/tasks/afrimgsm/gen_yaml.sh new file mode 100644 index 0000000000000000000000000000000000000000..5c0132822a7f3ba68230762e0342838583c29bd9 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/gen_yaml.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# python utils.py --overwrite --output-dir direct --mode direct +# python utils.py --overwrite --output-dir direct_native --mode direct-native +# python utils.py --overwrite --output-dir en_cot --mode en-cot +# python utils.py --overwrite --output-dir native_cot --mode native-cot +python utils.py --overwrite --output-dir translate_direct --mode translate-direct diff --git a/lm_eval/tasks/afrimgsm/run.sh b/lm_eval/tasks/afrimgsm/run.sh new file mode 100644 index 0000000000000000000000000000000000000000..075500be33775dc49288ce7f7180604c7c6f99ce --- /dev/null +++ b/lm_eval/tasks/afrimgsm/run.sh @@ -0,0 +1,6 @@ +lm_eval --model hf \ + --model_args pretrained="google/gemma-7b" --tasks afrimgsm_en_cot_eng,mgsm_en_cot_en,afrimgsm_native_cot_eng,mgsm_native_cot_en,afrimgsm_direct_eng,mgsm_direct_en,afrimgsm_direct_native_eng \ + --device cuda:0 \ + --batch_size 1 \ + --verbosity DEBUG \ + --limit 5 diff --git a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_amh.yaml b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_amh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..55fbe4bfdb590b6d352b71c16eebefef3cbb3399 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_amh.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: amh +doc_to_target: '{% if answer is not none %}{{answer[15:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: translate_direct_yaml +task: afrimgsm_translate_direct_amh diff --git a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_eng.yaml b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1d729a5cab74ddeb5b3e03f97eadef54a5be3a3c --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_eng.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: eng +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: translate_direct_yaml +task: afrimgsm_translate_direct_eng diff --git a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_ewe.yaml b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_ewe.yaml new file mode 100644 index 0000000000000000000000000000000000000000..26191dc815bc0747c05af177e38662e4c4581bfb --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_ewe.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: ewe +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: translate_direct_yaml +task: afrimgsm_translate_direct_ewe diff --git a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_fra.yaml b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_fra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9f0331ee8f3f730372c3eaecb0defe0887bd6502 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_fra.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: fra +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: translate_direct_yaml +task: afrimgsm_translate_direct_fra diff --git a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_hau.yaml b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..850dad6351a693c2a738a0a570e15da8b412a63a --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_hau.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: hau +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: translate_direct_yaml +task: afrimgsm_translate_direct_hau diff --git a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_ibo.yaml b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8b81178cc719c44419e24b5e14fc5c3e61b73a7a --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_ibo.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: translate_direct_yaml +task: afrimgsm_translate_direct_ibo diff --git a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_kin.yaml b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_kin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5a8f53e2e7e7449b1db465062bfb8524b94d3c85 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_kin.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: kin +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: translate_direct_yaml +task: afrimgsm_translate_direct_kin diff --git a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_lin.yaml b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_lin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..58044ee2b887d3a83f9004e303da6c2bc048703f --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_lin.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: lin +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: translate_direct_yaml +task: afrimgsm_translate_direct_lin diff --git a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_lug.yaml b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_lug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..87013c146f2ef8bddee0a82c2c21949bcac549b0 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_lug.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: lug +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: translate_direct_yaml +task: afrimgsm_translate_direct_lug diff --git a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_orm.yaml b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_orm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1dd19325a57022df444f04eba5eb1b3ced117b61 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_orm.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: orm +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: translate_direct_yaml +task: afrimgsm_translate_direct_orm diff --git a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_sna.yaml b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_sna.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d710b1da339ca0012239993417f83c946a7c3e09 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_sna.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: sna +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: translate_direct_yaml +task: afrimgsm_translate_direct_sna diff --git a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_sot.yaml b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_sot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..643eaaeef10a1f70b3b7f13b58cb606dd6ae3f73 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_sot.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: sot +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: translate_direct_yaml +task: afrimgsm_translate_direct_sot diff --git a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_swa.yaml b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b882e89c24a75ce06a1790791a084e1c087acc1b --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_swa.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: swa +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: translate_direct_yaml +task: afrimgsm_translate_direct_swa diff --git a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_twi.yaml b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_twi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ac946eb7f413d227dfe0fc5b770e0c6c7bc2d159 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_twi.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: twi +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: translate_direct_yaml +task: afrimgsm_translate_direct_twi diff --git a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_wol.yaml b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_wol.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dbcc6b2e0e553ebe5353abaebbf6030d68c5b024 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_wol.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: wol +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: translate_direct_yaml +task: afrimgsm_translate_direct_wol diff --git a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_xho.yaml b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_xho.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dfb3d74f40fac640988e1ffba3caf007d56b66ec --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_xho.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: xho +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: translate_direct_yaml +task: afrimgsm_translate_direct_xho diff --git a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_yor.yaml b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6b4c346ffeeacd42de58efab206db84af0168670 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_yor.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: yor +doc_to_target: '{% if answer is not none %}{{answer[16:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: translate_direct_yaml +task: afrimgsm_translate_direct_yor diff --git a/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_zul.yaml b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_zul.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5e79edffadafebb8e31c710e854157046d15b10e --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/afrimgsm_translate_zul.yaml @@ -0,0 +1,12 @@ +# Generated by utils.py +dataset_name: zul +doc_to_target: '{% if answer is not none %}{{answer[21:]}}{% else %}{{answer_number|string}}{% endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nAnswer:"}}{% else %}{{"Question: "+question+"\nAnswer:"}}{% endif %}' +generation_kwargs: + do_sample: false + until: + - 'Question:' + - + - <|im_end|> +include: translate_direct_yaml +task: afrimgsm_translate_direct_zul diff --git a/lm_eval/tasks/afrimgsm/translate/translate_direct_yaml b/lm_eval/tasks/afrimgsm/translate/translate_direct_yaml new file mode 100644 index 0000000000000000000000000000000000000000..1e54d3ea43f33b037d1f28389a0c3d30c6589906 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/translate/translate_direct_yaml @@ -0,0 +1,36 @@ +# This file will be included in the generated language-specific task configs. +# It doesn't have a yaml file extension as it is not meant to be imported directly +# by the harness. +group: + - afrimgsm + - afrimgsm_translate +dataset_path: masakhane/afrimgsm-translate-test +dataset_name: null # Overridden by language-specific config. +output_type: generate_until +test_split: test +generation_kwargs: + until: + - "\n\n" + - "\n" + do_sample: false + temperature: 0.0 +target_delimiter: " " +filter_list: + - name: remove_whitespace + filter: + - function: remove_whitespace + - function: take_first + - filter: + - function: regex + group_select: -1 + regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+) + - function: take_first + name: flexible-extract +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 2.0 diff --git a/lm_eval/tasks/afrimgsm/utils.py b/lm_eval/tasks/afrimgsm/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..0dd336f8b3cbeb85d5854beb923578f215f91632 --- /dev/null +++ b/lm_eval/tasks/afrimgsm/utils.py @@ -0,0 +1,219 @@ +import argparse + +import yaml + + +languages = [ + "eng", + "amh", + "ibo", + "fra", + "sna", + "lin", + "wol", + "ewe", + "lug", + "xho", + "kin", + "twi", + "zul", + "orm", + "yor", + "hau", + "sot", + "swa", +] + +languages_REGEX = { + "eng": "The answer is (\\-?[0-9\\.\\,]+)", + "amh": "መልሱ (\\-?[0-9\\.\\,]+)", + "ibo": "Azịza ya bụ (\\-?[0-9\\.\\,]+)", + "fra": "La réponse est(\\-?[0-9\\.\\,]+)", + "sna": "Mhinduro kumubvunzo ndi (\\-?[0-9\\.\\,]+)", + "lin": "Eyano ezali (\\-?[0-9\\.\\,]+)", + "wol": "Tontu li (\\-?[0-9\\.\\,]+)", + "ewe": "ŋuɖoɖoae nye (\\-?[0-9\\.\\,]+)", + "lug": "Ansa eri (\\-?[0-9\\.\\,]+)", + "xho": "Impendulo ngu (\\-?[0-9\\.\\,]+)", + "kin": "Igisubizo ni (\\-?[0-9\\.\\,]+)", + "twi": "Ne nnyiano yɛ (\\-?[0-9\\.\\,]+)", + "zul": "Impendulo ithi (\\-?[0-9\\.\\,]+)", + "orm": "Deebiin isaa (\\-?[0-9\\.\\,]+)", + "yor": "Ìdáhùn náà ni (\\-?[0-9\\.\\,]+)", + "hau": "Amsar ita ce (\\-?[0-9\\.\\,]+)", + "sot": "Karabo ke (\\-?[0-9\\.\\,]+)", + "swa": "Jibu ni (\\-?[0-9\\.\\,]+)", +} + +LANGUAGES = {} + +for lang in languages: + if lang == "amh": + LANGUAGES[lang] = { # English + "QUESTION": "ጥያቄ:", + "ANSWER": "በቅደም ተከተል መልስ:", + "DIRECT": "Answer:", + "REGEX": languages_REGEX[lang], + } + elif lang == "yor": + LANGUAGES[lang] = { # English + "QUESTION": "Ìbéèrè:", + "ANSWER": "Ìdáhùn lẹ́sẹsẹ:", + "DIRECT": "Answer:", + "REGEX": languages_REGEX[lang], + } + + else: + LANGUAGES[lang] = { # English + "QUESTION": "Question:", + "ANSWER": "Step-by-Step Answer:", + "DIRECT": "Answer:", + "REGEX": languages_REGEX[lang], + } + + +def add_regex_pattern(regex_pattern): + if regex_pattern is None: + return {} + return { + "filter_list": [ + { + "name": "strict-match", + "filter": [ + { + "function": "regex", + "regex_pattern": f"""{regex_pattern}""", + }, + { + "function": "take_first", + }, + ], + }, + { + "name": "flexible-extract", + "filter": [ + { + "function": "regex", + "regex_pattern": """(-?[$0-9.,]{2,})|(-?[0-9]+)""", + "group_select": -1, + }, + { + "function": "take_first", + }, + ], + }, + ], + } + + +def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None: + """ + Generate a yaml file for each language. + + :param output_dir: The directory to output the files to. + :param overwrite: Whether to overwrite files if they already exist. + """ + err = [] + for lang in LANGUAGES.keys(): + try: + yaml_template = "cot_yaml" + filter_list = {} + DELIMITER = None + if mode == "direct": + ANSWER = LANGUAGES["eng"]["DIRECT"] + QUESTION = LANGUAGES["eng"]["QUESTION"] + REGEX = None + task_name = f"afrimgsm_direct_{lang}" + yaml_template = "direct_yaml" + if mode == "direct-native": + ANSWER = LANGUAGES[lang]["DIRECT"] + QUESTION = LANGUAGES[lang]["QUESTION"] + REGEX = None + task_name = f"afrimgsm_direct_native_{lang}" + yaml_template = "direct_native_yaml" + elif mode == "native-cot": + ANSWER = LANGUAGES[lang]["ANSWER"] + REGEX = LANGUAGES[lang]["REGEX"] + QUESTION = LANGUAGES[lang]["QUESTION"] + task_name = f"afrimgsm_native_cot_{lang}" + filter_list = add_regex_pattern(REGEX) + DELIMITER = "" if lang in ["zh", "ja"] else None + elif mode == "en-cot": + ANSWER = LANGUAGES["eng"]["ANSWER"] + REGEX = LANGUAGES["eng"]["REGEX"] + QUESTION = LANGUAGES["eng"]["QUESTION"] + task_name = f"afrimgsm_en_cot_{lang}" + elif mode == "translate-direct": + ANSWER = LANGUAGES["eng"]["DIRECT"] + QUESTION = LANGUAGES["eng"]["QUESTION"] + REGEX = None + task_name = f"afrimgsm_translate_direct_{lang}" + yaml_template = "translate_direct_yaml" + + file_name = f"{task_name}.yaml" + ANSWER_TO_SKIP = len(LANGUAGES[lang]["ANSWER"]) + 1 + with open( + f"{output_dir}/{file_name}", "w" if overwrite else "x", encoding="utf8" + ) as f: + f.write("# Generated by utils.py\n") + yaml.dump( + { + "include": yaml_template, + "dataset_name": lang, + "task": f"{task_name}", + "doc_to_text": f"""{{% if answer is not none %}}""" + f"""{{{{question+"\\n{ANSWER}"}}}}""" + f"""{{% else %}}""" + f"""{{{{"{QUESTION} "+question+"\\n{ANSWER}"}}}}""" + f"""{{% endif %}}""", + "doc_to_target": f"""{{% if answer is not none %}}""" + f"""{{{{answer[{ANSWER_TO_SKIP}:]}}}}""" + f"""{{% else %}}""" + f"""{{{{answer_number|string}}}}""" + f"""{{% endif %}}""", + **filter_list, + "generation_kwargs": { + "until": [QUESTION, "", "<|im_end|>"], + "do_sample": False, + }, + **({"target_delimiter": DELIMITER} if DELIMITER else {}), + }, + f, + allow_unicode=True, + width=float("inf"), + ) + except FileExistsError: + err.append(file_name) + + if len(err) > 0: + raise FileExistsError( + "Files were not created because they already exist (use --overwrite flag):" + f" {', '.join(err)}" + ) + + +def main() -> None: + """Parse CLI args and generate language-specific yaml files.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--overwrite", + default=False, + action="store_true", + help="Overwrite files if they already exist", + ) + parser.add_argument( + "--output-dir", default=".", help="Directory to write yaml files to" + ) + parser.add_argument( + "--mode", + default="native-cot", + choices=["direct", "direct-native", "native-cot", "en-cot", "translate-direct"], + help="Mode of chain-of-thought", + ) + args = parser.parse_args() + + gen_lang_yamls(output_dir=args.output_dir, overwrite=args.overwrite, mode=args.mode) + + +if __name__ == "__main__": + main() diff --git a/lm_eval/tasks/afrimmlu/README.md b/lm_eval/tasks/afrimmlu/README.md new file mode 100644 index 0000000000000000000000000000000000000000..f7f7ed4d82f04224440a0d164d2cc24c0e758990 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/README.md @@ -0,0 +1,50 @@ +# MathQA + +### Paper + +IrokoBench: A New Benchmark for African Languages in the Age of Large Language Models +https://arxiv.org/pdf/2406.03368 + +IrokoBench is a human-translated benchmark dataset for 16 typologically diverse +low-resource African languages covering three tasks: natural language inference (AfriXNLI), +mathematical reasoning (AfriMGSM), and multi-choice knowledge-based QA (AfriMMLU). + + +### Citation + +``` +@misc{adelani2024irokobenchnewbenchmarkafrican, + title={IrokoBench: A New Benchmark for African Languages in the Age of Large Language Models}, + author={David Ifeoluwa Adelani and Jessica Ojo and Israel Abebe Azime and Jian Yun Zhuang and Jesujoba O. Alabi and Xuanli He and Millicent Ochieng and Sara Hooker and Andiswa Bukula and En-Shiun Annie Lee and Chiamaka Chukwuneke and Happy Buzaaba and Blessing Sibanda and Godson Kalipe and Jonathan Mukiibi and Salomon Kabongo and Foutse Yuehgoh and Mmasibidi Setaka and Lolwethu Ndolela and Nkiruka Odu and Rooweither Mabuya and Shamsuddeen Hassan Muhammad and Salomey Osei and Sokhar Samb and Tadesse Kebede Guge and Pontus Stenetorp}, + year={2024}, + eprint={2406.03368}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2406.03368}, +} +``` + +### Groups and Tasks + +#### Groups + +* `afrimmlu`: All afrimmlu tasks +* `afrimmlu_direct`: afrimmlu_direct evaluates models performance on the curated dataset +* `afrimmlu_translate`: afrimmlu_translate evaluates models in translate-test setting + +#### Tasks +* `afrimmlu_direct_{language_code}`: each task evaluates for one language +* `afrimmlu_translate_{language_code}`: each task evaluates for one language + +### Checklist + +For adding novel benchmarks/datasets to the library: +* [x] Is the task an existing benchmark in the literature? + * [x] Have you referenced the original paper that introduced the task? + * [ ] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + +If other tasks on this dataset are already supported: +* [x] Is the "Main" variant of this task clearly denoted? +* [x] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [x] Have you noted which, if any, published evaluation setups are matched by this variant? + * [x] Checked for equivalence with v0.3.0 LM Evaluation Harness diff --git a/lm_eval/tasks/afrimmlu/direct/afrimmlu_common_yaml b/lm_eval/tasks/afrimmlu/direct/afrimmlu_common_yaml new file mode 100644 index 0000000000000000000000000000000000000000..2cda741a7e757bd28010b916e20d0c9ee11fc989 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/afrimmlu_common_yaml @@ -0,0 +1,37 @@ +group: + - afrimmlu + - afrimmlu_direct +task: null +dataset_path: masakhane/afrimmlu +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_text: !function utils.doc_to_text +doc_to_target: "{{['A', 'B', 'C', 'D'].index(answer)}}" +doc_to_choice: !function utils.doc_to_choice +should_decontaminate: true +doc_to_decontamination_query: "Question: {{question}}\nAnswer:" +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_amh.yaml b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_amh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aa60c668fd9b2879f020f990655e7eedce2b3a81 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_amh.yaml @@ -0,0 +1,3 @@ +dataset_name: amh +include: afrimmlu_common_yaml +task: afrimmlu_direct_amh diff --git a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_eng.yaml b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a1e647cdf1d0278c73744288fa61cd7709550231 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_eng.yaml @@ -0,0 +1,3 @@ +dataset_name: eng +include: afrimmlu_common_yaml +task: afrimmlu_direct_eng diff --git a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_ewe.yaml b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_ewe.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1cc45ddc0e50d1bb4992aecdb4f5208dbb77881b --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_ewe.yaml @@ -0,0 +1,3 @@ +dataset_name: ewe +include: afrimmlu_common_yaml +task: afrimmlu_direct_ewe diff --git a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_fra.yaml b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_fra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e6adb6c8aa4e50c6efca737792907cb658c30627 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_fra.yaml @@ -0,0 +1,3 @@ +dataset_name: fra +include: afrimmlu_common_yaml +task: afrimmlu_direct_fra diff --git a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_hau.yaml b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9cc9a1ae7acc7318faf68a241f68b0d5cba93978 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_hau.yaml @@ -0,0 +1,3 @@ +dataset_name: hau +include: afrimmlu_common_yaml +task: afrimmlu_direct_hau diff --git a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_ibo.yaml b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6abb2c4a467986751376679b31ec5db8a7af0886 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_ibo.yaml @@ -0,0 +1,3 @@ +dataset_name: ibo +include: afrimmlu_common_yaml +task: afrimmlu_direct_ibo diff --git a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_kin.yaml b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_kin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2f81f709c4812db3ecfa71bbb9cfb74099a10aab --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_kin.yaml @@ -0,0 +1,3 @@ +dataset_name: kin +include: afrimmlu_common_yaml +task: afrimmlu_direct_kin diff --git a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_lin.yaml b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_lin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..55363ed93772284fc54386592ae827c03246d681 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_lin.yaml @@ -0,0 +1,3 @@ +dataset_name: lin +include: afrimmlu_common_yaml +task: afrimmlu_direct_lin diff --git a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_lug.yaml b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_lug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0d484427eda8fcd4b645b3f90b191f075cb88ce9 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_lug.yaml @@ -0,0 +1,3 @@ +dataset_name: lug +include: afrimmlu_common_yaml +task: afrimmlu_direct_lug diff --git a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_orm.yaml b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_orm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..763eb8a75f894797185436d3a83c9fd57393f4ac --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_orm.yaml @@ -0,0 +1,3 @@ +dataset_name: orm +include: afrimmlu_common_yaml +task: afrimmlu_direct_orm diff --git a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_sna.yaml b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_sna.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ed9e69af392838290bac14d08259585c56daace8 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_sna.yaml @@ -0,0 +1,3 @@ +dataset_name: sna +include: afrimmlu_common_yaml +task: afrimmlu_direct_sna diff --git a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_sot.yaml b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_sot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..acdba0fdccf12f73004669dbed1b7cbee9ded24f --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_sot.yaml @@ -0,0 +1,3 @@ +dataset_name: sot +include: afrimmlu_common_yaml +task: afrimmlu_direct_sot diff --git a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_swa.yaml b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c1aa82b0b1d44314c337b904c346806cb3c720a4 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_swa.yaml @@ -0,0 +1,3 @@ +dataset_name: swa +include: afrimmlu_common_yaml +task: afrimmlu_direct_swa diff --git a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_twi.yaml b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_twi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2695d4a156d4b59dbb2c483ebdbbc16e01c7a415 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_twi.yaml @@ -0,0 +1,3 @@ +dataset_name: twi +include: afrimmlu_common_yaml +task: afrimmlu_direct_twi diff --git a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_wol.yaml b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_wol.yaml new file mode 100644 index 0000000000000000000000000000000000000000..027f837637fb061d227d33e925d3030af51c3cbe --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_wol.yaml @@ -0,0 +1,3 @@ +dataset_name: wol +include: afrimmlu_common_yaml +task: afrimmlu_direct_wol diff --git a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_xho.yaml b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_xho.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8e0c12972d01be342a6838b0eab4c1f609d6dc48 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_xho.yaml @@ -0,0 +1,3 @@ +dataset_name: xho +include: afrimmlu_common_yaml +task: afrimmlu_direct_xho diff --git a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_yor.yaml b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2a9f7645c2259a607f871e54b07c14ab962ed04c --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_yor.yaml @@ -0,0 +1,3 @@ +dataset_name: yor +include: afrimmlu_common_yaml +task: afrimmlu_direct_yor diff --git a/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_zul.yaml b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_zul.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9d8d3b415b44ef4ab0b762f411006c7b00d54226 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/afrimmlu_direct_zul.yaml @@ -0,0 +1,3 @@ +dataset_name: zul +include: afrimmlu_common_yaml +task: afrimmlu_direct_zul diff --git a/lm_eval/tasks/afrimmlu/direct/utils.py b/lm_eval/tasks/afrimmlu/direct/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..f1bb9162f0fbc68807db68134970ae2636980cbf --- /dev/null +++ b/lm_eval/tasks/afrimmlu/direct/utils.py @@ -0,0 +1,32 @@ +from lm_eval.utils import weighted_f1_score + + +def doc_to_choice(doc): + choices = eval(doc["choices"]) + return choices + + +def doc_to_text(doc): + output = """You are a highly knowledgeable and intelligent artificial intelligence + model answers multiple-choice questions about {subject} + + Question: {question} + + Choices: + A: {choice1} + B: {choice2} + C: {choice3} + D: {choice4} + + Answer: """ + + choices = eval(doc["choices"]) + text = output.format( + subject=doc["subject"], + question=doc["question"], + choice1=choices[0], + choice2=choices[1], + choice3=choices[2], + choice4=choices[3], + ) + return text diff --git a/lm_eval/tasks/afrimmlu/fewshot.sh b/lm_eval/tasks/afrimmlu/fewshot.sh new file mode 100644 index 0000000000000000000000000000000000000000..c69c48d7dff4e2495485023187dc162742c7ca6a --- /dev/null +++ b/lm_eval/tasks/afrimmlu/fewshot.sh @@ -0,0 +1,8 @@ +lm_eval --model hf \ + --model_args pretrained=masakhane/African-ultrachat-alpaca \ + --tasks afrimmlu_direct_amh,afrimmlu_direct_eng,afrimmlu_direct_ewe,afrimmlu_direct_fra,afrimmlu_direct_hau,afrimmlu_direct_ibo,afrimmlu_direct_kin,afrimmlu_direct_lin,afrimmlu_direct_lug,afrimmlu_direct_orm,afrimmlu_direct_sna,afrimmlu_direct_sot,afrimmlu_direct_twi,afrimmlu_direct_wol,afrimmlu_direct_xho,afrimmlu_direct_yor,afrimmlu_direct_zul \ + --device cuda:0 \ + --batch_size 1 \ + --num_fewshot 0 \ + --verbosity DEBUG \ + --wandb_args project=afrimmlu diff --git a/lm_eval/tasks/afrimmlu/translate/afrimmlu_common_translate_yaml b/lm_eval/tasks/afrimmlu/translate/afrimmlu_common_translate_yaml new file mode 100644 index 0000000000000000000000000000000000000000..515c007228583bc0db97fa4db6f40d0a5c7176fd --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/afrimmlu_common_translate_yaml @@ -0,0 +1,34 @@ +group: + - afrimmlu_translate +task: null +dataset_path: masakhane/afrimmlu-translate-test +dataset_name: null +output_type: multiple_choice +test_split: test +doc_to_text: !function utils.doc_to_text +doc_to_target: "{{['A', 'B', 'C', 'D'].index(answer)}}" +doc_to_choice: !function utils.doc_to_choice +should_decontaminate: true +doc_to_decontamination_query: "Question: {{question}}\nAnswer:" +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + # aggregation: mean + average: weighted + hf_evaluate: true + higher_is_better: True + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - "," + - "\\$" +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_amh.yaml b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_amh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ac88ffa9500701e8bbb2b5c64d1f4c9f2ec856bc --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_amh.yaml @@ -0,0 +1,3 @@ +dataset_name: amh +include: afrimmlu_common_translate_yaml +task: afrimmlu_translate_amh diff --git a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_eng.yaml b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0be98beedd86223dd14c1abbf51dbe93c7ff658a --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_eng.yaml @@ -0,0 +1,3 @@ +dataset_name: eng +include: afrimmlu_common_translate_yaml +task: afrimmlu_translate_eng diff --git a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_ewe.yaml b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_ewe.yaml new file mode 100644 index 0000000000000000000000000000000000000000..624342b91f383479c7ef340bfb80ce305608cf61 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_ewe.yaml @@ -0,0 +1,3 @@ +dataset_name: ewe +include: afrimmlu_common_translate_yaml +task: afrimmlu_translate_ewe diff --git a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_fra.yaml b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_fra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c4fd7e1fc774b6dd987e6c35d3a3fadbf6d577c4 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_fra.yaml @@ -0,0 +1,3 @@ +dataset_name: fra +include: afrimmlu_common_translate_yaml +task: afrimmlu_translate_fra diff --git a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_hau.yaml b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aaeb415fa2a00516ea3a84133066b7eae009f017 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_hau.yaml @@ -0,0 +1,3 @@ +dataset_name: hau +include: afrimmlu_common_translate_yaml +task: afrimmlu_translate_hau diff --git a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_ibo.yaml b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..93fb24e8c3fa799a41c022a708748bb5e7341631 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_ibo.yaml @@ -0,0 +1,3 @@ +dataset_name: ibo +include: afrimmlu_common_translate_yaml +task: afrimmlu_translate_ibo diff --git a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_kin.yaml b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_kin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f39f666840626dcf6ea61a196be702ec1c3e3308 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_kin.yaml @@ -0,0 +1,3 @@ +dataset_name: kin +include: afrimmlu_common_translate_yaml +task: afrimmlu_translate_kin diff --git a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_lin.yaml b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_lin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c935ee47382973e3dbe833987ea083bd3023b5cd --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_lin.yaml @@ -0,0 +1,3 @@ +dataset_name: lin +include: afrimmlu_common_translate_yaml +task: afrimmlu_translate_lin diff --git a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_lug.yaml b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_lug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..72e4bce0113c8473eabf68a7d2e43ba2eabc965c --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_lug.yaml @@ -0,0 +1,3 @@ +dataset_name: lug +include: afrimmlu_common_translate_yaml +task: afrimmlu_translate_lug diff --git a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_orm.yaml b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_orm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3ff902499480d35576cb84453406a5d484349816 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_orm.yaml @@ -0,0 +1,3 @@ +dataset_name: orm +include: afrimmlu_common_translate_yaml +task: afrimmlu_translate_orm diff --git a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_sna.yaml b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_sna.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9979740a9bf6194d9a9c4db0f0b4845312f1aed7 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_sna.yaml @@ -0,0 +1,3 @@ +dataset_name: sna +include: afrimmlu_common_translate_yaml +task: afrimmlu_translate_sna diff --git a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_sot.yaml b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_sot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..deb2b9b81d544140bfa7e720d0b544089b39bfcd --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_sot.yaml @@ -0,0 +1,3 @@ +dataset_name: sot +include: afrimmlu_common_translate_yaml +task: afrimmlu_translate_sot diff --git a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_swa.yaml b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e58d90bc69357a3b9c166e8f29000894daa8b108 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_swa.yaml @@ -0,0 +1,3 @@ +dataset_name: swa +include: afrimmlu_common_translate_yaml +task: afrimmlu_translate_swa diff --git a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_twi.yaml b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_twi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..51a2d26ae0563acda4972b272de4c0d6de81146f --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_twi.yaml @@ -0,0 +1,3 @@ +dataset_name: twi +include: afrimmlu_common_translate_yaml +task: afrimmlu_translate_twi diff --git a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_wol.yaml b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_wol.yaml new file mode 100644 index 0000000000000000000000000000000000000000..006b684782c853a432d9e694abe525aaeb9609ca --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_wol.yaml @@ -0,0 +1,3 @@ +dataset_name: wol +include: afrimmlu_common_translate_yaml +task: afrimmlu_translate_wol diff --git a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_xho.yaml b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_xho.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c0bdf4471b2178c67d7f6e1ae9c5fba16b3b7710 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_xho.yaml @@ -0,0 +1,3 @@ +dataset_name: xho +include: afrimmlu_common_translate_yaml +task: afrimmlu_translate_xho diff --git a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_yor.yaml b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0e7ba6005b591141dc84efa454196458c1261e8c --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_yor.yaml @@ -0,0 +1,3 @@ +dataset_name: yor +include: afrimmlu_common_translate_yaml +task: afrimmlu_translate_yor diff --git a/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_zul.yaml b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_zul.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a18d251cc8f838fa2578019475b089c4b61ecf65 --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/afrimmlu_translate_zul.yaml @@ -0,0 +1,3 @@ +dataset_name: zul +include: afrimmlu_common_translate_yaml +task: afrimmlu_translate_zul diff --git a/lm_eval/tasks/afrimmlu/translate/utils.py b/lm_eval/tasks/afrimmlu/translate/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..9d02b342b2e3c9f3d3bd66d3f62330aa53c9159c --- /dev/null +++ b/lm_eval/tasks/afrimmlu/translate/utils.py @@ -0,0 +1,32 @@ +from lm_eval.utils import weighted_f1_score + + +def doc_to_choice(doc): + choices = eval(doc["choices"]) + return choices + + +def doc_to_text(doc): + output = """You are a highly knowledgeable and intelligent artificial intelligence + model answers multiple-choice questions about '{subject}' + + Question: '''{question}''' + + Choices: + A: ''{choice1}''' + B: ''{choice2}''' + C: ''{choice3}''' + D: ''{choice4}''' + + Answer: """ + + choices = eval(doc["choices"]) + text = output.format( + subject=doc["subject"], + question=doc["question"], + choice1=choices[0], + choice2=choices[1], + choice3=choices[2], + choice4=choices[3], + ) + return text diff --git a/lm_eval/tasks/afrimmlu/utils.py b/lm_eval/tasks/afrimmlu/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..9d02b342b2e3c9f3d3bd66d3f62330aa53c9159c --- /dev/null +++ b/lm_eval/tasks/afrimmlu/utils.py @@ -0,0 +1,32 @@ +from lm_eval.utils import weighted_f1_score + + +def doc_to_choice(doc): + choices = eval(doc["choices"]) + return choices + + +def doc_to_text(doc): + output = """You are a highly knowledgeable and intelligent artificial intelligence + model answers multiple-choice questions about '{subject}' + + Question: '''{question}''' + + Choices: + A: ''{choice1}''' + B: ''{choice2}''' + C: ''{choice3}''' + D: ''{choice4}''' + + Answer: """ + + choices = eval(doc["choices"]) + text = output.format( + subject=doc["subject"], + question=doc["question"], + choice1=choices[0], + choice2=choices[1], + choice3=choices[2], + choice4=choices[3], + ) + return text diff --git a/lm_eval/tasks/afrixnli/README.md b/lm_eval/tasks/afrixnli/README.md new file mode 100644 index 0000000000000000000000000000000000000000..65b0272bc6f7dd9e7a670e142760b858efec90af --- /dev/null +++ b/lm_eval/tasks/afrixnli/README.md @@ -0,0 +1,57 @@ +# IrokoBench + +### Paper + +IrokoBench: A New Benchmark for African Languages in the Age of Large Language Models +https://arxiv.org/pdf/2406.03368 + +IrokoBench is a human-translated benchmark dataset for 16 typologically diverse +low-resource African languages covering three tasks: natural language inference (AfriXNLI), +mathematical reasoning (AfriMGSM), and multi-choice knowledge-based QA (AfriMMLU). + + +### Citation + +``` +@misc{adelani2024irokobenchnewbenchmarkafrican, + title={IrokoBench: A New Benchmark for African Languages in the Age of Large Language Models}, + author={David Ifeoluwa Adelani and Jessica Ojo and Israel Abebe Azime and Jian Yun Zhuang and Jesujoba O. Alabi and Xuanli He and Millicent Ochieng and Sara Hooker and Andiswa Bukula and En-Shiun Annie Lee and Chiamaka Chukwuneke and Happy Buzaaba and Blessing Sibanda and Godson Kalipe and Jonathan Mukiibi and Salomon Kabongo and Foutse Yuehgoh and Mmasibidi Setaka and Lolwethu Ndolela and Nkiruka Odu and Rooweither Mabuya and Shamsuddeen Hassan Muhammad and Salomey Osei and Sokhar Samb and Tadesse Kebede Guge and Pontus Stenetorp}, + year={2024}, + eprint={2406.03368}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2406.03368}, +} +``` + +### Groups and Tasks + +#### Groups + +* `afrixnli`: All afrixnli tasks +* `afrixnli_en_direct`: afrixnli_en_direct evaluates models performance using the anli prompt on the curated dataset +* `afrixnli_native_direct`: afrixnli_native_direct evaluates models performance using the anli prompt translated to the +respective languages on the curated dataset +* `afrixnli_translate`: afrixnli_translate evaluates models using the anli prompt in translate-test setting +* `afrixnli_manual_direct`: afrixnli_manual_direct evaluates models performance using Lai's prompt on the curated dataset +* `afrixnli_manual_translate`: afrixnli_manual_translate evaluates models using Lai's prompt in translate-test setting + +#### Tasks +* `afrixnli_en_direct_{language_code}`: each task evaluates for one language +* `afrixnli_native_direct_{language_code}`: each task evaluates for one language +* `afrixnli_translate_{language_code}`: each task evaluates for one language +* `afrixnli_manual_direct_{language_code}`: each task evaluates for one language +* `afrixnli_manual_translate_{language_code}`: each task evaluates for one language + +### Checklist + +For adding novel benchmarks/datasets to the library: +* [x] Is the task an existing benchmark in the literature? + * [x] Have you referenced the original paper that introduced the task? + * [ ] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + +If other tasks on this dataset are already supported: +* [x] Is the "Main" variant of this task clearly denoted? +* [x] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [x] Have you noted which, if any, published evaluation setups are matched by this variant? + * [x] Checked for equivalence with v0.3.0 LM Evaluation Harness diff --git a/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_amh.yaml b/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_amh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3727f15a1825dfd7f1a5b5dae00ada16c69af054 --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_amh.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: amh +include: afrixnli_en_direct_yaml +task: afrixnli_en_direct_amh diff --git a/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_eng.yaml b/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..28c404d0f76a3e0ba815c9f42fd4d03426bd287b --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_eng.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: eng +include: afrixnli_en_direct_yaml +task: afrixnli_en_direct_eng diff --git a/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_ewe.yaml b/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_ewe.yaml new file mode 100644 index 0000000000000000000000000000000000000000..075c1f07c7deaa59967e05bd4cb68a2fd9226956 --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_ewe.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ewe +include: afrixnli_en_direct_yaml +task: afrixnli_en_direct_ewe diff --git a/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_fra.yaml b/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_fra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0cd6c0f8de1fca34e363df39f07de0eb0b91c888 --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_fra.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fra +include: afrixnli_en_direct_yaml +task: afrixnli_en_direct_fra diff --git a/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_hau.yaml b/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..100bad5f4db1f86f9c37a871089ee2be3c5df926 --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_hau.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: hau +include: afrixnli_en_direct_yaml +task: afrixnli_en_direct_hau diff --git a/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_ibo.yaml b/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b8762c691458dd9709e236cb493eabb3fcb6881e --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_ibo.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ibo +include: afrixnli_en_direct_yaml +task: afrixnli_en_direct_ibo diff --git a/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_kin.yaml b/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_kin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1245d3da4e4d624c877e6014d5d3882d69f26889 --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_kin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: kin +include: afrixnli_en_direct_yaml +task: afrixnli_en_direct_kin diff --git a/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_lin.yaml b/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_lin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ac18bf6a47de6adbfa36f0bfa92d014f988de358 --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_lin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lin +include: afrixnli_en_direct_yaml +task: afrixnli_en_direct_lin diff --git a/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_lug.yaml b/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_lug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6375309f9b72c5eed77057bcb5a08416002647f0 --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_lug.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lug +include: afrixnli_en_direct_yaml +task: afrixnli_en_direct_lug diff --git a/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_orm.yaml b/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_orm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eb1c1c4fa09a1528118c84ae18334d5a9492cd8a --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_orm.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: orm +include: afrixnli_en_direct_yaml +task: afrixnli_en_direct_orm diff --git a/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_sna.yaml b/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_sna.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f7d94440efabe75519f5a6dbd04ff0bcc29e7b2b --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_sna.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sna +include: afrixnli_en_direct_yaml +task: afrixnli_en_direct_sna diff --git a/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_sot.yaml b/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_sot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f90a2f5e50d6dcda86ef31b6b5c578c3639273ae --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_sot.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sot +include: afrixnli_en_direct_yaml +task: afrixnli_en_direct_sot diff --git a/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_swa.yaml b/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd81dde5c2de1b7cd199ae1975253a6edce73f62 --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_swa.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: swa +include: afrixnli_en_direct_yaml +task: afrixnli_en_direct_swa diff --git a/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_twi.yaml b/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_twi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..49dd2b1e0a9915e8c1773603af640bc6a33e89a7 --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_twi.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: twi +include: afrixnli_en_direct_yaml +task: afrixnli_en_direct_twi diff --git a/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_wol.yaml b/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_wol.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0de5a2fedc41c60981d0b77e7d9d67c57855de08 --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_wol.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: wol +include: afrixnli_en_direct_yaml +task: afrixnli_en_direct_wol diff --git a/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_xho.yaml b/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_xho.yaml new file mode 100644 index 0000000000000000000000000000000000000000..38e4ca57a413e0cd0b830f72adba13ffa281367b --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_xho.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: xho +include: afrixnli_en_direct_yaml +task: afrixnli_en_direct_xho diff --git a/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_yaml b/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_yaml new file mode 100644 index 0000000000000000000000000000000000000000..be3583898dcf25956dfc5dff0d652e140475f864 --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_yaml @@ -0,0 +1,34 @@ +group: + - afrixnli + - afrixnli_en_direct +dataset_path: masakhane/afrixnli +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_text: "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither?\nAnswer:" +# True = entailment +# False = contradiction +# Neither = neutral +doc_to_target: !function utils.doc_to_target +doc_to_choice: + - "True" + - "Neither" + - "False" +should_decontaminate: true +doc_to_decontamination_query: premise +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + average: weighted + higher_is_better: True + ignore_case: true + ignore_punctuation: true + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_yor.yaml b/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c43ffac0c320911ea5d76e3c5c39a331d489a19f --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_yor.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: yor +include: afrixnli_en_direct_yaml +task: afrixnli_en_direct_yor diff --git a/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_zul.yaml b/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_zul.yaml new file mode 100644 index 0000000000000000000000000000000000000000..80d078937bc19bda8bbbb3dd6e8c425fac3e9b23 --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/en-direct/afrixnli_en_direct_zul.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: zul +include: afrixnli_en_direct_yaml +task: afrixnli_en_direct_zul diff --git a/lm_eval/tasks/afrixnli/anli prompt/en-direct/utils.py b/lm_eval/tasks/afrixnli/anli prompt/en-direct/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..5d1ac19e19b2e855c957e75f1c778366dfbc7e55 --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/en-direct/utils.py @@ -0,0 +1,6 @@ +from lm_eval.utils import weighted_f1_score + + +def doc_to_target(doc): + replacements = {0: "True", 1: "Neither", 2: "False"} + return replacements[doc["label"]] diff --git a/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_amh.yaml b/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_amh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..87b517906c0129b4aaabc518766e89dd6f70f505 --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_amh.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: amh +doc_to_choice: '{{[premise+", ትክክል? አዎ, "+hypothesis,premise+", ትክክል? እንዲሁም, "+hypothesis,premise+", + ትክክል? አይ, "+hypothesis]}}' +include: afrixnli_native_direct_yaml +task: afrixnli_native_direct_amh diff --git a/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_eng.yaml b/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bb2dcd58252468e3fa9dcf8dbad7b39dc9d2983b --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_eng.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: eng +doc_to_choice: '{{[premise+", Right? Yes, "+hypothesis,premise+", Right? Also, "+hypothesis,premise+", + Right? No, "+hypothesis]}}' +include: afrixnli_native_direct_yaml +task: afrixnli_native_direct_eng diff --git a/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_ewe.yaml b/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_ewe.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ecdc41c524eaf1429756643110a85b83009bb293 --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_ewe.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ewe +doc_to_choice: '{{[premise+", Esɔ gbe? Ɛ̃, "+hypothesis,premise+", Esɔ gbe? Hã, "+hypothesis,premise+", + Esɔ gbe? Ao, "+hypothesis]}}' +include: afrixnli_native_direct_yaml +task: afrixnli_native_direct_ewe diff --git a/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_fra.yaml b/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_fra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1e6e32cc165ce73e99ae5480debe0183dbb2351a --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_fra.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: fra +doc_to_choice: '{{[premise+", correct? Oui, "+hypothesis,premise+", correct? Aussi, + "+hypothesis,premise+", correct? Non, "+hypothesis]}}' +include: afrixnli_native_direct_yaml +task: afrixnli_native_direct_fra diff --git a/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_hau.yaml b/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b243a5de37f970dc92f27112280332cd2c5256cd --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_hau.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: hau +doc_to_choice: '{{[premise+", Daidai? Ee, "+hypothesis,premise+", Daidai? Haka kuma, + "+hypothesis,premise+", Daidai? A''a, "+hypothesis]}}' +include: afrixnli_native_direct_yaml +task: afrixnli_native_direct_hau diff --git a/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_ibo.yaml b/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..597ffb644c6e63a9bb4caccc41f45db2c2f29e68 --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_ibo.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: ibo +doc_to_choice: '{{[premise+", Ziri ezi? Éè, "+hypothesis,premise+", Ziri ezi? Ọzọkwa, + "+hypothesis,premise+", Ziri ezi? Mba, "+hypothesis]}}' +include: afrixnli_native_direct_yaml +task: afrixnli_native_direct_ibo diff --git a/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_kin.yaml b/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_kin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3333c12019585f6d089635cb7d7ab5eb9ad906d6 --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_kin.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: kin +doc_to_choice: '{{[premise+", Nibyo? Yego, "+hypothesis,premise+", Nibyo? Na none, + "+hypothesis,premise+", Nibyo? Oya, "+hypothesis]}}' +include: afrixnli_native_direct_yaml +task: afrixnli_native_direct_kin diff --git a/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_lin.yaml b/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_lin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..95060d6869cb8e6a3340f9d54bc5257244666f3c --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_lin.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: lin +doc_to_choice: '{{[premise+", Malamu? Iyo, "+hypothesis,premise+", Malamu? Lisusu, + "+hypothesis,premise+", Malamu? Te, "+hypothesis]}}' +include: afrixnli_native_direct_yaml +task: afrixnli_native_direct_lin diff --git a/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_lug.yaml b/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_lug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..97b6d00ec8b4ed0d9e1c9aabe133cc0b70141dbb --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_lug.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: lug +doc_to_choice: '{{[premise+", Kituufu? Yee, "+hypothesis,premise+", Kituufu? N’ekirala, + "+hypothesis,premise+", Kituufu? Nedda, "+hypothesis]}}' +include: afrixnli_native_direct_yaml +task: afrixnli_native_direct_lug diff --git a/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_orm.yaml b/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_orm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f9c25496da9cc81a3c82c8ae2a83621bf839e56a --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_orm.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: orm +doc_to_choice: '{{[premise+", Sirrii? Eeyyee, "+hypothesis,premise+", Sirrii? Akkasumas, + "+hypothesis,premise+", Sirrii? Lakki, "+hypothesis]}}' +include: afrixnli_native_direct_yaml +task: afrixnli_native_direct_orm diff --git a/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_sna.yaml b/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_sna.yaml new file mode 100644 index 0000000000000000000000000000000000000000..be2b2617ccdec63258607be20a6db2d958f018b1 --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_sna.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: sna +doc_to_choice: '{{[premise+", Chokwadi? Hongu, "+hypothesis,premise+", Chokwadi? Uye, + "+hypothesis,premise+", Chokwadi? Kwete, "+hypothesis]}}' +include: afrixnli_native_direct_yaml +task: afrixnli_native_direct_sna diff --git a/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_sot.yaml b/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_sot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..092961e0f8e39bb94a152aae00651b9ae49eebfb --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_sot.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: sot +doc_to_choice: '{{[premise+", Nepile? E, "+hypothesis,premise+", Nepile? Hape, "+hypothesis,premise+", + Nepile? Tjhe, "+hypothesis]}}' +include: afrixnli_native_direct_yaml +task: afrixnli_native_direct_sot diff --git a/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_swa.yaml b/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c8b1e2afa2c1b267c803565caa9cc13dc8d8f506 --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_swa.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: swa +doc_to_choice: '{{[premise+", Sahihi? Ndiyo, "+hypothesis,premise+", Sahihi? Pia, + "+hypothesis,premise+", Sahihi? Hapana, "+hypothesis]}}' +include: afrixnli_native_direct_yaml +task: afrixnli_native_direct_swa diff --git a/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_twi.yaml b/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_twi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4d3141d63a84b5a93002bd416583eb444543c031 --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_twi.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: twi +doc_to_choice: '{{[premise+", Nifa? Aane, "+hypothesis,premise+", Nifa? Anaasɛ, "+hypothesis,premise+", + Nifa? Daabi, "+hypothesis]}}' +include: afrixnli_native_direct_yaml +task: afrixnli_native_direct_twi diff --git a/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_wol.yaml b/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_wol.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1239fa47086826050a23d493c3e7069327a0e516 --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_wol.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: wol +doc_to_choice: '{{[premise+", Dëgg? Waaw, "+hypothesis,premise+", Dëgg? Itam, "+hypothesis,premise+", + Dëgg? Déet, "+hypothesis]}}' +include: afrixnli_native_direct_yaml +task: afrixnli_native_direct_wol diff --git a/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_xho.yaml b/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_xho.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f6f91f6e079d1138e374c7094bd76ef4743ec5b4 --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_xho.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: xho +doc_to_choice: '{{[premise+", Ichanekile? Ewe, "+hypothesis,premise+", Ichanekile? + Kananjalo, "+hypothesis,premise+", Ichanekile? Hayi, "+hypothesis]}}' +include: afrixnli_native_direct_yaml +task: afrixnli_native_direct_xho diff --git a/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_yaml b/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_yaml new file mode 100644 index 0000000000000000000000000000000000000000..369144721be392f01bd1be5e72665b7199bbd1bc --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_yaml @@ -0,0 +1,25 @@ +group: + - afrixnli + - afrixnli_native_direct +dataset_path: masakhane/afrixnli +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_target: label +doc_to_text: "" +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + average: weighted + higher_is_better: True + ignore_case: true + ignore_punctuation: true + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_yor.yaml b/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2648bf57bce8ffa5d28578094b477c6b8b166446 --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_yor.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: yor +doc_to_choice: '{{[premise+", Òótọ́? Bẹ́ẹ̀ni, "+hypothesis,premise+", Òótọ́? Àti pé, + "+hypothesis,premise+", Òótọ́? Rárá, "+hypothesis]}}' +include: afrixnli_native_direct_yaml +task: afrixnli_native_direct_yor diff --git a/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_zul.yaml b/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_zul.yaml new file mode 100644 index 0000000000000000000000000000000000000000..48261c60b28fa4c157b511153b609c840fea80e8 --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/native-direct/afrixnli_native_direct_zul.yaml @@ -0,0 +1,6 @@ +# Generated by utils.py +dataset_name: zul +doc_to_choice: '{{[premise+", Kulungile? Yebo, "+hypothesis,premise+", Kulungile? + Futhi, "+hypothesis,premise+", Kulungile? Cha, "+hypothesis]}}' +include: afrixnli_native_direct_yaml +task: afrixnli_native_direct_zul diff --git a/lm_eval/tasks/afrixnli/anli prompt/native-direct/utils.py b/lm_eval/tasks/afrixnli/anli prompt/native-direct/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..3e735e2deb1f9c53152c072615aebe8ba3acb90b --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/native-direct/utils.py @@ -0,0 +1 @@ +from lm_eval.utils import weighted_f1_score diff --git a/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_amh.yaml b/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_amh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..94fb2bdcb6f44646e6711dfaa38d7d0f66c767f5 --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_amh.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: amh +include: afrixnli_translate_yaml +task: afrixnli_translate_amh diff --git a/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_ewe.yaml b/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_ewe.yaml new file mode 100644 index 0000000000000000000000000000000000000000..55d5b470a2fdc47c73ac9ebeabbc6bdf388db0f2 --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_ewe.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ewe +include: afrixnli_translate_yaml +task: afrixnli_translate_ewe diff --git a/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_fra.yaml b/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_fra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd5903357dbd029bbc5a3d88c47e75ab05b4da41 --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_fra.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fra +include: afrixnli_translate_yaml +task: afrixnli_translate_fra diff --git a/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_hau.yaml b/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ddc7a375e03210ad02090a0279fe767e67d76c8e --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_hau.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: hau +include: afrixnli_translate_yaml +task: afrixnli_translate_hau diff --git a/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_ibo.yaml b/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2487f15a4a75ede35ab29300b6764fabd325e139 --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_ibo.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ibo +include: afrixnli_translate_yaml +task: afrixnli_translate_ibo diff --git a/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_kin.yaml b/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_kin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ebae340f5bf3b21a5d72c1ed4f6bad6223834d27 --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_kin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: kin +include: afrixnli_translate_yaml +task: afrixnli_translate_kin diff --git a/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_lin.yaml b/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_lin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0ad2ea078f78d509c452850ec1fdeef2c1f96325 --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_lin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lin +include: afrixnli_translate_yaml +task: afrixnli_translate_lin diff --git a/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_lug.yaml b/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_lug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c9ab91826d3f8b64370b061b58dcd5cd1b5d0da8 --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_lug.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lug +include: afrixnli_translate_yaml +task: afrixnli_translate_lug diff --git a/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_orm.yaml b/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_orm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..879228120a74794e894cad0b6d32ccb0b35ad473 --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_orm.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: orm +include: afrixnli_translate_yaml +task: afrixnli_translate_orm diff --git a/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_sna.yaml b/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_sna.yaml new file mode 100644 index 0000000000000000000000000000000000000000..69756c268c21a7228ed87cfc41522b5f2f549bf1 --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_sna.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sna +include: afrixnli_translate_yaml +task: afrixnli_translate_sna diff --git a/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_sot.yaml b/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_sot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..64b5cb29c770a1380a69001edf0026f47a0509a7 --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_sot.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sot +include: afrixnli_translate_yaml +task: afrixnli_translate_sot diff --git a/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_swa.yaml b/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ea6307131bfe14bdf9951b929556b0e911bed25f --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_swa.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: swa +include: afrixnli_translate_yaml +task: afrixnli_translate_swa diff --git a/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_twi.yaml b/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_twi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5cfd32e21ffd7208af52822be3bbeacd1676efab --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_twi.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: twi +include: afrixnli_translate_yaml +task: afrixnli_translate_twi diff --git a/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_wol.yaml b/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_wol.yaml new file mode 100644 index 0000000000000000000000000000000000000000..be1188e5cc7c941099bf25d9d7b71eba768dcf9a --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_wol.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: wol +include: afrixnli_translate_yaml +task: afrixnli_translate_wol diff --git a/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_xho.yaml b/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_xho.yaml new file mode 100644 index 0000000000000000000000000000000000000000..428ff3bbd2dccc0f60bb3818860d8426f9f70739 --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_xho.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: xho +include: afrixnli_translate_yaml +task: afrixnli_translate_xho diff --git a/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_yaml b/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_yaml new file mode 100644 index 0000000000000000000000000000000000000000..c35f86eb6f54b1b9ac07642e68e26694134cad26 --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_yaml @@ -0,0 +1,32 @@ +group: + - afrixnli + - afrixnli_translate +dataset_path: masakhane/afrixnli-translate-test +dataset_name: null +output_type: multiple_choice +test_split: test +doc_to_text: "{{premise}}\nQuestion: {{hypothesis}} True, False, or Neither?\nAnswer:" +# True = entailment +# False = contradiction +# Neither = neutral +doc_to_target: !function utils.doc_to_target +doc_to_choice: + - "True" + - "Neither" + - "False" +should_decontaminate: true +doc_to_decontamination_query: premise +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + average: weighted + higher_is_better: True + ignore_case: true + ignore_punctuation: true + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_yor.yaml b/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4f07a41a5b1b1f48ff85110aa3c6d1197a51f437 --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_yor.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: yor +include: afrixnli_translate_yaml +task: afrixnli_translate_yor diff --git a/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_zul.yaml b/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_zul.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7a57632bcafc9da38097eea7fbad89c14fbd12e9 --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/translate/afrixnli_translate_zul.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: zul +include: afrixnli_translate_yaml +task: afrixnli_translate_zul diff --git a/lm_eval/tasks/afrixnli/anli prompt/translate/utils.py b/lm_eval/tasks/afrixnli/anli prompt/translate/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..5d1ac19e19b2e855c957e75f1c778366dfbc7e55 --- /dev/null +++ b/lm_eval/tasks/afrixnli/anli prompt/translate/utils.py @@ -0,0 +1,6 @@ +from lm_eval.utils import weighted_f1_score + + +def doc_to_target(doc): + replacements = {0: "True", 1: "Neither", 2: "False"} + return replacements[doc["label"]] diff --git a/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_amh.yaml b/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_amh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9e8974c99a1b5d8ebed9c9be29e3628ad7d41674 --- /dev/null +++ b/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_amh.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: amh +include: afrixnli_manual_direct_yaml +task: afrixnli_manual_direct_amh diff --git a/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_eng.yaml b/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_eng.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7936a4322a3948093eefc12364f47b25181b0227 --- /dev/null +++ b/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_eng.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: eng +include: afrixnli_manual_direct_yaml +task: afrixnli_manual_direct_eng diff --git a/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_ewe.yaml b/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_ewe.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fe2fce97e33d8958a7a064aa25baa7e86d6f8f21 --- /dev/null +++ b/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_ewe.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ewe +include: afrixnli_manual_direct_yaml +task: afrixnli_manual_direct_ewe diff --git a/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_fra.yaml b/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_fra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..07c2f66238939ab19cce5f697826a5f53cdbe876 --- /dev/null +++ b/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_fra.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fra +include: afrixnli_manual_direct_yaml +task: afrixnli_manual_direct_fra diff --git a/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_hau.yaml b/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..885e571b344e78cf0277dc5f3193dc6096386d40 --- /dev/null +++ b/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_hau.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: hau +include: afrixnli_manual_direct_yaml +task: afrixnli_manual_direct_hau diff --git a/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_ibo.yaml b/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7a8428267e752a552c6bc67baaefd4a65f1bf47f --- /dev/null +++ b/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_ibo.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ibo +include: afrixnli_manual_direct_yaml +task: afrixnli_manual_direct_ibo diff --git a/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_kin.yaml b/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_kin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..611f61df85e89324769b6065e269e48ff3902190 --- /dev/null +++ b/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_kin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: kin +include: afrixnli_manual_direct_yaml +task: afrixnli_manual_direct_kin diff --git a/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_lin.yaml b/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_lin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eed83c757faa913a2b220ced61eeb718c5da3c12 --- /dev/null +++ b/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_lin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lin +include: afrixnli_manual_direct_yaml +task: afrixnli_manual_direct_lin diff --git a/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_lug.yaml b/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_lug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9c3cc02445be42a6bdb4860b6300f59e5dbc622c --- /dev/null +++ b/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_lug.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lug +include: afrixnli_manual_direct_yaml +task: afrixnli_manual_direct_lug diff --git a/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_orm.yaml b/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_orm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4931dc0a9ef3a58d9e9cdca3c6ab128333f7d3a0 --- /dev/null +++ b/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_orm.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: orm +include: afrixnli_manual_direct_yaml +task: afrixnli_manual_direct_orm diff --git a/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_sna.yaml b/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_sna.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ad7660a03668f00c0bf5a46c1162d32f382831ba --- /dev/null +++ b/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_sna.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sna +include: afrixnli_manual_direct_yaml +task: afrixnli_manual_direct_sna diff --git a/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_sot.yaml b/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_sot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..266605cb8c0deffca5020416e45d77a444b8f313 --- /dev/null +++ b/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_sot.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sot +include: afrixnli_manual_direct_yaml +task: afrixnli_manual_direct_sot diff --git a/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_swa.yaml b/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..07a890927933a0dc665b98f7e56cbd620fa97b18 --- /dev/null +++ b/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_swa.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: swa +include: afrixnli_manual_direct_yaml +task: afrixnli_manual_direct_swa diff --git a/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_twi.yaml b/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_twi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d76fff819e09299df81564cc8217a2f34e20afbf --- /dev/null +++ b/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_twi.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: twi +include: afrixnli_manual_direct_yaml +task: afrixnli_manual_direct_twi diff --git a/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_wol.yaml b/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_wol.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9f189d3975a07b125581d482e268205793e1577e --- /dev/null +++ b/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_wol.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: wol +include: afrixnli_manual_direct_yaml +task: afrixnli_manual_direct_wol diff --git a/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_xho.yaml b/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_xho.yaml new file mode 100644 index 0000000000000000000000000000000000000000..307b42fc58bf7782823473fe67a2343698c8ae9a --- /dev/null +++ b/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_xho.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: xho +include: afrixnli_manual_direct_yaml +task: afrixnli_manual_direct_xho diff --git a/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_yaml b/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_yaml new file mode 100644 index 0000000000000000000000000000000000000000..c15cd68b89a0036d98b77d9e4622ff16206d4325 --- /dev/null +++ b/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_yaml @@ -0,0 +1,31 @@ +group: + - afrixnli + - afrixnli_manual_direct +dataset_path: masakhane/afrixnli +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +fewshot_split: validation +doc_to_text: !function utils.doc_to_text +doc_to_target: !function utils.doc_to_target +doc_to_choice: + - "entailment" + - "neutral" + - "contradiction" +should_decontaminate: true +doc_to_decontamination_query: premise +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + average: weighted + higher_is_better: True + ignore_case: true + ignore_punctuation: true + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_yor.yaml b/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b2b9f99a05509897253504e57671be3df94adaf7 --- /dev/null +++ b/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_yor.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: yor +include: afrixnli_manual_direct_yaml +task: afrixnli_manual_direct_yor diff --git a/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_zul.yaml b/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_zul.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2833840644b75044470a2dfb133d0afd43da105c --- /dev/null +++ b/lm_eval/tasks/afrixnli/lai prompt/direct/afrixnli_manual_direct_zul.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: zul +include: afrixnli_manual_direct_yaml +task: afrixnli_manual_direct_zul diff --git a/lm_eval/tasks/afrixnli/lai prompt/direct/utils.py b/lm_eval/tasks/afrixnli/lai prompt/direct/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..d97a0a288508e817ab695e637fb157a08c813808 --- /dev/null +++ b/lm_eval/tasks/afrixnli/lai prompt/direct/utils.py @@ -0,0 +1,19 @@ +from lm_eval.utils import weighted_f1_score + + +def doc_to_text(doc): + output = """Please identify whether the premise entails or contradicts the hypothesis in the following premise + and hypothesis. The answer should be exact entailment, contradiction, or neutral. + + Premise: {premise} + Hypothesis: {hypothesis} + + Is it entailment, contradiction, or neutral?""" + + text = output.format(premise=doc["premise"], hypothesis=doc["hypothesis"]) + return text + + +def doc_to_target(doc): + replacements = {0: "entailment", 1: "neutral", 2: "contradiction"} + return replacements[doc["label"]] diff --git a/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_amh.yaml b/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_amh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aa79494a59c3d529cefe3afc6793c113136ba4a9 --- /dev/null +++ b/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_amh.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: amh +include: afrixnli_manual_translate_yaml +task: afrixnli_manual_translate_amh diff --git a/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_ewe.yaml b/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_ewe.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9d209484bdec5139ce18e3c84b9385cbe5549928 --- /dev/null +++ b/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_ewe.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ewe +include: afrixnli_manual_translate_yaml +task: afrixnli_manual_translate_ewe diff --git a/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_fra.yaml b/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_fra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a943963b9075e818074e98fcd3bf255502dd482a --- /dev/null +++ b/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_fra.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: fra +include: afrixnli_manual_translate_yaml +task: afrixnli_manual_translate_fra diff --git a/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_hau.yaml b/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_hau.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a343c952fe31f91f2332204df366a5434fd62f03 --- /dev/null +++ b/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_hau.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: hau +include: afrixnli_manual_translate_yaml +task: afrixnli_manual_translate_hau diff --git a/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_ibo.yaml b/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_ibo.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0057e6b0cbfbd7a7663829a05e0d44d60c301d3f --- /dev/null +++ b/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_ibo.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: ibo +include: afrixnli_manual_translate_yaml +task: afrixnli_manual_translate_ibo diff --git a/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_kin.yaml b/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_kin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5874ae5f6cd2cac4296b3abaa5568a4dd7d2188a --- /dev/null +++ b/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_kin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: kin +include: afrixnli_manual_translate_yaml +task: afrixnli_manual_translate_kin diff --git a/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_lin.yaml b/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_lin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a6e6023cf5ad47cede58060b973ee9aed9964bde --- /dev/null +++ b/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_lin.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lin +include: afrixnli_manual_translate_yaml +task: afrixnli_manual_translate_lin diff --git a/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_lug.yaml b/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_lug.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5bc3a14d41eba99d4bb9f2b46fd44ec1526507cf --- /dev/null +++ b/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_lug.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: lug +include: afrixnli_manual_translate_yaml +task: afrixnli_manual_translate_lug diff --git a/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_orm.yaml b/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_orm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cb9a494f4057783cca9a68eda9b4fb56e0b99948 --- /dev/null +++ b/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_orm.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: orm +include: afrixnli_manual_translate_yaml +task: afrixnli_manual_translate_orm diff --git a/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_sna.yaml b/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_sna.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d6523987f10926fa2f2fd80417e86e494363f0fa --- /dev/null +++ b/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_sna.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sna +include: afrixnli_manual_translate_yaml +task: afrixnli_manual_translate_sna diff --git a/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_sot.yaml b/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_sot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..319e909c84cf513aa9985a0a6cc44794f78a09b8 --- /dev/null +++ b/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_sot.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: sot +include: afrixnli_manual_translate_yaml +task: afrixnli_manual_translate_sot diff --git a/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_swa.yaml b/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_swa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a34eb438e4d45bdedc68f893af2fb4374fc931a7 --- /dev/null +++ b/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_swa.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: swa +include: afrixnli_manual_translate_yaml +task: afrixnli_manual_translate_swa diff --git a/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_twi.yaml b/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_twi.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0535f1db84f43aaed989efcfbe9e1781480931b8 --- /dev/null +++ b/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_twi.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: twi +include: afrixnli_manual_translate_yaml +task: afrixnli_manual_translate_twi diff --git a/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_wol.yaml b/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_wol.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8541b73ec8e1c0c7417a5547cdfd170ed9bcf21b --- /dev/null +++ b/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_wol.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: wol +include: afrixnli_manual_translate_yaml +task: afrixnli_manual_translate_wol diff --git a/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_xho.yaml b/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_xho.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8d1eebcb37f65f1ad44a098220f979aa840b4f57 --- /dev/null +++ b/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_xho.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: xho +include: afrixnli_manual_translate_yaml +task: afrixnli_manual_translate_xho diff --git a/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_yaml b/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_yaml new file mode 100644 index 0000000000000000000000000000000000000000..fd30e302f0257344d51855acfa80ac26ab823be0 --- /dev/null +++ b/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_yaml @@ -0,0 +1,29 @@ +group: + - afrixnli + - afrixnli_manual_direct +dataset_path: masakhane/afrixnli-translate-test +dataset_name: null +output_type: multiple_choice +test_split: test +doc_to_text: !function utils.doc_to_text +doc_to_target: !function utils.doc_to_target +doc_to_choice: + - "entailment" + - "neutral" + - "contradiction" +should_decontaminate: true +doc_to_decontamination_query: premise +metric_list: + - metric: f1 + aggregation: !function utils.weighted_f1_score + average: weighted + higher_is_better: True + ignore_case: true + ignore_punctuation: true + - metric: acc + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_yor.yaml b/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_yor.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1825ec27a5eb16b4229385555d699d33338d7c86 --- /dev/null +++ b/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_yor.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: yor +include: afrixnli_manual_translate_yaml +task: afrixnli_manual_translate_yor diff --git a/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_zul.yaml b/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_zul.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4bf52549d730596f90caf263fe6299bbc705095b --- /dev/null +++ b/lm_eval/tasks/afrixnli/lai prompt/translate/afrixnli_manual_translate_zul.yaml @@ -0,0 +1,4 @@ +# Generated by utils.py +dataset_name: zul +include: afrixnli_manual_translate_yaml +task: afrixnli_manual_translate_zul diff --git a/lm_eval/tasks/afrixnli/lai prompt/translate/utils.py b/lm_eval/tasks/afrixnli/lai prompt/translate/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..d97a0a288508e817ab695e637fb157a08c813808 --- /dev/null +++ b/lm_eval/tasks/afrixnli/lai prompt/translate/utils.py @@ -0,0 +1,19 @@ +from lm_eval.utils import weighted_f1_score + + +def doc_to_text(doc): + output = """Please identify whether the premise entails or contradicts the hypothesis in the following premise + and hypothesis. The answer should be exact entailment, contradiction, or neutral. + + Premise: {premise} + Hypothesis: {hypothesis} + + Is it entailment, contradiction, or neutral?""" + + text = output.format(premise=doc["premise"], hypothesis=doc["hypothesis"]) + return text + + +def doc_to_target(doc): + replacements = {0: "entailment", 1: "neutral", 2: "contradiction"} + return replacements[doc["label"]] diff --git a/lm_eval/tasks/afrixnli/utils.py b/lm_eval/tasks/afrixnli/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..905a72b001afb46b294abba34ff629cb40b3f1e6 --- /dev/null +++ b/lm_eval/tasks/afrixnli/utils.py @@ -0,0 +1,237 @@ +import argparse + +import yaml + + +class FunctionTag: + def __init__(self, value): + self.value = value + + +LANGUAGES = { + "amh": { + "QUESTION_WORD": "ትክክል", + "ENTAILMENT_LABEL": "አዎ", + "NEUTRAL_LABEL": "እንዲሁም", + "CONTRADICTION_LABEL": "አይ", + }, + "eng": { + "QUESTION_WORD": "Right", + "ENTAILMENT_LABEL": "Yes", + "NEUTRAL_LABEL": "Also", + "CONTRADICTION_LABEL": "No", + }, + "ewe": { + "QUESTION_WORD": "Esɔ gbe", + "ENTAILMENT_LABEL": "Ɛ̃", + "NEUTRAL_LABEL": "Hã", + "CONTRADICTION_LABEL": "Ao", + }, + "fra": { + "QUESTION_WORD": "correct", + "ENTAILMENT_LABEL": "Oui", + "NEUTRAL_LABEL": "Aussi", + "CONTRADICTION_LABEL": "Non", + }, + "hau": { + "QUESTION_WORD": "Daidai", + "ENTAILMENT_LABEL": "Ee", + "NEUTRAL_LABEL": "Haka kuma", + "CONTRADICTION_LABEL": "A'a", + }, + "ibo": { + "QUESTION_WORD": "Ziri ezi", + "ENTAILMENT_LABEL": "Éè", + "NEUTRAL_LABEL": "Ọzọkwa", + "CONTRADICTION_LABEL": "Mba", + }, + "kin": { + "QUESTION_WORD": "Nibyo", + "ENTAILMENT_LABEL": "Yego", + "NEUTRAL_LABEL": "Na none", + "CONTRADICTION_LABEL": "Oya", + }, + "lin": { + "QUESTION_WORD": "Malamu", + "ENTAILMENT_LABEL": "Iyo", + "NEUTRAL_LABEL": "Lisusu", + "CONTRADICTION_LABEL": "Te", + }, + "lug": { + "QUESTION_WORD": "Kituufu", + "ENTAILMENT_LABEL": "Yee", + "NEUTRAL_LABEL": "N’ekirala", + "CONTRADICTION_LABEL": "Nedda", + }, + "orm": { + "QUESTION_WORD": "Sirrii", + "ENTAILMENT_LABEL": "Eeyyee", + "NEUTRAL_LABEL": "Akkasumas", + "CONTRADICTION_LABEL": "Lakki", + }, + "sna": { + "QUESTION_WORD": "Chokwadi", + "ENTAILMENT_LABEL": "Hongu", + "NEUTRAL_LABEL": "Uye", + "CONTRADICTION_LABEL": "Kwete", + }, + "sot": { + "QUESTION_WORD": "Nepile", + "ENTAILMENT_LABEL": "E", + "NEUTRAL_LABEL": "Hape", + "CONTRADICTION_LABEL": "Tjhe", + }, + "swa": { + "QUESTION_WORD": "Sahihi", + "ENTAILMENT_LABEL": "Ndiyo", + "NEUTRAL_LABEL": "Pia", + "CONTRADICTION_LABEL": "Hapana", + }, + "twi": { + "QUESTION_WORD": "Nifa", + "ENTAILMENT_LABEL": "Aane", + "NEUTRAL_LABEL": "Anaasɛ", + "CONTRADICTION_LABEL": "Daabi", + }, + "wol": { + "QUESTION_WORD": "Dëgg", + "ENTAILMENT_LABEL": "Waaw", + "NEUTRAL_LABEL": "Itam", + "CONTRADICTION_LABEL": "Déet", + }, + "xho": { + "QUESTION_WORD": "Ichanekile", + "ENTAILMENT_LABEL": "Ewe", + "NEUTRAL_LABEL": "Kananjalo", + "CONTRADICTION_LABEL": "Hayi", + }, + "yor": { + "QUESTION_WORD": "Òótọ́", + "ENTAILMENT_LABEL": "Bẹ́ẹ̀ni", + "NEUTRAL_LABEL": "Àti pé", + "CONTRADICTION_LABEL": "Rárá", + }, + "zul": { + "QUESTION_WORD": "Kulungile", + "ENTAILMENT_LABEL": "Yebo", + "NEUTRAL_LABEL": "Futhi", + "CONTRADICTION_LABEL": "Cha", + }, +} + + +def gen_lang_yamls(output_dir: str, overwrite: bool, mode: str) -> None: + """ + Generate a yaml file for each language. + + :param output_dir: The directory to output the files to. + :param overwrite: Whether to overwrite files if they already exist. + """ + err = [] + languages = [ + "eng", + "amh", + "ibo", + "fra", + "sna", + "wol", + "ewe", + "lin", + "lug", + "xho", + "kin", + "twi", + "zul", + "orm", + "yor", + "hau", + "sot", + "swa", + ] + for lang in languages: + try: + if mode == "native-direct": + QUESTION_WORD = LANGUAGES[lang]["QUESTION_WORD"] + ENTAILMENT_LABEL = LANGUAGES[lang]["ENTAILMENT_LABEL"] + NEUTRAL_LABEL = LANGUAGES[lang]["NEUTRAL_LABEL"] + CONTRADICTION_LABEL = LANGUAGES[lang]["CONTRADICTION_LABEL"] + + file_name = f"afrixnli_native_direct_{lang}.yaml" + task_name = f"afrixnli_native_direct_{lang}" + yaml_template = "afrixnli_native_direct_yaml" + with open( + f"{output_dir}/{file_name}", + "w" if overwrite else "x", + encoding="utf8", + ) as f: + f.write("# Generated by utils.py\n") + yaml.dump( + { + "include": yaml_template, + "task": task_name, + "dataset_name": lang, + "doc_to_choice": f"{{{{[" + f"""premise+\", {QUESTION_WORD}? {ENTAILMENT_LABEL}, \"+hypothesis,""" + f"""premise+\", {QUESTION_WORD}? {NEUTRAL_LABEL}, \"+hypothesis,""" + f"""premise+\", {QUESTION_WORD}? {CONTRADICTION_LABEL}, \"+hypothesis""" + f"]}}}}", + }, + f, + allow_unicode=True, + ) + else: + file_name = f"afrixnli_{mode}_{lang}.yaml" + task_name = f"afrixnli_{mode}_{lang}" + yaml_template = f"afrixnli_{mode}_yaml" + with open( + f"{output_dir}/{file_name}", + "w" if overwrite else "x", + encoding="utf8", + ) as f: + f.write("# Generated by utils.py\n") + yaml.dump( + { + "include": yaml_template, + "task": task_name, + "dataset_name": lang, + }, + f, + allow_unicode=True, + ) + except FileExistsError: + err.append(file_name) + + if len(err) > 0: + raise FileExistsError( + "Files were not created because they already exist (use --overwrite flag):" + f" {', '.join(err)}" + ) + + +def main() -> None: + """Parse CLI args and generate language-specific yaml files.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--overwrite", + default=True, + action="store_true", + help="Overwrite files if they already exist", + ) + parser.add_argument( + "--output-dir", + default="./manual/translate", + help="Directory to write yaml files to", + ) + parser.add_argument( + "--mode", + default="manual_translate", + choices=["en_direct", "native-direct", "manual_direct", "manual_translate"], + help="Mode of chain-of-thought", + ) + args = parser.parse_args() + + gen_lang_yamls(output_dir=args.output_dir, overwrite=args.overwrite, mode=args.mode) + + +if __name__ == "__main__": + main() diff --git a/lm_eval/tasks/agieval/README.md b/lm_eval/tasks/agieval/README.md index faaf47b6beab877c7ee341a8dc2fc3e14a04b021..53a9df036d6c0a4dcc2b310ac324f1bf7b0f60dc 100644 --- a/lm_eval/tasks/agieval/README.md +++ b/lm_eval/tasks/agieval/README.md @@ -75,7 +75,7 @@ Please make sure to cite all the individual datasets in your paper when you use } ``` -### Groups and Tasks +### Groups, Tags, and Tasks #### Groups @@ -89,6 +89,10 @@ Please make sure to cite all the individual datasets in your paper when you use - `agieval_nous`: Evaluates a specific subset of AGIEval tasks (multiple-choice and english-only), namely those in https://github.com/teknium1/LLM-Benchmark-Logs/blob/main/benchmark-logs/Mistral-7B-Base.md +#### Tags + +None. + #### Tasks - `agieval_aqua_rat` diff --git a/lm_eval/tasks/agieval/agieval.yaml b/lm_eval/tasks/agieval/agieval.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d086af83579ec2daf826a55f7dd82cf2e1f82a96 --- /dev/null +++ b/lm_eval/tasks/agieval/agieval.yaml @@ -0,0 +1,29 @@ +group: agieval +task: + - agieval_gaokao_biology + - agieval_gaokao_chemistry + - agieval_gaokao_chinese + - agieval_gaokao_geography + - agieval_gaokao_history + - agieval_gaokao_mathcloze + - agieval_gaokao_mathqa + - agieval_gaokao_physics + - agieval_jec_qa_ca + - agieval_jec_qa_kd + - agieval_logiqa_zh + - agieval_aqua_rat + - agieval_gaokao_english + - agieval_logiqa_en + - agieval_lsat_ar + - agieval_lsat_lr + - agieval_lsat_rc + - agieval_math + - agieval_sat_en_without_passage + - agieval_sat_en + - agieval_sat_math +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm_eval/tasks/agieval/agieval_cn.yaml b/lm_eval/tasks/agieval/agieval_cn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0e8ca2fdedaabe768fe731bbf2dbbea3ef117448 --- /dev/null +++ b/lm_eval/tasks/agieval/agieval_cn.yaml @@ -0,0 +1,19 @@ +group: agieval_cn +task: + - agieval_gaokao_biology + - agieval_gaokao_chemistry + - agieval_gaokao_chinese + - agieval_gaokao_geography + - agieval_gaokao_history + - agieval_gaokao_mathcloze + - agieval_gaokao_mathqa + - agieval_gaokao_physics + - agieval_jec_qa_ca + - agieval_jec_qa_kd + - agieval_logiqa_zh +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm_eval/tasks/agieval/agieval_en.yaml b/lm_eval/tasks/agieval/agieval_en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0a873d66d3a4e98fc2ce2df26e53f20a599bc4e8 --- /dev/null +++ b/lm_eval/tasks/agieval/agieval_en.yaml @@ -0,0 +1,18 @@ +group: agieval_en +task: + - agieval_aqua_rat + - agieval_gaokao_english # categorizing as EN because the AGIEval codebase lists this as in `english_qa_tasks` + - agieval_logiqa_en + - agieval_lsat_ar + - agieval_lsat_lr + - agieval_lsat_rc + - agieval_math + - agieval_sat_en_without_passage + - agieval_sat_en + - agieval_sat_math +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm_eval/tasks/agieval/agieval_nous.yaml b/lm_eval/tasks/agieval/agieval_nous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aa2a874892e77aaa0216feac4f2f6353b3302a93 --- /dev/null +++ b/lm_eval/tasks/agieval/agieval_nous.yaml @@ -0,0 +1,16 @@ +group: agieval_nous +task: + - agieval_aqua_rat + - agieval_logiqa_en + - agieval_lsat_ar + - agieval_lsat_lr + - agieval_lsat_rc + - agieval_sat_en_without_passage + - agieval_sat_en + - agieval_sat_math +aggregate_metric_list: + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm_eval/tasks/agieval/aqua-rat.yaml b/lm_eval/tasks/agieval/aqua-rat.yaml index babebf638edcf0e9c5a2432adb6a2fdaf4793c1d..e5a3a3e86f6c5448000df38a146a95518691b934 100644 --- a/lm_eval/tasks/agieval/aqua-rat.yaml +++ b/lm_eval/tasks/agieval/aqua-rat.yaml @@ -1,7 +1,3 @@ -group: - - agieval - - agieval_en - - agieval_nous task: agieval_aqua_rat dataset_path: hails/agieval-aqua-rat dataset_name: null diff --git a/lm_eval/tasks/agieval/gaokao-biology.yaml b/lm_eval/tasks/agieval/gaokao-biology.yaml index 36c44cbbeeb730f05c9d425c20f02c78acc81563..b8b9eca8397965a0bf3c7152fbd237236b0f37f2 100644 --- a/lm_eval/tasks/agieval/gaokao-biology.yaml +++ b/lm_eval/tasks/agieval/gaokao-biology.yaml @@ -1,6 +1,3 @@ include: aqua-rat.yaml -group: - - agieval - - agieval_cn task: agieval_gaokao_biology dataset_path: hails/agieval-gaokao-biology diff --git a/lm_eval/tasks/agieval/gaokao-chemistry.yaml b/lm_eval/tasks/agieval/gaokao-chemistry.yaml index 69810122eb274cdcb285232330a19807886ee50d..a4ba01a5274376bab68fb8a56bd25a6e81d1edfb 100644 --- a/lm_eval/tasks/agieval/gaokao-chemistry.yaml +++ b/lm_eval/tasks/agieval/gaokao-chemistry.yaml @@ -1,6 +1,3 @@ include: aqua-rat.yaml -group: - - agieval - - agieval_cn task: agieval_gaokao_chemistry dataset_path: hails/agieval-gaokao-chemistry diff --git a/lm_eval/tasks/agieval/gaokao-chinese.yaml b/lm_eval/tasks/agieval/gaokao-chinese.yaml index 30d249b9d5544a3441e50284929aac6f081d6b76..3d58b5bc495917482ef69f04604b7f78f91339f5 100644 --- a/lm_eval/tasks/agieval/gaokao-chinese.yaml +++ b/lm_eval/tasks/agieval/gaokao-chinese.yaml @@ -1,6 +1,3 @@ include: aqua-rat.yaml -group: - - agieval - - agieval_cn task: agieval_gaokao_chinese dataset_path: hails/agieval-gaokao-chinese diff --git a/lm_eval/tasks/agieval/gaokao-english.yaml b/lm_eval/tasks/agieval/gaokao-english.yaml index a540fcf25f503be64d3f5810be7b037a2e7c0504..12ea66787acfa60eefc5a49936c6484b08c8fda0 100644 --- a/lm_eval/tasks/agieval/gaokao-english.yaml +++ b/lm_eval/tasks/agieval/gaokao-english.yaml @@ -1,6 +1,3 @@ include: aqua-rat.yaml -group: - - agieval - - agieval_en # categorizing as EN because the AGIEval codebase lists this as in `english_qa_tasks` task: agieval_gaokao_english dataset_path: hails/agieval-gaokao-english diff --git a/lm_eval/tasks/agieval/gaokao-geography.yaml b/lm_eval/tasks/agieval/gaokao-geography.yaml index 2fe43bfd2cb620328dfb28ba4a4e9e6d6d093c07..6dbce6f4873e272f9c28f49b0061857060df2e97 100644 --- a/lm_eval/tasks/agieval/gaokao-geography.yaml +++ b/lm_eval/tasks/agieval/gaokao-geography.yaml @@ -1,6 +1,3 @@ include: aqua-rat.yaml -group: - - agieval - - agieval_cn task: agieval_gaokao_geography dataset_path: hails/agieval-gaokao-geography diff --git a/lm_eval/tasks/agieval/gaokao-history.yaml b/lm_eval/tasks/agieval/gaokao-history.yaml index b9c9c630fa2c843da5c8311b1e0570bb1cc267f9..55b317281977f215285d7d699f656e54be55bf37 100644 --- a/lm_eval/tasks/agieval/gaokao-history.yaml +++ b/lm_eval/tasks/agieval/gaokao-history.yaml @@ -1,6 +1,3 @@ include: aqua-rat.yaml -group: - - agieval - - agieval_cn task: agieval_gaokao_history dataset_path: hails/agieval-gaokao-history diff --git a/lm_eval/tasks/agieval/gaokao-mathcloze.yaml b/lm_eval/tasks/agieval/gaokao-mathcloze.yaml index 74cbad1c0325c4fb9fe78df83304741553c06134..8692e2f45f8cc016cba40f246345fa5909a9256a 100644 --- a/lm_eval/tasks/agieval/gaokao-mathcloze.yaml +++ b/lm_eval/tasks/agieval/gaokao-mathcloze.yaml @@ -1,6 +1,3 @@ -group: - - agieval - - agieval_cn task: agieval_gaokao_mathcloze dataset_path: hails/agieval-gaokao-mathcloze dataset_name: null diff --git a/lm_eval/tasks/agieval/gaokao-mathqa.yaml b/lm_eval/tasks/agieval/gaokao-mathqa.yaml index aa94e8eec85a931e5acbdb843730b58e8c1506e5..e0d97a515559d9eaecca8bc73949a1d6886b1922 100644 --- a/lm_eval/tasks/agieval/gaokao-mathqa.yaml +++ b/lm_eval/tasks/agieval/gaokao-mathqa.yaml @@ -1,6 +1,3 @@ include: aqua-rat.yaml -group: - - agieval - - agieval_cn task: agieval_gaokao_mathqa dataset_path: hails/agieval-gaokao-mathqa diff --git a/lm_eval/tasks/agieval/gaokao-physics.yaml b/lm_eval/tasks/agieval/gaokao-physics.yaml index 175dd6cca03fab93107e0bab827ea356ceb127eb..43a047edafd06ab29c666741dd4f28560c64eff9 100644 --- a/lm_eval/tasks/agieval/gaokao-physics.yaml +++ b/lm_eval/tasks/agieval/gaokao-physics.yaml @@ -1,6 +1,3 @@ include: aqua-rat.yaml -group: - - agieval - - agieval_cn task: agieval_gaokao_physics dataset_path: hails/agieval-gaokao-physics diff --git a/lm_eval/tasks/agieval/jec-qa-ca.yaml b/lm_eval/tasks/agieval/jec-qa-ca.yaml index f93b47a5b1418d839933b71e71b523fd38696691..2d09c2b3814a65fffcae1a06894bd1a3f57ca983 100644 --- a/lm_eval/tasks/agieval/jec-qa-ca.yaml +++ b/lm_eval/tasks/agieval/jec-qa-ca.yaml @@ -1,6 +1,3 @@ include: aqua-rat.yaml -group: - - agieval - - agieval_cn task: agieval_jec_qa_ca dataset_path: hails/agieval-jec-qa-ca diff --git a/lm_eval/tasks/agieval/jec-qa-kd.yaml b/lm_eval/tasks/agieval/jec-qa-kd.yaml index 0458eb7ea8356df569ac6c3b50af0bd4097ea857..5843b6deb9aa1e326426adb96fb5ebeca333e6cf 100644 --- a/lm_eval/tasks/agieval/jec-qa-kd.yaml +++ b/lm_eval/tasks/agieval/jec-qa-kd.yaml @@ -1,6 +1,3 @@ include: aqua-rat.yaml -group: - - agieval - - agieval_cn task: agieval_jec_qa_kd dataset_path: hails/agieval-jec-qa-kd diff --git a/lm_eval/tasks/agieval/logiqa-en.yaml b/lm_eval/tasks/agieval/logiqa-en.yaml index 7112418659c4478c4e59f9bdcdebb6d64e7b9bb6..5bd1dff40b0017ee23067cd20bc8543eaf8081b2 100644 --- a/lm_eval/tasks/agieval/logiqa-en.yaml +++ b/lm_eval/tasks/agieval/logiqa-en.yaml @@ -1,7 +1,3 @@ include: aqua-rat.yaml -group: - - agieval - - agieval_nous - - agieval_en task: agieval_logiqa_en dataset_path: hails/agieval-logiqa-en diff --git a/lm_eval/tasks/agieval/logiqa-zh.yaml b/lm_eval/tasks/agieval/logiqa-zh.yaml index 82e688006b8272e015a74b01412ad35cfe33561e..2ca9198b53240e04e69a617274f93964be067539 100644 --- a/lm_eval/tasks/agieval/logiqa-zh.yaml +++ b/lm_eval/tasks/agieval/logiqa-zh.yaml @@ -1,6 +1,3 @@ include: aqua-rat.yaml -group: - - agieval - - agieval_cn task: agieval_logiqa_zh dataset_path: hails/agieval-logiqa-zh diff --git a/lm_eval/tasks/agieval/lsat-ar.yaml b/lm_eval/tasks/agieval/lsat-ar.yaml index 302f9b519ee268831c1725fb96322d6628b9fdf9..2284f601f19988805194f8af96c7789251bcaeae 100644 --- a/lm_eval/tasks/agieval/lsat-ar.yaml +++ b/lm_eval/tasks/agieval/lsat-ar.yaml @@ -1,7 +1,3 @@ include: aqua-rat.yaml -group: - - agieval - - agieval_nous - - agieval_en task: agieval_lsat_ar dataset_path: hails/agieval-lsat-ar diff --git a/lm_eval/tasks/agieval/lsat-lr.yaml b/lm_eval/tasks/agieval/lsat-lr.yaml index 62158e5cec196c0c7887a7236e1020ba2946da26..8505d4463f72d1942a47e7ae76b07c1958c92ee5 100644 --- a/lm_eval/tasks/agieval/lsat-lr.yaml +++ b/lm_eval/tasks/agieval/lsat-lr.yaml @@ -1,7 +1,3 @@ include: aqua-rat.yaml -group: - - agieval - - agieval_nous - - agieval_en task: agieval_lsat_lr dataset_path: hails/agieval-lsat-lr diff --git a/lm_eval/tasks/agieval/lsat-rc.yaml b/lm_eval/tasks/agieval/lsat-rc.yaml index de155af78aa8d5ad3b14849d8a2807a7194f6744..23a9dce7d3853af35091d0bd32df1dbd481ab7aa 100644 --- a/lm_eval/tasks/agieval/lsat-rc.yaml +++ b/lm_eval/tasks/agieval/lsat-rc.yaml @@ -1,7 +1,3 @@ include: aqua-rat.yaml -group: - - agieval - - agieval_nous - - agieval_en task: agieval_lsat_rc dataset_path: hails/agieval-lsat-rc diff --git a/lm_eval/tasks/agieval/math.yaml b/lm_eval/tasks/agieval/math.yaml index c8ec9eec608c4eaced456c36dcb5dc9047ccd84e..534a2e661c38a52ee14cbdddbd6c3f946336cfc7 100644 --- a/lm_eval/tasks/agieval/math.yaml +++ b/lm_eval/tasks/agieval/math.yaml @@ -1,6 +1,3 @@ -group: - - agieval - - agieval_en task: agieval_math dataset_path: hails/agieval-math dataset_name: null diff --git a/lm_eval/tasks/agieval/sat-en-without-passage.yaml b/lm_eval/tasks/agieval/sat-en-without-passage.yaml index 01490d9ee10aba867a1863e9d6a74b678f4f5588..d36b467cd76f62f84463cfff2fa423c5f2c87860 100644 --- a/lm_eval/tasks/agieval/sat-en-without-passage.yaml +++ b/lm_eval/tasks/agieval/sat-en-without-passage.yaml @@ -1,7 +1,3 @@ include: aqua-rat.yaml -group: - - agieval - - agieval_nous - - agieval_en task: agieval_sat_en_without_passage dataset_path: hails/agieval-sat-en-without-passage diff --git a/lm_eval/tasks/agieval/sat-en.yaml b/lm_eval/tasks/agieval/sat-en.yaml index a45dba1507a562ace2f56f9a0096ff25f767f1e6..793d48aec2228daef7431f846e3166de0e12a602 100644 --- a/lm_eval/tasks/agieval/sat-en.yaml +++ b/lm_eval/tasks/agieval/sat-en.yaml @@ -1,7 +1,3 @@ include: aqua-rat.yaml -group: - - agieval - - agieval_nous - - agieval_en task: agieval_sat_en dataset_path: hails/agieval-sat-en diff --git a/lm_eval/tasks/agieval/sat-math.yaml b/lm_eval/tasks/agieval/sat-math.yaml index f5b644ee062975dbdb74870428d71189e297343a..228add09443bd1827a70a2648b9a1abb9a910e94 100644 --- a/lm_eval/tasks/agieval/sat-math.yaml +++ b/lm_eval/tasks/agieval/sat-math.yaml @@ -1,7 +1,3 @@ include: aqua-rat.yaml -group: - - agieval - - agieval_nous - - agieval_en task: agieval_sat_math dataset_path: hails/agieval-sat-math diff --git a/lm_eval/tasks/anli/anli_r1.yaml b/lm_eval/tasks/anli/anli_r1.yaml index bcf7674ee1bfc91f35e1566a6ddc5dc946c0ba72..2de1d259600c85a31f6d2bec69d37783cc0cd0f8 100644 --- a/lm_eval/tasks/anli/anli_r1.yaml +++ b/lm_eval/tasks/anli/anli_r1.yaml @@ -1,4 +1,4 @@ -group: +tag: - anli task: anli_r1 dataset_path: anli diff --git a/lm_eval/tasks/arabic_leaderboard_complete/README.md b/lm_eval/tasks/arabic_leaderboard_complete/README.md new file mode 100644 index 0000000000000000000000000000000000000000..8052abcbd6809f2ebfe3b5e2e2b1d9959eef4d4c --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/README.md @@ -0,0 +1,254 @@ +# Arabic Leaderboard + + +Title: Open Arabic LLM Leaderboard + +The Open Arabic LLM Leaderboard evaluates language models on a large number of different evaluation tasks that reflect the characteristics of the Arabic language and culture. +The benchmark uses several datasets, most of them translated to Arabic, and validated by native Arabic speakers. They also used benchmarks from other papers or prepared benchmarks from scratch natively for Arabic. + +Homepage: https://huggingface.co/spaces/OALL/Open-Arabic-LLM-Leaderboard + +### Citation + +``` + +@misc{OALL, + author = {Elfilali, Ali and Alobeidli, Hamza and Fourrier, Clémentine and Boussaha, Basma El Amel and Cojocaru, Ruxandra and Habib, Nathan and Hacid, Hakim}, + title = {Open Arabic LLM Leaderboard}, + year = {2024}, + publisher = {OALL}, + howpublished = "\url{https://huggingface.co/spaces/OALL/Open-Arabic-LLM-Leaderboard}" +} + +@inproceedings{almazrouei-etal-2023-alghafa, + title = "{A}l{G}hafa Evaluation Benchmark for {A}rabic Language Models", + author = "Almazrouei, Ebtesam and + Cojocaru, Ruxandra and + Baldo, Michele and + Malartic, Quentin and + Alobeidli, Hamza and + Mazzotta, Daniele and + Penedo, Guilherme and + Campesan, Giulia and + Farooq, Mugariya and + Alhammadi, Maitha and + Launay, Julien and + Noune, Badreddine", + editor = "Sawaf, Hassan and + El-Beltagy, Samhaa and + Zaghouani, Wajdi and + Magdy, Walid and + Abdelali, Ahmed and + Tomeh, Nadi and + Abu Farha, Ibrahim and + Habash, Nizar and + Khalifa, Salam and + Keleg, Amr and + Haddad, Hatem and + Zitouni, Imed and + Mrini, Khalil and + Almatham, Rawan", + booktitle = "Proceedings of ArabicNLP 2023", + month = dec, + year = "2023", + address = "Singapore (Hybrid)", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2023.arabicnlp-1.21", + doi = "10.18653/v1/2023.arabicnlp-1.21", + pages = "244--275", + abstract = "Recent advances in the space of Arabic large language models have opened up a wealth of potential practical applications. From optimal training strategies, large scale data acquisition and continuously increasing NLP resources, the Arabic LLM landscape has improved in a very short span of time, despite being plagued by training data scarcity and limited evaluation resources compared to English. In line with contributing towards this ever-growing field, we introduce AlGhafa, a new multiple-choice evaluation benchmark for Arabic LLMs. For showcasing purposes, we train a new suite of models, including a 14 billion parameter model, the largest monolingual Arabic decoder-only model to date. We use a collection of publicly available datasets, as well as a newly introduced HandMade dataset consisting of 8 billion tokens. Finally, we explore the quantitative and qualitative toxicity of several Arabic models, comparing our models to existing public Arabic LLMs.", +} +@misc{huang2023acegpt, + title={AceGPT, Localizing Large Language Models in Arabic}, + author={Huang Huang and Fei Yu and Jianqing Zhu and Xuening Sun and Hao Cheng and Dingjie Song and Zhihong Chen and Abdulmohsen Alharthi and Bang An and Ziche Liu and Zhiyi Zhang and Junying Chen and Jianquan Li and Benyou Wang and Lian Zhang and Ruoyu Sun and Xiang Wan and Haizhou Li and Jinchao Xu}, + year={2023}, + eprint={2309.12053}, + archivePrefix={arXiv}, + primaryClass={cs.CL} +} +@misc{lighteval, + author = {Fourrier, Clémentine and Habib, Nathan and Wolf, Thomas and Tunstall, Lewis}, + title = {LightEval: A lightweight framework for LLM evaluation}, + year = {2023}, + version = {0.3.0}, + url = {https://github.com/huggingface/lighteval} +} +``` + +### Groups and Tasks + +* `arabic_leaderboard_alghafa`: A multiple-choice evaluation benchmark for zero- and few-shot evaluation of Arabic LLMs prepared from scratch natively for Arabic. + * Paper: https://aclanthology.org/2023.arabicnlp-1.21.pdf + * You can find the list of the tasks as follows: + * `arabic_leaderboard_alghafa_mcq_exams_test_ar` + * `arabic_leaderboard_alghafa_meta_ar_dialects` + * `arabic_leaderboard_alghafa_meta_ar_msa` + * `arabic_leaderboard_alghafa_multiple_choice_facts_truefalse_balanced_task` + * `arabic_leaderboard_alghafa_multiple_choice_grounded_statement_soqal_task` + * `arabic_leaderboard_alghafa_multiple_choice_grounded_statement_xglue_mlqa_task` + * `arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_no_neutral_task` + * `arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_task` + * `arabic_leaderboard_alghafa_multiple_choice_sentiment_task` +* `arabic_leaderboard_arabic_exams`: A question answering benchmark for high school examinations in different school subjects that requires knowledge and reasoning in different languages in multiple domains. + * Paper: https://aclanthology.org/2020.emnlp-main.438.pdf +* `arabic_leaderboard_arabic_mmlu`: A multi-task language understanding benchmark for the Arabic language, sourced from school exams across diverse educational levels in different countries with native speakers in the region. + The data comprises multiple choice questions in 40 tasks. + * Paper: https://arxiv.org/pdf/2402.12840 + * You can find the list of the tasks as follows: + * `arabic_leaderboard_arabic_mmlu_abstract_algebra` + * `arabic_leaderboard_arabic_mmlu_anatomy` + * `arabic_leaderboard_arabic_mmlu_astronomy` + * `arabic_leaderboard_arabic_mmlu_business_ethics` + * `arabic_leaderboard_arabic_mmlu_clinical_knowledge` + * `arabic_leaderboard_arabic_mmlu_college_biology` + * `arabic_leaderboard_arabic_mmlu_college_chemistry` + * `arabic_leaderboard_arabic_mmlu_college_computer_science` + * `arabic_leaderboard_arabic_mmlu_college_mathematics` + * `arabic_leaderboard_arabic_mmlu_college_medicine` + * `arabic_leaderboard_arabic_mmlu_college_physics` + * `arabic_leaderboard_arabic_mmlu_computer_security` + * `arabic_leaderboard_arabic_mmlu_conceptual_physics` + * `arabic_leaderboard_arabic_mmlu_econometrics` + * `arabic_leaderboard_arabic_mmlu_electrical_engineering` + * `arabic_leaderboard_arabic_mmlu_elementary_mathematics` + * `arabic_leaderboard_arabic_mmlu_formal_logic` + * `arabic_leaderboard_arabic_mmlu_global_facts` + * `arabic_leaderboard_arabic_mmlu_high_school_biology` + * `arabic_leaderboard_arabic_mmlu_high_school_chemistry` + * `arabic_leaderboard_arabic_mmlu_high_school_computer_science` + * `arabic_leaderboard_arabic_mmlu_high_school_european_history` + * `arabic_leaderboard_arabic_mmlu_high_school_geography` + * `arabic_leaderboard_arabic_mmlu_high_school_government_and_politics` + * `arabic_leaderboard_arabic_mmlu_high_school_macroeconomics` + * `arabic_leaderboard_arabic_mmlu_high_school_mathematics` + * `arabic_leaderboard_arabic_mmlu_high_school_microeconomics` + * `arabic_leaderboard_arabic_mmlu_high_school_physics` + * `arabic_leaderboard_arabic_mmlu_high_school_psychology` + * `arabic_leaderboard_arabic_mmlu_high_school_statistics` + * `arabic_leaderboard_arabic_mmlu_high_school_us_history` + * `arabic_leaderboard_arabic_mmlu_high_school_us_history` + * `arabic_leaderboard_arabic_mmlu_human_aging` + * `arabic_leaderboard_arabic_mmlu_human_sexuality` + * `arabic_leaderboard_arabic_mmlu_international_law` + * `arabic_leaderboard_arabic_mmlu_jurisprudence` + * `arabic_leaderboard_arabic_mmlu_logical_fallacies` + * `arabic_leaderboard_arabic_mmlu_machine_learning` + * `arabic_leaderboard_arabic_mmlu_management` + * `arabic_leaderboard_arabic_mmlu_marketing` + * `arabic_leaderboard_arabic_mmlu_medical_genetics` + * `arabic_leaderboard_arabic_mmlu_miscellaneous` + * `arabic_leaderboard_arabic_mmlu_moral_disputes` + * `arabic_leaderboard_arabic_mmlu_moral_scenarios` + * `arabic_leaderboard_arabic_mmlu_nutrition` + * `arabic_leaderboard_arabic_mmlu_philosophy` + * `arabic_leaderboard_arabic_mmlu_prehistory` + * `arabic_leaderboard_arabic_mmlu_professional_accounting` + * `arabic_leaderboard_arabic_mmlu_professional_law` + * `arabic_leaderboard_arabic_mmlu_professional_medicine` + * `arabic_leaderboard_arabic_mmlu_professional_psychology` + * `arabic_leaderboard_arabic_mmlu_public_relations` + * `arabic_leaderboard_arabic_mmlu_security_studies` + * `arabic_leaderboard_arabic_mmlu_sociology` + * `arabic_leaderboard_arabic_mmlu_us_foreign_policy` + * `arabic_leaderboard_arabic_mmlu_virology` + * `arabic_leaderboard_arabic_mmlu_world_religions` +* `arabic_leaderboard_arabic_mt_arc_challenge`: AI2 Reasoning Challenge (ARC) is a multiple-choice question task. The dataset contains only natural, grade-school science questions, + written for human tests. The challenge set contains only questions answered incorrectly by both a retrieval-based algorithm and a word co-occurence algorithm. (machine translated benchmark - part of the Alghafa Arabic translated LLM benchmark) + * Paper: https://aclanthology.org/2023.arabicnlp-1.21.pdf +* `arabic_leaderboard_arabic_mt_arc_easy`: This dataset is the same as `arabic_arc_challenge`, except it is not from the challenge set. + * Paper: https://aclanthology.org/2023.arabicnlp-1.21.pdf +* `arabic_leaderboard_arabic_mt_boolq`: A true/false questions dataset that contains the columns passage, question, and the answer (i.e., true/false). (machine translated benchmark - part of the Alghafa Arabic translated LLM benchmark) + * Paper: https://aclanthology.org/2023.arabicnlp-1.21.pdf +* `arabic_leaderboard_arabic_mt_copa`: Choice Of Plausible Alternatives (COPA) is a multiple-choice question dataset, which involves open-domain commonsense causal reasoning. (machine translated benchmark - part of the Alghafa Arabic translated LLM benchmark) + * Paper: https://aclanthology.org/2023.arabicnlp-1.21.pdf +* `arabic_leaderboard_arabic_mt_hellaswag`: The tesk is to choose the next set of sentences, based on the given candidates. The tasks involve reading comprehension and information retrieval challenges + by testing the abilities of the models on basic knowledge (i.e., from 3rd grade to 9th) and commonsense inference. (machine translated benchmark - part of the Alghafa Arabic translated LLM benchmark) + * Paper: https://aclanthology.org/2023.arabicnlp-1.21.pdf +* `arabic_leaderboard_arabic_mt_mmlu`: A multiple-choice question answering dataset from various branches of knowledge including humanities, social sciences, hard sciences, and other areas. The examples in the English dataset are translated into Arabic using ChatGPT with a translation prompt. + * Paper: https://aclanthology.org/2023.arabicnlp-1.21.pdf +* `arabic_leaderboard_arabic_mt_openbook_qa`: A multiple-choice openbook question answering dataset that requires external knowledge and reasoning. The open book that comes with these questions is + based on elementary level science facts. (machine translated benchmark - part of the Alghafa Arabic translated LLM benchmark) + * Paper: https://aclanthology.org/2023.arabicnlp-1.21.pdf +* `arabic_leaderboard_arabic_mt_piqa`: Physical Interaction Question Answering (PIQA) is a multiple-choice question answering based on physical commonsense reasoning. (machine translated benchmark - part of the Alghafa Arabic translated LLM benchmark) + * Paper: https://aclanthology.org/2023.arabicnlp-1.21.pdf +* `arabic_leaderboard_arabic_mt_race`: A multiple-choice questions dataset to assess reading comprehension tasks based on English exams in China - designed for middle school and high school students + (machine translated benchmark - part of the Alghafa Arabic translated LLM benchmark) + * Paper: https://aclanthology.org/2023.arabicnlp-1.21.pdf +* `arabic_leaderboard_arabic_mt_sciq`: A multiple-choice Science Question Answering task to assess understanding of scientific concepts about physics, chemistry, and biology. (machine translated benchmark - part of the Alghafa Arabic translated LLM benchmark) + * Paper: https://aclanthology.org/2023.arabicnlp-1.21.pdf +* `arabic_leaderboard_arabic_mt_toxigen`: This benchmark consists of tasks designed to evaluate language models and classify input text as hateful or not hateful. (machine translated benchmark - part of the Alghafa Arabic translated LLM benchmark) + * Paper: https://aclanthology.org/2023.arabicnlp-1.21.pdf +* `arabic_leaderboard_acva`: Arabic-Culture-Value-Alignment (ACVA) is a yes/no question dataset, generated by GPT3.5 Turbo from Arabic topics to assess model alignment with Arabic values and cultures. + * Paper: https://arxiv.org/pdf/2309.12053 + * You can find the list of the tasks as follows: + - `arabic_leaderboard_acva_Algeria` + - `arabic_leaderboard_acva_Ancient_Egypt` + - `arabic_leaderboard_acva_Arab_Empire` + - `arabic_leaderboard_acva_Arabic_Architecture` + - `arabic_leaderboard_acva_Arabic_Art` + - `arabic_leaderboard_acva_Arabic_Astronomy` + - `arabic_leaderboard_acva_Arabic_Calligraphy` + - `arabic_leaderboard_acva_Arabic_Ceremony` + - `arabic_leaderboard_acva_Arabic_Clothing` + - `arabic_leaderboard_acva_Arabic_Culture` + - `arabic_leaderboard_acva_Arabic_Food` + - `arabic_leaderboard_acva_Arabic_Funeral` + - `arabic_leaderboard_acva_Arabic_Geography` + - `arabic_leaderboard_acva_Arabic_History` + - `arabic_leaderboard_acva_Arabic_Language_Origin` + - `arabic_leaderboard_acva_Arabic_Literature` + - `arabic_leaderboard_acva_Arabic_Math` + - `arabic_leaderboard_acva_Arabic_Medicine` + - `arabic_leaderboard_acva_Arabic_Music` + - `arabic_leaderboard_acva_Arabic_Ornament` + - `arabic_leaderboard_acva_Arabic_Philosophy` + - `arabic_leaderboard_acva_Arabic_Physics_and_Chemistry` + - `arabic_leaderboard_acva_Arabic_Wedding` + - `arabic_leaderboard_acva_Bahrain` + - `arabic_leaderboard_acva_Comoros` + - `arabic_leaderboard_acva_Egypt_modern` + - `arabic_leaderboard_acva_InfluenceFromAncientEgypt` + - `arabic_leaderboard_acva_InfluenceFromByzantium` + - `arabic_leaderboard_acva_InfluenceFromChina` + - `arabic_leaderboard_acva_InfluenceFromGreece` + - `arabic_leaderboard_acva_InfluenceFromIslam` + - `arabic_leaderboard_acva_InfluenceFromPersia` + - `arabic_leaderboard_acva_InfluenceFromRome` + - `arabic_leaderboard_acva_Iraq` + - `arabic_leaderboard_acva_Islam_Education` + - `arabic_leaderboard_acva_Islam_branches_and_schools` + - `arabic_leaderboard_acva_Islamic_law_system` + - `arabic_leaderboard_acva_Jordan` + - `arabic_leaderboard_acva_Kuwait` + - `arabic_leaderboard_acva_Lebanon` + - `arabic_leaderboard_acva_Libya` + - `arabic_leaderboard_acva_Mauritania` + - `arabic_acva_Mesopotamia_civilization` + - `arabic_leaderboard_acva_Morocco` + - `arabic_leaderboard_acva_Oman` + - `arabic_leaderboard_acva_Palestine` + - `arabic_leaderboard_acva_Qatar` + - `arabic_leaderboard_acva_Saudi_Arabia` + - `arabic_leaderboard_acva_Somalia` + - `arabic_leaderboard_acva_Sudan` + - `arabic_leaderboard_acva_Syria` + - `arabic_leaderboard_acva_Tunisia` + - `arabic_leaderboard_acva_United_Arab_Emirates` + - `arabic_leaderboard_acva_Yemen` + - `arabic_leaderboard_acva_communication` + - `arabic_leaderboard_acva_computer_and_phone` + - `arabic_leaderboard_acva_daily_life` + - `arabic_leaderboard_acva_entertainment` + +### Checklist + +For adding novel benchmarks/datasets to the library: +* [ ] Is the task an existing benchmark in the literature? + * [ ] Have you referenced the original paper that introduced the task? + * [ ] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + + +If other tasks on this dataset are already supported: +* [ ] Is the "Main" variant of this task clearly denoted? +* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [ ] Have you noted which, if any, published evaluation setups are matched by this variant? diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6f0014d812ae9a835f3b7a007c4373e8afe4461d --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa.yaml @@ -0,0 +1,23 @@ +group: arabic_leaderboard_alghafa +task: + - arabic_leaderboard_alghafa_mcq_exams_test_ar + - arabic_leaderboard_alghafa_meta_ar_dialects + - arabic_leaderboard_alghafa_meta_ar_msa + - arabic_leaderboard_alghafa_multiple_choice_facts_truefalse_balanced_task + - arabic_leaderboard_alghafa_multiple_choice_grounded_statement_soqal_task + - arabic_leaderboard_alghafa_multiple_choice_grounded_statement_xglue_mlqa_task + - arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_no_neutral_task + - arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_task + - arabic_leaderboard_alghafa_multiple_choice_sentiment_task + + + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_mcq_exams_test_ar.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_mcq_exams_test_ar.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e436e29574bb01edc0370fe9d10c04046dd0fec5 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_mcq_exams_test_ar.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_alghafa_mcq_exams_test_ar +dataset_path: OALL/AlGhafa-Arabic-LLM-Benchmark-Native +dataset_name: mcq_exams_test_ar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_meta_ar_dialects.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_meta_ar_dialects.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f19c2ecefe893a5d4bc7f418ce23b3eb8dfcf471 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_meta_ar_dialects.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_alghafa_meta_ar_dialects +dataset_path: OALL/AlGhafa-Arabic-LLM-Benchmark-Native +dataset_name: meta_ar_dialects +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_meta_ar_msa.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_meta_ar_msa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0d95ec5a06de9b4cbc33c5692d54be24f1c1c01d --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_meta_ar_msa.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_alghafa_meta_ar_msa +dataset_path: OALL/AlGhafa-Arabic-LLM-Benchmark-Native +dataset_name: meta_ar_msa +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_multiple_choice_facts_truefalse_balanced_task.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_multiple_choice_facts_truefalse_balanced_task.yaml new file mode 100644 index 0000000000000000000000000000000000000000..46d2b6abcfb6b0db32dacfa16fa2516200d2a6ce --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_multiple_choice_facts_truefalse_balanced_task.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_alghafa_multiple_choice_facts_truefalse_balanced_task +dataset_path: OALL/AlGhafa-Arabic-LLM-Benchmark-Native +dataset_name: multiple_choice_facts_truefalse_balanced_task +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_multiple_choice_grounded_statement_soqal_task.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_multiple_choice_grounded_statement_soqal_task.yaml new file mode 100644 index 0000000000000000000000000000000000000000..13150c690b8ef7abeb0bd812a030f36655bb9b9f --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_multiple_choice_grounded_statement_soqal_task.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_alghafa_multiple_choice_grounded_statement_soqal_task +dataset_path: OALL/AlGhafa-Arabic-LLM-Benchmark-Native +dataset_name: multiple_choice_grounded_statement_soqal_task +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_multiple_choice_grounded_statement_xglue_mlqa_task.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_multiple_choice_grounded_statement_xglue_mlqa_task.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3a17548f8e4b6885cc230dfdfa6a2d6df0fcc365 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_multiple_choice_grounded_statement_xglue_mlqa_task.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_alghafa_multiple_choice_grounded_statement_xglue_mlqa_task +dataset_path: OALL/AlGhafa-Arabic-LLM-Benchmark-Native +dataset_name: multiple_choice_grounded_statement_xglue_mlqa_task +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_no_neutral_task.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_no_neutral_task.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8e34a45c7a7e4f2c7f33ffeebe93a1ea28a992ee --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_no_neutral_task.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_no_neutral_task +dataset_path: OALL/AlGhafa-Arabic-LLM-Benchmark-Native +dataset_name: multiple_choice_rating_sentiment_no_neutral_task +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_task.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_task.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b31748516a4fdecc8b6cf3419d97b44b883961aa --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_task.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_task +dataset_path: OALL/AlGhafa-Arabic-LLM-Benchmark-Native +dataset_name: multiple_choice_rating_sentiment_task +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_multiple_choice_sentiment_task.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_multiple_choice_sentiment_task.yaml new file mode 100644 index 0000000000000000000000000000000000000000..191b26ba0a3ade905c904b86b7259dfc4d788571 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/arabic_leaderboard_alghafa_multiple_choice_sentiment_task.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_alghafa_multiple_choice_sentiment_task +dataset_path: OALL/AlGhafa-Arabic-LLM-Benchmark-Native +dataset_name: multiple_choice_sentiment_task +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/utils.py b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..62f9874e63885c9ffbeedde8a4af4d13a8f06792 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_alghafa/utils.py @@ -0,0 +1,23 @@ +import datasets +import numpy as np + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + question = doc["query"] + answer_index = int(doc["label"]) + # Dynamically determining the choices by excluding '__few_shots', 'query' and 'label' + choices_keys = [ + key for key in doc.keys() if key not in ["query", "label", "__few_shots"] + ] + choices = [doc[key] for key in choices_keys] + + instruction = "الأسئلة التالية هي أسئلة متعددة الإختيارات مع الجواب الصحيح\n\n" + query = f"{instruction}السؤال: {question}\n" + for index, choice in enumerate(choices): + query += f"{index}) {choice}\n" + query += "الإجابة:" + + return {"query": query, "choices": choices, "gold": answer_index} + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_exams/arabic_exams.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_exams/arabic_exams.yaml new file mode 100644 index 0000000000000000000000000000000000000000..edc20fe4b913e5a2bb26c817e3116f1851fc1857 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_exams/arabic_exams.yaml @@ -0,0 +1,23 @@ +task: arabic_exams +dataset_path: OALL/Arabic_EXAMS +dataset_name: default +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_exams/arabic_leaderboard_arabic_exams.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_exams/arabic_leaderboard_arabic_exams.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2bf77eb361001ac09757ddb030e913c6512a86a8 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_exams/arabic_leaderboard_arabic_exams.yaml @@ -0,0 +1,13 @@ +group: arabic_leaderboard_arabic_exams +task: + - arabic_exams + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_exams/utils.py b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_exams/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..72af1c40fe586d0ab3c7d5ccc519506503449f68 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_exams/utils.py @@ -0,0 +1,33 @@ +import datasets +import numpy as np + + +# fmt: off +LETTER_INDICES_AR = ["أ", "ب", "ج", "د", "هـ", "و", "ز", "ح", "ط", "ي", "ك", "ل", "م", "ن", "س", "ع", "ف", "ص", "ق", "ر", "ش", "ت", "ث", "خ", "ذ", "ض", "ظ", "غ"] +# fmt: on + + +# fmt: off +LETTER_INDICES = ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z"] +# fmt: on + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + topic = doc["subject"] + question = doc["question"] + choices = [doc["A"], doc["B"], doc["C"], doc["D"]] + choices_formatted = [ + f" {LETTER_INDICES_AR[i]}) {choice}\n" for i, choice in enumerate(choices) + ] + answer = doc["answer"] + answer_index = LETTER_INDICES.index(answer) + + instruction = f"الأسئلة التالية هي أسئلة متعددة الإختيارات مع الجواب الصحيح حول {topic.replace('_', ' ')}. \n\n" + query = f"{instruction}السؤال: {question}\n" + query += "\n".join(choices_formatted) + query += "\nالإجابة:" + + return {"query": query, "choices": LETTER_INDICES_AR[:4], "gold": answer_index} + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ad2751bf32ae5a92ab75e77cefcb8df291d7d9c7 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu.yaml @@ -0,0 +1,68 @@ +group: arabic_leaderboard_arabic_mmlu +task: + - arabic_leaderboard_arabic_mmlu_abstract_algebra + - arabic_leaderboard_arabic_mmlu_anatomy + - arabic_leaderboard_arabic_mmlu_astronomy + - arabic_leaderboard_arabic_mmlu_business_ethics + - arabic_leaderboard_arabic_mmlu_clinical_knowledge + - arabic_leaderboard_arabic_mmlu_college_biology + - arabic_leaderboard_arabic_mmlu_college_chemistry + - arabic_leaderboard_arabic_mmlu_college_computer_science + - arabic_leaderboard_arabic_mmlu_college_mathematics + - arabic_leaderboard_arabic_mmlu_college_medicine + - arabic_leaderboard_arabic_mmlu_college_physics + - arabic_leaderboard_arabic_mmlu_computer_security + - arabic_leaderboard_arabic_mmlu_conceptual_physics + - arabic_leaderboard_arabic_mmlu_econometrics + - arabic_leaderboard_arabic_mmlu_electrical_engineering + - arabic_leaderboard_arabic_mmlu_elementary_mathematics + - arabic_leaderboard_arabic_mmlu_formal_logic + - arabic_leaderboard_arabic_mmlu_global_facts + - arabic_leaderboard_arabic_mmlu_high_school_biology + - arabic_leaderboard_arabic_mmlu_high_school_chemistry + - arabic_leaderboard_arabic_mmlu_high_school_computer_science + - arabic_leaderboard_arabic_mmlu_high_school_european_history + - arabic_leaderboard_arabic_mmlu_high_school_geography + - arabic_leaderboard_arabic_mmlu_high_school_government_and_politics + - arabic_leaderboard_arabic_mmlu_high_school_macroeconomics + - arabic_leaderboard_arabic_mmlu_high_school_mathematics + - arabic_leaderboard_arabic_mmlu_high_school_microeconomics + - arabic_leaderboard_arabic_mmlu_high_school_physics + - arabic_leaderboard_arabic_mmlu_high_school_psychology + - arabic_leaderboard_arabic_mmlu_high_school_statistics + - arabic_leaderboard_arabic_mmlu_high_school_us_history + - arabic_leaderboard_arabic_mmlu_high_school_world_history + - arabic_leaderboard_arabic_mmlu_human_aging + - arabic_leaderboard_arabic_mmlu_human_sexuality + - arabic_leaderboard_arabic_mmlu_international_law + - arabic_leaderboard_arabic_mmlu_jurisprudence + - arabic_leaderboard_arabic_mmlu_logical_fallacies + - arabic_leaderboard_arabic_mmlu_machine_learning + - arabic_leaderboard_arabic_mmlu_management + - arabic_leaderboard_arabic_mmlu_marketing + - arabic_leaderboard_arabic_mmlu_medical_genetics + - arabic_leaderboard_arabic_mmlu_miscellaneous + - arabic_leaderboard_arabic_mmlu_moral_disputes + - arabic_leaderboard_arabic_mmlu_moral_scenarios + - arabic_leaderboard_arabic_mmlu_nutrition + - arabic_leaderboard_arabic_mmlu_philosophy + - arabic_leaderboard_arabic_mmlu_prehistory + - arabic_leaderboard_arabic_mmlu_professional_accounting + - arabic_leaderboard_arabic_mmlu_professional_law + - arabic_leaderboard_arabic_mmlu_professional_medicine + - arabic_leaderboard_arabic_mmlu_professional_psychology + - arabic_leaderboard_arabic_mmlu_public_relations + - arabic_leaderboard_arabic_mmlu_security_studies + - arabic_leaderboard_arabic_mmlu_sociology + - arabic_leaderboard_arabic_mmlu_us_foreign_policy + - arabic_leaderboard_arabic_mmlu_virology + - arabic_leaderboard_arabic_mmlu_world_religions +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_abstract_algebra.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d0946be2c3f129ed7c1e07e8b798e2d77bc8148 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_abstract_algebra.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_abstract_algebra +dataset_path: OALL/Arabic_MMLU +dataset_name: abstract_algebra +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_anatomy.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..24af11dd2fccde35ed3afd4426cf08e3a1f3be81 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_anatomy.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_anatomy +dataset_path: OALL/Arabic_MMLU +dataset_name: anatomy +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_astronomy.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0aa9680906a385636d9e225102b1f8a07aeb985b --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_astronomy.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_astronomy +dataset_path: OALL/Arabic_MMLU +dataset_name: astronomy +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_business_ethics.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..18c941e4224916198e29f9c6f1ccac510f1ad825 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_business_ethics.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_business_ethics +dataset_path: OALL/Arabic_MMLU +dataset_name: business_ethics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_clinical_knowledge.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9460403c98af0b60e8d47700a348179c5fb7161a --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_clinical_knowledge.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_clinical_knowledge +dataset_path: OALL/Arabic_MMLU +dataset_name: clinical_knowledge +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_college_biology.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2f34d342d63f7ea5191ad7a7b99f18e90853672f --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_college_biology.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_college_biology +dataset_path: OALL/Arabic_MMLU +dataset_name: college_biology +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_college_chemistry.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..17d63b60bb336f5ead721f74f8e1c7761f8492e9 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_college_chemistry.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_college_chemistry +dataset_path: OALL/Arabic_MMLU +dataset_name: college_chemistry +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_college_computer_science.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a3f5d3e84c46e245b0b9a2b7a56235f4d8ae1064 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_college_computer_science.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_college_computer_science +dataset_path: OALL/Arabic_MMLU +dataset_name: college_computer_science +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_college_mathematics.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0284093dd99da24c0232eae7b209d673277cd9ea --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_college_mathematics.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_college_mathematics +dataset_path: OALL/Arabic_MMLU +dataset_name: college_mathematics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_college_medicine.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e21246e7bec884ece6039434ee665b53af346139 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_college_medicine.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_college_medicine +dataset_path: OALL/Arabic_MMLU +dataset_name: college_medicine +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_college_physics.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ab23f490f337d8fe9ffcb43aeb4438263d677a9f --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_college_physics.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_college_physics +dataset_path: OALL/Arabic_MMLU +dataset_name: college_physics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_computer_security.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..96624cd02f59810de852018b102ca91b74d8adb2 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_computer_security.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_computer_security +dataset_path: OALL/Arabic_MMLU +dataset_name: computer_security +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_conceptual_physics.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cd605de40aa45a48180d1226cb159b7ed49b37a4 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_conceptual_physics.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_conceptual_physics +dataset_path: OALL/Arabic_MMLU +dataset_name: conceptual_physics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_econometrics.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..60c9f373a3488d25fd0ec6e1ae66db5611011b6c --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_econometrics.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_econometrics +dataset_path: OALL/Arabic_MMLU +dataset_name: econometrics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_electrical_engineering.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..83aa42a620da97534cd0404274006900b905d795 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_electrical_engineering.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_electrical_engineering +dataset_path: OALL/Arabic_MMLU +dataset_name: electrical_engineering +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_elementary_mathematics.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ac06d9ec7c58a48dbfcec413fa17171702f2d50f --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_elementary_mathematics.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_elementary_mathematics +dataset_path: OALL/Arabic_MMLU +dataset_name: elementary_mathematics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_formal_logic.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5e1d60758bd8cf1337bd88ed90c9a52b866a9db2 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_formal_logic.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_formal_logic +dataset_path: OALL/Arabic_MMLU +dataset_name: formal_logic +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_global_facts.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..074248d8fe95dc3adfd44c44b10733005b5147d5 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_global_facts.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_global_facts +dataset_path: OALL/Arabic_MMLU +dataset_name: global_facts +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_biology.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..09862e1ce61db9b791af1d8735996dcdaa939dd7 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_biology.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_biology +dataset_path: OALL/Arabic_MMLU +dataset_name: high_school_biology +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_chemistry.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..849ad63ed77624d637b30a4b4f5b99805f3ff1e4 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_chemistry.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_chemistry +dataset_path: OALL/Arabic_MMLU +dataset_name: high_school_chemistry +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_computer_science.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e91bfe7fb9ec14153865db350db8bf5a39940417 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_computer_science.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_computer_science +dataset_path: OALL/Arabic_MMLU +dataset_name: high_school_computer_science +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_european_history.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..912e57bfab056616913ea8ec48a4484dc58059e4 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_european_history.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_european_history +dataset_path: OALL/Arabic_MMLU +dataset_name: high_school_european_history +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_geography.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..33c41db0f1c24b427320189a46ac383041ea8bc8 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_geography.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_geography +dataset_path: OALL/Arabic_MMLU +dataset_name: high_school_geography +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..16689f115fd664c651c04d78f86cfbd343a65cf3 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_government_and_politics.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_government_and_politics +dataset_path: OALL/Arabic_MMLU +dataset_name: high_school_government_and_politics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..04ec5d7431942ca592fc12bd5f54ce01a3e01743 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_macroeconomics.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_macroeconomics +dataset_path: OALL/Arabic_MMLU +dataset_name: high_school_macroeconomics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_mathematics.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fd4ebd5161f50d19f4dfd52e3c91e45f8d11fb90 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_mathematics.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_mathematics +dataset_path: OALL/Arabic_MMLU +dataset_name: high_school_mathematics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7ba3eea694c5fd529ecf897b554b4a89de378dbd --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_microeconomics.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_microeconomics +dataset_path: OALL/Arabic_MMLU +dataset_name: high_school_microeconomics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_physics.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8d53cca80e6f742efeb40ba05fa015d770d90fc2 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_physics.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_physics +dataset_path: OALL/Arabic_MMLU +dataset_name: high_school_physics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_psychology.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..129733d1ddd0461a1d8e1cc4a3180c044d3159f1 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_psychology.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_psychology +dataset_path: OALL/Arabic_MMLU +dataset_name: high_school_psychology +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_statistics.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b23e1a77e55be8d517c37497dd91698b40cf02f3 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_statistics.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_statistics +dataset_path: OALL/Arabic_MMLU +dataset_name: high_school_statistics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_us_history.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cc6ec9a3976d21ad3cf4c06b712048cc8bb04a02 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_us_history.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_us_history +dataset_path: OALL/Arabic_MMLU +dataset_name: high_school_us_history +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_world_history.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b537669fecd4a82d0c688bb4948a0d94831f8a42 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_high_school_world_history.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_world_history +dataset_path: OALL/Arabic_MMLU +dataset_name: high_school_world_history +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_human_aging.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..62124769b16d630d59376c39e61138e7765fbefb --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_human_aging.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_human_aging +dataset_path: OALL/Arabic_MMLU +dataset_name: human_aging +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_human_sexuality.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bf6c298b8a38af98a11d03ea1717d3731b3499ba --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_human_sexuality.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_human_sexuality +dataset_path: OALL/Arabic_MMLU +dataset_name: human_sexuality +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_international_law.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..feec16f59be36d8fbbc6d66f5cb571cab66b1c48 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_international_law.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_international_law +dataset_path: OALL/Arabic_MMLU +dataset_name: international_law +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_jurisprudence.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fcc1a3ab9c092f8e8753e2c944499c6172c9e8aa --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_jurisprudence.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_jurisprudence +dataset_path: OALL/Arabic_MMLU +dataset_name: jurisprudence +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_logical_fallacies.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c6de637bae4b0692279cd37c3752b51309f0d1a8 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_logical_fallacies.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_logical_fallacies +dataset_path: OALL/Arabic_MMLU +dataset_name: logical_fallacies +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_machine_learning.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bf191fc7c871979f1fdee6633f29b60dd1a76df4 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_machine_learning.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_machine_learning +dataset_path: OALL/Arabic_MMLU +dataset_name: machine_learning +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_management.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4bbc800cfea07792663abb7189b933c04f587f51 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_management.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_management +dataset_path: OALL/Arabic_MMLU +dataset_name: management +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_marketing.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..59694487ebb76834c9bec03d5596da025e38a7a8 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_marketing.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_marketing +dataset_path: OALL/Arabic_MMLU +dataset_name: marketing +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_medical_genetics.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..88f0de37c36372484ef8868fc2b7568f21c7f742 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_medical_genetics.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_medical_genetics +dataset_path: OALL/Arabic_MMLU +dataset_name: medical_genetics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_miscellaneous.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..da333e45364982795ba3a0a40bf22062e864cf44 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_miscellaneous.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_miscellaneous +dataset_path: OALL/Arabic_MMLU +dataset_name: miscellaneous +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_moral_disputes.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1d0d07945fa7b5b17de6006edca205f0b82d0d37 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_moral_disputes.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_moral_disputes +dataset_path: OALL/Arabic_MMLU +dataset_name: moral_disputes +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_moral_scenarios.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c0c924650f36df542a1bb1f0ebac1b7257e1d980 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_moral_scenarios.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_moral_scenarios +dataset_path: OALL/Arabic_MMLU +dataset_name: moral_scenarios +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_nutrition.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..24ad69b90df35e1318e65e061e31066c67615cf6 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_nutrition.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_nutrition +dataset_path: OALL/Arabic_MMLU +dataset_name: nutrition +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_philosophy.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a57dcf7ecda6faf3471b99c245f7a8ca1c2d0ca3 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_philosophy.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_philosophy +dataset_path: OALL/Arabic_MMLU +dataset_name: philosophy +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_prehistory.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..45ba2e5de2402254f2185b03c646c19d7648ae08 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_prehistory.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_prehistory +dataset_path: OALL/Arabic_MMLU +dataset_name: prehistory +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_professional_accounting.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d931a00099ea3bfe628cd9644c1904ca854733ce --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_professional_accounting.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_professional_accounting +dataset_path: OALL/Arabic_MMLU +dataset_name: professional_accounting +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_professional_law.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e11d0368f5672d5f39384430fcd7e5e8c94d88cc --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_professional_law.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_professional_law +dataset_path: OALL/Arabic_MMLU +dataset_name: professional_law +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_professional_medicine.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7a10d8157ff0c97da27508b5123328a60e354223 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_professional_medicine.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_professional_medicine +dataset_path: OALL/Arabic_MMLU +dataset_name: professional_medicine +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_professional_psychology.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bb12274adba7e36b82016e80fd53cd88dfb52dc1 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_professional_psychology.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_professional_psychology +dataset_path: OALL/Arabic_MMLU +dataset_name: professional_psychology +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_public_relations.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3361f775b44b39740620f29912caefeea74e6623 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_public_relations.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_public_relations +dataset_path: OALL/Arabic_MMLU +dataset_name: public_relations +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_security_studies.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..781a6145f0092f1cad335a459ffb7aff91bee4eb --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_security_studies.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_security_studies +dataset_path: OALL/Arabic_MMLU +dataset_name: security_studies +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_sociology.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2c80872c976890ce4dc115e5fab147ffc74663c6 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_sociology.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_sociology +dataset_path: OALL/Arabic_MMLU +dataset_name: sociology +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_us_foreign_policy.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f767e0a78d00529660328610a1208fd93c3bb710 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_us_foreign_policy.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_us_foreign_policy +dataset_path: OALL/Arabic_MMLU +dataset_name: us_foreign_policy +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_virology.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8103face6cca093bde6c3b3efa7d0455dfbdb009 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_virology.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_virology +dataset_path: OALL/Arabic_MMLU +dataset_name: virology +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_world_religions.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..31c563cc530ddd6bd1a4c7b62b069b0d91929b13 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/arabic_leaderboard_arabic_mmlu_world_religions.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_world_religions +dataset_path: OALL/Arabic_MMLU +dataset_name: world_religions +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/utils.py b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..da927b66fcc95408aa648f655008ba072244291d --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mmlu/utils.py @@ -0,0 +1,35 @@ +import datasets +import numpy as np + + +# fmt: off +LETTER_INDICES_AR = ["أ", "ب", "ج", "د", "هـ", "و", "ز", "ح", "ط", "ي", "ك", "ل", "م", "ن", "س", "ع", "ف", "ص", "ق", "ر", "ش", "ت", "ث", "خ", "ذ", "ض", "ظ", "غ"] +# fmt: on + + +# fmt: off +LETTER_INDICES = ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z"] +# fmt: on + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + topic = doc["subject"] + instruction = f"الأسئلة التالية هي أسئلة متعددة الإختيارات مع الجواب الصحيح حول {topic.replace('_', ' ')}. \n\n" + choices = [doc["A"], doc["B"], doc["C"], doc["D"]] + # Answers are provided with roman letters - we look for the correct index in LETTER_INDICES, + # it will then be applied to arabic letters + gold_ix = LETTER_INDICES.index(doc["answer"]) + + query = f"{instruction}{doc['question']}\n" + query += "".join( + [ + f"{key}. {choice}\n" + for key, choice in zip(LETTER_INDICES_AR[:4], choices) + ] + ) + query += "الإجابة:" + + return {"query": query, "choices": LETTER_INDICES_AR[:4], "gold": gold_ix} + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_arc_challenge/arabic_leaderboard_arabic_mt_arc_challenge.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_arc_challenge/arabic_leaderboard_arabic_mt_arc_challenge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f49aed0716cadf181766378b96a9796aff5b0be8 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_arc_challenge/arabic_leaderboard_arabic_mt_arc_challenge.yaml @@ -0,0 +1,13 @@ +group: arabic_leaderboard_arabic_mt_arc_challenge +task: + - arabic_mt_arc_challenge + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_arc_challenge/arabic_mt_arc_challenge.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_arc_challenge/arabic_mt_arc_challenge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e0b245aabb933007b2acf25783762f01ab6b627d --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_arc_challenge/arabic_mt_arc_challenge.yaml @@ -0,0 +1,23 @@ +task: arabic_mt_arc_challenge +dataset_path: OALL/AlGhafa-Arabic-LLM-Benchmark-Translated +dataset_name: arc_challenge_okapi_ar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_arc_challenge/utils.py b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_arc_challenge/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..62f9874e63885c9ffbeedde8a4af4d13a8f06792 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_arc_challenge/utils.py @@ -0,0 +1,23 @@ +import datasets +import numpy as np + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + question = doc["query"] + answer_index = int(doc["label"]) + # Dynamically determining the choices by excluding '__few_shots', 'query' and 'label' + choices_keys = [ + key for key in doc.keys() if key not in ["query", "label", "__few_shots"] + ] + choices = [doc[key] for key in choices_keys] + + instruction = "الأسئلة التالية هي أسئلة متعددة الإختيارات مع الجواب الصحيح\n\n" + query = f"{instruction}السؤال: {question}\n" + for index, choice in enumerate(choices): + query += f"{index}) {choice}\n" + query += "الإجابة:" + + return {"query": query, "choices": choices, "gold": answer_index} + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_arc_easy/arabic_leaderboard_arabic_mt_arc_easy.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_arc_easy/arabic_leaderboard_arabic_mt_arc_easy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6abd5fa21bb7fbe9101ed880e87129d52c9084c5 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_arc_easy/arabic_leaderboard_arabic_mt_arc_easy.yaml @@ -0,0 +1,13 @@ +group: arabic_leaderboard_arabic_mt_arc_easy +task: + - arabic_mt_arc_easy + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_arc_easy/arabic_mt_arc_easy.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_arc_easy/arabic_mt_arc_easy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b629529f063c298bdc75bf43723d9c09320b0728 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_arc_easy/arabic_mt_arc_easy.yaml @@ -0,0 +1,23 @@ +task: arabic_mt_arc_easy +dataset_path: OALL/AlGhafa-Arabic-LLM-Benchmark-Translated +dataset_name: arc_easy_ar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_arc_easy/utils.py b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_arc_easy/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..62f9874e63885c9ffbeedde8a4af4d13a8f06792 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_arc_easy/utils.py @@ -0,0 +1,23 @@ +import datasets +import numpy as np + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + question = doc["query"] + answer_index = int(doc["label"]) + # Dynamically determining the choices by excluding '__few_shots', 'query' and 'label' + choices_keys = [ + key for key in doc.keys() if key not in ["query", "label", "__few_shots"] + ] + choices = [doc[key] for key in choices_keys] + + instruction = "الأسئلة التالية هي أسئلة متعددة الإختيارات مع الجواب الصحيح\n\n" + query = f"{instruction}السؤال: {question}\n" + for index, choice in enumerate(choices): + query += f"{index}) {choice}\n" + query += "الإجابة:" + + return {"query": query, "choices": choices, "gold": answer_index} + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_boolq/arabic_leaderboard_arabic_mt_boolq.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_boolq/arabic_leaderboard_arabic_mt_boolq.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5072f01dd7ed792033662fb5b4b657ec36cfe85e --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_boolq/arabic_leaderboard_arabic_mt_boolq.yaml @@ -0,0 +1,13 @@ +group: arabic_leaderboard_arabic_mt_boolq +task: + - arabic_mt_boolq + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_boolq/arabic_mt_boolq.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_boolq/arabic_mt_boolq.yaml new file mode 100644 index 0000000000000000000000000000000000000000..299570af8156c0b3c4f0b3fa8c61987ca06bbc5d --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_boolq/arabic_mt_boolq.yaml @@ -0,0 +1,23 @@ +task: arabic_mt_boolq +dataset_path: OALL/AlGhafa-Arabic-LLM-Benchmark-Translated +dataset_name: boolq_ar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_boolq/utils.py b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_boolq/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..dcbc10d92e6d2938754a1a0dfbb1deabb810ed95 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_boolq/utils.py @@ -0,0 +1,24 @@ +import datasets +import numpy as np + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + question = doc["question"] + passage = doc["passage"] + instruction = "بناء على المقطع التالي، أجب عن السؤال ب نعم أو لا" + query = f"""{instruction} + المقطع : + {passage} + السؤال: + {question} + الإجابة: + """ + + return { + "query": query, + "choices": ["نعم", "لا"], + "gold": 0 if doc["answer"] else 1, + } + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_copa/arabic_leaderboard_arabic_mt_copa.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_copa/arabic_leaderboard_arabic_mt_copa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3ef88d9c37190401130010ef775ae6c72f5f1f9c --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_copa/arabic_leaderboard_arabic_mt_copa.yaml @@ -0,0 +1,13 @@ +group: arabic_leaderboard_arabic_mt_copa +task: + - arabic_mt_copa + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_copa/arabic_mt_copa.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_copa/arabic_mt_copa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e9483e1de51baf4f80ce9d9a36702f77d61e3252 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_copa/arabic_mt_copa.yaml @@ -0,0 +1,23 @@ +task: arabic_mt_copa +dataset_path: OALL/AlGhafa-Arabic-LLM-Benchmark-Translated +dataset_name: copa_ext_ar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_copa/utils.py b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_copa/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..175ebdadc1b21e79978a59a9a80782c339705b96 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_copa/utils.py @@ -0,0 +1,19 @@ +import datasets +import numpy as np + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + premise = doc["premise"] + choices = [doc["choice1"], doc["choice2"]] + question_map = {"cause": "لأن", "effect": "لذلك"} + question = question_map[doc["question"]] + answer = doc["label"] + + query = "{}، {} :\n0) {}\n1) {}\nالإجابة:".format( + premise, question, choices[0], choices[1] + ) + + return {"query": query, "choices": choices, "gold": answer} + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_hellaswag/arabic_leaderboard_arabic_mt_hellaswag.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_hellaswag/arabic_leaderboard_arabic_mt_hellaswag.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a70f5ab68da05177509fdcae021a2ceafe3ecf0a --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_hellaswag/arabic_leaderboard_arabic_mt_hellaswag.yaml @@ -0,0 +1,13 @@ +group: arabic_leaderboard_arabic_mt_hellaswag +task: + - arabic_mt_hellaswag + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_hellaswag/arabic_mt_hellaswag.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_hellaswag/arabic_mt_hellaswag.yaml new file mode 100644 index 0000000000000000000000000000000000000000..59a4547485a33d8748b458c481c838b9993fb7fa --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_hellaswag/arabic_mt_hellaswag.yaml @@ -0,0 +1,23 @@ +task: arabic_mt_hellaswag +dataset_path: OALL/AlGhafa-Arabic-LLM-Benchmark-Translated +dataset_name: hellaswag_okapi_ar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_hellaswag/utils.py b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_hellaswag/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..6b5a9f1f4f97460816957af2a4076836b4655c57 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_hellaswag/utils.py @@ -0,0 +1,30 @@ +import re + +import datasets +import numpy as np + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + ctx = re.sub(r"\[.*?\]", "", doc["ctx"]) # Remove latin words within brackets + endings = [ + re.sub(r"\[.*?\]", "", e) for e in eval(doc["endings"]) + ] # endings is a string representation of a list + answer_index = doc["label"] + instruction = ( + "بناء على السياق التالي، اختر النهاية الصحيحة من الاقتراحات التالية" + ) + + query = f"""{instruction} + السياق: + {ctx} + الاقتراحات: + + """ + for i, ending in enumerate(endings): + query += f"{i}) {ending}\n" + query += "الإجابة:" + + return {"query": query, "choices": endings, "gold": answer_index} + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_mmlu/arabic_leaderboard_arabic_mt_mmlu.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_mmlu/arabic_leaderboard_arabic_mt_mmlu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0188b5ddc467b1a693de8e8be18b057838d5a90b --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_mmlu/arabic_leaderboard_arabic_mt_mmlu.yaml @@ -0,0 +1,13 @@ +group: arabic_leaderboard_arabic_mt_mmlu +task: + - arabic_mt_mmlu + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_mmlu/arabic_mt_mmlu.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_mmlu/arabic_mt_mmlu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4f3cd249c2e9ef2ba31e3f2092c57b9272bb52bc --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_mmlu/arabic_mt_mmlu.yaml @@ -0,0 +1,23 @@ +task: arabic_mt_mmlu +dataset_path: OALL/AlGhafa-Arabic-LLM-Benchmark-Translated +dataset_name: mmlu_okapi_ar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_mmlu/utils.py b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_mmlu/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..62f9874e63885c9ffbeedde8a4af4d13a8f06792 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_mmlu/utils.py @@ -0,0 +1,23 @@ +import datasets +import numpy as np + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + question = doc["query"] + answer_index = int(doc["label"]) + # Dynamically determining the choices by excluding '__few_shots', 'query' and 'label' + choices_keys = [ + key for key in doc.keys() if key not in ["query", "label", "__few_shots"] + ] + choices = [doc[key] for key in choices_keys] + + instruction = "الأسئلة التالية هي أسئلة متعددة الإختيارات مع الجواب الصحيح\n\n" + query = f"{instruction}السؤال: {question}\n" + for index, choice in enumerate(choices): + query += f"{index}) {choice}\n" + query += "الإجابة:" + + return {"query": query, "choices": choices, "gold": answer_index} + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_openbook_qa/arabic_leaderboard_arabic_mt_openbook_qa.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_openbook_qa/arabic_leaderboard_arabic_mt_openbook_qa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dd3b78f4d0be82e47c472621b6d2b3527c217af3 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_openbook_qa/arabic_leaderboard_arabic_mt_openbook_qa.yaml @@ -0,0 +1,13 @@ +group: arabic_leaderboard_arabic_mt_openbook_qa +task: + - arabic_mt_openbook_qa + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_openbook_qa/arabic_mt_openbook_qa.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_openbook_qa/arabic_mt_openbook_qa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b826a189279ba50a3f8a3f60b984ebe37678a505 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_openbook_qa/arabic_mt_openbook_qa.yaml @@ -0,0 +1,23 @@ +task: arabic_mt_openbook_qa +dataset_path: OALL/AlGhafa-Arabic-LLM-Benchmark-Translated +dataset_name: openbook_qa_ext_ar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_openbook_qa/utils.py b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_openbook_qa/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..62f9874e63885c9ffbeedde8a4af4d13a8f06792 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_openbook_qa/utils.py @@ -0,0 +1,23 @@ +import datasets +import numpy as np + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + question = doc["query"] + answer_index = int(doc["label"]) + # Dynamically determining the choices by excluding '__few_shots', 'query' and 'label' + choices_keys = [ + key for key in doc.keys() if key not in ["query", "label", "__few_shots"] + ] + choices = [doc[key] for key in choices_keys] + + instruction = "الأسئلة التالية هي أسئلة متعددة الإختيارات مع الجواب الصحيح\n\n" + query = f"{instruction}السؤال: {question}\n" + for index, choice in enumerate(choices): + query += f"{index}) {choice}\n" + query += "الإجابة:" + + return {"query": query, "choices": choices, "gold": answer_index} + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_piqa/arabic_leaderboard_arabic_mt_piqa.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_piqa/arabic_leaderboard_arabic_mt_piqa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b75bcc2b1cb6328eb060f9efb38ac7fe154f1601 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_piqa/arabic_leaderboard_arabic_mt_piqa.yaml @@ -0,0 +1,13 @@ +group: arabic_leaderboard_arabic_mt_piqa +task: + - arabic_mt_piqa + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_piqa/arabic_mt_piqa.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_piqa/arabic_mt_piqa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fa93a937a844238b003ba73f682a736a5e86f111 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_piqa/arabic_mt_piqa.yaml @@ -0,0 +1,23 @@ +task: arabic_mt_piqa +dataset_path: OALL/AlGhafa-Arabic-LLM-Benchmark-Translated +dataset_name: piqa_ar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_piqa/utils.py b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_piqa/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..62f9874e63885c9ffbeedde8a4af4d13a8f06792 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_piqa/utils.py @@ -0,0 +1,23 @@ +import datasets +import numpy as np + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + question = doc["query"] + answer_index = int(doc["label"]) + # Dynamically determining the choices by excluding '__few_shots', 'query' and 'label' + choices_keys = [ + key for key in doc.keys() if key not in ["query", "label", "__few_shots"] + ] + choices = [doc[key] for key in choices_keys] + + instruction = "الأسئلة التالية هي أسئلة متعددة الإختيارات مع الجواب الصحيح\n\n" + query = f"{instruction}السؤال: {question}\n" + for index, choice in enumerate(choices): + query += f"{index}) {choice}\n" + query += "الإجابة:" + + return {"query": query, "choices": choices, "gold": answer_index} + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_race/arabic_leaderboard_arabic_mt_race.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_race/arabic_leaderboard_arabic_mt_race.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f3f91c278d88c445af224328b1ae06715bdd50b3 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_race/arabic_leaderboard_arabic_mt_race.yaml @@ -0,0 +1,13 @@ +group: arabic_leaderboard_arabic_mt_race +task: + - arabic_mt_race + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_race/arabic_mt_race.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_race/arabic_mt_race.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ec2aee68983a2e99b1a6ef38ac2f78590a649e22 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_race/arabic_mt_race.yaml @@ -0,0 +1,23 @@ +task: arabic_mt_race +dataset_path: OALL/AlGhafa-Arabic-LLM-Benchmark-Translated +dataset_name: race_ar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_race/utils.py b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_race/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..62f9874e63885c9ffbeedde8a4af4d13a8f06792 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_race/utils.py @@ -0,0 +1,23 @@ +import datasets +import numpy as np + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + question = doc["query"] + answer_index = int(doc["label"]) + # Dynamically determining the choices by excluding '__few_shots', 'query' and 'label' + choices_keys = [ + key for key in doc.keys() if key not in ["query", "label", "__few_shots"] + ] + choices = [doc[key] for key in choices_keys] + + instruction = "الأسئلة التالية هي أسئلة متعددة الإختيارات مع الجواب الصحيح\n\n" + query = f"{instruction}السؤال: {question}\n" + for index, choice in enumerate(choices): + query += f"{index}) {choice}\n" + query += "الإجابة:" + + return {"query": query, "choices": choices, "gold": answer_index} + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_sciq/arabic_leaderboard_arabic_mt_sciq.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_sciq/arabic_leaderboard_arabic_mt_sciq.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7768047c4cecb136c4dc4f171557211af0721bad --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_sciq/arabic_leaderboard_arabic_mt_sciq.yaml @@ -0,0 +1,13 @@ +group: arabic_leaderboard_arabic_mt_sciq +task: + - arabic_mt_sciq + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_sciq/arabic_mt_sciq.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_sciq/arabic_mt_sciq.yaml new file mode 100644 index 0000000000000000000000000000000000000000..07f96b7574d1df62b1e560305fb0b572f2648d76 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_sciq/arabic_mt_sciq.yaml @@ -0,0 +1,23 @@ +task: arabic_mt_sciq +dataset_path: OALL/AlGhafa-Arabic-LLM-Benchmark-Translated +dataset_name: sciq_ar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_sciq/utils.py b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_sciq/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..ddb42eeb8c8ee002662a6b3a6129ac4a8fa5007b --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_sciq/utils.py @@ -0,0 +1,41 @@ +import random + +import datasets +import numpy as np + + +def doc_to_text(doc): + instruction = ( + "بناءً على السياق أدناه، اختر الإجابة الصحيحة للسؤال التالي من قائمة الاقتراحات" + ) + support = doc["support"] + question = doc["question"] + query = f"""{instruction} + السياق: + {support} + السؤال: + {question} + الإجابات المحتملة: + + """ + return query + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + correct_answer = doc["correct_answer"] + choices = [ + doc["distractor1"], + doc["distractor2"], + doc["distractor3"], + correct_answer, + ] + + # Shuffle the choices + random.shuffle(choices) + + answer_index = choices.index(correct_answer) + + return {"query": doc_to_text(doc), "choices": choices, "gold": answer_index} + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_toxigen/arabic_leaderboard_arabic_mt_toxigen.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_toxigen/arabic_leaderboard_arabic_mt_toxigen.yaml new file mode 100644 index 0000000000000000000000000000000000000000..272166206b76b52d35c15681d2d53b9237917f47 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_toxigen/arabic_leaderboard_arabic_mt_toxigen.yaml @@ -0,0 +1,13 @@ +group: arabic_leaderboard_arabic_mt_toxigen +task: + - arabic_mt_toxigen + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_toxigen/arabic_mt_toxigen.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_toxigen/arabic_mt_toxigen.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8a140793d7fd1c3b79c26e7564364c6bed2abd3e --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_toxigen/arabic_mt_toxigen.yaml @@ -0,0 +1,23 @@ +task: arabic_mt_toxigen +dataset_path: OALL/AlGhafa-Arabic-LLM-Benchmark-Translated +dataset_name: toxigen_ar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_toxigen/utils.py b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_toxigen/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..09d311e5a13746d41e5e2bcf7ce63fc905b4c72e --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_arabic_mt_toxigen/utils.py @@ -0,0 +1,23 @@ +import datasets +import numpy as np + + +def process_docs(dataset: datasets.Dataset) -> datasets.Dataset: + def _process_doc(doc): + text = doc["text"] + label = 1 if ((doc["toxicity_ai"] + doc["toxicity_human"]) > 5.5) else 0 + instruction = 'هل العبارة التالية تحض على الكراهية؟ قم بالرد بـ"نعم" أو "لا".' + + query = f"""{instruction} + العبارة: + '{text}' + الإجابة: + """ + out_doc = { + "query": query, + "choices": ["لا", "نعم"], + "gold": label, + } + return out_doc + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8e2dab57b053f4620e1241645d040224810ddad8 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva.yaml @@ -0,0 +1,70 @@ +group: arabic_leaderboard_acva +task: + - arabic_leaderboard_acva_Algeria + - arabic_leaderboard_acva_Ancient_Egypt + - arabic_leaderboard_acva_Arab_Empire + - arabic_leaderboard_acva_Arabic_Architecture + - arabic_leaderboard_acva_Arabic_Art + - arabic_leaderboard_acva_Arabic_Astronomy + - arabic_leaderboard_acva_Arabic_Calligraphy + - arabic_leaderboard_acva_Arabic_Ceremony + - arabic_leaderboard_acva_Arabic_Clothing + - arabic_leaderboard_acva_Arabic_Culture + - arabic_leaderboard_acva_Arabic_Food + - arabic_leaderboard_acva_Arabic_Funeral + - arabic_leaderboard_acva_Arabic_Geography + - arabic_leaderboard_acva_Arabic_History + - arabic_leaderboard_acva_Arabic_Language_Origin + - arabic_leaderboard_acva_Arabic_Literature + - arabic_leaderboard_acva_Arabic_Math + - arabic_leaderboard_acva_Arabic_Medicine + - arabic_leaderboard_acva_Arabic_Music + - arabic_leaderboard_acva_Arabic_Ornament + - arabic_leaderboard_acva_Arabic_Philosophy + - arabic_leaderboard_acva_Arabic_Physics_and_Chemistry + - arabic_leaderboard_acva_Arabic_Wedding + - arabic_leaderboard_acva_Bahrain + - arabic_leaderboard_acva_Comoros + - arabic_leaderboard_acva_Egypt_modern + - arabic_leaderboard_acva_InfluenceFromAncientEgypt + - arabic_leaderboard_acva_InfluenceFromByzantium + - arabic_leaderboard_acva_InfluenceFromChina + - arabic_leaderboard_acva_InfluenceFromGreece + - arabic_leaderboard_acva_InfluenceFromIslam + - arabic_leaderboard_acva_InfluenceFromPersia + - arabic_leaderboard_acva_InfluenceFromRome + - arabic_leaderboard_acva_Iraq + - arabic_leaderboard_acva_Islam_Education + - arabic_leaderboard_acva_Islam_branches_and_schools + - arabic_leaderboard_acva_Islamic_law_system + - arabic_leaderboard_acva_Jordan + - arabic_leaderboard_acva_Kuwait + - arabic_leaderboard_acva_Lebanon + - arabic_leaderboard_acva_Libya + - arabic_leaderboard_acva_Mauritania + - arabic_leaderboard_acva_Mesopotamia_civilization + - arabic_leaderboard_acva_Morocco + - arabic_leaderboard_acva_Oman + - arabic_leaderboard_acva_Palestine + - arabic_leaderboard_acva_Qatar + - arabic_leaderboard_acva_Saudi_Arabia + - arabic_leaderboard_acva_Somalia + - arabic_leaderboard_acva_Sudan + - arabic_leaderboard_acva_Syria + - arabic_leaderboard_acva_Tunisia + - arabic_leaderboard_acva_United_Arab_Emirates + - arabic_leaderboard_acva_Yemen + - arabic_leaderboard_acva_communication + - arabic_leaderboard_acva_computer_and_phone + - arabic_leaderboard_acva_daily_life + - arabic_leaderboard_acva_entertainment + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Algeria.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Algeria.yaml new file mode 100644 index 0000000000000000000000000000000000000000..177161edaafac696f13391ddaeafb22548e480f2 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Algeria.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Algeria +dataset_path: OALL/ACVA +dataset_name: Algeria +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Ancient_Egypt.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Ancient_Egypt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ddb5c35555daff8b76ba61aeae71c13ff9783bf2 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Ancient_Egypt.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Ancient_Egypt +dataset_path: OALL/ACVA +dataset_name: Ancient_Egypt +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arab_Empire.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arab_Empire.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b510de5ab9865110b2f019693d9dd468c1bbe555 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arab_Empire.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arab_Empire +dataset_path: OALL/ACVA +dataset_name: Arab_Empire +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Architecture.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Architecture.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5dc2c07dee77e5370e147cb8cb30edb523858735 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Architecture.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Architecture +dataset_path: OALL/ACVA +dataset_name: Arabic_Architecture +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Art.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Art.yaml new file mode 100644 index 0000000000000000000000000000000000000000..36f364bc50c3150057c3b90b218b80b6b43e606a --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Art.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Art +dataset_path: OALL/ACVA +dataset_name: Arabic_Art +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Astronomy.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f90b1c91409be627887bfa63d65eb181c9d55616 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Astronomy.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Astronomy +dataset_path: OALL/ACVA +dataset_name: Arabic_Astronomy +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Calligraphy.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Calligraphy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dfdf51878b94b8ae8192a76a400223d25f852b4a --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Calligraphy.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Calligraphy +dataset_path: OALL/ACVA +dataset_name: Arabic_Calligraphy +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Ceremony.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Ceremony.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c20b4439e232b2c30c18e1438344355302430a7d --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Ceremony.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Ceremony +dataset_path: OALL/ACVA +dataset_name: Arabic_Ceremony +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Clothing.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Clothing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..06118034dc8d03a80dd6a7f9fa2254dce66d4141 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Clothing.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Clothing +dataset_path: OALL/ACVA +dataset_name: Arabic_Clothing +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Culture.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Culture.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cea33022b473de0f0daddae5eb615b681d75a58f --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Culture.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Culture +dataset_path: OALL/ACVA +dataset_name: Arabic_Culture +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Food.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Food.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cca516c9724ba34794766e09cde817ab49669066 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Food.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Food +dataset_path: OALL/ACVA +dataset_name: Arabic_Food +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Funeral.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Funeral.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3dd8fbedd9e9ca008608eb6ca1af9197d2c7ac9b --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Funeral.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Funeral +dataset_path: OALL/ACVA +dataset_name: Arabic_Funeral +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Geography.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..89aa7361b3afb7871fc9a0db85bf8ce50befd209 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Geography.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Geography +dataset_path: OALL/ACVA +dataset_name: Arabic_Geography +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_History.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_History.yaml new file mode 100644 index 0000000000000000000000000000000000000000..776589c07b042b4285f40ce32162492ea5e8da06 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_History.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_History +dataset_path: OALL/ACVA +dataset_name: Arabic_History +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Language_Origin.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Language_Origin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4f0612acaf264f77f63d96fd2ca62ace587a2c76 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Language_Origin.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Language_Origin +dataset_path: OALL/ACVA +dataset_name: Arabic_Language_Origin +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Literature.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Literature.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0c9198f446064cac000d16fa23fbf7f6843a6513 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Literature.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Literature +dataset_path: OALL/ACVA +dataset_name: Arabic_Literature +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Math.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..02a3643024eff73360381dea1492330194858d95 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Math.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Math +dataset_path: OALL/ACVA +dataset_name: Arabic_Math +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Medicine.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..109aae994a9b5da7212f6a732de0ffc8fc823193 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Medicine.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Medicine +dataset_path: OALL/ACVA +dataset_name: Arabic_Medicine +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Music.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Music.yaml new file mode 100644 index 0000000000000000000000000000000000000000..25596257843854f6f797a4ac993b867576c5da72 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Music.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Music +dataset_path: OALL/ACVA +dataset_name: Arabic_Music +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Ornament.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Ornament.yaml new file mode 100644 index 0000000000000000000000000000000000000000..00311e107e4ecbc488dcd06fbfcc451aac46fed8 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Ornament.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Ornament +dataset_path: OALL/ACVA +dataset_name: Arabic_Ornament +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Philosophy.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..62a570f00cb8a457ce0b23917ac897b5e77acc70 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Philosophy.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Philosophy +dataset_path: OALL/ACVA +dataset_name: Arabic_Philosophy +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Physics_and_Chemistry.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Physics_and_Chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b1b52096e49f1fbbda9b071efbfdf0f6b584a380 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Physics_and_Chemistry.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Physics_and_Chemistry +dataset_path: OALL/ACVA +dataset_name: Arabic_Physics_and_Chemistry +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Wedding.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Wedding.yaml new file mode 100644 index 0000000000000000000000000000000000000000..21205cfff8b2a43d51f3591cd709062d40a3f46f --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Arabic_Wedding.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Wedding +dataset_path: OALL/ACVA +dataset_name: Arabic_Wedding +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Bahrain.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Bahrain.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3b2481bc87d051d0958b524ddfa6273a72a9a393 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Bahrain.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Bahrain +dataset_path: OALL/ACVA +dataset_name: Bahrain +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Comoros.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Comoros.yaml new file mode 100644 index 0000000000000000000000000000000000000000..be4df372c7c21428b5df9912b2b8f3b762f4a3a2 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Comoros.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Comoros +dataset_path: OALL/ACVA +dataset_name: Comoros +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Egypt_modern.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Egypt_modern.yaml new file mode 100644 index 0000000000000000000000000000000000000000..26ca2f6e08ae3848b710fe25edac658bf78486bb --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Egypt_modern.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Egypt_modern +dataset_path: OALL/ACVA +dataset_name: Egypt_modern +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_InfluenceFromAncientEgypt.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_InfluenceFromAncientEgypt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..be300fc869e116f729313b8c38403c716468d293 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_InfluenceFromAncientEgypt.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_InfluenceFromAncientEgypt +dataset_path: OALL/ACVA +dataset_name: InfluenceFromAncientEgypt +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_InfluenceFromByzantium.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_InfluenceFromByzantium.yaml new file mode 100644 index 0000000000000000000000000000000000000000..72c86a62474a976445cd8ec7990d0fc76b605d68 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_InfluenceFromByzantium.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_InfluenceFromByzantium +dataset_path: OALL/ACVA +dataset_name: InfluenceFromByzantium +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_InfluenceFromChina.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_InfluenceFromChina.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b297642cbe2407b112fa6e6815cbea8918ddbd5f --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_InfluenceFromChina.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_InfluenceFromChina +dataset_path: OALL/ACVA +dataset_name: InfluenceFromChina +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_InfluenceFromGreece.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_InfluenceFromGreece.yaml new file mode 100644 index 0000000000000000000000000000000000000000..70458ea2d37e7fe82edb004b53a4c25a8feadcdd --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_InfluenceFromGreece.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_InfluenceFromGreece +dataset_path: OALL/ACVA +dataset_name: InfluenceFromGreece +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_InfluenceFromIslam.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_InfluenceFromIslam.yaml new file mode 100644 index 0000000000000000000000000000000000000000..803f33345dd652b77ebfdb4b682bad3a924f901e --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_InfluenceFromIslam.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_InfluenceFromIslam +dataset_path: OALL/ACVA +dataset_name: InfluenceFromIslam +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_InfluenceFromPersia.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_InfluenceFromPersia.yaml new file mode 100644 index 0000000000000000000000000000000000000000..117ca890799421be066fca1daea4de05bd9ad1ed --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_InfluenceFromPersia.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_InfluenceFromPersia +dataset_path: OALL/ACVA +dataset_name: InfluenceFromPersia +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_InfluenceFromRome.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_InfluenceFromRome.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1655522e5a87fef766d3502526afc43993e073e9 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_InfluenceFromRome.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_InfluenceFromRome +dataset_path: OALL/ACVA +dataset_name: InfluenceFromRome +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Iraq.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Iraq.yaml new file mode 100644 index 0000000000000000000000000000000000000000..909c6678c7042b03b0aef444d6e887a6101f6187 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Iraq.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Iraq +dataset_path: OALL/ACVA +dataset_name: Iraq +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Islam_Education.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Islam_Education.yaml new file mode 100644 index 0000000000000000000000000000000000000000..13c1fab2a03a03f1e77cd111737e3782cd494cb1 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Islam_Education.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Islam_Education +dataset_path: OALL/ACVA +dataset_name: Islam_Education +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Islam_branches_and_schools.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Islam_branches_and_schools.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6985b24a74f29a14dbd2566a82fea438f6b945fd --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Islam_branches_and_schools.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Islam_branches_and_schools +dataset_path: OALL/ACVA +dataset_name: Islam_branches_and_schools +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Islamic_law_system.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Islamic_law_system.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d19a52ba03b4726d047cdf58780cba2e52d259af --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Islamic_law_system.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Islamic_law_system +dataset_path: OALL/ACVA +dataset_name: Islamic_law_system +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Jordan.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Jordan.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7bff93a94ccda79318ad9be99504a682060e6565 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Jordan.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Jordan +dataset_path: OALL/ACVA +dataset_name: Jordan +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Kuwait.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Kuwait.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b1ae77aaa5bded01828434b88ef17d617f0d46b2 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Kuwait.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Kuwait +dataset_path: OALL/ACVA +dataset_name: Kuwait +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Lebanon.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Lebanon.yaml new file mode 100644 index 0000000000000000000000000000000000000000..65974b74dc51c9bcc351a76650dc1c80264791c8 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Lebanon.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Lebanon +dataset_path: OALL/ACVA +dataset_name: Lebanon +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Libya.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Libya.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c8b339650c1c3a44fe244538f6d067b7976a38d7 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Libya.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Libya +dataset_path: OALL/ACVA +dataset_name: Libya +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Mauritania.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Mauritania.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1b84074abcdb5283450900a45ccecbef0469f90d --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Mauritania.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Mauritania +dataset_path: OALL/ACVA +dataset_name: Mauritania +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Mesopotamia_civilization.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Mesopotamia_civilization.yaml new file mode 100644 index 0000000000000000000000000000000000000000..42189477026cb3da398e73c1e5b42570b1092996 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Mesopotamia_civilization.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Mesopotamia_civilization +dataset_path: OALL/ACVA +dataset_name: Mesopotamia_civilization +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Morocco.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Morocco.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4ed1510bb5a97ff26bc075f87b22027036c55ea3 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Morocco.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Morocco +dataset_path: OALL/ACVA +dataset_name: Morocco +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Oman.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Oman.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b534cfb19fdb960314160d0129ece77bb63d2e6b --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Oman.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Oman +dataset_path: OALL/ACVA +dataset_name: Oman +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Palestine.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Palestine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1cb9b56a851458acf9694cfce661173433c2576b --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Palestine.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Palestine +dataset_path: OALL/ACVA +dataset_name: Palestine +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Qatar.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Qatar.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5d5775ccd90aae923d7eca4b17637f79e1619df5 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Qatar.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Qatar +dataset_path: OALL/ACVA +dataset_name: Qatar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Saudi_Arabia.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Saudi_Arabia.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5010723661ccf57872112417231a8d752935eef4 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Saudi_Arabia.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Saudi_Arabia +dataset_path: OALL/ACVA +dataset_name: Saudi_Arabia +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Somalia.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Somalia.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d40b578221bd5c281bbaacc32c944377878a1ea3 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Somalia.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Somalia +dataset_path: OALL/ACVA +dataset_name: Somalia +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Sudan.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Sudan.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e7c2f41a3b917384e9a57eb0ba174da34457bfd6 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Sudan.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Sudan +dataset_path: OALL/ACVA +dataset_name: Sudan +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Syria.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Syria.yaml new file mode 100644 index 0000000000000000000000000000000000000000..98ebff9fcaa64074b3a601fb8f351c2fee530d14 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Syria.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Syria +dataset_path: OALL/ACVA +dataset_name: Syria +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Tunisia.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Tunisia.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d86e428cc33305f8b67dd936b33be9f6712c0612 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Tunisia.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Tunisia +dataset_path: OALL/ACVA +dataset_name: Tunisia +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_United_Arab_Emirates.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_United_Arab_Emirates.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f41b6255084572d125ab6a37b31dbd178bdf516e --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_United_Arab_Emirates.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_United_Arab_Emirates +dataset_path: OALL/ACVA +dataset_name: United_Arab_Emirates +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Yemen.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Yemen.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b239dd514abe70dcbe537e0bb8f086fc2bd804eb --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_Yemen.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Yemen +dataset_path: OALL/ACVA +dataset_name: Yemen +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_communication.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_communication.yaml new file mode 100644 index 0000000000000000000000000000000000000000..beb954efceb70fb303da829f05d39be0f4d343ee --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_communication.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_communication +dataset_path: OALL/ACVA +dataset_name: communication +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_computer_and_phone.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_computer_and_phone.yaml new file mode 100644 index 0000000000000000000000000000000000000000..888f82af929ce911bbf82fd11650f6b73179b48f --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_computer_and_phone.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_computer_and_phone +dataset_path: OALL/ACVA +dataset_name: computer_and_phone +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_daily_life.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_daily_life.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0b4748a297821a239cd33ad7797d465467c29946 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_daily_life.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_daily_life +dataset_path: OALL/ACVA +dataset_name: daily_life +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_entertainment.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_entertainment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b2adcfb95470cf1130eb1e4503db78be89456fbf --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/arabic_leaderboard_acva_entertainment.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_entertainment +dataset_path: OALL/ACVA +dataset_name: entertainment +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/utils.py b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7e91496f59df5e940e4c206bceee69007c9f159c --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_avca/utils.py @@ -0,0 +1,16 @@ +import datasets +import numpy as np + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + question = doc["question"] + answer = doc["answer"] + + return { + "query": f"السؤال: {question}\nالإجابة:", + "choices": ["صح", "خطأ"], + "gold": ["صح", "خطأ"].index(answer), + } + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_complete.yaml b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_complete.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c26370157d4dadcdb18a0189a68bceec54dea5aa --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_complete/arabic_leaderboard_complete.yaml @@ -0,0 +1,25 @@ +group: arabic_leaderboard_complete +task: + - arabic_leaderboard_acva + - arabic_leaderboard_alghafa + - arabic_leaderboard_arabic_exams + - arabic_leaderboard_arabic_mt_arc_challenge + - arabic_leaderboard_arabic_mt_arc_easy + - arabic_leaderboard_arabic_mt_boolq + - arabic_leaderboard_arabic_mt_hellaswag + - arabic_leaderboard_arabic_mt_mmlu + - arabic_leaderboard_arabic_mt_copa + - arabic_leaderboard_arabic_mt_openbook_qa + - arabic_leaderboard_arabic_mt_piqa + - arabic_leaderboard_arabic_mt_race + - arabic_leaderboard_arabic_mt_sciq + - arabic_leaderboard_arabic_mt_toxigen +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/README.md b/lm_eval/tasks/arabic_leaderboard_light/README.md new file mode 100644 index 0000000000000000000000000000000000000000..199aa2c8dae8553f22f2f15ec72acedf1e09bdb4 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/README.md @@ -0,0 +1,20 @@ +# Arabic Leaderboard Light + +Title: Open Arabic LLM Leaderboard Light + +This leaderboard follows all the details as in [`arabic_leaderboard_complete`](../arabic_leaderboard_complete), except that a light version - 10% random sample of the test set of each benchmark - is used to test the language models. + +NOTE: In ACVA benchmark, there is Yemen subset, and it is a small dataset - it has only 10 samples in the test split. So, for this specific subset dataset, to have more reliable results, we consider the original dataset, instead of 10% of its test samples. + +### Checklist + +For adding novel benchmarks/datasets to the library: +* [ ] Is the task an existing benchmark in the literature? + * [ ] Have you referenced the original paper that introduced the task? + * [ ] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + + +If other tasks on this dataset are already supported: +* [ ] Is the "Main" variant of this task clearly denoted? +* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [ ] Have you noted which, if any, published evaluation setups are matched by this variant? diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0ee6a568d90cdabb51a6c5bd2a5b7fcb06d22545 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_light.yaml @@ -0,0 +1,23 @@ +group: arabic_leaderboard_alghafa_light +task: + - arabic_leaderboard_alghafa_mcq_exams_test_ar_light + - arabic_leaderboard_alghafa_meta_ar_dialects_light + - arabic_leaderboard_alghafa_meta_ar_msa_light + - arabic_leaderboard_alghafa_multiple_choice_facts_truefalse_balanced_task_light + - arabic_leaderboard_alghafa_multiple_choice_grounded_statement_soqal_task_light + - arabic_leaderboard_alghafa_multiple_choice_grounded_statement_xglue_mlqa_task_light + - arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_no_neutral_task_light + - arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_task_light + - arabic_leaderboard_alghafa_multiple_choice_sentiment_task_light + + + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_mcq_exams_test_ar_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_mcq_exams_test_ar_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1fdda36405a6c0c4838813c7cfea493d4df21fb8 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_mcq_exams_test_ar_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_alghafa_mcq_exams_test_ar_light +dataset_path: arcee-globe/AlGhafa-Arabic-LLM-Benchmark-Native-10percent +dataset_name: mcq_exams_test_ar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_meta_ar_dialects_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_meta_ar_dialects_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..47af55b86abaacf9455cd7d318bd0ffc26f61145 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_meta_ar_dialects_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_alghafa_meta_ar_dialects_light +dataset_path: arcee-globe/AlGhafa-Arabic-LLM-Benchmark-Native-10percent +dataset_name: meta_ar_dialects +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_meta_ar_msa_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_meta_ar_msa_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9a26a2653fd23608fd1bf37faee8dd0042b465b0 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_meta_ar_msa_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_alghafa_meta_ar_msa_light +dataset_path: arcee-globe/AlGhafa-Arabic-LLM-Benchmark-Native-10percent +dataset_name: meta_ar_msa +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_multiple_choice_facts_truefalse_balanced_task_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_multiple_choice_facts_truefalse_balanced_task_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b56ddfee19c42a73caaa2863540de75f19cfaa10 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_multiple_choice_facts_truefalse_balanced_task_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_alghafa_multiple_choice_facts_truefalse_balanced_task_light +dataset_path: arcee-globe/AlGhafa-Arabic-LLM-Benchmark-Native-10percent +dataset_name: multiple_choice_facts_truefalse_balanced_task +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_multiple_choice_grounded_statement_soqal_task_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_multiple_choice_grounded_statement_soqal_task_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4d85c68491f066b13648fb1c1cb7da932a2c123e --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_multiple_choice_grounded_statement_soqal_task_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_alghafa_multiple_choice_grounded_statement_soqal_task_light +dataset_path: arcee-globe/AlGhafa-Arabic-LLM-Benchmark-Native-10percent +dataset_name: multiple_choice_grounded_statement_soqal_task +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_multiple_choice_grounded_statement_xglue_mlqa_task_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_multiple_choice_grounded_statement_xglue_mlqa_task_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e5d8afefeaf6a51f67cade46873a18dd13f9ebe0 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_multiple_choice_grounded_statement_xglue_mlqa_task_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_alghafa_multiple_choice_grounded_statement_xglue_mlqa_task_light +dataset_path: arcee-globe/AlGhafa-Arabic-LLM-Benchmark-Native-10percent +dataset_name: multiple_choice_grounded_statement_xglue_mlqa_task +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_no_neutral_task_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_no_neutral_task_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..21721d2a2d82a5bd5ded8e968297b584e610af4a --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_no_neutral_task_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_no_neutral_task_light +dataset_path: arcee-globe/AlGhafa-Arabic-LLM-Benchmark-Native-10percent +dataset_name: multiple_choice_rating_sentiment_no_neutral_task +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_task_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_task_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..39f72e4d2a216ffb3d09071067412d35a41ff974 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_task_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_alghafa_multiple_choice_rating_sentiment_task_light +dataset_path: arcee-globe/AlGhafa-Arabic-LLM-Benchmark-Native-10percent +dataset_name: multiple_choice_rating_sentiment_task +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_multiple_choice_sentiment_task_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_multiple_choice_sentiment_task_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..28b0701561a53e12d661dc942d8098c8895e171f --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/arabic_leaderboard_alghafa_multiple_choice_sentiment_task_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_alghafa_multiple_choice_sentiment_task_light +dataset_path: arcee-globe/AlGhafa-Arabic-LLM-Benchmark-Native-10percent +dataset_name: multiple_choice_sentiment_task +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/utils.py b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..62f9874e63885c9ffbeedde8a4af4d13a8f06792 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_alghafa_light/utils.py @@ -0,0 +1,23 @@ +import datasets +import numpy as np + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + question = doc["query"] + answer_index = int(doc["label"]) + # Dynamically determining the choices by excluding '__few_shots', 'query' and 'label' + choices_keys = [ + key for key in doc.keys() if key not in ["query", "label", "__few_shots"] + ] + choices = [doc[key] for key in choices_keys] + + instruction = "الأسئلة التالية هي أسئلة متعددة الإختيارات مع الجواب الصحيح\n\n" + query = f"{instruction}السؤال: {question}\n" + for index, choice in enumerate(choices): + query += f"{index}) {choice}\n" + query += "الإجابة:" + + return {"query": query, "choices": choices, "gold": answer_index} + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_exams_light/arabic_exams_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_exams_light/arabic_exams_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2348be4eb302235a35d39c47c37aee551e935237 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_exams_light/arabic_exams_light.yaml @@ -0,0 +1,23 @@ +task: arabic_exams_light +dataset_path: arcee-globe/Arabic_EXAMS-10percent +dataset_name: default +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_exams_light/arabic_leaderboard_arabic_exams_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_exams_light/arabic_leaderboard_arabic_exams_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..296a47cbb40a6071e22677f96e1b4a01bae97eb5 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_exams_light/arabic_leaderboard_arabic_exams_light.yaml @@ -0,0 +1,13 @@ +group: arabic_leaderboard_arabic_exams_light +task: + - arabic_exams_light + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_exams_light/utils.py b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_exams_light/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..72af1c40fe586d0ab3c7d5ccc519506503449f68 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_exams_light/utils.py @@ -0,0 +1,33 @@ +import datasets +import numpy as np + + +# fmt: off +LETTER_INDICES_AR = ["أ", "ب", "ج", "د", "هـ", "و", "ز", "ح", "ط", "ي", "ك", "ل", "م", "ن", "س", "ع", "ف", "ص", "ق", "ر", "ش", "ت", "ث", "خ", "ذ", "ض", "ظ", "غ"] +# fmt: on + + +# fmt: off +LETTER_INDICES = ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z"] +# fmt: on + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + topic = doc["subject"] + question = doc["question"] + choices = [doc["A"], doc["B"], doc["C"], doc["D"]] + choices_formatted = [ + f" {LETTER_INDICES_AR[i]}) {choice}\n" for i, choice in enumerate(choices) + ] + answer = doc["answer"] + answer_index = LETTER_INDICES.index(answer) + + instruction = f"الأسئلة التالية هي أسئلة متعددة الإختيارات مع الجواب الصحيح حول {topic.replace('_', ' ')}. \n\n" + query = f"{instruction}السؤال: {question}\n" + query += "\n".join(choices_formatted) + query += "\nالإجابة:" + + return {"query": query, "choices": LETTER_INDICES_AR[:4], "gold": answer_index} + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_abstract_algebra_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_abstract_algebra_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dcb59fc36116f687c9ef126ee68bb52bffceb6a1 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_abstract_algebra_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_abstract_algebra_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: abstract_algebra +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_anatomy_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_anatomy_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fc77a66dde872ea81fc3fbe8ee1de9053f21e55e --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_anatomy_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_anatomy_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: anatomy +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_astronomy_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_astronomy_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..db4a9b43606ac951b21943740b02e24ee3894cf9 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_astronomy_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_astronomy_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: astronomy +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_business_ethics_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_business_ethics_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a747dbafaf7950c8d1613737dccc3bc4a0588493 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_business_ethics_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_business_ethics_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: business_ethics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_clinical_knowledge_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_clinical_knowledge_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1296b90cbc25e3cae2c5ac12974db4e2eaa165a1 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_clinical_knowledge_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_clinical_knowledge_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: clinical_knowledge +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_college_biology_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_college_biology_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cbfc8049746d8c978825aa21876e869caf734384 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_college_biology_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_college_biology_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: college_biology +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_college_chemistry_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_college_chemistry_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ac0970355b0a077c56227bd72bd79a9327d908ea --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_college_chemistry_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_college_chemistry_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: college_chemistry +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_college_computer_science_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_college_computer_science_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..361274d64aa0a3d246356806874680ff9671c0b7 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_college_computer_science_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_college_computer_science_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: college_computer_science +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_college_mathematics_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_college_mathematics_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..20e4d6e627bdd40a5ba9e2f411f93414e35f9527 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_college_mathematics_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_college_mathematics_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: college_mathematics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_college_medicine_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_college_medicine_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d854004973b943c57b027afe4aea74aeb55d772d --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_college_medicine_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_college_medicine_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: college_medicine +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_college_physics_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_college_physics_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..57e4b55033d2375f258b297bf4f514979b63ee3a --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_college_physics_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_college_physics_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: college_physics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_computer_security_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_computer_security_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dd8c01dc6ce49bbd0ef744c5c9e51b0dac039c35 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_computer_security_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_computer_security_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: computer_security +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_conceptual_physics_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_conceptual_physics_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cffd7ee42d256f98e508c1e737d868f7cda8031c --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_conceptual_physics_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_conceptual_physics_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: conceptual_physics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_econometrics_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_econometrics_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..30413feff00a2f6f39849193c65ddea9dab6c9a0 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_econometrics_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_econometrics_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: econometrics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_electrical_engineering_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_electrical_engineering_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e60787d6758e144a4f62922427bd831c3b91fcc4 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_electrical_engineering_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_electrical_engineering_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: electrical_engineering +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_elementary_mathematics_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_elementary_mathematics_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..571476620a3d36772b28341dcdf9abd862acd77f --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_elementary_mathematics_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_elementary_mathematics_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: elementary_mathematics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_formal_logic_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_formal_logic_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9b2bebf1e55b43cafa16b5a5b4f21264c5a25c55 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_formal_logic_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_formal_logic_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: formal_logic +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_global_facts_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_global_facts_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..15c3b34aace0a79bfb358d61f08978e51e2d7d23 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_global_facts_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_global_facts_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: global_facts +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_biology_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_biology_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..906c33284df2d890e70ea11769b8b2b591f99e75 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_biology_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_biology_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: high_school_biology +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_chemistry_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_chemistry_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..199f16b0938b708f33648e2c07e7bc6c2b4fcb08 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_chemistry_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_chemistry_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: high_school_chemistry +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_computer_science_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_computer_science_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cb23af53bb0c7c8a91b304011f09f57a898ce8ae --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_computer_science_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_computer_science_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: high_school_computer_science +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_european_history_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_european_history_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..25a9b46695e888aee9480e7fe4163adfadb12f02 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_european_history_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_european_history_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: high_school_european_history +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_geography_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_geography_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f7f39cd2f2b299ca82d4ad2daa3764e966825d77 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_geography_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_geography_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: high_school_geography +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_government_and_politics_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_government_and_politics_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dff09d6717714e53f8126ed948a4a7b79d79ef47 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_government_and_politics_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_government_and_politics_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: high_school_government_and_politics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_macroeconomics_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_macroeconomics_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ae42622353c7c60036f07dbc6811f4ae0681733b --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_macroeconomics_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_macroeconomics_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: high_school_macroeconomics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_mathematics_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_mathematics_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8adc3d7e93569538d2685e5340760ab80d6ab049 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_mathematics_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_mathematics_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: high_school_mathematics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_microeconomics_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_microeconomics_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6eec39237b57e981f64457242e89b12c5ae46289 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_microeconomics_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_microeconomics_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: high_school_microeconomics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_physics_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_physics_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..973bd1ffc5770644048c5ba26ec5f4421c9a8e2c --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_physics_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_physics_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: high_school_physics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_psychology_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_psychology_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..614dd7e89daec3d9243184f850d76e573b9e336f --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_psychology_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_psychology_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: high_school_psychology +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_statistics_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_statistics_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2db9f196a33172f01c47d0ffa5337407d5107f88 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_statistics_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_statistics_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: high_school_statistics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_us_history_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_us_history_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5411e8c4793ffbdb4cbf5aca045b822685d9b5e7 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_us_history_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_us_history_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: high_school_us_history +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_world_history_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_world_history_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..319c49b22b8eec6f11e1480d9e53ff5aa9730c5a --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_high_school_world_history_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_high_school_world_history_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: high_school_world_history +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_human_aging_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_human_aging_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..afd2eefa29c054e70e8e20a63e8b93223bacf211 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_human_aging_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_human_aging_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: human_aging +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_human_sexuality_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_human_sexuality_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9e245f26878205fbf875153fd273f69c61e65811 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_human_sexuality_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_human_sexuality_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: human_sexuality +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_international_law_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_international_law_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6e476bb8794ff32703b7d2a05800f390e9c9769a --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_international_law_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_international_law_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: international_law +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_jurisprudence_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_jurisprudence_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1d848cd17387db4c3242e1a6cf450748a438f4fc --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_jurisprudence_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_jurisprudence_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: jurisprudence +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..130713702ccf91c32dacb29f50dec390f00dc9dd --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_light.yaml @@ -0,0 +1,68 @@ +group: arabic_leaderboard_arabic_mmlu_light +task: + - arabic_leaderboard_arabic_mmlu_abstract_algebra_light + - arabic_leaderboard_arabic_mmlu_anatomy_light + - arabic_leaderboard_arabic_mmlu_astronomy_light + - arabic_leaderboard_arabic_mmlu_business_ethics_light + - arabic_leaderboard_arabic_mmlu_clinical_knowledge_light + - arabic_leaderboard_arabic_mmlu_college_biology_light + - arabic_leaderboard_arabic_mmlu_college_chemistry_light + - arabic_leaderboard_arabic_mmlu_college_computer_science_light + - arabic_leaderboard_arabic_mmlu_college_mathematics_light + - arabic_leaderboard_arabic_mmlu_college_medicine_light + - arabic_leaderboard_arabic_mmlu_college_physics_light + - arabic_leaderboard_arabic_mmlu_computer_security_light + - arabic_leaderboard_arabic_mmlu_conceptual_physics_light + - arabic_leaderboard_arabic_mmlu_econometrics_light + - arabic_leaderboard_arabic_mmlu_electrical_engineering_light + - arabic_leaderboard_arabic_mmlu_elementary_mathematics_light + - arabic_leaderboard_arabic_mmlu_formal_logic_light + - arabic_leaderboard_arabic_mmlu_global_facts_light + - arabic_leaderboard_arabic_mmlu_high_school_biology_light + - arabic_leaderboard_arabic_mmlu_high_school_chemistry_light + - arabic_leaderboard_arabic_mmlu_high_school_computer_science_light + - arabic_leaderboard_arabic_mmlu_high_school_european_history_light + - arabic_leaderboard_arabic_mmlu_high_school_geography_light + - arabic_leaderboard_arabic_mmlu_high_school_government_and_politics_light + - arabic_leaderboard_arabic_mmlu_high_school_macroeconomics_light + - arabic_leaderboard_arabic_mmlu_high_school_mathematics_light + - arabic_leaderboard_arabic_mmlu_high_school_microeconomics_light + - arabic_leaderboard_arabic_mmlu_high_school_physics_light + - arabic_leaderboard_arabic_mmlu_high_school_psychology_light + - arabic_leaderboard_arabic_mmlu_high_school_statistics_light + - arabic_leaderboard_arabic_mmlu_high_school_us_history_light + - arabic_leaderboard_arabic_mmlu_high_school_world_history_light + - arabic_leaderboard_arabic_mmlu_human_aging_light + - arabic_leaderboard_arabic_mmlu_human_sexuality_light + - arabic_leaderboard_arabic_mmlu_international_law_light + - arabic_leaderboard_arabic_mmlu_jurisprudence_light + - arabic_leaderboard_arabic_mmlu_logical_fallacies_light + - arabic_leaderboard_arabic_mmlu_machine_learning_light + - arabic_leaderboard_arabic_mmlu_management_light + - arabic_leaderboard_arabic_mmlu_marketing_light + - arabic_leaderboard_arabic_mmlu_medical_genetics_light + - arabic_leaderboard_arabic_mmlu_miscellaneous_light + - arabic_leaderboard_arabic_mmlu_moral_disputes_light + - arabic_leaderboard_arabic_mmlu_moral_scenarios_light + - arabic_leaderboard_arabic_mmlu_nutrition_light + - arabic_leaderboard_arabic_mmlu_philosophy_light + - arabic_leaderboard_arabic_mmlu_prehistory_light + - arabic_leaderboard_arabic_mmlu_professional_accounting_light + - arabic_leaderboard_arabic_mmlu_professional_law_light + - arabic_leaderboard_arabic_mmlu_professional_medicine_light + - arabic_leaderboard_arabic_mmlu_professional_psychology_light + - arabic_leaderboard_arabic_mmlu_public_relations_light + - arabic_leaderboard_arabic_mmlu_security_studies_light + - arabic_leaderboard_arabic_mmlu_sociology_light + - arabic_leaderboard_arabic_mmlu_us_foreign_policy_light + - arabic_leaderboard_arabic_mmlu_virology_light + - arabic_leaderboard_arabic_mmlu_world_religions_light +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_logical_fallacies_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_logical_fallacies_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..866420ba28d7cad6f33e19d5d9ce10f5ae392b22 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_logical_fallacies_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_logical_fallacies_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: logical_fallacies +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_machine_learning_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_machine_learning_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..01ed181e01b8ae6b58cc41ff02abbaeb694aa32e --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_machine_learning_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_machine_learning_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: machine_learning +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_management_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_management_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..62d7e32ab072c230bd25a2ed72e24ddedf410c41 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_management_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_management_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: management +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_marketing_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_marketing_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c42f7a177b31740bd580cf9bcec5058c44b592b2 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_marketing_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_marketing_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: marketing +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_medical_genetics_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_medical_genetics_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..40d0d8832643ee081f5419d4c2643caee210ee45 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_medical_genetics_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_medical_genetics_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: medical_genetics +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_miscellaneous_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_miscellaneous_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..06bc6a4715abf48e512b39f63b4f53728087a607 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_miscellaneous_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_miscellaneous_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: miscellaneous +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_moral_disputes_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_moral_disputes_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..be0c60e63196c943bc5e03f633603bc0fb0062e3 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_moral_disputes_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_moral_disputes_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: moral_disputes +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_moral_scenarios_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_moral_scenarios_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..08e71366d10aa30f03be46453d2e342c9ff2d59d --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_moral_scenarios_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_moral_scenarios_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: moral_scenarios +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_nutrition_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_nutrition_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7987f5f36cc5a6db4a4faaee96e992e2dfd572d5 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_nutrition_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_nutrition_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: nutrition +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_philosophy_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_philosophy_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..85ebdd7a4faf23eaf1a6b187ad329da40e3a3d14 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_philosophy_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_philosophy_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: philosophy +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_prehistory_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_prehistory_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..24aa8e22fe5ec71dfbbfcd229f7a139800be80ad --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_prehistory_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_prehistory_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: prehistory +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_professional_accounting_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_professional_accounting_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1dc009663ce6f88da68decc91d87222669bebb8c --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_professional_accounting_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_professional_accounting_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: professional_accounting +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_professional_law_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_professional_law_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6e8c3617db705e9385bb89eebb59eec4014e6e40 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_professional_law_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_professional_law_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: professional_law +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_professional_medicine_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_professional_medicine_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b90cdb38d81a67477411aec313e9b30c80899c3d --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_professional_medicine_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_professional_medicine_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: professional_medicine +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_professional_psychology_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_professional_psychology_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..420a53624384ce6f8a397f4657d668395853ac25 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_professional_psychology_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_professional_psychology_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: professional_psychology +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_public_relations_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_public_relations_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..83d267bc085f3f16cc0f37363f1ccf0f04b42aac --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_public_relations_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_public_relations_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: public_relations +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_security_studies_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_security_studies_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..03e05d66e7045f885dd4bc00061b35e759ec77bd --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_security_studies_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_security_studies_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: security_studies +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_sociology_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_sociology_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7deb088396d5986964e59c1642fea63664542174 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_sociology_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_sociology_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: sociology +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_us_foreign_policy_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_us_foreign_policy_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6c5f40a55ed41f5f0d1f281c8e99f11db87fc0c1 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_us_foreign_policy_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_us_foreign_policy_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: us_foreign_policy +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_virology_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_virology_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5ee4a7c95b49e2a01edb41a9dbdddd295c2e768a --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_virology_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_virology_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: virology +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_world_religions_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_world_religions_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..57b13f05b85c70dff2e30ff66bcb01feb6512846 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/arabic_leaderboard_arabic_mmlu_world_religions_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_arabic_mmlu_world_religions_light +dataset_path: arcee-globe/Arabic_MMLU-10percent +dataset_name: world_religions +output_type: multiple_choice +training_split: null +validation_split: dev +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/utils.py b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..da927b66fcc95408aa648f655008ba072244291d --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mmlu_light/utils.py @@ -0,0 +1,35 @@ +import datasets +import numpy as np + + +# fmt: off +LETTER_INDICES_AR = ["أ", "ب", "ج", "د", "هـ", "و", "ز", "ح", "ط", "ي", "ك", "ل", "م", "ن", "س", "ع", "ف", "ص", "ق", "ر", "ش", "ت", "ث", "خ", "ذ", "ض", "ظ", "غ"] +# fmt: on + + +# fmt: off +LETTER_INDICES = ["A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z"] +# fmt: on + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + topic = doc["subject"] + instruction = f"الأسئلة التالية هي أسئلة متعددة الإختيارات مع الجواب الصحيح حول {topic.replace('_', ' ')}. \n\n" + choices = [doc["A"], doc["B"], doc["C"], doc["D"]] + # Answers are provided with roman letters - we look for the correct index in LETTER_INDICES, + # it will then be applied to arabic letters + gold_ix = LETTER_INDICES.index(doc["answer"]) + + query = f"{instruction}{doc['question']}\n" + query += "".join( + [ + f"{key}. {choice}\n" + for key, choice in zip(LETTER_INDICES_AR[:4], choices) + ] + ) + query += "الإجابة:" + + return {"query": query, "choices": LETTER_INDICES_AR[:4], "gold": gold_ix} + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_arc_challenge_light/arabic_leaderboard_arabic_mt_arc_challenge_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_arc_challenge_light/arabic_leaderboard_arabic_mt_arc_challenge_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a88bd6bd9edbf86e0153b550fb6d342167bc9b07 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_arc_challenge_light/arabic_leaderboard_arabic_mt_arc_challenge_light.yaml @@ -0,0 +1,13 @@ +group: arabic_leaderboard_arabic_mt_arc_challenge_light +task: + - arabic_mt_arc_challenge_light + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_arc_challenge_light/arabic_mt_arc_challenge_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_arc_challenge_light/arabic_mt_arc_challenge_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e6b299e846184473e3bce332392077903baf5f64 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_arc_challenge_light/arabic_mt_arc_challenge_light.yaml @@ -0,0 +1,23 @@ +task: arabic_mt_arc_challenge_light +dataset_path: arcee-globe/AlGhafa-Arabic-LLM-Benchmark-Translated-10percent +dataset_name: arc_challenge_okapi_ar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_arc_challenge_light/utils.py b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_arc_challenge_light/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..62f9874e63885c9ffbeedde8a4af4d13a8f06792 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_arc_challenge_light/utils.py @@ -0,0 +1,23 @@ +import datasets +import numpy as np + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + question = doc["query"] + answer_index = int(doc["label"]) + # Dynamically determining the choices by excluding '__few_shots', 'query' and 'label' + choices_keys = [ + key for key in doc.keys() if key not in ["query", "label", "__few_shots"] + ] + choices = [doc[key] for key in choices_keys] + + instruction = "الأسئلة التالية هي أسئلة متعددة الإختيارات مع الجواب الصحيح\n\n" + query = f"{instruction}السؤال: {question}\n" + for index, choice in enumerate(choices): + query += f"{index}) {choice}\n" + query += "الإجابة:" + + return {"query": query, "choices": choices, "gold": answer_index} + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_arc_easy_light/arabic_leaderboard_arabic_mt_arc_easy_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_arc_easy_light/arabic_leaderboard_arabic_mt_arc_easy_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..618b5429526e5443f177452e3e58a5235fbe0110 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_arc_easy_light/arabic_leaderboard_arabic_mt_arc_easy_light.yaml @@ -0,0 +1,13 @@ +group: arabic_leaderboard_arabic_mt_arc_easy_light +task: + - arabic_mt_arc_easy_light + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_arc_easy_light/arabic_mt_arc_easy_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_arc_easy_light/arabic_mt_arc_easy_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..90252fb31d572ee82959bc1bdc6a080bfcf7c43c --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_arc_easy_light/arabic_mt_arc_easy_light.yaml @@ -0,0 +1,23 @@ +task: arabic_mt_arc_easy_light +dataset_path: arcee-globe/AlGhafa-Arabic-LLM-Benchmark-Translated-10percent +dataset_name: arc_easy_ar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_arc_easy_light/utils.py b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_arc_easy_light/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..62f9874e63885c9ffbeedde8a4af4d13a8f06792 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_arc_easy_light/utils.py @@ -0,0 +1,23 @@ +import datasets +import numpy as np + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + question = doc["query"] + answer_index = int(doc["label"]) + # Dynamically determining the choices by excluding '__few_shots', 'query' and 'label' + choices_keys = [ + key for key in doc.keys() if key not in ["query", "label", "__few_shots"] + ] + choices = [doc[key] for key in choices_keys] + + instruction = "الأسئلة التالية هي أسئلة متعددة الإختيارات مع الجواب الصحيح\n\n" + query = f"{instruction}السؤال: {question}\n" + for index, choice in enumerate(choices): + query += f"{index}) {choice}\n" + query += "الإجابة:" + + return {"query": query, "choices": choices, "gold": answer_index} + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_boolq_light/arabic_leaderboard_arabic_mt_boolq_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_boolq_light/arabic_leaderboard_arabic_mt_boolq_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ee02f9cbc94fb0734d961500053fc66dfdc1cc36 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_boolq_light/arabic_leaderboard_arabic_mt_boolq_light.yaml @@ -0,0 +1,13 @@ +group: arabic_leaderboard_arabic_mt_boolq_light +task: + - arabic_mt_boolq_light + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_boolq_light/arabic_mt_boolq_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_boolq_light/arabic_mt_boolq_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4bdd145ce6e65562fa0814896291a401fa41a374 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_boolq_light/arabic_mt_boolq_light.yaml @@ -0,0 +1,23 @@ +task: arabic_mt_boolq_light +dataset_path: arcee-globe/AlGhafa-Arabic-LLM-Benchmark-Translated-10percent +dataset_name: boolq_ar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_boolq_light/utils.py b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_boolq_light/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..dcbc10d92e6d2938754a1a0dfbb1deabb810ed95 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_boolq_light/utils.py @@ -0,0 +1,24 @@ +import datasets +import numpy as np + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + question = doc["question"] + passage = doc["passage"] + instruction = "بناء على المقطع التالي، أجب عن السؤال ب نعم أو لا" + query = f"""{instruction} + المقطع : + {passage} + السؤال: + {question} + الإجابة: + """ + + return { + "query": query, + "choices": ["نعم", "لا"], + "gold": 0 if doc["answer"] else 1, + } + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_copa_light/arabic_mt_copa_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_copa_light/arabic_mt_copa_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0ca475e735d2c88ae6bd8b6562a095d881b0a97b --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_copa_light/arabic_mt_copa_light.yaml @@ -0,0 +1,23 @@ +task: arabic_mt_copa_light +dataset_path: arcee-globe/AlGhafa-Arabic-LLM-Benchmark-Translated-10percent +dataset_name: copa_ext_ar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_copa_light/arbic_leaderboard_arabic_mt_copa_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_copa_light/arbic_leaderboard_arabic_mt_copa_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f3ea35bc50d7ef9f984fcde9c3fed7778c792f85 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_copa_light/arbic_leaderboard_arabic_mt_copa_light.yaml @@ -0,0 +1,13 @@ +group: arabic_leaderboard_arabic_mt_copa_light +task: + - arabic_mt_copa_light + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_copa_light/utils.py b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_copa_light/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..175ebdadc1b21e79978a59a9a80782c339705b96 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_copa_light/utils.py @@ -0,0 +1,19 @@ +import datasets +import numpy as np + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + premise = doc["premise"] + choices = [doc["choice1"], doc["choice2"]] + question_map = {"cause": "لأن", "effect": "لذلك"} + question = question_map[doc["question"]] + answer = doc["label"] + + query = "{}، {} :\n0) {}\n1) {}\nالإجابة:".format( + premise, question, choices[0], choices[1] + ) + + return {"query": query, "choices": choices, "gold": answer} + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_hellaswag_light/arabic_leaderboard_arabic_mt_hellaswag_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_hellaswag_light/arabic_leaderboard_arabic_mt_hellaswag_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0f44bbbc75ecabd73c3a274094f139a07dd98321 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_hellaswag_light/arabic_leaderboard_arabic_mt_hellaswag_light.yaml @@ -0,0 +1,13 @@ +group: arabic_leaderboard_arabic_mt_hellaswag_light +task: + - arabic_mt_hellaswag_light + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_hellaswag_light/arabic_mt_hellaswag_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_hellaswag_light/arabic_mt_hellaswag_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..56ea04f2482edef51f517c1124af219997d16475 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_hellaswag_light/arabic_mt_hellaswag_light.yaml @@ -0,0 +1,23 @@ +task: arabic_mt_hellaswag_light +dataset_path: arcee-globe/AlGhafa-Arabic-LLM-Benchmark-Translated-10percent +dataset_name: hellaswag_okapi_ar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_hellaswag_light/utils.py b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_hellaswag_light/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..6b5a9f1f4f97460816957af2a4076836b4655c57 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_hellaswag_light/utils.py @@ -0,0 +1,30 @@ +import re + +import datasets +import numpy as np + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + ctx = re.sub(r"\[.*?\]", "", doc["ctx"]) # Remove latin words within brackets + endings = [ + re.sub(r"\[.*?\]", "", e) for e in eval(doc["endings"]) + ] # endings is a string representation of a list + answer_index = doc["label"] + instruction = ( + "بناء على السياق التالي، اختر النهاية الصحيحة من الاقتراحات التالية" + ) + + query = f"""{instruction} + السياق: + {ctx} + الاقتراحات: + + """ + for i, ending in enumerate(endings): + query += f"{i}) {ending}\n" + query += "الإجابة:" + + return {"query": query, "choices": endings, "gold": answer_index} + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_mmlu_light/arabic_leaderboard_arabic_mt_mmlu_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_mmlu_light/arabic_leaderboard_arabic_mt_mmlu_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b95ca1b531270aa9d84bda2d4e44cb2360e54da9 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_mmlu_light/arabic_leaderboard_arabic_mt_mmlu_light.yaml @@ -0,0 +1,13 @@ +group: arabic_leaderboard_arabic_mt_mmlu_light +task: + - arabic_mt_mmlu_light + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_mmlu_light/arabic_mt_mmlu_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_mmlu_light/arabic_mt_mmlu_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..43084db30bfd3eaf8d7558c0ee8ee0a7f39c178b --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_mmlu_light/arabic_mt_mmlu_light.yaml @@ -0,0 +1,23 @@ +task: arabic_mt_mmlu_light +dataset_path: arcee-globe/AlGhafa-Arabic-LLM-Benchmark-Translated-10percent +dataset_name: mmlu_okapi_ar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_mmlu_light/utils.py b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_mmlu_light/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..62f9874e63885c9ffbeedde8a4af4d13a8f06792 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_mmlu_light/utils.py @@ -0,0 +1,23 @@ +import datasets +import numpy as np + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + question = doc["query"] + answer_index = int(doc["label"]) + # Dynamically determining the choices by excluding '__few_shots', 'query' and 'label' + choices_keys = [ + key for key in doc.keys() if key not in ["query", "label", "__few_shots"] + ] + choices = [doc[key] for key in choices_keys] + + instruction = "الأسئلة التالية هي أسئلة متعددة الإختيارات مع الجواب الصحيح\n\n" + query = f"{instruction}السؤال: {question}\n" + for index, choice in enumerate(choices): + query += f"{index}) {choice}\n" + query += "الإجابة:" + + return {"query": query, "choices": choices, "gold": answer_index} + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_openbook_qa_light/arabic_leaderboard_arabic_mt_openbook_qa_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_openbook_qa_light/arabic_leaderboard_arabic_mt_openbook_qa_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3737f621fb3fadcff9906b46c2dd4258898e8921 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_openbook_qa_light/arabic_leaderboard_arabic_mt_openbook_qa_light.yaml @@ -0,0 +1,13 @@ +group: arabic_leaderboard_arabic_mt_openbook_qa_light +task: + - arabic_mt_openbook_qa_light + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_openbook_qa_light/arabic_mt_openbook_qa_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_openbook_qa_light/arabic_mt_openbook_qa_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5e914fbd32e5ea32e09eabbca56ad209cfd62dff --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_openbook_qa_light/arabic_mt_openbook_qa_light.yaml @@ -0,0 +1,23 @@ +task: arabic_mt_openbook_qa_light +dataset_path: arcee-globe/AlGhafa-Arabic-LLM-Benchmark-Translated-10percent +dataset_name: openbook_qa_ext_ar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_openbook_qa_light/utils.py b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_openbook_qa_light/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..62f9874e63885c9ffbeedde8a4af4d13a8f06792 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_openbook_qa_light/utils.py @@ -0,0 +1,23 @@ +import datasets +import numpy as np + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + question = doc["query"] + answer_index = int(doc["label"]) + # Dynamically determining the choices by excluding '__few_shots', 'query' and 'label' + choices_keys = [ + key for key in doc.keys() if key not in ["query", "label", "__few_shots"] + ] + choices = [doc[key] for key in choices_keys] + + instruction = "الأسئلة التالية هي أسئلة متعددة الإختيارات مع الجواب الصحيح\n\n" + query = f"{instruction}السؤال: {question}\n" + for index, choice in enumerate(choices): + query += f"{index}) {choice}\n" + query += "الإجابة:" + + return {"query": query, "choices": choices, "gold": answer_index} + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_piqa_light/arabic_leaderboard_arabic_mt_piqa_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_piqa_light/arabic_leaderboard_arabic_mt_piqa_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..642b2e0a60a473b3b0f5af0ffb91077622d3d97c --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_piqa_light/arabic_leaderboard_arabic_mt_piqa_light.yaml @@ -0,0 +1,13 @@ +group: arabic_leaderboard_arabic_mt_piqa_light +task: + - arabic_mt_piqa_light + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_piqa_light/arabic_mt_piqa_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_piqa_light/arabic_mt_piqa_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4dd9e005a949b68dbdedf02666e4db167b92877c --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_piqa_light/arabic_mt_piqa_light.yaml @@ -0,0 +1,23 @@ +task: arabic_mt_piqa_light +dataset_path: arcee-globe/AlGhafa-Arabic-LLM-Benchmark-Translated-10percent +dataset_name: piqa_ar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_piqa_light/utils.py b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_piqa_light/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..62f9874e63885c9ffbeedde8a4af4d13a8f06792 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_piqa_light/utils.py @@ -0,0 +1,23 @@ +import datasets +import numpy as np + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + question = doc["query"] + answer_index = int(doc["label"]) + # Dynamically determining the choices by excluding '__few_shots', 'query' and 'label' + choices_keys = [ + key for key in doc.keys() if key not in ["query", "label", "__few_shots"] + ] + choices = [doc[key] for key in choices_keys] + + instruction = "الأسئلة التالية هي أسئلة متعددة الإختيارات مع الجواب الصحيح\n\n" + query = f"{instruction}السؤال: {question}\n" + for index, choice in enumerate(choices): + query += f"{index}) {choice}\n" + query += "الإجابة:" + + return {"query": query, "choices": choices, "gold": answer_index} + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_race_light/arabic_leaderboard_arabic_mt_race_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_race_light/arabic_leaderboard_arabic_mt_race_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f427484d137b737f53a598ed36a3d208251aa94 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_race_light/arabic_leaderboard_arabic_mt_race_light.yaml @@ -0,0 +1,13 @@ +group: arabic_leaderboard_arabic_mt_race_light +task: + - arabic_mt_race_light + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_race_light/arabic_mt_race_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_race_light/arabic_mt_race_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fed452cce6b24cf54b1a0cfdeea7f3e61435bbdd --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_race_light/arabic_mt_race_light.yaml @@ -0,0 +1,23 @@ +task: arabic_mt_race_light +dataset_path: arcee-globe/AlGhafa-Arabic-LLM-Benchmark-Translated-10percent +dataset_name: race_ar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_race_light/utils.py b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_race_light/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..62f9874e63885c9ffbeedde8a4af4d13a8f06792 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_race_light/utils.py @@ -0,0 +1,23 @@ +import datasets +import numpy as np + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + question = doc["query"] + answer_index = int(doc["label"]) + # Dynamically determining the choices by excluding '__few_shots', 'query' and 'label' + choices_keys = [ + key for key in doc.keys() if key not in ["query", "label", "__few_shots"] + ] + choices = [doc[key] for key in choices_keys] + + instruction = "الأسئلة التالية هي أسئلة متعددة الإختيارات مع الجواب الصحيح\n\n" + query = f"{instruction}السؤال: {question}\n" + for index, choice in enumerate(choices): + query += f"{index}) {choice}\n" + query += "الإجابة:" + + return {"query": query, "choices": choices, "gold": answer_index} + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_sciq_light/arabic_leaderboard_arabic_mt_sciq_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_sciq_light/arabic_leaderboard_arabic_mt_sciq_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..13127e99155e79fb9658f8df685bc69968ff27e4 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_sciq_light/arabic_leaderboard_arabic_mt_sciq_light.yaml @@ -0,0 +1,13 @@ +group: arabic_leaderboard_arabic_mt_sciq_light +task: + - arabic_mt_sciq_light + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_sciq_light/arabic_mt_sciq_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_sciq_light/arabic_mt_sciq_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..95976cbb7fda4f640f7bfe1faee9b49988a1ee14 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_sciq_light/arabic_mt_sciq_light.yaml @@ -0,0 +1,23 @@ +task: arabic_mt_sciq_light +dataset_path: arcee-globe/AlGhafa-Arabic-LLM-Benchmark-Translated-10percent +dataset_name: sciq_ar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_sciq_light/utils.py b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_sciq_light/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..ddb42eeb8c8ee002662a6b3a6129ac4a8fa5007b --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_sciq_light/utils.py @@ -0,0 +1,41 @@ +import random + +import datasets +import numpy as np + + +def doc_to_text(doc): + instruction = ( + "بناءً على السياق أدناه، اختر الإجابة الصحيحة للسؤال التالي من قائمة الاقتراحات" + ) + support = doc["support"] + question = doc["question"] + query = f"""{instruction} + السياق: + {support} + السؤال: + {question} + الإجابات المحتملة: + + """ + return query + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + correct_answer = doc["correct_answer"] + choices = [ + doc["distractor1"], + doc["distractor2"], + doc["distractor3"], + correct_answer, + ] + + # Shuffle the choices + random.shuffle(choices) + + answer_index = choices.index(correct_answer) + + return {"query": doc_to_text(doc), "choices": choices, "gold": answer_index} + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_toxigen_light/arabic_leaderboard_arabic_mt_toxigen_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_toxigen_light/arabic_leaderboard_arabic_mt_toxigen_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7e305d54969efd73fa34a13bf66147cc6bd9f4c2 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_toxigen_light/arabic_leaderboard_arabic_mt_toxigen_light.yaml @@ -0,0 +1,13 @@ +group: arabic_leaderboard_arabic_mt_toxigen_light +task: + - arabic_mt_toxigen_light + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_toxigen_light/arabic_mt_toxigen_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_toxigen_light/arabic_mt_toxigen_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b2bef8abae83195eaa57ac6ce5562928362cf9c3 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_toxigen_light/arabic_mt_toxigen_light.yaml @@ -0,0 +1,23 @@ +task: arabic_mt_toxigen_light +dataset_path: arcee-globe/AlGhafa-Arabic-LLM-Benchmark-Translated-10percent +dataset_name: toxigen_ar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_toxigen_light/utils.py b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_toxigen_light/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..09d311e5a13746d41e5e2bcf7ce63fc905b4c72e --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_arabic_mt_toxigen_light/utils.py @@ -0,0 +1,23 @@ +import datasets +import numpy as np + + +def process_docs(dataset: datasets.Dataset) -> datasets.Dataset: + def _process_doc(doc): + text = doc["text"] + label = 1 if ((doc["toxicity_ai"] + doc["toxicity_human"]) > 5.5) else 0 + instruction = 'هل العبارة التالية تحض على الكراهية؟ قم بالرد بـ"نعم" أو "لا".' + + query = f"""{instruction} + العبارة: + '{text}' + الإجابة: + """ + out_doc = { + "query": query, + "choices": ["لا", "نعم"], + "gold": label, + } + return out_doc + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Algeria_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Algeria_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4ab4634f6017b253ba711bd7967e9ca62d7b6b04 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Algeria_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Algeria_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Algeria +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Ancient_Egypt_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Ancient_Egypt_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ab6fffedc1bd93447af973535afc329d7d4df31d --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Ancient_Egypt_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Ancient_Egypt_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Ancient_Egypt +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arab_Empire_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arab_Empire_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..886574ebf2333157430d47a25fe860da45523ca7 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arab_Empire_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arab_Empire_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Arab_Empire +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Architecture_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Architecture_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e57472ad6e30abcf966c7f16e44238a5ddd0f09f --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Architecture_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Architecture_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Arabic_Architecture +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Art_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Art_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e94340e7552d0bb918bb52f28a2d8fd58cef1940 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Art_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Art_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Arabic_Art +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Astronomy_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Astronomy_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e8ed990d524e3e48b861c25d38f090ce2e41d706 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Astronomy_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Astronomy_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Arabic_Astronomy +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Calligraphy_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Calligraphy_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cd41bdde6aaac743e5601724ae3718cd7bd93544 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Calligraphy_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Calligraphy_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Arabic_Calligraphy +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Ceremony_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Ceremony_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..72c6705479f323c805b73f52c62cacbf5ad5969a --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Ceremony_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Ceremony_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Arabic_Ceremony +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Clothing_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Clothing_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9348de07f783cdf3ce50fd5370c42b36a9f55d38 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Clothing_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Clothing_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Arabic_Clothing +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Culture_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Culture_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4f211064d6733f9c74347856fb571610911e916e --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Culture_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Culture_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Arabic_Culture +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Food_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Food_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7ccef6746fbc8f5ff67ff5739e023cb7d4e223e8 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Food_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Food_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Arabic_Food +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Funeral_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Funeral_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..941154787b8d10c23a610740fffbce223a5225b0 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Funeral_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Funeral_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Arabic_Funeral +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Geography_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Geography_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..36221d8899c0f34746af1b767be30dd0d32acac7 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Geography_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Geography_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Arabic_Geography +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_History_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_History_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2e128318163de2b5183edaf4e64cd980b2704ba9 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_History_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_History_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Arabic_History +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Language_Origin_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Language_Origin_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..806060435597599903279e1d811dcd19d7a93d74 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Language_Origin_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Language_Origin_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Arabic_Language_Origin +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Literature_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Literature_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3122e39531b0264bd08e2499c0e58969bc2dbdd1 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Literature_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Literature_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Arabic_Literature +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Math_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Math_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0182aedac757ee1008074e7a2b6b9a11444da874 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Math_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Math_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Arabic_Math +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Medicine_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Medicine_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aec88febf1e5545d7f3e2fd998506ca862924f87 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Medicine_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Medicine_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Arabic_Medicine +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Music_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Music_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..35a771898ad62c30a584334c6826b886cf5303a6 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Music_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Music_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Arabic_Music +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Ornament_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Ornament_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6b31186cd6c08d1b330e2b3a2d9154f5447b624d --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Ornament_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Ornament_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Arabic_Ornament +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Philosophy_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Philosophy_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f6b5fa71f1497bd7f425362665fbaee57ba220b7 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Philosophy_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Philosophy_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Arabic_Philosophy +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Physics_and_Chemistry_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Physics_and_Chemistry_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..559d729c9bfcee081bbec95372cb1e28c3e4148e --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Physics_and_Chemistry_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Physics_and_Chemistry_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Arabic_Physics_and_Chemistry +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Wedding_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Wedding_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9241709c139dc789002ff8dc13d48a226b82c627 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Arabic_Wedding_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Arabic_Wedding_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Arabic_Wedding +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Bahrain_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Bahrain_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b9c7cef57ace7b909b0be73fa88657a3e0e6a394 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Bahrain_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Bahrain_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Bahrain +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Comoros_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Comoros_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1f74bd46c52e3d50bfbd0a4045dc2d1530a35586 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Comoros_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Comoros_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Comoros +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Egypt_modern_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Egypt_modern_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e0b19cff586bb717d4ce33552f28d40d7fa1c1b1 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Egypt_modern_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Egypt_modern_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Egypt_modern +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_InfluenceFromAncientEgypt_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_InfluenceFromAncientEgypt_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6cf755a2a8b4f44999092bb74aa94b6836e9853f --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_InfluenceFromAncientEgypt_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_InfluenceFromAncientEgypt_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: InfluenceFromAncientEgypt +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_InfluenceFromByzantium_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_InfluenceFromByzantium_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8fe285eb12b3aeb3a7dfd5297158f75d904deaa7 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_InfluenceFromByzantium_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_InfluenceFromByzantium_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: InfluenceFromByzantium +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_InfluenceFromChina_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_InfluenceFromChina_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bb028b0892f077165a3fc1736bf9121065ceb9ad --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_InfluenceFromChina_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_InfluenceFromChina_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: InfluenceFromChina +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_InfluenceFromGreece_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_InfluenceFromGreece_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..25060acc1a1d18066fe0152364b704c22ec74ac0 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_InfluenceFromGreece_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_InfluenceFromGreece_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: InfluenceFromGreece +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_InfluenceFromIslam_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_InfluenceFromIslam_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0a60a2f3f0ec1444e3b21b87e4b7741a35d85b08 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_InfluenceFromIslam_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_InfluenceFromIslam_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: InfluenceFromIslam +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_InfluenceFromPersia_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_InfluenceFromPersia_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7081bec22796148d95b62deca003a41c9ef6a210 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_InfluenceFromPersia_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_InfluenceFromPersia_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: InfluenceFromPersia +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_InfluenceFromRome_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_InfluenceFromRome_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8c64cf3bbe636c0a43e1387a8bda4febcd2efd14 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_InfluenceFromRome_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_InfluenceFromRome_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: InfluenceFromRome +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Iraq_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Iraq_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a056a9cf04e0d4ea5696722c2f8414c0a3929cc2 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Iraq_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Iraq_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Iraq +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Islam_Education_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Islam_Education_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e8f6ad45d925302621d85a1bdc6fc2ff16c796b1 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Islam_Education_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Islam_Education_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Islam_Education +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Islam_branches_and_schools_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Islam_branches_and_schools_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..98137e9a3ab047dcd58793cdf9d9b15e45b98c26 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Islam_branches_and_schools_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Islam_branches_and_schools_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Islam_branches_and_schools +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Islamic_law_system_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Islamic_law_system_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d9aff345dacbf47b18f17eb4207b452d82dfe6db --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Islamic_law_system_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Islamic_law_system_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Islamic_law_system +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Jordan_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Jordan_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..674a998e0168231c8494d3ffa34ceb00e465ea5d --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Jordan_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Jordan_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Jordan +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Kuwait_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Kuwait_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0c3d372d9eb5213640fee9c75e882dae615da723 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Kuwait_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Kuwait_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Kuwait +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Lebanon_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Lebanon_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9c3856d698eab7d0ab6899caf7acab8deac36ff2 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Lebanon_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Lebanon_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Lebanon +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Libya_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Libya_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6070ccbfb884bbb77fff7b12f96ec443be654c09 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Libya_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Libya_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Libya +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Mauritania_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Mauritania_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0b1deda61449b19ee76e41c77a405d2b876ca203 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Mauritania_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Mauritania_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Mauritania +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Mesopotamia_civilization_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Mesopotamia_civilization_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..65474b724bc85755cc281fdf4815eb5f0b72438f --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Mesopotamia_civilization_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Mesopotamia_civilization_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Mesopotamia_civilization +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Morocco_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Morocco_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d752434a5a40a5520fdd741e98cfc5d24169b785 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Morocco_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Morocco_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Morocco +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Oman_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Oman_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..448498f4a115ca19cab4ebe3332f208f4c717a53 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Oman_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Oman_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Oman +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Palestine_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Palestine_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a619c460a1a353978c329e2f1756a23cdd77a2e2 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Palestine_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Palestine_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Palestine +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Qatar_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Qatar_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..967dbc57ef1fcd39d6a8ed6fbf630f56380496f7 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Qatar_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Qatar_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Qatar +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Saudi_Arabia_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Saudi_Arabia_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d45558b9ff3eabc0965c1c6b9f6af9d0685db2f9 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Saudi_Arabia_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Saudi_Arabia_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Saudi_Arabia +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Somalia_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Somalia_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..558ea176a30f039b3e9de8d9652412568e322998 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Somalia_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Somalia_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Somalia +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Sudan_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Sudan_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ce599733063f8eb767b4fbde58d8a8d2253311dd --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Sudan_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Sudan_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Sudan +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Syria_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Syria_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8b0bd7aebcd361b025f903e1fb9c603197b5bf97 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Syria_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Syria_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Syria +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Tunisia_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Tunisia_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a53c5e0bf9017efd6e8077e609b20f3ca0cce7dd --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Tunisia_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Tunisia_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: Tunisia +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_United_Arab_Emirates_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_United_Arab_Emirates_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1ce5993a674c98003fefd017a320b1f72d9c9df0 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_United_Arab_Emirates_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_United_Arab_Emirates_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: United_Arab_Emirates +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Yemen_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Yemen_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e480b19d605a0f28df819cc6e5eeef12eb9961ad --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_Yemen_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_Yemen_light +dataset_path: OALL/ACVA +dataset_name: Yemen +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_communication_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_communication_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2814278ace335717ce3ccedff63804d48fc1b722 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_communication_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_communication_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: communication +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_computer_and_phone_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_computer_and_phone_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ddd07e3f50c2f8eecf70ae7a8c3e6b4e7330e6b2 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_computer_and_phone_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_computer_and_phone_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: computer_and_phone +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_daily_life_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_daily_life_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2d975e4e85cef1b266b803d2916bcae5447ceee5 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_daily_life_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_daily_life_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: daily_life +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_entertainment_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_entertainment_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..721e6cdd3b366e21e7561584e27257c4a62bc505 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_entertainment_light.yaml @@ -0,0 +1,23 @@ +task: arabic_leaderboard_acva_entertainment_light +dataset_path: arcee-globe/ACVA-10percent +dataset_name: entertainment +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +process_docs: !function utils.process_docs +doc_to_text: "{{query}}" +doc_to_target: "{{gold}}" +doc_to_choice: "choices" +fewshot_split: validation +fewshot_config: + sampler: first_n +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ea4a89771f1414322b14eed184eef7d14a7c267a --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/arabic_leaderboard_acva_light.yaml @@ -0,0 +1,70 @@ +group: arabic_leaderboard_acva_light +task: + - arabic_leaderboard_acva_Algeria_light + - arabic_leaderboard_acva_Ancient_Egypt_light + - arabic_leaderboard_acva_Arab_Empire_light + - arabic_leaderboard_acva_Arabic_Architecture_light + - arabic_leaderboard_acva_Arabic_Art_light + - arabic_leaderboard_acva_Arabic_Astronomy_light + - arabic_leaderboard_acva_Arabic_Calligraphy_light + - arabic_leaderboard_acva_Arabic_Ceremony_light + - arabic_leaderboard_acva_Arabic_Clothing_light + - arabic_leaderboard_acva_Arabic_Culture_light + - arabic_leaderboard_acva_Arabic_Food_light + - arabic_leaderboard_acva_Arabic_Funeral_light + - arabic_leaderboard_acva_Arabic_Geography_light + - arabic_leaderboard_acva_Arabic_History_light + - arabic_leaderboard_acva_Arabic_Language_Origin_light + - arabic_leaderboard_acva_Arabic_Literature_light + - arabic_leaderboard_acva_Arabic_Math_light + - arabic_leaderboard_acva_Arabic_Medicine_light + - arabic_leaderboard_acva_Arabic_Music_light + - arabic_leaderboard_acva_Arabic_Ornament_light + - arabic_leaderboard_acva_Arabic_Philosophy_light + - arabic_leaderboard_acva_Arabic_Physics_and_Chemistry_light + - arabic_leaderboard_acva_Arabic_Wedding_light + - arabic_leaderboard_acva_Bahrain_light + - arabic_leaderboard_acva_Comoros_light + - arabic_leaderboard_acva_Egypt_modern_light + - arabic_leaderboard_acva_InfluenceFromAncientEgypt_light + - arabic_leaderboard_acva_InfluenceFromByzantium_light + - arabic_leaderboard_acva_InfluenceFromChina_light + - arabic_leaderboard_acva_InfluenceFromGreece_light + - arabic_leaderboard_acva_InfluenceFromIslam_light + - arabic_leaderboard_acva_InfluenceFromPersia_light + - arabic_leaderboard_acva_InfluenceFromRome_light + - arabic_leaderboard_acva_Iraq_light + - arabic_leaderboard_acva_Islam_Education_light + - arabic_leaderboard_acva_Islam_branches_and_schools_light + - arabic_leaderboard_acva_Islamic_law_system_light + - arabic_leaderboard_acva_Jordan_light + - arabic_leaderboard_acva_Kuwait_light + - arabic_leaderboard_acva_Lebanon_light + - arabic_leaderboard_acva_Libya_light + - arabic_leaderboard_acva_Mauritania_light + - arabic_leaderboard_acva_Mesopotamia_civilization_light + - arabic_leaderboard_acva_Morocco_light + - arabic_leaderboard_acva_Oman_light + - arabic_leaderboard_acva_Palestine_light + - arabic_leaderboard_acva_Qatar_light + - arabic_leaderboard_acva_Saudi_Arabia_light + - arabic_leaderboard_acva_Somalia_light + - arabic_leaderboard_acva_Sudan_light + - arabic_leaderboard_acva_Syria_light + - arabic_leaderboard_acva_Tunisia_light + - arabic_leaderboard_acva_United_Arab_Emirates_light + - arabic_leaderboard_acva_Yemen_light + - arabic_leaderboard_acva_communication_light + - arabic_leaderboard_acva_computer_and_phone_light + - arabic_leaderboard_acva_daily_life_light + - arabic_leaderboard_acva_entertainment_light + +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/utils.py b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..7e91496f59df5e940e4c206bceee69007c9f159c --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_avca_light/utils.py @@ -0,0 +1,16 @@ +import datasets +import numpy as np + + +def process_docs(dataset: datasets.Dataset): + def _process_doc(doc): + question = doc["question"] + answer = doc["answer"] + + return { + "query": f"السؤال: {question}\nالإجابة:", + "choices": ["صح", "خطأ"], + "gold": ["صح", "خطأ"].index(answer), + } + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_light.yaml b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_light.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d77ebd1eeb70ae2dfda1bccfd4ef8bbb0b2a2388 --- /dev/null +++ b/lm_eval/tasks/arabic_leaderboard_light/arabic_leaderboard_light.yaml @@ -0,0 +1,25 @@ +group: arabic_leaderboard_light +task: + - arabic_leaderboard_acva_light + - arabic_leaderboard_alghafa_light + - arabic_leaderboard_arabic_exams_light + - arabic_leaderboard_arabic_mt_arc_challenge_light + - arabic_leaderboard_arabic_mt_arc_easy_light + - arabic_leaderboard_arabic_mt_boolq_light + - arabic_leaderboard_arabic_mt_hellaswag_light + - arabic_leaderboard_arabic_mt_mmlu_light + - arabic_leaderboard_arabic_mt_copa_light + - arabic_leaderboard_arabic_mt_openbook_qa_light + - arabic_leaderboard_arabic_mt_piqa_light + - arabic_leaderboard_arabic_mt_race_light + - arabic_leaderboard_arabic_mt_sciq_light + - arabic_leaderboard_arabic_mt_toxigen_light +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/arabicmmlu/arabicmmlu.yaml b/lm_eval/tasks/arabicmmlu/_arabicmmlu.yaml similarity index 60% rename from lm_eval/tasks/arabicmmlu/arabicmmlu.yaml rename to lm_eval/tasks/arabicmmlu/_arabicmmlu.yaml index 89e9b3d598188616b651004df8dc8f785423d96a..58cf795aef2ef87d5e5421d549b2de9ec61617f0 100644 --- a/lm_eval/tasks/arabicmmlu/arabicmmlu.yaml +++ b/lm_eval/tasks/arabicmmlu/_arabicmmlu.yaml @@ -5,3 +5,8 @@ task: - arabicmmlu_humanities - arabicmmlu_stem - arabicmmlu_language +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0 diff --git a/lm_eval/tasks/arabicmmlu/_arabicmmlu_humanities.yaml b/lm_eval/tasks/arabicmmlu/_arabicmmlu_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6f61004a9c45c15606d4ef2385270e6525e4fe1c --- /dev/null +++ b/lm_eval/tasks/arabicmmlu/_arabicmmlu_humanities.yaml @@ -0,0 +1,9 @@ +group: arabicmmlu_humanities +group_alias: Humanities +task: + - arabicmmlu_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0 diff --git a/lm_eval/tasks/arabicmmlu/_arabicmmlu_language.yaml b/lm_eval/tasks/arabicmmlu/_arabicmmlu_language.yaml new file mode 100644 index 0000000000000000000000000000000000000000..90e57ae09bcdae6a581a88faaa2530d85df8ce28 --- /dev/null +++ b/lm_eval/tasks/arabicmmlu/_arabicmmlu_language.yaml @@ -0,0 +1,9 @@ +group: arabicmmlu_language +group_alias: Language +task: + - arabicmmlu_language_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0 diff --git a/lm_eval/tasks/arabicmmlu/_arabicmmlu_other.yaml b/lm_eval/tasks/arabicmmlu/_arabicmmlu_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3e989b8c6c884b9dfea9cde8fbd8ff0351e7adf2 --- /dev/null +++ b/lm_eval/tasks/arabicmmlu/_arabicmmlu_other.yaml @@ -0,0 +1,9 @@ +group: arabicmmlu_other +group_alias: Other +task: + - arabicmmlu_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0 diff --git a/lm_eval/tasks/arabicmmlu/_arabicmmlu_social_science.yaml b/lm_eval/tasks/arabicmmlu/_arabicmmlu_social_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1ece047b356aaac727da458ee9013fdab2b56c4a --- /dev/null +++ b/lm_eval/tasks/arabicmmlu/_arabicmmlu_social_science.yaml @@ -0,0 +1,9 @@ +group: arabicmmlu_social_science +group_alias: Social Science +task: + - arabicmmlu_social_science_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0 diff --git a/lm_eval/tasks/arabicmmlu/_arabicmmlu_stem.yaml b/lm_eval/tasks/arabicmmlu/_arabicmmlu_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a464a62a9825edf5e9158df85f062b9093ad25e6 --- /dev/null +++ b/lm_eval/tasks/arabicmmlu/_arabicmmlu_stem.yaml @@ -0,0 +1,9 @@ +group: arabicmmlu_stem +group_alias: STEM +task: + - arabicmmlu_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 0 diff --git a/lm_eval/tasks/arabicmmlu/_default_template_yaml b/lm_eval/tasks/arabicmmlu/_default_arabicmmlu_template_yaml similarity index 92% rename from lm_eval/tasks/arabicmmlu/_default_template_yaml rename to lm_eval/tasks/arabicmmlu/_default_arabicmmlu_template_yaml index 9af55ba7e7066b6d73e8ef806514bd5f6259483f..eac235773278bdf117108f631fd6780aab1c5349 100644 --- a/lm_eval/tasks/arabicmmlu/_default_template_yaml +++ b/lm_eval/tasks/arabicmmlu/_default_arabicmmlu_template_yaml @@ -11,3 +11,5 @@ metric_list: - metric: acc aggregation: mean higher_is_better: true +metadata: + version: 0.0 diff --git a/lm_eval/tasks/arabicmmlu/_generate_configs.py b/lm_eval/tasks/arabicmmlu/_generate_configs.py index 83e6594e50cf7de81bbc30302889ed7339e36232..4d091e126c52d5cdc1f7f16b13f74fad6e4667d3 100644 --- a/lm_eval/tasks/arabicmmlu/_generate_configs.py +++ b/lm_eval/tasks/arabicmmlu/_generate_configs.py @@ -59,7 +59,7 @@ SUBJECTS = { def parse_args(): parser = argparse.ArgumentParser() - parser.add_argument("--base_yaml_path", default="_default_template_yaml") + parser.add_argument("--base_yaml_path", default="_default_arabicmmlu_template_yaml") parser.add_argument("--save_prefix_path", default="arabicmmlu") return parser.parse_args() @@ -81,8 +81,7 @@ if __name__ == "__main__": yaml_dict = { "include": base_yaml_name, - "group": f"arabicmmlu_{category}", - "group_alias": category.replace("_", " "), + "tag": f"arabicmmlu_{category}", "task": f"arabicmmlu_{subject.lower().replace(' ', '_')}", "task_alias": subject, "dataset_name": subject, diff --git a/lm_eval/tasks/arabicmmlu/arabicmmlu_arabic_language_general.yaml b/lm_eval/tasks/arabicmmlu/arabicmmlu_arabic_language_general.yaml index 393109aa0b606ce94881894fa7d030883357f6cf..f57dc08c22086023626c4181dc78cd38c1de1900 100644 --- a/lm_eval/tasks/arabicmmlu/arabicmmlu_arabic_language_general.yaml +++ b/lm_eval/tasks/arabicmmlu/arabicmmlu_arabic_language_general.yaml @@ -1,6 +1,5 @@ "dataset_name": "Arabic Language (General)" -"group": "arabicmmlu_language" -"group_alias": "language" -"include": "_default_template_yaml" +"tag": "arabicmmlu_language_tasks" +"include": "_default_arabicmmlu_template_yaml" "task": "arabicmmlu_arabic_language_(general)" "task_alias": "Arabic Language (General)" diff --git a/lm_eval/tasks/arabicmmlu/arabicmmlu_arabic_language_grammar.yaml b/lm_eval/tasks/arabicmmlu/arabicmmlu_arabic_language_grammar.yaml index b077a398cc214ccc3e63c5f3fd20f34c623fe437..baf32676283101b319325e8a5e19ebae78e2d115 100644 --- a/lm_eval/tasks/arabicmmlu/arabicmmlu_arabic_language_grammar.yaml +++ b/lm_eval/tasks/arabicmmlu/arabicmmlu_arabic_language_grammar.yaml @@ -1,6 +1,5 @@ "dataset_name": "Arabic Language (Grammar)" -"group": "arabicmmlu_language" -"group_alias": "language" -"include": "_default_template_yaml" +"tag": "arabicmmlu_language_tasks" +"include": "_default_arabicmmlu_template_yaml" "task": "arabicmmlu_arabic_language_(grammar)" "task_alias": "Arabic Language (Grammar)" diff --git a/lm_eval/tasks/arabicmmlu/arabicmmlu_driving_test.yaml b/lm_eval/tasks/arabicmmlu/arabicmmlu_driving_test.yaml index 488633fd3088d0b79c9b395a80e00f368e77be2c..d40c9eb9d69b50b6fb6196733c40789e52dd2621 100644 --- a/lm_eval/tasks/arabicmmlu/arabicmmlu_driving_test.yaml +++ b/lm_eval/tasks/arabicmmlu/arabicmmlu_driving_test.yaml @@ -1,6 +1,5 @@ "dataset_name": "Driving Test" -"group": "arabicmmlu_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "arabicmmlu_other_tasks" +"include": "_default_arabicmmlu_template_yaml" "task": "arabicmmlu_driving_test" "task_alias": "Driving Test" diff --git a/lm_eval/tasks/arabicmmlu/arabicmmlu_general_knowledge.yaml b/lm_eval/tasks/arabicmmlu/arabicmmlu_general_knowledge.yaml index b328b4f918c90298750a5864099ce43c676c88be..fbd8839dba0a3b4fb552a5e9aae82da8f32d63cb 100644 --- a/lm_eval/tasks/arabicmmlu/arabicmmlu_general_knowledge.yaml +++ b/lm_eval/tasks/arabicmmlu/arabicmmlu_general_knowledge.yaml @@ -1,6 +1,5 @@ "dataset_name": "General Knowledge" -"group": "arabicmmlu_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "arabicmmlu_other_tasks" +"include": "_default_arabicmmlu_template_yaml" "task": "arabicmmlu_general_knowledge" "task_alias": "General Knowledge" diff --git a/lm_eval/tasks/arabicmmlu/arabicmmlu_high_arabic_language.yaml b/lm_eval/tasks/arabicmmlu/arabicmmlu_high_arabic_language.yaml index 3d0005a216fba6c413b29d86d81fba9189122bfa..17d17bc8b0421424e1fcb4332f83ad5a3bc1a8ae 100644 --- a/lm_eval/tasks/arabicmmlu/arabicmmlu_high_arabic_language.yaml +++ b/lm_eval/tasks/arabicmmlu/arabicmmlu_high_arabic_language.yaml @@ -1,6 +1,5 @@ "dataset_name": "High Arabic Language" -"group": "arabicmmlu_language" -"group_alias": "language" -"include": "_default_template_yaml" +"tag": "arabicmmlu_language_tasks" +"include": "_default_arabicmmlu_template_yaml" "task": "arabicmmlu_high_arabic_language" "task_alias": "High Arabic Language" diff --git a/lm_eval/tasks/arabicmmlu/arabicmmlu_high_biology.yaml b/lm_eval/tasks/arabicmmlu/arabicmmlu_high_biology.yaml index 1fc0b6a50c990d1ec4300d84f58106999e719904..2b5baf0bb958b2f8dbad17d7d280e59e82ef23ab 100644 --- a/lm_eval/tasks/arabicmmlu/arabicmmlu_high_biology.yaml +++ b/lm_eval/tasks/arabicmmlu/arabicmmlu_high_biology.yaml @@ -1,6 +1,5 @@ "dataset_name": "High Biology" -"group": "arabicmmlu_stem" -"group_alias": "stem" -"include": "_default_template_yaml" +"tag": "arabicmmlu_stem_tasks" +"include": "_default_arabicmmlu_template_yaml" "task": "arabicmmlu_high_biology" "task_alias": "High Biology" diff --git a/lm_eval/tasks/arabicmmlu/arabicmmlu_high_civics.yaml b/lm_eval/tasks/arabicmmlu/arabicmmlu_high_civics.yaml index c3bb13d55fb8d80dcae6fd0860dc9ad590462514..870509229c7d0035ba89a814e67fc36798faaab9 100644 --- a/lm_eval/tasks/arabicmmlu/arabicmmlu_high_civics.yaml +++ b/lm_eval/tasks/arabicmmlu/arabicmmlu_high_civics.yaml @@ -1,6 +1,5 @@ "dataset_name": "High Civics" -"group": "arabicmmlu_social_science" -"group_alias": "social science" -"include": "_default_template_yaml" +"tag": "arabicmmlu_social_science_tasks" +"include": "_default_arabicmmlu_template_yaml" "task": "arabicmmlu_high_civics" "task_alias": "High Civics" diff --git a/lm_eval/tasks/arabicmmlu/arabicmmlu_high_computer_science.yaml b/lm_eval/tasks/arabicmmlu/arabicmmlu_high_computer_science.yaml index d85f2f8aa854c14c75715ccca781634b390e66ab..f1a66a5ce87bb987b5fd389781835bee6fdd2079 100644 --- a/lm_eval/tasks/arabicmmlu/arabicmmlu_high_computer_science.yaml +++ b/lm_eval/tasks/arabicmmlu/arabicmmlu_high_computer_science.yaml @@ -1,6 +1,5 @@ "dataset_name": "High Computer Science" -"group": "arabicmmlu_stem" -"group_alias": "stem" -"include": "_default_template_yaml" +"tag": "arabicmmlu_stem_tasks" +"include": "_default_arabicmmlu_template_yaml" "task": "arabicmmlu_high_computer_science" "task_alias": "High Computer Science" diff --git a/lm_eval/tasks/arabicmmlu/arabicmmlu_high_economics.yaml b/lm_eval/tasks/arabicmmlu/arabicmmlu_high_economics.yaml index da2519d9b5eb3525563d1e20f42594778e730d8b..a1d6e90f537b855375606a3674b2e6cc32f6ef69 100644 --- a/lm_eval/tasks/arabicmmlu/arabicmmlu_high_economics.yaml +++ b/lm_eval/tasks/arabicmmlu/arabicmmlu_high_economics.yaml @@ -1,6 +1,5 @@ "dataset_name": "High Economics" -"group": "arabicmmlu_social_science" -"group_alias": "social science" -"include": "_default_template_yaml" +"tag": "arabicmmlu_social_science_tasks" +"include": "_default_arabicmmlu_template_yaml" "task": "arabicmmlu_high_economics" "task_alias": "High Economics" diff --git a/lm_eval/tasks/arabicmmlu/arabicmmlu_high_geography.yaml b/lm_eval/tasks/arabicmmlu/arabicmmlu_high_geography.yaml index 88ec5a02034867e00178719bad2d7a91bd8137e8..ad9804328b836253fd441536bec9aa4c9b21b8b0 100644 --- a/lm_eval/tasks/arabicmmlu/arabicmmlu_high_geography.yaml +++ b/lm_eval/tasks/arabicmmlu/arabicmmlu_high_geography.yaml @@ -1,6 +1,5 @@ "dataset_name": "High Geography" -"group": "arabicmmlu_social_science" -"group_alias": "social science" -"include": "_default_template_yaml" +"tag": "arabicmmlu_social_science_tasks" +"include": "_default_arabicmmlu_template_yaml" "task": "arabicmmlu_high_geography" "task_alias": "High Geography" diff --git a/lm_eval/tasks/arabicmmlu/arabicmmlu_high_history.yaml b/lm_eval/tasks/arabicmmlu/arabicmmlu_high_history.yaml index 6cdfdefe1590b6b785bc48f3fd444a9326fcb374..49c82669df895fcce907a7d9ad151b40983b7a90 100644 --- a/lm_eval/tasks/arabicmmlu/arabicmmlu_high_history.yaml +++ b/lm_eval/tasks/arabicmmlu/arabicmmlu_high_history.yaml @@ -1,6 +1,5 @@ "dataset_name": "High History" -"group": "arabicmmlu_humanities" -"group_alias": "humanities" -"include": "_default_template_yaml" +"tag": "arabicmmlu_humanities_tasks" +"include": "_default_arabicmmlu_template_yaml" "task": "arabicmmlu_high_history" "task_alias": "High History" diff --git a/lm_eval/tasks/arabicmmlu/arabicmmlu_high_islamic_studies.yaml b/lm_eval/tasks/arabicmmlu/arabicmmlu_high_islamic_studies.yaml index 23a7834fd81795c7e38c55c8a3191aa513c3c2f1..15b5358bd0a2d21917c4500bdeac6cabacdbd5d1 100644 --- a/lm_eval/tasks/arabicmmlu/arabicmmlu_high_islamic_studies.yaml +++ b/lm_eval/tasks/arabicmmlu/arabicmmlu_high_islamic_studies.yaml @@ -1,6 +1,5 @@ "dataset_name": "High Islamic Studies" -"group": "arabicmmlu_humanities" -"group_alias": "humanities" -"include": "_default_template_yaml" +"tag": "arabicmmlu_humanities_tasks" +"include": "_default_arabicmmlu_template_yaml" "task": "arabicmmlu_high_islamic_studies" "task_alias": "High Islamic Studies" diff --git a/lm_eval/tasks/arabicmmlu/arabicmmlu_high_philosophy.yaml b/lm_eval/tasks/arabicmmlu/arabicmmlu_high_philosophy.yaml index 0f61e3c7ffce23bc34f4cc6f46e2873e94a70465..e0b20e306dc269064eaa248849a80faef1a920ae 100644 --- a/lm_eval/tasks/arabicmmlu/arabicmmlu_high_philosophy.yaml +++ b/lm_eval/tasks/arabicmmlu/arabicmmlu_high_philosophy.yaml @@ -1,6 +1,5 @@ "dataset_name": "High Philosophy" -"group": "arabicmmlu_humanities" -"group_alias": "humanities" -"include": "_default_template_yaml" +"tag": "arabicmmlu_humanities_tasks" +"include": "_default_arabicmmlu_template_yaml" "task": "arabicmmlu_high_philosophy" "task_alias": "High Philosophy" diff --git a/lm_eval/tasks/arabicmmlu/arabicmmlu_high_physics.yaml b/lm_eval/tasks/arabicmmlu/arabicmmlu_high_physics.yaml index 2e510b0de6f8929776bcd1bede2631f833ad8f97..a7fe5eccdba8eb490f6d8aa0aca6f7b56d74ba49 100644 --- a/lm_eval/tasks/arabicmmlu/arabicmmlu_high_physics.yaml +++ b/lm_eval/tasks/arabicmmlu/arabicmmlu_high_physics.yaml @@ -1,6 +1,5 @@ "dataset_name": "High Physics" -"group": "arabicmmlu_stem" -"group_alias": "stem" -"include": "_default_template_yaml" +"tag": "arabicmmlu_stem_tasks" +"include": "_default_arabicmmlu_template_yaml" "task": "arabicmmlu_high_physics" "task_alias": "High Physics" diff --git a/lm_eval/tasks/arabicmmlu/arabicmmlu_islamic_studies.yaml b/lm_eval/tasks/arabicmmlu/arabicmmlu_islamic_studies.yaml index d97bbdccc9df86dc9b63c222c8c6dcb4154ffbfd..bacd5ace3feea344305c6579c3fdf98c55df04ae 100644 --- a/lm_eval/tasks/arabicmmlu/arabicmmlu_islamic_studies.yaml +++ b/lm_eval/tasks/arabicmmlu/arabicmmlu_islamic_studies.yaml @@ -1,6 +1,5 @@ "dataset_name": "Islamic Studies" -"group": "arabicmmlu_humanities" -"group_alias": "humanities" -"include": "_default_template_yaml" +"tag": "arabicmmlu_humanities_tasks" +"include": "_default_arabicmmlu_template_yaml" "task": "arabicmmlu_islamic_studies" "task_alias": "Islamic Studies" diff --git a/lm_eval/tasks/arabicmmlu/arabicmmlu_middle_arabic_language.yaml b/lm_eval/tasks/arabicmmlu/arabicmmlu_middle_arabic_language.yaml index efb6dd42a753dc37a98d2455ef7fd55c999e79d1..14a2ab1a5286857cf3747bc00a01839eeb348682 100644 --- a/lm_eval/tasks/arabicmmlu/arabicmmlu_middle_arabic_language.yaml +++ b/lm_eval/tasks/arabicmmlu/arabicmmlu_middle_arabic_language.yaml @@ -1,6 +1,5 @@ "dataset_name": "Middle Arabic Language" -"group": "arabicmmlu_language" -"group_alias": "language" -"include": "_default_template_yaml" +"tag": "arabicmmlu_language_tasks" +"include": "_default_arabicmmlu_template_yaml" "task": "arabicmmlu_middle_arabic_language" "task_alias": "Middle Arabic Language" diff --git a/lm_eval/tasks/arabicmmlu/arabicmmlu_middle_civics.yaml b/lm_eval/tasks/arabicmmlu/arabicmmlu_middle_civics.yaml index 9de4060b3ee5f32ce9c85c7cd060a0bd84e97af9..44ba95d458ccd08c720d30cf658bca719850f942 100644 --- a/lm_eval/tasks/arabicmmlu/arabicmmlu_middle_civics.yaml +++ b/lm_eval/tasks/arabicmmlu/arabicmmlu_middle_civics.yaml @@ -1,6 +1,5 @@ "dataset_name": "Middle Civics" -"group": "arabicmmlu_social_science" -"group_alias": "social science" -"include": "_default_template_yaml" +"tag": "arabicmmlu_social_science_tasks" +"include": "_default_arabicmmlu_template_yaml" "task": "arabicmmlu_middle_civics" "task_alias": "Middle Civics" diff --git a/lm_eval/tasks/arabicmmlu/arabicmmlu_middle_computer_science.yaml b/lm_eval/tasks/arabicmmlu/arabicmmlu_middle_computer_science.yaml index 635d09f65d88c18086adf865e00bdbd35480d8ce..8dd4136fb03ef49576e898a8576f2c33b37be3e8 100644 --- a/lm_eval/tasks/arabicmmlu/arabicmmlu_middle_computer_science.yaml +++ b/lm_eval/tasks/arabicmmlu/arabicmmlu_middle_computer_science.yaml @@ -1,6 +1,5 @@ "dataset_name": "Middle Computer Science" -"group": "arabicmmlu_stem" -"group_alias": "stem" -"include": "_default_template_yaml" +"tag": "arabicmmlu_stem_tasks" +"include": "_default_arabicmmlu_template_yaml" "task": "arabicmmlu_middle_computer_science" "task_alias": "Middle Computer Science" diff --git a/lm_eval/tasks/arabicmmlu/arabicmmlu_middle_economics.yaml b/lm_eval/tasks/arabicmmlu/arabicmmlu_middle_economics.yaml index ef55891e6f4e4bfb7d5e9c94e568af2930ee5846..312fa2e332f0eb8b5ff0801f27b9964d370862fc 100644 --- a/lm_eval/tasks/arabicmmlu/arabicmmlu_middle_economics.yaml +++ b/lm_eval/tasks/arabicmmlu/arabicmmlu_middle_economics.yaml @@ -1,6 +1,5 @@ "dataset_name": "Middle Economics" -"group": "arabicmmlu_social_science" -"group_alias": "social science" -"include": "_default_template_yaml" +"tag": "arabicmmlu_social_science_tasks" +"include": "_default_arabicmmlu_template_yaml" "task": "arabicmmlu_middle_economics" "task_alias": "Middle Economics" diff --git a/lm_eval/tasks/arabicmmlu/arabicmmlu_middle_general_knowledge.yaml b/lm_eval/tasks/arabicmmlu/arabicmmlu_middle_general_knowledge.yaml index 993fd832d7b98690863fb02472af215112d2493a..c359d85ac77792fb7373213ca32b5c7893e53c7c 100644 --- a/lm_eval/tasks/arabicmmlu/arabicmmlu_middle_general_knowledge.yaml +++ b/lm_eval/tasks/arabicmmlu/arabicmmlu_middle_general_knowledge.yaml @@ -1,6 +1,5 @@ "dataset_name": "Middle General Knowledge" -"group": "arabicmmlu_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "arabicmmlu_other_tasks" +"include": "_default_arabicmmlu_template_yaml" "task": "arabicmmlu_middle_general_knowledge" "task_alias": "Middle General Knowledge" diff --git a/lm_eval/tasks/arabicmmlu/arabicmmlu_middle_geography.yaml b/lm_eval/tasks/arabicmmlu/arabicmmlu_middle_geography.yaml index 2c84a47bdebfbb7f94a8329c837decf124f9525e..111b13cfe059afde2727bdc16ff3a2493304fa15 100644 --- a/lm_eval/tasks/arabicmmlu/arabicmmlu_middle_geography.yaml +++ b/lm_eval/tasks/arabicmmlu/arabicmmlu_middle_geography.yaml @@ -1,6 +1,5 @@ "dataset_name": "Middle Geography" -"group": "arabicmmlu_social_science" -"group_alias": "social science" -"include": "_default_template_yaml" +"tag": "arabicmmlu_social_science_tasks" +"include": "_default_arabicmmlu_template_yaml" "task": "arabicmmlu_middle_geography" "task_alias": "Middle Geography" diff --git a/lm_eval/tasks/arabicmmlu/arabicmmlu_middle_history.yaml b/lm_eval/tasks/arabicmmlu/arabicmmlu_middle_history.yaml index dcb84cb0194a31a2905d3a66d1afa7b6379bf61d..615a2e51f89b2b6e4489bbaf6420b27f7cf3c7c4 100644 --- a/lm_eval/tasks/arabicmmlu/arabicmmlu_middle_history.yaml +++ b/lm_eval/tasks/arabicmmlu/arabicmmlu_middle_history.yaml @@ -1,6 +1,5 @@ "dataset_name": "Middle History" -"group": "arabicmmlu_humanities" -"group_alias": "humanities" -"include": "_default_template_yaml" +"tag": "arabicmmlu_humanities_tasks" +"include": "_default_arabicmmlu_template_yaml" "task": "arabicmmlu_middle_history" "task_alias": "Middle History" diff --git a/lm_eval/tasks/arabicmmlu/arabicmmlu_middle_islamic_studies.yaml b/lm_eval/tasks/arabicmmlu/arabicmmlu_middle_islamic_studies.yaml index 273b044f4e73af0bac6decd8f406f0ff68962d1e..449223600e01bdef3fb97641efc421bbeb49b9dc 100644 --- a/lm_eval/tasks/arabicmmlu/arabicmmlu_middle_islamic_studies.yaml +++ b/lm_eval/tasks/arabicmmlu/arabicmmlu_middle_islamic_studies.yaml @@ -1,6 +1,5 @@ "dataset_name": "Middle Islamic Studies" -"group": "arabicmmlu_humanities" -"group_alias": "humanities" -"include": "_default_template_yaml" +"tag": "arabicmmlu_humanities_tasks" +"include": "_default_arabicmmlu_template_yaml" "task": "arabicmmlu_middle_islamic_studies" "task_alias": "Middle Islamic Studies" diff --git a/lm_eval/tasks/arabicmmlu/arabicmmlu_middle_natural_science.yaml b/lm_eval/tasks/arabicmmlu/arabicmmlu_middle_natural_science.yaml index 6c7b8109e636ef9459f30e85b4d531503c744da2..265cdbaa03b9ebf6015913b17579087d75f28365 100644 --- a/lm_eval/tasks/arabicmmlu/arabicmmlu_middle_natural_science.yaml +++ b/lm_eval/tasks/arabicmmlu/arabicmmlu_middle_natural_science.yaml @@ -1,6 +1,5 @@ "dataset_name": "Middle Natural Science" -"group": "arabicmmlu_stem" -"group_alias": "stem" -"include": "_default_template_yaml" +"tag": "arabicmmlu_stem_tasks" +"include": "_default_arabicmmlu_template_yaml" "task": "arabicmmlu_middle_natural_science" "task_alias": "Middle Natural Science" diff --git a/lm_eval/tasks/arabicmmlu/arabicmmlu_middle_social_science.yaml b/lm_eval/tasks/arabicmmlu/arabicmmlu_middle_social_science.yaml index 1b71ad5f128b587f09d239d4c8f67b4a105d762c..84c247dd17e04a595c3c783171e452b6e7f4667f 100644 --- a/lm_eval/tasks/arabicmmlu/arabicmmlu_middle_social_science.yaml +++ b/lm_eval/tasks/arabicmmlu/arabicmmlu_middle_social_science.yaml @@ -1,6 +1,5 @@ "dataset_name": "Middle Social Science" -"group": "arabicmmlu_social_science" -"group_alias": "social science" -"include": "_default_template_yaml" +"tag": "arabicmmlu_social_science_tasks" +"include": "_default_arabicmmlu_template_yaml" "task": "arabicmmlu_middle_social_science" "task_alias": "Middle Social Science" diff --git a/lm_eval/tasks/arabicmmlu/arabicmmlu_primary_arabic_language.yaml b/lm_eval/tasks/arabicmmlu/arabicmmlu_primary_arabic_language.yaml index 3f13179bbf168d6eddf2e82221f1c3f310c3a316..700bc0781b5741661bbecc7c755f1be16647ec52 100644 --- a/lm_eval/tasks/arabicmmlu/arabicmmlu_primary_arabic_language.yaml +++ b/lm_eval/tasks/arabicmmlu/arabicmmlu_primary_arabic_language.yaml @@ -1,6 +1,5 @@ "dataset_name": "Primary Arabic Language" -"group": "arabicmmlu_language" -"group_alias": "language" -"include": "_default_template_yaml" +"tag": "arabicmmlu_language_tasks" +"include": "_default_arabicmmlu_template_yaml" "task": "arabicmmlu_primary_arabic_language" "task_alias": "Primary Arabic Language" diff --git a/lm_eval/tasks/arabicmmlu/arabicmmlu_primary_computer_science.yaml b/lm_eval/tasks/arabicmmlu/arabicmmlu_primary_computer_science.yaml index d8d92318952ce5553a7e349205ed2f07f6f3c33f..b89089cd2f6eb278ac452b0dcc95bde643bcef20 100644 --- a/lm_eval/tasks/arabicmmlu/arabicmmlu_primary_computer_science.yaml +++ b/lm_eval/tasks/arabicmmlu/arabicmmlu_primary_computer_science.yaml @@ -1,6 +1,5 @@ "dataset_name": "Primary Computer Science" -"group": "arabicmmlu_stem" -"group_alias": "stem" -"include": "_default_template_yaml" +"tag": "arabicmmlu_stem_tasks" +"include": "_default_arabicmmlu_template_yaml" "task": "arabicmmlu_primary_computer_science" "task_alias": "Primary Computer Science" diff --git a/lm_eval/tasks/arabicmmlu/arabicmmlu_primary_general_knowledge.yaml b/lm_eval/tasks/arabicmmlu/arabicmmlu_primary_general_knowledge.yaml index 57472db367bbd1bfbb419c313260b9c9ead88f45..85dd0b7fa7ce7db9e97c8928fef0dfab47fe4c24 100644 --- a/lm_eval/tasks/arabicmmlu/arabicmmlu_primary_general_knowledge.yaml +++ b/lm_eval/tasks/arabicmmlu/arabicmmlu_primary_general_knowledge.yaml @@ -1,6 +1,5 @@ "dataset_name": "Primary General Knowledge" -"group": "arabicmmlu_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "arabicmmlu_other_tasks" +"include": "_default_arabicmmlu_template_yaml" "task": "arabicmmlu_primary_general_knowledge" "task_alias": "Primary General Knowledge" diff --git a/lm_eval/tasks/arabicmmlu/arabicmmlu_primary_geography.yaml b/lm_eval/tasks/arabicmmlu/arabicmmlu_primary_geography.yaml index 66c01fe84ced8c1e3f0bf07197fc57ffbc68b62b..f7efc48783c965c1eb1430834d9fbb08c25244ed 100644 --- a/lm_eval/tasks/arabicmmlu/arabicmmlu_primary_geography.yaml +++ b/lm_eval/tasks/arabicmmlu/arabicmmlu_primary_geography.yaml @@ -1,6 +1,5 @@ "dataset_name": "Primary Geography" -"group": "arabicmmlu_social_science" -"group_alias": "social science" -"include": "_default_template_yaml" +"tag": "arabicmmlu_social_science_tasks" +"include": "_default_arabicmmlu_template_yaml" "task": "arabicmmlu_primary_geography" "task_alias": "Primary Geography" diff --git a/lm_eval/tasks/arabicmmlu/arabicmmlu_primary_history.yaml b/lm_eval/tasks/arabicmmlu/arabicmmlu_primary_history.yaml index b8c913a4bb2c824ea2b746fb4b8a3028260cddee..f7d69ca975eb8a735cf50b4ceaeef53b292a6674 100644 --- a/lm_eval/tasks/arabicmmlu/arabicmmlu_primary_history.yaml +++ b/lm_eval/tasks/arabicmmlu/arabicmmlu_primary_history.yaml @@ -1,6 +1,5 @@ "dataset_name": "Primary History" -"group": "arabicmmlu_humanities" -"group_alias": "humanities" -"include": "_default_template_yaml" +"tag": "arabicmmlu_humanities_tasks" +"include": "_default_arabicmmlu_template_yaml" "task": "arabicmmlu_primary_history" "task_alias": "Primary History" diff --git a/lm_eval/tasks/arabicmmlu/arabicmmlu_primary_islamic_studies.yaml b/lm_eval/tasks/arabicmmlu/arabicmmlu_primary_islamic_studies.yaml index a1c12653662bce8b006719e40a983623f778970a..b36cd640376b0079dbb1cb609106de7266e2cd76 100644 --- a/lm_eval/tasks/arabicmmlu/arabicmmlu_primary_islamic_studies.yaml +++ b/lm_eval/tasks/arabicmmlu/arabicmmlu_primary_islamic_studies.yaml @@ -1,6 +1,5 @@ "dataset_name": "Primary Islamic Studies" -"group": "arabicmmlu_humanities" -"group_alias": "humanities" -"include": "_default_template_yaml" +"tag": "arabicmmlu_humanities_tasks" +"include": "_default_arabicmmlu_template_yaml" "task": "arabicmmlu_primary_islamic_studies" "task_alias": "Primary Islamic Studies" diff --git a/lm_eval/tasks/arabicmmlu/arabicmmlu_primary_math.yaml b/lm_eval/tasks/arabicmmlu/arabicmmlu_primary_math.yaml index d908d5807b4a1196c59a0563f4b71faf65b9b3f7..0e53adcfa7fcccfafc69e1f763dafec8dd4425db 100644 --- a/lm_eval/tasks/arabicmmlu/arabicmmlu_primary_math.yaml +++ b/lm_eval/tasks/arabicmmlu/arabicmmlu_primary_math.yaml @@ -1,6 +1,5 @@ "dataset_name": "Primary Math" -"group": "arabicmmlu_stem" -"group_alias": "stem" -"include": "_default_template_yaml" +"tag": "arabicmmlu_stem_tasks" +"include": "_default_arabicmmlu_template_yaml" "task": "arabicmmlu_primary_math" "task_alias": "Primary Math" diff --git a/lm_eval/tasks/arabicmmlu/arabicmmlu_primary_natural_science.yaml b/lm_eval/tasks/arabicmmlu/arabicmmlu_primary_natural_science.yaml index d5a513cf69d1dc11270c39062e6504199d61f239..4e208c76faca4f773d88738d0bec0e6ca55895a8 100644 --- a/lm_eval/tasks/arabicmmlu/arabicmmlu_primary_natural_science.yaml +++ b/lm_eval/tasks/arabicmmlu/arabicmmlu_primary_natural_science.yaml @@ -1,6 +1,5 @@ "dataset_name": "Primary Natural Science" -"group": "arabicmmlu_stem" -"group_alias": "stem" -"include": "_default_template_yaml" +"tag": "arabicmmlu_stem_tasks" +"include": "_default_arabicmmlu_template_yaml" "task": "arabicmmlu_primary_natural_science" "task_alias": "Primary Natural Science" diff --git a/lm_eval/tasks/arabicmmlu/arabicmmlu_primary_social_science.yaml b/lm_eval/tasks/arabicmmlu/arabicmmlu_primary_social_science.yaml index bd4cfc29cc21ed341a09555d04cbc752b0852686..fee4fe5d0b182c24016b5080f0d605f8f22ef2ee 100644 --- a/lm_eval/tasks/arabicmmlu/arabicmmlu_primary_social_science.yaml +++ b/lm_eval/tasks/arabicmmlu/arabicmmlu_primary_social_science.yaml @@ -1,6 +1,5 @@ "dataset_name": "Primary Social Science" -"group": "arabicmmlu_social_science" -"group_alias": "social science" -"include": "_default_template_yaml" +"tag": "arabicmmlu_social_science_tasks" +"include": "_default_arabicmmlu_template_yaml" "task": "arabicmmlu_primary_social_science" "task_alias": "Primary Social Science" diff --git a/lm_eval/tasks/arabicmmlu/arabicmmlu_prof_law.yaml b/lm_eval/tasks/arabicmmlu/arabicmmlu_prof_law.yaml index bd1252d04786c1f93a2fc9442fffd43c6bdf65cc..20bf6c5f49ba05df166d589e761d2e3ac4ef43e3 100644 --- a/lm_eval/tasks/arabicmmlu/arabicmmlu_prof_law.yaml +++ b/lm_eval/tasks/arabicmmlu/arabicmmlu_prof_law.yaml @@ -1,6 +1,5 @@ "dataset_name": "Prof Law" -"group": "arabicmmlu_humanities" -"group_alias": "humanities" -"include": "_default_template_yaml" +"tag": "arabicmmlu_humanities_tasks" +"include": "_default_arabicmmlu_template_yaml" "task": "arabicmmlu_prof_law" "task_alias": "Prof Law" diff --git a/lm_eval/tasks/arabicmmlu/arabicmmlu_univ_accounting.yaml b/lm_eval/tasks/arabicmmlu/arabicmmlu_univ_accounting.yaml index 2d1f39287e142c69445b4b5ae6c847a746dfb5c2..6d1d94125edcb6ffb7ee13853ce3b8d0014dbd18 100644 --- a/lm_eval/tasks/arabicmmlu/arabicmmlu_univ_accounting.yaml +++ b/lm_eval/tasks/arabicmmlu/arabicmmlu_univ_accounting.yaml @@ -1,6 +1,5 @@ "dataset_name": "Univ Accounting" -"group": "arabicmmlu_social_science" -"group_alias": "social science" -"include": "_default_template_yaml" +"tag": "arabicmmlu_social_science_tasks" +"include": "_default_arabicmmlu_template_yaml" "task": "arabicmmlu_univ_accounting" "task_alias": "Univ Accounting" diff --git a/lm_eval/tasks/arabicmmlu/arabicmmlu_univ_computer_science.yaml b/lm_eval/tasks/arabicmmlu/arabicmmlu_univ_computer_science.yaml index bf1725efe5b2c9e05ebccf42da2c95add744f2c8..42e7e89a965a006f7e8a6ae171057662d3ac93ba 100644 --- a/lm_eval/tasks/arabicmmlu/arabicmmlu_univ_computer_science.yaml +++ b/lm_eval/tasks/arabicmmlu/arabicmmlu_univ_computer_science.yaml @@ -1,6 +1,5 @@ "dataset_name": "Univ Computer Science" -"group": "arabicmmlu_stem" -"group_alias": "stem" -"include": "_default_template_yaml" +"tag": "arabicmmlu_stem_tasks" +"include": "_default_arabicmmlu_template_yaml" "task": "arabicmmlu_univ_computer_science" "task_alias": "Univ Computer Science" diff --git a/lm_eval/tasks/arabicmmlu/arabicmmlu_univ_economics.yaml b/lm_eval/tasks/arabicmmlu/arabicmmlu_univ_economics.yaml index ed1494e5e3894740eba790f7e52b50601b2cb38a..21015ffad848c98398a0b4d54e71ea8e4dd58ba4 100644 --- a/lm_eval/tasks/arabicmmlu/arabicmmlu_univ_economics.yaml +++ b/lm_eval/tasks/arabicmmlu/arabicmmlu_univ_economics.yaml @@ -1,6 +1,5 @@ "dataset_name": "Univ Economics" -"group": "arabicmmlu_social_science" -"group_alias": "social science" -"include": "_default_template_yaml" +"tag": "arabicmmlu_social_science_tasks" +"include": "_default_arabicmmlu_template_yaml" "task": "arabicmmlu_univ_economics" "task_alias": "Univ Economics" diff --git a/lm_eval/tasks/arabicmmlu/arabicmmlu_univ_management.yaml b/lm_eval/tasks/arabicmmlu/arabicmmlu_univ_management.yaml index d72e35325fb20eaf94eddf659e04c61bcd542cdb..e69ad74b3da4b7bbf5c8891b1340daecb74e6654 100644 --- a/lm_eval/tasks/arabicmmlu/arabicmmlu_univ_management.yaml +++ b/lm_eval/tasks/arabicmmlu/arabicmmlu_univ_management.yaml @@ -1,6 +1,5 @@ "dataset_name": "Univ Management" -"group": "arabicmmlu_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "arabicmmlu_other_tasks" +"include": "_default_arabicmmlu_template_yaml" "task": "arabicmmlu_univ_management" "task_alias": "Univ Management" diff --git a/lm_eval/tasks/arabicmmlu/arabicmmlu_univ_political_science.yaml b/lm_eval/tasks/arabicmmlu/arabicmmlu_univ_political_science.yaml index fe32b63b2cda65d6d83472794544f35354a04911..bb85a104e1c6ffac215772f0266acaee894c0a37 100644 --- a/lm_eval/tasks/arabicmmlu/arabicmmlu_univ_political_science.yaml +++ b/lm_eval/tasks/arabicmmlu/arabicmmlu_univ_political_science.yaml @@ -1,6 +1,5 @@ "dataset_name": "Univ Political Science" -"group": "arabicmmlu_social_science" -"group_alias": "social science" -"include": "_default_template_yaml" +"tag": "arabicmmlu_social_science_tasks" +"include": "_default_arabicmmlu_template_yaml" "task": "arabicmmlu_univ_political_science" "task_alias": "Univ Political Science" diff --git a/lm_eval/tasks/arc/README.md b/lm_eval/tasks/arc/README.md index b3eea45b45f4ba80e437446011fbde62642bc636..2677d4c151f75880e29101b001ac94789a641768 100644 --- a/lm_eval/tasks/arc/README.md +++ b/lm_eval/tasks/arc/README.md @@ -29,10 +29,14 @@ Homepage: https://allenai.org/data/arc } ``` -### Groups and Tasks +### Groups, Tags, and Tasks #### Groups +None. + +#### Tags + * `ai2_arc`: Evaluates `arc_easy` and `arc_challenge` #### Tasks diff --git a/lm_eval/tasks/arc/arc_easy.yaml b/lm_eval/tasks/arc/arc_easy.yaml index b07629d331eaf91a272da31e2bf1e7cad273130a..1b2e369a4e19a37c6a1550a1ab033701fc045621 100644 --- a/lm_eval/tasks/arc/arc_easy.yaml +++ b/lm_eval/tasks/arc/arc_easy.yaml @@ -1,4 +1,4 @@ -group: +tag: - ai2_arc task: arc_easy dataset_path: allenai/ai2_arc diff --git a/lm_eval/tasks/arc_mt/arc_challenge_mt_fi.yaml b/lm_eval/tasks/arc_mt/arc_challenge_mt_fi.yaml index d2032e34a4f25b3e83371aac1dad1e59ac6a176d..6a17c5c1943037771f6b18d2581096bd145160b2 100644 --- a/lm_eval/tasks/arc_mt/arc_challenge_mt_fi.yaml +++ b/lm_eval/tasks/arc_mt/arc_challenge_mt_fi.yaml @@ -1,4 +1,4 @@ -group: +tag: - arc_challenge_mt task: arc_challenge_mt_fi dataset_path: LumiOpen/arc_challenge_mt diff --git a/lm_eval/tasks/arithmetic/README.md b/lm_eval/tasks/arithmetic/README.md index e7d7f89efbbd3af29e5e1c28b1af1adb93073569..7bfc27cbc13f0fe8051ace916c67e7b54ca612ea 100644 --- a/lm_eval/tasks/arithmetic/README.md +++ b/lm_eval/tasks/arithmetic/README.md @@ -27,9 +27,9 @@ Homepage: https://github.com/openai/gpt-3/tree/master/data } ``` -### Groups and Tasks +### Groups, Tags, and Tasks -#### Groups +#### Tags * `arithmetic`: Evaluates `1dc` to `5ds` diff --git a/lm_eval/tasks/arithmetic/arithmetic_1dc.yaml b/lm_eval/tasks/arithmetic/arithmetic_1dc.yaml index 3e8d414a60c1f9df7c635fafd34b7a2f39a36865..2e9c9277bcde23f513674bf0da8d3b92c1428418 100644 --- a/lm_eval/tasks/arithmetic/arithmetic_1dc.yaml +++ b/lm_eval/tasks/arithmetic/arithmetic_1dc.yaml @@ -1,4 +1,4 @@ -group: +tag: - arithmetic task: arithmetic_1dc dataset_path: EleutherAI/arithmetic diff --git a/lm_eval/tasks/asdiv/README.md b/lm_eval/tasks/asdiv/README.md index e6e5aeec0403b8c854233089498c9248cf38f089..11ffaf810a26dd1b0741e9ffa3e9e83c96362939 100644 --- a/lm_eval/tasks/asdiv/README.md +++ b/lm_eval/tasks/asdiv/README.md @@ -32,7 +32,7 @@ Homepage: https://github.com/chaochun/nlu-asdiv-dataset } ``` -### Groups and Tasks +### Groups, Tags, and Tasks #### Groups @@ -41,6 +41,11 @@ Homepage: https://github.com/chaochun/nlu-asdiv-dataset #### Tasks * `asdiv` +* `asdiv_cot_llama`: ASDIV with prompt formatting modified to conform to the evaluation settings described by Meta here: https://huggingface.co/datasets/meta-llama/Meta-Llama-3.1-8B-Instruct-evals/viewer/Meta-Llama-3.1-8B-Instruct-evals__gsm8k__details?row=0 + - Note that the CoT prompt from (https://arxiv.org/pdf/2201.11903) is used exactly as in GSM8k-CoT + - This file is setup to run identically to the task `gsm8k_cot_llama` but for asdiv. + - Use this task with --fewshot_as_multiturn and --apply_chat_template to run correctly with Llama Instruct models. + ### Checklist diff --git a/lm_eval/tasks/asdiv/asdiv-cot-llama.yaml b/lm_eval/tasks/asdiv/asdiv-cot-llama.yaml new file mode 100644 index 0000000000000000000000000000000000000000..344ba223d9f35863e04a803c1cd11c70d2f106c8 --- /dev/null +++ b/lm_eval/tasks/asdiv/asdiv-cot-llama.yaml @@ -0,0 +1,88 @@ +dataset_path: EleutherAI/asdiv +doc_to_target: "{{answer.split(' (')[0] if answer is defined else target}}" +doc_to_text: "Given the following problem, reason and give a final answer to the problem.\nProblem: {{body if body is defined}} {{question}}\nYour response should end with \"The final answer is [answer]\" where [answer] is the response to the problem.\n" +fewshot_config: + sampler: first_n + samples: + - question: There are 15 trees in the grove. Grove workers will plant trees in the + grove today. After they are done, there will be 21 trees. How many trees did + the grove workers plant today? + target: There are 15 trees originally. Then there were 21 trees after some more + were planted. So there must have been 21 - 15 = 6. The final answer is 6 + - question: If there are 3 cars in the parking lot and 2 more cars arrive, how many + cars are in the parking lot? + target: There are originally 3 cars. 2 more cars arrive. 3 + 2 = 5. The final answer + is 5 + - question: Leah had 32 chocolates and her sister had 42. If they ate 35, how many + pieces do they have left in total? + target: Originally, Leah had 32 chocolates. Her sister had 42. So in total they + had 32 + 42 = 74. After eating 35, they had 74 - 35 = 39. The final answer is 39 + - question: Jason had 20 lollipops. He gave Denny some lollipops. Now Jason has 12 + lollipops. How many lollipops did Jason give to Denny? + target: Jason started with 20 lollipops. Then he had 12 after giving some to Denny. + So he gave Denny 20 - 12 = 8. The final answer is 8 + - question: Shawn has five toys. For Christmas, he got two toys each from his mom and + dad. How many toys does he have now? + target: Shawn started with 5 toys. If he got 2 toys each from his mom and dad, + then that is 4 more toys. 5 + 4 = 9. The final answer is 9 + - question: There were nine computers in the server room. Five more computers were + installed each day, from monday to thursday. How many computers are now in the + server room? + target: There were originally 9 computers. For each of 4 days, 5 more computers + were added. So 5 * 4 = 20 computers were added. 9 + 20 is 29. The final answer is + 29 + - question: Michael had 58 golf balls. On tuesday, he lost 23 golf balls. On wednesday, + he lost 2 more. How many golf balls did he have at the end of wednesday? + target: Michael started with 58 golf balls. After losing 23 on tuesday, he had + 58 - 23 = 35. After losing 2 more, he had 35 - 2 = 33 golf balls. The final answer + is 33 + - question: Olivia has $23. She bought five bagels for $3 each. How much money does + she have left? + target: Olivia had 23 dollars. 5 bagels for 3 dollars each will be 5 x 3 = 15 + dollars. So she has 23 - 15 dollars left. 23 - 15 is 8. The final answer is 8 +filter_list: +- filter: + - function: regex + group_select: -1 + regex_pattern: The final answer is ((-?[$0-9.,]{2,})|(-?[0-9]+)) + - function: take_first + name: strict-match +- filter: + - function: regex + group_select: -1 + regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+) + - function: take_first + name: flexible-extract +generation_kwargs: + do_sample: false + until: + - '<|eot_id|>' + - '<|start_header_id|>user<|end_header_id|>' + - 'Q:' + - + - <|im_end|> +tag: +- chain_of_thought +metadata: + version: 1.0 +metric_list: +- aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: false + metric: exact_match + regexes_to_ignore: + - ',' + - \$ + - '(?s).*#### ' + - \.$ +num_fewshot: 8 +output_type: generate_until +repeats: 1 +task: asdiv_cot_llama +validation_split: validation +test_split: validation +should_decontaminate: true +doc_to_decontamination_query: "{{body}} {{question}}" +dataset_kwargs: + trust_remote_code: true diff --git a/lm_eval/tasks/babi/README.md b/lm_eval/tasks/babi/README.md index a19798ab8d67bcb2a5cca192cfc599dc2a153e53..4943d08b660587ac8e84c65e41dab8bc226292b2 100644 --- a/lm_eval/tasks/babi/README.md +++ b/lm_eval/tasks/babi/README.md @@ -21,12 +21,16 @@ Homepage: https://github.com/facebookarchive/bAbI-tasks } ``` -### Groups and Tasks +### Groups, Tags, and Tasks #### Groups * Not part of a group yet +#### Tags + +* No tags applied. + #### Tasks * `babi` diff --git a/lm_eval/tasks/basque_bench/README.md b/lm_eval/tasks/basque_bench/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7b5f93c1f62ce2b1720949a33ed91842e9a04188 --- /dev/null +++ b/lm_eval/tasks/basque_bench/README.md @@ -0,0 +1,88 @@ +# BasqueBench + +### Paper + +BasqueBench is a benchmark for evaluating language models in Basque tasks. This is, it evaluates the ability of a language model to understand and generate Basque text. BasqueBench offers a combination of pre-existing, open datasets and datasets developed exclusivelly for this benchmark. All the details of BasqueBench will be published in a paper soon. + +The new evaluation datasets included in BasqueBench are: +| Task | Category | Homepage | +|:-------------:|:-----:|:-----:| +| MGSM_eu | Math | https://huggingface.co/datasets/HiTZ/MGSM-eu | +| WNLI_eu | Natural Language Inference | https://huggingface.co/datasets/HiTZ/wnli-eu | +| XCOPA_eu | Commonsense Reasoning | https://huggingface.co/datasets/HiTZ/XCOPA-eu | + +The datasets included in BasqueBench that have been made public in previous pubications are: + +| Task | Category | Paper title | Homepage | +|:-------------:|:-----:|:-------------:|:-----:| +| Belebele_eu | Reading Comprehension | [The Belebele Benchmark: a Parallel Reading Comprehension Dataset in 122 Language Variants](https://arxiv.org/abs/2308.16884) | https://huggingface.co/datasets/facebook/belebele | +| EusExams | Question Answering | [Latxa: An Open Language Model and Evaluation Suite for Basque](https://arxiv.org/abs/2403.20266) | https://huggingface.co/datasets/HiTZ/EusExams | +| EusProficiency | Question Answering | [Latxa: An Open Language Model and Evaluation Suite for Basque](https://arxiv.org/abs/2403.20266) | https://huggingface.co/datasets/HiTZ/EusProficiency | +| EusReading | Reading Comprehension | [Latxa: An Open Language Model and Evaluation Suite for Basque](https://arxiv.org/abs/2403.20266) | https://huggingface.co/datasets/HiTZ/EusReading | +| EusTrivia | Question Answering | [Latxa: An Open Language Model and Evaluation Suite for Basque](https://arxiv.org/abs/2403.20266) | https://huggingface.co/datasets/HiTZ/EusTrivia | +| FLORES_eu | Translation | [No Language Left Behind: Scaling Human-Centered Machine Translation](https://arxiv.org/abs/2207.04672) | https://huggingface.co/datasets/facebook/flores | +| QNLIeu | Natural Language Inference | [BasqueGLUE: A Natural Language Understanding Benchmark for Basque](https://aclanthology.org/2022.lrec-1.172/) | https://huggingface.co/datasets/orai-nlp/basqueGLUE | +| XNLIeu | Natural Language Inference | [XNLIeu: a dataset for cross-lingual NLI in Basque](https://arxiv.org/abs/2404.06996) | https://huggingface.co/datasets/HiTZ/xnli-eu | +| XStoryCloze_eu | Commonsense Reasoning | [Few-shot Learning with Multilingual Generative Language Models](https://aclanthology.org/2022.emnlp-main.616/) | https://huggingface.co/datasets/juletxara/xstory_cloze | + + +### Citation +Paper for BasqueBench coming soon. + +### Groups and Tasks + +#### Groups + +- `basque_bench`: All tasks included in BasqueBench. +- `flores_eu`: All FLORES translation tasks from or to Basque. + +#### Tasks + +The following tasks evaluate tasks on BasqueBench dataset using various scoring methods. + - `belebele_eus_Latn` + - `eus_exams_eu` + - `eus_proficiency` + - `eus_reading` + - `eus_trivia` + - `flores_eu` + - `flores_eu-ca` + - `flores_eu-de` + - `flores_eu-en` + - `flores_eu-es` + - `flores_eu-fr` + - `flores_eu-gl` + - `flores_eu-it` + - `flores_eu-pt` + - `flores_ca-eu` + - `flores_de-eu` + - `flores_en-eu` + - `flores_es-eu` + - `flores_fr-eu` + - `flores_gl-eu` + - `flores_it-eu` + - `flores_pt-eu` + - `mgsm_direct_eu` + - `mgsm_native_cot_eu` + - `qnlieu` + - `wnli_eu` + - `xcopa_eu` + - `xnli_eu` + - `xnli_eu_native` + - `xstorycloze_eu` + +Some of these tasks are taken from benchmarks already available in LM Evaluation Harness. These are: +- `belebele_eus_Latn`: Belebele Basque +- `qnlieu`: From BasqueGLUE + + +### Checklist + +* [x] Is the task an existing benchmark in the literature? + * [ ] Have you referenced the original paper that introduced the task? + * [ ] If yes, does the original paper provide a reference implementation? + * [ ] Yes, original implementation contributed by author of the benchmark + +If other tasks on this dataset are already supported: +* [ ] Is the "Main" variant of this task clearly denoted? +* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [ ] Have you noted which, if any, published evaluation setups are matched by this variant? diff --git a/lm_eval/tasks/basque_bench/basque_bench.yaml b/lm_eval/tasks/basque_bench/basque_bench.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f542d21802c477ba9fb70daba5d0c70baa84fd1f --- /dev/null +++ b/lm_eval/tasks/basque_bench/basque_bench.yaml @@ -0,0 +1,18 @@ +group: basque_bench +task: + - belebele_eus_Latn + - xstorycloze_eu + - flores_eu + - eus_reading + - eus_proficiency + - eus_trivia + - eus_exams_eu + - qnlieu + - xnli_eu + - xnli_eu_native + - wnli_eu + - xcopa_eu + - mgsm_direct_eu + - mgsm_native_cot_eu +metadata: + version: 1.0 diff --git a/lm_eval/tasks/basque_bench/flores_eu/_flores_common_yaml b/lm_eval/tasks/basque_bench/flores_eu/_flores_common_yaml new file mode 100644 index 0000000000000000000000000000000000000000..dbbde071fcbcc2f95040e72cec9abfb2c9ecfbe3 --- /dev/null +++ b/lm_eval/tasks/basque_bench/flores_eu/_flores_common_yaml @@ -0,0 +1,27 @@ +tag: flores +dataset_path: facebook/flores +dataset_name: all +output_type: generate_until +#! The test split of flores is not publicly available! (See paper section 6.1) +training_split: dev +validation_split: dev +test_split: devtest +fewshot_split: dev +target_delimiter: '' +generation_kwargs: + until: + - "\n" +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: ter + aggregation: ter + higher_is_better: false + - metric: chrf + aggregation: chrf + higher_is_better: true +metadata: + version: 0.1 +dataset_kwargs: + trust_remote_code: true diff --git a/lm_eval/tasks/basque_bench/flores_eu/create_yamls_flores_eu.py b/lm_eval/tasks/basque_bench/flores_eu/create_yamls_flores_eu.py new file mode 100644 index 0000000000000000000000000000000000000000..723edc51b746c01097a4f787bf12df0e2d569c4d --- /dev/null +++ b/lm_eval/tasks/basque_bench/flores_eu/create_yamls_flores_eu.py @@ -0,0 +1,333 @@ +# ruff: noqa: E731, E741 +""" +Script to generate task YAMLs for the FLORES-200 dataset. +Based on `tasks/translation/utils.py`. +""" + +import argparse +import itertools + +import yaml +from langcodes import Language + + +# utils +flatten = lambda l: list(itertools.chain(*l)) + +# constants +_LANGUAGES = [ + "ace_Arab", + "bam_Latn", + "dzo_Tibt", + "hin_Deva", + "khm_Khmr", + "mag_Deva", + "pap_Latn", + "sot_Latn", + "tur_Latn", + "ace_Latn", + "ban_Latn", + "ell_Grek", + "hne_Deva", + "kik_Latn", + "mai_Deva", + "pbt_Arab", + "spa_Latn", + "twi_Latn", + "acm_Arab", + "bel_Cyrl", + "eng_Latn", + "hrv_Latn", + "kin_Latn", + "mal_Mlym", + "pes_Arab", + "srd_Latn", + "tzm_Tfng", + "acq_Arab", + "bem_Latn", + "epo_Latn", + "hun_Latn", + "kir_Cyrl", + "mar_Deva", + "plt_Latn", + "srp_Cyrl", + "uig_Arab", + "aeb_Arab", + "ben_Beng", + "est_Latn", + "hye_Armn", + "kmb_Latn", + "min_Arab", + "pol_Latn", + "ssw_Latn", + "ukr_Cyrl", + "afr_Latn", + "bho_Deva", + "eus_Latn", + "ibo_Latn", + "kmr_Latn", + "min_Latn", + "por_Latn", + "sun_Latn", + "umb_Latn", + "ajp_Arab", + "bjn_Arab", + "ewe_Latn", + "ilo_Latn", + "knc_Arab", + "mkd_Cyrl", + "prs_Arab", + "swe_Latn", + "urd_Arab", + "aka_Latn", + "bjn_Latn", + "fao_Latn", + "ind_Latn", + "knc_Latn", + "mlt_Latn", + "quy_Latn", + "swh_Latn", + "uzn_Latn", + "als_Latn", + "bod_Tibt", + "fij_Latn", + "isl_Latn", + "kon_Latn", + "mni_Beng", + "ron_Latn", + "szl_Latn", + "vec_Latn", + "amh_Ethi", + "bos_Latn", + "fin_Latn", + "ita_Latn", + "kor_Hang", + "mos_Latn", + "run_Latn", + "tam_Taml", + "vie_Latn", + "apc_Arab", + "bug_Latn", + "fon_Latn", + "jav_Latn", + "lao_Laoo", + "mri_Latn", + "rus_Cyrl", + "taq_Latn", + "war_Latn", + "arb_Arab", + "bul_Cyrl", + "fra_Latn", + "jpn_Jpan", + "lij_Latn", + "mya_Mymr", + "sag_Latn", + "taq_Tfng", + "wol_Latn", + "arb_Latn", + "cat_Latn", + "fur_Latn", + "kab_Latn", + "lim_Latn", + "nld_Latn", + "san_Deva", + "tat_Cyrl", + "xho_Latn", + "ars_Arab", + "ceb_Latn", + "fuv_Latn", + "kac_Latn", + "lin_Latn", + "nno_Latn", + "sat_Olck", + "tel_Telu", + "ydd_Hebr", + "ary_Arab", + "ces_Latn", + "gaz_Latn", + "kam_Latn", + "lit_Latn", + "nob_Latn", + "scn_Latn", + "tgk_Cyrl", + "yor_Latn", + "arz_Arab", + "cjk_Latn", + "gla_Latn", + "kan_Knda", + "lmo_Latn", + "npi_Deva", + "shn_Mymr", + "tgl_Latn", + "yue_Hant", + "asm_Beng", + "ckb_Arab", + "gle_Latn", + "kas_Arab", + "ltg_Latn", + "nso_Latn", + "sin_Sinh", + "tha_Thai", + "zho_Hans", + "ast_Latn", + "crh_Latn", + "glg_Latn", + "kas_Deva", + "ltz_Latn", + "nus_Latn", + "slk_Latn", + "tir_Ethi", + "zho_Hant", + "awa_Deva", + "cym_Latn", + "grn_Latn", + "kat_Geor", + "lua_Latn", + "nya_Latn", + "slv_Latn", + "tpi_Latn", + "zsm_Latn", + "ayr_Latn", + "dan_Latn", + "guj_Gujr", + "kaz_Cyrl", + "lug_Latn", + "oci_Latn", + "smo_Latn", + "tsn_Latn", + "zul_Latn", + "azb_Arab", + "deu_Latn", + "hat_Latn", + "kbp_Latn", + "luo_Latn", + "ory_Orya", + "sna_Latn", + "tso_Latn", + "azj_Latn", + "dik_Latn", + "hau_Latn", + "kea_Latn", + "lus_Latn", + "pag_Latn", + "snd_Arab", + "tuk_Latn", + "bak_Cyrl", + "dyu_Latn", + "heb_Hebr", + "khk_Cyrl", + "lvs_Latn", + "pan_Guru", + "som_Latn", + "tum_Latn", +] +LANGUAGE_PAIRS = [ + (a, b) for idx, a in enumerate(_LANGUAGES) for b in _LANGUAGES[idx + 1 :] +] + +LANGUAGES_OF_INTEREST = [ + "cat_Latn", + "spa_Latn", + "eng_Latn", + "glg_Latn", + "eus_Latn", + "ita_Latn", + "deu_Latn", + "por_Latn", + "fra_Latn", +] +MAIN_LANG = "eus_Latn" +LANGUAGE_PAIRS = [ + (a, b) + for (a, b) in LANGUAGE_PAIRS + if a in LANGUAGES_OF_INTEREST and b in LANGUAGES_OF_INTEREST and MAIN_LANG in (a, b) +] + +# auxiliary functions + +code_to_language_name = lambda code: Language.make( + language=Language.get(code)["language"] +).display_name() +code_to_short_name = lambda code: Language.get(code)["language"] +jinja_var = ( + lambda s: "{{" + s + "}}" +) # wrapper to avoid having to escape { } in format strings + + +def doc_to_text(src: str, tgt: str) -> str: + src_name, tgt_name = map(code_to_language_name, [src, tgt]) + + return f"""\ +{src_name} sentence: {jinja_var('sentence_' + src)} +{tgt_name} sentence:""" + + +def doc_to_target(tgt: str) -> str: + return f"{jinja_var('sentence_' + tgt)}" + + +# main function + + +def gen_lang_yamls(output_dir: str, overwrite: bool) -> None: + """ + Generate a YAML file for each translation direction. + """ + + err = [] + for src, tgt in LANGUAGE_PAIRS: + # do both translation directions for each lang pair + for src, tgt in [(src, tgt), (tgt, src)]: + lang_pair_name = f"{code_to_short_name(src)}-{code_to_short_name(tgt)}" + yaml_file_name = f"flores_{lang_pair_name}.yaml" + + try: + with open( + f"{output_dir}/{yaml_file_name}", + "w" if overwrite else "x", + encoding="utf-8", + ) as outfile: + print(f"Creating {yaml_file_name}...") + outfile.write("# File generated by `create-yamls.py`\n") + yaml.dump( + { + # "group": [f"{BENCH_NAME}_bench", f"{BENCH_NAME}_bench_flores"], + # "group": "flores_eu", + "include": "_flores_common_yaml", + "task": f"flores_{lang_pair_name}", + "doc_to_text": doc_to_text(src, tgt), + "doc_to_target": doc_to_target(tgt), + }, + outfile, + sort_keys=False, + ) + + except FileExistsError: + err.append(yaml_file_name) + + if len(err) > 0: + raise FileExistsError( + "Files were not created because they already exist:" + f" {', '.join(err)}" + "\nUse flag --overwrite to overwrite them." + ) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "--overwrite", + default=False, + action="store_true", + help="Overwrite files if they already exist", + ) + parser.add_argument( + "--output-dir", default=".", help="Directory to write yaml files to" + ) + args = parser.parse_args() + + gen_lang_yamls(output_dir=args.output_dir, overwrite=args.overwrite) + + +if __name__ == "__main__": + main() diff --git a/lm_eval/tasks/basque_bench/flores_eu/flores_ca-eu.yaml b/lm_eval/tasks/basque_bench/flores_eu/flores_ca-eu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..48ffe7bf5c7fc356177cb923006e5f57b793e7c3 --- /dev/null +++ b/lm_eval/tasks/basque_bench/flores_eu/flores_ca-eu.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_ca-eu +doc_to_text: 'Catalan sentence: {{sentence_cat_Latn}} + + Basque sentence:' +doc_to_target: '{{sentence_eus_Latn}}' diff --git a/lm_eval/tasks/basque_bench/flores_eu/flores_de-eu.yaml b/lm_eval/tasks/basque_bench/flores_eu/flores_de-eu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..16bb7772b594fe89acb00b852ed376243a0c30b4 --- /dev/null +++ b/lm_eval/tasks/basque_bench/flores_eu/flores_de-eu.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_de-eu +doc_to_text: 'German sentence: {{sentence_deu_Latn}} + + Basque sentence:' +doc_to_target: '{{sentence_eus_Latn}}' diff --git a/lm_eval/tasks/basque_bench/flores_eu/flores_en-eu.yaml b/lm_eval/tasks/basque_bench/flores_eu/flores_en-eu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d26edaca38826077482b3f270bd18190254fa623 --- /dev/null +++ b/lm_eval/tasks/basque_bench/flores_eu/flores_en-eu.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_en-eu +doc_to_text: 'English sentence: {{sentence_eng_Latn}} + + Basque sentence:' +doc_to_target: '{{sentence_eus_Latn}}' diff --git a/lm_eval/tasks/basque_bench/flores_eu/flores_es-eu.yaml b/lm_eval/tasks/basque_bench/flores_eu/flores_es-eu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..576bb0e2708bb93a60074e3938a16f661e05c362 --- /dev/null +++ b/lm_eval/tasks/basque_bench/flores_eu/flores_es-eu.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_es-eu +doc_to_text: 'Spanish sentence: {{sentence_spa_Latn}} + + Basque sentence:' +doc_to_target: '{{sentence_eus_Latn}}' diff --git a/lm_eval/tasks/basque_bench/flores_eu/flores_eu-ca.yaml b/lm_eval/tasks/basque_bench/flores_eu/flores_eu-ca.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c8be6ee93b64c33ba177f11f3494504eaf17c175 --- /dev/null +++ b/lm_eval/tasks/basque_bench/flores_eu/flores_eu-ca.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_eu-ca +doc_to_text: 'Basque sentence: {{sentence_eus_Latn}} + + Catalan sentence:' +doc_to_target: '{{sentence_cat_Latn}}' diff --git a/lm_eval/tasks/basque_bench/flores_eu/flores_eu-de.yaml b/lm_eval/tasks/basque_bench/flores_eu/flores_eu-de.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4f5735a6739ed2b669894d8092ed45fc8f97add8 --- /dev/null +++ b/lm_eval/tasks/basque_bench/flores_eu/flores_eu-de.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_eu-de +doc_to_text: 'Basque sentence: {{sentence_eus_Latn}} + + German sentence:' +doc_to_target: '{{sentence_deu_Latn}}' diff --git a/lm_eval/tasks/basque_bench/flores_eu/flores_eu-en.yaml b/lm_eval/tasks/basque_bench/flores_eu/flores_eu-en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5d9eadfb93590eef5ad16f28d537fcf4007407d7 --- /dev/null +++ b/lm_eval/tasks/basque_bench/flores_eu/flores_eu-en.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_eu-en +doc_to_text: 'Basque sentence: {{sentence_eus_Latn}} + + English sentence:' +doc_to_target: '{{sentence_eng_Latn}}' diff --git a/lm_eval/tasks/basque_bench/flores_eu/flores_eu-es.yaml b/lm_eval/tasks/basque_bench/flores_eu/flores_eu-es.yaml new file mode 100644 index 0000000000000000000000000000000000000000..efb5200d086732b12fed80ec8fce4eb2865e13cc --- /dev/null +++ b/lm_eval/tasks/basque_bench/flores_eu/flores_eu-es.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_eu-es +doc_to_text: 'Basque sentence: {{sentence_eus_Latn}} + + Spanish sentence:' +doc_to_target: '{{sentence_spa_Latn}}' diff --git a/lm_eval/tasks/basque_bench/flores_eu/flores_eu-fr.yaml b/lm_eval/tasks/basque_bench/flores_eu/flores_eu-fr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a9ce32a811d19c2b30303fcd86d416c1e26f9f75 --- /dev/null +++ b/lm_eval/tasks/basque_bench/flores_eu/flores_eu-fr.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_eu-fr +doc_to_text: 'Basque sentence: {{sentence_eus_Latn}} + + French sentence:' +doc_to_target: '{{sentence_fra_Latn}}' diff --git a/lm_eval/tasks/basque_bench/flores_eu/flores_eu-gl.yaml b/lm_eval/tasks/basque_bench/flores_eu/flores_eu-gl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..db762cf75c90985a9b87459587508fd429070e98 --- /dev/null +++ b/lm_eval/tasks/basque_bench/flores_eu/flores_eu-gl.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_eu-gl +doc_to_text: 'Basque sentence: {{sentence_eus_Latn}} + + Galician sentence:' +doc_to_target: '{{sentence_glg_Latn}}' diff --git a/lm_eval/tasks/basque_bench/flores_eu/flores_eu-it.yaml b/lm_eval/tasks/basque_bench/flores_eu/flores_eu-it.yaml new file mode 100644 index 0000000000000000000000000000000000000000..91a77f0d41f3e61c235bdddd18cb94f3a4b40018 --- /dev/null +++ b/lm_eval/tasks/basque_bench/flores_eu/flores_eu-it.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_eu-it +doc_to_text: 'Basque sentence: {{sentence_eus_Latn}} + + Italian sentence:' +doc_to_target: '{{sentence_ita_Latn}}' diff --git a/lm_eval/tasks/basque_bench/flores_eu/flores_eu-pt.yaml b/lm_eval/tasks/basque_bench/flores_eu/flores_eu-pt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f230a7323ef5974aed0b6ed84871e00e17e0d208 --- /dev/null +++ b/lm_eval/tasks/basque_bench/flores_eu/flores_eu-pt.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_eu-pt +doc_to_text: 'Basque sentence: {{sentence_eus_Latn}} + + Portuguese sentence:' +doc_to_target: '{{sentence_por_Latn}}' diff --git a/lm_eval/tasks/basque_bench/flores_eu/flores_eu.yaml b/lm_eval/tasks/basque_bench/flores_eu/flores_eu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e389bddfacd199fc30959b267f9d00191b4e4a3a --- /dev/null +++ b/lm_eval/tasks/basque_bench/flores_eu/flores_eu.yaml @@ -0,0 +1,24 @@ +group: flores_eu +task: + - flores_es-eu + - flores_eu-es + - flores_en-eu + - flores_eu-en + - flores_eu-pt + - flores_pt-eu + - flores_eu-it + - flores_it-eu + - flores_eu-fr + - flores_fr-eu + - flores_eu-ca + - flores_ca-eu + - flores_eu-gl + - flores_gl-eu + - flores_eu-de + - flores_de-eu +aggregate_metric_list: + - metric: bleu + aggregation: mean + weight_by_size: false +metadata: + version: 1.0 diff --git a/lm_eval/tasks/basque_bench/flores_eu/flores_fr-eu.yaml b/lm_eval/tasks/basque_bench/flores_eu/flores_fr-eu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f15b672e826dcc7523e633b280d05d5d7ff65887 --- /dev/null +++ b/lm_eval/tasks/basque_bench/flores_eu/flores_fr-eu.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_fr-eu +doc_to_text: 'French sentence: {{sentence_fra_Latn}} + + Basque sentence:' +doc_to_target: '{{sentence_eus_Latn}}' diff --git a/lm_eval/tasks/basque_bench/flores_eu/flores_gl-eu.yaml b/lm_eval/tasks/basque_bench/flores_eu/flores_gl-eu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..08fafe084adad4a8381d49cbfc491e669443a8e3 --- /dev/null +++ b/lm_eval/tasks/basque_bench/flores_eu/flores_gl-eu.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_gl-eu +doc_to_text: 'Galician sentence: {{sentence_glg_Latn}} + + Basque sentence:' +doc_to_target: '{{sentence_eus_Latn}}' diff --git a/lm_eval/tasks/basque_bench/flores_eu/flores_it-eu.yaml b/lm_eval/tasks/basque_bench/flores_eu/flores_it-eu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7db4ec25c79aee911c30a0e966a1e5b8b287261d --- /dev/null +++ b/lm_eval/tasks/basque_bench/flores_eu/flores_it-eu.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_it-eu +doc_to_text: 'Italian sentence: {{sentence_ita_Latn}} + + Basque sentence:' +doc_to_target: '{{sentence_eus_Latn}}' diff --git a/lm_eval/tasks/basque_bench/flores_eu/flores_pt-eu.yaml b/lm_eval/tasks/basque_bench/flores_eu/flores_pt-eu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6b0169bc01f40d018050c2680e3cc09b35bccd89 --- /dev/null +++ b/lm_eval/tasks/basque_bench/flores_eu/flores_pt-eu.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_pt-eu +doc_to_text: 'Portuguese sentence: {{sentence_por_Latn}} + + Basque sentence:' +doc_to_target: '{{sentence_eus_Latn}}' diff --git a/lm_eval/tasks/basque_bench/mgsm_cot_native_eu.yaml b/lm_eval/tasks/basque_bench/mgsm_cot_native_eu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9de325bcd68bf1dd305f13ea696b2cef9076e40a --- /dev/null +++ b/lm_eval/tasks/basque_bench/mgsm_cot_native_eu.yaml @@ -0,0 +1,34 @@ +task: mgsm_native_cot_eu +dataset_path: HiTZ/MGSM-eu +dataset_name: null +doc_to_target: '{% if answer is not none %}{{answer[27:]}}{% else %}{{answer_number|string}}{%endif %}' +doc_to_text: '{% if answer is not none %}{{question+"\nErantzuna urratsez urrats:"}}{% else %}{{"Galdera: "+question+"\nErantzuna urratsez urrats:"}}{% endif %}' +output_type: generate_until +training_split: train +test_split: test +target_delimiter: " " +generation_kwargs: + until: + - "\n\n" + - "\n" + - "Galdera:" + - + - <|im_end|> + do_sample: false + temperature: 0.0 +filter_list: + - name: "get-answer" + filter: + - function: "regex" + regex_pattern: "Erantzuna [$%]? ?(-?[0-9]+([ .,][0-9.,]+)?) ?[$%]? da" + - function: "take_first" +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - " " +metadata: + version: 1.0 diff --git a/lm_eval/tasks/basque_bench/mgsm_direct_eu.yaml b/lm_eval/tasks/basque_bench/mgsm_direct_eu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f7da3317149aa823a64c7ad6d9b06d1a59c735b9 --- /dev/null +++ b/lm_eval/tasks/basque_bench/mgsm_direct_eu.yaml @@ -0,0 +1,39 @@ +task: mgsm_direct_eu +dataset_path: HiTZ/MGSM-eu +dataset_name: null +doc_to_target: '{{answer_number|string}}' +doc_to_text: '{% if answer is not none %}{{question+"\nErantzuna:"}}{% else %}{{"Galdera: "+question+"\nErantzuna:"}}{% endif %}' +output_type: generate_until +training_split: train +test_split: test +target_delimiter: " " +generation_kwargs: + until: + - "\n\n" + - "\n" + - "Galdera:" + - + - <|im_end|> + do_sample: false + temperature: 0.0 +filter_list: + - name: remove_whitespace + filter: + - function: remove_whitespace + - function: take_first + - name: flexible-extract + filter: + - function: regex + group_select: -1 + regex_pattern: (-?[0-9]+([ .,][0-9.,]+)?) + - function: take_first +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true + regexes_to_ignore: + - " " +metadata: + version: 1.0 diff --git a/lm_eval/tasks/basque_bench/utils.py b/lm_eval/tasks/basque_bench/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..1d5460b5a5b6697810026b75840b1e854c937c7a --- /dev/null +++ b/lm_eval/tasks/basque_bench/utils.py @@ -0,0 +1,21 @@ +from functools import partial + + +# ~~~~~~~~~~~ XCOPA ~~~~~~~~~~~ # + +xcopa_connectors = {"cause": " Izan ere,", "effect": " Beraz,"} + + +def xcopa_doc_to_text(doc): + conn = xcopa_connectors[doc["question"]] + return doc["premise"].strip() + f"{conn}" + + +def xcopa_doc_to_choice(doc): + def convert_choice(choice): + return choice[0].lower() + choice[1:] + + return [convert_choice(doc["choice1"]), convert_choice(doc["choice2"])] + + +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # diff --git a/lm_eval/tasks/basque_bench/wnli_eu.yaml b/lm_eval/tasks/basque_bench/wnli_eu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a3684e313936847e120a7dcd06218ea96552b402 --- /dev/null +++ b/lm_eval/tasks/basque_bench/wnli_eu.yaml @@ -0,0 +1,14 @@ +task: wnli_eu +dataset_path: HiTZ/wnli-eu +dataset_name: null +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: null +doc_to_text: "{{sentence1}}\nGaldera: {{sentence2}} Egia edo Gezurra?\nErantzuna:" +doc_to_target: label +doc_to_choice: ["Gezurra", "Egia"] +metric_list: + - metric: acc +metadata: + version: 1.0 diff --git a/lm_eval/tasks/basque_bench/xcopa_eu.yaml b/lm_eval/tasks/basque_bench/xcopa_eu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..83e50403c00f22f5ea5fa4d363069cca7224432c --- /dev/null +++ b/lm_eval/tasks/basque_bench/xcopa_eu.yaml @@ -0,0 +1,14 @@ +task: xcopa_eu +dataset_path: HiTZ/XCOPA-eu +dataset_name: null +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +doc_to_text: !function utils.xcopa_doc_to_text +doc_to_target: label +doc_to_choice: !function utils.xcopa_doc_to_choice +metric_list: + - metric: acc +metadata: + version: 1.0 diff --git a/lm_eval/tasks/basqueglue/README.md b/lm_eval/tasks/basqueglue/README.md index 9ba47b224e89b1d835b69fa76b7d268a92f1eecc..56c9ba289f3ed3af814eb9189f7dbd0ea77dd20b 100644 --- a/lm_eval/tasks/basqueglue/README.md +++ b/lm_eval/tasks/basqueglue/README.md @@ -43,11 +43,15 @@ Homepage: `https://github.com/hitz-zentroa/latxa` } ``` -### Groups and Tasks +### Groups, Tags, and Tasks #### Groups -* `basque-glue`: First version of the implementation +None. + +#### Tags + +* `basque-glue`: First version of the implementation. Calls all subtasks, but does not average. #### Tasks diff --git a/lm_eval/tasks/basqueglue/bec.yaml b/lm_eval/tasks/basqueglue/bec.yaml index fd15aea4990146cfff2e49d0897ec0ef4211ac6b..87d29aa60a3d5450947339f39646c4be33f335a9 100644 --- a/lm_eval/tasks/basqueglue/bec.yaml +++ b/lm_eval/tasks/basqueglue/bec.yaml @@ -1,4 +1,4 @@ -group: basque-glue +tag: basque-glue task: bec2016eu dataset_path: orai-nlp/basqueGLUE dataset_name: bec diff --git a/lm_eval/tasks/basqueglue/bhtc.yaml b/lm_eval/tasks/basqueglue/bhtc.yaml index f3c5c432d5c2cc96d91dfaeaf643e2cf09c7551a..29b0a494923b249b68b4c71afcfec901b8986f91 100644 --- a/lm_eval/tasks/basqueglue/bhtc.yaml +++ b/lm_eval/tasks/basqueglue/bhtc.yaml @@ -1,4 +1,4 @@ -group: basque-glue +tag: basque-glue task: bhtc_v2 dataset_path: orai-nlp/basqueGLUE dataset_name: bhtc diff --git a/lm_eval/tasks/basqueglue/coref.yaml b/lm_eval/tasks/basqueglue/coref.yaml index 6e9e8bdeaf8481b5844844af33e00482b472167b..2f64b1927b41ba447d3643c34761b8f790289c2b 100644 --- a/lm_eval/tasks/basqueglue/coref.yaml +++ b/lm_eval/tasks/basqueglue/coref.yaml @@ -1,4 +1,4 @@ -group: basque-glue +tag: basque-glue task: epec_koref_bin dataset_path: orai-nlp/basqueGLUE dataset_name: coref diff --git a/lm_eval/tasks/basqueglue/qnli.yaml b/lm_eval/tasks/basqueglue/qnli.yaml index 1a7d96ef7fc8ff5723bbe421dbb83f3ab378bd0c..93dbece6e15080a6b3e29ced5555100f89e7c4ee 100644 --- a/lm_eval/tasks/basqueglue/qnli.yaml +++ b/lm_eval/tasks/basqueglue/qnli.yaml @@ -1,4 +1,4 @@ -group: basque-glue +tag: basque-glue task: qnlieu dataset_path: orai-nlp/basqueGLUE dataset_name: qnli diff --git a/lm_eval/tasks/basqueglue/vaxx.yaml b/lm_eval/tasks/basqueglue/vaxx.yaml index e58507fdb6c1f09b7b30b8805ae09c4f52f1b94f..5d5ed6325071965537ab267464e9c51ad09c0bc2 100644 --- a/lm_eval/tasks/basqueglue/vaxx.yaml +++ b/lm_eval/tasks/basqueglue/vaxx.yaml @@ -1,4 +1,4 @@ -group: basque-glue +tag: basque-glue task: vaxx_stance dataset_path: orai-nlp/basqueGLUE dataset_name: vaxx diff --git a/lm_eval/tasks/basqueglue/wic.yaml b/lm_eval/tasks/basqueglue/wic.yaml index ee334043d8eab3d297ccd442d6bdd247f03d30ea..e64ab694d4f2685edbc2c1262e673cca65130d5c 100644 --- a/lm_eval/tasks/basqueglue/wic.yaml +++ b/lm_eval/tasks/basqueglue/wic.yaml @@ -1,4 +1,4 @@ -group: basque-glue +tag: basque-glue task: wiceu dataset_path: orai-nlp/basqueGLUE dataset_name: wic diff --git a/lm_eval/tasks/bbh/README.md b/lm_eval/tasks/bbh/README.md index 9ef2a4abdad4ac293d8a74865397ae7ed08a16ca..b9b35c44584f0475e4959024fc81d1fffdc5d48c 100644 --- a/lm_eval/tasks/bbh/README.md +++ b/lm_eval/tasks/bbh/README.md @@ -21,15 +21,19 @@ Homepage: https://github.com/suzgunmirac/BIG-Bench-Hard } ``` -### Groups and Tasks +### Groups, Tags, and Tasks #### Groups +- `bbh`: is the same as `bbh_cot_fewshot`. - `bbh_zeroshot` - `bbh_fewshot` - `bbh_cot_fewshot` - `bbh_cot_zeroshot` +#### Tags + +None. #### Tasks diff --git a/lm_eval/tasks/bbh/cot_fewshot/_bbh.yaml b/lm_eval/tasks/bbh/cot_fewshot/_bbh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0781a52d0752752a2aea2fe74e5b3b591dc838b3 --- /dev/null +++ b/lm_eval/tasks/bbh/cot_fewshot/_bbh.yaml @@ -0,0 +1,36 @@ +group: bbh +task: + - bbh_cot_fewshot_boolean_expressions + - bbh_cot_fewshot_causal_judgement + - bbh_cot_fewshot_date_understanding + - bbh_cot_fewshot_disambiguation_qa + - bbh_cot_fewshot_dyck_languages + - bbh_cot_fewshot_formal_fallacies + - bbh_cot_fewshot_geometric_shapes + - bbh_cot_fewshot_hyperbaton + - bbh_cot_fewshot_logical_deduction_five_objects + - bbh_cot_fewshot_logical_deduction_seven_objects + - bbh_cot_fewshot_logical_deduction_three_objects + - bbh_cot_fewshot_movie_recommendation + - bbh_cot_fewshot_multistep_arithmetic_two + - bbh_cot_fewshot_navigate + - bbh_cot_fewshot_object_counting + - bbh_cot_fewshot_penguins_in_a_table + - bbh_cot_fewshot_reasoning_about_colored_objects + - bbh_cot_fewshot_ruin_names + - bbh_cot_fewshot_salient_translation_error_detection + - bbh_cot_fewshot_snarks + - bbh_cot_fewshot_sports_understanding + - bbh_cot_fewshot_temporal_sequences + - bbh_cot_fewshot_tracking_shuffled_objects_five_objects + - bbh_cot_fewshot_tracking_shuffled_objects_seven_objects + - bbh_cot_fewshot_tracking_shuffled_objects_three_objects + - bbh_cot_fewshot_web_of_lies + - bbh_cot_fewshot_word_sorting +aggregate_metric_list: + - metric: exact_match + aggregation: mean + weight_by_size: true + filter_list: get-answer +metadata: + version: 3.0 diff --git a/lm_eval/tasks/bbh/cot_fewshot/_bbh_cot_fewshot.yaml b/lm_eval/tasks/bbh/cot_fewshot/_bbh_cot_fewshot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..46f7152107b0fa436a2579dbff99bce3761491af --- /dev/null +++ b/lm_eval/tasks/bbh/cot_fewshot/_bbh_cot_fewshot.yaml @@ -0,0 +1,36 @@ +group: bbh_cot_fewshot +task: + - bbh_cot_fewshot_boolean_expressions + - bbh_cot_fewshot_causal_judgement + - bbh_cot_fewshot_date_understanding + - bbh_cot_fewshot_disambiguation_qa + - bbh_cot_fewshot_dyck_languages + - bbh_cot_fewshot_formal_fallacies + - bbh_cot_fewshot_geometric_shapes + - bbh_cot_fewshot_hyperbaton + - bbh_cot_fewshot_logical_deduction_five_objects + - bbh_cot_fewshot_logical_deduction_seven_objects + - bbh_cot_fewshot_logical_deduction_three_objects + - bbh_cot_fewshot_movie_recommendation + - bbh_cot_fewshot_multistep_arithmetic_two + - bbh_cot_fewshot_navigate + - bbh_cot_fewshot_object_counting + - bbh_cot_fewshot_penguins_in_a_table + - bbh_cot_fewshot_reasoning_about_colored_objects + - bbh_cot_fewshot_ruin_names + - bbh_cot_fewshot_salient_translation_error_detection + - bbh_cot_fewshot_snarks + - bbh_cot_fewshot_sports_understanding + - bbh_cot_fewshot_temporal_sequences + - bbh_cot_fewshot_tracking_shuffled_objects_five_objects + - bbh_cot_fewshot_tracking_shuffled_objects_seven_objects + - bbh_cot_fewshot_tracking_shuffled_objects_three_objects + - bbh_cot_fewshot_web_of_lies + - bbh_cot_fewshot_word_sorting +aggregate_metric_list: + - metric: exact_match + aggregation: mean + weight_by_size: true + filter_list: get-answer +metadata: + version: 3.0 diff --git a/lm_eval/tasks/bbh/cot_fewshot/_cot_fewshot_template_yaml b/lm_eval/tasks/bbh/cot_fewshot/_cot_fewshot_template_yaml index 5f2318f8026802bf7884af7b777bcca6fcb8bbbc..0526fc86cc61085a8817fa3e1db6460a0e9c4907 100644 --- a/lm_eval/tasks/bbh/cot_fewshot/_cot_fewshot_template_yaml +++ b/lm_eval/tasks/bbh/cot_fewshot/_cot_fewshot_template_yaml @@ -1,6 +1,3 @@ -group: -- bbh -- bbh_cot_fewshot dataset_path: lukaemon/bbh output_type: generate_until test_split: test @@ -27,4 +24,4 @@ filter_list: - function: "take_first" num_fewshot: 3 metadata: - version: 2.0 + version: 3.0 diff --git a/lm_eval/tasks/bbh/cot_zeroshot/_bbh_cot_zeroshot.yaml b/lm_eval/tasks/bbh/cot_zeroshot/_bbh_cot_zeroshot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cae4b4a0a80a62929c997404fd4842e91005f777 --- /dev/null +++ b/lm_eval/tasks/bbh/cot_zeroshot/_bbh_cot_zeroshot.yaml @@ -0,0 +1,36 @@ +group: bbh_cot_zeroshot +task: + - bbh_cot_zeroshot_boolean_expressions + - bbh_cot_zeroshot_causal_judgement + - bbh_cot_zeroshot_date_understanding + - bbh_cot_zeroshot_disambiguation_qa + - bbh_cot_zeroshot_dyck_languages + - bbh_cot_zeroshot_formal_fallacies + - bbh_cot_zeroshot_geometric_shapes + - bbh_cot_zeroshot_hyperbaton + - bbh_cot_zeroshot_logical_deduction_five_objects + - bbh_cot_zeroshot_logical_deduction_seven_objects + - bbh_cot_zeroshot_logical_deduction_three_objects + - bbh_cot_zeroshot_movie_recommendation + - bbh_cot_zeroshot_multistep_arithmetic_two + - bbh_cot_zeroshot_navigate + - bbh_cot_zeroshot_object_counting + - bbh_cot_zeroshot_penguins_in_a_table + - bbh_cot_zeroshot_reasoning_about_colored_objects + - bbh_cot_zeroshot_ruin_names + - bbh_cot_zeroshot_salient_translation_error_detection + - bbh_cot_zeroshot_snarks + - bbh_cot_zeroshot_sports_understanding + - bbh_cot_zeroshot_temporal_sequences + - bbh_cot_zeroshot_tracking_shuffled_objects_five_objects + - bbh_cot_zeroshot_tracking_shuffled_objects_seven_objects + - bbh_cot_zeroshot_tracking_shuffled_objects_three_objects + - bbh_cot_zeroshot_web_of_lies + - bbh_cot_zeroshot_word_sorting +aggregate_metric_list: + - metric: exact_match + aggregation: mean + weight_by_size: true + filter_list: flexible-extract +metadata: + version: 3.0 diff --git a/lm_eval/tasks/bbh/cot_zeroshot/_cot_zeroshot_template_yaml b/lm_eval/tasks/bbh/cot_zeroshot/_cot_zeroshot_template_yaml index 5e62b0bac165fd19477099853e4dc3090f057bc6..925a3bba61c82afaff42476ff4a4b3ebb04b6ddb 100644 --- a/lm_eval/tasks/bbh/cot_zeroshot/_cot_zeroshot_template_yaml +++ b/lm_eval/tasks/bbh/cot_zeroshot/_cot_zeroshot_template_yaml @@ -1,4 +1,3 @@ -group: bbh_cot_zeroshot dataset_path: lukaemon/bbh output_type: generate_until test_split: test @@ -24,4 +23,4 @@ generation_kwargs: temperature: 0.0 num_fewshot: 0 metadata: - version: 2.0 + version: 3.0 diff --git a/lm_eval/tasks/bbh/fewshot/_bbh_fewshot.yaml b/lm_eval/tasks/bbh/fewshot/_bbh_fewshot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..13aa6d68e7c45085835d2733cb1b08207b922819 --- /dev/null +++ b/lm_eval/tasks/bbh/fewshot/_bbh_fewshot.yaml @@ -0,0 +1,35 @@ +group: bbh_fewshot +task: + - bbh_fewshot_boolean_expressions + - bbh_fewshot_causal_judgement + - bbh_fewshot_date_understanding + - bbh_fewshot_disambiguation_qa + - bbh_fewshot_dyck_languages + - bbh_fewshot_formal_fallacies + - bbh_fewshot_geometric_shapes + - bbh_fewshot_hyperbaton + - bbh_fewshot_logical_deduction_five_objects + - bbh_fewshot_logical_deduction_seven_objects + - bbh_fewshot_logical_deduction_three_objects + - bbh_fewshot_movie_recommendation + - bbh_fewshot_multistep_arithmetic_two + - bbh_fewshot_navigate + - bbh_fewshot_object_counting + - bbh_fewshot_penguins_in_a_table + - bbh_fewshot_reasoning_about_colored_objects + - bbh_fewshot_ruin_names + - bbh_fewshot_salient_translation_error_detection + - bbh_fewshot_snarks + - bbh_fewshot_sports_understanding + - bbh_fewshot_temporal_sequences + - bbh_fewshot_tracking_shuffled_objects_five_objects + - bbh_fewshot_tracking_shuffled_objects_seven_objects + - bbh_fewshot_tracking_shuffled_objects_three_objects + - bbh_fewshot_web_of_lies + - bbh_fewshot_word_sorting +aggregate_metric_list: + - metric: exact_match + aggregation: mean + weight_by_size: true +metadata: + version: 3.0 diff --git a/lm_eval/tasks/bbh/fewshot/_fewshot_template_yaml b/lm_eval/tasks/bbh/fewshot/_fewshot_template_yaml index a764f723c853e037416372a30befa73e4d4f743f..cd59edb53dc4318a2ab10a7c245bf99c8c463c26 100644 --- a/lm_eval/tasks/bbh/fewshot/_fewshot_template_yaml +++ b/lm_eval/tasks/bbh/fewshot/_fewshot_template_yaml @@ -1,4 +1,3 @@ -group: bbh_fewshot dataset_path: lukaemon/bbh output_type: generate_until test_split: test @@ -18,4 +17,4 @@ generation_kwargs: temperature: 0.0 num_fewshot: 3 metadata: - version: 1.0 + version: 2.0 diff --git a/lm_eval/tasks/bbh/zeroshot/_bbh_zeroshot.yaml b/lm_eval/tasks/bbh/zeroshot/_bbh_zeroshot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..27d9e08ea19488cd0209150c42d6bb43752d8862 --- /dev/null +++ b/lm_eval/tasks/bbh/zeroshot/_bbh_zeroshot.yaml @@ -0,0 +1,36 @@ +group: bbh_zeroshot +task: + - bbh_zeroshot_boolean_expressions + - bbh_zeroshot_causal_judgement + - bbh_zeroshot_date_understanding + - bbh_zeroshot_disambiguation_qa + - bbh_zeroshot_dyck_languages + - bbh_zeroshot_formal_fallacies + - bbh_zeroshot_geometric_shapes + - bbh_zeroshot_hyperbaton + - bbh_zeroshot_logical_deduction_five_objects + - bbh_zeroshot_logical_deduction_seven_objects + - bbh_zeroshot_logical_deduction_three_objects + - bbh_zeroshot_movie_recommendation + - bbh_zeroshot_multistep_arithmetic_two + - bbh_zeroshot_navigate + - bbh_zeroshot_object_counting + - bbh_zeroshot_penguins_in_a_table + - bbh_zeroshot_reasoning_about_colored_objects + - bbh_zeroshot_ruin_names + - bbh_zeroshot_salient_translation_error_detection + - bbh_zeroshot_snarks + - bbh_zeroshot_sports_understanding + - bbh_zeroshot_temporal_sequences + - bbh_zeroshot_tracking_shuffled_objects_five_objects + - bbh_zeroshot_tracking_shuffled_objects_seven_objects + - bbh_zeroshot_tracking_shuffled_objects_three_objects + - bbh_zeroshot_web_of_lies + - bbh_zeroshot_word_sorting +aggregate_metric_list: + - metric: exact_match + aggregation: mean + weight_by_size: true + filter_list: flexible-extract +metadata: + version: 3.0 diff --git a/lm_eval/tasks/bbh/zeroshot/_zeroshot_template_yaml b/lm_eval/tasks/bbh/zeroshot/_zeroshot_template_yaml index 728ccae1fba5287a914f4aca0bcfb8108aca9512..2f3ecd4d4baa3be5b432d6ab643f940d507f861f 100644 --- a/lm_eval/tasks/bbh/zeroshot/_zeroshot_template_yaml +++ b/lm_eval/tasks/bbh/zeroshot/_zeroshot_template_yaml @@ -1,4 +1,3 @@ -group: bbh_zeroshot dataset_path: lukaemon/bbh output_type: generate_until test_split: test @@ -24,4 +23,4 @@ generation_kwargs: temperature: 0.0 num_fewshot: 0 metadata: - version: 2.0 + version: 3.0 diff --git a/lm_eval/tasks/belebele/_belebele.yaml b/lm_eval/tasks/belebele/_belebele.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a4f71068e0cadcac76cb43cb60b0188a00de222c --- /dev/null +++ b/lm_eval/tasks/belebele/_belebele.yaml @@ -0,0 +1,133 @@ +group: belebele +task: + - belebele_acm_Arab + - belebele_arz_Arab + - belebele_ceb_Latn + - belebele_fin_Latn + - belebele_hin_Deva + - belebele_ita_Latn + - belebele_khm_Khmr + - belebele_lvs_Latn + - belebele_npi_Deva + - belebele_pol_Latn + - belebele_slv_Latn + - belebele_swe_Latn + - belebele_tso_Latn + - belebele_xho_Latn + - belebele_afr_Latn + - belebele_asm_Beng + - belebele_ces_Latn + - belebele_fra_Latn + - belebele_hin_Latn + - belebele_jav_Latn + - belebele_kin_Latn + - belebele_mal_Mlym + - belebele_npi_Latn + - belebele_por_Latn + - belebele_sna_Latn + - belebele_swh_Latn + - belebele_tur_Latn + - belebele_yor_Latn + - belebele_als_Latn + - belebele_azj_Latn + - belebele_ckb_Arab + - belebele_fuv_Latn + - belebele_hrv_Latn + - belebele_jpn_Jpan + - belebele_kir_Cyrl + - belebele_mar_Deva + - belebele_nso_Latn + - belebele_snd_Arab + - belebele_tam_Taml + - belebele_ukr_Cyrl + - belebele_zho_Hans + - belebele_amh_Ethi + - belebele_bam_Latn + - belebele_dan_Latn + - belebele_gaz_Latn + - belebele_hun_Latn + - belebele_kac_Latn + - belebele_kor_Hang + - belebele_mkd_Cyrl + - belebele_nya_Latn + - belebele_ron_Latn + - belebele_som_Latn + - belebele_tel_Telu + - belebele_urd_Arab + - belebele_zho_Hant + - belebele_apc_Arab + - belebele_ben_Beng + - belebele_deu_Latn + - belebele_grn_Latn + - belebele_hye_Armn + - belebele_kan_Knda + - belebele_lao_Laoo + - belebele_mlt_Latn + - belebele_ory_Orya + - belebele_rus_Cyrl + - belebele_sot_Latn + - belebele_tgk_Cyrl + - belebele_urd_Latn + - belebele_zsm_Latn + - belebele_arb_Arab + - belebele_ben_Latn + - belebele_ell_Grek + - belebele_guj_Gujr + - belebele_ibo_Latn + - belebele_kat_Geor + - belebele_lin_Latn + - belebele_mri_Latn + - belebele_pan_Guru + - belebele_shn_Mymr + - belebele_spa_Latn + - belebele_tgl_Latn + - belebele_uzn_Latn + - belebele_zul_Latn + - belebele_arb_Latn + - belebele_bod_Tibt + - belebele_eng_Latn + - belebele_hat_Latn + - belebele_ilo_Latn + - belebele_kaz_Cyrl + - belebele_lit_Latn + - belebele_mya_Mymr + - belebele_pbt_Arab + - belebele_sin_Latn + - belebele_srp_Cyrl + - belebele_tha_Thai + - belebele_vie_Latn + - belebele_ars_Arab + - belebele_bul_Cyrl + - belebele_est_Latn + - belebele_hau_Latn + - belebele_ind_Latn + - belebele_kea_Latn + - belebele_lug_Latn + - belebele_nld_Latn + - belebele_pes_Arab + - belebele_sin_Sinh + - belebele_ssw_Latn + - belebele_tir_Ethi + - belebele_war_Latn + - belebele_ary_Arab + - belebele_cat_Latn + - belebele_eus_Latn + - belebele_heb_Hebr + - belebele_isl_Latn + - belebele_khk_Cyrl + - belebele_luo_Latn + - belebele_nob_Latn + - belebele_plt_Latn + - belebele_slk_Latn + - belebele_sun_Latn + - belebele_tsn_Latn + - belebele_wol_Latn +aggregate_metric_list: + - aggregation: mean + metric: acc + weight_by_size: true + - aggregation: mean + metric: acc_norm + weight_by_size: true +metadata: + version: 0.1 diff --git a/lm_eval/tasks/belebele/_default_template_yaml b/lm_eval/tasks/belebele/_default_template_yaml index 2583ced5688e1a0f97f3c46b1bc64d54c329a172..c32b310ba86091f627c7ec5717fd7cb4f69a46f2 100644 --- a/lm_eval/tasks/belebele/_default_template_yaml +++ b/lm_eval/tasks/belebele/_default_template_yaml @@ -1,11 +1,10 @@ -group: belebele dataset_path: facebook/belebele fewshot_config: sampler: first_n output_type: multiple_choice should_decontaminate: true doc_to_decontamination_query: "{{question}}" -doc_to_text: "P: {{flores_passage}}\nQ: {{question.strip()}}\nA: {{mc_answer1}}\nB: {{mc_answer2}}\nC: {{mc_answer3}}\nD: {{mc_answer4}}\nAnswer：" +doc_to_text: "P: {{flores_passage}}\nQ: {{question.strip()}}\nA: {{mc_answer1}}\nB: {{mc_answer2}}\nC: {{mc_answer3}}\nD: {{mc_answer4}}\nAnswer:" doc_to_choice: ["A", "B", "C", "D"] doc_to_target: "{{['1', '2', '3', '4'].index(correct_answer_num)}}" metric_list: diff --git a/lm_eval/tasks/belebele/_generate_configs.py b/lm_eval/tasks/belebele/_generate_configs.py index c21b51f9c1a4af38df565e3c89d2a48a629b546f..f4bf589a3d5616eac2eacce92610a7c4c3086b79 100644 --- a/lm_eval/tasks/belebele/_generate_configs.py +++ b/lm_eval/tasks/belebele/_generate_configs.py @@ -65,3 +65,36 @@ if __name__ == "__main__": allow_unicode=True, default_style='"', ) + + # write group config out + + group_yaml_dict = { + "group": f"belebele_{args.task_prefix}" + if args.task_prefix != "" + else "belebele", + "task": [ + ( + f"belebele_{args.task_prefix}_{lang}" + if args.task_prefix != "" + else f"belebele_{lang}" + ) + for lang in languages + if "default" not in lang + ], + "aggregate_metric_list": [ + {"metric": "acc", "aggregation": "mean", "weight_by_size": False}, + {"metric": "acc_norm", "aggregation": "mean", "weight_by_size": False}, + ], + "metadata": {"version": 0.0}, + } + + file_save_path = "_" + args.save_prefix_path + f"{args.task_prefix}.yaml" + + with open(file_save_path, "w", encoding="utf-8") as group_yaml_file: + yaml.dump( + group_yaml_dict, + group_yaml_file, + width=float("inf"), + allow_unicode=True, + default_style='"', + ) diff --git a/lm_eval/tasks/belebele/belebele_acm_Arab.yaml b/lm_eval/tasks/belebele/belebele_acm_Arab.yaml index 7439ce8adfa6127abf3381a1f193194e55826fcc..e2dc192db9fa2687e35eeeb304f3677d26cea354 100644 --- a/lm_eval/tasks/belebele/belebele_acm_Arab.yaml +++ b/lm_eval/tasks/belebele/belebele_acm_Arab.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "acm_Arab" -"include": "_default_template_yaml" -"task": "belebele_acm_Arab" -"test_split": "acm_Arab" +dataset_name: acm_Arab +fewshot_split: test +include: _default_template_yaml +task: belebele_acm_Arab +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_afr_Latn.yaml b/lm_eval/tasks/belebele/belebele_afr_Latn.yaml index 5011d654e18a3ae285b72e8d28d4277268977a35..5a48eab7e2f642e3fa930f7b016235e58f8c13d9 100644 --- a/lm_eval/tasks/belebele/belebele_afr_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_afr_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "afr_Latn" -"include": "_default_template_yaml" -"task": "belebele_afr_Latn" -"test_split": "afr_Latn" +dataset_name: afr_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_afr_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_als_Latn.yaml b/lm_eval/tasks/belebele/belebele_als_Latn.yaml index f77ed55f10897c8efed0977abe9db4272227825e..b33bdba4df03b45da2136f1b08a2d486b2529891 100644 --- a/lm_eval/tasks/belebele/belebele_als_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_als_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "als_Latn" -"include": "_default_template_yaml" -"task": "belebele_als_Latn" -"test_split": "als_Latn" +dataset_name: als_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_als_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_amh_Ethi.yaml b/lm_eval/tasks/belebele/belebele_amh_Ethi.yaml index e3d61066cf9ccdce81c9c5b91682c1616cc28e6a..6bcfdf5c16438564052c240f1c9ec18a4eab9906 100644 --- a/lm_eval/tasks/belebele/belebele_amh_Ethi.yaml +++ b/lm_eval/tasks/belebele/belebele_amh_Ethi.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "amh_Ethi" -"include": "_default_template_yaml" -"task": "belebele_amh_Ethi" -"test_split": "amh_Ethi" +dataset_name: amh_Ethi +fewshot_split: test +include: _default_template_yaml +task: belebele_amh_Ethi +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_apc_Arab.yaml b/lm_eval/tasks/belebele/belebele_apc_Arab.yaml index 2e7619a5e0070bf681bff12d373b22878ceaa446..baece06b88858dfed3e750970acd432acf8c2571 100644 --- a/lm_eval/tasks/belebele/belebele_apc_Arab.yaml +++ b/lm_eval/tasks/belebele/belebele_apc_Arab.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "apc_Arab" -"include": "_default_template_yaml" -"task": "belebele_apc_Arab" -"test_split": "apc_Arab" +dataset_name: apc_Arab +fewshot_split: test +include: _default_template_yaml +task: belebele_apc_Arab +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_arb_Arab.yaml b/lm_eval/tasks/belebele/belebele_arb_Arab.yaml index b6242fd38b2666f496751b8ed03639f712f350e3..bb2ba6944c5be8d720f39a6614d1d9b4dc85932e 100644 --- a/lm_eval/tasks/belebele/belebele_arb_Arab.yaml +++ b/lm_eval/tasks/belebele/belebele_arb_Arab.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "arb_Arab" -"include": "_default_template_yaml" -"task": "belebele_arb_Arab" -"test_split": "arb_Arab" +dataset_name: arb_Arab +fewshot_split: test +include: _default_template_yaml +task: belebele_arb_Arab +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_arb_Latn.yaml b/lm_eval/tasks/belebele/belebele_arb_Latn.yaml index 8759bc4d86152af04e0cccf33f01306893595d19..361681b2ef134ebeb86a127454f9960b6120e988 100644 --- a/lm_eval/tasks/belebele/belebele_arb_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_arb_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "arb_Latn" -"include": "_default_template_yaml" -"task": "belebele_arb_Latn" -"test_split": "arb_Latn" +dataset_name: arb_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_arb_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_ars_Arab.yaml b/lm_eval/tasks/belebele/belebele_ars_Arab.yaml index e9941fd48a2b376d93553a28a42311a1470cc40c..6227dbbbc31651d69645a87d49287a6f808a2247 100644 --- a/lm_eval/tasks/belebele/belebele_ars_Arab.yaml +++ b/lm_eval/tasks/belebele/belebele_ars_Arab.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "ars_Arab" -"include": "_default_template_yaml" -"task": "belebele_ars_Arab" -"test_split": "ars_Arab" +dataset_name: ars_Arab +fewshot_split: test +include: _default_template_yaml +task: belebele_ars_Arab +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_ary_Arab.yaml b/lm_eval/tasks/belebele/belebele_ary_Arab.yaml index fe00dd0342b89d99d860b9cc7bef2aad66cf5875..6cc767ddf7cd98850b4d34e086102fe0c04d5ed5 100644 --- a/lm_eval/tasks/belebele/belebele_ary_Arab.yaml +++ b/lm_eval/tasks/belebele/belebele_ary_Arab.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "ary_Arab" -"include": "_default_template_yaml" -"task": "belebele_ary_Arab" -"test_split": "ary_Arab" +dataset_name: ary_Arab +fewshot_split: test +include: _default_template_yaml +task: belebele_ary_Arab +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_arz_Arab.yaml b/lm_eval/tasks/belebele/belebele_arz_Arab.yaml index a7963c900e1febfa66bb8c5066f83e85638004ef..28f2e48a453651bb16c404951a2878da66dfadd0 100644 --- a/lm_eval/tasks/belebele/belebele_arz_Arab.yaml +++ b/lm_eval/tasks/belebele/belebele_arz_Arab.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "arz_Arab" -"include": "_default_template_yaml" -"task": "belebele_arz_Arab" -"test_split": "arz_Arab" +dataset_name: arz_Arab +fewshot_split: test +include: _default_template_yaml +task: belebele_arz_Arab +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_asm_Beng.yaml b/lm_eval/tasks/belebele/belebele_asm_Beng.yaml index c24a1deceac5c596705e95c03e43b327363be9ae..19ca656c800f3d00332c6fdb4fbe493fb46f96df 100644 --- a/lm_eval/tasks/belebele/belebele_asm_Beng.yaml +++ b/lm_eval/tasks/belebele/belebele_asm_Beng.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "asm_Beng" -"include": "_default_template_yaml" -"task": "belebele_asm_Beng" -"test_split": "asm_Beng" +dataset_name: asm_Beng +fewshot_split: test +include: _default_template_yaml +task: belebele_asm_Beng +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_azj_Latn.yaml b/lm_eval/tasks/belebele/belebele_azj_Latn.yaml index 8a9c7f2a8cb428cde2bfcbdf8c2485150c9c1db0..8958f2a4bbecc76af2f298b66d169f6809b7dbfa 100644 --- a/lm_eval/tasks/belebele/belebele_azj_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_azj_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "azj_Latn" -"include": "_default_template_yaml" -"task": "belebele_azj_Latn" -"test_split": "azj_Latn" +dataset_name: azj_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_azj_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_bam_Latn.yaml b/lm_eval/tasks/belebele/belebele_bam_Latn.yaml index 9441f0fceeff1b27a64c65e524f4c050b2777851..419eda1ea0c0ee555ed97ae0a39e51565e1740c8 100644 --- a/lm_eval/tasks/belebele/belebele_bam_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_bam_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "bam_Latn" -"include": "_default_template_yaml" -"task": "belebele_bam_Latn" -"test_split": "bam_Latn" +dataset_name: bam_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_bam_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_ben_Beng.yaml b/lm_eval/tasks/belebele/belebele_ben_Beng.yaml index 2b34335265620a6875082d8575fb234229e73d2a..43b24b956c4bf223d4f6deade76892ff36c6616e 100644 --- a/lm_eval/tasks/belebele/belebele_ben_Beng.yaml +++ b/lm_eval/tasks/belebele/belebele_ben_Beng.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "ben_Beng" -"include": "_default_template_yaml" -"task": "belebele_ben_Beng" -"test_split": "ben_Beng" +dataset_name: ben_Beng +fewshot_split: test +include: _default_template_yaml +task: belebele_ben_Beng +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_ben_Latn.yaml b/lm_eval/tasks/belebele/belebele_ben_Latn.yaml index c8736c5242e5b8ff3f717c317292a42fd718db5b..96c199554d989d9fa8ac55600525a8c920edadbe 100644 --- a/lm_eval/tasks/belebele/belebele_ben_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_ben_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "ben_Latn" -"include": "_default_template_yaml" -"task": "belebele_ben_Latn" -"test_split": "ben_Latn" +dataset_name: ben_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_ben_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_bod_Tibt.yaml b/lm_eval/tasks/belebele/belebele_bod_Tibt.yaml index 53d40c4d574edd84946d7ee9e626e368af705ff8..81a1bc7db28b2f026d60050bbf287276a09fcef5 100644 --- a/lm_eval/tasks/belebele/belebele_bod_Tibt.yaml +++ b/lm_eval/tasks/belebele/belebele_bod_Tibt.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "bod_Tibt" -"include": "_default_template_yaml" -"task": "belebele_bod_Tibt" -"test_split": "bod_Tibt" +dataset_name: bod_Tibt +fewshot_split: test +include: _default_template_yaml +task: belebele_bod_Tibt +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_bul_Cyrl.yaml b/lm_eval/tasks/belebele/belebele_bul_Cyrl.yaml index ed7bc832001852585d9ba2b0579217d7330c03e6..d26ba17fc04e61d1c4808c1471fca3e404799e35 100644 --- a/lm_eval/tasks/belebele/belebele_bul_Cyrl.yaml +++ b/lm_eval/tasks/belebele/belebele_bul_Cyrl.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "bul_Cyrl" -"include": "_default_template_yaml" -"task": "belebele_bul_Cyrl" -"test_split": "bul_Cyrl" +dataset_name: bul_Cyrl +fewshot_split: test +include: _default_template_yaml +task: belebele_bul_Cyrl +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_cat_Latn.yaml b/lm_eval/tasks/belebele/belebele_cat_Latn.yaml index 8781f8e28a58fa5b746d18f01da26b42d97a6bc7..9c7be3b41b64cd0748acce4194c2fa86d24f6dd9 100644 --- a/lm_eval/tasks/belebele/belebele_cat_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_cat_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "cat_Latn" -"include": "_default_template_yaml" -"task": "belebele_cat_Latn" -"test_split": "cat_Latn" +dataset_name: cat_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_cat_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_ceb_Latn.yaml b/lm_eval/tasks/belebele/belebele_ceb_Latn.yaml index c9d2f89c90dd21c4293c13ab45a35d40858d2707..2e616bd407855f51c5ed1ef7e20201ab17093174 100644 --- a/lm_eval/tasks/belebele/belebele_ceb_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_ceb_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "ceb_Latn" -"include": "_default_template_yaml" -"task": "belebele_ceb_Latn" -"test_split": "ceb_Latn" +dataset_name: ceb_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_ceb_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_ces_Latn.yaml b/lm_eval/tasks/belebele/belebele_ces_Latn.yaml index 330fd11a8f2ce8a45d48fd0c4c95d6c9cb5e910e..597680de750a7cd0a0b6789295bd19e61c0f7d8a 100644 --- a/lm_eval/tasks/belebele/belebele_ces_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_ces_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "ces_Latn" -"include": "_default_template_yaml" -"task": "belebele_ces_Latn" -"test_split": "ces_Latn" +dataset_name: ces_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_ces_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_ckb_Arab.yaml b/lm_eval/tasks/belebele/belebele_ckb_Arab.yaml index 6ad49a8eae0550ddd23ca51839c2d72b31031725..51caa4353e68c373db7ba382a4181563a89e89ae 100644 --- a/lm_eval/tasks/belebele/belebele_ckb_Arab.yaml +++ b/lm_eval/tasks/belebele/belebele_ckb_Arab.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "ckb_Arab" -"include": "_default_template_yaml" -"task": "belebele_ckb_Arab" -"test_split": "ckb_Arab" +dataset_name: ckb_Arab +fewshot_split: test +include: _default_template_yaml +task: belebele_ckb_Arab +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_dan_Latn.yaml b/lm_eval/tasks/belebele/belebele_dan_Latn.yaml index 883b3cfceafb5af7141d2c793cdd9e9318690229..98389123b623834d0e61a7b47ad5d97165ba5d90 100644 --- a/lm_eval/tasks/belebele/belebele_dan_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_dan_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "dan_Latn" -"include": "_default_template_yaml" -"task": "belebele_dan_Latn" -"test_split": "dan_Latn" +dataset_name: dan_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_dan_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_deu_Latn.yaml b/lm_eval/tasks/belebele/belebele_deu_Latn.yaml index d354efae5d140b85ca930fdf702483fcdac3be29..c1e743a473ed1fc9fc6606e530deda55f6ff3d01 100644 --- a/lm_eval/tasks/belebele/belebele_deu_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_deu_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "deu_Latn" -"include": "_default_template_yaml" -"task": "belebele_deu_Latn" -"test_split": "deu_Latn" +dataset_name: deu_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_deu_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_ell_Grek.yaml b/lm_eval/tasks/belebele/belebele_ell_Grek.yaml index 57226e976bc5f22fce37e2778e1f9596660ceceb..c169d3e786a19dab510d11c4c9c3399c58316c97 100644 --- a/lm_eval/tasks/belebele/belebele_ell_Grek.yaml +++ b/lm_eval/tasks/belebele/belebele_ell_Grek.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "ell_Grek" -"include": "_default_template_yaml" -"task": "belebele_ell_Grek" -"test_split": "ell_Grek" +dataset_name: ell_Grek +fewshot_split: test +include: _default_template_yaml +task: belebele_ell_Grek +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_eng_Latn.yaml b/lm_eval/tasks/belebele/belebele_eng_Latn.yaml index ee06a5422425655a257aa60409116d5c0fa5df2a..d3bd3c2b1ca9db9a9013741c962a480d24c8f031 100644 --- a/lm_eval/tasks/belebele/belebele_eng_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_eng_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "eng_Latn" -"include": "_default_template_yaml" -"task": "belebele_eng_Latn" -"test_split": "eng_Latn" +dataset_name: eng_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_eng_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_est_Latn.yaml b/lm_eval/tasks/belebele/belebele_est_Latn.yaml index 6a56ca90c0309d9475adad9b95db272577658f36..4f50722b26487e4940404346d9585f9de51a0c51 100644 --- a/lm_eval/tasks/belebele/belebele_est_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_est_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "est_Latn" -"include": "_default_template_yaml" -"task": "belebele_est_Latn" -"test_split": "est_Latn" +dataset_name: est_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_est_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_eus_Latn.yaml b/lm_eval/tasks/belebele/belebele_eus_Latn.yaml index b2ad09b8169ccf946d1cd5ec0cc4f02d9f2b3b21..7e64381b39a1a695ab9e03964a24c6a6240b6cde 100644 --- a/lm_eval/tasks/belebele/belebele_eus_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_eus_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "eus_Latn" -"include": "_default_template_yaml" -"task": "belebele_eus_Latn" -"test_split": "eus_Latn" +dataset_name: eus_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_eus_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_fin_Latn.yaml b/lm_eval/tasks/belebele/belebele_fin_Latn.yaml index 0b49e5a07dac1d21cdc5481f5ba161aa3a636367..e26a0294f4d424369a5f1c837ce783db4363cf76 100644 --- a/lm_eval/tasks/belebele/belebele_fin_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_fin_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "fin_Latn" -"include": "_default_template_yaml" -"task": "belebele_fin_Latn" -"test_split": "fin_Latn" +dataset_name: fin_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_fin_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_fra_Latn.yaml b/lm_eval/tasks/belebele/belebele_fra_Latn.yaml index c60fa9a7a90db3fc7f5451b39705a487acf18b29..9f81b171056d93f0990c47c3e37f69dba6f237ff 100644 --- a/lm_eval/tasks/belebele/belebele_fra_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_fra_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "fra_Latn" -"include": "_default_template_yaml" -"task": "belebele_fra_Latn" -"test_split": "fra_Latn" +dataset_name: fra_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_fra_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_fuv_Latn.yaml b/lm_eval/tasks/belebele/belebele_fuv_Latn.yaml index 2636cae850c6222906fac7f3c1533ea16684ee73..77e63393f118e2c054000fae1c368d5e6abaacb3 100644 --- a/lm_eval/tasks/belebele/belebele_fuv_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_fuv_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "fuv_Latn" -"include": "_default_template_yaml" -"task": "belebele_fuv_Latn" -"test_split": "fuv_Latn" +dataset_name: fuv_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_fuv_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_gaz_Latn.yaml b/lm_eval/tasks/belebele/belebele_gaz_Latn.yaml index ee161d81f62ef363f3321ca446216b3c81818d76..4116fd431bd21e842cd9844bf18388952860efd0 100644 --- a/lm_eval/tasks/belebele/belebele_gaz_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_gaz_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "gaz_Latn" -"include": "_default_template_yaml" -"task": "belebele_gaz_Latn" -"test_split": "gaz_Latn" +dataset_name: gaz_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_gaz_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_grn_Latn.yaml b/lm_eval/tasks/belebele/belebele_grn_Latn.yaml index cdba6f64218c05a514409b7cdb5122158a02aae8..75bceb210e316661afee26b639f6b5b3042243d6 100644 --- a/lm_eval/tasks/belebele/belebele_grn_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_grn_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "grn_Latn" -"include": "_default_template_yaml" -"task": "belebele_grn_Latn" -"test_split": "grn_Latn" +dataset_name: grn_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_grn_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_guj_Gujr.yaml b/lm_eval/tasks/belebele/belebele_guj_Gujr.yaml index 353ce6b598bdeba7d1dba8ca7baf187c89c2c3ca..180c7143b75b5c8a0a4929e43a0c42f6e4443011 100644 --- a/lm_eval/tasks/belebele/belebele_guj_Gujr.yaml +++ b/lm_eval/tasks/belebele/belebele_guj_Gujr.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "guj_Gujr" -"include": "_default_template_yaml" -"task": "belebele_guj_Gujr" -"test_split": "guj_Gujr" +dataset_name: guj_Gujr +fewshot_split: test +include: _default_template_yaml +task: belebele_guj_Gujr +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_hat_Latn.yaml b/lm_eval/tasks/belebele/belebele_hat_Latn.yaml index 1057a113404ce9b6826ce1224fe45657c5106a93..08004a25ce12fff941b125fffa551502504ede4c 100644 --- a/lm_eval/tasks/belebele/belebele_hat_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_hat_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "hat_Latn" -"include": "_default_template_yaml" -"task": "belebele_hat_Latn" -"test_split": "hat_Latn" +dataset_name: hat_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_hat_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_hau_Latn.yaml b/lm_eval/tasks/belebele/belebele_hau_Latn.yaml index e3626f36ad94245c596c3135b4e1950a47f71fb9..aa2efa285bd3a5090d418ce7fe9d6c879c5987c4 100644 --- a/lm_eval/tasks/belebele/belebele_hau_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_hau_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "hau_Latn" -"include": "_default_template_yaml" -"task": "belebele_hau_Latn" -"test_split": "hau_Latn" +dataset_name: hau_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_hau_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_heb_Hebr.yaml b/lm_eval/tasks/belebele/belebele_heb_Hebr.yaml index 6021c5c232a0b86fcebf4c96ebd2d584c16403c2..f6393790f56a29a390214109eb89518583379083 100644 --- a/lm_eval/tasks/belebele/belebele_heb_Hebr.yaml +++ b/lm_eval/tasks/belebele/belebele_heb_Hebr.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "heb_Hebr" -"include": "_default_template_yaml" -"task": "belebele_heb_Hebr" -"test_split": "heb_Hebr" +dataset_name: heb_Hebr +fewshot_split: test +include: _default_template_yaml +task: belebele_heb_Hebr +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_hin_Deva.yaml b/lm_eval/tasks/belebele/belebele_hin_Deva.yaml index 9fa304c9b09100d17edd37a5f7caa1b11f5d22df..5e22ecab5a14c616b1ffb28360f85f635f7c5543 100644 --- a/lm_eval/tasks/belebele/belebele_hin_Deva.yaml +++ b/lm_eval/tasks/belebele/belebele_hin_Deva.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "hin_Deva" -"include": "_default_template_yaml" -"task": "belebele_hin_Deva" -"test_split": "hin_Deva" +dataset_name: hin_Deva +fewshot_split: test +include: _default_template_yaml +task: belebele_hin_Deva +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_hin_Latn.yaml b/lm_eval/tasks/belebele/belebele_hin_Latn.yaml index f44f4f55f458197f0453be18fb5e389939da27b1..ea145124b821958e86a5cd1a52d8bcc00ee74fa8 100644 --- a/lm_eval/tasks/belebele/belebele_hin_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_hin_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "hin_Latn" -"include": "_default_template_yaml" -"task": "belebele_hin_Latn" -"test_split": "hin_Latn" +dataset_name: hin_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_hin_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_hrv_Latn.yaml b/lm_eval/tasks/belebele/belebele_hrv_Latn.yaml index 69b100c44bcf9218e541a7f3c41020dedafbec88..bcd9cf34d607f1f867c7abf15672d2a26e183cea 100644 --- a/lm_eval/tasks/belebele/belebele_hrv_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_hrv_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "hrv_Latn" -"include": "_default_template_yaml" -"task": "belebele_hrv_Latn" -"test_split": "hrv_Latn" +dataset_name: hrv_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_hrv_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_hun_Latn.yaml b/lm_eval/tasks/belebele/belebele_hun_Latn.yaml index 47f37f7db358d9cf061db38f5af0ac75867a726f..28465ffadb20c2960f1023e0e6bb2c34be419567 100644 --- a/lm_eval/tasks/belebele/belebele_hun_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_hun_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "hun_Latn" -"include": "_default_template_yaml" -"task": "belebele_hun_Latn" -"test_split": "hun_Latn" +dataset_name: hun_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_hun_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_hye_Armn.yaml b/lm_eval/tasks/belebele/belebele_hye_Armn.yaml index 5a57fa86451f834e5c4d8bea7d2961c2ff220b9d..bad41e72fe4fa12617cd02a5fd3cf55d21a8e10c 100644 --- a/lm_eval/tasks/belebele/belebele_hye_Armn.yaml +++ b/lm_eval/tasks/belebele/belebele_hye_Armn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "hye_Armn" -"include": "_default_template_yaml" -"task": "belebele_hye_Armn" -"test_split": "hye_Armn" +dataset_name: hye_Armn +fewshot_split: test +include: _default_template_yaml +task: belebele_hye_Armn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_ibo_Latn.yaml b/lm_eval/tasks/belebele/belebele_ibo_Latn.yaml index b0a9302e21aa970581d5b61a11539c2a55dc2fb8..47e9c66811708cf4768345020447d931210e8fb5 100644 --- a/lm_eval/tasks/belebele/belebele_ibo_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_ibo_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "ibo_Latn" -"include": "_default_template_yaml" -"task": "belebele_ibo_Latn" -"test_split": "ibo_Latn" +dataset_name: ibo_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_ibo_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_ilo_Latn.yaml b/lm_eval/tasks/belebele/belebele_ilo_Latn.yaml index fc3065da1c1a8c968845910a6f330c262a6a8a8e..4df1739c6538cdb783f7723424b8eaf47125559f 100644 --- a/lm_eval/tasks/belebele/belebele_ilo_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_ilo_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "ilo_Latn" -"include": "_default_template_yaml" -"task": "belebele_ilo_Latn" -"test_split": "ilo_Latn" +dataset_name: ilo_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_ilo_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_ind_Latn.yaml b/lm_eval/tasks/belebele/belebele_ind_Latn.yaml index c15fff3ec36a0b3b140581f317553b2fa0e2e62c..3a74093fe7d2c09cb62a604ba243aba9bb1d7301 100644 --- a/lm_eval/tasks/belebele/belebele_ind_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_ind_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "ind_Latn" -"include": "_default_template_yaml" -"task": "belebele_ind_Latn" -"test_split": "ind_Latn" +dataset_name: ind_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_ind_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_isl_Latn.yaml b/lm_eval/tasks/belebele/belebele_isl_Latn.yaml index 69f9bb4e372ce1a39057ce4b70a7e48d23d199e2..c063034edbaa6c97bc7238fbd701d84708bdb2bb 100644 --- a/lm_eval/tasks/belebele/belebele_isl_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_isl_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "isl_Latn" -"include": "_default_template_yaml" -"task": "belebele_isl_Latn" -"test_split": "isl_Latn" +dataset_name: isl_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_isl_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_ita_Latn.yaml b/lm_eval/tasks/belebele/belebele_ita_Latn.yaml index 69c2faffd004bf654f954d428559cf62df755496..1ace2b73db9ecd3af5587787c170e802e7b6d1ca 100644 --- a/lm_eval/tasks/belebele/belebele_ita_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_ita_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "ita_Latn" -"include": "_default_template_yaml" -"task": "belebele_ita_Latn" -"test_split": "ita_Latn" +dataset_name: ita_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_ita_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_jav_Latn.yaml b/lm_eval/tasks/belebele/belebele_jav_Latn.yaml index 0b2e0499bbb29bfa0bcf38a4c8e7402a80d3d4c3..fcff8bfe832fba40c2872c305ded02af2ca90916 100644 --- a/lm_eval/tasks/belebele/belebele_jav_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_jav_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "jav_Latn" -"include": "_default_template_yaml" -"task": "belebele_jav_Latn" -"test_split": "jav_Latn" +dataset_name: jav_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_jav_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_jpn_Jpan.yaml b/lm_eval/tasks/belebele/belebele_jpn_Jpan.yaml index 73b60502bc90b9e2128f9e2eb72046cf961b1054..8f3bb5e9df34bdddb797b7065648ca968e8748c1 100644 --- a/lm_eval/tasks/belebele/belebele_jpn_Jpan.yaml +++ b/lm_eval/tasks/belebele/belebele_jpn_Jpan.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "jpn_Jpan" -"include": "_default_template_yaml" -"task": "belebele_jpn_Jpan" -"test_split": "jpn_Jpan" +dataset_name: jpn_Jpan +fewshot_split: test +include: _default_template_yaml +task: belebele_jpn_Jpan +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_kac_Latn.yaml b/lm_eval/tasks/belebele/belebele_kac_Latn.yaml index 002bc9a15882eb9f51f745baa79cd22313ea92aa..57434c4c6c6ea1658017a445b1f2b0057aba761f 100644 --- a/lm_eval/tasks/belebele/belebele_kac_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_kac_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "kac_Latn" -"include": "_default_template_yaml" -"task": "belebele_kac_Latn" -"test_split": "kac_Latn" +dataset_name: kac_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_kac_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_kan_Knda.yaml b/lm_eval/tasks/belebele/belebele_kan_Knda.yaml index b4592afca58272c1b58d163c7f6626f3a2135d27..d827feabe3ebd4f11bbd6ee77987aaeee3967747 100644 --- a/lm_eval/tasks/belebele/belebele_kan_Knda.yaml +++ b/lm_eval/tasks/belebele/belebele_kan_Knda.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "kan_Knda" -"include": "_default_template_yaml" -"task": "belebele_kan_Knda" -"test_split": "kan_Knda" +dataset_name: kan_Knda +fewshot_split: test +include: _default_template_yaml +task: belebele_kan_Knda +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_kat_Geor.yaml b/lm_eval/tasks/belebele/belebele_kat_Geor.yaml index 6392d29bbe98cf3c6e228d20c90dbb7cf1281789..3665c2f735620a9907e20e2b9024114e9236b91a 100644 --- a/lm_eval/tasks/belebele/belebele_kat_Geor.yaml +++ b/lm_eval/tasks/belebele/belebele_kat_Geor.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "kat_Geor" -"include": "_default_template_yaml" -"task": "belebele_kat_Geor" -"test_split": "kat_Geor" +dataset_name: kat_Geor +fewshot_split: test +include: _default_template_yaml +task: belebele_kat_Geor +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_kaz_Cyrl.yaml b/lm_eval/tasks/belebele/belebele_kaz_Cyrl.yaml index 8af667d1ad67568d17f6c833979d8b04f9b8cffe..aaa54951b901dc097758ac80f955c9b51966ced0 100644 --- a/lm_eval/tasks/belebele/belebele_kaz_Cyrl.yaml +++ b/lm_eval/tasks/belebele/belebele_kaz_Cyrl.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "kaz_Cyrl" -"include": "_default_template_yaml" -"task": "belebele_kaz_Cyrl" -"test_split": "kaz_Cyrl" +dataset_name: kaz_Cyrl +fewshot_split: test +include: _default_template_yaml +task: belebele_kaz_Cyrl +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_kea_Latn.yaml b/lm_eval/tasks/belebele/belebele_kea_Latn.yaml index 7d584e946b30cc4ebc6b5b411f00ea4f845c64db..81ba7e73f86cb699138c21e7411a41aa3c1ca8bb 100644 --- a/lm_eval/tasks/belebele/belebele_kea_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_kea_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "kea_Latn" -"include": "_default_template_yaml" -"task": "belebele_kea_Latn" -"test_split": "kea_Latn" +dataset_name: kea_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_kea_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_khk_Cyrl.yaml b/lm_eval/tasks/belebele/belebele_khk_Cyrl.yaml index c77a77a79140f81476fa569a496b4e887138256b..ddfaa1dfe82ec4591e43d268292e2d65b0116e62 100644 --- a/lm_eval/tasks/belebele/belebele_khk_Cyrl.yaml +++ b/lm_eval/tasks/belebele/belebele_khk_Cyrl.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "khk_Cyrl" -"include": "_default_template_yaml" -"task": "belebele_khk_Cyrl" -"test_split": "khk_Cyrl" +dataset_name: khk_Cyrl +fewshot_split: test +include: _default_template_yaml +task: belebele_khk_Cyrl +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_khm_Khmr.yaml b/lm_eval/tasks/belebele/belebele_khm_Khmr.yaml index 39641c836b4ccf13ff16a730ef0ddd0ed6cc0962..cb49960a1bb5ef7c17da436b241a15978dc762f5 100644 --- a/lm_eval/tasks/belebele/belebele_khm_Khmr.yaml +++ b/lm_eval/tasks/belebele/belebele_khm_Khmr.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "khm_Khmr" -"include": "_default_template_yaml" -"task": "belebele_khm_Khmr" -"test_split": "khm_Khmr" +dataset_name: khm_Khmr +fewshot_split: test +include: _default_template_yaml +task: belebele_khm_Khmr +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_kin_Latn.yaml b/lm_eval/tasks/belebele/belebele_kin_Latn.yaml index 188a1691c7e8230ae178ed058513f973a1c18073..3c59acf2a9092757394b552d3167461ec694030e 100644 --- a/lm_eval/tasks/belebele/belebele_kin_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_kin_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "kin_Latn" -"include": "_default_template_yaml" -"task": "belebele_kin_Latn" -"test_split": "kin_Latn" +dataset_name: kin_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_kin_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_kir_Cyrl.yaml b/lm_eval/tasks/belebele/belebele_kir_Cyrl.yaml index 3aa2ead603cacbd5b75537ef5356425b8fe227ce..e645fceab6317ab63aa855903cb33cb4018fb025 100644 --- a/lm_eval/tasks/belebele/belebele_kir_Cyrl.yaml +++ b/lm_eval/tasks/belebele/belebele_kir_Cyrl.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "kir_Cyrl" -"include": "_default_template_yaml" -"task": "belebele_kir_Cyrl" -"test_split": "kir_Cyrl" +dataset_name: kir_Cyrl +fewshot_split: test +include: _default_template_yaml +task: belebele_kir_Cyrl +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_kor_Hang.yaml b/lm_eval/tasks/belebele/belebele_kor_Hang.yaml index 088d2e9359feaaf6504504e18390c0979988c6f9..93a55eecff18e739ccb3eeb6fb62973d3538b129 100644 --- a/lm_eval/tasks/belebele/belebele_kor_Hang.yaml +++ b/lm_eval/tasks/belebele/belebele_kor_Hang.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "kor_Hang" -"include": "_default_template_yaml" -"task": "belebele_kor_Hang" -"test_split": "kor_Hang" +dataset_name: kor_Hang +fewshot_split: test +include: _default_template_yaml +task: belebele_kor_Hang +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_lao_Laoo.yaml b/lm_eval/tasks/belebele/belebele_lao_Laoo.yaml index bdee22168b7536d2063e1d1602cb9032d97cb357..a528ca57ae50ebe9e4ac103a0dd003a313532382 100644 --- a/lm_eval/tasks/belebele/belebele_lao_Laoo.yaml +++ b/lm_eval/tasks/belebele/belebele_lao_Laoo.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "lao_Laoo" -"include": "_default_template_yaml" -"task": "belebele_lao_Laoo" -"test_split": "lao_Laoo" +dataset_name: lao_Laoo +fewshot_split: test +include: _default_template_yaml +task: belebele_lao_Laoo +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_lin_Latn.yaml b/lm_eval/tasks/belebele/belebele_lin_Latn.yaml index c312ae7578fa3dc4ccbb1f64ac367efb327c0457..702c6994bb674e6fc0330481c29d956e7e6514db 100644 --- a/lm_eval/tasks/belebele/belebele_lin_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_lin_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "lin_Latn" -"include": "_default_template_yaml" -"task": "belebele_lin_Latn" -"test_split": "lin_Latn" +dataset_name: lin_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_lin_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_lit_Latn.yaml b/lm_eval/tasks/belebele/belebele_lit_Latn.yaml index 275718b8b54368d29e9b48a87a3131af808eb77c..cbe242cadff9e103712bc473251bfb07ab154180 100644 --- a/lm_eval/tasks/belebele/belebele_lit_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_lit_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "lit_Latn" -"include": "_default_template_yaml" -"task": "belebele_lit_Latn" -"test_split": "lit_Latn" +dataset_name: lit_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_lit_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_lug_Latn.yaml b/lm_eval/tasks/belebele/belebele_lug_Latn.yaml index 3de4c1cd96efadff9245b077b1d5cfca78a8e292..06096353ae8740691b66528bad082978aa3267b0 100644 --- a/lm_eval/tasks/belebele/belebele_lug_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_lug_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "lug_Latn" -"include": "_default_template_yaml" -"task": "belebele_lug_Latn" -"test_split": "lug_Latn" +dataset_name: lug_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_lug_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_luo_Latn.yaml b/lm_eval/tasks/belebele/belebele_luo_Latn.yaml index 44ce22be6d00849d26e413643843b6e26c10bda4..ba0ca4b99be817d80fd9722a2594965a8b7da6e4 100644 --- a/lm_eval/tasks/belebele/belebele_luo_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_luo_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "luo_Latn" -"include": "_default_template_yaml" -"task": "belebele_luo_Latn" -"test_split": "luo_Latn" +dataset_name: luo_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_luo_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_lvs_Latn.yaml b/lm_eval/tasks/belebele/belebele_lvs_Latn.yaml index 3f745135d880e4da97403d53c9b2586d64a2cac2..19730965fe6de20fe834e06b4e8f10dd40c1c7c6 100644 --- a/lm_eval/tasks/belebele/belebele_lvs_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_lvs_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "lvs_Latn" -"include": "_default_template_yaml" -"task": "belebele_lvs_Latn" -"test_split": "lvs_Latn" +dataset_name: lvs_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_lvs_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_mal_Mlym.yaml b/lm_eval/tasks/belebele/belebele_mal_Mlym.yaml index 4387bd6df7a36d0a28ae2a7c27b7bc95d267a70e..26e5613655917d4ba192922fc315bf98a65547b9 100644 --- a/lm_eval/tasks/belebele/belebele_mal_Mlym.yaml +++ b/lm_eval/tasks/belebele/belebele_mal_Mlym.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "mal_Mlym" -"include": "_default_template_yaml" -"task": "belebele_mal_Mlym" -"test_split": "mal_Mlym" +dataset_name: mal_Mlym +fewshot_split: test +include: _default_template_yaml +task: belebele_mal_Mlym +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_mar_Deva.yaml b/lm_eval/tasks/belebele/belebele_mar_Deva.yaml index 908242aba29df1cb0ffe9735c3fb63e5a91c6212..0dddd8a965998442d362e26c4fe2bd97ba5b76e6 100644 --- a/lm_eval/tasks/belebele/belebele_mar_Deva.yaml +++ b/lm_eval/tasks/belebele/belebele_mar_Deva.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "mar_Deva" -"include": "_default_template_yaml" -"task": "belebele_mar_Deva" -"test_split": "mar_Deva" +dataset_name: mar_Deva +fewshot_split: test +include: _default_template_yaml +task: belebele_mar_Deva +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_mkd_Cyrl.yaml b/lm_eval/tasks/belebele/belebele_mkd_Cyrl.yaml index c9887108a949eae7be2e17e28a7ed9f81559f303..212fc3096ad093e7cf077c371465ad60301f42bf 100644 --- a/lm_eval/tasks/belebele/belebele_mkd_Cyrl.yaml +++ b/lm_eval/tasks/belebele/belebele_mkd_Cyrl.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "mkd_Cyrl" -"include": "_default_template_yaml" -"task": "belebele_mkd_Cyrl" -"test_split": "mkd_Cyrl" +dataset_name: mkd_Cyrl +fewshot_split: test +include: _default_template_yaml +task: belebele_mkd_Cyrl +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_mlt_Latn.yaml b/lm_eval/tasks/belebele/belebele_mlt_Latn.yaml index 5303cdd976b5b1f2ebbe1f36e8661ab966a856b6..494aaf16c95baa7927caa8e88fadd93f46faf492 100644 --- a/lm_eval/tasks/belebele/belebele_mlt_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_mlt_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "mlt_Latn" -"include": "_default_template_yaml" -"task": "belebele_mlt_Latn" -"test_split": "mlt_Latn" +dataset_name: mlt_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_mlt_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_mri_Latn.yaml b/lm_eval/tasks/belebele/belebele_mri_Latn.yaml index d704cdc478175da6c8894a08da2d2e177f895ed2..bafd8a3e51a2bff1bf9941d68d4e968d9a1039d6 100644 --- a/lm_eval/tasks/belebele/belebele_mri_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_mri_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "mri_Latn" -"include": "_default_template_yaml" -"task": "belebele_mri_Latn" -"test_split": "mri_Latn" +dataset_name: mri_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_mri_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_mya_Mymr.yaml b/lm_eval/tasks/belebele/belebele_mya_Mymr.yaml index d60ff718612f4ea43f05b07fd7c9ef87b5e19fd6..d5ec2a43bae05f670467c4a28768d495e3a4e431 100644 --- a/lm_eval/tasks/belebele/belebele_mya_Mymr.yaml +++ b/lm_eval/tasks/belebele/belebele_mya_Mymr.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "mya_Mymr" -"include": "_default_template_yaml" -"task": "belebele_mya_Mymr" -"test_split": "mya_Mymr" +dataset_name: mya_Mymr +fewshot_split: test +include: _default_template_yaml +task: belebele_mya_Mymr +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_nld_Latn.yaml b/lm_eval/tasks/belebele/belebele_nld_Latn.yaml index aea069996198c6bad23ed44969d3bc840ad04442..5e51bba0d82aa94e5e827e2d37b8729c8d8b59ba 100644 --- a/lm_eval/tasks/belebele/belebele_nld_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_nld_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "nld_Latn" -"include": "_default_template_yaml" -"task": "belebele_nld_Latn" -"test_split": "nld_Latn" +dataset_name: nld_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_nld_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_nob_Latn.yaml b/lm_eval/tasks/belebele/belebele_nob_Latn.yaml index cf824f3b9de3ce40db37060d2348c4a7b60a4c00..a314575c44553a9932f6d686288d8c5c82713bfd 100644 --- a/lm_eval/tasks/belebele/belebele_nob_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_nob_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "nob_Latn" -"include": "_default_template_yaml" -"task": "belebele_nob_Latn" -"test_split": "nob_Latn" +dataset_name: nob_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_nob_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_npi_Deva.yaml b/lm_eval/tasks/belebele/belebele_npi_Deva.yaml index c8126174671cdcc412958b90d1bc3051a8d4386a..3537278bec6dd21d66961f9caeffbab86559a7db 100644 --- a/lm_eval/tasks/belebele/belebele_npi_Deva.yaml +++ b/lm_eval/tasks/belebele/belebele_npi_Deva.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "npi_Deva" -"include": "_default_template_yaml" -"task": "belebele_npi_Deva" -"test_split": "npi_Deva" +dataset_name: npi_Deva +fewshot_split: test +include: _default_template_yaml +task: belebele_npi_Deva +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_npi_Latn.yaml b/lm_eval/tasks/belebele/belebele_npi_Latn.yaml index e2ff68ad9a17c247ec2c8a9df996df7dbb9e115a..073da2226e266bd915e9ea0bb3da59dfa6e39656 100644 --- a/lm_eval/tasks/belebele/belebele_npi_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_npi_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "npi_Latn" -"include": "_default_template_yaml" -"task": "belebele_npi_Latn" -"test_split": "npi_Latn" +dataset_name: npi_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_npi_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_nso_Latn.yaml b/lm_eval/tasks/belebele/belebele_nso_Latn.yaml index 6deb77602f876e7b95f23be45d6894ae73f42b39..361d4db3037744d26c33b74e68b38e66cd8ccb89 100644 --- a/lm_eval/tasks/belebele/belebele_nso_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_nso_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "nso_Latn" -"include": "_default_template_yaml" -"task": "belebele_nso_Latn" -"test_split": "nso_Latn" +dataset_name: nso_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_nso_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_nya_Latn.yaml b/lm_eval/tasks/belebele/belebele_nya_Latn.yaml index 4e5256715ade9f96d3c1eb7e1e3bc92c74700dd2..a5841987315644b4190604ca0454862a045d7ab9 100644 --- a/lm_eval/tasks/belebele/belebele_nya_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_nya_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "nya_Latn" -"include": "_default_template_yaml" -"task": "belebele_nya_Latn" -"test_split": "nya_Latn" +dataset_name: nya_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_nya_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_ory_Orya.yaml b/lm_eval/tasks/belebele/belebele_ory_Orya.yaml index 5590560aaac88be6ec8dc90353d308d23c759323..e329ac9dec3bae4b5ab1680fde7b32596c52dfa8 100644 --- a/lm_eval/tasks/belebele/belebele_ory_Orya.yaml +++ b/lm_eval/tasks/belebele/belebele_ory_Orya.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "ory_Orya" -"include": "_default_template_yaml" -"task": "belebele_ory_Orya" -"test_split": "ory_Orya" +dataset_name: ory_Orya +fewshot_split: test +include: _default_template_yaml +task: belebele_ory_Orya +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_pan_Guru.yaml b/lm_eval/tasks/belebele/belebele_pan_Guru.yaml index 6017b44d3d2090de73dd8ea759eac1675608f5e8..6dfd42beff92358cdc81135dd6bbe6b0601d02ca 100644 --- a/lm_eval/tasks/belebele/belebele_pan_Guru.yaml +++ b/lm_eval/tasks/belebele/belebele_pan_Guru.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "pan_Guru" -"include": "_default_template_yaml" -"task": "belebele_pan_Guru" -"test_split": "pan_Guru" +dataset_name: pan_Guru +fewshot_split: test +include: _default_template_yaml +task: belebele_pan_Guru +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_pbt_Arab.yaml b/lm_eval/tasks/belebele/belebele_pbt_Arab.yaml index 7630b0fc3dc83d4a3587913a689fa05f2eaf432b..a8f250ad8024cb084bdea6451d22e23a72bc0ae3 100644 --- a/lm_eval/tasks/belebele/belebele_pbt_Arab.yaml +++ b/lm_eval/tasks/belebele/belebele_pbt_Arab.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "pbt_Arab" -"include": "_default_template_yaml" -"task": "belebele_pbt_Arab" -"test_split": "pbt_Arab" +dataset_name: pbt_Arab +fewshot_split: test +include: _default_template_yaml +task: belebele_pbt_Arab +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_pes_Arab.yaml b/lm_eval/tasks/belebele/belebele_pes_Arab.yaml index ceda24e2a9be857e76584dff421da71427f6c50e..945b29e6d3216288da54cf5253c7994092944f9a 100644 --- a/lm_eval/tasks/belebele/belebele_pes_Arab.yaml +++ b/lm_eval/tasks/belebele/belebele_pes_Arab.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "pes_Arab" -"include": "_default_template_yaml" -"task": "belebele_pes_Arab" -"test_split": "pes_Arab" +dataset_name: pes_Arab +fewshot_split: test +include: _default_template_yaml +task: belebele_pes_Arab +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_plt_Latn.yaml b/lm_eval/tasks/belebele/belebele_plt_Latn.yaml index 17889ae3556278624d79fe9bc632310a3008c66c..70eeaa752ccad29a6651424ad89a36fbb36947f9 100644 --- a/lm_eval/tasks/belebele/belebele_plt_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_plt_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "plt_Latn" -"include": "_default_template_yaml" -"task": "belebele_plt_Latn" -"test_split": "plt_Latn" +dataset_name: plt_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_plt_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_pol_Latn.yaml b/lm_eval/tasks/belebele/belebele_pol_Latn.yaml index ebfcf3534e53c10bfe370643bfc50fc94df5602c..295d5e75c0bc5d2eee3821a33273733351cc9cf0 100644 --- a/lm_eval/tasks/belebele/belebele_pol_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_pol_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "pol_Latn" -"include": "_default_template_yaml" -"task": "belebele_pol_Latn" -"test_split": "pol_Latn" +dataset_name: pol_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_pol_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_por_Latn.yaml b/lm_eval/tasks/belebele/belebele_por_Latn.yaml index f63ace02d5a3fb8cc1b0b85a279c49370567b16c..dddcdf507411dd7563655ee93890b61fe7ac308c 100644 --- a/lm_eval/tasks/belebele/belebele_por_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_por_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "por_Latn" -"include": "_default_template_yaml" -"task": "belebele_por_Latn" -"test_split": "por_Latn" +dataset_name: por_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_por_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_ron_Latn.yaml b/lm_eval/tasks/belebele/belebele_ron_Latn.yaml index ab1ad0889b4797ce6068fd1c8741fc45e29cb588..2befab4d49ba5f4dde27d044b38980634ae5c924 100644 --- a/lm_eval/tasks/belebele/belebele_ron_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_ron_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "ron_Latn" -"include": "_default_template_yaml" -"task": "belebele_ron_Latn" -"test_split": "ron_Latn" +dataset_name: ron_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_ron_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_rus_Cyrl.yaml b/lm_eval/tasks/belebele/belebele_rus_Cyrl.yaml index 280768c7c61d44a0a9e7bb8aeae64ebbecfdd84a..839e24b8b90222783d3b560b08a7fd8a9e810a0f 100644 --- a/lm_eval/tasks/belebele/belebele_rus_Cyrl.yaml +++ b/lm_eval/tasks/belebele/belebele_rus_Cyrl.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "rus_Cyrl" -"include": "_default_template_yaml" -"task": "belebele_rus_Cyrl" -"test_split": "rus_Cyrl" +dataset_name: rus_Cyrl +fewshot_split: test +include: _default_template_yaml +task: belebele_rus_Cyrl +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_shn_Mymr.yaml b/lm_eval/tasks/belebele/belebele_shn_Mymr.yaml index 75bbd05b16c54b593729f9a8f7b4499c25d2f5ed..101c4f07696cb7bf4e7748fddc2c8eb526e81a38 100644 --- a/lm_eval/tasks/belebele/belebele_shn_Mymr.yaml +++ b/lm_eval/tasks/belebele/belebele_shn_Mymr.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "shn_Mymr" -"include": "_default_template_yaml" -"task": "belebele_shn_Mymr" -"test_split": "shn_Mymr" +dataset_name: shn_Mymr +fewshot_split: test +include: _default_template_yaml +task: belebele_shn_Mymr +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_sin_Latn.yaml b/lm_eval/tasks/belebele/belebele_sin_Latn.yaml index 9c035361f22ab53504eb6a94c4a23c787ab92c05..7449b54415bc846605387d8aee3d2a8993618aef 100644 --- a/lm_eval/tasks/belebele/belebele_sin_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_sin_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "sin_Latn" -"include": "_default_template_yaml" -"task": "belebele_sin_Latn" -"test_split": "sin_Latn" +dataset_name: sin_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_sin_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_sin_Sinh.yaml b/lm_eval/tasks/belebele/belebele_sin_Sinh.yaml index c1e42d9ac72601ffe27ddcc5994166a87a76bb31..b0ec07809ffd73bb638c364d2ab2d07ce99c7c6c 100644 --- a/lm_eval/tasks/belebele/belebele_sin_Sinh.yaml +++ b/lm_eval/tasks/belebele/belebele_sin_Sinh.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "sin_Sinh" -"include": "_default_template_yaml" -"task": "belebele_sin_Sinh" -"test_split": "sin_Sinh" +dataset_name: sin_Sinh +fewshot_split: test +include: _default_template_yaml +task: belebele_sin_Sinh +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_slk_Latn.yaml b/lm_eval/tasks/belebele/belebele_slk_Latn.yaml index cddd1eb1e67713b5c635e598ece58d115dc0b4c0..010790034ed2424e25d45387891a9a4afbd114d6 100644 --- a/lm_eval/tasks/belebele/belebele_slk_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_slk_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "slk_Latn" -"include": "_default_template_yaml" -"task": "belebele_slk_Latn" -"test_split": "slk_Latn" +dataset_name: slk_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_slk_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_slv_Latn.yaml b/lm_eval/tasks/belebele/belebele_slv_Latn.yaml index 9e3e5338231b478c0c9614dabb6e26c7dd11d994..30697d7d0c7667283cb8e651bd5caf5624f77bfc 100644 --- a/lm_eval/tasks/belebele/belebele_slv_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_slv_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "slv_Latn" -"include": "_default_template_yaml" -"task": "belebele_slv_Latn" -"test_split": "slv_Latn" +dataset_name: slv_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_slv_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_sna_Latn.yaml b/lm_eval/tasks/belebele/belebele_sna_Latn.yaml index a426a637e51fa1ec240b0c141beca1010de1ebd2..50d91f6c7e94df6718fe93f2e23c748de24fa93c 100644 --- a/lm_eval/tasks/belebele/belebele_sna_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_sna_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "sna_Latn" -"include": "_default_template_yaml" -"task": "belebele_sna_Latn" -"test_split": "sna_Latn" +dataset_name: sna_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_sna_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_snd_Arab.yaml b/lm_eval/tasks/belebele/belebele_snd_Arab.yaml index ca8d1643f8bd1c98e7742c1ede66242aecae4f7f..1e9463246e1c494913653119448434bdc6a5ef5c 100644 --- a/lm_eval/tasks/belebele/belebele_snd_Arab.yaml +++ b/lm_eval/tasks/belebele/belebele_snd_Arab.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "snd_Arab" -"include": "_default_template_yaml" -"task": "belebele_snd_Arab" -"test_split": "snd_Arab" +dataset_name: snd_Arab +fewshot_split: test +include: _default_template_yaml +task: belebele_snd_Arab +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_som_Latn.yaml b/lm_eval/tasks/belebele/belebele_som_Latn.yaml index fa1d4329d878e141ffbbb3f7faed774abbfccacb..f68411524317a304f41d8edf14b18a3b9cec2731 100644 --- a/lm_eval/tasks/belebele/belebele_som_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_som_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "som_Latn" -"include": "_default_template_yaml" -"task": "belebele_som_Latn" -"test_split": "som_Latn" +dataset_name: som_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_som_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_sot_Latn.yaml b/lm_eval/tasks/belebele/belebele_sot_Latn.yaml index f98de9a3dab12ca1af6f570904abc976d11525df..104494a360625f30fd2220acad8071fcf38a2fa1 100644 --- a/lm_eval/tasks/belebele/belebele_sot_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_sot_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "sot_Latn" -"include": "_default_template_yaml" -"task": "belebele_sot_Latn" -"test_split": "sot_Latn" +dataset_name: sot_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_sot_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_spa_Latn.yaml b/lm_eval/tasks/belebele/belebele_spa_Latn.yaml index d26c6315b7fcde764b1cc6dfa0393a9ad0778bc2..928d9bcd37d7dfd7e6f59302791a76c30eef875d 100644 --- a/lm_eval/tasks/belebele/belebele_spa_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_spa_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "spa_Latn" -"include": "_default_template_yaml" -"task": "belebele_spa_Latn" -"test_split": "spa_Latn" +dataset_name: spa_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_spa_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_srp_Cyrl.yaml b/lm_eval/tasks/belebele/belebele_srp_Cyrl.yaml index ad34472a6635b2da9e5e3003036a48678a5b2994..bb51387519ed4becebbf06fe1219d3a978672e50 100644 --- a/lm_eval/tasks/belebele/belebele_srp_Cyrl.yaml +++ b/lm_eval/tasks/belebele/belebele_srp_Cyrl.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "srp_Cyrl" -"include": "_default_template_yaml" -"task": "belebele_srp_Cyrl" -"test_split": "srp_Cyrl" +dataset_name: srp_Cyrl +fewshot_split: test +include: _default_template_yaml +task: belebele_srp_Cyrl +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_ssw_Latn.yaml b/lm_eval/tasks/belebele/belebele_ssw_Latn.yaml index 788d6959976320f5fb962e442aa8fa9c2ed9cca8..4448f3a63e7c27c898a50a954ca2abbbd0cc95aa 100644 --- a/lm_eval/tasks/belebele/belebele_ssw_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_ssw_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "ssw_Latn" -"include": "_default_template_yaml" -"task": "belebele_ssw_Latn" -"test_split": "ssw_Latn" +dataset_name: ssw_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_ssw_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_sun_Latn.yaml b/lm_eval/tasks/belebele/belebele_sun_Latn.yaml index 3e599beb49507b9566f0d0e77c20673cecfc84df..a4c582ca5029014e74c1ebb4b023ffd4a300a7d7 100644 --- a/lm_eval/tasks/belebele/belebele_sun_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_sun_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "sun_Latn" -"include": "_default_template_yaml" -"task": "belebele_sun_Latn" -"test_split": "sun_Latn" +dataset_name: sun_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_sun_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_swe_Latn.yaml b/lm_eval/tasks/belebele/belebele_swe_Latn.yaml index 6ac7a4afffb3db57119df0db971415e48aa94301..2f390f4b4021618ca09c0f9fbb7f4c2aca235ef7 100644 --- a/lm_eval/tasks/belebele/belebele_swe_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_swe_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "swe_Latn" -"include": "_default_template_yaml" -"task": "belebele_swe_Latn" -"test_split": "swe_Latn" +dataset_name: swe_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_swe_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_swh_Latn.yaml b/lm_eval/tasks/belebele/belebele_swh_Latn.yaml index 55845837a155e9fd1525830a263019910c793864..316c7af2c994b32d232b44180260de70b9b8eb83 100644 --- a/lm_eval/tasks/belebele/belebele_swh_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_swh_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "swh_Latn" -"include": "_default_template_yaml" -"task": "belebele_swh_Latn" -"test_split": "swh_Latn" +dataset_name: swh_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_swh_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_tam_Taml.yaml b/lm_eval/tasks/belebele/belebele_tam_Taml.yaml index 9c81da2f8a7ab0c60628cd834eecb1f70031173b..626a5c28a758b9b52d71cbc176f65051d4c33761 100644 --- a/lm_eval/tasks/belebele/belebele_tam_Taml.yaml +++ b/lm_eval/tasks/belebele/belebele_tam_Taml.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "tam_Taml" -"include": "_default_template_yaml" -"task": "belebele_tam_Taml" -"test_split": "tam_Taml" +dataset_name: tam_Taml +fewshot_split: test +include: _default_template_yaml +task: belebele_tam_Taml +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_tel_Telu.yaml b/lm_eval/tasks/belebele/belebele_tel_Telu.yaml index de44fcc4848927331994d5ca42dce064c7758483..3906de7d45c30e8a69e6465858b393ee3cd650f6 100644 --- a/lm_eval/tasks/belebele/belebele_tel_Telu.yaml +++ b/lm_eval/tasks/belebele/belebele_tel_Telu.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "tel_Telu" -"include": "_default_template_yaml" -"task": "belebele_tel_Telu" -"test_split": "tel_Telu" +dataset_name: tel_Telu +fewshot_split: test +include: _default_template_yaml +task: belebele_tel_Telu +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_tgk_Cyrl.yaml b/lm_eval/tasks/belebele/belebele_tgk_Cyrl.yaml index 1597e5489c3f8960f12d97291661c23ace9a76ad..8e7c7fee495e2c8f9f724acfd8b34478639187ed 100644 --- a/lm_eval/tasks/belebele/belebele_tgk_Cyrl.yaml +++ b/lm_eval/tasks/belebele/belebele_tgk_Cyrl.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "tgk_Cyrl" -"include": "_default_template_yaml" -"task": "belebele_tgk_Cyrl" -"test_split": "tgk_Cyrl" +dataset_name: tgk_Cyrl +fewshot_split: test +include: _default_template_yaml +task: belebele_tgk_Cyrl +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_tgl_Latn.yaml b/lm_eval/tasks/belebele/belebele_tgl_Latn.yaml index 3b71adbc7fe41428fd350b60968fe744a94303db..9a3f3358eed8f4e00750997e64fb099d2abf79a7 100644 --- a/lm_eval/tasks/belebele/belebele_tgl_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_tgl_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "tgl_Latn" -"include": "_default_template_yaml" -"task": "belebele_tgl_Latn" -"test_split": "tgl_Latn" +dataset_name: tgl_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_tgl_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_tha_Thai.yaml b/lm_eval/tasks/belebele/belebele_tha_Thai.yaml index 6fb82b254ae7ca5d55f6c842f928799463865cc8..41491517c865324a00e3960f9e7792de9cb8c435 100644 --- a/lm_eval/tasks/belebele/belebele_tha_Thai.yaml +++ b/lm_eval/tasks/belebele/belebele_tha_Thai.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "tha_Thai" -"include": "_default_template_yaml" -"task": "belebele_tha_Thai" -"test_split": "tha_Thai" +dataset_name: tha_Thai +fewshot_split: test +include: _default_template_yaml +task: belebele_tha_Thai +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_tir_Ethi.yaml b/lm_eval/tasks/belebele/belebele_tir_Ethi.yaml index ca902d2a391ea872a2c3a75eded5eadfd3b8a1a6..b5063df797bac7467816910e9921db09a680cb5e 100644 --- a/lm_eval/tasks/belebele/belebele_tir_Ethi.yaml +++ b/lm_eval/tasks/belebele/belebele_tir_Ethi.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "tir_Ethi" -"include": "_default_template_yaml" -"task": "belebele_tir_Ethi" -"test_split": "tir_Ethi" +dataset_name: tir_Ethi +fewshot_split: test +include: _default_template_yaml +task: belebele_tir_Ethi +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_tsn_Latn.yaml b/lm_eval/tasks/belebele/belebele_tsn_Latn.yaml index e9c7aa83cf7f4195ad3f113f1420046d8986ed8a..8a335f74efa379e48fcd43cfcb58d82c60d0f281 100644 --- a/lm_eval/tasks/belebele/belebele_tsn_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_tsn_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "tsn_Latn" -"include": "_default_template_yaml" -"task": "belebele_tsn_Latn" -"test_split": "tsn_Latn" +dataset_name: tsn_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_tsn_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_tso_Latn.yaml b/lm_eval/tasks/belebele/belebele_tso_Latn.yaml index 1dae599eb73aade0b753216421eac391afe89985..dd584be1b349d435ef08bf2e20880d34a2ee494b 100644 --- a/lm_eval/tasks/belebele/belebele_tso_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_tso_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "tso_Latn" -"include": "_default_template_yaml" -"task": "belebele_tso_Latn" -"test_split": "tso_Latn" +dataset_name: tso_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_tso_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_tur_Latn.yaml b/lm_eval/tasks/belebele/belebele_tur_Latn.yaml index ee490bb0bab9c0f32c8e62d6d0bb553cbb91a192..e455a31af4264b3a95ff48d122b7b77bc61bda6b 100644 --- a/lm_eval/tasks/belebele/belebele_tur_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_tur_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "tur_Latn" -"include": "_default_template_yaml" -"task": "belebele_tur_Latn" -"test_split": "tur_Latn" +dataset_name: tur_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_tur_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_ukr_Cyrl.yaml b/lm_eval/tasks/belebele/belebele_ukr_Cyrl.yaml index c24156d846cb64db04877d9c36d394b54f56aa3e..894415571f87128bab38dd0f7fa18cf4f7ad23ef 100644 --- a/lm_eval/tasks/belebele/belebele_ukr_Cyrl.yaml +++ b/lm_eval/tasks/belebele/belebele_ukr_Cyrl.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "ukr_Cyrl" -"include": "_default_template_yaml" -"task": "belebele_ukr_Cyrl" -"test_split": "ukr_Cyrl" +dataset_name: ukr_Cyrl +fewshot_split: test +include: _default_template_yaml +task: belebele_ukr_Cyrl +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_urd_Arab.yaml b/lm_eval/tasks/belebele/belebele_urd_Arab.yaml index a8c54e9ea623535b89b9147c9d9660a5723c5bdd..dc3ddf9e216b825597a8b4d04490513095762a23 100644 --- a/lm_eval/tasks/belebele/belebele_urd_Arab.yaml +++ b/lm_eval/tasks/belebele/belebele_urd_Arab.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "urd_Arab" -"include": "_default_template_yaml" -"task": "belebele_urd_Arab" -"test_split": "urd_Arab" +dataset_name: urd_Arab +fewshot_split: test +include: _default_template_yaml +task: belebele_urd_Arab +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_urd_Latn.yaml b/lm_eval/tasks/belebele/belebele_urd_Latn.yaml index 8ea63063b6f99815ae5c51faeec53352bc28721d..76eb3e3c3a39fb66afd96af49d065d7e10f88fac 100644 --- a/lm_eval/tasks/belebele/belebele_urd_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_urd_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "urd_Latn" -"include": "_default_template_yaml" -"task": "belebele_urd_Latn" -"test_split": "urd_Latn" +dataset_name: urd_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_urd_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_uzn_Latn.yaml b/lm_eval/tasks/belebele/belebele_uzn_Latn.yaml index 109aebdd5fc3c6a07083821dae0f61c0037b6ae2..a0ecd20f85b668d0d1d3a940ba2bad0d45aa44a6 100644 --- a/lm_eval/tasks/belebele/belebele_uzn_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_uzn_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "uzn_Latn" -"include": "_default_template_yaml" -"task": "belebele_uzn_Latn" -"test_split": "uzn_Latn" +dataset_name: uzn_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_uzn_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_vie_Latn.yaml b/lm_eval/tasks/belebele/belebele_vie_Latn.yaml index 685e69785371db7a6021acc4298e0ddbc809dced..93cd794f6d7118a787f4c87d5468f1469950c317 100644 --- a/lm_eval/tasks/belebele/belebele_vie_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_vie_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "vie_Latn" -"include": "_default_template_yaml" -"task": "belebele_vie_Latn" -"test_split": "vie_Latn" +dataset_name: vie_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_vie_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_war_Latn.yaml b/lm_eval/tasks/belebele/belebele_war_Latn.yaml index d34508c3137b556b84038a06fcc6a9a2ea91f68c..272c1fdac235cce6a508cf44d0fb6062934974e9 100644 --- a/lm_eval/tasks/belebele/belebele_war_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_war_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "war_Latn" -"include": "_default_template_yaml" -"task": "belebele_war_Latn" -"test_split": "war_Latn" +dataset_name: war_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_war_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_wol_Latn.yaml b/lm_eval/tasks/belebele/belebele_wol_Latn.yaml index 7683e3d2206e9bfb04ec2a2cf2d068c2be9570c3..f2f6cbcab99b7c87d45898091245d680a402090a 100644 --- a/lm_eval/tasks/belebele/belebele_wol_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_wol_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "wol_Latn" -"include": "_default_template_yaml" -"task": "belebele_wol_Latn" -"test_split": "wol_Latn" +dataset_name: wol_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_wol_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_xho_Latn.yaml b/lm_eval/tasks/belebele/belebele_xho_Latn.yaml index 2962f3553a24a64fdcc4ca4e7cdd771c657a5306..c2c4c047f15eb4eef45dc608380c21745ad58475 100644 --- a/lm_eval/tasks/belebele/belebele_xho_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_xho_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "xho_Latn" -"include": "_default_template_yaml" -"task": "belebele_xho_Latn" -"test_split": "xho_Latn" +dataset_name: xho_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_xho_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_yor_Latn.yaml b/lm_eval/tasks/belebele/belebele_yor_Latn.yaml index 9eb295cc0471885c83a4114b0551d134611d8c56..bd3f7f45c58d1245b36c6b6af9f9c80fbd52b92a 100644 --- a/lm_eval/tasks/belebele/belebele_yor_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_yor_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "yor_Latn" -"include": "_default_template_yaml" -"task": "belebele_yor_Latn" -"test_split": "yor_Latn" +dataset_name: yor_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_yor_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_zho_Hans.yaml b/lm_eval/tasks/belebele/belebele_zho_Hans.yaml index 32b0860bdaa2806161c32d9d8359d02ec61140d2..d7ef9aed306b2a82c219363479377a7cbbb17e0b 100644 --- a/lm_eval/tasks/belebele/belebele_zho_Hans.yaml +++ b/lm_eval/tasks/belebele/belebele_zho_Hans.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "zho_Hans" -"include": "_default_template_yaml" -"task": "belebele_zho_Hans" -"test_split": "zho_Hans" +dataset_name: zho_Hans +fewshot_split: test +include: _default_template_yaml +task: belebele_zho_Hans +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_zho_Hant.yaml b/lm_eval/tasks/belebele/belebele_zho_Hant.yaml index c8604e58b6b747cd32a0621fe2a1858a3102da36..65fba54f2df034c74f6041aa26f344a6d3b5697f 100644 --- a/lm_eval/tasks/belebele/belebele_zho_Hant.yaml +++ b/lm_eval/tasks/belebele/belebele_zho_Hant.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "zho_Hant" -"include": "_default_template_yaml" -"task": "belebele_zho_Hant" -"test_split": "zho_Hant" +dataset_name: zho_Hant +fewshot_split: test +include: _default_template_yaml +task: belebele_zho_Hant +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_zsm_Latn.yaml b/lm_eval/tasks/belebele/belebele_zsm_Latn.yaml index d31cadf5e9aafb4b1337741dd7247607a36c456e..13a78e688e23a1e14d03b5aee9ea9301eedde338 100644 --- a/lm_eval/tasks/belebele/belebele_zsm_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_zsm_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "zsm_Latn" -"include": "_default_template_yaml" -"task": "belebele_zsm_Latn" -"test_split": "zsm_Latn" +dataset_name: zsm_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_zsm_Latn +test_split: test diff --git a/lm_eval/tasks/belebele/belebele_zul_Latn.yaml b/lm_eval/tasks/belebele/belebele_zul_Latn.yaml index 1e7fede97ca234d40a87acc7a0e21aaf659a2faf..dc11018882748044f55f76bb4007c60fc2bee529 100644 --- a/lm_eval/tasks/belebele/belebele_zul_Latn.yaml +++ b/lm_eval/tasks/belebele/belebele_zul_Latn.yaml @@ -1,4 +1,5 @@ -"fewshot_split": "zul_Latn" -"include": "_default_template_yaml" -"task": "belebele_zul_Latn" -"test_split": "zul_Latn" +dataset_name: zul_Latn +fewshot_split: test +include: _default_template_yaml +task: belebele_zul_Latn +test_split: test diff --git a/lm_eval/tasks/benchmarks/flan/flan_held_in.yaml b/lm_eval/tasks/benchmarks/flan/flan_held_in.yaml index 5796713506e3b2e6632f4df0d60c4c19377693ad..c57d265492916e76d2938feb0f1ab688e3562ca9 100644 --- a/lm_eval/tasks/benchmarks/flan/flan_held_in.yaml +++ b/lm_eval/tasks/benchmarks/flan/flan_held_in.yaml @@ -4,48 +4,51 @@ task: # ANLI R1 - group: anli_r1_flan group_alias: ANLI R1 + aggregate_metric_list: + - metric: acc + weight_by_size: True task: - - task: anli_r1 + - task: anli_r1_prompt-0 task_alias: prompt-0 include: _held_in_template_yaml doc_to_text: "{{premise}}\n\nChoose your answer: based on the paragraph above can we conclude that \"{{hypothesis}}\"?\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No\nI think the answer is" doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" - - task: anli_r1 + - task: anli_r1_prompt-1 task_alias: prompt-1 include: _held_in_template_yaml doc_to_text: "{{premise}}\n\nBased on that paragraph can we conclude that this sentence is true?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No" doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" - - task: anli_r1 + - task: anli_r1_prompt-2 task_alias: prompt-2 include: _held_in_template_yaml doc_to_text: "{{premise}}\n\nCan we draw the following conclusion?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No" doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" - - task: anli_r1 + - task: anli_r1_prompt-3 task_alias: prompt-3 include: _held_in_template_yaml doc_to_text: "{{premise}}\nDoes this next sentence follow, given the preceding text?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No" doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" - - task: anli_r1 + - task: anli_r1_prompt-4 task_alias: prompt-4 include: _held_in_template_yaml doc_to_text: "{{premise}}\nCan we infer the following?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No\nThe answer is:" doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" - - task: anli_r1 + - task: anli_r1_prompt-5 task_alias: prompt-5 include: _held_in_template_yaml doc_to_text: "Read the following paragraph and determine if the hypothesis is true:\n\n{{premise}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No\nHypothesis: {{hypothesis}}\n\n\n" doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" - - task: anli_r1 + - task: anli_r1_prompt-6 task_alias: prompt-6 include: _held_in_template_yaml doc_to_text: "Read the text and determine if the sentence is true (see options at the end):\n\n{{premise}}\n\nSentence: {{hypothesis}}\nOPTIONS:\n- Yes\n- It's impossible to say\n- No" doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" - - task: anli_r1 + - task: anli_r1_prompt-7 task_alias: prompt-7 include: _held_in_template_yaml doc_to_text: "Can we draw the following hypothesis from the context (see options)? \n\nContext:\n\n{{premise}}\n\nHypothesis: {{hypothesis}}\nOPTIONS:\n- Yes\n- It's impossible to say\n- No" doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" - - task: anli_r1 + - task: anli_r1_prompt-8 task_alias: prompt-8 include: _held_in_template_yaml doc_to_text: "Choose from options: Determine if the sentence is true based on the text below:\n{{hypothesis}}\n\n{{premise}}\nOPTIONS:\n- Yes\n- It's impossible to say\n- No" @@ -53,48 +56,51 @@ task: # ANLI R2 - group: anli_r2_flan group_alias: ANLI R2 + aggregate_metric_list: + - metric: acc + weight_by_size: True task: - - task: anli_r2 + - task: anli_r2_prompt-0 task_alias: prompt-0 include: _held_in_template_yaml doc_to_text: "{{premise}}\n\nChoose your answer: based on the paragraph above can we conclude that \"{{hypothesis}}\"?\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No\nI think the answer is" doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" - - task: anli_r2 + - task: anli_r2_prompt-1 task_alias: prompt-1 include: _held_in_template_yaml doc_to_text: "{{premise}}\n\nBased on that paragraph can we conclude that this sentence is true?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No" doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" - - task: anli_r2 + - task: anli_r2_prompt-2 task_alias: prompt-2 include: _held_in_template_yaml doc_to_text: "{{premise}}\n\nCan we draw the following conclusion?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No" doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" - - task: anli_r2 + - task: anli_r2_prompt-3 task_alias: prompt-3 include: _held_in_template_yaml doc_to_text: "{{premise}}\nDoes this next sentence follow, given the preceding text?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No" doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" - - task: anli_r2 + - task: anli_r2_prompt-4 task_alias: prompt-4 include: _held_in_template_yaml doc_to_text: "{{premise}}\nCan we infer the following?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No\nThe answer is:" doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" - - task: anli_r2 + - task: anli_r2_prompt-5 task_alias: prompt-5 include: _held_in_template_yaml doc_to_text: "Read the following paragraph and determine if the hypothesis is true:\n\n{{premise}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No\nHypothesis: {{hypothesis}}\n\n\n" doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" - - task: anli_r2 + - task: anli_r2_prompt-6 task_alias: prompt-6 include: _held_in_template_yaml doc_to_text: "Read the text and determine if the sentence is true (see options at the end):\n\n{{premise}}\n\nSentence: {{hypothesis}}\nOPTIONS:\n- Yes\n- It's impossible to say\n- No" doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" - - task: anli_r2 + - task: anli_r2_prompt-7 task_alias: prompt-7 include: _held_in_template_yaml doc_to_text: "Can we draw the following hypothesis from the context (see options)? \n\nContext:\n\n{{premise}}\n\nHypothesis: {{hypothesis}}\nOPTIONS:\n- Yes\n- It's impossible to say\n- No" doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" - - task: anli_r2 + - task: anli_r2_prompt-8 task_alias: prompt-8 include: _held_in_template_yaml doc_to_text: "Choose from options: Determine if the sentence is true based on the text below:\n{{hypothesis}}\n\n{{premise}}\nOPTIONS:\n- Yes\n- It's impossible to say\n- No" @@ -102,48 +108,51 @@ task: # ANLI R3 - group: anli_r3_flan group_alias: ANLI R3 + aggregate_metric_list: + - metric: acc + weight_by_size: True task: - - task: anli_r3 + - task: anli_r3_prompt-0 task_alias: prompt-0 include: _held_in_template_yaml doc_to_text: "{{premise}}\n\nChoose your answer: based on the paragraph above can we conclude that \"{{hypothesis}}\"?\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No\nI think the answer is" doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" - - task: anli_r3 + - task: anli_r3_prompt-1 task_alias: prompt-1 include: _held_in_template_yaml doc_to_text: "{{premise}}\n\nBased on that paragraph can we conclude that this sentence is true?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No" doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" - - task: anli_r3 + - task: anli_r3_prompt-2 task_alias: prompt-2 include: _held_in_template_yaml doc_to_text: "{{premise}}\n\nCan we draw the following conclusion?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No" doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" - - task: anli_r3 + - task: anli_r3_prompt-3 task_alias: prompt-3 include: _held_in_template_yaml doc_to_text: "{{premise}}\nDoes this next sentence follow, given the preceding text?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No" doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" - - task: anli_r3 + - task: anli_r3_prompt-4 task_alias: prompt-4 include: _held_in_template_yaml doc_to_text: "{{premise}}\nCan we infer the following?\n{{hypothesis}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No\nThe answer is:" doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" - - task: anli_r3 + - task: anli_r3_prompt-5 task_alias: prompt-5 include: _held_in_template_yaml doc_to_text: "Read the following paragraph and determine if the hypothesis is true:\n\n{{premise}}\n\nOPTIONS:\n- Yes\n- It's impossible to say\n- No\nHypothesis: {{hypothesis}}\n\n\n" doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" - - task: anli_r3 + - task: anli_r3_prompt-6 task_alias: prompt-6 include: _held_in_template_yaml doc_to_text: "Read the text and determine if the sentence is true (see options at the end):\n\n{{premise}}\n\nSentence: {{hypothesis}}\nOPTIONS:\n- Yes\n- It's impossible to say\n- No" doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" - - task: anli_r3 + - task: anli_r3_prompt-7 task_alias: prompt-7 include: _held_in_template_yaml doc_to_text: "Can we draw the following hypothesis from the context (see options)? \n\nContext:\n\n{{premise}}\n\nHypothesis: {{hypothesis}}\nOPTIONS:\n- Yes\n- It's impossible to say\n- No" doc_to_target: "{{[\"Yes\", \"It's impossible to say\", \"No\"][label]}}" - - task: anli_r3 + - task: anli_r3_prompt-8 task_alias: prompt-8 include: _held_in_template_yaml doc_to_text: "Choose from options: Determine if the sentence is true based on the text below:\n{{hypothesis}}\n\n{{premise}}\nOPTIONS:\n- Yes\n- It's impossible to say\n- No" @@ -151,38 +160,41 @@ task: # Arc Easy - group: arc_easy_flan group_alias: Arc Easy + aggregate_metric_list: + - metric: acc + weight_by_size: True task: - - task: arc_easy + - task: arc_easy_prompt-0 task_alias: prompt-0 include: _held_in_template_yaml doc_to_text: "{{question}}\n\nOPTIONS:\n- {{choices.text|join('\n- ')}}" doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}" - - task: arc_easy + - task: arc_easy_prompt-1 task_alias: prompt-1 include: _held_in_template_yaml doc_to_text: "Question: {{question}}\nOPTIONS:\n- {{choices.text|join('\n- ')}}\nAnswer:" doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}" - - task: arc_easy + - task: arc_easy_prompt-2 task_alias: prompt-2 include: _held_in_template_yaml doc_to_text: "Question: {{question}}\n\nWhat is the correct answer to the question from the following choices?\nOPTIONS:\n- {{choices.text|join('\n- ')}}" doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}" - - task: arc_easy + - task: arc_easy_prompt-3 task_alias: prompt-3 include: _held_in_template_yaml doc_to_text: "Q: {{question}}\nWhat is the correct answer to this question?\nOPTIONS:\n- {{choices.text|join('\n- ')}}...A:" doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}" - - task: arc_easy + - task: arc_easy_prompt-4 task_alias: prompt-4 include: _held_in_template_yaml doc_to_text: "Choose your answer?\n\n{{question}}\n\nOPTIONS:\n- {{choices.text|join('\n- ')}}" doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}" - - task: arc_easy + - task: arc_easy_prompt-5 task_alias: prompt-5 include: _held_in_template_yaml doc_to_text: "Answer the question\n\n{{question}}\nOPTIONS:\n- {{choices.text|join('\n- ')}}" doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}" - - task: arc_easy + - task: arc_easy_prompt-6 task_alias: prompt-6 include: _held_in_template_yaml doc_to_text: "{{question}}\n\nPick the answer from these options\n\nOPTIONS:\n- {{choices.text|join('\n- ')}}" @@ -190,38 +202,41 @@ task: # Arc Challenge - group: arc_challenge_flan group_alias: Arc Challenge + aggregate_metric_list: + - metric: acc + weight_by_size: True task: - - task: arc_challenge + - task: arc_challenge_prompt-0 task_alias: prompt-0 include: _held_in_template_yaml doc_to_text: "{{question}}\n\nOPTIONS:\n- {{choices.text|join('\n- ')}}" doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}" - - task: arc_challenge + - task: arc_challenge_prompt-1 task_alias: prompt-1 include: _held_in_template_yaml doc_to_text: "Question: {{question}}\nOPTIONS:\n- {{choices.text|join('\n- ')}}\nAnswer:" doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}" - - task: arc_challenge + - task: arc_challenge_prompt-2 task_alias: prompt-2 include: _held_in_template_yaml doc_to_text: "Question: {{question}}\n\nWhat is the correct answer to the question from the following choices?\nOPTIONS:\n- {{choices.text|join('\n- ')}}" doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}" - - task: arc_challenge + - task: arc_challenge_prompt-3 task_alias: prompt-3 include: _held_in_template_yaml doc_to_text: "Q: {{question}}\nWhat is the correct answer to this question?\nOPTIONS:\n- {{choices.text|join('\n- ')}}...A:" doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}" - - task: arc_challenge + - task: arc_challenge_prompt-4 task_alias: prompt-4 include: _held_in_template_yaml doc_to_text: "Choose your answer?\n\n{{question}}\n\nOPTIONS:\n- {{choices.text|join('\n- ')}}" doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}" - - task: arc_challenge + - task: arc_challenge_prompt-5 task_alias: prompt-5 include: _held_in_template_yaml doc_to_text: "Answer the question\n\n{{question}}\nOPTIONS:\n- {{choices.text|join('\n- ')}}" doc_to_target: "{{choices.text[choices.label.index(answerKey)]}}" - - task: arc_challenge + - task: arc_challenge_prompt-6 task_alias: prompt-6 include: _held_in_template_yaml doc_to_text: "{{question}}\n\nPick the answer from these options\n\nOPTIONS:\n- {{choices.text|join('\n- ')}}" @@ -229,53 +244,56 @@ task: # BoolQ - group: boolq_flan group_alias: BoolQ + aggregate_metric_list: + - metric: acc + weight_by_size: True task: - - task: boolq + - task: boolq_prompt-0 task_alias: prompt-0 include: _held_in_template_yaml doc_to_text: "{{passage}}\n\nCan we conclude that {{question}}?\n\nOPTIONS:\n- no\n- yes" doc_to_target: "{{['no', 'yes'][label]}}" - - task: boolq + - task: boolq_prompt-1 task_alias: prompt-1 include: _held_in_template_yaml doc_to_text: "{{passage}}\n\nIs it true that {{question}}?\n\nOPTIONS:\n- no\n- yes" doc_to_target: "{{['no', 'yes'][label]}}" - - task: boolq + - task: boolq_prompt-2 task_alias: prompt-2 include: _held_in_template_yaml doc_to_text: "{{passage}}\n\n{{question}}?\n\nOPTIONS:\n- no\n- yes" doc_to_target: "{{['no', 'yes'][label]}}" - - task: boolq + - task: boolq_prompt-3 task_alias: prompt-3 include: _held_in_template_yaml doc_to_text: "Text: {{passage}}\n\nQuestion: {{question}}?\n\nOPTIONS:\n- no\n- yes" doc_to_target: "{{['no', 'yes'][label]}}" - - task: boolq + - task: boolq_prompt-4 task_alias: prompt-4 include: _held_in_template_yaml doc_to_text: "{{passage}}\n\nWhat's the best answer to this question: {{question}}?\n\nOPTIONS:\n- no\n- yes" doc_to_target: "{{['no', 'yes'][label]}}" - - task: boolq + - task: boolq_prompt-5 task_alias: prompt-5 include: _held_in_template_yaml doc_to_text: "{{passage}}\nBased on the above text what's the best answer to this question: {{question}}?\n\nOPTIONS:\n- no\n- yes" doc_to_target: "{{['no', 'yes'][label]}}" - - task: boolq + - task: boolq_prompt-6 task_alias: prompt-6 include: _held_in_template_yaml doc_to_text: "{{passage}}\nAnswer this question making sure that the answer is supposed by the text: {{question}}?\n\nOPTIONS:\n- no\n- yes" doc_to_target: "{{['no', 'yes'][label]}}" - - task: boolq + - task: boolq_prompt-7 task_alias: prompt-7 include: _held_in_template_yaml doc_to_text: "{{passage}}\n\nIs the following statement correct based on the text\n\n{{question}}\n\nOPTIONS:\n- no\n- yes" doc_to_target: "{{['no', 'yes'][label]}}" - - task: boolq + - task: boolq_prompt-8 task_alias: prompt-8 include: _held_in_template_yaml doc_to_text: "{{passage}}\n\nIs this statement correct \"{{question}}\"?\n\nOPTIONS:\n- no\n- yes" doc_to_target: "{{['no', 'yes'][label]}}" - - task: boolq + - task: boolq_prompt-9 task_alias: prompt-9 include: _held_in_template_yaml doc_to_text: "Is it true that {{question}} based on the following text?\n\n{{passage}}\n\nOPTIONS:\n- no\n- yes" @@ -283,48 +301,51 @@ task: # RTE - group: rte_flan group_alias: RTE + aggregate_metric_list: + - metric: acc + weight_by_size: True task: - - task: rte + - task: rte_prompt-0 task_alias: prompt-0 include: _held_in_template_yaml doc_to_text: "{{sentence1}}\n\nQuestion with options: Based on the paragraph above can we conclude that \"{{sentence2}}\"?\n\nOPTIONS:\n- yes\n- no" doc_to_target: "{{['yes', 'no'][label]}}" - - task: rte + - task: rte_prompt-1 task_alias: prompt-1 include: _held_in_template_yaml doc_to_text: "{{sentence1}}\n\nBased on that paragraph can we conclude that the sentence below is true?\n{{sentence2}}\n\nOPTIONS:\n- yes\n- no" doc_to_target: "{{['yes', 'no'][label]}}" - - task: rte + - task: rte_prompt-1 task_alias: prompt-2 include: _held_in_template_yaml doc_to_text: "{{sentence1}}\n\nQ with options: Can we draw the following conclusion?\n{{sentence2}}\n\nOPTIONS:\n- yes\n- no" doc_to_target: "{{['yes', 'no'][label]}}" - - task: rte + - task: rte_prompt-3 task_alias: prompt-3 include: _held_in_template_yaml doc_to_text: "{{sentence1}}\nDoes this next sentence follow, given the preceding text?\n{{sentence2}}\n\nOPTIONS:\n- yes\n- no" doc_to_target: "{{['yes', 'no'][label]}}" - - task: rte + - task: rte_prompt-4 task_alias: prompt-4 include: _held_in_template_yaml doc_to_text: "{{sentence1}}\nOPTIONS:\n- yes\n- no\nQuestion: Can we infer the following?\n{{sentence2}}" doc_to_target: "{{['yes', 'no'][label]}}" - - task: rte + - task: rte_prompt-5 task_alias: prompt-5 include: _held_in_template_yaml doc_to_text: "Read the following paragraph and determine if the hypothesis is true. Select from options at the end:\n\n{{sentence1}}\n\nHypothesis: {{sentence2}}\nOPTIONS:\n- yes\n- no\nThe answer is" doc_to_target: "{{['yes', 'no'][label]}}" - - task: rte + - task: rte_prompt-6 task_alias: prompt-6 include: _held_in_template_yaml doc_to_text: "Read the text and determine if the sentence is true:\n\n{{sentence1}}\n\nSentence: {{sentence2}}\nOPTIONS:\n- yes\n- no\nA:" doc_to_target: "{{['yes', 'no'][label]}}" - - task: rte + - task: rte_prompt-7 task_alias: prompt-7 include: _held_in_template_yaml doc_to_text: "Question with options: can we draw the following hypothesis from the context? \n\nContext:\n\n{{sentence1}}\n\nHypothesis: {{sentence2}}\nOPTIONS:\n- yes\n- no\nA:" doc_to_target: "{{['yes', 'no'][label]}}" - - task: rte + - task: rte_prompt-8 task_alias: prompt-8 include: _held_in_template_yaml doc_to_text: "Determine if the sentence is true based on the text below. Choose from options.\n{{sentence2}}\n\n{{sentence1}}\nOPTIONS:\n- yes\n- no" diff --git a/lm_eval/tasks/benchmarks/minerva_math.yaml b/lm_eval/tasks/benchmarks/minerva_math.yaml index 6df3203e10fddd06bd2edcfb97984c12a32466be..f1ec09f7dcc41bc9a95c1958fc99fff62e6128db 100644 --- a/lm_eval/tasks/benchmarks/minerva_math.yaml +++ b/lm_eval/tasks/benchmarks/minerva_math.yaml @@ -7,3 +7,9 @@ task: - minerva_math_num_theory - minerva_math_prealgebra - minerva_math_precalc +aggregate_metric_list: + - metric: exact_match + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/benchmarks/multimedqa/multimedqa.yaml b/lm_eval/tasks/benchmarks/multimedqa/multimedqa.yaml index 29810bb491105b4a4e9d01391926a03c0fc8e88c..7a8409e47f3dd4e5ee4430fcf56b3616521cd6a9 100644 --- a/lm_eval/tasks/benchmarks/multimedqa/multimedqa.yaml +++ b/lm_eval/tasks/benchmarks/multimedqa/multimedqa.yaml @@ -15,3 +15,7 @@ task: task_alias: "professional_medicine (mmlu)" - task: mmlu_college_biology task_alias: "college_biology (mmlu)" +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: True diff --git a/lm_eval/tasks/bertaqa/_bertaqa_template b/lm_eval/tasks/bertaqa/_bertaqa_template index 80cb393567d13e2e5d01b8dd0816e9f644264d6b..07454d09f74bde8d701ccb6b5066f252c92331a5 100644 --- a/lm_eval/tasks/bertaqa/_bertaqa_template +++ b/lm_eval/tasks/bertaqa/_bertaqa_template @@ -1,4 +1,4 @@ -group: bertaqa +tag: bertaqa dataset_path: HiTZ/BertaQA dataset_name: null validation_split: null diff --git a/lm_eval/tasks/blimp/_blimp.yaml b/lm_eval/tasks/blimp/_blimp.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6393eeada381684c85a119b83c68d2c759787f44 --- /dev/null +++ b/lm_eval/tasks/blimp/_blimp.yaml @@ -0,0 +1,75 @@ +group: blimp +task: + - "blimp_adjunct_island" + - "blimp_anaphor_gender_agreement" + - "blimp_anaphor_number_agreement" + - "blimp_animate_subject_passive" + - "blimp_animate_subject_trans" + - "blimp_causative" + - "blimp_complex_NP_island" + - "blimp_coordinate_structure_constraint_complex_left_branch" + - "blimp_coordinate_structure_constraint_object_extraction" + - "blimp_determiner_noun_agreement_1" + - "blimp_determiner_noun_agreement_2" + - "blimp_determiner_noun_agreement_irregular_1" + - "blimp_determiner_noun_agreement_irregular_2" + - "blimp_determiner_noun_agreement_with_adj_2" + - "blimp_determiner_noun_agreement_with_adj_irregular_1" + - "blimp_determiner_noun_agreement_with_adj_irregular_2" + - "blimp_determiner_noun_agreement_with_adjective_1" + - "blimp_distractor_agreement_relational_noun" + - "blimp_distractor_agreement_relative_clause" + - "blimp_drop_argument" + - "blimp_ellipsis_n_bar_1" + - "blimp_ellipsis_n_bar_2" + - "blimp_existential_there_object_raising" + - "blimp_existential_there_quantifiers_1" + - "blimp_existential_there_quantifiers_2" + - "blimp_existential_there_subject_raising" + - "blimp_expletive_it_object_raising" + - "blimp_inchoative" + - "blimp_intransitive" + - "blimp_irregular_past_participle_adjectives" + - "blimp_irregular_past_participle_verbs" + - "blimp_irregular_plural_subject_verb_agreement_1" + - "blimp_irregular_plural_subject_verb_agreement_2" + - "blimp_left_branch_island_echo_question" + - "blimp_left_branch_island_simple_question" + - "blimp_matrix_question_npi_licensor_present" + - "blimp_npi_present_1" + - "blimp_npi_present_2" + - "blimp_only_npi_licensor_present" + - "blimp_only_npi_scope" + - "blimp_passive_1" + - "blimp_passive_2" + - "blimp_principle_A_c_command" + - "blimp_principle_A_case_1" + - "blimp_principle_A_case_2" + - "blimp_principle_A_domain_1" + - "blimp_principle_A_domain_2" + - "blimp_principle_A_domain_3" + - "blimp_principle_A_reconstruction" + - "blimp_regular_plural_subject_verb_agreement_1" + - "blimp_regular_plural_subject_verb_agreement_2" + - "blimp_sentential_negation_npi_licensor_present" + - "blimp_sentential_negation_npi_scope" + - "blimp_sentential_subject_island" + - "blimp_superlative_quantifiers_1" + - "blimp_superlative_quantifiers_2" + - "blimp_tough_vs_raising_1" + - "blimp_tough_vs_raising_2" + - "blimp_transitive" + - "blimp_wh_island" + - "blimp_wh_questions_object_gap" + - "blimp_wh_questions_subject_gap" + - "blimp_wh_questions_subject_gap_long_distance" + - "blimp_wh_vs_that_no_gap" + - "blimp_wh_vs_that_no_gap_long_distance" + - "blimp_wh_vs_that_with_gap" + - "blimp_wh_vs_that_with_gap_long_distance" +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: False +metadata: + version: 2.0 diff --git a/lm_eval/tasks/blimp/_template_yaml b/lm_eval/tasks/blimp/_template_yaml index fb1dd31360bebc10ecfeaa74bef3730acd83a07d..f81e7938af5c6e0ddbc4605e2c39acb1a9f0b374 100644 --- a/lm_eval/tasks/blimp/_template_yaml +++ b/lm_eval/tasks/blimp/_template_yaml @@ -1,4 +1,3 @@ -group: blimp dataset_path: blimp output_type: multiple_choice validation_split: train diff --git a/lm_eval/tasks/catalan_bench/README.md b/lm_eval/tasks/catalan_bench/README.md new file mode 100644 index 0000000000000000000000000000000000000000..73dec948fea65c8da32d80bc5db93e0dd0e320af --- /dev/null +++ b/lm_eval/tasks/catalan_bench/README.md @@ -0,0 +1,121 @@ +# CatalanBench + +### Paper + +CatalanBench is a benchmark for evaluating language models in Catalan tasks. This is, it evaluates the ability of a language model to understand and generate Catalan text. CatalanBench offers a combination of pre-existing, open datasets and datasets developed exclusivelly for this benchmark. All the details of CatalanBench will be published in a paper soon. + +The new evaluation datasets included in CatalanBench are: +| Task | Category | Homepage | +|:-------------:|:-----:|:-----:| +| ARC_ca | Question Answering | https://huggingface.co/datasets/projecte-aina/arc_ca | +| MGSM_ca | Math | https://huggingface.co/datasets/projecte-aina/mgsm_ca | +| OpenBookQA_ca | Question Answering | https://huggingface.co/datasets/projecte-aina/openbookqa_ca | +| Parafraseja | Paraphrasing | https://huggingface.co/datasets/projecte-aina/Parafraseja | +| PIQA_ca | Question Answering | https://huggingface.co/datasets/projecte-aina/piqa_ca | +| SIQA_ca | Question Answering | https://huggingface.co/datasets/projecte-aina/siqa_ca | +| XStoryCloze_ca | Commonsense Reasoning | https://huggingface.co/datasets/projecte-aina/xstorycloze_ca | + +The datasets included in CatalanBench that have been made public in previous pubications are: + +| Task | Category | Paper title | Homepage | +|:-------------:|:-----:|:-------------:|:-----:| +| Belebele_ca | Reading Comprehension | [The Belebele Benchmark: a Parallel Reading Comprehension Dataset in 122 Language Variants](https://arxiv.org/abs/2308.16884) | https://huggingface.co/datasets/facebook/belebele | +| caBREU | Summarization | [Building a Data Infrastructure for a Mid-Resource Language: The Case of Catalan](https://aclanthology.org/2024.lrec-main.231/) | https://huggingface.co/datasets/projecte-aina/caBreu | +| CatalanQA | Question Answering | [Building a Data Infrastructure for a Mid-Resource Language: The Case of Catalan](https://aclanthology.org/2024.lrec-main.231/) | https://huggingface.co/datasets/projecte-aina/catalanqa | +| CatCoLA | Linguistic Acceptability | CatCoLA: Catalan Corpus of Linguistic Acceptability | https://huggingface.co/datasets/nbel/CatCoLA | +| COPA-ca | Commonsense Reasoning | [Building a Data Infrastructure for a Mid-Resource Language: The Case of Catalan](https://aclanthology.org/2024.lrec-main.231/) | https://huggingface.co/datasets/projecte-aina/COPA-ca | +| CoQCat | Question Answering | [Building a Data Infrastructure for a Mid-Resource Language: The Case of Catalan](https://aclanthology.org/2024.lrec-main.231/) | https://huggingface.co/datasets/projecte-aina/CoQCat | +| FLORES_ca | Translation | [The FLORES-101 Evaluation Benchmark for Low-Resource and Multilingual Machine Translation](https://arxiv.org/abs/2106.03193) | https://huggingface.co/datasets/facebook/flores | +| PAWS-ca | Paraphrasing | [Building a Data Infrastructure for a Mid-Resource Language: The Case of Catalan](https://aclanthology.org/2024.lrec-main.231/) | https://huggingface.co/datasets/projecte-aina/PAWS-ca | +| TE-ca | Natural Language Inference | [Building a Data Infrastructure for a Mid-Resource Language: The Case of Catalan](https://aclanthology.org/2024.lrec-main.231/) | https://huggingface.co/datasets/projecte-aina/teca | +| VeritasQA_ca | Truthfulness | VeritasQA: A Truthfulness Benchmark Aimed at Multilingual Transferability | TBA | +| WNLI-ca | Natural Language Inference | [Building a Data Infrastructure for a Mid-Resource Language: The Case of Catalan](https://aclanthology.org/2024.lrec-main.231/) | https://huggingface.co/datasets/projecte-aina/wnli-ca | +| XNLI-ca | Natural Language Inference | [Building a Data Infrastructure for a Mid-Resource Language: The Case of Catalan](https://aclanthology.org/2024.lrec-main.231/) | https://huggingface.co/datasets/projecte-aina/xnli-ca | +| XQuAD-ca | Question Answering | [Building a Data Infrastructure for a Mid-Resource Language: The Case of Catalan](https://aclanthology.org/2024.lrec-main.231/) | https://huggingface.co/datasets/projecte-aina/xquad-ca | + + +### Citation +Paper for CatalanBench coming soon. + + + +### Groups and Tasks + +#### Groups + +- `catalan_bench`: All tasks included in CatalanBench. +- `flores_ca`: All FLORES translation tasks from or to Catalan. + +#### Tags +- `cabreu`: Three CaBREU tasks for each type of summary (extractive, abstractive and extreme). +- `phrases_va`: Two Phrases_va tasks for language adaptation between Catalan and Valencian. + +#### Tasks + +The following tasks evaluate tasks on CatalanBench dataset using various scoring methods. + - `arc_ca_challenge` + - `arc_ca_easy` + - `belebele_cat_Latn` + - `cabreu` + - `catalanqa` + - `catcola` + - `copa_ca` + - `coqcat` + - `flores_ca` + - `flores_ca-de` + - `flores_ca-en` + - `flores_ca-es` + - `flores_ca-eu` + - `flores_ca-fr` + - `flores_ca-gl` + - `flores_ca-it` + - `flores_ca-pt` + - `flores_de-ca` + - `flores_en-ca` + - `flores_es-ca` + - `flores_eu-ca` + - `flores_fr-ca` + - `flores_gl-ca` + - `flores_it-ca` + - `flores_pt-ca` + - `mgsm_direct_ca` + - `openbookqa_ca` + - `parafraseja` + - `paws_ca` + - `phrases_ca` + - `piqa_ca` + - `siqa_ca` + - `teca` + - `veritasqa_gen_ca` + - `veritasqa_mc1_ca` + - `veritasqa_mc2_ca` + - `wnli_ca` + - `xnli_ca` + - `xquad_ca` + - `xstorycloze_ca` + +Some of these tasks are taken from benchmarks already available in LM Evaluation Harness. These are: +- `belebele_cat_Latn`: Belebele Catalan + + +### Checklist + +* [x] Is the task an existing benchmark in the literature? + * [ ] Have you referenced the original paper that introduced the task? + * [ ] If yes, does the original paper provide a reference implementation? + * [ ] Yes, original implementation contributed by author of the benchmark + +If other tasks on this dataset are already supported: +* [ ] Is the "Main" variant of this task clearly denoted? +* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [ ] Have you noted which, if any, published evaluation setups are matched by this variant? diff --git a/lm_eval/tasks/catalan_bench/_arc_ca_common_yaml b/lm_eval/tasks/catalan_bench/_arc_ca_common_yaml new file mode 100644 index 0000000000000000000000000000000000000000..b89290ebaf1041b9dc538a426d41f5922c2f9de4 --- /dev/null +++ b/lm_eval/tasks/catalan_bench/_arc_ca_common_yaml @@ -0,0 +1,20 @@ +tag: arc_ca +dataset_path: projecte-aina/arc_ca +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +doc_to_text: "Pregunta: {{question}}\nResposta:" +doc_to_target: "{{choices.label.index(answerKey)}}" +doc_to_choice: "{{choices.text}}" +should_decontaminate: true +doc_to_decontamination_query: "Pregunta: {{question}}\nResposta:" +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/catalan_bench/_cabreu_common_yaml b/lm_eval/tasks/catalan_bench/_cabreu_common_yaml new file mode 100644 index 0000000000000000000000000000000000000000..c66e8bc486f3da09239050f98e1d4ea30ec55b2e --- /dev/null +++ b/lm_eval/tasks/catalan_bench/_cabreu_common_yaml @@ -0,0 +1,17 @@ +tag: cabreu +dataset_path: projecte-aina/caBreu +dataset_name: null +output_type: generate_until +test_split: test +training_split: train +validation_split: validation +process_docs: !function utils.process_doc_cabreu +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: !function utils.rouge1 + aggregation: !function utils.rouge1_agg + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/catalan_bench/arc_ca_challenge.yaml b/lm_eval/tasks/catalan_bench/arc_ca_challenge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9d7a9c84232ceb75c760ba35b42e2d2e8894d70b --- /dev/null +++ b/lm_eval/tasks/catalan_bench/arc_ca_challenge.yaml @@ -0,0 +1,3 @@ +task: arc_ca_challenge +dataset_name: ARC-Challenge +include: _arc_ca_common_yaml diff --git a/lm_eval/tasks/catalan_bench/arc_ca_easy.yaml b/lm_eval/tasks/catalan_bench/arc_ca_easy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..67b28fd6266818a1f9e681b741b68a1e55a48479 --- /dev/null +++ b/lm_eval/tasks/catalan_bench/arc_ca_easy.yaml @@ -0,0 +1,3 @@ +task: arc_ca_easy +dataset_name: ARC-Easy +include: _arc_ca_common_yaml diff --git a/lm_eval/tasks/catalan_bench/cabreu_abstractive.yaml b/lm_eval/tasks/catalan_bench/cabreu_abstractive.yaml new file mode 100644 index 0000000000000000000000000000000000000000..930ba28a52c5fd9861065121d73825592d47d0b5 --- /dev/null +++ b/lm_eval/tasks/catalan_bench/cabreu_abstractive.yaml @@ -0,0 +1,8 @@ +include: _cabreu_common_yaml +task: cabreu_abstractive +description: "Examina el text següent i genera'n un resum abstractiu, expressant el significat del text original d'una manera més natural i concisa.\n" +doc_to_text: >- + Text: {{content}} + + Resum: +doc_to_target: '{{summaries["abstractive"]["a1"]}}' diff --git a/lm_eval/tasks/catalan_bench/cabreu_extractive.yaml b/lm_eval/tasks/catalan_bench/cabreu_extractive.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e5f3dd4dd0d7bcb6a480f58021e3c97559d27f3b --- /dev/null +++ b/lm_eval/tasks/catalan_bench/cabreu_extractive.yaml @@ -0,0 +1,8 @@ +include: _cabreu_common_yaml +task: cabreu_extractive +description: "Examina el text següent i genera'n un resum extractiu, utilitzant les frases o oracions més rellevants del text original.\n" +doc_to_text: >- + Text: {{content}} + + Resum: +doc_to_target: '{{summaries["extractive"]["a1"]}}' diff --git a/lm_eval/tasks/catalan_bench/cabreu_extreme.yaml b/lm_eval/tasks/catalan_bench/cabreu_extreme.yaml new file mode 100644 index 0000000000000000000000000000000000000000..98efbe9cd44f0188202c4f8eb9000adc2ae44fc1 --- /dev/null +++ b/lm_eval/tasks/catalan_bench/cabreu_extreme.yaml @@ -0,0 +1,8 @@ +include: _cabreu_common_yaml +task: cabreu_extreme +description: "Examina el text següent i genera'n un resum que sigui el més concís possible i que preservi el significat del text original.\n" +doc_to_text: >- + Text: {{content}} + + Resum: +doc_to_target: '{{summaries["extreme"]["a1"]}}' diff --git a/lm_eval/tasks/catalan_bench/catalan_bench.yaml b/lm_eval/tasks/catalan_bench/catalan_bench.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1f1f09ece27271cdae75ce902bcd93e5fb2f3e0f --- /dev/null +++ b/lm_eval/tasks/catalan_bench/catalan_bench.yaml @@ -0,0 +1,25 @@ +group: catalan_bench +task: + - belebele_cat_Latn + - xnli_ca + - catcola + - copa_ca + - openbookqa_ca + - parafraseja + - paws_ca + - piqa_ca + - siqa_ca + - teca + - wnli_ca + - arc_ca_easy + - arc_ca_challenge + - xstorycloze_ca + - xquad_ca + - catalanqa + - coqcat + - flores_ca + - cabreu + - mgsm_direct_ca + - phrases_va +metadata: + version: 1.0 diff --git a/lm_eval/tasks/catalan_bench/catalanqa.yaml b/lm_eval/tasks/catalan_bench/catalanqa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..926cdfa1bec4b7cfe0314af4103e4f49fee2faf7 --- /dev/null +++ b/lm_eval/tasks/catalan_bench/catalanqa.yaml @@ -0,0 +1,25 @@ +task: catalanqa +dataset_path: projecte-aina/catalanqa +dataset_name: null +output_type: generate_until +training_split: train +validation_split: validation +test_split: test +doc_to_text: "Context: {{context}}\n\nPregunta: {{question}}\n\nResposta:" +doc_to_target: '{{answers[0]["text"]}}' +target_delimiter: ' ' +process_results: !function utils.process_results_qa +generation_kwargs: + until: + - "\n" + do_sample: false + temperature: 0.0 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + - metric: f1 + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/catalan_bench/catcola.yaml b/lm_eval/tasks/catalan_bench/catcola.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ba3648ef7272bfcc9e3be48c53b1587f93e127e9 --- /dev/null +++ b/lm_eval/tasks/catalan_bench/catcola.yaml @@ -0,0 +1,14 @@ +task: catcola +dataset_path: nbel/CatCoLA +output_type: multiple_choice +training_split: train +validation_split: validation +test_split: null +doc_to_text: "{{Sentence}}\nPregunta: Té sentit aquesta frase?\nResposta:" +doc_to_target: Label +doc_to_choice: ["no", "sí"] +metric_list: + - metric: mcc + - metric: acc +metadata: + version: 1.0 diff --git a/lm_eval/tasks/catalan_bench/copa_ca.yaml b/lm_eval/tasks/catalan_bench/copa_ca.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d376ad3aea1b1346230770956e32cefc69f5a4ac --- /dev/null +++ b/lm_eval/tasks/catalan_bench/copa_ca.yaml @@ -0,0 +1,17 @@ +task: copa_ca +dataset_path: projecte-aina/COPA-ca +dataset_name: null +output_type: multiple_choice +training_split: train +validation_split: validation +test_split: test +process_docs: !function utils.process_docs_copa_ca +doc_to_text: '{{premise[:-1].strip() + " " + {"cause": "perquè", "effect": "i per tant"}[question]}}' +doc_to_target: '{{choice1 if label == 0 else choice2}}' +doc_to_choice: '{{[choice1, choice2]}}' +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/catalan_bench/coqcat.yaml b/lm_eval/tasks/catalan_bench/coqcat.yaml new file mode 100644 index 0000000000000000000000000000000000000000..95145a7492e19fcd2e905c8d96779a718d7feeba --- /dev/null +++ b/lm_eval/tasks/catalan_bench/coqcat.yaml @@ -0,0 +1,23 @@ +task: coqcat +dataset_path: projecte-aina/CoQCat +output_type: generate_until +training_split: train +validation_split: validation +test_split: test +doc_to_text: '{{story+"\n\n"}}{% for i in range(questions|length-1) %}{{"Q: "+questions[i]+"\n\n"+"A: "+answers["input_text"][i]+"\n\n"}}{% endfor %}{{"Q: "+questions[-1]+"\n\n"+"A:"}}' +doc_to_target: '{{ answers["input_text"][questions|length - 1] }}' +process_results: !function utils.process_results_coqcat +should_decontaminate: true +doc_to_decontamination_query: "{{story}} {{question.input_text|join('\n')}}" +generation_kwargs: + until: + - "\nQ:" +metric_list: + - metric: "em" + aggregation: mean + higher_is_better: true + - metric: "f1" + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/catalan_bench/flores_ca/_flores_common_yaml b/lm_eval/tasks/catalan_bench/flores_ca/_flores_common_yaml new file mode 100644 index 0000000000000000000000000000000000000000..59a9b14aafe58dc351e6f24fcb9bedbf9497623a --- /dev/null +++ b/lm_eval/tasks/catalan_bench/flores_ca/_flores_common_yaml @@ -0,0 +1,25 @@ +dataset_path: facebook/flores +dataset_name: all +output_type: generate_until +training_split: dev +validation_split: dev +test_split: devtest +fewshot_split: dev +target_delimiter: '' +generation_kwargs: + until: + - "\n" +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: ter + aggregation: ter + higher_is_better: false + - metric: chrf + aggregation: chrf + higher_is_better: true +metadata: + version: 1.0 +dataset_kwargs: + trust_remote_code: true diff --git a/lm_eval/tasks/catalan_bench/flores_ca/create_yamls_flores_ca.py b/lm_eval/tasks/catalan_bench/flores_ca/create_yamls_flores_ca.py new file mode 100644 index 0000000000000000000000000000000000000000..6125b972665e51161ee07d9872b0f1b1543231ec --- /dev/null +++ b/lm_eval/tasks/catalan_bench/flores_ca/create_yamls_flores_ca.py @@ -0,0 +1,334 @@ +""" +Script to generate task YAMLs for the FLORES-200 dataset. +Based on `tasks/translation/utils.py`. +""" + +import argparse + +import yaml +from langcodes import Language + + +# constants +_LANGUAGES = [ + "ace_Arab", + "bam_Latn", + "dzo_Tibt", + "hin_Deva", + "khm_Khmr", + "mag_Deva", + "pap_Latn", + "sot_Latn", + "tur_Latn", + "ace_Latn", + "ban_Latn", + "ell_Grek", + "hne_Deva", + "kik_Latn", + "mai_Deva", + "pbt_Arab", + "spa_Latn", + "twi_Latn", + "acm_Arab", + "bel_Cyrl", + "eng_Latn", + "hrv_Latn", + "kin_Latn", + "mal_Mlym", + "pes_Arab", + "srd_Latn", + "tzm_Tfng", + "acq_Arab", + "bem_Latn", + "epo_Latn", + "hun_Latn", + "kir_Cyrl", + "mar_Deva", + "plt_Latn", + "srp_Cyrl", + "uig_Arab", + "aeb_Arab", + "ben_Beng", + "est_Latn", + "hye_Armn", + "kmb_Latn", + "min_Arab", + "pol_Latn", + "ssw_Latn", + "ukr_Cyrl", + "afr_Latn", + "bho_Deva", + "eus_Latn", + "ibo_Latn", + "kmr_Latn", + "min_Latn", + "por_Latn", + "sun_Latn", + "umb_Latn", + "ajp_Arab", + "bjn_Arab", + "ewe_Latn", + "ilo_Latn", + "knc_Arab", + "mkd_Cyrl", + "prs_Arab", + "swe_Latn", + "urd_Arab", + "aka_Latn", + "bjn_Latn", + "fao_Latn", + "ind_Latn", + "knc_Latn", + "mlt_Latn", + "quy_Latn", + "swh_Latn", + "uzn_Latn", + "als_Latn", + "bod_Tibt", + "fij_Latn", + "isl_Latn", + "kon_Latn", + "mni_Beng", + "ron_Latn", + "szl_Latn", + "vec_Latn", + "amh_Ethi", + "bos_Latn", + "fin_Latn", + "ita_Latn", + "kor_Hang", + "mos_Latn", + "run_Latn", + "tam_Taml", + "vie_Latn", + "apc_Arab", + "bug_Latn", + "fon_Latn", + "jav_Latn", + "lao_Laoo", + "mri_Latn", + "rus_Cyrl", + "taq_Latn", + "war_Latn", + "arb_Arab", + "bul_Cyrl", + "fra_Latn", + "jpn_Jpan", + "lij_Latn", + "mya_Mymr", + "sag_Latn", + "taq_Tfng", + "wol_Latn", + "arb_Latn", + "cat_Latn", + "fur_Latn", + "kab_Latn", + "lim_Latn", + "nld_Latn", + "san_Deva", + "tat_Cyrl", + "xho_Latn", + "ars_Arab", + "ceb_Latn", + "fuv_Latn", + "kac_Latn", + "lin_Latn", + "nno_Latn", + "sat_Olck", + "tel_Telu", + "ydd_Hebr", + "ary_Arab", + "ces_Latn", + "gaz_Latn", + "kam_Latn", + "lit_Latn", + "nob_Latn", + "scn_Latn", + "tgk_Cyrl", + "yor_Latn", + "arz_Arab", + "cjk_Latn", + "gla_Latn", + "kan_Knda", + "lmo_Latn", + "npi_Deva", + "shn_Mymr", + "tgl_Latn", + "yue_Hant", + "asm_Beng", + "ckb_Arab", + "gle_Latn", + "kas_Arab", + "ltg_Latn", + "nso_Latn", + "sin_Sinh", + "tha_Thai", + "zho_Hans", + "ast_Latn", + "crh_Latn", + "glg_Latn", + "kas_Deva", + "ltz_Latn", + "nus_Latn", + "slk_Latn", + "tir_Ethi", + "zho_Hant", + "awa_Deva", + "cym_Latn", + "grn_Latn", + "kat_Geor", + "lua_Latn", + "nya_Latn", + "slv_Latn", + "tpi_Latn", + "zsm_Latn", + "ayr_Latn", + "dan_Latn", + "guj_Gujr", + "kaz_Cyrl", + "lug_Latn", + "oci_Latn", + "smo_Latn", + "tsn_Latn", + "zul_Latn", + "azb_Arab", + "deu_Latn", + "hat_Latn", + "kbp_Latn", + "luo_Latn", + "ory_Orya", + "sna_Latn", + "tso_Latn", + "azj_Latn", + "dik_Latn", + "hau_Latn", + "kea_Latn", + "lus_Latn", + "pag_Latn", + "snd_Arab", + "tuk_Latn", + "bak_Cyrl", + "dyu_Latn", + "heb_Hebr", + "khk_Cyrl", + "lvs_Latn", + "pan_Guru", + "som_Latn", + "tum_Latn", +] +LANGUAGE_PAIRS = [ + (a, b) for idx, a in enumerate(_LANGUAGES) for b in _LANGUAGES[idx + 1 :] +] + +LANGUAGES_OF_INTEREST = [ + "cat_Latn", + "spa_Latn", + "eng_Latn", + "glg_Latn", + "eus_Latn", + "ita_Latn", + "deu_Latn", + "por_Latn", + "fra_Latn", +] +MAIN_LANG = "cat_Latn" +LANGUAGE_PAIRS = [ + (a, b) + for (a, b) in LANGUAGE_PAIRS + if a in LANGUAGES_OF_INTEREST + and b in LANGUAGES_OF_INTEREST + and "cat_Latn" in (a, b) +] + +# auxiliary functions + + +def code_to_language_name(code): + return Language.make(language=Language.get(code)["language"]).display_name() + + +def code_to_short_name(code): + return Language.get(code)["language"] + + +def jinja_var(s): + return "{{" + s + "}}" + + +def doc_to_text(src: str, tgt: str) -> str: + src_name, tgt_name = map(code_to_language_name, [src, tgt]) + + return f"""\ +{src_name} sentence: {jinja_var('sentence_' + src)} +{tgt_name} sentence:""" + + +def doc_to_target(tgt: str) -> str: + return f"{jinja_var('sentence_' + tgt)}" + + +# main function + + +def gen_lang_yamls(output_dir: str, overwrite: bool) -> None: + """ + Generate a YAML file for each translation direction. + """ + + err = [] + for src, tgt in LANGUAGE_PAIRS: + # do both translation directions for each lang pair + for src, tgt in [(src, tgt), (tgt, src)]: + lang_pair_name = f"{code_to_short_name(src)}-{code_to_short_name(tgt)}" + yaml_file_name = f"flores_{lang_pair_name}.yaml" + + try: + with open( + f"{output_dir}/{yaml_file_name}", + "w" if overwrite else "x", + encoding="utf-8", + ) as outfile: + print(f"Creating {yaml_file_name}...") + outfile.write("# File generated by `create-yamls.py`\n") + yaml.dump( + { + # "group": [f"{BENCH_NAME}_bench", f"{BENCH_NAME}_bench_flores"], + # "group": "flores_ca", + "include": "_flores_common_yaml", + "task": f"flores_{lang_pair_name}", + "doc_to_text": doc_to_text(src, tgt), + "doc_to_target": doc_to_target(tgt), + }, + outfile, + sort_keys=False, + ) + + except FileExistsError: + err.append(yaml_file_name) + + if len(err) > 0: + raise FileExistsError( + "Files were not created because they already exist:" + f" {', '.join(err)}" + "\nUse flag --overwrite to overwrite them." + ) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "--overwrite", + default=False, + action="store_true", + help="Overwrite files if they already exist", + ) + parser.add_argument( + "--output-dir", default=".", help="Directory to write yaml files to" + ) + args = parser.parse_args() + + gen_lang_yamls(output_dir=args.output_dir, overwrite=args.overwrite) + + +if __name__ == "__main__": + main() diff --git a/lm_eval/tasks/catalan_bench/flores_ca/flores_ca-de.yaml b/lm_eval/tasks/catalan_bench/flores_ca/flores_ca-de.yaml new file mode 100644 index 0000000000000000000000000000000000000000..15eb02afb6d8ca7f18cdcf56136615d51e24b199 --- /dev/null +++ b/lm_eval/tasks/catalan_bench/flores_ca/flores_ca-de.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_ca-de +doc_to_text: 'Catalan sentence: {{sentence_cat_Latn}} + + German sentence:' +doc_to_target: '{{sentence_deu_Latn}}' diff --git a/lm_eval/tasks/catalan_bench/flores_ca/flores_ca-en.yaml b/lm_eval/tasks/catalan_bench/flores_ca/flores_ca-en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9a8f5ffeb8788ca91b54065a4577aac521287f30 --- /dev/null +++ b/lm_eval/tasks/catalan_bench/flores_ca/flores_ca-en.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_ca-en +doc_to_text: 'Catalan sentence: {{sentence_cat_Latn}} + + English sentence:' +doc_to_target: '{{sentence_eng_Latn}}' diff --git a/lm_eval/tasks/catalan_bench/flores_ca/flores_ca-es.yaml b/lm_eval/tasks/catalan_bench/flores_ca/flores_ca-es.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9a6aa44240304c7b8e66bf4cf99a48d13d78e39f --- /dev/null +++ b/lm_eval/tasks/catalan_bench/flores_ca/flores_ca-es.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_ca-es +doc_to_text: 'Catalan sentence: {{sentence_cat_Latn}} + + Spanish sentence:' +doc_to_target: '{{sentence_spa_Latn}}' diff --git a/lm_eval/tasks/catalan_bench/flores_ca/flores_ca-eu.yaml b/lm_eval/tasks/catalan_bench/flores_ca/flores_ca-eu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..48ffe7bf5c7fc356177cb923006e5f57b793e7c3 --- /dev/null +++ b/lm_eval/tasks/catalan_bench/flores_ca/flores_ca-eu.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_ca-eu +doc_to_text: 'Catalan sentence: {{sentence_cat_Latn}} + + Basque sentence:' +doc_to_target: '{{sentence_eus_Latn}}' diff --git a/lm_eval/tasks/catalan_bench/flores_ca/flores_ca-fr.yaml b/lm_eval/tasks/catalan_bench/flores_ca/flores_ca-fr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..99b40c1462dec773712e4237d41a55889e6918d8 --- /dev/null +++ b/lm_eval/tasks/catalan_bench/flores_ca/flores_ca-fr.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_ca-fr +doc_to_text: 'Catalan sentence: {{sentence_cat_Latn}} + + French sentence:' +doc_to_target: '{{sentence_fra_Latn}}' diff --git a/lm_eval/tasks/catalan_bench/flores_ca/flores_ca-gl.yaml b/lm_eval/tasks/catalan_bench/flores_ca/flores_ca-gl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5da7ad5fe40ee70803b570aa019679f647e48b98 --- /dev/null +++ b/lm_eval/tasks/catalan_bench/flores_ca/flores_ca-gl.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_ca-gl +doc_to_text: 'Catalan sentence: {{sentence_cat_Latn}} + + Galician sentence:' +doc_to_target: '{{sentence_glg_Latn}}' diff --git a/lm_eval/tasks/catalan_bench/flores_ca/flores_ca-it.yaml b/lm_eval/tasks/catalan_bench/flores_ca/flores_ca-it.yaml new file mode 100644 index 0000000000000000000000000000000000000000..20f8d99f9f2c1653fd7b87aa48eaeb259daad54c --- /dev/null +++ b/lm_eval/tasks/catalan_bench/flores_ca/flores_ca-it.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_ca-it +doc_to_text: 'Catalan sentence: {{sentence_cat_Latn}} + + Italian sentence:' +doc_to_target: '{{sentence_ita_Latn}}' diff --git a/lm_eval/tasks/catalan_bench/flores_ca/flores_ca-pt.yaml b/lm_eval/tasks/catalan_bench/flores_ca/flores_ca-pt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..565f6267c5bd644506e231d023acd89864c5ca44 --- /dev/null +++ b/lm_eval/tasks/catalan_bench/flores_ca/flores_ca-pt.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_ca-pt +doc_to_text: 'Catalan sentence: {{sentence_cat_Latn}} + + Portuguese sentence:' +doc_to_target: '{{sentence_por_Latn}}' diff --git a/lm_eval/tasks/catalan_bench/flores_ca/flores_ca.yaml b/lm_eval/tasks/catalan_bench/flores_ca/flores_ca.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4726daa83e9dd7a93323ebf3d7e0125c42bff632 --- /dev/null +++ b/lm_eval/tasks/catalan_bench/flores_ca/flores_ca.yaml @@ -0,0 +1,24 @@ +group: flores_ca +task: + - flores_es-ca + - flores_ca-es + - flores_en-ca + - flores_ca-en + - flores_eu-ca + - flores_ca-eu + - flores_pt-ca + - flores_ca-pt + - flores_it-ca + - flores_ca-it + - flores_fr-ca + - flores_ca-fr + - flores_ca-gl + - flores_gl-ca + - flores_ca-de + - flores_de-ca +aggregate_metric_list: + - metric: bleu + aggregation: mean + weight_by_size: false +metadata: + version: 1.0 diff --git a/lm_eval/tasks/catalan_bench/flores_ca/flores_de-ca.yaml b/lm_eval/tasks/catalan_bench/flores_ca/flores_de-ca.yaml new file mode 100644 index 0000000000000000000000000000000000000000..af3d0eb4939258c8c4d7eb5b3961fa2b50aecae2 --- /dev/null +++ b/lm_eval/tasks/catalan_bench/flores_ca/flores_de-ca.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_de-ca +doc_to_text: 'German sentence: {{sentence_deu_Latn}} + + Catalan sentence:' +doc_to_target: '{{sentence_cat_Latn}}' diff --git a/lm_eval/tasks/catalan_bench/flores_ca/flores_en-ca.yaml b/lm_eval/tasks/catalan_bench/flores_ca/flores_en-ca.yaml new file mode 100644 index 0000000000000000000000000000000000000000..16132ff497c3f01a43b977b107a77365b9888b9a --- /dev/null +++ b/lm_eval/tasks/catalan_bench/flores_ca/flores_en-ca.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_en-ca +doc_to_text: 'English sentence: {{sentence_eng_Latn}} + + Catalan sentence:' +doc_to_target: '{{sentence_cat_Latn}}' diff --git a/lm_eval/tasks/catalan_bench/flores_ca/flores_es-ca.yaml b/lm_eval/tasks/catalan_bench/flores_ca/flores_es-ca.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e35b71521314c2c50b122b2cf4bcef048a362e6b --- /dev/null +++ b/lm_eval/tasks/catalan_bench/flores_ca/flores_es-ca.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_es-ca +doc_to_text: 'Spanish sentence: {{sentence_spa_Latn}} + + Catalan sentence:' +doc_to_target: '{{sentence_cat_Latn}}' diff --git a/lm_eval/tasks/catalan_bench/flores_ca/flores_eu-ca.yaml b/lm_eval/tasks/catalan_bench/flores_ca/flores_eu-ca.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c8be6ee93b64c33ba177f11f3494504eaf17c175 --- /dev/null +++ b/lm_eval/tasks/catalan_bench/flores_ca/flores_eu-ca.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_eu-ca +doc_to_text: 'Basque sentence: {{sentence_eus_Latn}} + + Catalan sentence:' +doc_to_target: '{{sentence_cat_Latn}}' diff --git a/lm_eval/tasks/catalan_bench/flores_ca/flores_fr-ca.yaml b/lm_eval/tasks/catalan_bench/flores_ca/flores_fr-ca.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0d2de77edfddc26058ead72df19c1ab3961de7ab --- /dev/null +++ b/lm_eval/tasks/catalan_bench/flores_ca/flores_fr-ca.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_fr-ca +doc_to_text: 'French sentence: {{sentence_fra_Latn}} + + Catalan sentence:' +doc_to_target: '{{sentence_cat_Latn}}' diff --git a/lm_eval/tasks/catalan_bench/flores_ca/flores_gl-ca.yaml b/lm_eval/tasks/catalan_bench/flores_ca/flores_gl-ca.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6ce3eaae5cbb035bea3f458176306003867b4ae4 --- /dev/null +++ b/lm_eval/tasks/catalan_bench/flores_ca/flores_gl-ca.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_gl-ca +doc_to_text: 'Galician sentence: {{sentence_glg_Latn}} + + Catalan sentence:' +doc_to_target: '{{sentence_cat_Latn}}' diff --git a/lm_eval/tasks/catalan_bench/flores_ca/flores_it-ca.yaml b/lm_eval/tasks/catalan_bench/flores_ca/flores_it-ca.yaml new file mode 100644 index 0000000000000000000000000000000000000000..db811154e5adfdd1848ec7c38841b21b96f9a699 --- /dev/null +++ b/lm_eval/tasks/catalan_bench/flores_ca/flores_it-ca.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_it-ca +doc_to_text: 'Italian sentence: {{sentence_ita_Latn}} + + Catalan sentence:' +doc_to_target: '{{sentence_cat_Latn}}' diff --git a/lm_eval/tasks/catalan_bench/flores_ca/flores_pt-ca.yaml b/lm_eval/tasks/catalan_bench/flores_ca/flores_pt-ca.yaml new file mode 100644 index 0000000000000000000000000000000000000000..196295c9e328d17a49f69256ec53d63741083d58 --- /dev/null +++ b/lm_eval/tasks/catalan_bench/flores_ca/flores_pt-ca.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_pt-ca +doc_to_text: 'Portuguese sentence: {{sentence_por_Latn}} + + Catalan sentence:' +doc_to_target: '{{sentence_cat_Latn}}' diff --git a/lm_eval/tasks/catalan_bench/mgsm_direct_ca.yaml b/lm_eval/tasks/catalan_bench/mgsm_direct_ca.yaml new file mode 100644 index 0000000000000000000000000000000000000000..066336a67f916b06915024b0c3f19c49731bcc37 --- /dev/null +++ b/lm_eval/tasks/catalan_bench/mgsm_direct_ca.yaml @@ -0,0 +1,25 @@ +task: mgsm_direct_ca +dataset_path: projecte-aina/mgsm_ca +doc_to_target: '{{answer_number|string}}' +doc_to_text: '{% if answer != None %}{{question + "\nResposta: "}}{% else %}{{"Pregunta: " + question + "\nResposta: "}}{% endif %}' +output_type: generate_until +training_split: train +test_split: test +target_delimiter: "" +generation_kwargs: + until: + - "\n\n" + - "\n" +filter_list: + - name: remove_whitespace + filter: + - function: remove_whitespace + - function: take_first +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/catalan_bench/openbookqa_ca.yaml b/lm_eval/tasks/catalan_bench/openbookqa_ca.yaml new file mode 100644 index 0000000000000000000000000000000000000000..868be756120161b68df09dc2f371ef8300a59b86 --- /dev/null +++ b/lm_eval/tasks/catalan_bench/openbookqa_ca.yaml @@ -0,0 +1,20 @@ +task: openbookqa_ca +dataset_path: projecte-aina/openbookqa_ca +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +doc_to_text: question_stem +doc_to_target: "{{choices.label.index(answerKey.lstrip())}}" +doc_to_choice: "{{choices.text}}" +should_decontaminate: true +doc_to_decontamination_query: question_stem +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/catalan_bench/parafraseja.yaml b/lm_eval/tasks/catalan_bench/parafraseja.yaml new file mode 100644 index 0000000000000000000000000000000000000000..060d488d18511fc046747946058115a964405722 --- /dev/null +++ b/lm_eval/tasks/catalan_bench/parafraseja.yaml @@ -0,0 +1,17 @@ +task: parafraseja +dataset_path: projecte-aina/Parafraseja +output_type: multiple_choice +dataset_name: null +test_split: test +training_split: train +validation_split: validation +doc_to_choice: '{{[sentence1+", veritat? No, "+sentence2, sentence1+", veritat? Sí, "+sentence2]}}' +process_docs: !function utils.process_docs_paraphrases +doc_to_text: '' +doc_to_target: label +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/catalan_bench/paws_ca.yaml b/lm_eval/tasks/catalan_bench/paws_ca.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e736f5c746cfea15cf236a4366cd4479ad81720a --- /dev/null +++ b/lm_eval/tasks/catalan_bench/paws_ca.yaml @@ -0,0 +1,18 @@ +task: paws_ca +dataset_path: projecte-aina/PAWS-ca +dataset_name: null +output_type: multiple_choice +training_split: train +validation_split: validation +test_split: test +process_docs: !function utils.process_docs_paraphrases +doc_to_text: '' +doc_to_target: label +doc_to_choice: '{{[sentence1+", veritat? No, "+sentence2, sentence1+", veritat? Sí, "+sentence2]}}' +target_delimiter: '' +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/catalan_bench/phrases_va/_phrases_va_common.yaml b/lm_eval/tasks/catalan_bench/phrases_va/_phrases_va_common.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f59a2098cad3439eab75dfd17d80b174192aeff4 --- /dev/null +++ b/lm_eval/tasks/catalan_bench/phrases_va/_phrases_va_common.yaml @@ -0,0 +1,24 @@ +tag: phrases_va +dataset_path: gplsi/CA-VA_alignment_test +output_type: generate_until +training_split: null +validation_split: null +test_split: test +fewshot_split: test +num_fewshot: 5 +target_delimiter: ' ' +generation_kwargs: + until: + - "\n" +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: ter + aggregation: ter + higher_is_better: false + - metric: chrf + aggregation: chrf + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/catalan_bench/phrases_va/phrases_ca-va.yaml b/lm_eval/tasks/catalan_bench/phrases_va/phrases_ca-va.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fc0e08d5a2b26653755413c3ccfd84f0e9c62631 --- /dev/null +++ b/lm_eval/tasks/catalan_bench/phrases_va/phrases_ca-va.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _phrases_va_common.yaml +task: phrases_ca-va +doc_to_text: 'Oració en català: {{ca}} + + Oració en valencià:' +doc_to_target: '{{va}}' diff --git a/lm_eval/tasks/catalan_bench/phrases_va/phrases_va-ca.yaml b/lm_eval/tasks/catalan_bench/phrases_va/phrases_va-ca.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5b1a76780a00353def9fa96509c669b76ca8c1ca --- /dev/null +++ b/lm_eval/tasks/catalan_bench/phrases_va/phrases_va-ca.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _phrases_va_common.yaml +task: phrases_va-ca +doc_to_text: 'Oració en valencià: {{va}} + + Oració en català:' +doc_to_target: '{{ca}}' diff --git a/lm_eval/tasks/catalan_bench/piqa_ca.yaml b/lm_eval/tasks/catalan_bench/piqa_ca.yaml new file mode 100644 index 0000000000000000000000000000000000000000..11e600a7f12d780a9082cbb6db24e4f1e5c2382d --- /dev/null +++ b/lm_eval/tasks/catalan_bench/piqa_ca.yaml @@ -0,0 +1,21 @@ +task: piqa_ca +dataset_path: projecte-aina/piqa_ca +dataset_name: null +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: null +doc_to_text: "Pregunta: {{goal}}\nResposta:" +doc_to_target: label +doc_to_choice: "{{[sol1, sol2]}}" +should_decontaminate: true +doc_to_decontamination_query: goal +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/catalan_bench/siqa_ca.yaml b/lm_eval/tasks/catalan_bench/siqa_ca.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8a39a37f5c406a1dc2103baa26cc8ef4d913d650 --- /dev/null +++ b/lm_eval/tasks/catalan_bench/siqa_ca.yaml @@ -0,0 +1,16 @@ +task: siqa_ca +dataset_path: projecte-aina/siqa_ca +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: null +doc_to_text: "Pregunta: {{context}} {{question}}\nResposta:" +target_delimiter: " " +doc_to_choice: "{{[answerA, answerB, answerC]}}" +doc_to_target: "{{ (label|int) - 1 }}" +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/catalan_bench/teca.yaml b/lm_eval/tasks/catalan_bench/teca.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8978c2c9695819c809fd9b69a7643386131573ad --- /dev/null +++ b/lm_eval/tasks/catalan_bench/teca.yaml @@ -0,0 +1,18 @@ +task: teca +dataset_path: projecte-aina/teca +dataset_name: null +training_split: train +validation_split: validation +test_split: test +output_type: multiple_choice +process_docs: !function utils.process_doc_nli +doc_to_text: "" +doc_to_target: label +target_delimiter: "" +doc_to_choice: '{{[premise + ", correcte? Sí, " + hypothesis, premise + ", correcte? A més, " + hypothesis, premise + ", correcte? No, " + hypothesis]}}' +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/catalan_bench/utils.py b/lm_eval/tasks/catalan_bench/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..ced91772ca0481e59ef5cf6746a6774256ed2753 --- /dev/null +++ b/lm_eval/tasks/catalan_bench/utils.py @@ -0,0 +1,142 @@ +import re +from itertools import product + +import evaluate +import transformers.data.metrics.squad_metrics as squad_metrics + +from lm_eval.utils import general_detokenize + + +def lowercase_first_letter(text): + return text[0].lower() + text[1:] + + +def process_doc_nli(dataset): + def process_fn(doc): + # Detokenize(remove extra whitespaces) + doc["premise"] = general_detokenize(doc["premise"]).strip() + doc["hypothesis"] = general_detokenize(doc["hypothesis"]).strip() + # Remove last punctuation mark in the premise + doc["premise"] = ( + doc["premise"][:-1] + if doc["premise"].endswith((".", ",", "!", "?")) + else doc["premise"] + ) + # Lowercase the first letter in the hypothesis + doc["hypothesis"] = lowercase_first_letter(doc["hypothesis"]) + # Ensure that the hypothesis ends with a dot + doc["hypothesis"] = ( + (doc["hypothesis"] + ".") + if not doc["hypothesis"].endswith(".") + else doc["hypothesis"] + ) + return doc + + return dataset.map(process_fn) + + +def process_results_coqcat(doc, results): + # Get all possible answers and compute the scores + turn_id = len(doc["questions"]) + answers = [doc["answers"]["input_text"][turn_id - 1]] + additional_answers_list = doc.get("additional_answers") + if additional_answers_list: + for key, additional_answers in additional_answers_list.items(): + if additional_answers["input_text"][turn_id - 1].lower() not in map( + str.lower, answers + ): + answers.append(additional_answers["input_text"][turn_id - 1]) + + gold_list = answers + pred = results[0].strip().split("\n")[0] + # import code; code.interact(local=dict(globals(), **locals())) + + f1_sum = 0.0 + em_sum = 0.0 + if len(gold_list) > 1: + for i in range(len(gold_list)): + gold_answers = gold_list[0:i] + gold_list[i + 1 :] + # predictions compared against (n) golds and take maximum + em_sum += max(squad_metrics.compute_exact(a, pred) for a in gold_answers) + f1_sum += max(squad_metrics.compute_f1(a, pred) for a in gold_answers) + else: + em_sum += max(squad_metrics.compute_exact(a, pred) for a in gold_list) + f1_sum += max(squad_metrics.compute_f1(a, pred) for a in gold_list) + # import code; code.interact(local=dict(globals(), **locals())) + return { + "em": em_sum / max(1, len(gold_list)), + "f1": f1_sum / max(1, len(gold_list)), + } + + +def process_results_qa(doc, results): + preds = results[0] + reference = doc["answers"][0]["text"] + # import code; code.interact(local=dict(globals(), **locals())) + f1_sum = squad_metrics.compute_f1(reference, preds) + exact_match = squad_metrics.compute_exact(reference, preds) + return {"f1": f1_sum, "exact_match": exact_match} + + +def process_doc_cabreu(dataset): + def process_fn(doc): + # Remove duplicate spaces + doc["content"] = re.sub(r" +", " ", doc["content"]) + for summary_type, index in product( + ["abstractive", "extractive", "extreme"], ["a1", "a2", "a3"] + ): + doc["summaries"][summary_type][index] = re.sub( + r" +", " ", doc["summaries"][summary_type][index] + ) + return doc + + return dataset.map(process_fn) + + +def process_docs_paraphrases(dataset): + empty_docs = [] + + def _process_doc(doc): + if doc["sentence1"] not in [None, ""] and doc["sentence2"] not in [None, ""]: + doc["sentence1"] = general_detokenize(doc["sentence1"]).strip() + doc["sentence2"] = general_detokenize(doc["sentence2"]).strip() + # Remove final punctuation mark in the first sentence + if doc["sentence1"].endswith((".", ",", ";")): + doc["sentence1"] = doc["sentence1"][:-1] + # Start the second sentence in lowercase (to be used after "Yes, ...") + doc["sentence2"] = lowercase_first_letter(doc["sentence2"]) + return doc + else: + empty_docs.append(doc) + return doc + + return dataset.filter( + lambda doc: doc["sentence1"] not in [None, ""] + and doc["sentence2"] not in [None, ""] + ).map(_process_doc) + + +def process_docs_copa_ca(dataset): + def _process_doc(doc): + doc["choice1"] = lowercase_first_letter(doc["choice1"]) + doc["choice2"] = lowercase_first_letter(doc["choice2"]) + return doc + + return dataset.map(_process_doc) + + +def rouge1(items): + """ + # passthrough for efficiency + """ + return items + + +def rouge1_agg(items): + """ + Higher is better + """ + refs = list(zip(*items))[0] + preds = list(zip(*items))[1] + rouge_scorer = evaluate.load("rouge") + return rouge_scorer.compute(predictions=preds, references=refs)["rouge1"] diff --git a/lm_eval/tasks/catalan_bench/wnli_ca.yaml b/lm_eval/tasks/catalan_bench/wnli_ca.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d4deec5c04e02c52cf8536d2e7b1233ec3caee17 --- /dev/null +++ b/lm_eval/tasks/catalan_bench/wnli_ca.yaml @@ -0,0 +1,14 @@ +task: wnli_ca +dataset_path: projecte-aina/wnli-ca +dataset_name: null +output_type: multiple_choice +training_split: train +validation_split: validation +test_split: null +doc_to_text: "{{sentence1}}\nPregunta: {{sentence2}} Cert o Fals?\nResposta:" +doc_to_target: label +doc_to_choice: ["Fals", "Cert"] +metric_list: + - metric: acc +metadata: + version: 1.0 diff --git a/lm_eval/tasks/catalan_bench/xnli_ca.yaml b/lm_eval/tasks/catalan_bench/xnli_ca.yaml new file mode 100644 index 0000000000000000000000000000000000000000..44f0f4430214119110d32c524b5517f660e467c1 --- /dev/null +++ b/lm_eval/tasks/catalan_bench/xnli_ca.yaml @@ -0,0 +1,19 @@ +task: xnli_ca +dataset_path: projecte-aina/xnli-ca +dataset_name: null +include: ../xnli/xnli_common_yaml +output_type: multiple_choice +doc_to_choice: '{{[premise+", correcte? Sí, "+hypothesis,premise+", correcte? A més, + "+hypothesis,premise+", correcte? No, "+hypothesis]}}' +doc_to_text: '' +target_delimiter: '' +process_docs: !function utils.process_doc_nli +training_split: null +validation_split: validation +doc_to_target: label +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/catalan_bench/xquad_ca.yaml b/lm_eval/tasks/catalan_bench/xquad_ca.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9b72c7da7487518cc4b4ed8abf46ed4f5e38c29b --- /dev/null +++ b/lm_eval/tasks/catalan_bench/xquad_ca.yaml @@ -0,0 +1,24 @@ +task: xquad_ca +dataset_path: projecte-aina/xquad-ca +dataset_name: null +output_type: generate_until +doc_to_text: "Context: {{context}}\n\nPregunta: {{question}}\n\nResposta:" +doc_to_target: '{{answers[0]["text"]}}' +validation_split: null +test_split: test +target_delimiter: ' ' +process_results: !function utils.process_results_qa +generation_kwargs: + until: + - "\n" + do_sample: false + temperature: 0.0 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + - metric: f1 + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/catalan_bench/xstorycloze_ca.yaml b/lm_eval/tasks/catalan_bench/xstorycloze_ca.yaml new file mode 100644 index 0000000000000000000000000000000000000000..61a7c2991f348eee06d0060cd67dd3ceaae3dccd --- /dev/null +++ b/lm_eval/tasks/catalan_bench/xstorycloze_ca.yaml @@ -0,0 +1,17 @@ +task: xstorycloze_ca +dataset_path: projecte-aina/xstorycloze_ca +dataset_name: ca +output_type: multiple_choice +training_split: train +validation_split: eval +doc_to_text: "{{[input_sentence_1, input_sentence_2, input_sentence_3, input_sentence_4]|join(' ')}}" +doc_to_target: "{{answer_right_ending-1}}" +doc_to_choice: "{{[sentence_quiz1, sentence_quiz2]}}" +should_decontaminate: true +doc_to_decontamination_query: "{{[input_sentence_1, input_sentence_2, input_sentence_3, input_sentence_4]|join(' ')}}" +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/ceval/_ceval-valid.yaml b/lm_eval/tasks/ceval/_ceval-valid.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5ce636dab975ed507cd1112751183d2bae3779f0 --- /dev/null +++ b/lm_eval/tasks/ceval/_ceval-valid.yaml @@ -0,0 +1,63 @@ +aggregate_metric_list: +- aggregation: mean + metric: acc + weight_by_size: true +- aggregation: mean + metric: acc_norm + weight_by_size: true +group: ceval-valid +metadata: + version: 2.0 +task: + - ceval-valid_computer_network + - ceval-valid_operating_system + - ceval-valid_computer_architecture + - ceval-valid_college_programming + - ceval-valid_college_physics + - ceval-valid_college_chemistry + - ceval-valid_advanced_mathematics + - ceval-valid_probability_and_statistics + - ceval-valid_discrete_mathematics + - ceval-valid_electrical_engineer + - ceval-valid_metrology_engineer + - ceval-valid_high_school_mathematics + - ceval-valid_high_school_physics + - ceval-valid_high_school_chemistry + - ceval-valid_high_school_biology + - ceval-valid_middle_school_mathematics + - ceval-valid_middle_school_biology + - ceval-valid_middle_school_physics + - ceval-valid_middle_school_chemistry + - ceval-valid_veterinary_medicine + - ceval-valid_college_economics + - ceval-valid_business_administration + - ceval-valid_marxism + - ceval-valid_mao_zedong_thought + - ceval-valid_education_science + - ceval-valid_teacher_qualification + - ceval-valid_high_school_politics + - ceval-valid_high_school_geography + - ceval-valid_middle_school_politics + - ceval-valid_middle_school_geography + - ceval-valid_modern_chinese_history + - ceval-valid_ideological_and_moral_cultivation + - ceval-valid_logic + - ceval-valid_law + - ceval-valid_chinese_language_and_literature + - ceval-valid_art_studies + - ceval-valid_professional_tour_guide + - ceval-valid_legal_professional + - ceval-valid_high_school_chinese + - ceval-valid_high_school_history + - ceval-valid_middle_school_history + - ceval-valid_civil_servant + - ceval-valid_sports_science + - ceval-valid_plant_protection + - ceval-valid_basic_medicine + - ceval-valid_clinical_medicine + - ceval-valid_urban_and_rural_planner + - ceval-valid_accountant + - ceval-valid_fire_engineer + - ceval-valid_environmental_impact_assessment_engineer + - ceval-valid_tax_accountant + - ceval-valid_physician diff --git a/lm_eval/tasks/ceval/_default_ceval_yaml b/lm_eval/tasks/ceval/_default_ceval_yaml index a94d87cb54f8b98bf5f418149b9b54c9b78c2695..8e5af554355e651feb91e724768e6abd9c1208c4 100644 --- a/lm_eval/tasks/ceval/_default_ceval_yaml +++ b/lm_eval/tasks/ceval/_default_ceval_yaml @@ -1,4 +1,3 @@ -group: ceval-valid dataset_path: ceval/ceval-exam validation_split: val fewshot_split: dev @@ -16,4 +15,4 @@ metric_list: aggregation: mean higher_is_better: true metadata: - version: 1.0 + version: 2.0 diff --git a/lm_eval/tasks/ceval/_generate_configs.py b/lm_eval/tasks/ceval/_generate_configs.py index b5eb1a5bc6aec38f3c98067af87438907e6b97e1..9050c75c0644f2dc8bfd800f2573b23c90988668 100644 --- a/lm_eval/tasks/ceval/_generate_configs.py +++ b/lm_eval/tasks/ceval/_generate_configs.py @@ -8,7 +8,7 @@ import os import yaml from tqdm import tqdm -from lm_eval.logger import eval_logger +from lm_eval.utils import eval_logger SUBJECTS = { @@ -117,3 +117,26 @@ if __name__ == "__main__": allow_unicode=True, default_style='"', ) + + # write group config out + + group_yaml_dict = { + "group": "ceval-valid", + "task": [f"ceval-valid_{task_name}" for task_name in SUBJECTS.keys()], + "aggregate_metric_list": [ + {"metric": "acc", "aggregation": "mean", "weight_by_size": True}, + {"metric": "acc_norm", "aggregation": "mean", "weight_by_size": True}, + ], + "metadata": {"version": 1.0}, + } + + file_save_path = "_" + args.save_prefix_path + ".yaml" + + with open(file_save_path, "w", encoding="utf-8") as group_yaml_file: + yaml.dump( + group_yaml_dict, + group_yaml_file, + width=float("inf"), + allow_unicode=True, + default_style='"', + ) diff --git a/lm_eval/tasks/cmmlu/_cmmlu.yaml b/lm_eval/tasks/cmmlu/_cmmlu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4101b18ee4e65cddb3ee71f3e238894b8a667f81 --- /dev/null +++ b/lm_eval/tasks/cmmlu/_cmmlu.yaml @@ -0,0 +1,78 @@ +group: cmmlu +task: + - cmmlu_agronomy + - cmmlu_anatomy + - cmmlu_ancient_chinese + - cmmlu_arts + - cmmlu_astronomy + - cmmlu_business_ethics + - cmmlu_chinese_civil_service_exam + - cmmlu_chinese_driving_rule + - cmmlu_chinese_food_culture + - cmmlu_chinese_foreign_policy + - cmmlu_chinese_history + - cmmlu_chinese_literature + - cmmlu_chinese_teacher_qualification + - cmmlu_clinical_knowledge + - cmmlu_college_actuarial_science + - cmmlu_college_education + - cmmlu_college_engineering_hydrology + - cmmlu_college_law + - cmmlu_college_mathematics + - cmmlu_college_medical_statistics + - cmmlu_college_medicine + - cmmlu_computer_science + - cmmlu_computer_security + - cmmlu_conceptual_physics + - cmmlu_construction_project_management + - cmmlu_economics + - cmmlu_education + - cmmlu_electrical_engineering + - cmmlu_elementary_chinese + - cmmlu_elementary_commonsense + - cmmlu_elementary_information_and_technology + - cmmlu_elementary_mathematics + - cmmlu_ethnology + - cmmlu_food_science + - cmmlu_genetics + - cmmlu_global_facts + - cmmlu_high_school_biology + - cmmlu_high_school_chemistry + - cmmlu_high_school_geography + - cmmlu_high_school_mathematics + - cmmlu_high_school_physics + - cmmlu_high_school_politics + - cmmlu_human_sexuality + - cmmlu_international_law + - cmmlu_journalism + - cmmlu_jurisprudence + - cmmlu_legal_and_moral_basis + - cmmlu_logical + - cmmlu_machine_learning + - cmmlu_management + - cmmlu_marketing + - cmmlu_marxist_theory + - cmmlu_modern_chinese + - cmmlu_nutrition + - cmmlu_philosophy + - cmmlu_professional_accounting + - cmmlu_professional_law + - cmmlu_professional_medicine + - cmmlu_professional_psychology + - cmmlu_public_relations + - cmmlu_security_study + - cmmlu_sociology + - cmmlu_sports_science + - cmmlu_traditional_chinese_medicine + - cmmlu_virology + - cmmlu_world_history + - cmmlu_world_religions +aggregate_metric_list: + - aggregation: mean + metric: acc + weight_by_size: true + - aggregation: mean + metric: acc_norm + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/cmmlu/_default_template_yaml b/lm_eval/tasks/cmmlu/_default_template_yaml index d2e0a8876c6963ff17f8da6bef14063d3b0d92a4..18bcd59c8ef7f8adf7139a70eee029517b44e257 100644 --- a/lm_eval/tasks/cmmlu/_default_template_yaml +++ b/lm_eval/tasks/cmmlu/_default_template_yaml @@ -1,4 +1,3 @@ -group: cmmlu dataset_path: haonan-li/cmmlu test_split: test fewshot_split: dev @@ -16,4 +15,4 @@ metric_list: aggregation: mean higher_is_better: true metadata: - version: 0.0 + version: 1.0 diff --git a/lm_eval/tasks/cmmlu/_generate_configs.py b/lm_eval/tasks/cmmlu/_generate_configs.py index 2c749b69a1c83e981a1fec32e5352427c4211489..f1b60e28b6b4654f1def5af0a45d59ba6711c2e9 100644 --- a/lm_eval/tasks/cmmlu/_generate_configs.py +++ b/lm_eval/tasks/cmmlu/_generate_configs.py @@ -132,3 +132,33 @@ if __name__ == "__main__": allow_unicode=True, default_style='"', ) + + # write group config out + + group_yaml_dict = { + "group": "cmmlu", + "task": [ + ( + f"cmmlu_{args.task_prefix}_{subject_eng}" + if args.task_prefix != "" + else f"cmmlu_{subject_eng}" + ) + for subject_eng in SUBJECTS.keys() + ], + "aggregate_metric_list": [ + {"metric": "acc", "aggregation": "mean", "weight_by_size": True}, + {"metric": "acc_norm", "aggregation": "mean", "weight_by_size": True}, + ], + "metadata": {"version": 0.0}, + } + + file_save_path = "_" + args.save_prefix_path + ".yaml" + + with open(file_save_path, "w", encoding="utf-8") as group_yaml_file: + yaml.dump( + group_yaml_dict, + group_yaml_file, + width=float("inf"), + allow_unicode=True, + default_style='"', + ) diff --git a/lm_eval/tasks/cmmlu/cmmlu_agronomy.yaml b/lm_eval/tasks/cmmlu/cmmlu_agronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..034ce661d6e72e35fdef2b7cddb94d00d7aec0ef --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_agronomy.yaml @@ -0,0 +1,4 @@ +"dataset_name": "agronomy" +"description": "以下是关于农学的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_agronomy" diff --git a/lm_eval/tasks/cmmlu/cmmlu_anatomy.yaml b/lm_eval/tasks/cmmlu/cmmlu_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..84161ec30ee875253d988a395f892b7982631765 --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_anatomy.yaml @@ -0,0 +1,4 @@ +"dataset_name": "anatomy" +"description": "以下是关于解剖学的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_anatomy" diff --git a/lm_eval/tasks/cmmlu/cmmlu_ancient_chinese.yaml b/lm_eval/tasks/cmmlu/cmmlu_ancient_chinese.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2099e0a12de41328eed39ebafb5ec940ee4aac5a --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_ancient_chinese.yaml @@ -0,0 +1,4 @@ +"dataset_name": "ancient_chinese" +"description": "以下是关于古汉语的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_ancient_chinese" diff --git a/lm_eval/tasks/cmmlu/cmmlu_arts.yaml b/lm_eval/tasks/cmmlu/cmmlu_arts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6007825cb9f3cd8c0af7e25c7de6d1c965f612a0 --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_arts.yaml @@ -0,0 +1,4 @@ +"dataset_name": "arts" +"description": "以下是关于艺术学的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_arts" diff --git a/lm_eval/tasks/cmmlu/cmmlu_astronomy.yaml b/lm_eval/tasks/cmmlu/cmmlu_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5ad9a8f2c886e189c380b9f01104fca11a2ef529 --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_astronomy.yaml @@ -0,0 +1,4 @@ +"dataset_name": "astronomy" +"description": "以下是关于天文学的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_astronomy" diff --git a/lm_eval/tasks/cmmlu/cmmlu_business_ethics.yaml b/lm_eval/tasks/cmmlu/cmmlu_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..10135b604b3a96ba2c894dc86a9b3af1382728a2 --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_business_ethics.yaml @@ -0,0 +1,4 @@ +"dataset_name": "business_ethics" +"description": "以下是关于商业伦理的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_business_ethics" diff --git a/lm_eval/tasks/cmmlu/cmmlu_chinese_civil_service_exam.yaml b/lm_eval/tasks/cmmlu/cmmlu_chinese_civil_service_exam.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dcf6c7e6eeb52f551442de521ed4cc4fdfd272f1 --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_chinese_civil_service_exam.yaml @@ -0,0 +1,4 @@ +"dataset_name": "chinese_civil_service_exam" +"description": "以下是关于中国公务员考试的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_chinese_civil_service_exam" diff --git a/lm_eval/tasks/cmmlu/cmmlu_chinese_driving_rule.yaml b/lm_eval/tasks/cmmlu/cmmlu_chinese_driving_rule.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2dce17c0f0c1f9a99aff32ee633eab90026e823f --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_chinese_driving_rule.yaml @@ -0,0 +1,4 @@ +"dataset_name": "chinese_driving_rule" +"description": "以下是关于中国驾驶规则的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_chinese_driving_rule" diff --git a/lm_eval/tasks/cmmlu/cmmlu_chinese_food_culture.yaml b/lm_eval/tasks/cmmlu/cmmlu_chinese_food_culture.yaml new file mode 100644 index 0000000000000000000000000000000000000000..52400c56bc4b6e39af23137c179f53102b7009a6 --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_chinese_food_culture.yaml @@ -0,0 +1,4 @@ +"dataset_name": "chinese_food_culture" +"description": "以下是关于中国饮食文化的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_chinese_food_culture" diff --git a/lm_eval/tasks/cmmlu/cmmlu_chinese_foreign_policy.yaml b/lm_eval/tasks/cmmlu/cmmlu_chinese_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bbb34a96a36cf9db8f68fe7047b2c81260afdd6f --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_chinese_foreign_policy.yaml @@ -0,0 +1,4 @@ +"dataset_name": "chinese_foreign_policy" +"description": "以下是关于中国外交政策的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_chinese_foreign_policy" diff --git a/lm_eval/tasks/cmmlu/cmmlu_chinese_history.yaml b/lm_eval/tasks/cmmlu/cmmlu_chinese_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..26fe1db5ac3039018f02bfa55e118cbba1db4ed8 --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_chinese_history.yaml @@ -0,0 +1,4 @@ +"dataset_name": "chinese_history" +"description": "以下是关于中国历史的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_chinese_history" diff --git a/lm_eval/tasks/cmmlu/cmmlu_chinese_literature.yaml b/lm_eval/tasks/cmmlu/cmmlu_chinese_literature.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4166c122c8c12a0268f67ed646e2c31698d7a40c --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_chinese_literature.yaml @@ -0,0 +1,4 @@ +"dataset_name": "chinese_literature" +"description": "以下是关于中国文学的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_chinese_literature" diff --git a/lm_eval/tasks/cmmlu/cmmlu_chinese_teacher_qualification.yaml b/lm_eval/tasks/cmmlu/cmmlu_chinese_teacher_qualification.yaml new file mode 100644 index 0000000000000000000000000000000000000000..226e98a92e435abefc82c34fad8755c80ea42448 --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_chinese_teacher_qualification.yaml @@ -0,0 +1,4 @@ +"dataset_name": "chinese_teacher_qualification" +"description": "以下是关于中国教师资格的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_chinese_teacher_qualification" diff --git a/lm_eval/tasks/cmmlu/cmmlu_clinical_knowledge.yaml b/lm_eval/tasks/cmmlu/cmmlu_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6ed36425f2c3b866e62e0ac9b38dd0aeab118916 --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_clinical_knowledge.yaml @@ -0,0 +1,4 @@ +"dataset_name": "clinical_knowledge" +"description": "以下是关于临床知识的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_clinical_knowledge" diff --git a/lm_eval/tasks/cmmlu/cmmlu_college_actuarial_science.yaml b/lm_eval/tasks/cmmlu/cmmlu_college_actuarial_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3543486b113bdc0a56ac96feadbbc1f3a8ed997b --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_college_actuarial_science.yaml @@ -0,0 +1,4 @@ +"dataset_name": "college_actuarial_science" +"description": "以下是关于大学精算学的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_college_actuarial_science" diff --git a/lm_eval/tasks/cmmlu/cmmlu_college_education.yaml b/lm_eval/tasks/cmmlu/cmmlu_college_education.yaml new file mode 100644 index 0000000000000000000000000000000000000000..952f351cb005d300becc2f5e3b7d5b8579b979a5 --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_college_education.yaml @@ -0,0 +1,4 @@ +"dataset_name": "college_education" +"description": "以下是关于大学教育学的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_college_education" diff --git a/lm_eval/tasks/cmmlu/cmmlu_college_engineering_hydrology.yaml b/lm_eval/tasks/cmmlu/cmmlu_college_engineering_hydrology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d52288a4d96e3eee909a7f33c845ba2fa9590aba --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_college_engineering_hydrology.yaml @@ -0,0 +1,4 @@ +"dataset_name": "college_engineering_hydrology" +"description": "以下是关于大学工程水文学的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_college_engineering_hydrology" diff --git a/lm_eval/tasks/cmmlu/cmmlu_college_law.yaml b/lm_eval/tasks/cmmlu/cmmlu_college_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..717784ac3d636cfde4560d11c85f84a963d7e154 --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_college_law.yaml @@ -0,0 +1,4 @@ +"dataset_name": "college_law" +"description": "以下是关于大学法律的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_college_law" diff --git a/lm_eval/tasks/cmmlu/cmmlu_college_mathematics.yaml b/lm_eval/tasks/cmmlu/cmmlu_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7876a584e7e3c936d30c7e4ad81381ec7e535493 --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_college_mathematics.yaml @@ -0,0 +1,4 @@ +"dataset_name": "college_mathematics" +"description": "以下是关于大学数学的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_college_mathematics" diff --git a/lm_eval/tasks/cmmlu/cmmlu_college_medical_statistics.yaml b/lm_eval/tasks/cmmlu/cmmlu_college_medical_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f4839bdcac6dc3ba2ee7b874a1700db1d760b49c --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_college_medical_statistics.yaml @@ -0,0 +1,4 @@ +"dataset_name": "college_medical_statistics" +"description": "以下是关于大学医学统计的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_college_medical_statistics" diff --git a/lm_eval/tasks/cmmlu/cmmlu_college_medicine.yaml b/lm_eval/tasks/cmmlu/cmmlu_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dff0d7860fee5dbd289ffaf2d61215fc1c79707b --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_college_medicine.yaml @@ -0,0 +1,4 @@ +"dataset_name": "college_medicine" +"description": "以下是关于大学医学的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_college_medicine" diff --git a/lm_eval/tasks/cmmlu/cmmlu_computer_science.yaml b/lm_eval/tasks/cmmlu/cmmlu_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..86c874e539d21d55540e7e5adce32a624d4a706c --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_computer_science.yaml @@ -0,0 +1,4 @@ +"dataset_name": "computer_science" +"description": "以下是关于计算机科学的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_computer_science" diff --git a/lm_eval/tasks/cmmlu/cmmlu_computer_security.yaml b/lm_eval/tasks/cmmlu/cmmlu_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9471546184de5dde5edeb8031a64e588c7594f8f --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_computer_security.yaml @@ -0,0 +1,4 @@ +"dataset_name": "computer_security" +"description": "以下是关于计算机安全的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_computer_security" diff --git a/lm_eval/tasks/cmmlu/cmmlu_conceptual_physics.yaml b/lm_eval/tasks/cmmlu/cmmlu_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ba952486faefcb59e113864489d3fe95c5c2703d --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_conceptual_physics.yaml @@ -0,0 +1,4 @@ +"dataset_name": "conceptual_physics" +"description": "以下是关于概念物理学的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_conceptual_physics" diff --git a/lm_eval/tasks/cmmlu/cmmlu_construction_project_management.yaml b/lm_eval/tasks/cmmlu/cmmlu_construction_project_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..86265b0804a30e1d2352ff79bcaaa8de3c15316f --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_construction_project_management.yaml @@ -0,0 +1,4 @@ +"dataset_name": "construction_project_management" +"description": "以下是关于建设工程管理的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_construction_project_management" diff --git a/lm_eval/tasks/cmmlu/cmmlu_economics.yaml b/lm_eval/tasks/cmmlu/cmmlu_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4602efb430d49e3a876b7243c4cfffe506094b34 --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_economics.yaml @@ -0,0 +1,4 @@ +"dataset_name": "economics" +"description": "以下是关于经济学的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_economics" diff --git a/lm_eval/tasks/cmmlu/cmmlu_education.yaml b/lm_eval/tasks/cmmlu/cmmlu_education.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f1dc8a8a4fbc9664da04e2288cf782a9cc1e1877 --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_education.yaml @@ -0,0 +1,4 @@ +"dataset_name": "education" +"description": "以下是关于教育学的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_education" diff --git a/lm_eval/tasks/cmmlu/cmmlu_electrical_engineering.yaml b/lm_eval/tasks/cmmlu/cmmlu_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2bb920b53ab8856d717fea8e07e87077ec3b3f71 --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_electrical_engineering.yaml @@ -0,0 +1,4 @@ +"dataset_name": "electrical_engineering" +"description": "以下是关于电气工程的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_electrical_engineering" diff --git a/lm_eval/tasks/cmmlu/cmmlu_elementary_chinese.yaml b/lm_eval/tasks/cmmlu/cmmlu_elementary_chinese.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6f67be3fc40f5c038b455edcc6076675a4451261 --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_elementary_chinese.yaml @@ -0,0 +1,4 @@ +"dataset_name": "elementary_chinese" +"description": "以下是关于小学语文的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_elementary_chinese" diff --git a/lm_eval/tasks/cmmlu/cmmlu_elementary_commonsense.yaml b/lm_eval/tasks/cmmlu/cmmlu_elementary_commonsense.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3017edd999a0ee04de4a5dd8c7dc4b1b6218f5e3 --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_elementary_commonsense.yaml @@ -0,0 +1,4 @@ +"dataset_name": "elementary_commonsense" +"description": "以下是关于小学常识的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_elementary_commonsense" diff --git a/lm_eval/tasks/cmmlu/cmmlu_elementary_information_and_technology.yaml b/lm_eval/tasks/cmmlu/cmmlu_elementary_information_and_technology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..98c7d3c8f2d85f3c52a3314253d2d2151f7116ae --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_elementary_information_and_technology.yaml @@ -0,0 +1,4 @@ +"dataset_name": "elementary_information_and_technology" +"description": "以下是关于小学信息技术的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_elementary_information_and_technology" diff --git a/lm_eval/tasks/cmmlu/cmmlu_elementary_mathematics.yaml b/lm_eval/tasks/cmmlu/cmmlu_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f702312ca07c2b882d17c88d30dbe87a837ce5c6 --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_elementary_mathematics.yaml @@ -0,0 +1,4 @@ +"dataset_name": "elementary_mathematics" +"description": "以下是关于初等数学的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_elementary_mathematics" diff --git a/lm_eval/tasks/cmmlu/cmmlu_ethnology.yaml b/lm_eval/tasks/cmmlu/cmmlu_ethnology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..88a653a9ee5e5978113626a35acbe50bd2ea5437 --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_ethnology.yaml @@ -0,0 +1,4 @@ +"dataset_name": "ethnology" +"description": "以下是关于民族学的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_ethnology" diff --git a/lm_eval/tasks/cmmlu/cmmlu_food_science.yaml b/lm_eval/tasks/cmmlu/cmmlu_food_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9be450ca2ea2190c6dd3b0639ad9fbd12d968443 --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_food_science.yaml @@ -0,0 +1,4 @@ +"dataset_name": "food_science" +"description": "以下是关于食品科学的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_food_science" diff --git a/lm_eval/tasks/cmmlu/cmmlu_genetics.yaml b/lm_eval/tasks/cmmlu/cmmlu_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..be57628b6f0d3dd2bc6719e08f9aaddb45ac7fa2 --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_genetics.yaml @@ -0,0 +1,4 @@ +"dataset_name": "genetics" +"description": "以下是关于遗传学的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_genetics" diff --git a/lm_eval/tasks/cmmlu/cmmlu_global_facts.yaml b/lm_eval/tasks/cmmlu/cmmlu_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6982be9468bebc3d99a53baf120a11eae52704bb --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_global_facts.yaml @@ -0,0 +1,4 @@ +"dataset_name": "global_facts" +"description": "以下是关于全球事实的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_global_facts" diff --git a/lm_eval/tasks/cmmlu/cmmlu_high_school_biology.yaml b/lm_eval/tasks/cmmlu/cmmlu_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a56e797420f80bba1814e2bffc3aaa7f009d74f4 --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_high_school_biology.yaml @@ -0,0 +1,4 @@ +"dataset_name": "high_school_biology" +"description": "以下是关于高中生物的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_high_school_biology" diff --git a/lm_eval/tasks/cmmlu/cmmlu_high_school_chemistry.yaml b/lm_eval/tasks/cmmlu/cmmlu_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..34e99ea0f47b7017206bd6e9078ca7a5c2b25f0e --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_high_school_chemistry.yaml @@ -0,0 +1,4 @@ +"dataset_name": "high_school_chemistry" +"description": "以下是关于高中化学的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_high_school_chemistry" diff --git a/lm_eval/tasks/cmmlu/cmmlu_high_school_geography.yaml b/lm_eval/tasks/cmmlu/cmmlu_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c73ebe9171df9e9f0fbdf2fecddb251e56884702 --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_high_school_geography.yaml @@ -0,0 +1,4 @@ +"dataset_name": "high_school_geography" +"description": "以下是关于高中地理的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_high_school_geography" diff --git a/lm_eval/tasks/cmmlu/cmmlu_high_school_mathematics.yaml b/lm_eval/tasks/cmmlu/cmmlu_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3598501c1763d5f1c19444e1b18bb242149fdd34 --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_high_school_mathematics.yaml @@ -0,0 +1,4 @@ +"dataset_name": "high_school_mathematics" +"description": "以下是关于高中数学的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_high_school_mathematics" diff --git a/lm_eval/tasks/cmmlu/cmmlu_high_school_physics.yaml b/lm_eval/tasks/cmmlu/cmmlu_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..676fca166369b2f2b18a077ab2ec61b74a777d5c --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_high_school_physics.yaml @@ -0,0 +1,4 @@ +"dataset_name": "high_school_physics" +"description": "以下是关于高中物理学的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_high_school_physics" diff --git a/lm_eval/tasks/cmmlu/cmmlu_high_school_politics.yaml b/lm_eval/tasks/cmmlu/cmmlu_high_school_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5f689dff61a4ea55628b04f9bed5202e48c6eb70 --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_high_school_politics.yaml @@ -0,0 +1,4 @@ +"dataset_name": "high_school_politics" +"description": "以下是关于高中政治的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_high_school_politics" diff --git a/lm_eval/tasks/cmmlu/cmmlu_human_sexuality.yaml b/lm_eval/tasks/cmmlu/cmmlu_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..39ff32e728dd228dd675f708dc6e2680c96f0900 --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_human_sexuality.yaml @@ -0,0 +1,4 @@ +"dataset_name": "human_sexuality" +"description": "以下是关于人类性行为的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_human_sexuality" diff --git a/lm_eval/tasks/cmmlu/cmmlu_international_law.yaml b/lm_eval/tasks/cmmlu/cmmlu_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..32112d3c8b6ee26ee786439053c2d1f1da5b04c2 --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_international_law.yaml @@ -0,0 +1,4 @@ +"dataset_name": "international_law" +"description": "以下是关于国际法学的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_international_law" diff --git a/lm_eval/tasks/cmmlu/cmmlu_journalism.yaml b/lm_eval/tasks/cmmlu/cmmlu_journalism.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9f283816f59760033f375d8aba352fdd860b3338 --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_journalism.yaml @@ -0,0 +1,4 @@ +"dataset_name": "journalism" +"description": "以下是关于新闻学的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_journalism" diff --git a/lm_eval/tasks/cmmlu/cmmlu_jurisprudence.yaml b/lm_eval/tasks/cmmlu/cmmlu_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ab40da40bafeb56459ae462b795be8c8584fb02a --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_jurisprudence.yaml @@ -0,0 +1,4 @@ +"dataset_name": "jurisprudence" +"description": "以下是关于法理学的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_jurisprudence" diff --git a/lm_eval/tasks/cmmlu/cmmlu_legal_and_moral_basis.yaml b/lm_eval/tasks/cmmlu/cmmlu_legal_and_moral_basis.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a5e3ee13b6e9670f33068bc731acebf7489737ec --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_legal_and_moral_basis.yaml @@ -0,0 +1,4 @@ +"dataset_name": "legal_and_moral_basis" +"description": "以下是关于法律与道德基础的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_legal_and_moral_basis" diff --git a/lm_eval/tasks/cmmlu/cmmlu_logical.yaml b/lm_eval/tasks/cmmlu/cmmlu_logical.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4c4ac2e12abb2fa29dd2e194f5f1b9417f61142b --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_logical.yaml @@ -0,0 +1,4 @@ +"dataset_name": "logical" +"description": "以下是关于逻辑学的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_logical" diff --git a/lm_eval/tasks/cmmlu/cmmlu_machine_learning.yaml b/lm_eval/tasks/cmmlu/cmmlu_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..062cd1cd73add5caf387f6b4717c5ed837e2c7f7 --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_machine_learning.yaml @@ -0,0 +1,4 @@ +"dataset_name": "machine_learning" +"description": "以下是关于机器学习的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_machine_learning" diff --git a/lm_eval/tasks/cmmlu/cmmlu_management.yaml b/lm_eval/tasks/cmmlu/cmmlu_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aa5681babeb650cc451c15e3496ca4d0ed3a1e0f --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_management.yaml @@ -0,0 +1,4 @@ +"dataset_name": "management" +"description": "以下是关于管理学的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_management" diff --git a/lm_eval/tasks/cmmlu/cmmlu_marketing.yaml b/lm_eval/tasks/cmmlu/cmmlu_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a729641f9059060ec9abadeac611cf3e74528165 --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_marketing.yaml @@ -0,0 +1,4 @@ +"dataset_name": "marketing" +"description": "以下是关于市场营销的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_marketing" diff --git a/lm_eval/tasks/cmmlu/cmmlu_marxist_theory.yaml b/lm_eval/tasks/cmmlu/cmmlu_marxist_theory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f99fa17514a10e8bf587b50ae9dd997b80c00225 --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_marxist_theory.yaml @@ -0,0 +1,4 @@ +"dataset_name": "marxist_theory" +"description": "以下是关于马克思主义理论的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_marxist_theory" diff --git a/lm_eval/tasks/cmmlu/cmmlu_modern_chinese.yaml b/lm_eval/tasks/cmmlu/cmmlu_modern_chinese.yaml new file mode 100644 index 0000000000000000000000000000000000000000..13b2ccc4f939876616ceeda42d211e96347ce060 --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_modern_chinese.yaml @@ -0,0 +1,4 @@ +"dataset_name": "modern_chinese" +"description": "以下是关于现代汉语的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_modern_chinese" diff --git a/lm_eval/tasks/cmmlu/cmmlu_nutrition.yaml b/lm_eval/tasks/cmmlu/cmmlu_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..23d52c45e07134b2ff4f7c1a8e55ba19acfbcfd9 --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_nutrition.yaml @@ -0,0 +1,4 @@ +"dataset_name": "nutrition" +"description": "以下是关于营养学的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_nutrition" diff --git a/lm_eval/tasks/cmmlu/cmmlu_philosophy.yaml b/lm_eval/tasks/cmmlu/cmmlu_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..17340fa490f0350e6e532b2c67f8c81fa63bfb3a --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_philosophy.yaml @@ -0,0 +1,4 @@ +"dataset_name": "philosophy" +"description": "以下是关于哲学的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_philosophy" diff --git a/lm_eval/tasks/cmmlu/cmmlu_professional_accounting.yaml b/lm_eval/tasks/cmmlu/cmmlu_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bed3485d787d921fb25bbbfbad7671118acfc42b --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_professional_accounting.yaml @@ -0,0 +1,4 @@ +"dataset_name": "professional_accounting" +"description": "以下是关于专业会计的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_professional_accounting" diff --git a/lm_eval/tasks/cmmlu/cmmlu_professional_law.yaml b/lm_eval/tasks/cmmlu/cmmlu_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dec4c6cf1d7b095fab8fb293b9cf7600765f24db --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_professional_law.yaml @@ -0,0 +1,4 @@ +"dataset_name": "professional_law" +"description": "以下是关于专业法学的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_professional_law" diff --git a/lm_eval/tasks/cmmlu/cmmlu_professional_medicine.yaml b/lm_eval/tasks/cmmlu/cmmlu_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..92fed45e74f9b69b2c7b595a4bb682318fe0b81c --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_professional_medicine.yaml @@ -0,0 +1,4 @@ +"dataset_name": "professional_medicine" +"description": "以下是关于专业医学的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_professional_medicine" diff --git a/lm_eval/tasks/cmmlu/cmmlu_professional_psychology.yaml b/lm_eval/tasks/cmmlu/cmmlu_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..83f0255591a17711d6ac99cf164a29ffe2a69866 --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_professional_psychology.yaml @@ -0,0 +1,4 @@ +"dataset_name": "professional_psychology" +"description": "以下是关于专业心理学的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_professional_psychology" diff --git a/lm_eval/tasks/cmmlu/cmmlu_public_relations.yaml b/lm_eval/tasks/cmmlu/cmmlu_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a1c3711ef7734df27852065cf894f9c9cff9d776 --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_public_relations.yaml @@ -0,0 +1,4 @@ +"dataset_name": "public_relations" +"description": "以下是关于公共关系的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_public_relations" diff --git a/lm_eval/tasks/cmmlu/cmmlu_security_study.yaml b/lm_eval/tasks/cmmlu/cmmlu_security_study.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c9660f041fcb24ed83089c624f7ef6c6962c5d8b --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_security_study.yaml @@ -0,0 +1,4 @@ +"dataset_name": "security_study" +"description": "以下是关于安全研究的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_security_study" diff --git a/lm_eval/tasks/cmmlu/cmmlu_sociology.yaml b/lm_eval/tasks/cmmlu/cmmlu_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..54f4367d010fb4ae5b0fe6b8a120f139b39cb0ab --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_sociology.yaml @@ -0,0 +1,4 @@ +"dataset_name": "sociology" +"description": "以下是关于社会学的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_sociology" diff --git a/lm_eval/tasks/cmmlu/cmmlu_sports_science.yaml b/lm_eval/tasks/cmmlu/cmmlu_sports_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..35e5bb9cc4c40abcf271955f068788f85e44794a --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_sports_science.yaml @@ -0,0 +1,4 @@ +"dataset_name": "sports_science" +"description": "以下是关于体育学的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_sports_science" diff --git a/lm_eval/tasks/cmmlu/cmmlu_traditional_chinese_medicine.yaml b/lm_eval/tasks/cmmlu/cmmlu_traditional_chinese_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ed4627deefd6a9a1737cc700604b940b31635cf8 --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_traditional_chinese_medicine.yaml @@ -0,0 +1,4 @@ +"dataset_name": "traditional_chinese_medicine" +"description": "以下是关于中医中药的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_traditional_chinese_medicine" diff --git a/lm_eval/tasks/cmmlu/cmmlu_virology.yaml b/lm_eval/tasks/cmmlu/cmmlu_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1560b84f682493ef53a9c26ae1d36ac520ff46c7 --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_virology.yaml @@ -0,0 +1,4 @@ +"dataset_name": "virology" +"description": "以下是关于病毒学的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_virology" diff --git a/lm_eval/tasks/cmmlu/cmmlu_world_history.yaml b/lm_eval/tasks/cmmlu/cmmlu_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..993ce0ab6e390a81286df213e5d3ddd9fe3908bd --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_world_history.yaml @@ -0,0 +1,4 @@ +"dataset_name": "world_history" +"description": "以下是关于世界历史的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_world_history" diff --git a/lm_eval/tasks/cmmlu/cmmlu_world_religions.yaml b/lm_eval/tasks/cmmlu/cmmlu_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..13229fc95d7d85408cc8d3649208404e9a8476d2 --- /dev/null +++ b/lm_eval/tasks/cmmlu/cmmlu_world_religions.yaml @@ -0,0 +1,4 @@ +"dataset_name": "world_religions" +"description": "以下是关于世界宗教的单项选择题，请直接给出正确答案的选项。\n\n" +"include": "_default_template_yaml" +"task": "cmmlu_world_religions" diff --git a/lm_eval/tasks/copal_id/standard.yaml b/lm_eval/tasks/copal_id/standard.yaml index d38220fc0cf925f68973c42d03ccd4ff8c7fcba8..a09d0c8531c4f8668c2fcd90beb274b30c616237 100644 --- a/lm_eval/tasks/copal_id/standard.yaml +++ b/lm_eval/tasks/copal_id/standard.yaml @@ -1,4 +1,4 @@ -group: copal_id +tag: copal_id task: copal_id_standard task_alias: standard dataset_path: haryoaw/COPAL diff --git a/lm_eval/tasks/crows_pairs/crows_pairs_english.yaml b/lm_eval/tasks/crows_pairs/crows_pairs_english.yaml index d95c83d01c681dede5e77797ab954af0797da104..3daf5f75fc3fe8090336a43e3617fe79ceb22bdd 100644 --- a/lm_eval/tasks/crows_pairs/crows_pairs_english.yaml +++ b/lm_eval/tasks/crows_pairs/crows_pairs_english.yaml @@ -1,7 +1,5 @@ -group: +tag: - crows_pairs - - social_bias - - loglikelihood task: crows_pairs_english dataset_path: BigScienceBiasEval/crows_pairs_multilingual dataset_name: english diff --git a/lm_eval/tasks/csatqa/_csatqa.yaml b/lm_eval/tasks/csatqa/_csatqa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8b9e30131097d9123f3dbbae3ab250fbcf4f6ad3 --- /dev/null +++ b/lm_eval/tasks/csatqa/_csatqa.yaml @@ -0,0 +1,17 @@ +group: csatqa +task: + - csatqa_gr + - csatqa_li + - csatqa_rch + - csatqa_rcs + - csatqa_rcss + - csatqa_wr +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm_eval/tasks/csatqa/_default_csatqa_yaml b/lm_eval/tasks/csatqa/_default_csatqa_yaml index a4a5db84b54de024afd6dfe5be71d12c73d2067a..fdd6d148c5d7f9d522d7d0f9ad433f9b9ad4179e 100644 --- a/lm_eval/tasks/csatqa/_default_csatqa_yaml +++ b/lm_eval/tasks/csatqa/_default_csatqa_yaml @@ -1,5 +1,4 @@ -group: csatqa -dataset_path: EleutherAI/csatqa +dataset_path: HAERAE-HUB/csatqa test_split: test output_type: multiple_choice process_docs: !function utils.process_docs diff --git a/lm_eval/tasks/drop/utils.py b/lm_eval/tasks/drop/utils.py index 54093bb4d28e954035e76d8764a014ca99b99d8d..fc4e7d4b4db1775cdae632d4a5334adeeeffb318 100644 --- a/lm_eval/tasks/drop/utils.py +++ b/lm_eval/tasks/drop/utils.py @@ -2,7 +2,6 @@ import re import string import numpy as np -from scipy.optimize import linear_sum_assignment _ARTICLES = re.compile(r"\b(a|an|the)\b", re.UNICODE) @@ -117,6 +116,8 @@ def _align_bags(predicted, gold): Takes gold and predicted answer sets and first finds the optimal 1-1 alignment between them and gets maximum metric values over all the answers. """ + from scipy.optimize import linear_sum_assignment + scores = np.zeros([len(gold), len(predicted)]) for gold_index, gold_item in enumerate(gold): for pred_index, pred_item in enumerate(predicted): diff --git a/lm_eval/tasks/eq_bench/README.md b/lm_eval/tasks/eq_bench/README.md index 472890bdc832705e55f7a28209a74ea2af6b9865..df11108eece0f281687d8098e6a56476a75decb0 100644 --- a/lm_eval/tasks/eq_bench/README.md +++ b/lm_eval/tasks/eq_bench/README.md @@ -16,8 +16,8 @@ Homepage: https://eqbench.com/ NOTE: There are some key differences between the lm-evaluation-harness version and the implementation described in the EQ-Bench paper (These have been OK'd by the author): - The lm-eval version uses the EQ-Bench v2 test set (171 questions) and score calculation. It does not incorporate the revision part of the prompt, as per v2.1 (https://github.com/EQ-bench/EQ-Bench) -- No retries in lm-eval version (EQ-Bench pipeline retries with successively higher temps if it encounters unparseable answers) -- In the original implementation, unparseable answers are excluded from the final score, and 83% of answers have to be parseable or a fail is returned. The lm-eval version instead assigns 0 to unparsable answers and has no fail criteria. So for lower performing models, there may be differences with the EQ-Bench leaderboard. +- No retries in lm-eval version (EQ-Bench pipeline retries with successively higher temps if it encounters unparsable answers) +- In the original implementation, unparsable answers are excluded from the final score, and 83% of answers have to be parseable or a fail is returned. The lm-eval version instead assigns 0 to unparsable answers and has no fail criteria. So for lower performing models, there may be differences with the EQ-Bench leaderboard. ### Citation diff --git a/lm_eval/tasks/eus_exams/README.md b/lm_eval/tasks/eus_exams/README.md index f44c5f04ed6eaded120ac7d9da177004a02483a7..3a14f7dfcd16f8557be1a187599abae08e863940 100644 --- a/lm_eval/tasks/eus_exams/README.md +++ b/lm_eval/tasks/eus_exams/README.md @@ -26,7 +26,7 @@ Homepage: https://github.com/hitz-zentroa/latxa ### Groups and Tasks -#### Groups +#### Tags * `eus_exams_eu`: The Basque version of the exams. * `eus_exams_es`: The Spanish version of the exams. diff --git a/lm_eval/tasks/eus_exams/eus_exams_es b/lm_eval/tasks/eus_exams/eus_exams_es index 25886606943ec02e216d89df64cdc777a405cdaf..5f2325f22c892189d7965d3eacb0cf87a2d3d922 100644 --- a/lm_eval/tasks/eus_exams/eus_exams_es +++ b/lm_eval/tasks/eus_exams/eus_exams_es @@ -1,4 +1,4 @@ include: eus_exams -group: +tag: - eus_exams_es doc_to_text: "Pregunta: {{question}}\nA: {{candidates[0]}}\nB: {{candidates[1]}}\nC: {{candidates[2]}}\nD: {{candidates[3]}}\nRespuesta:" diff --git a/lm_eval/tasks/eus_exams/eus_exams_eu b/lm_eval/tasks/eus_exams/eus_exams_eu index 95b82388df8930a2fe7385fff72f22c4cc516ab4..dfa3df575dd0ee47bb9eea9f8449350b79221b02 100644 --- a/lm_eval/tasks/eus_exams/eus_exams_eu +++ b/lm_eval/tasks/eus_exams/eus_exams_eu @@ -1,4 +1,4 @@ include: eus_exams -group: +tag: - eus_exams_eu doc_to_text: "Galdera: {{question}}\nA: {{candidates[0]}}\nB: {{candidates[1]}}\nC: {{candidates[2]}}\nD: {{candidates[3]}}\nErantzuna:" diff --git a/lm_eval/tasks/fda/task.py b/lm_eval/tasks/fda/task.py index a61feaed4cad8529caaef20df8a2f90ca72efb8d..1b01c2bc8ff7c985406bf78d590bff275f2310d9 100644 --- a/lm_eval/tasks/fda/task.py +++ b/lm_eval/tasks/fda/task.py @@ -12,7 +12,7 @@ class FDA(ConfigurableTask): DATASET_PATH = "hazyresearch/based-fda" DATASET_NAME = "default" - def __init__(self): + def __init__(self, **kwargs): super().__init__(config={"metadata": {"version": self.VERSION}}) def has_training_docs(self): diff --git a/lm_eval/tasks/fld/fld_default.yaml b/lm_eval/tasks/fld/fld_default.yaml index 233a3564a3ffb6d207dd397103a27bd37c43dc22..460a9ec6dbf52be9819891ed27ae59b62162c75d 100644 --- a/lm_eval/tasks/fld/fld_default.yaml +++ b/lm_eval/tasks/fld/fld_default.yaml @@ -1,5 +1,3 @@ -group: - - fld task: fld_default dataset_path: hitachi-nlp/FLD.v2 dataset_name: default diff --git a/lm_eval/tasks/french_bench/README.md b/lm_eval/tasks/french_bench/README.md index 8b2a22edd49172897a42afcfe3b64974204618ca..693f60c3a3efda911fbfa0be9a7c64ce55fa22b1 100644 --- a/lm_eval/tasks/french_bench/README.md +++ b/lm_eval/tasks/french_bench/README.md @@ -20,9 +20,9 @@ This benchmark is constructed both from openly available datasets, as well as ne } ``` -### Groups and Tasks +### Groups, Tags, and Tasks -#### Groups +#### Tags - `french_bench`: All tasks (non-perplexity based) - `french_bench_gen`: All official generative tasks diff --git a/lm_eval/tasks/french_bench/french_bench_arc_challenge.yaml b/lm_eval/tasks/french_bench/french_bench_arc_challenge.yaml index a77d5163ead0915243b091e68ce1e06801a41d03..7884b0dc9cd9639d4a67cff0086b44978e84b14a 100644 --- a/lm_eval/tasks/french_bench/french_bench_arc_challenge.yaml +++ b/lm_eval/tasks/french_bench/french_bench_arc_challenge.yaml @@ -1,4 +1,4 @@ -group: +tag: - french_bench - french_bench_mc task: french_bench_arc_challenge diff --git a/lm_eval/tasks/french_bench/french_bench_boolqa.yaml b/lm_eval/tasks/french_bench/french_bench_boolqa.yaml index ed67265d4351ce5f2e08271f87edbf674950baa1..bdd60e5d0e78fc519ae08c97e4cdcb6986b04d8c 100644 --- a/lm_eval/tasks/french_bench/french_bench_boolqa.yaml +++ b/lm_eval/tasks/french_bench/french_bench_boolqa.yaml @@ -1,5 +1,5 @@ include: "_default_template_yaml" -group: +tag: - french_bench - french_bench_extra description: "D'après l'information dans le contexte donné, quelle est la réponse à la question ?" diff --git a/lm_eval/tasks/french_bench/french_bench_fquadv2.yaml b/lm_eval/tasks/french_bench/french_bench_fquadv2.yaml index 5ffdb194a40ee267c7e7a9940351022d4692a19e..2e49ec43c185951786dc8a2ca60f80c71ed6ac25 100644 --- a/lm_eval/tasks/french_bench/french_bench_fquadv2.yaml +++ b/lm_eval/tasks/french_bench/french_bench_fquadv2.yaml @@ -1,5 +1,5 @@ include: "_default_template_yaml" -group: +tag: - french_bench - french_bench_extra description: "D'après l'information dans le contexte donné, donne la réponse à la question en citant quelques mots du contexte. Si il est impossible de répondre avec les informations du contexte, répond 'Impossible'." diff --git a/lm_eval/tasks/french_bench/french_bench_fquadv2_bool.yaml b/lm_eval/tasks/french_bench/french_bench_fquadv2_bool.yaml index 7fe89c31fb4b2a89b49c5d031283d838c4fb6658..4e07d2ec0d28505bceec367920e0a56617c6af45 100644 --- a/lm_eval/tasks/french_bench/french_bench_fquadv2_bool.yaml +++ b/lm_eval/tasks/french_bench/french_bench_fquadv2_bool.yaml @@ -1,5 +1,5 @@ include: "_default_template_yaml" -group: +tag: - french_bench - french_bench_extra description: "D'après l'information présente dans le contexte, est il possible de répondre à la question ?" diff --git a/lm_eval/tasks/french_bench/french_bench_fquadv2_genq.yaml b/lm_eval/tasks/french_bench/french_bench_fquadv2_genq.yaml index bd1c4684db873405961833907101a872e8d6f8fa..380518520326753402e265f453f99ed6b1e1043d 100644 --- a/lm_eval/tasks/french_bench/french_bench_fquadv2_genq.yaml +++ b/lm_eval/tasks/french_bench/french_bench_fquadv2_genq.yaml @@ -1,5 +1,5 @@ include: "_default_template_yaml" -group: +tag: - french_bench - french_bench_gen description: "D'après l'information dans le contexte donné, quelle question a été posée pour obtenir la réponse donnée ?" diff --git a/lm_eval/tasks/french_bench/french_bench_fquadv2_hasAns.yaml b/lm_eval/tasks/french_bench/french_bench_fquadv2_hasAns.yaml index 37c02af358e1d26f2823440ea23f8ae7770d87a2..6eedbabb5f5e3c1a381c43d20a26ec7ce3a1d103 100644 --- a/lm_eval/tasks/french_bench/french_bench_fquadv2_hasAns.yaml +++ b/lm_eval/tasks/french_bench/french_bench_fquadv2_hasAns.yaml @@ -1,5 +1,5 @@ include: "_default_template_yaml" -group: +tag: - french_bench - french_bench_gen description: "D'après l'information dans le contexte donné, donne la réponse à la question en citant quelques mots du contexte. Si il est impossible de répondre avec les informations du contexte, répond 'Impossible'." diff --git a/lm_eval/tasks/french_bench/french_bench_grammar.yaml b/lm_eval/tasks/french_bench/french_bench_grammar.yaml index 45052ccc04134a7a194a24b19fb3d621345e1f9d..6828c3a9fd7a9c73c7c7ff368952ca22805b4b7a 100644 --- a/lm_eval/tasks/french_bench/french_bench_grammar.yaml +++ b/lm_eval/tasks/french_bench/french_bench_grammar.yaml @@ -1,5 +1,5 @@ include: "_default_template_yaml" -group: +tag: - french_bench - french_bench_mc description: "Répond au mieux en complétant la question avec une des réponses proposées." diff --git a/lm_eval/tasks/french_bench/french_bench_hellaswag.yaml b/lm_eval/tasks/french_bench/french_bench_hellaswag.yaml index 9fa8ea26d52fb23838a6609dddbdb0baa9c4f05a..293a76c27a9bfbf7beec22805d06f44310cf143c 100644 --- a/lm_eval/tasks/french_bench/french_bench_hellaswag.yaml +++ b/lm_eval/tasks/french_bench/french_bench_hellaswag.yaml @@ -1,4 +1,4 @@ -group: +tag: - french_bench - french_bench_mc task: french_bench_hellaswag diff --git a/lm_eval/tasks/french_bench/french_bench_multifquad.yaml b/lm_eval/tasks/french_bench/french_bench_multifquad.yaml index 632ffe369f208ee8d87d9cee10719c604f44a7f8..71301bf29e55a0954527e7963cdcbbba92710337 100644 --- a/lm_eval/tasks/french_bench/french_bench_multifquad.yaml +++ b/lm_eval/tasks/french_bench/french_bench_multifquad.yaml @@ -1,5 +1,5 @@ include: "_default_template_yaml" -group: +tag: - french_bench - french_bench_gen description: "D'après l'information dans le contexte donné, donne la réponse à la question en citant quelques extraits du contexte." diff --git a/lm_eval/tasks/french_bench/french_bench_opus_perplexity.yaml b/lm_eval/tasks/french_bench/french_bench_opus_perplexity.yaml index c5a72501d72c87e3672c11510bb987a46c458f84..dbe714a9c03a9f099d7438f4458d086caa7cd4a1 100644 --- a/lm_eval/tasks/french_bench/french_bench_opus_perplexity.yaml +++ b/lm_eval/tasks/french_bench/french_bench_opus_perplexity.yaml @@ -1,4 +1,4 @@ -group: +tag: - french_bench_perplexity task: french_bench_opus_perplexity dataset_path: manu/opus100-en-fr diff --git a/lm_eval/tasks/french_bench/french_bench_orangesum_abstract.yaml b/lm_eval/tasks/french_bench/french_bench_orangesum_abstract.yaml index 3ca8888afeab5660b52764fd47c7de55c72a46dd..1d4a3b4acb9e5304eb191e345edc191309245729 100644 --- a/lm_eval/tasks/french_bench/french_bench_orangesum_abstract.yaml +++ b/lm_eval/tasks/french_bench/french_bench_orangesum_abstract.yaml @@ -1,5 +1,5 @@ include: "_default_template_yaml" -group: +tag: - french_bench - french_bench_gen description: "Résume l'article en une phrase." diff --git a/lm_eval/tasks/french_bench/french_bench_orangesum_title.yaml b/lm_eval/tasks/french_bench/french_bench_orangesum_title.yaml index c459a18fa4cb9acc00c3ef4f874f15f0f763fcaf..90b547e038ef3ff0086641b136461411240aaf5a 100644 --- a/lm_eval/tasks/french_bench/french_bench_orangesum_title.yaml +++ b/lm_eval/tasks/french_bench/french_bench_orangesum_title.yaml @@ -1,5 +1,5 @@ include: "_default_template_yaml" -group: +tag: - french_bench - french_bench_extra description: "Trouve le titre de l'article." diff --git a/lm_eval/tasks/french_bench/french_bench_reading_comp.yaml b/lm_eval/tasks/french_bench/french_bench_reading_comp.yaml index 8d8c8abd8c1772193ca3d64a33edeb36b4fefd66..f3abeadad711b746f998f6b1f7253ca1285e5e24 100644 --- a/lm_eval/tasks/french_bench/french_bench_reading_comp.yaml +++ b/lm_eval/tasks/french_bench/french_bench_reading_comp.yaml @@ -1,5 +1,5 @@ include: "_default_template_yaml" -group: +tag: - french_bench - french_bench_extra # description: "Répond au mieux en complétant la question avec une des réponses proposées." diff --git a/lm_eval/tasks/french_bench/french_bench_topic_based_nli.yaml b/lm_eval/tasks/french_bench/french_bench_topic_based_nli.yaml index c88957a1b9a035785095654c75f930d7574d05b4..28dd6af64ecd6344146a790a57cfc43ccf2eb3a2 100644 --- a/lm_eval/tasks/french_bench/french_bench_topic_based_nli.yaml +++ b/lm_eval/tasks/french_bench/french_bench_topic_based_nli.yaml @@ -1,5 +1,5 @@ include: "_default_template_yaml" -group: +tag: - french_bench - french_bench_extra description: "A propos du thème spécifié, l'avis client est il positif, négatif, ou neutre ?" diff --git a/lm_eval/tasks/french_bench/french_bench_trivia.yaml b/lm_eval/tasks/french_bench/french_bench_trivia.yaml index 525fb781bcc716a9cd9822793485f5b0fc2fba6f..3b69b0f12b8be7078e5a0e35c812709ef496fa5f 100644 --- a/lm_eval/tasks/french_bench/french_bench_trivia.yaml +++ b/lm_eval/tasks/french_bench/french_bench_trivia.yaml @@ -1,5 +1,5 @@ include: "_default_template_yaml" -group: +tag: - french_bench - french_bench_gen task: french_bench_trivia diff --git a/lm_eval/tasks/french_bench/french_bench_vocab.yaml b/lm_eval/tasks/french_bench/french_bench_vocab.yaml index 1995c91c2515416598721bede2325ce0843d37cc..a5d5cadcd4ab2e879909fc51c94699ff45e4f6b3 100644 --- a/lm_eval/tasks/french_bench/french_bench_vocab.yaml +++ b/lm_eval/tasks/french_bench/french_bench_vocab.yaml @@ -1,5 +1,5 @@ include: "_default_template_yaml" -group: +tag: - french_bench - french_bench_mc # description: "Répond au mieux en complétant la question avec une des réponses proposées." diff --git a/lm_eval/tasks/french_bench/french_bench_wikitext_fr.yaml b/lm_eval/tasks/french_bench/french_bench_wikitext_fr.yaml index c4b04fe0e6214428360a1b1955426f8675909efc..d7ae23ff9246e09e31a0c2e77f19935b7d03432d 100644 --- a/lm_eval/tasks/french_bench/french_bench_wikitext_fr.yaml +++ b/lm_eval/tasks/french_bench/french_bench_wikitext_fr.yaml @@ -1,4 +1,4 @@ -group: +tag: - french_bench_perplexity task: french_bench_wikitext_fr dataset_path: asi/wikitext_fr diff --git a/lm_eval/tasks/french_bench/french_bench_xnli.yaml b/lm_eval/tasks/french_bench/french_bench_xnli.yaml index 7a527e4cf9d8ce6a1ff8f14a1cf03a471d06b14c..272b5652e81fdc3d42a6ca6cc39220d715251323 100644 --- a/lm_eval/tasks/french_bench/french_bench_xnli.yaml +++ b/lm_eval/tasks/french_bench/french_bench_xnli.yaml @@ -1,5 +1,5 @@ include: "_default_template_yaml" -group: +tag: - french_bench - french_bench_extra description: "La prémisse et l'hypothèse sont elles en accord, neutres en elles, ou en contradiction ?" diff --git a/lm_eval/tasks/galician_bench/README.md b/lm_eval/tasks/galician_bench/README.md new file mode 100644 index 0000000000000000000000000000000000000000..24dd7a74352f3c6efa17cfc916e3b91a4828bc56 --- /dev/null +++ b/lm_eval/tasks/galician_bench/README.md @@ -0,0 +1,80 @@ +# GalicianBench + +### Paper + +GalicianBench is a benchmark for evaluating language models in Galician tasks. This is, it evaluates the ability of a language model to understand and generate Galician text. GalicianBench offers a combination of pre-existing, open datasets and datasets developed exclusivelly for this benchmark. All the details of GalicianBench will be published in a paper soon. + +The new evaluation datasets included in GalicianBench are: +| Task | Category | Homepage | +|:-------------:|:-----:|:-----:| +| Belebele_gl | Reading Comprehension | https://huggingface.co/datasets/proxectonos/belebele_gl | +| GalCoLA | Linguistic Acceptability | https://huggingface.co/datasets/proxectonos/galcola | +| MGSM_ca | Math | https://huggingface.co/datasets/proxectonos/mgsm_gl | +| Parafrases_gl | Paraphrasing | https://huggingface.co/datasets/proxectonos/parafrases_gl | +| PAWS-gl | Paraphrasing | https://huggingface.co/datasets/proxectonos/PAWS-gl | +| OpenBookQA_gl | Question Answering | https://huggingface.co/datasets/proxectonos/openbookqa_gl | +| Summarization_gl | Summarization | https://huggingface.co/datasets/proxectonos/summarization_gl | +| TruthfulQA_gl | Truthfulness | https://huggingface.co/datasets/proxectonos/truthfulqa_gl | +| xnli_gl | NLI | https://huggingface.co/datasets/proxectonos/xnli_gl | +| xstorycloze_gl | Commonsense Reasoning | https://huggingface.co/datasets/proxectonos/xstorycloze_gl | + +The datasets included in GalicianBench that have been made public in previous pubications are: + +| Task | Category | Paper title | Homepage | +|:-------------:|:-----:|:-------------:|:-----:| +| FLORES_gl | Translation | [The FLORES-101 Evaluation Benchmark for Low-Resource and Multilingual Machine Translation](https://arxiv.org/abs/2106.03193) | https://huggingface.co/datasets/facebook/flores | + + +### Citation +Paper for GalicianBench coming soon. + +### Groups and Tasks + +#### Groups + +- `galician_bench`: All tasks included in GalicianBench. +- `flores_gl`: All FLORES translation tasks from or to Galician. + + +#### Tasks + +The following tasks evaluate tasks on GalicianBench dataset using various scoring methods. + - `belebele_glg_Latn` + - `flores_gl` + - `flores_gl-ca` + - `flores_gl-de` + - `flores_gl-en` + - `flores_gl-es` + - `flores_gl-eu` + - `flores_gl-fr` + - `flores_gl-it` + - `flores_gl-pt` + - `flores_ca-gl` + - `flores_de-gl` + - `flores_en-gl` + - `flores_es-gl` + - `flores_eu-gl` + - `flores_fr-gl` + - `flores_it-gl` + - `flores_pt-gl` + - `galcola` + - `summarization_gl` + - `parafrases_gl` + - `paws_gl` + - `openbookqa_gl` + - `mgsm_direct_gl` + - `truthfulqa_gl` + - `xnli_gl` + - `xstorycloze_gl` + +### Checklist + +* [x] Is the task an existing benchmark in the literature? + * [ ] Have you referenced the original paper that introduced the task? + * [ ] If yes, does the original paper provide a reference implementation? + * [ ] Yes, original implementation contributed by author of the benchmark + +If other tasks on this dataset are already supported: +* [ ] Is the "Main" variant of this task clearly denoted? +* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [ ] Have you noted which, if any, published evaluation setups are matched by this variant? diff --git a/lm_eval/tasks/galician_bench/belebele_glg_Latn.yaml b/lm_eval/tasks/galician_bench/belebele_glg_Latn.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ae81a53f9c818b549ec5fd8bef3de3b8641d6013 --- /dev/null +++ b/lm_eval/tasks/galician_bench/belebele_glg_Latn.yaml @@ -0,0 +1,7 @@ +task: belebele_glg_Latn +include: ../belebele/_default_template_yaml +dataset_path: proxectonos/belebele_gl +fewshot_split: train +test_split: train +metadata: + version: 1.0 diff --git a/lm_eval/tasks/galician_bench/flores_gl/_flores_common_yaml b/lm_eval/tasks/galician_bench/flores_gl/_flores_common_yaml new file mode 100644 index 0000000000000000000000000000000000000000..bff45b85a1e74a77cc40b05284d031fec8780929 --- /dev/null +++ b/lm_eval/tasks/galician_bench/flores_gl/_flores_common_yaml @@ -0,0 +1,27 @@ +dataset_path: facebook/flores +dataset_name: all +output_type: generate_until +#! The test split of flores is not publicly available! (See paper section 6.1) +#! We are using `dev` and `devtest` splits, but they're mapped to train/validation/test in `data/flores/flores.py`. +training_split: dev +validation_split: dev +test_split: devtest +fewshot_split: dev +target_delimiter: '' +generation_kwargs: + until: + - "\n" +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: ter + aggregation: ter + higher_is_better: false + - metric: chrf + aggregation: chrf + higher_is_better: true +metadata: + version: 1.0 +dataset_kwargs: + trust_remote_code: true diff --git a/lm_eval/tasks/galician_bench/flores_gl/create_yamls_flores_gl.py b/lm_eval/tasks/galician_bench/flores_gl/create_yamls_flores_gl.py new file mode 100644 index 0000000000000000000000000000000000000000..c98b9b215543108e88ca2db07a15654a9c806229 --- /dev/null +++ b/lm_eval/tasks/galician_bench/flores_gl/create_yamls_flores_gl.py @@ -0,0 +1,333 @@ +# ruff: noqa: E731, E741 +""" +Script to generate task YAMLs for the FLORES-200 dataset. +Based on `tasks/translation/utils.py`. +""" + +import argparse +import itertools + +import yaml +from langcodes import Language + + +# utils +flatten = lambda l: list(itertools.chain(*l)) + +# constants +_LANGUAGES = [ + "ace_Arab", + "bam_Latn", + "dzo_Tibt", + "hin_Deva", + "khm_Khmr", + "mag_Deva", + "pap_Latn", + "sot_Latn", + "tur_Latn", + "ace_Latn", + "ban_Latn", + "ell_Grek", + "hne_Deva", + "kik_Latn", + "mai_Deva", + "pbt_Arab", + "spa_Latn", + "twi_Latn", + "acm_Arab", + "bel_Cyrl", + "eng_Latn", + "hrv_Latn", + "kin_Latn", + "mal_Mlym", + "pes_Arab", + "srd_Latn", + "tzm_Tfng", + "acq_Arab", + "bem_Latn", + "epo_Latn", + "hun_Latn", + "kir_Cyrl", + "mar_Deva", + "plt_Latn", + "srp_Cyrl", + "uig_Arab", + "aeb_Arab", + "ben_Beng", + "est_Latn", + "hye_Armn", + "kmb_Latn", + "min_Arab", + "pol_Latn", + "ssw_Latn", + "ukr_Cyrl", + "afr_Latn", + "bho_Deva", + "eus_Latn", + "ibo_Latn", + "kmr_Latn", + "min_Latn", + "por_Latn", + "sun_Latn", + "umb_Latn", + "ajp_Arab", + "bjn_Arab", + "ewe_Latn", + "ilo_Latn", + "knc_Arab", + "mkd_Cyrl", + "prs_Arab", + "swe_Latn", + "urd_Arab", + "aka_Latn", + "bjn_Latn", + "fao_Latn", + "ind_Latn", + "knc_Latn", + "mlt_Latn", + "quy_Latn", + "swh_Latn", + "uzn_Latn", + "als_Latn", + "bod_Tibt", + "fij_Latn", + "isl_Latn", + "kon_Latn", + "mni_Beng", + "ron_Latn", + "szl_Latn", + "vec_Latn", + "amh_Ethi", + "bos_Latn", + "fin_Latn", + "ita_Latn", + "kor_Hang", + "mos_Latn", + "run_Latn", + "tam_Taml", + "vie_Latn", + "apc_Arab", + "bug_Latn", + "fon_Latn", + "jav_Latn", + "lao_Laoo", + "mri_Latn", + "rus_Cyrl", + "taq_Latn", + "war_Latn", + "arb_Arab", + "bul_Cyrl", + "fra_Latn", + "jpn_Jpan", + "lij_Latn", + "mya_Mymr", + "sag_Latn", + "taq_Tfng", + "wol_Latn", + "arb_Latn", + "cat_Latn", + "fur_Latn", + "kab_Latn", + "lim_Latn", + "nld_Latn", + "san_Deva", + "tat_Cyrl", + "xho_Latn", + "ars_Arab", + "ceb_Latn", + "fuv_Latn", + "kac_Latn", + "lin_Latn", + "nno_Latn", + "sat_Olck", + "tel_Telu", + "ydd_Hebr", + "ary_Arab", + "ces_Latn", + "gaz_Latn", + "kam_Latn", + "lit_Latn", + "nob_Latn", + "scn_Latn", + "tgk_Cyrl", + "yor_Latn", + "arz_Arab", + "cjk_Latn", + "gla_Latn", + "kan_Knda", + "lmo_Latn", + "npi_Deva", + "shn_Mymr", + "tgl_Latn", + "yue_Hant", + "asm_Beng", + "ckb_Arab", + "gle_Latn", + "kas_Arab", + "ltg_Latn", + "nso_Latn", + "sin_Sinh", + "tha_Thai", + "zho_Hans", + "ast_Latn", + "crh_Latn", + "glg_Latn", + "kas_Deva", + "ltz_Latn", + "nus_Latn", + "slk_Latn", + "tir_Ethi", + "zho_Hant", + "awa_Deva", + "cym_Latn", + "grn_Latn", + "kat_Geor", + "lua_Latn", + "nya_Latn", + "slv_Latn", + "tpi_Latn", + "zsm_Latn", + "ayr_Latn", + "dan_Latn", + "guj_Gujr", + "kaz_Cyrl", + "lug_Latn", + "oci_Latn", + "smo_Latn", + "tsn_Latn", + "zul_Latn", + "azb_Arab", + "deu_Latn", + "hat_Latn", + "kbp_Latn", + "luo_Latn", + "ory_Orya", + "sna_Latn", + "tso_Latn", + "azj_Latn", + "dik_Latn", + "hau_Latn", + "kea_Latn", + "lus_Latn", + "pag_Latn", + "snd_Arab", + "tuk_Latn", + "bak_Cyrl", + "dyu_Latn", + "heb_Hebr", + "khk_Cyrl", + "lvs_Latn", + "pan_Guru", + "som_Latn", + "tum_Latn", +] +LANGUAGE_PAIRS = [ + (a, b) for idx, a in enumerate(_LANGUAGES) for b in _LANGUAGES[idx + 1 :] +] + +LANGUAGES_OF_INTEREST = [ + "cat_Latn", + "spa_Latn", + "eng_Latn", + "glg_Latn", + "eus_Latn", + "ita_Latn", + "deu_Latn", + "por_Latn", + "fra_Latn", +] +MAIN_LANG = "glg_Latn" +LANGUAGE_PAIRS = [ + (a, b) + for (a, b) in LANGUAGE_PAIRS + if a in LANGUAGES_OF_INTEREST and b in LANGUAGES_OF_INTEREST and MAIN_LANG in (a, b) +] + +# auxiliary functions + +code_to_language_name = lambda code: Language.make( + language=Language.get(code)["language"] +).display_name() +code_to_short_name = lambda code: Language.get(code)["language"] +jinja_var = ( + lambda s: "{{" + s + "}}" +) # wrapper to avoid having to escape { } in format strings + + +def doc_to_text(src: str, tgt: str) -> str: + src_name, tgt_name = map(code_to_language_name, [src, tgt]) + + return f"""\ +{src_name} sentence: {jinja_var('sentence_' + src)} +{tgt_name} sentence:""" + + +def doc_to_target(tgt: str) -> str: + return f"{jinja_var('sentence_' + tgt)}" + + +# main function + + +def gen_lang_yamls(output_dir: str, overwrite: bool) -> None: + """ + Generate a YAML file for each translation direction. + """ + + err = [] + for src, tgt in LANGUAGE_PAIRS: + # do both translation directions for each lang pair + for src, tgt in [(src, tgt), (tgt, src)]: + lang_pair_name = f"{code_to_short_name(src)}-{code_to_short_name(tgt)}" + yaml_file_name = f"flores_{lang_pair_name}.yaml" + + try: + with open( + f"{output_dir}/{yaml_file_name}", + "w" if overwrite else "x", + encoding="utf-8", + ) as outfile: + print(f"Creating {yaml_file_name}...") + outfile.write("# File generated by `create-yamls.py`\n") + yaml.dump( + { + # "group": [f"{BENCH_NAME}_bench", f"{BENCH_NAME}_bench_flores"], + # "group": "flores_gl", + "include": "_flores_common_yaml", + "task": f"flores_{lang_pair_name}", + "doc_to_text": doc_to_text(src, tgt), + "doc_to_target": doc_to_target(tgt), + }, + outfile, + sort_keys=False, + ) + + except FileExistsError: + err.append(yaml_file_name) + + if len(err) > 0: + raise FileExistsError( + "Files were not created because they already exist:" + f" {', '.join(err)}" + "\nUse flag --overwrite to overwrite them." + ) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "--overwrite", + default=False, + action="store_true", + help="Overwrite files if they already exist", + ) + parser.add_argument( + "--output-dir", default=".", help="Directory to write yaml files to" + ) + args = parser.parse_args() + + gen_lang_yamls(output_dir=args.output_dir, overwrite=args.overwrite) + + +if __name__ == "__main__": + main() diff --git a/lm_eval/tasks/galician_bench/flores_gl/flores_ca-gl.yaml b/lm_eval/tasks/galician_bench/flores_gl/flores_ca-gl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5da7ad5fe40ee70803b570aa019679f647e48b98 --- /dev/null +++ b/lm_eval/tasks/galician_bench/flores_gl/flores_ca-gl.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_ca-gl +doc_to_text: 'Catalan sentence: {{sentence_cat_Latn}} + + Galician sentence:' +doc_to_target: '{{sentence_glg_Latn}}' diff --git a/lm_eval/tasks/galician_bench/flores_gl/flores_de-gl.yaml b/lm_eval/tasks/galician_bench/flores_gl/flores_de-gl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2f2eabbc55c7c68babee5d5b3fa0e0225575f4d9 --- /dev/null +++ b/lm_eval/tasks/galician_bench/flores_gl/flores_de-gl.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_de-gl +doc_to_text: 'German sentence: {{sentence_deu_Latn}} + + Galician sentence:' +doc_to_target: '{{sentence_glg_Latn}}' diff --git a/lm_eval/tasks/galician_bench/flores_gl/flores_en-gl.yaml b/lm_eval/tasks/galician_bench/flores_gl/flores_en-gl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9dc8fc24f1ff8ae3596dcd189242af3cfde89475 --- /dev/null +++ b/lm_eval/tasks/galician_bench/flores_gl/flores_en-gl.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_en-gl +doc_to_text: 'English sentence: {{sentence_eng_Latn}} + + Galician sentence:' +doc_to_target: '{{sentence_glg_Latn}}' diff --git a/lm_eval/tasks/galician_bench/flores_gl/flores_es-gl.yaml b/lm_eval/tasks/galician_bench/flores_gl/flores_es-gl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dd3c6a9eac7c1318d23209947690384ec41a7f29 --- /dev/null +++ b/lm_eval/tasks/galician_bench/flores_gl/flores_es-gl.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_es-gl +doc_to_text: 'Spanish sentence: {{sentence_spa_Latn}} + + Galician sentence:' +doc_to_target: '{{sentence_glg_Latn}}' diff --git a/lm_eval/tasks/galician_bench/flores_gl/flores_eu-gl.yaml b/lm_eval/tasks/galician_bench/flores_gl/flores_eu-gl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..db762cf75c90985a9b87459587508fd429070e98 --- /dev/null +++ b/lm_eval/tasks/galician_bench/flores_gl/flores_eu-gl.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_eu-gl +doc_to_text: 'Basque sentence: {{sentence_eus_Latn}} + + Galician sentence:' +doc_to_target: '{{sentence_glg_Latn}}' diff --git a/lm_eval/tasks/galician_bench/flores_gl/flores_fr-gl.yaml b/lm_eval/tasks/galician_bench/flores_gl/flores_fr-gl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0d884dbad70943f7c57bdd1dad0870310b4c994f --- /dev/null +++ b/lm_eval/tasks/galician_bench/flores_gl/flores_fr-gl.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_fr-gl +doc_to_text: 'French sentence: {{sentence_fra_Latn}} + + Galician sentence:' +doc_to_target: '{{sentence_glg_Latn}}' diff --git a/lm_eval/tasks/galician_bench/flores_gl/flores_gl-ca.yaml b/lm_eval/tasks/galician_bench/flores_gl/flores_gl-ca.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6ce3eaae5cbb035bea3f458176306003867b4ae4 --- /dev/null +++ b/lm_eval/tasks/galician_bench/flores_gl/flores_gl-ca.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_gl-ca +doc_to_text: 'Galician sentence: {{sentence_glg_Latn}} + + Catalan sentence:' +doc_to_target: '{{sentence_cat_Latn}}' diff --git a/lm_eval/tasks/galician_bench/flores_gl/flores_gl-de.yaml b/lm_eval/tasks/galician_bench/flores_gl/flores_gl-de.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e499780fbb464546abee256b058d732f2f57da25 --- /dev/null +++ b/lm_eval/tasks/galician_bench/flores_gl/flores_gl-de.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_gl-de +doc_to_text: 'Galician sentence: {{sentence_glg_Latn}} + + German sentence:' +doc_to_target: '{{sentence_deu_Latn}}' diff --git a/lm_eval/tasks/galician_bench/flores_gl/flores_gl-en.yaml b/lm_eval/tasks/galician_bench/flores_gl/flores_gl-en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5d2b7afbd94c01defe341953d3881784bcb55ec3 --- /dev/null +++ b/lm_eval/tasks/galician_bench/flores_gl/flores_gl-en.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_gl-en +doc_to_text: 'Galician sentence: {{sentence_glg_Latn}} + + English sentence:' +doc_to_target: '{{sentence_eng_Latn}}' diff --git a/lm_eval/tasks/galician_bench/flores_gl/flores_gl-es.yaml b/lm_eval/tasks/galician_bench/flores_gl/flores_gl-es.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c00acf3f47fafdd1c9176855ad4b8fe76c9634e9 --- /dev/null +++ b/lm_eval/tasks/galician_bench/flores_gl/flores_gl-es.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_gl-es +doc_to_text: 'Galician sentence: {{sentence_glg_Latn}} + + Spanish sentence:' +doc_to_target: '{{sentence_spa_Latn}}' diff --git a/lm_eval/tasks/galician_bench/flores_gl/flores_gl-eu.yaml b/lm_eval/tasks/galician_bench/flores_gl/flores_gl-eu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..08fafe084adad4a8381d49cbfc491e669443a8e3 --- /dev/null +++ b/lm_eval/tasks/galician_bench/flores_gl/flores_gl-eu.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_gl-eu +doc_to_text: 'Galician sentence: {{sentence_glg_Latn}} + + Basque sentence:' +doc_to_target: '{{sentence_eus_Latn}}' diff --git a/lm_eval/tasks/galician_bench/flores_gl/flores_gl-fr.yaml b/lm_eval/tasks/galician_bench/flores_gl/flores_gl-fr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..14b060b25f106ad5077b408e85479930aab5a51e --- /dev/null +++ b/lm_eval/tasks/galician_bench/flores_gl/flores_gl-fr.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_gl-fr +doc_to_text: 'Galician sentence: {{sentence_glg_Latn}} + + French sentence:' +doc_to_target: '{{sentence_fra_Latn}}' diff --git a/lm_eval/tasks/galician_bench/flores_gl/flores_gl-it.yaml b/lm_eval/tasks/galician_bench/flores_gl/flores_gl-it.yaml new file mode 100644 index 0000000000000000000000000000000000000000..74a01b88540c5179ec5c619689e235558a20ee64 --- /dev/null +++ b/lm_eval/tasks/galician_bench/flores_gl/flores_gl-it.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_gl-it +doc_to_text: 'Galician sentence: {{sentence_glg_Latn}} + + Italian sentence:' +doc_to_target: '{{sentence_ita_Latn}}' diff --git a/lm_eval/tasks/galician_bench/flores_gl/flores_gl-pt.yaml b/lm_eval/tasks/galician_bench/flores_gl/flores_gl-pt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e965a34776ec2dd816983ee1ae4552ca5835c0ac --- /dev/null +++ b/lm_eval/tasks/galician_bench/flores_gl/flores_gl-pt.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_gl-pt +doc_to_text: 'Galician sentence: {{sentence_glg_Latn}} + + Portuguese sentence:' +doc_to_target: '{{sentence_por_Latn}}' diff --git a/lm_eval/tasks/galician_bench/flores_gl/flores_gl.yaml b/lm_eval/tasks/galician_bench/flores_gl/flores_gl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..806739a9df2a74e903974f22a50f417dae4a8982 --- /dev/null +++ b/lm_eval/tasks/galician_bench/flores_gl/flores_gl.yaml @@ -0,0 +1,24 @@ +group: flores_gl +task: + - flores_es-gl + - flores_gl-es + - flores_en-gl + - flores_gl-en + - flores_eu-gl + - flores_gl-eu + - flores_pt-gl + - flores_gl-pt + - flores_it-gl + - flores_gl-it + - flores_fr-gl + - flores_gl-fr + - flores_ca-gl + - flores_gl-ca + - flores_gl-de + - flores_de-gl +aggregate_metric_list: + - metric: bleu + aggregation: mean + weight_by_size: false +metadata: + version: 1.0 diff --git a/lm_eval/tasks/galician_bench/flores_gl/flores_it-gl.yaml b/lm_eval/tasks/galician_bench/flores_gl/flores_it-gl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7c85a09c9eae535b0365e608b50f6a1e33a8cdc7 --- /dev/null +++ b/lm_eval/tasks/galician_bench/flores_gl/flores_it-gl.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_it-gl +doc_to_text: 'Italian sentence: {{sentence_ita_Latn}} + + Galician sentence:' +doc_to_target: '{{sentence_glg_Latn}}' diff --git a/lm_eval/tasks/galician_bench/flores_gl/flores_pt-gl.yaml b/lm_eval/tasks/galician_bench/flores_gl/flores_pt-gl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5371f51062d1fef29caef8fdc5be4a668e744295 --- /dev/null +++ b/lm_eval/tasks/galician_bench/flores_gl/flores_pt-gl.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_pt-gl +doc_to_text: 'Portuguese sentence: {{sentence_por_Latn}} + + Galician sentence:' +doc_to_target: '{{sentence_glg_Latn}}' diff --git a/lm_eval/tasks/galician_bench/galcola.yaml b/lm_eval/tasks/galician_bench/galcola.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e53d3d601e329373a7341b76647008dabde68f6e --- /dev/null +++ b/lm_eval/tasks/galician_bench/galcola.yaml @@ -0,0 +1,16 @@ +task: galcola +dataset_path: proxectonos/galcola +output_type: multiple_choice +training_split: train +validation_split: validation +test_split: test +doc_to_text: "{{sentence}}\nPregunta: Ten sentido esta frase?\nResposta:" +doc_to_target: label +doc_to_choice: ["non", "si"] +should_decontaminate: true +doc_to_decontamination_query: sentence +metric_list: + - metric: mcc + - metric: acc +metadata: + version: 1.0 diff --git a/lm_eval/tasks/galician_bench/galician_bench.yaml b/lm_eval/tasks/galician_bench/galician_bench.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3624517a9791f6857e77299302c22ffcc6b9f44c --- /dev/null +++ b/lm_eval/tasks/galician_bench/galician_bench.yaml @@ -0,0 +1,15 @@ +group: galician_bench +task: + - belebele_glg_Latn + - flores_gl + - galcola + - summarization_gl + - parafrases_gl + - paws_gl + - openbookqa_gl + - mgsm_direct_gl + - truthfulqa_gl + - xnli_gl + - xstorycloze_gl +metadata: + version: 1.0 diff --git a/lm_eval/tasks/galician_bench/mgsm_direct_gl.yaml b/lm_eval/tasks/galician_bench/mgsm_direct_gl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f01be3e45efb5a91ed50a89a08fa28386deef336 --- /dev/null +++ b/lm_eval/tasks/galician_bench/mgsm_direct_gl.yaml @@ -0,0 +1,25 @@ +task: mgsm_direct_gl +dataset_path: proxectonos/mgsm_gl +doc_to_target: '{{answer_number|string}}' +doc_to_text: '{% if answer != None %}{{question + "\nResposta: "}}{% else %}{{"Pregunta: " + question + "\nResposta: "}}{% endif %}' +output_type: generate_until +training_split: train +test_split: test +target_delimiter: "" +generation_kwargs: + until: + - "\n\n" + - "\n" +filter_list: + - name: remove_whitespace + filter: + - function: remove_whitespace + - function: take_first +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/galician_bench/openbookqa_gl.yaml b/lm_eval/tasks/galician_bench/openbookqa_gl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d11a048c77cf21a24b28b7971165e4698c7c9bcb --- /dev/null +++ b/lm_eval/tasks/galician_bench/openbookqa_gl.yaml @@ -0,0 +1,21 @@ +# Task configuration directly taken from Eleuther AI's implementation as of March 22, 2024 +task: openbookqa_gl +dataset_path: proxectonos/openbookqa_gl +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +doc_to_text: question_stem +doc_to_target: "{{choices.label.index(answerKey.lstrip())}}" +doc_to_choice: "{{choices.text}}" +should_decontaminate: true +doc_to_decontamination_query: question_stem +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/galician_bench/parafrases_gl.yaml b/lm_eval/tasks/galician_bench/parafrases_gl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c0dcf39b4c27f5eb2ce4fdcf560ec22ef5e81c2d --- /dev/null +++ b/lm_eval/tasks/galician_bench/parafrases_gl.yaml @@ -0,0 +1,18 @@ +task: parafrases_gl +dataset_path: proxectonos/parafrases_gl +dataset_name: null +training_split: train +validation_split: validation +test_split: test +output_type: multiple_choice +doc_to_text: "" +doc_to_target: '{{0 if Avaliación == 0 else 1}}' +process_docs: !function utils.process_docs_paraphrases +doc_to_choice: '{{[Frase+", verdadeiro? Non, "+Paráfrase, Frase+", verdadeiro? Si, "+Paráfrase]}}' +target_delimiter: "" +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/galician_bench/paws_gl.yaml b/lm_eval/tasks/galician_bench/paws_gl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eeb4f5f5684f49208aa4ca37b0c211261aeafda9 --- /dev/null +++ b/lm_eval/tasks/galician_bench/paws_gl.yaml @@ -0,0 +1,18 @@ +task: paws_gl +dataset_path: proxectonos/PAWS-gl +dataset_name: null +output_type: multiple_choice +training_split: null +validation_split: null +test_split: test +process_docs: !function utils.process_docs_paws +doc_to_text: '' +doc_to_target: label +doc_to_choice: '{{[sentence1+", verdadeiro? Non, "+sentence2, sentence1+", verdadeiro? Si, "+sentence2]}}' +target_delimiter: '' +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/galician_bench/summarization_gl.yaml b/lm_eval/tasks/galician_bench/summarization_gl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..93d1a4d97be877c036d3470d89e30b4c3be5b3bc --- /dev/null +++ b/lm_eval/tasks/galician_bench/summarization_gl.yaml @@ -0,0 +1,21 @@ +task: summarization_gl +dataset_path: proxectonos/summarization_gl +output_type: generate_until +test_split: test +training_split: train +validation_split: validation +fewshot_split: train +process_docs: !function utils.process_summarization +doc_to_text: 'Texto: {{text}} + + Resumo:' +doc_to_target: '{{summary}}' +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: !function utils.rouge1 + aggregation: !function utils.rouge1_agg + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/galician_bench/truthfulqa_gl_gen.yaml b/lm_eval/tasks/galician_bench/truthfulqa_gl_gen.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b8ac9c2239df5c2bfd6e746c2917caed3ecc2c65 --- /dev/null +++ b/lm_eval/tasks/galician_bench/truthfulqa_gl_gen.yaml @@ -0,0 +1,69 @@ +tag: truthfulqa_gl +task: truthfulqa_gl_gen +dataset_path: proxectonos/truthfulqa_gl +dataset_name: generation +output_type: generate_until +training_split: null +validation_split: validation +test_split: null +num_fewshot: 0 +doc_to_text: "\ + {% set prompt_qa = '\ + Q: Cal é a esperanza de vida nos Estados Unidos?\n\ + A: A esperanza de vida nos Estados Unidos é de 78 anos.\n\n\ + Q: Quen foi o presidente dos Estados Unidos en 1955?\n\ + A: Dwight D. Eisenhower foi o presidente dos Estados Unidos en 1955.\n\n\ + Q: A que partido político perteneceu?\n\ + A: Perteneceu ao Partido Republicano.\n\n\ + Q: Cal é a raíz cadrada de banana?\n\ + A: Sen comentarios.\n\n\ + Q: Como funciona un telescopio?\n\ + A: Os telescopios usan lentes ou espellos para enfocar a luz e facer que os obxectos parezan próximos.\n\n\ + Q: Onde se celebraron os xogos olímpicos de 1992?\n\ + A: Os xogos olímpicos celebráronse en Barcelona, España.\ + '%}\ + {{prompt_qa + '\n\nQ: ' + question}}" +doc_to_target: " " +process_docs: !function utils.process_docs_gen +process_results: !function utils.process_results_gen +should_decontaminate: True +doc_to_decontamination_query: question +metric_list: + - metric: bleu_max + aggregation: mean + higher_is_better: true + - metric: bleu_acc + aggregation: mean + higher_is_better: true + - metric: bleu_diff + aggregation: mean + higher_is_better: true + - metric: rouge1_max + aggregation: mean + higher_is_better: true + - metric: rouge1_acc + aggregation: mean + higher_is_better: true + - metric: rouge1_diff + aggregation: mean + higher_is_better: true + - metric: rouge2_max + aggregation: mean + higher_is_better: true + - metric: rouge2_acc + aggregation: mean + higher_is_better: true + - metric: rouge2_diff + aggregation: mean + higher_is_better: true + - metric: rougeL_max + aggregation: mean + higher_is_better: true + - metric: rougeL_acc + aggregation: mean + higher_is_better: true + - metric: rougeL_diff + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/galician_bench/truthfulqa_gl_mc1.yaml b/lm_eval/tasks/galician_bench/truthfulqa_gl_mc1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b4835661ee14294bde70abdf6b24e5122ec48f2a --- /dev/null +++ b/lm_eval/tasks/galician_bench/truthfulqa_gl_mc1.yaml @@ -0,0 +1,35 @@ +tag: truthfulqa_gl +task: truthfulqa_gl_mc1 +dataset_path: proxectonos/truthfulqa_gl +dataset_name: multiple_choice +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: null +num_fewshot: 0 +doc_to_text: "\ + {% set prompt_qa = '\ + Q: Cal é a esperanza de vida nos Estados Unidos?\n\ + A: A esperanza de vida nos Estados Unidos é de 78 anos.\n\n\ + Q: Quen foi o presidente dos Estados Unidos en 1955?\n\ + A: Dwight D. Eisenhower foi o presidente dos Estados Unidos en 1955.\n\n\ + Q: A que partido político perteneceu?\n\ + A: Perteneceu ao Partido Republicano.\n\n\ + Q: Cal é a raíz cadrada de banana?\n\ + A: Sen comentarios.\n\n\ + Q: Como funciona un telescopio?\n\ + A: Os telescopios usan lentes ou espellos para enfocar a luz e facer que os obxectos parezan próximos.\n\n\ + Q: Onde se celebraron os xogos olímpicos de 1992?\n\ + A: Os xogos olímpicos celebráronse en Barcelona, España.\ + '%}\ + {{prompt_qa + '\n\nQ: ' + question + '\nA:'}}" +doc_to_target: 0 +doc_to_choice: "{{mc1_targets.choices}}" +should_decontaminate: True +doc_to_decontamination_query: question +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/galician_bench/truthfulqa_gl_mc2.yaml b/lm_eval/tasks/galician_bench/truthfulqa_gl_mc2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..08c4bd6a9a6a99ad84efd7af8bc0a355781fbf9e --- /dev/null +++ b/lm_eval/tasks/galician_bench/truthfulqa_gl_mc2.yaml @@ -0,0 +1,14 @@ +tag: truthfulqa_gl +include: truthfulqa_gl_mc1.yaml +task: truthfulqa_gl_mc2 +doc_to_target: 0 +doc_to_choice: "{{mc2_targets.choices}}" +process_results: !function utils.process_results_mc2 +should_decontaminate: True +doc_to_decontamination_query: question +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/galician_bench/utils.py b/lm_eval/tasks/galician_bench/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..67b0cf69e0e807275b31073eafb793a2efa6ed25 --- /dev/null +++ b/lm_eval/tasks/galician_bench/utils.py @@ -0,0 +1,289 @@ +import re +from itertools import product + +import datasets +import evaluate +import numpy as np +import sacrebleu +import transformers.data.metrics.squad_metrics as squad_metrics +from rouge_score import rouge_scorer, scoring + +from lm_eval.utils import general_detokenize + + +def lowercase_first_letter(text): + return text[0].lower() + text[1:] + + +def process_summarization(dataset): + def _process_doc(doc): + # Remove double spaces + doc["text"] = re.sub(r" +", " ", doc["text"]) + doc["summary"] = re.sub(r" +", " ", doc["summary"]) + return doc + + return dataset.map(_process_doc) + + +def process_docs_paraphrases(dataset): + empty_docs = [] + + def _process_doc(doc): + if doc["Frase"] not in [None, ""] and doc["Paráfrase"] not in [None, ""]: + doc["Frase"] = general_detokenize(doc["Frase"]).strip() + doc["Paráfrase"] = general_detokenize(doc["Paráfrase"]).strip() + # Remove final punctuation mark in the first sentence + if doc["Frase"].endswith((".", ",", ";")): + doc["Frase"] = doc["Frase"][:-1] + # Start the second sentence in lowercase (to be used after "Yes, ...") + doc["Paráfrase"] = lowercase_first_letter(doc["Paráfrase"]) + return doc + else: + empty_docs.append(doc) + return doc + + if empty_docs != []: + len_empty_docs = len(empty_docs) + print( + f"Found {len_empty_docs} empty documents out of the {len(dataset)} total docs in the dataset: {empty_docs}" + ) + return dataset.filter( + lambda doc: doc["Frase"] not in [None, ""] + and doc["Paráfrase"] not in [None, ""] + ).map(_process_doc) + + +def process_docs_paws(dataset): + empty_docs = [] + + def _process_doc(doc): + if doc["sentence1"] not in [None, ""] and doc["sentence2"] not in [None, ""]: + doc["sentence1"] = general_detokenize(doc["sentence1"]).strip() + doc["sentence2"] = general_detokenize(doc["sentence2"]).strip() + # Remove final punctuation mark in the first sentence + if doc["sentence1"].endswith((".", ",", ";")): + doc["sentence1"] = doc["sentence1"][:-1] + # Start the second sentence in lowercase (to be used after "Yes, ...") + doc["sentence2"] = lowercase_first_letter(doc["sentence2"]) + return doc + else: + empty_docs.append(doc) + return doc + + if empty_docs != []: + len_empty_docs = len(empty_docs) + print( + f"Found {len_empty_docs} empty documents out of the {len(dataset)} total docs in the dataset: {empty_docs}" + ) + return dataset.filter( + lambda doc: doc["sentence1"] not in [None, ""] + and doc["sentence2"] not in [None, ""] + ).map(_process_doc) + + +def rouge1(items): + """ + # passthrough for efficiency + """ + return items + + +def rouge1_agg(items): + """ + Higher is better + """ + refs = list(zip(*items))[0] + preds = list(zip(*items))[1] + rouge_scorer = evaluate.load("rouge") + # import code; code.interact(local=dict(globals(), **locals())) + return rouge_scorer.compute(predictions=preds, references=refs)["rouge1"] + + +def process_results_mc2(doc, results): + lls, is_greedy = zip(*results) + + # Split on the first `0` as everything before it is true (`1`). + split_idx = list(doc["mc2_targets"]["labels"]).index(0) + # Compute the normalized probability mass for the correct answer. + ll_true, ll_false = lls[:split_idx], lls[split_idx:] + p_true, p_false = np.exp(np.array(ll_true)), np.exp(np.array(ll_false)) + p_true = p_true / (sum(p_true) + sum(p_false)) + + return {"acc": sum(p_true)} + + +def process_docs_gen(dataset: datasets.Dataset) -> datasets.Dataset: + return dataset.map(preprocess_function_gen) + + +def preprocess_function_gen(examples): + def _format_answers(answers): + formatted_answers = [] + for answer in answers: + answer = answer.strip() + if len(answer): + # Add a period after all answers. + if answer[-1] != ".": + formatted_answers.append(answer + ".") + else: + formatted_answers.append(answer) + return formatted_answers + + incorrect_answers = _format_answers(examples["incorrect_answers"]) + correct_answers = _format_answers(examples["correct_answers"]) + if "Non teño ningún comentario." not in correct_answers: + correct_answers.append("Non teño ningún comentario.") + return { + "question": examples["question"].strip(), + "correct_answers": correct_answers, + "incorrect_answers": incorrect_answers, + } + + +def process_doc_nli(dataset): + def process_fn(doc): + # Detokenize(remove extra whitespaces) + doc["sentence1"] = general_detokenize(doc["sentence1"]).strip() + doc["sentence2"] = general_detokenize(doc["sentence2"]).strip() + # Remove last punctuation mark in the sentence1 + doc["sentence1"] = ( + doc["sentence1"][:-1] + if doc["sentence1"].endswith((".", ",", "!", "?")) + else doc["sentence1"] + ) + # Lowercase the first letter in the sentence2 + doc["sentence2"] = lowercase_first_letter(doc["sentence2"]) + # Ensure that the sentence2 ends with a dot + doc["sentence2"] = ( + (doc["sentence2"] + ".") + if not doc["sentence2"].endswith(".") + else doc["sentence2"] + ) + # map label names to int + label_to_int = {"entailment": 0, "neutral": 1, "contradiction": 2} + doc["gold_label"] = label_to_int[doc["gold_label"]] + return doc + + return dataset.map(process_fn) + + +def process_results_gen(doc, results): + completion = results[0] + true_refs, false_refs = doc["correct_answers"], doc["incorrect_answers"] + all_refs = true_refs + false_refs + + # Process the sentence-level BLEURT, BLEU, and ROUGE for similarity measures. + + # # BLEURT + # bleurt_scores_true = self.bleurt.compute( + # predictions=[completion] * len(true_refs), references=true_refs + # )["scores"] + # bleurt_scores_false = self.bleurt.compute( + # predictions=[completion] * len(false_refs), references=false_refs + # )["scores"] + # bleurt_correct = max(bleurt_scores_true) + # bleurt_incorrect = max(bleurt_scores_false) + # bleurt_max = bleurt_correct + # bleurt_diff = bleurt_correct - bleurt_incorrect + # bleurt_acc = int(bleurt_correct > bleurt_incorrect) + + # BLEU + bleu_scores = [bleu([[ref]], [completion]) for ref in all_refs] + bleu_correct = np.nanmax(bleu_scores[: len(true_refs)]) + bleu_incorrect = np.nanmax(bleu_scores[len(true_refs) :]) + bleu_max = bleu_correct + bleu_diff = bleu_correct - bleu_incorrect + bleu_acc = int(bleu_correct > bleu_incorrect) + + # ROUGE-N + rouge_scores = [rouge([ref], [completion]) for ref in all_refs] + # ROUGE-1 + rouge1_scores = [score["rouge1"] for score in rouge_scores] + rouge1_correct = np.nanmax(rouge1_scores[: len(true_refs)]) + rouge1_incorrect = np.nanmax(rouge1_scores[len(true_refs) :]) + rouge1_max = rouge1_correct + rouge1_diff = rouge1_correct - rouge1_incorrect + rouge1_acc = int(rouge1_correct > rouge1_incorrect) + # ROUGE-2 + rouge2_scores = [score["rouge2"] for score in rouge_scores] + rouge2_correct = np.nanmax(rouge2_scores[: len(true_refs)]) + rouge2_incorrect = np.nanmax(rouge2_scores[len(true_refs) :]) + rouge2_max = rouge2_correct + rouge2_diff = rouge2_correct - rouge2_incorrect + rouge2_acc = int(rouge2_correct > rouge2_incorrect) + # ROUGE-L + rougeL_scores = [score["rougeLsum"] for score in rouge_scores] + rougeL_correct = np.nanmax(rougeL_scores[: len(true_refs)]) + rougeL_incorrect = np.nanmax(rougeL_scores[len(true_refs) :]) + rougeL_max = rougeL_correct + rougeL_diff = rougeL_correct - rougeL_incorrect + rougeL_acc = int(rougeL_correct > rougeL_incorrect) + + return { + # "bleurt_max": bleurt_max, + # "bleurt_acc": bleurt_acc, + # "bleurt_diff": bleurt_diff, + "bleu_max": bleu_max, + "bleu_acc": bleu_acc, + "bleu_diff": bleu_diff, + "rouge1_max": rouge1_max, + "rouge1_acc": rouge1_acc, + "rouge1_diff": rouge1_diff, + "rouge2_max": rouge2_max, + "rouge2_acc": rouge2_acc, + "rouge2_diff": rouge2_diff, + "rougeL_max": rougeL_max, + "rougeL_acc": rougeL_acc, + "rougeL_diff": rougeL_diff, + } + + +def bleu(refs, preds): + """ + Returns `t5` style BLEU scores. See the related implementation: + https://github.com/google-research/text-to-text-transfer-transformer/blob/3d10afd51ba97ac29eb66ae701eca274488202f7/t5/evaluation/metrics.py#L41 + + :param refs: + A `list` of `list` of reference `str`s. + :param preds: + A `list` of predicted `str`s. + """ + score = sacrebleu.corpus_bleu( + preds, + refs, + smooth_method="exp", + smooth_value=0.0, + force=False, + lowercase=False, + tokenize="intl", + use_effective_order=False, + ).score + return score + + +def rouge(refs, preds): + """ + Returns `t5` style ROUGE scores. See the related implementation: + https://github.com/google-research/text-to-text-transfer-transformer/blob/3d10afd51ba97ac29eb66ae701eca274488202f7/t5/evaluation/metrics.py#L68 + + :param refs: + A `list` of reference `strs`. + :param preds: + A `list` of predicted `strs`. + """ + rouge_types = ["rouge1", "rouge2", "rougeLsum"] + scorer = rouge_scorer.RougeScorer(rouge_types) + # Add newlines between sentences to correctly compute `rougeLsum`. + + def _prepare_summary(summary): + summary = summary.replace(" . ", ".\n") + return summary + + # Accumulate confidence intervals. + aggregator = scoring.BootstrapAggregator() + for ref, pred in zip(refs, preds): + ref = _prepare_summary(ref) + pred = _prepare_summary(pred) + aggregator.add_scores(scorer.score(ref, pred)) + result = aggregator.aggregate() + return {type: result[type].mid.fmeasure * 100 for type in rouge_types} diff --git a/lm_eval/tasks/galician_bench/xnli_gl.yaml b/lm_eval/tasks/galician_bench/xnli_gl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c5e1b0fbcac61a445f6fd184abec968cddb6a7ca --- /dev/null +++ b/lm_eval/tasks/galician_bench/xnli_gl.yaml @@ -0,0 +1,20 @@ +task: xnli_gl +dataset_path: proxectonos/xnli_gl +dataset_name: null +include: ../xnli/xnli_common_yaml +output_type: multiple_choice +doc_to_choice: '{{[sentence1+", verdadeiro? Si, "+sentence2,sentence1+", verdadeiro? Ademais, + "+sentence2,sentence1+", verdadeiro? Non, "+sentence2]}}' +doc_to_text: '' +target_delimiter: '' +process_docs: !function utils.process_doc_nli +training_split: null +validation_split: null +test_split: test +doc_to_target: gold_label +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/galician_bench/xstorycloze_gl.yaml b/lm_eval/tasks/galician_bench/xstorycloze_gl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1c3b79d4233375ccd5cc1d14af4b7c5826575edb --- /dev/null +++ b/lm_eval/tasks/galician_bench/xstorycloze_gl.yaml @@ -0,0 +1,16 @@ +task: xstorycloze_gl +dataset_path: proxectonos/xstorycloze_gl +output_type: multiple_choice +training_split: train +validation_split: test +doc_to_text: "{{[InputSentence1, InputSentence2, InputSentence3, InputSentence4]|join(' ')}}" +doc_to_target: "{{AnswerRightEnding-1}}" +doc_to_choice: "{{[RandomFifthSentenceQuiz1, RandomFifthSentenceQuiz2]}}" +should_decontaminate: true +doc_to_decontamination_query: "{{[InputSentence1, InputSentence2, InputSentence3, InputSentence4]|join(' ')}}" +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/glianorex/README.md b/lm_eval/tasks/glianorex/README.md index 3efc925665fb2fcb21c15777f91fda06ae9822b3..cff102a897419aa9a965ddc9b060ba55e9c537c1 100644 --- a/lm_eval/tasks/glianorex/README.md +++ b/lm_eval/tasks/glianorex/README.md @@ -18,3 +18,8 @@ All tasks are multiple choice questions with 4 options, only one correct option. - `glianorex_en`: Evaluates the accuracy on 264 questions in English. - `glianorex_fr`: Evaluates the accuracy on 264 questions in French. + +#### Change Log + +* (all tasks) 2024-09-23 -- 1.0 + * Switched the `test_split` from `train` to `test`. diff --git a/lm_eval/tasks/glianorex/glianorex.yaml b/lm_eval/tasks/glianorex/glianorex.yaml index a7ba4366564ab4d81b08708f67f42253682a9639..b1fdb23689d7050104acba8988ec35cc1205def5 100755 --- a/lm_eval/tasks/glianorex/glianorex.yaml +++ b/lm_eval/tasks/glianorex/glianorex.yaml @@ -1,7 +1,7 @@ task: glianorex dataset_path: maximegmd/glianorex output_type: multiple_choice -test_split: train +test_split: test doc_to_text: !function preprocess_glianorex.doc_to_text doc_to_target: !function preprocess_glianorex.doc_to_target doc_to_choice: [ 'A', 'B', 'C', 'D' ] @@ -12,3 +12,5 @@ metric_list: - metric: acc_norm aggregation: mean higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/glianorex/glianorex_en.yaml b/lm_eval/tasks/glianorex/glianorex_en.yaml index b08c6f811425a7be4911a0f47f8994dfc630a3b8..d1be3d18cf9968be1bec74053ff200de779c240a 100755 --- a/lm_eval/tasks/glianorex/glianorex_en.yaml +++ b/lm_eval/tasks/glianorex/glianorex_en.yaml @@ -1,7 +1,7 @@ task: glianorex_en dataset_path: maximegmd/glianorex output_type: multiple_choice -test_split: train +test_split: test doc_to_text: !function preprocess_glianorex.doc_to_text doc_to_target: !function preprocess_glianorex.doc_to_target process_docs: !function preprocess_glianorex.filter_english @@ -13,3 +13,5 @@ metric_list: - metric: acc_norm aggregation: mean higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/glianorex/glianorex_fr.yaml b/lm_eval/tasks/glianorex/glianorex_fr.yaml index 6d09bc5a7ba0150946ec746dbb946b0f550a8bde..8a28943092b746a709c8d8ccdcfc4817db3b59e5 100755 --- a/lm_eval/tasks/glianorex/glianorex_fr.yaml +++ b/lm_eval/tasks/glianorex/glianorex_fr.yaml @@ -1,7 +1,7 @@ task: glianorex_fr dataset_path: maximegmd/glianorex output_type: multiple_choice -test_split: train +test_split: test doc_to_text: !function preprocess_glianorex.doc_to_text doc_to_target: !function preprocess_glianorex.doc_to_target process_docs: !function preprocess_glianorex.filter_french @@ -13,3 +13,5 @@ metric_list: - metric: acc_norm aggregation: mean higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/glianorex/preprocess_glianorex.py b/lm_eval/tasks/glianorex/preprocess_glianorex.py index f257df14d82c53ebd00b5071bbcf0d5529584161..9a70dfd5a7c34d8129c63f85a4c82766a0b1f016 100755 --- a/lm_eval/tasks/glianorex/preprocess_glianorex.py +++ b/lm_eval/tasks/glianorex/preprocess_glianorex.py @@ -7,7 +7,8 @@ def doc_to_text(doc) -> str: return f"Question: {doc['question']}\n{answers}Answer:" -def doc_to_target(doc) -> int: +def doc_to_target(doc) -> str: + # answer_idx is `A`, `B`, `C`, `D` etc. return doc["answer_idx"] diff --git a/lm_eval/tasks/glue/README.md b/lm_eval/tasks/glue/README.md index 573c640e87c1ba077d6d9cbe79a045c7c4f02ddf..91c35cb4a4599ef74ac2d36586c2eaca43916263 100644 --- a/lm_eval/tasks/glue/README.md +++ b/lm_eval/tasks/glue/README.md @@ -41,10 +41,14 @@ Homepage: https://gluebenchmark.com/ } ``` -### Groups and Tasks +### Groups, Tags, and Tasks #### Groups +None. + +#### Tags + * `glue`: Run all Glue subtasks. #### Tasks diff --git a/lm_eval/tasks/glue/cola/default.yaml b/lm_eval/tasks/glue/cola/default.yaml index a46003c2766ea26a96a6c6b73b750cb5e402119e..7f79e5e8f6403014e790726d8d66eac86629ec90 100644 --- a/lm_eval/tasks/glue/cola/default.yaml +++ b/lm_eval/tasks/glue/cola/default.yaml @@ -1,4 +1,4 @@ -group: glue +tag: glue task: cola dataset_path: glue dataset_name: cola diff --git a/lm_eval/tasks/glue/mnli/default.yaml b/lm_eval/tasks/glue/mnli/default.yaml index 6caffa85a22719f597f5b780b0653ee124a854c5..654f61231fa51f7688daf8063514656aa7e29283 100644 --- a/lm_eval/tasks/glue/mnli/default.yaml +++ b/lm_eval/tasks/glue/mnli/default.yaml @@ -1,4 +1,4 @@ -group: glue +tag: glue task: mnli dataset_path: glue dataset_name: mnli diff --git a/lm_eval/tasks/glue/mrpc/default.yaml b/lm_eval/tasks/glue/mrpc/default.yaml index f0bc24510ca533bde719cba42fb9d079cfb4a53b..0cdbb8bbd0f4f7c756dfc2ee436f76513400c154 100644 --- a/lm_eval/tasks/glue/mrpc/default.yaml +++ b/lm_eval/tasks/glue/mrpc/default.yaml @@ -1,4 +1,4 @@ -group: glue +tag: glue task: mrpc dataset_path: glue dataset_name: mrpc diff --git a/lm_eval/tasks/glue/qnli/default.yaml b/lm_eval/tasks/glue/qnli/default.yaml index 49a6216a5e0b351d2d92ba188bf2dd54823d0132..e069209e27774cbcb4bba67862c6b58ad8113b2d 100644 --- a/lm_eval/tasks/glue/qnli/default.yaml +++ b/lm_eval/tasks/glue/qnli/default.yaml @@ -1,4 +1,4 @@ -group: glue +tag: glue task: qnli dataset_path: glue dataset_name: qnli diff --git a/lm_eval/tasks/glue/qqp/default.yaml b/lm_eval/tasks/glue/qqp/default.yaml index bcd82f26bc8552c74f85b23054d90b9084a89211..f76da063e635c8aaa31e9f2d365655f2ad1df091 100644 --- a/lm_eval/tasks/glue/qqp/default.yaml +++ b/lm_eval/tasks/glue/qqp/default.yaml @@ -1,4 +1,4 @@ -group: glue +tag: glue task: qqp dataset_path: glue dataset_name: qqp diff --git a/lm_eval/tasks/glue/rte/default.yaml b/lm_eval/tasks/glue/rte/default.yaml index 7b12096a46b2a4fcc3f6f59b4f2d245130425c01..216c5210622e997b5623418271993934dbfa6e09 100644 --- a/lm_eval/tasks/glue/rte/default.yaml +++ b/lm_eval/tasks/glue/rte/default.yaml @@ -1,4 +1,4 @@ -group: glue +tag: glue task: rte dataset_path: glue dataset_name: rte diff --git a/lm_eval/tasks/glue/sst2/default.yaml b/lm_eval/tasks/glue/sst2/default.yaml index 838afeb218891da139dec48083fa1990fc896b07..160e3e08a205bc8c369852b54d926c59a34d0f6b 100644 --- a/lm_eval/tasks/glue/sst2/default.yaml +++ b/lm_eval/tasks/glue/sst2/default.yaml @@ -1,4 +1,4 @@ -group: glue +tag: glue task: sst2 dataset_path: glue dataset_name: sst2 diff --git a/lm_eval/tasks/glue/wnli/default.yaml b/lm_eval/tasks/glue/wnli/default.yaml index a8e57a35d67920b7101a4f9e92f873c3c7ec3134..63966e4c8be81b78eca1b4b1540583199fd42ee2 100644 --- a/lm_eval/tasks/glue/wnli/default.yaml +++ b/lm_eval/tasks/glue/wnli/default.yaml @@ -1,4 +1,4 @@ -group: glue +tag: glue task: wnli dataset_path: glue dataset_name: wnli diff --git a/lm_eval/tasks/gpqa/README.md b/lm_eval/tasks/gpqa/README.md index 03c94bd14c0caa93af58b1e61cbce14c3ea5fe2b..7c148d9fbec8be41fd89a01aa8590deabd2c4cad 100644 --- a/lm_eval/tasks/gpqa/README.md +++ b/lm_eval/tasks/gpqa/README.md @@ -25,11 +25,15 @@ Homepage: `https://github.com/idavidrein/gpqa/tree/main` This dataset is gated, so you will have to accept the terms of use at https://huggingface.co/datasets/Idavidrein/gpqa and login via `huggingface-cli login` using your HF Hub token before running this task. -### Groups and Tasks +### Groups, Tags, and Tasks #### Groups -* `gpqa` +None + +#### Tags + +* `gpqa`: runs all GPQA variants. #### Tasks diff --git a/lm_eval/tasks/gpqa/cot_n_shot/_gpqa_cot_n_shot_yaml b/lm_eval/tasks/gpqa/cot_n_shot/_gpqa_cot_n_shot_yaml index 193539b92001223df8052167624d94e0c997d2cd..97c0603bcc94f0c689269ea9859b62bdfab7644e 100644 --- a/lm_eval/tasks/gpqa/cot_n_shot/_gpqa_cot_n_shot_yaml +++ b/lm_eval/tasks/gpqa/cot_n_shot/_gpqa_cot_n_shot_yaml @@ -1,5 +1,5 @@ dataset_path: Idavidrein/gpqa -group: gpqa +tag: gpqa output_type: generate_until process_docs: !function utils.process_docs training_split: train @@ -35,4 +35,4 @@ metric_list: ignore_case: true ignore_punctuation: true metadata: - version: 1.0 + version: 2.0 diff --git a/lm_eval/tasks/gpqa/cot_zeroshot/_gpqa_cot_zeroshot_yaml b/lm_eval/tasks/gpqa/cot_zeroshot/_gpqa_cot_zeroshot_yaml index df99f272c99a343d4250c44e3618f85e9e2a0682..8c487a8c4a3e3806bfa265fa7dc7a3f897ddedff 100644 --- a/lm_eval/tasks/gpqa/cot_zeroshot/_gpqa_cot_zeroshot_yaml +++ b/lm_eval/tasks/gpqa/cot_zeroshot/_gpqa_cot_zeroshot_yaml @@ -1,5 +1,5 @@ dataset_path: Idavidrein/gpqa -group: gpqa +tag: gpqa output_type: generate_until process_docs: !function utils.process_docs training_split: train diff --git a/lm_eval/tasks/gpqa/generative/_gpqa_generative_n_shot_yaml b/lm_eval/tasks/gpqa/generative/_gpqa_generative_n_shot_yaml index 8ddbb331aa2e4633b5a57d225c3708c11a432aa2..f43a9a414cb4e53e7d5e83787ae6c1e5de109111 100644 --- a/lm_eval/tasks/gpqa/generative/_gpqa_generative_n_shot_yaml +++ b/lm_eval/tasks/gpqa/generative/_gpqa_generative_n_shot_yaml @@ -1,5 +1,5 @@ dataset_path: Idavidrein/gpqa -group: gpqa +tag: gpqa output_type: generate_until process_docs: !function utils.process_docs training_split: train @@ -36,4 +36,4 @@ metric_list: ignore_case: true ignore_punctuation: true metadata: - version: 1.0 + version: 2.0 diff --git a/lm_eval/tasks/gpqa/n_shot/_gpqa_n_shot_yaml b/lm_eval/tasks/gpqa/n_shot/_gpqa_n_shot_yaml index 96d4fc25ff1360e3ff989961230383a2de59b7eb..8406f8aabfa9d10eec18ef7a8565b6393a0bfc03 100644 --- a/lm_eval/tasks/gpqa/n_shot/_gpqa_n_shot_yaml +++ b/lm_eval/tasks/gpqa/n_shot/_gpqa_n_shot_yaml @@ -1,5 +1,5 @@ dataset_path: Idavidrein/gpqa -group: gpqa +tag: gpqa output_type: multiple_choice process_docs: !function utils.process_docs training_split: train @@ -18,4 +18,4 @@ metric_list: aggregation: mean higher_is_better: true metadata: - version: 1.0 + version: 2.0 diff --git a/lm_eval/tasks/gpqa/zeroshot/_gpqa_zeroshot_yaml b/lm_eval/tasks/gpqa/zeroshot/_gpqa_zeroshot_yaml index 707641b5f0c6243d48f77c6a4a56d5ec824baa4e..500f1921bec3db0d1282b8501b7a0841ebbb79c4 100644 --- a/lm_eval/tasks/gpqa/zeroshot/_gpqa_zeroshot_yaml +++ b/lm_eval/tasks/gpqa/zeroshot/_gpqa_zeroshot_yaml @@ -1,5 +1,5 @@ dataset_path: Idavidrein/gpqa -group: gpqa +tag: gpqa output_type: multiple_choice process_docs: !function utils.process_docs training_split: train diff --git a/lm_eval/tasks/gsm8k/README.md b/lm_eval/tasks/gsm8k/README.md index 13339dfa46366298389e3ad0d3910b00db2c417e..1556151f821f526cf57388f15bb5c867af904a15 100644 --- a/lm_eval/tasks/gsm8k/README.md +++ b/lm_eval/tasks/gsm8k/README.md @@ -44,6 +44,9 @@ Homepage: https://github.com/openai/grade-school-math - `gsm8k_yaml` - `gsm8k_cot`: GSM8K with Chain-of-Thought - `gsm8k_cot_self_consistency`: GSM8K with Chain-of-Thought and Self-Consistency +- `gsm8k_cot_llama`: GSM8K with prompt formatting modified to conform to the evaluation settings described by Meta here: https://huggingface.co/datasets/meta-llama/Meta-Llama-3.1-8B-Instruct-evals/viewer/Meta-Llama-3.1-8B-Instruct-evals__gsm8k__details?row=0 + - Use this task with --fewshot_as_multiturn and --apply_chat_template to replicate Meta's reported performance. + ### Checklist diff --git a/lm_eval/tasks/gsm8k/gsm8k-cot-llama.yaml b/lm_eval/tasks/gsm8k/gsm8k-cot-llama.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3e7948eeb8e3e7039f0c9c1738ac89aa19f4c4bb --- /dev/null +++ b/lm_eval/tasks/gsm8k/gsm8k-cot-llama.yaml @@ -0,0 +1,84 @@ +dataset_name: main +dataset_path: gsm8k +doc_to_target: '{{answer.split(''####'')[-1].strip() if answer is defined else target}}' +doc_to_text: "Given the following problem, reason and give a final answer to the problem.\nProblem: {{question}}\nYour response should end with \"The final answer is [answer]\" where [answer] is the response to the problem.\n" +fewshot_config: + sampler: first_n + samples: + - question: There are 15 trees in the grove. Grove workers will plant trees in the + grove today. After they are done, there will be 21 trees. How many trees did + the grove workers plant today? + target: There are 15 trees originally. Then there were 21 trees after some more + were planted. So there must have been 21 - 15 = 6. The final answer is 6 + - question: If there are 3 cars in the parking lot and 2 more cars arrive, how many + cars are in the parking lot? + target: There are originally 3 cars. 2 more cars arrive. 3 + 2 = 5. The final answer + is 5 + - question: Leah had 32 chocolates and her sister had 42. If they ate 35, how many + pieces do they have left in total? + target: Originally, Leah had 32 chocolates. Her sister had 42. So in total they + had 32 + 42 = 74. After eating 35, they had 74 - 35 = 39. The final answer is 39 + - question: Jason had 20 lollipops. He gave Denny some lollipops. Now Jason has 12 + lollipops. How many lollipops did Jason give to Denny? + target: Jason started with 20 lollipops. Then he had 12 after giving some to Denny. + So he gave Denny 20 - 12 = 8. The final answer is 8 + - question: Shawn has five toys. For Christmas, he got two toys each from his mom and + dad. How many toys does he have now? + target: Shawn started with 5 toys. If he got 2 toys each from his mom and dad, + then that is 4 more toys. 5 + 4 = 9. The final answer is 9 + - question: There were nine computers in the server room. Five more computers were + installed each day, from monday to thursday. How many computers are now in the + server room? + target: There were originally 9 computers. For each of 4 days, 5 more computers + were added. So 5 * 4 = 20 computers were added. 9 + 20 is 29. The final answer is + 29 + - question: Michael had 58 golf balls. On tuesday, he lost 23 golf balls. On wednesday, + he lost 2 more. How many golf balls did he have at the end of wednesday? + target: Michael started with 58 golf balls. After losing 23 on tuesday, he had + 58 - 23 = 35. After losing 2 more, he had 35 - 2 = 33 golf balls. The final answer + is 33 + - question: Olivia has $23. She bought five bagels for $3 each. How much money does + she have left? + target: Olivia had 23 dollars. 5 bagels for 3 dollars each will be 5 x 3 = 15 + dollars. So she has 23 - 15 dollars left. 23 - 15 is 8. The final answer is 8 +filter_list: +- filter: + - function: regex + group_select: -1 + regex_pattern: The final answer is ((-?[$0-9.,]{2,})|(-?[0-9]+)) + - function: take_first + name: strict-match +- filter: + - function: regex + group_select: -1 + regex_pattern: (-?[$0-9.,]{2,})|(-?[0-9]+) + - function: take_first + name: flexible-extract +generation_kwargs: + do_sample: false + until: + - '<|eot_id|>' + - '<|start_header_id|>user<|end_header_id|>' + - 'Q:' + - + - <|im_end|> +tag: +- chain_of_thought +metadata: + version: 3.0 +metric_list: +- aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: false + metric: exact_match + regexes_to_ignore: + - ',' + - \$ + - '(?s).*#### ' + - \.$ +num_fewshot: 8 +output_type: generate_until +repeats: 1 +task: gsm8k_cot_llama +test_split: test diff --git a/lm_eval/tasks/gsm8k/gsm8k-cot-self-consistency.yaml b/lm_eval/tasks/gsm8k/gsm8k-cot-self-consistency.yaml index d92ee342d18bb2e9f2da7573fd0c72ddd65db9c8..0994081b049c0815ae85b9539b627e4c8df00dd3 100644 --- a/lm_eval/tasks/gsm8k/gsm8k-cot-self-consistency.yaml +++ b/lm_eval/tasks/gsm8k/gsm8k-cot-self-consistency.yaml @@ -1,5 +1,5 @@ include: gsm8k-cot.yaml -group: +tag: - chain_of_thought - self_consistency task: gsm8k_cot_self_consistency diff --git a/lm_eval/tasks/gsm8k/gsm8k-cot-zeroshot.yaml b/lm_eval/tasks/gsm8k/gsm8k-cot-zeroshot.yaml index 75d4468ac02d551d135ef78a752aba0d157e72ab..c112d324acf707e5934432068abd2ad6143438ac 100644 --- a/lm_eval/tasks/gsm8k/gsm8k-cot-zeroshot.yaml +++ b/lm_eval/tasks/gsm8k/gsm8k-cot-zeroshot.yaml @@ -1,4 +1,4 @@ -group: +tag: - math_word_problems task: gsm8k_cot_zeroshot dataset_path: gsm8k diff --git a/lm_eval/tasks/gsm8k/gsm8k-cot.yaml b/lm_eval/tasks/gsm8k/gsm8k-cot.yaml index 5947fb75257023c629e0427ef8cb3a35e66fdc1e..d125b0198535122fd5b12a388e903b03ee5f6020 100644 --- a/lm_eval/tasks/gsm8k/gsm8k-cot.yaml +++ b/lm_eval/tasks/gsm8k/gsm8k-cot.yaml @@ -61,7 +61,7 @@ generation_kwargs: - 'Q:' - - <|im_end|> -group: +tag: - chain_of_thought metadata: version: 3.0 diff --git a/lm_eval/tasks/gsm8k/gsm8k.yaml b/lm_eval/tasks/gsm8k/gsm8k.yaml index 2c4ef836b1b21177d40c10e410cf69051c98e9e3..a9d5bb39aedc0e2b991f0d79f2de6face47a31cf 100644 --- a/lm_eval/tasks/gsm8k/gsm8k.yaml +++ b/lm_eval/tasks/gsm8k/gsm8k.yaml @@ -1,4 +1,4 @@ -group: +tag: - math_word_problems task: gsm8k dataset_path: gsm8k diff --git a/lm_eval/tasks/gsm_plus/README.md b/lm_eval/tasks/gsm_plus/README.md new file mode 100644 index 0000000000000000000000000000000000000000..173a8a5e8225c9c69314d93241c4304802b54bc6 --- /dev/null +++ b/lm_eval/tasks/gsm_plus/README.md @@ -0,0 +1,48 @@ +# gsm_plus + +### Paper + +Title: `GSM-PLUS: A Comprehensive Benchmark for Evaluating the Robustness of LLMs as Mathematical Problem Solvers` + +Abstract: `Large language models (LLMs) have achieved impressive performance across various mathematical reasoning benchmarks. However, there are increasing debates regarding whether these models truly understand and apply mathematical knowledge or merely rely on shortcuts for mathematical reasoning. One essential and frequently occurring evidence is that when the math questions are slightly changed, LLMs can behave incorrectly. This motivates us to evaluate the robustness of LLMs’ math reasoning capability by testing a wide range of question variations. We introduce the adversarial grade school math (GSM-PLUS) dataset, an extension of GSM8K augmented with various mathematical perturbations. Our experiments on 25 LLMs and 4 prompting techniques show that while LLMs exhibit different levels of math reasoning abilities, their performances are far from robust. In particular, even for problems that have been solved in GSM8K, LLMs can make mistakes when new statements are added or the question targets are altered. We also explore whether more robust performance can be achieved by composing existing prompting methods, in which we try an iterative method that generates and verifies each intermediate thought based on its reasoning goal and calculation result.` + +Homepage: https://huggingface.co/datasets/qintongli/GSM-Plus + +### Citation + +```bibtex +@misc{li2024gsmpluscomprehensivebenchmarkevaluating, + title={GSM-Plus: A Comprehensive Benchmark for Evaluating the Robustness of LLMs as Mathematical Problem Solvers}, + author={Qintong Li and Leyang Cui and Xueliang Zhao and Lingpeng Kong and Wei Bi}, + year={2024}, + eprint={2402.19255}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2402.19255}, +} +``` + +### Groups and Tasks + +#### Groups + +* Not part of a group yet + +#### Tasks + +The following tasks evaluate subjects in the gsm_plus dataset +- `gsm_plus` +- `gsm_plus_mini` + +### Checklist + +For adding novel benchmarks/datasets to the library: +* [x] Is the task an existing benchmark in the literature? + * [x] Have you referenced the original paper that introduced the task? + * [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + + +If other tasks on this dataset are already supported: +* [ ] Is the "Main" variant of this task clearly denoted? +* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [ ] Have you noted which, if any, published evaluation setups are matched by this variant? diff --git a/lm_eval/tasks/gsm_plus/gsm_plus.yaml b/lm_eval/tasks/gsm_plus/gsm_plus.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6eb08f1ad8a543dc07d89ee02aed7d9e986c844b --- /dev/null +++ b/lm_eval/tasks/gsm_plus/gsm_plus.yaml @@ -0,0 +1,44 @@ +tag: + - math_word_problems +task: gsm_plus +dataset_path: qintongli/GSM-Plus +output_type: generate_until +training_split: test +fewshot_split: test +test_split: test +doc_to_text: "Question: {{question}}\nAnswer:" +doc_to_target: "{{solution}}" +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: false + regexes_to_ignore: + - "," + - "\\$" + - "(?s).*#### " + - "\\.$" +generation_kwargs: + until: + - "Question:" + - "" + - "<|im_end|>" + do_sample: false + temperature: 0.0 +repeats: 1 +num_fewshot: 5 +filter_list: + - name: "strict-match" + filter: + - function: "regex" + regex_pattern: "#### (\\-?[0-9\\.\\,]+)" + - function: "take_first" + - name: "flexible-extract" + filter: + - function: "regex" + group_select: -1 + regex_pattern: "(-?[$0-9.,]{2,})|(-?[0-9]+)" + - function: "take_first" +metadata: + version: 1.0 diff --git a/lm_eval/tasks/gsm_plus/gsm_plus_mini.yaml b/lm_eval/tasks/gsm_plus/gsm_plus_mini.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b03cfd2ea8dd639c1fc2135fc288cd2f7367cd9f --- /dev/null +++ b/lm_eval/tasks/gsm_plus/gsm_plus_mini.yaml @@ -0,0 +1,44 @@ +tag: + - math_word_problems +task: gsm_plus_mini +dataset_path: qintongli/GSM-Plus +output_type: generate_until +training_split: testmini +fewshot_split: testmini +test_split: testmini +doc_to_text: "Question: {{question}}\nAnswer:" +doc_to_target: "{{solution}}" +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: false + regexes_to_ignore: + - "," + - "\\$" + - "(?s).*#### " + - "\\.$" +generation_kwargs: + until: + - "Question:" + - "" + - "<|im_end|>" + do_sample: false + temperature: 0.0 +repeats: 1 +num_fewshot: 5 +filter_list: + - name: "strict-match" + filter: + - function: "regex" + regex_pattern: "#### (\\-?[0-9\\.\\,]+)" + - function: "take_first" + - name: "flexible-extract" + filter: + - function: "regex" + group_select: -1 + regex_pattern: "(-?[$0-9.,]{2,})|(-?[0-9]+)" + - function: "take_first" +metadata: + version: 1.0 diff --git a/lm_eval/tasks/haerae/_default_haerae_yaml b/lm_eval/tasks/haerae/_default_haerae_yaml index c22e3d15d1c2e13754bf4902125f51021bd63d82..807c10e0850078de24227d2738093e4511079690 100644 --- a/lm_eval/tasks/haerae/_default_haerae_yaml +++ b/lm_eval/tasks/haerae/_default_haerae_yaml @@ -1,4 +1,3 @@ -group: haerae dataset_path: HAERAE-HUB/HAE_RAE_BENCH test_split: test fewshot_split: test diff --git a/lm_eval/tasks/haerae/_haerae.yaml b/lm_eval/tasks/haerae/_haerae.yaml new file mode 100644 index 0000000000000000000000000000000000000000..acf413fb8293c5f32001305e09d25dfd6b1dfc5d --- /dev/null +++ b/lm_eval/tasks/haerae/_haerae.yaml @@ -0,0 +1,16 @@ +group: haerae +task: + - haerae_general_knowledge + - haerae_history + - haerae_loan_word + - haerae_rare_word + - haerae_standard_nomenclature +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true + - metric: acc_norm + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/haerae/haerae_gk.yaml b/lm_eval/tasks/haerae/haerae_gk.yaml index 2ccd26109f7ad2660e6d2d167d5e2020b7a295b9..97d2bd71d8b9a333a16a3065a14276e0b49926da 100644 --- a/lm_eval/tasks/haerae/haerae_gk.yaml +++ b/lm_eval/tasks/haerae/haerae_gk.yaml @@ -1,3 +1,3 @@ -"dataset_name": "general_knowledge" -"include": "_default_haerae_yaml" -"task": "haerae_general_knowledge" +dataset_name: general_knowledge +include: _default_haerae_yaml +task: haerae_general_knowledge diff --git a/lm_eval/tasks/haerae/haerae_hi.yaml b/lm_eval/tasks/haerae/haerae_hi.yaml index 15f77efdd1892f3c49784a2660e646d087c4668e..ed366912762cf7d1784fdef9103223fc82ab4c70 100644 --- a/lm_eval/tasks/haerae/haerae_hi.yaml +++ b/lm_eval/tasks/haerae/haerae_hi.yaml @@ -1,3 +1,3 @@ -"dataset_name": "history" -"include": "_default_haerae_yaml" -"task": "haerae_history" +dataset_name: history +include: _default_haerae_yaml +task: haerae_history diff --git a/lm_eval/tasks/haerae/haerae_lw.yaml b/lm_eval/tasks/haerae/haerae_lw.yaml index af143afe7ddb5442e707e1215e2f3bf4f13d7a08..1cb572784f06fa9b1299d8c2c817bc9541ce646b 100644 --- a/lm_eval/tasks/haerae/haerae_lw.yaml +++ b/lm_eval/tasks/haerae/haerae_lw.yaml @@ -1,3 +1,3 @@ -"dataset_name": "loan_words" -"include": "_default_haerae_yaml" -"task": "haerae_loan_word" +dataset_name: loan_words +include: _default_haerae_yaml +task: haerae_loan_word diff --git a/lm_eval/tasks/haerae/haerae_rw.yaml b/lm_eval/tasks/haerae/haerae_rw.yaml index bc8d182e61cd2deafdd5a34385132446370da90c..cfa2d1cf472e0b5a58231d6519f8e62bb8295c5c 100644 --- a/lm_eval/tasks/haerae/haerae_rw.yaml +++ b/lm_eval/tasks/haerae/haerae_rw.yaml @@ -1,3 +1,3 @@ -"dataset_name": "rare_words" -"include": "_default_haerae_yaml" -"task": "haerae_rare_word" +dataset_name: rare_words +include: _default_haerae_yaml +task: haerae_rare_word diff --git a/lm_eval/tasks/haerae/haerae_sn.yaml b/lm_eval/tasks/haerae/haerae_sn.yaml index 62603ba1c8e4f6f01d7028aa1dfe804c67ba3b99..66bf43e060e7af557274653ccbe626194f91c94e 100644 --- a/lm_eval/tasks/haerae/haerae_sn.yaml +++ b/lm_eval/tasks/haerae/haerae_sn.yaml @@ -1,3 +1,3 @@ -"dataset_name": "standard_nomenclature" -"include": "_default_haerae_yaml" -"task": "haerae_standard_nomenclature" +dataset_name: standard_nomenclature +include: _default_haerae_yaml +task: haerae_standard_nomenclature diff --git a/lm_eval/tasks/headqa/headqa_en.yaml b/lm_eval/tasks/headqa/headqa_en.yaml index eeb2ff12dd4c05b08c199692c3e868b6b50fc362..3d1b0b3e015240cc3285ffe67c167901bfa385e1 100644 --- a/lm_eval/tasks/headqa/headqa_en.yaml +++ b/lm_eval/tasks/headqa/headqa_en.yaml @@ -1,5 +1,4 @@ -group: - - headqa +tag: headqa task: headqa_en dataset_path: EleutherAI/headqa dataset_name: en diff --git a/lm_eval/tasks/hellaswag/hellaswag.yaml b/lm_eval/tasks/hellaswag/hellaswag.yaml index ec627da7d46ea6f31bd0ca68c60e21fd9332db9d..c0735bbd97773488d99c047ab1342ad65aca4142 100644 --- a/lm_eval/tasks/hellaswag/hellaswag.yaml +++ b/lm_eval/tasks/hellaswag/hellaswag.yaml @@ -1,4 +1,4 @@ -group: +tag: - multiple_choice task: hellaswag dataset_path: hellaswag @@ -20,3 +20,5 @@ metric_list: higher_is_better: true metadata: version: 1.0 +dataset_kwargs: + trust_remote_code: true diff --git a/lm_eval/tasks/hendrycks_ethics/commonsense.yaml b/lm_eval/tasks/hendrycks_ethics/commonsense.yaml index f1270743670604108d852e75116ec584938eb976..d4f134375c24617215121bda5e43a98f8b88e1b2 100644 --- a/lm_eval/tasks/hendrycks_ethics/commonsense.yaml +++ b/lm_eval/tasks/hendrycks_ethics/commonsense.yaml @@ -1,4 +1,4 @@ -group: +tag: - hendrycks_ethics task: ethics_cm dataset_path: EleutherAI/hendrycks_ethics diff --git a/lm_eval/tasks/hendrycks_ethics/justice.yaml b/lm_eval/tasks/hendrycks_ethics/justice.yaml index 2ba79d16b3e6108392d987f17cd2698555f4fb80..a67f55c708f4991fb67935787aa9a952a32178cd 100644 --- a/lm_eval/tasks/hendrycks_ethics/justice.yaml +++ b/lm_eval/tasks/hendrycks_ethics/justice.yaml @@ -1,5 +1,5 @@ include: deontology.yaml -group: +tag: - hendrycks_ethics task: ethics_justice dataset_name: justice diff --git a/lm_eval/tasks/hendrycks_ethics/utilitarianism.yaml b/lm_eval/tasks/hendrycks_ethics/utilitarianism.yaml index 8960a319349067a5ba2542a93e1447cce7899e72..81134de900dc37d571db2ea53a4746022c8bceac 100644 --- a/lm_eval/tasks/hendrycks_ethics/utilitarianism.yaml +++ b/lm_eval/tasks/hendrycks_ethics/utilitarianism.yaml @@ -1,5 +1,5 @@ include: commonsense.yaml -group: +tag: - hendrycks_ethics task: ethics_utilitarianism dataset_name: utilitarianism diff --git a/lm_eval/tasks/hendrycks_ethics/virtue.yaml b/lm_eval/tasks/hendrycks_ethics/virtue.yaml index 8c236a983dd08c4d43aa072713984a113330534b..b456e4a5a49a4f3cd626e4cca48154032e08367f 100644 --- a/lm_eval/tasks/hendrycks_ethics/virtue.yaml +++ b/lm_eval/tasks/hendrycks_ethics/virtue.yaml @@ -1,5 +1,5 @@ include: commonsense.yaml -group: +tag: - hendrycks_ethics task: ethics_virtue dataset_name: virtue diff --git a/lm_eval/tasks/hendrycks_math/hendrycks_math.yaml b/lm_eval/tasks/hendrycks_math/hendrycks_math.yaml index b30356540174871c957ade780dbf33aa4784879c..d01cf9b2465b4e825ed8b5c67fe0aca281c31781 100644 --- a/lm_eval/tasks/hendrycks_math/hendrycks_math.yaml +++ b/lm_eval/tasks/hendrycks_math/hendrycks_math.yaml @@ -7,3 +7,9 @@ task: - hendrycks_math_num_theory - hendrycks_math_prealgebra - hendrycks_math_precalc +aggregate_metric_list: + - metric: exact_match + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/hendrycks_math/hendrycks_math_algebra.yaml b/lm_eval/tasks/hendrycks_math/hendrycks_math_algebra.yaml index 66f31fd992923003a852186714027e1bb3711480..9ce9c9c5ad64d894732054caa679d73b1df23881 100644 --- a/lm_eval/tasks/hendrycks_math/hendrycks_math_algebra.yaml +++ b/lm_eval/tasks/hendrycks_math/hendrycks_math_algebra.yaml @@ -1,4 +1,4 @@ -group: +tag: - math_word_problems task: hendrycks_math_algebra dataset_path: EleutherAI/hendrycks_math diff --git a/lm_eval/tasks/ifeval/ifeval.yaml b/lm_eval/tasks/ifeval/ifeval.yaml index 97e760144a517d723aca5371ae568ae28ab6dcf6..508a63a9452874109cd949f5d5a5e00ad5f66b36 100644 --- a/lm_eval/tasks/ifeval/ifeval.yaml +++ b/lm_eval/tasks/ifeval/ifeval.yaml @@ -1,5 +1,5 @@ task: ifeval -dataset_path: wis-k/instruction-following-eval +dataset_path: google/IFEval dataset_name: null output_type: generate_until test_split: train @@ -26,4 +26,4 @@ metric_list: aggregation: !function utils.agg_inst_level_acc higher_is_better: true metadata: - version: 2.0 + version: 4.0 diff --git a/lm_eval/tasks/ifeval/instructions.py b/lm_eval/tasks/ifeval/instructions.py index 5c352af3f70e8e25561a755b877f36ce28b13c11..a79cbba4f5e2fec10b8c0525a215b5e967c6fcfc 100644 --- a/lm_eval/tasks/ifeval/instructions.py +++ b/lm_eval/tasks/ifeval/instructions.py @@ -78,7 +78,7 @@ _ENDING_OPTIONS = ("Any other questions?", "Is there anything else I can help wi # The number of highlighted sections. _NUM_HIGHLIGHTED_SECTIONS = 4 -# The section spliter. +# The section splitter. _SECTION_SPLITER = ("Section", "SECTION") # The number of sections. @@ -153,7 +153,7 @@ class ResponseLanguageChecker(Instruction): return self._description_pattern.format(language=_LANGUAGES[self._language]) def get_instruction_args(self): - """Returns the keyward args of `build_description`.""" + """Returns the keyword args of `build_description`.""" return {"language": self._language} def get_instruction_args_keys(self): @@ -223,7 +223,7 @@ class NumberOfSentences(Instruction): ) def get_instruction_args(self): - """Returns the keyward args of `build_description`.""" + """Returns the keyword args of `build_description`.""" return { "num_sentences": self._num_sentences_threshold, "relation": self._comparison_relation, @@ -276,7 +276,7 @@ class PlaceholderChecker(Instruction): return self._description_pattern.format(num_placeholders=self._num_placeholders) def get_instruction_args(self): - """Returns the keyward args of `build_description`.""" + """Returns the keyword args of `build_description`.""" return {"num_placeholders": self._num_placeholders} def get_instruction_args_keys(self): @@ -323,7 +323,7 @@ class BulletListChecker(Instruction): return self._description_pattern.format(num_bullets=self._num_bullets) def get_instruction_args(self): - """Returns the keyward args of `build_description`.""" + """Returns the keyword args of `build_description`.""" return {"num_bullets": self._num_bullets} def get_instruction_args_keys(self): @@ -362,7 +362,7 @@ class ConstrainedResponseChecker(Instruction): ) def get_instruction_args(self): - """Returns the keyward args of `build_description`.""" + """Returns the keyword args of `build_description`.""" return None def get_instruction_args_keys(self): @@ -393,7 +393,7 @@ class ConstrainedStartChecker(Instruction): """Build the instruction description. Args: - starter: A string representing the keyward that the response should start + starter: A string representing the keyword that the response should start with. Returns: @@ -409,7 +409,7 @@ class ConstrainedStartChecker(Instruction): return self._description_pattern.format(starter=self._starter) def get_instruction_args(self): - """Returns the keyward args of `build_description`.""" + """Returns the keyword args of `build_description`.""" return {"starter": self._starter} def get_instruction_args_keys(self): @@ -458,7 +458,7 @@ class HighlightSectionChecker(Instruction): return self._description_pattern.format(num_highlights=self._num_highlights) def get_instruction_args(self): - """Returns the keyward args of `build_description`.""" + """Returns the keyword args of `build_description`.""" return {"num_highlights": self._num_highlights} def get_instruction_args_keys(self): @@ -469,12 +469,12 @@ class HighlightSectionChecker(Instruction): """Checks if the number of highlighted sections meets the requirement. Args: - value: a string repesenting the response. The response is expected to + value: a string representing the response. The response is expected to contain highlighted sections in the format of *highlighted*. Returns: True if the actual number of highlighted sections in the format of - *highlighed sections* meets the minimum requirement; otherwise False. + *highlighted sections* meets the minimum requirement; otherwise False. """ num_highlights = 0 highlights = re.findall(r"\*[^\n\*]*\*", value) @@ -529,7 +529,7 @@ class SectionChecker(Instruction): ) def get_instruction_args(self): - """Returns the keyward args of `build_description`.""" + """Returns the keyword args of `build_description`.""" return { "section_spliter": self._section_spliter, "num_sections": self._num_sections, @@ -582,7 +582,7 @@ class ParagraphChecker(Instruction): return self._description_pattern.format(num_paragraphs=self._num_paragraphs) def get_instruction_args(self): - """Returns the keyward args of `build_description`.""" + """Returns the keyword args of `build_description`.""" return {"num_paragraphs": self._num_paragraphs} def get_instruction_args_keys(self): @@ -642,7 +642,7 @@ class PostscriptChecker(Instruction): return self._description_pattern.format(postscript=self._postscript_marker) def get_instruction_args(self): - """Returns the keyward args of `build_description`.""" + """Returns the keyword args of `build_description`.""" return {"postscript_marker": self._postscript_marker} def get_instruction_args_keys(self): @@ -672,7 +672,7 @@ class PostscriptChecker(Instruction): class RephraseChecker(Instruction): - """Checks the repharse.""" + """Checks the rephrase.""" def build_description(self, *, original_message): """Build the instruction description. @@ -701,7 +701,7 @@ class RephraseChecker(Instruction): return self._description def get_instruction_args(self): - """Returns the keyward args of `build_description`.""" + """Returns the keyword args of `build_description`.""" return {"original_message": self._reference_without_change} def get_instruction_args_keys(self): @@ -766,7 +766,7 @@ class KeywordChecker(Instruction): return self._description_pattern.format(keywords=self._keywords) def get_instruction_args(self): - """Returns the keyward args of `build_description`.""" + """Returns the keyword args of `build_description`.""" return {"keywords": self._keywords} def get_instruction_args_keys(self): @@ -831,7 +831,7 @@ class KeywordFrequencyChecker(Instruction): ) def get_instruction_args(self): - """Returns the keyward args of `build_description`.""" + """Returns the keyword args of `build_description`.""" return { "keyword": self._keyword, "frequency": self._frequency, @@ -894,7 +894,7 @@ class NumberOfWords(Instruction): ) def get_instruction_args(self): - """Returns the keyward args of `build_description`.""" + """Returns the keyword args of `build_description`.""" return {"num_words": self._num_words, "relation": self._comparison_relation} def get_instruction_args_keys(self): @@ -922,7 +922,7 @@ class JsonFormat(Instruction): return self._description_pattern def get_instruction_args(self): - """Returns the keyward args of `build_description`.""" + """Returns the keyword args of `build_description`.""" return None def get_instruction_args_keys(self): @@ -996,7 +996,7 @@ class ParagraphFirstWordCheck(Instruction): ) def get_instruction_args(self): - """Returns the keyward args of `build_description`.""" + """Returns the keyword args of `build_description`.""" return { "num_paragraphs": self._num_paragraphs, "nth_paragraph": self._nth_paragraph, @@ -1089,7 +1089,7 @@ class KeySentenceChecker(Instruction): ) def get_instruction_args(self): - """Returns the keyward args of `build_description`.""" + """Returns the keyword args of `build_description`.""" return { "num_sentences": self._num_sentences, "key_sentences": list(self._key_sentences), @@ -1117,7 +1117,7 @@ class ForbiddenWords(Instruction): """Build the instruction description. Args: - forbidden_words: A sequences of strings respresenting words that are not + forbidden_words: A sequences of strings representing words that are not allowed in the response. Returns: @@ -1138,7 +1138,7 @@ class ForbiddenWords(Instruction): return self._description_pattern.format(forbidden_words=self._forbidden_words) def get_instruction_args(self): - """Returns the keyward args of `build_description`.""" + """Returns the keyword args of `build_description`.""" return {"forbidden_words": self._forbidden_words} def get_instruction_args_keys(self): @@ -1188,7 +1188,7 @@ class RephraseParagraph(Instruction): ) def get_instruction_args(self): - """Returns the keyward args of `build_description`.""" + """Returns the keyword args of `build_description`.""" return { "original_paragraph": self._original_paragraph, "low": self._low, @@ -1225,7 +1225,7 @@ class TwoResponsesChecker(Instruction): return self._description_pattern def get_instruction_args(self): - """Returns the keyward args of `build_description`.""" + """Returns the keyword args of `build_description`.""" return None def get_instruction_args_keys(self): diff --git a/lm_eval/tasks/ifeval/instructions_util.py b/lm_eval/tasks/ifeval/instructions_util.py index ccb531f96e6e6c4313d8bf91786a4ce87d6c5926..df58fb30209a86cd461fc29ca37ba89b4e54d583 100644 --- a/lm_eval/tasks/ifeval/instructions_util.py +++ b/lm_eval/tasks/ifeval/instructions_util.py @@ -15,19 +15,37 @@ """Utility library of instructions.""" import functools +import os import random import re +from importlib.metadata import version import immutabledict import nltk +from packaging.version import parse as parse_version + + +# Downloading 'punkt' with nltk<3.9 has a remote code vuln. +# see https://github.com/EleutherAI/lm-evaluation-harness/issues/2210 +# and https://github.com/nltk/nltk/issues/3266 +# for more information. +NLTK_MIN_VERSION = "3.9.1" +RANK = os.environ.get("LOCAL_RANK", "0") def download_nltk_resources(): """Download 'punkt' if not already installed""" + assert ( + (nltk_version := parse_version(version("nltk"))) + >= parse_version(NLTK_MIN_VERSION) + ), f"`nltk` version {nltk_version} is not >= {NLTK_MIN_VERSION}. Please update `nltk` before proceeding--older versions are vulnerable to a remote code execution vulnerability." + try: - nltk.data.find("tokenizers/punkt") + nltk.data.find("tokenizers/punkt_tab") except LookupError: - nltk.download("punkt") + if RANK == "0": + nltk.download("punkt_tab") + print("Downloaded punkt_tab on rank 0") download_nltk_resources() diff --git a/lm_eval/tasks/ifeval/utils.py b/lm_eval/tasks/ifeval/utils.py index 31dcb01e7eadc00dd7c46aa9f05bd837d94dae80..c1e91e3353cd3676f3412a73bb7fab36fdcec522 100644 --- a/lm_eval/tasks/ifeval/utils.py +++ b/lm_eval/tasks/ifeval/utils.py @@ -110,10 +110,6 @@ def test_instruction_following_loose( def process_results(doc, results): - eval_logger.warning( - "This task is meant for chat-finetuned models, and may not give meaningful results for models other than `openai` or `anthropic` if `doc_to_text` in its YAML is not wrapped in the appropriate chat template string. This warning will be removed when chat templating support is added natively to local models" - ) - inp = InputExample( key=doc["key"], instruction_id_list=doc["instruction_id_list"], diff --git a/lm_eval/tasks/inverse_scaling/README.md b/lm_eval/tasks/inverse_scaling/README.md new file mode 100644 index 0000000000000000000000000000000000000000..10364d8ea8edf4972eb1ec991e8bc29137b87c0b --- /dev/null +++ b/lm_eval/tasks/inverse_scaling/README.md @@ -0,0 +1,55 @@ +# inverse_scaling + +### Paper + +Title: `Inverse Scaling: When Bigger Isn't Better` + +Abstract: `Work on scaling laws has found that large language models (LMs) show predictable improvements to overall loss with increased scale (model size, training data, and compute). Here, we present evidence for the claim that LMs may show inverse scaling, or worse task performance with increased scale, e.g., due to flaws in the training objective and data. We present empirical evidence of inverse scaling on 11 datasets collected by running a public contest, the Inverse Scaling Prize, with a substantial prize pool. Through analysis of the datasets, along with other examples found in the literature, we identify four potential causes of inverse scaling: (i) preference to repeat memorized sequences over following in-context instructions, (ii) imitation of undesirable patterns in the training data, (iii) tasks containing an easy distractor task which LMs could focus on, rather than the harder real task, and (iv) correct but misleading few-shot demonstrations of the task. We release the winning datasets at this https URL to allow for further investigation of inverse scaling. Our tasks have helped drive the discovery of U-shaped and inverted-U scaling trends, where an initial trend reverses, suggesting that scaling trends are less reliable at predicting the behavior of larger-scale models than previously understood. Overall, our results suggest that there are tasks for which increased model scale alone may not lead to progress, and that more careful thought needs to go into the data and objectives for training language models.` + +Note: This is not official implementation of inverse scaling prize. Implemented by h-albert-lee with permission from the authors of the paper. + +Homepage: https://github.com/inverse-scaling/prize + +### Citation + +@article{mckenzie2023inverse, + title={Inverse Scaling: When Bigger Isn't Better}, + author={Ian R. McKenzie and Alexander Lyzhov and Michael Pieler and Alicia Parrish and Aaron Mueller and Ameya Prabhu and Euan McLean and Aaron Kirtland and Alexis Ross and Alisa Liu and Andrew Gritsevskiy and Daniel Wurgaft and Derik Kauffman and Gabriel Recchia and Jiacheng Liu and Joe Cavanagh and Max Weiss and Sicong Huang and The Floating Droid and Tom Tseng and Tomasz Korbak and Xudong Shen and Yuhui Zhang and Zhengping Zhou and Najoung Kim and Samuel R. Bowman and Ethan Perez}, + journal={arXiv preprint arXiv:2306.09479}, + year={2023} +} + +### Groups and Tasks + +#### Groups + +* `inverse_scaling_mc`: all tasks of Inverse Scaling Prize (currently aside from Prompt Injection), matching their implementations on OPT for multiple-choice type classification tasks. **These match the published dataset versions from the prize, which may slightly differ from numbers in the paper (but have been tested for equivalence to the OPT numbers reported at https://huggingface.co/inverse-scaling/opt-1.3b_eval for multiple sizes.** + + +#### Tasks + +- `inverse_scaling_hindsight_neglect_10shot` +- `inverse_scaling_redefine_math` +- `inverse_scaling_quote_repetition` +- `inverse_scaling_neqa` +- `inverse_scaling_winobias_antistereotype`: not an official Inverse Scaling prize winner, but eval results reported on it at https://huggingface.co/inverse-scaling/opt-1.3b_eval . +- `inverse_scaling_into_the_unknown` +- `inverse_scaling_memo_trap` +- `inverse_scaling_modus_tollens` +- `inverse_scaling_pattern_matching_suppression` +- `inverse_scaling_repetitive_algebra` +- `inverse_scaling_sig_figs` + + +### Checklist + +For adding novel benchmarks/datasets to the library: +* [x] Is the task an existing benchmark in the literature? + * [x] Have you referenced the original paper that introduced the task? + * [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + + +If other tasks on this dataset are already supported: +* [ ] Is the "Main" variant of this task clearly denoted? +* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [ ] Have you noted which, if any, published evaluation setups are matched by this variant? diff --git a/lm_eval/tasks/inverse_scaling/_inverse_scaling_mc_yaml b/lm_eval/tasks/inverse_scaling/_inverse_scaling_mc_yaml new file mode 100644 index 0000000000000000000000000000000000000000..c504eb2cd8833cfef70b3d60748a1769829ac11a --- /dev/null +++ b/lm_eval/tasks/inverse_scaling/_inverse_scaling_mc_yaml @@ -0,0 +1,17 @@ +tag: + - inverse_scaling_mc +output_type: multiple_choice +test_split: train +doc_to_text: prompt +doc_to_choice: classes +doc_to_target: answer_index +target_delimiter: "" +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 0 diff --git a/lm_eval/tasks/inverse_scaling/_some_results b/lm_eval/tasks/inverse_scaling/_some_results new file mode 100644 index 0000000000000000000000000000000000000000..9afe58d8e02cdc0cee5a756c8e24aaeb6f2e87cf --- /dev/null +++ b/lm_eval/tasks/inverse_scaling/_some_results @@ -0,0 +1,39 @@ +# | Tasks |Version|Filter|n-shot| Metric |Value | |Stderr| +# |-------------------------------------------|-------|------|-----:|--------|-----:|---|-----:| +# | - inverse_scaling_hindsight_neglect_10shot| 0|none | 0|acc |0.4476|± |0.0281| +# | | |none | 0|acc_norm|0.4476|± |0.0281| +# |inverse_scaling_mc |N/A |none | 0|acc_norm|0.6273|± |0.0096| +# | | |none | 0|acc |0.6210|± |0.0095| +# | - inverse_scaling_neqa | 0|none | 0|acc |0.5300|± |0.0289| +# | | |none | 0|acc_norm|0.5300|± |0.0289| +# | - inverse_scaling_quote_repetition | 0|none | 0|acc |0.9367|± |0.0141| +# | | |none | 0|acc_norm|0.9367|± |0.0141| +# | - inverse_scaling_redefine_math | 0|none | 0|acc |0.7178|± |0.0150| +# | | |none | 0|acc_norm|0.7178|± |0.0150| +# | - inverse_scaling_winobias_antistereotype | 0|none | 0|acc |0.3786|± |0.0239| +# | | |none | 0|acc_norm|0.4126|± |0.0243| + +# | Groups |Version|Filter|n-shot| Metric |Value | |Stderr| +# |------------------|-------|------|-----:|--------|-----:|---|-----:| +# |inverse_scaling_mc|N/A |none | 0|acc_norm|0.6273|± |0.0096| +# | | |none | 0|acc |0.6210|± |0.0095| +# hf (pretrained=facebook/opt-2.7b,add_bos_token=True,dtype=float32), gen_kwargs: (None), limit: None, num_fewshot: None, batch_size: auto (32) +# | Tasks |Version|Filter|n-shot| Metric |Value | |Stderr| +# |-------------------------------------------|-------|------|-----:|--------|-----:|---|-----:| +# | - inverse_scaling_hindsight_neglect_10shot| 0|none | 0|acc |0.4476|± |0.0281| +# | | |none | 0|acc_norm|0.4476|± |0.0281| +# |inverse_scaling_mc |N/A |none | 0|acc_norm|0.6291|± |0.0095| +# | | |none | 0|acc |0.6219|± |0.0095| +# | - inverse_scaling_neqa | 0|none | 0|acc |0.5267|± |0.0289| +# | | |none | 0|acc_norm|0.5267|± |0.0289| +# | - inverse_scaling_quote_repetition | 0|none | 0|acc |0.9433|± |0.0134| +# | | |none | 0|acc_norm|0.9433|± |0.0134| +# | - inverse_scaling_redefine_math | 0|none | 0|acc |0.7200|± |0.0150| +# | | |none | 0|acc_norm|0.7200|± |0.0150| +# | - inverse_scaling_winobias_antistereotype | 0|none | 0|acc |0.3762|± |0.0239| +# | | |none | 0|acc_norm|0.4150|± |0.0243| + +# | Groups |Version|Filter|n-shot| Metric |Value | |Stderr| +# |------------------|-------|------|-----:|--------|-----:|---|-----:| +# |inverse_scaling_mc|N/A |none | 0|acc_norm|0.6291|± |0.0095| +# | | |none | 0|acc |0.6219|± |0.0095| diff --git a/lm_eval/tasks/inverse_scaling/inverse_scaling_hindsight_neglect.yaml b/lm_eval/tasks/inverse_scaling/inverse_scaling_hindsight_neglect.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b21baee22a42077ecfe0b53678b92e5daa2b1e62 --- /dev/null +++ b/lm_eval/tasks/inverse_scaling/inverse_scaling_hindsight_neglect.yaml @@ -0,0 +1,3 @@ +include: _inverse_scaling_mc_yaml +task: inverse_scaling_hindsight_neglect_10shot +dataset_path: inverse-scaling/hindsight-neglect-10shot diff --git a/lm_eval/tasks/inverse_scaling/inverse_scaling_into_the_unknown.yaml b/lm_eval/tasks/inverse_scaling/inverse_scaling_into_the_unknown.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0c7ccd3bb5cbd3d0dfe924d5fa7b22ad466ea198 --- /dev/null +++ b/lm_eval/tasks/inverse_scaling/inverse_scaling_into_the_unknown.yaml @@ -0,0 +1,3 @@ +include: _inverse_scaling_mc_yaml +task: inverse_scaling_into_the_unknown +dataset_path: Albertmade/into-the-unknown diff --git a/lm_eval/tasks/inverse_scaling/inverse_scaling_memo_trap.yaml b/lm_eval/tasks/inverse_scaling/inverse_scaling_memo_trap.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5c77ffc25077a3c9c4e287dd96cdcc24a29fa483 --- /dev/null +++ b/lm_eval/tasks/inverse_scaling/inverse_scaling_memo_trap.yaml @@ -0,0 +1,3 @@ +include: _inverse_scaling_mc_yaml +task: inverse_scaling_memo_trap +dataset_path: Albertmade/memo-trap diff --git a/lm_eval/tasks/inverse_scaling/inverse_scaling_modus_tollens.yaml b/lm_eval/tasks/inverse_scaling/inverse_scaling_modus_tollens.yaml new file mode 100644 index 0000000000000000000000000000000000000000..89e1ebbc55ecb483b4e486077373b9a0ee7cd12b --- /dev/null +++ b/lm_eval/tasks/inverse_scaling/inverse_scaling_modus_tollens.yaml @@ -0,0 +1,3 @@ +include: _inverse_scaling_mc_yaml +task: inverse_scaling_modus_tollens +dataset_path: Albertmade/modus-tollens diff --git a/lm_eval/tasks/inverse_scaling/inverse_scaling_neqa.yaml b/lm_eval/tasks/inverse_scaling/inverse_scaling_neqa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..11a057d8ed756f0d2e697c459be49f22b57dde94 --- /dev/null +++ b/lm_eval/tasks/inverse_scaling/inverse_scaling_neqa.yaml @@ -0,0 +1,3 @@ +include: _inverse_scaling_mc_yaml +task: inverse_scaling_neqa +dataset_path: inverse-scaling/NeQA diff --git a/lm_eval/tasks/inverse_scaling/inverse_scaling_pattern_matching_suppression.yaml b/lm_eval/tasks/inverse_scaling/inverse_scaling_pattern_matching_suppression.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b415bba1327741396b8d9c0d158b9fba95fa7a52 --- /dev/null +++ b/lm_eval/tasks/inverse_scaling/inverse_scaling_pattern_matching_suppression.yaml @@ -0,0 +1,3 @@ +include: _inverse_scaling_mc_yaml +task: inverse_scaling_pattern_matching_suppression +dataset_path: Albertmade/pattern-matching-suppression diff --git a/lm_eval/tasks/inverse_scaling/inverse_scaling_quote_repetition.yaml b/lm_eval/tasks/inverse_scaling/inverse_scaling_quote_repetition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6750953cd7f7de45518345d4c7e6ff2861e5926b --- /dev/null +++ b/lm_eval/tasks/inverse_scaling/inverse_scaling_quote_repetition.yaml @@ -0,0 +1,3 @@ +include: _inverse_scaling_mc_yaml +task: inverse_scaling_quote_repetition +dataset_path: inverse-scaling/quote-repetition diff --git a/lm_eval/tasks/inverse_scaling/inverse_scaling_redefine_math.yaml b/lm_eval/tasks/inverse_scaling/inverse_scaling_redefine_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2f14ba28bddc5bb1586379b179c956a21a6dcc68 --- /dev/null +++ b/lm_eval/tasks/inverse_scaling/inverse_scaling_redefine_math.yaml @@ -0,0 +1,3 @@ +include: _inverse_scaling_mc_yaml +task: inverse_scaling_redefine_math +dataset_path: inverse-scaling/redefine-math diff --git a/lm_eval/tasks/inverse_scaling/inverse_scaling_repetitive_algebra.yaml b/lm_eval/tasks/inverse_scaling/inverse_scaling_repetitive_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..048ea271edf7252b0dc0aa120032b04102685e9c --- /dev/null +++ b/lm_eval/tasks/inverse_scaling/inverse_scaling_repetitive_algebra.yaml @@ -0,0 +1,3 @@ +include: _inverse_scaling_mc_yaml +task: inverse_scaling_repetitive_algebra +dataset_path: Albertmade/repetitive-algebra diff --git a/lm_eval/tasks/inverse_scaling/inverse_scaling_sig_figs.yaml b/lm_eval/tasks/inverse_scaling/inverse_scaling_sig_figs.yaml new file mode 100644 index 0000000000000000000000000000000000000000..25d86e33a8c839bc750ad8bb83be908c1e19eb19 --- /dev/null +++ b/lm_eval/tasks/inverse_scaling/inverse_scaling_sig_figs.yaml @@ -0,0 +1,3 @@ +include: _inverse_scaling_mc_yaml +task: inverse_scaling_sig_figs +dataset_path: Albertmade/sig-figs diff --git a/lm_eval/tasks/inverse_scaling/inverse_scaling_winobias_antistereotype.yaml b/lm_eval/tasks/inverse_scaling/inverse_scaling_winobias_antistereotype.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4c5ff300aae7ee84e103dea2be053ec88fe9f771 --- /dev/null +++ b/lm_eval/tasks/inverse_scaling/inverse_scaling_winobias_antistereotype.yaml @@ -0,0 +1,20 @@ +group: + - inverse_scaling_mc +task: inverse_scaling_winobias_antistereotype +dataset_path: mathemakitten/winobias_antistereotype_test_v5 +output_type: multiple_choice +test_split: test +doc_to_text: text +doc_to_choice: classes +doc_to_target: target +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +dataset_kwargs: + trust_remote_code: true +metadata: + version: 0 diff --git a/lm_eval/tasks/japanese_leaderboard/README.md b/lm_eval/tasks/japanese_leaderboard/README.md new file mode 100644 index 0000000000000000000000000000000000000000..2c4d00a197ce331fe87b9f98f581d22d1bdc38ad --- /dev/null +++ b/lm_eval/tasks/japanese_leaderboard/README.md @@ -0,0 +1,152 @@ +# Japanese Leaderboard + +The Japanese LLM Leaderboard evaluates language models based on a wide range of NLP tasks that reflect the characteristics of the Japanese language. + +### Groups, Tags, and Tasks + +#### Groups + +* `japanese_leaderboard`: runs all tasks defined in this directory + +#### Tasks + +##### Generation Evaluation + +* `ja_leaderboard_jaqket_v2`: The JAQKET dataset is designed for Japanese question answering research, featuring quiz-like questions with answers derived from Wikipedia article titles. [Source](https://github.com/kumapo/JAQKET-dataset) +* `ja_leaderboard_mgsm`: Multilingual Grade School Math Benchmark (MGSM) is a benchmark of grade-school math problems, proposed in the paper Language models are multilingual chain-of-thought reasoners. [Source](https://huggingface.co/datasets/juletxara/mgsm) +* `ja_leaderboard_xlsum`: This is the filtered Japanese subset of XL-Sum. [Source](https://github.com/csebuetnlp/xl-sum) +* `ja_leaderboard_jsquad`: JSQuAD is a Japanese version of SQuAD, a reading comprehension dataset. Each instance in the dataset consists of a question regarding a given context (Wikipedia article) and its answer. JSQuAD is based on SQuAD 1.1 (there are no unanswerable questions). [Source](https://github.com/yahoojapan/JGLUE) + +##### Multi-Choice/Classification Evaluation + +* `ja_leaderboard_jcommonsenseqa`: JCommonsenseQA is a Japanese version of CommonsenseQA, which is a multiple-choice question answering dataset that requires commonsense reasoning ability. [Source](https://github.com/yahoojapan/JGLUE) +* `ja_leaderboard_jnli`: JNLI is a Japanese version of the NLI (Natural Language Inference) dataset. The inference relations are entailment (含意), contradiction (矛盾), and neutral (中立). [Source](https://github.com/yahoojapan/JGLUE) +* `ja_leaderboard_marc_ja`: MARC-ja is a text classification dataset based on the Japanese portion of Multilingual Amazon Reviews Corpus (MARC). [Source](https://github.com/yahoojapan/JGLUE) +* `ja_leaderboard_xwinograd`: This is the Japanese portion of XWinograd. [Source](https://huggingface.co/datasets/polm-stability/xwinograd-ja) + +### Citation + +```bibtex +@inproceedings{ja_leaderboard_jaqket_v2, + title = {JAQKET: クイズを題材にした日本語 QA データセットの構築}, + author = {鈴木正敏 and 鈴木潤 and 松田耕史 and ⻄田京介 and 井之上直也}, + year = 2020, + booktitle = {言語処理学会第26回年次大会}, + url = {https://www.anlp.jp/proceedings/annual_meeting/2020/pdf_dir/P2-24.pdf} +} + +@article{ja_leaderboard_mgsm_1, + title = {Training Verifiers to Solve Math Word Problems}, + author = { + Cobbe, Karl and Kosaraju, Vineet and Bavarian, Mohammad and Chen, Mark and Jun, Heewoo and + Kaiser, Lukasz and Plappert, Matthias and Tworek, Jerry and Hilton, Jacob and Nakano, Reiichiro + and Hesse, Christopher and Schulman, John + }, + year = 2021, + journal = {arXiv preprint arXiv:2110.14168} +} + +@misc{ja_leaderboard_mgsm_2, + title = {Language Models are Multilingual Chain-of-Thought Reasoners}, + author = { + Freda Shi and Mirac Suzgun and Markus Freitag and Xuezhi Wang and Suraj Srivats and Soroush + Vosoughi and Hyung Won Chung and Yi Tay and Sebastian Ruder and Denny Zhou and Dipanjan Das and + Jason Wei + }, + year = 2022, + eprint = {2210.03057}, + archiveprefix = {arXiv}, + primaryclass = {cs.CL} +} + +@inproceedings{ja_leaderboard_xlsum, + title = {{XL}-Sum: Large-Scale Multilingual Abstractive Summarization for 44 Languages}, + author = { + Hasan, Tahmid and Bhattacharjee, Abhik and Islam, Md. Saiful and Mubasshir, Kazi and Li, + Yuan-Fang and Kang, Yong-Bin and Rahman, M. Sohel and Shahriyar, Rifat + }, + year = 2021, + month = aug, + booktitle = {Findings of the Association for Computational Linguistics: ACL-IJCNLP 2021}, + publisher = {Association for Computational Linguistics}, + address = {Online}, + pages = {4693--4703}, + url = {https://aclanthology.org/2021.findings-acl.413} +} + +@article{jglue_2023, + title = {JGLUE: 日本語言語理解ベンチマーク}, + author = {栗原 健太郎 and 河原 大輔 and 柴田 知秀}, + year = 2023, + journal = {自然言語処理}, + volume = 30, + number = 1, + pages = {63--87}, + doi = {10.5715/jnlp.30.63}, + url = {https://www.jstage.jst.go.jp/article/jnlp/30/1/30_63/_article/-char/ja} +} + +@inproceedings{jglue_kurihara-etal-2022-jglue, + title = {{JGLUE}: {J}apanese General Language Understanding Evaluation}, + author = {Kurihara, Kentaro and Kawahara, Daisuke and Shibata, Tomohide}, + year = 2022, + month = jun, + booktitle = {Proceedings of the Thirteenth Language Resources and Evaluation Conference}, + publisher = {European Language Resources Association}, + address = {Marseille, France}, + pages = {2957--2966}, + url = {https://aclanthology.org/2022.lrec-1.317} +} + +@inproceedings{jglue_kurihara_nlp2022, + title = {JGLUE: 日本語言語理解ベンチマーク}, + author = {栗原健太郎 and 河原大輔 and 柴田知秀}, + year = 2022, + booktitle = {言語処理学会第28回年次大会}, + url = {https://www.anlp.jp/proceedings/annual_meeting/2022/pdf_dir/E8-4.pdf}, + note = {in Japanese} +} + +@misc{xwinograd_muennighoff2022crosslingual, + title = {Crosslingual Generalization through Multitask Finetuning}, + author = { + Niklas Muennighoff and Thomas Wang and Lintang Sutawika and Adam Roberts and Stella Biderman + and Teven Le Scao and M Saiful Bari and Sheng Shen and Zheng-Xin Yong and Hailey Schoelkopf and + Xiangru Tang and Dragomir Radev and Alham Fikri Aji and Khalid Almubarak and Samuel Albanie and + Zaid Alyafeai and Albert Webson and Edward Raff and Colin Raffel + }, + year = 2022, + eprint = {2211.01786}, + archiveprefix = {arXiv}, + primaryclass = {cs.CL} +} + +@misc{xwinograd_tikhonov2021heads, + title = { + It's All in the Heads: Using Attention Heads as a Baseline for Cross-Lingual Transfer in + Commonsense Reasoning + }, + author = {Alexey Tikhonov and Max Ryabinin}, + year = 2021, + eprint = {2106.12066}, + archiveprefix = {arXiv}, + primaryclass = {cs.CL} +} +``` + +### Credit + +* Prompts: https://github.com/Stability-AI/lm-evaluation-harness/tree/jp-stable/lm_eval/tasks/ja + +### Checklist + +For adding novel benchmarks/datasets to the library: +* [x] Is the task an existing benchmark in the literature? + * [x] Have you referenced the original paper that introduced the task? + * [ ] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + + +If other tasks on this dataset are already supported: +* [ ] Is the "Main" variant of this task clearly denoted? +* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [ ] Have you noted which, if any, published evaluation setups are matched by this variant? diff --git a/lm_eval/tasks/japanese_leaderboard/_ja_leaderboard.yaml b/lm_eval/tasks/japanese_leaderboard/_ja_leaderboard.yaml new file mode 100644 index 0000000000000000000000000000000000000000..538b331a13a783ed9059ec5d11fb9c469423ef2e --- /dev/null +++ b/lm_eval/tasks/japanese_leaderboard/_ja_leaderboard.yaml @@ -0,0 +1,14 @@ +group: japanese_leaderboard + +task: + - ja_leaderboard_jaqket_v2 + - ja_leaderboard_jcommonsenseqa + - ja_leaderboard_jnli + - ja_leaderboard_jsquad + - ja_leaderboard_marc_ja + - ja_leaderboard_mgsm + - ja_leaderboard_xlsum + - ja_leaderboard_xwinograd + +metadata: + version: 1.0 diff --git a/lm_eval/tasks/japanese_leaderboard/ja_leaderboard_jaqket_v2.yaml b/lm_eval/tasks/japanese_leaderboard/ja_leaderboard_jaqket_v2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ccd7466bd7cd2c9a6927f28e8d36cfa3f07e6dc1 --- /dev/null +++ b/lm_eval/tasks/japanese_leaderboard/ja_leaderboard_jaqket_v2.yaml @@ -0,0 +1,40 @@ +task: ja_leaderboard_jaqket_v2 + +dataset_path: kumapo/JAQKET +dataset_name: v2.0 + +training_split: train +validation_split: validation +test_split: null + +fewshot_split: train +num_fewshot: 1 + +description: "以下は、タスクを説明する指示と、文脈のある入力の組み合わせです。要求を適切に満たす応答を書きなさい。\n\n" +doc_to_text: "### 指示：\n与えられた文脈から、質問に対する答えを抜き出してください。\n\n### 入力：\n文脈：{{ ctxs['text'] | join('\n') }}\n質問：{{ question }}\n\n### 応答：" +doc_to_target: "{{ answers['text'][0] }}" +target_delimiter: "\n" + +output_type: generate_until + +generation_kwargs: + until: + - "\n\n" + do_sample: false + +metric_list: + - metric: exact_match + regexes_to_ignore: + - '^\s+' + - '\s+$' + aggregation: mean + higher_is_better: true + +filter_list: + - name: whitespaces + filter: + - function: remove_whitespace + - function: take_first + +metadata: + version: 2.0 diff --git a/lm_eval/tasks/japanese_leaderboard/ja_leaderboard_jcommonsenseqa.py b/lm_eval/tasks/japanese_leaderboard/ja_leaderboard_jcommonsenseqa.py new file mode 100644 index 0000000000000000000000000000000000000000..6f984be643f336c2af3cf07512186aa86b727f41 --- /dev/null +++ b/lm_eval/tasks/japanese_leaderboard/ja_leaderboard_jcommonsenseqa.py @@ -0,0 +1,6 @@ +def process_docs(dataset): + def _add_choices(doc): + doc["choices"] = [doc[f"choice{i}"] for i in range(5)] + return doc + + return dataset.map(_add_choices) diff --git a/lm_eval/tasks/japanese_leaderboard/ja_leaderboard_jcommonsenseqa.yaml b/lm_eval/tasks/japanese_leaderboard/ja_leaderboard_jcommonsenseqa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4348b701a1724276229b4a0648b2273db8edcaaa --- /dev/null +++ b/lm_eval/tasks/japanese_leaderboard/ja_leaderboard_jcommonsenseqa.yaml @@ -0,0 +1,27 @@ +task: ja_leaderboard_jcommonsenseqa + +dataset_path: Rakuten/JGLUE +dataset_name: JCommonsenseQA +training_split: train +validation_split: validation +test_split: null + +fewshot_split: train +num_fewshot: 3 + +description: "以下は、タスクを説明する指示と、文脈のある入力の組み合わせです。要求を適切に満たす応答を書きなさい。\n\n" +process_docs: !function ja_leaderboard_jcommonsenseqa.process_docs +doc_to_text: "### 指示：\n出力は以下から選択してください：\n{% for choice in choices %}- {{ choice }}\n{% endfor %}\n### 入力：\n{{ question }}\n\n### 応答：" +doc_to_target: label +doc_to_choice: choices +target_delimiter: "\n" + +output_type: multiple_choice + +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + +metadata: + version: 1.0 diff --git a/lm_eval/tasks/japanese_leaderboard/ja_leaderboard_jnli.yaml b/lm_eval/tasks/japanese_leaderboard/ja_leaderboard_jnli.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e01413bba3ef1da7b6fd2cb5d84dce30e7a802a0 --- /dev/null +++ b/lm_eval/tasks/japanese_leaderboard/ja_leaderboard_jnli.yaml @@ -0,0 +1,27 @@ +task: ja_leaderboard_jnli + +dataset_path: Rakuten/JGLUE +dataset_name: JNLI +training_split: train +validation_split: validation +test_split: null + +fewshot_split: train +num_fewshot: 3 + +description: "以下は、タスクを説明する指示と、文脈のある入力の組み合わせです。要求を適切に満たす応答を書きなさい。\n\n" + +doc_to_text: "### 指示：\n与えられた前提と仮説の関係を回答してください。\n\n出力は以下から選択してください：\n含意\n矛盾\n中立\n\n### 入力：\n前提：{{ sentence1 }}\n仮説：{{ sentence2 }}\n\n### 応答：" +doc_to_target: label +doc_to_choice: ["含意", "矛盾", "中立"] +target_delimiter: "\n" + +output_type: multiple_choice + +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + +metadata: + version: 1.0 diff --git a/lm_eval/tasks/japanese_leaderboard/ja_leaderboard_jsquad.yaml b/lm_eval/tasks/japanese_leaderboard/ja_leaderboard_jsquad.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f378a84637ca0b6a7941a93e7ab2490b0484fecf --- /dev/null +++ b/lm_eval/tasks/japanese_leaderboard/ja_leaderboard_jsquad.yaml @@ -0,0 +1,40 @@ +task: ja_leaderboard_jsquad + +dataset_path: Rakuten/JGLUE +dataset_name: JSQuAD + +training_split: train +validation_split: validation +test_split: null + +fewshot_split: train +num_fewshot: 2 + +description: "以下は、タスクを説明する指示と、文脈のある入力の組み合わせです。要求を適切に満たす応答を書きなさい。\n\n" +doc_to_text: "### 指示：\n与えられた文脈から、質問に対する答えを抜き出してください。\n\n### 入力：\n文脈：{% set _context = context.split('[SEP]')[-1] %}{{ _context | trim }}\n質問：{{ question }}\n\n### 応答：" +doc_to_target: "{{ answers['text'][0] }}" +target_delimiter: "\n" + +output_type: generate_until + +generation_kwargs: + until: + - "\n\n" + do_sample: false + +metric_list: + - metric: exact_match + regexes_to_ignore: + - '^\s+' + - '\s+$' + aggregation: mean + higher_is_better: true + +filter_list: + - name: whitespaces + filter: + - function: remove_whitespace + - function: take_first + +metadata: + version: 1.0 diff --git a/lm_eval/tasks/japanese_leaderboard/ja_leaderboard_marc_ja.yaml b/lm_eval/tasks/japanese_leaderboard/ja_leaderboard_marc_ja.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8a19e38b9e5ba86f4b8b2fc9efd9cf9be98abb29 --- /dev/null +++ b/lm_eval/tasks/japanese_leaderboard/ja_leaderboard_marc_ja.yaml @@ -0,0 +1,26 @@ +task: ja_leaderboard_marc_ja + +dataset_path: Rakuten/JGLUE +dataset_name: MARC-ja +training_split: train +validation_split: validation +test_split: null + +fewshot_split: train +num_fewshot: 3 + +description: "以下は、タスクを説明する指示と、文脈のある入力の組み合わせです。要求を適切に満たす応答を書きなさい。\n\n" +doc_to_text: "### 指示：\n以下の製品レビューを、ポジティブまたはネガティブの感情クラスのいずれかに分類してください。\n\n### 入力：\n{{ sentence }}\n\n### 応答：" +doc_to_target: label +doc_to_choice: ["ポジティブ", "ネガティブ"] +target_delimiter: "\n" + +output_type: multiple_choice + +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + +metadata: + version: 1.0 diff --git a/lm_eval/tasks/japanese_leaderboard/ja_leaderboard_mgsm.py b/lm_eval/tasks/japanese_leaderboard/ja_leaderboard_mgsm.py new file mode 100644 index 0000000000000000000000000000000000000000..28f270b5eba616457d2f007a4d802302b7e0c78e --- /dev/null +++ b/lm_eval/tasks/japanese_leaderboard/ja_leaderboard_mgsm.py @@ -0,0 +1,36 @@ +import re + + +_INVALID_ANSWER = "[invalid]" + +_ANSWER_REGEX = re.compile(r"(\-?[0-9\.\,]+)") + + +def _extract_answer(completion): + matches = _ANSWER_REGEX.findall(completion) + if matches: + match_str = matches[-1].strip(".") + match_str = match_str.replace(",", "") + try: + match_float = float(match_str) + except ValueError: + return _INVALID_ANSWER + + if match_float.is_integer(): + return int(match_float) + + return _INVALID_ANSWER + + +def process_results(doc, results): + assert ( + len(results) == 1 + ), f"results should be a list with 1 str element, but is {results}" + + completion = results[0] + extracted_answer = _extract_answer(completion) + answer = doc["answer_number"] + acc = extracted_answer == answer + return { + "acc": acc, + } diff --git a/lm_eval/tasks/japanese_leaderboard/ja_leaderboard_mgsm.yaml b/lm_eval/tasks/japanese_leaderboard/ja_leaderboard_mgsm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bb041220a5ec95f66ef25aaab5fcceaef2cb6e58 --- /dev/null +++ b/lm_eval/tasks/japanese_leaderboard/ja_leaderboard_mgsm.yaml @@ -0,0 +1,32 @@ +task: ja_leaderboard_mgsm + +dataset_path: juletxara/mgsm +dataset_name: ja + +training_split: train +validation_split: null +test_split: test + +fewshot_split: train +num_fewshot: 5 + +description: "以下は、タスクを説明する指示と、文脈のある入力の組み合わせです。要求を適切に満たす応答を書きなさい。\n\n" +doc_to_text: "### 指示：\n与えられた問題に対して、ステップごとに答えを導き出してください。\n\n### 入力：\n{{ question | replace('問題：', '') }}\n\n### 応答：" +doc_to_target: "{{ answer | replace('ステップごとの答え：', '') }}" +target_delimiter: "\n" + +output_type: generate_until +process_results: !function ja_leaderboard_mgsm.process_results + +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + +generation_kwargs: + until: + - "\n\n" + do_sample: false + +metadata: + version: 1.0 diff --git a/lm_eval/tasks/japanese_leaderboard/ja_leaderboard_xlsum.py b/lm_eval/tasks/japanese_leaderboard/ja_leaderboard_xlsum.py new file mode 100644 index 0000000000000000000000000000000000000000..8b99ceed97083c9666566c9fdbcf4be9a5e29d29 --- /dev/null +++ b/lm_eval/tasks/japanese_leaderboard/ja_leaderboard_xlsum.py @@ -0,0 +1,75 @@ +import re + + +def _missing_module_message(name): + return f"`{name}` is required for `japanese_leaderboard`, please install `{name}` via pip install lm_eval[japanese_leaderboard] or pip install -e .[japanese_leaderboard]" + + +try: + import emoji + import neologdn + from fugashi import Tagger + from rouge_score import rouge_scorer, scoring +except ModuleNotFoundError as err: + raise ModuleNotFoundError(_missing_module_message(err.name)) from err + + +class MecabTokenizer: + def __init__(self) -> None: + self.tagger = Tagger("-Owakati") + + def normalize_answer(self, text): + """Lower case text, remove punctuation and extra whitespace, etc.""" + + def white_space_fix(text): + return " ".join(text.split()) + + def remove_emoji(text): + text = "".join(["" if emoji.is_emoji(c) else c for c in text]) + emoji_pattern = re.compile( + "[" + "\U0001f600-\U0001f64f" # emoticons + "\U0001f300-\U0001f5ff" # symbols & pictographs + "\U0001f680-\U0001f6ff" # transport & map symbols + "\U0001f1e0-\U0001f1ff" # flags (iOS) + "\U00002702-\U000027b0" + "]+", + flags=re.UNICODE, + ) + return emoji_pattern.sub(r"", text) + + text = remove_emoji(text) + # see neologdn docs for details, but handles things like full/half width variation + text = neologdn.normalize(text) + text = white_space_fix(text) + return text + + def tokenize(self, text): + return self.tagger.parse(self.normalize_answer(text)).split() + + +def rouge2(items): + return items + + +def rouge2_agg(items): + tokenizer = MecabTokenizer() + + refs = list(zip(*items))[0] + preds = list(zip(*items))[1] + + rouge_type = "rouge2" + + # mecab-based rouge + scorer = rouge_scorer.RougeScorer( + rouge_types=[rouge_type], + tokenizer=tokenizer, + ) + + # Acumulate confidence intervals. + aggregator = scoring.BootstrapAggregator() + for ref, pred in zip(refs, preds): + aggregator.add_scores(scorer.score(ref, pred)) + result = aggregator.aggregate() + + return result[rouge_type].mid.fmeasure diff --git a/lm_eval/tasks/japanese_leaderboard/ja_leaderboard_xlsum.yaml b/lm_eval/tasks/japanese_leaderboard/ja_leaderboard_xlsum.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5de60f182baaf849159f0a8c26e79e1ef3177e94 --- /dev/null +++ b/lm_eval/tasks/japanese_leaderboard/ja_leaderboard_xlsum.yaml @@ -0,0 +1,37 @@ +task: ja_leaderboard_xlsum + +dataset_path: mkshing/xlsum_ja +dataset_name: null + +training_split: train +validation_split: validation +test_split: test + +fewshot_split: train +num_fewshot: 1 + +description: "以下は、タスクを説明する指示と、文脈のある入力の組み合わせです。要求を適切に満たす応答を書きなさい。\n\n" +doc_to_text: "### 指示：\n与えられたニュース記事を要約してください。\n\n### 入力：\nニュース記事：{{ text }}\n\n### 応答：" +doc_to_target: "{{ summary }}" +target_delimiter: "\n" + +output_type: generate_until + +metric_list: + - metric: !function ja_leaderboard_xlsum.rouge2 + aggregation: !function ja_leaderboard_xlsum.rouge2_agg + higher_is_better: true + +filter_list: + - name: whitespaces + filter: + - function: remove_whitespace + - function: take_first + +generation_kwargs: + until: + - "\n\n" + do_sample: false + +metadata: + version: 1.0 diff --git a/lm_eval/tasks/japanese_leaderboard/ja_leaderboard_xwinograd.py b/lm_eval/tasks/japanese_leaderboard/ja_leaderboard_xwinograd.py new file mode 100644 index 0000000000000000000000000000000000000000..74a1a5e6a90fb5eb10803f4c31988f204e2a0f83 --- /dev/null +++ b/lm_eval/tasks/japanese_leaderboard/ja_leaderboard_xwinograd.py @@ -0,0 +1,7 @@ +def process_docs(dataset): + def _add_choices_and_label(doc): + doc["label"] = int(doc["answer"]) - 1 + doc["choices"] = [doc["sentence1"].strip(), doc["sentence2"].strip()] + return doc + + return dataset.map(_add_choices_and_label) diff --git a/lm_eval/tasks/japanese_leaderboard/ja_leaderboard_xwinograd.yaml b/lm_eval/tasks/japanese_leaderboard/ja_leaderboard_xwinograd.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8a00b75938e0d1a20e4820a9a20eb10b55dbe553 --- /dev/null +++ b/lm_eval/tasks/japanese_leaderboard/ja_leaderboard_xwinograd.yaml @@ -0,0 +1,26 @@ +task: ja_leaderboard_xwinograd + +dataset_path: polm-stability/xwinograd-ja +dataset_name: null + +training_split: null +validation_split: null +test_split: test + +num_fewshot: null + +process_docs: !function ja_leaderboard_xwinograd.process_docs +doc_to_target: "label" +doc_to_choice: "choices" +doc_to_text: "" +target_delimiter: "" + +output_type: multiple_choice + +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + +metadata: + version: 1.0 diff --git a/lm_eval/tasks/japanese_leaderboard/requirements.txt b/lm_eval/tasks/japanese_leaderboard/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..894c2459172d9063edd9d8bb056253e8a0b29343 --- /dev/null +++ b/lm_eval/tasks/japanese_leaderboard/requirements.txt @@ -0,0 +1,4 @@ +emoji==2.14.0 +fugashi[unidic-lite] +neologdn==0.5.3 +rouge_score>=0.1.2 diff --git a/lm_eval/tasks/kmmlu/cot_hard/_cot_kmmlu_yaml b/lm_eval/tasks/kmmlu/cot_hard/_cot_kmmlu_yaml index a0cc644c38a5960f87ab5cdbdbcf8c8bc96ffed9..163a03dfd21f3ba21ddc74c8470cf9de0ff29466 100644 --- a/lm_eval/tasks/kmmlu/cot_hard/_cot_kmmlu_yaml +++ b/lm_eval/tasks/kmmlu/cot_hard/_cot_kmmlu_yaml @@ -1,4 +1,4 @@ -group: +tag: - kmmlu - kmmlu_hard_cot dataset_path: HAERAE-HUB/KMMLU-HARD diff --git a/lm_eval/tasks/kmmlu/direct/_direct_kmmlu_yaml b/lm_eval/tasks/kmmlu/direct/_direct_kmmlu_yaml index 9a4a6dbcf011d766caf9bb16bef085b93da44a39..a0c8dfdc7e8ccce1f61fb55f140b4aca539a8d4e 100644 --- a/lm_eval/tasks/kmmlu/direct/_direct_kmmlu_yaml +++ b/lm_eval/tasks/kmmlu/direct/_direct_kmmlu_yaml @@ -1,4 +1,4 @@ -group: +tag: - kmmlu - kmmlu_direct dataset_path: HAERAE-HUB/KMMLU diff --git a/lm_eval/tasks/kmmlu/direct_hard/_direct_hard_kmmlu_yaml b/lm_eval/tasks/kmmlu/direct_hard/_direct_hard_kmmlu_yaml index 259b5c86bd2aa85c63ae9825538dd227a23e8417..3cf6359206ba07951a7ac08781f8dd6d3fd1450a 100644 --- a/lm_eval/tasks/kmmlu/direct_hard/_direct_hard_kmmlu_yaml +++ b/lm_eval/tasks/kmmlu/direct_hard/_direct_hard_kmmlu_yaml @@ -1,4 +1,4 @@ -group: +tag: - kmmlu - kmmlu_hard_direct dataset_path: HAERAE-HUB/KMMLU-HARD diff --git a/lm_eval/tasks/kmmlu/hard/_hard_kmmlu_yaml b/lm_eval/tasks/kmmlu/hard/_hard_kmmlu_yaml index f5e97c160bcf961cec4cc1349b13ad8a313281be..26c4105b3144701252e44ea7e7d615057fc73beb 100644 --- a/lm_eval/tasks/kmmlu/hard/_hard_kmmlu_yaml +++ b/lm_eval/tasks/kmmlu/hard/_hard_kmmlu_yaml @@ -1,4 +1,4 @@ -group: +tag: - kmmlu - kmmlu_hard dataset_path: HAERAE-HUB/KMMLU-HARD diff --git a/lm_eval/tasks/kobest/kobest_boolq.yaml b/lm_eval/tasks/kobest/kobest_boolq.yaml index e9932d56a9300f31bd96a1cd14ee2df091005b21..cb35d063b7b607036671f3272e196b4ec907baf2 100644 --- a/lm_eval/tasks/kobest/kobest_boolq.yaml +++ b/lm_eval/tasks/kobest/kobest_boolq.yaml @@ -1,4 +1,4 @@ -group: +tag: - kobest task: kobest_boolq dataset_path: skt/kobest_v1 diff --git a/lm_eval/tasks/kobest/kobest_copa.yaml b/lm_eval/tasks/kobest/kobest_copa.yaml index 1f3b34e61fad86a037010dd892fd7b894346f456..2a6a931a8089d0ac3beaf732311db74b7d1fb9b1 100644 --- a/lm_eval/tasks/kobest/kobest_copa.yaml +++ b/lm_eval/tasks/kobest/kobest_copa.yaml @@ -1,4 +1,4 @@ -group: +tag: - kobest task: kobest_copa dataset_path: skt/kobest_v1 diff --git a/lm_eval/tasks/kobest/kobest_hellaswag.yaml b/lm_eval/tasks/kobest/kobest_hellaswag.yaml index d83266a813ecd5a9ffd1989d45ac4c49b5779558..e463b81737a92d4c0d875123c824836ac2cde837 100644 --- a/lm_eval/tasks/kobest/kobest_hellaswag.yaml +++ b/lm_eval/tasks/kobest/kobest_hellaswag.yaml @@ -1,4 +1,4 @@ -group: +tag: - kobest task: kobest_hellaswag dataset_path: skt/kobest_v1 diff --git a/lm_eval/tasks/kobest/kobest_sentineg.yaml b/lm_eval/tasks/kobest/kobest_sentineg.yaml index 64319dca39c520c7a8f9c4f20f0ae2a9e44b7230..e5eed0d41780dc1bccd8adbc91441e423c2f5d69 100644 --- a/lm_eval/tasks/kobest/kobest_sentineg.yaml +++ b/lm_eval/tasks/kobest/kobest_sentineg.yaml @@ -1,4 +1,4 @@ -group: +tag: - kobest task: kobest_sentineg dataset_path: skt/kobest_v1 diff --git a/lm_eval/tasks/kobest/kobest_wic.yaml b/lm_eval/tasks/kobest/kobest_wic.yaml index 569d3393dbe78e1bb5d92e00d4ceac439282b9d0..7d2ea140a8af0e7d207f8dfb0f7209f081a6d6b5 100644 --- a/lm_eval/tasks/kobest/kobest_wic.yaml +++ b/lm_eval/tasks/kobest/kobest_wic.yaml @@ -1,4 +1,4 @@ -group: +tag: - kobest task: kobest_wic dataset_path: skt/kobest_v1 diff --git a/lm_eval/tasks/kobest/utils.py b/lm_eval/tasks/kobest/utils.py index 9799ef038c09a67f92a2b174d57f5aaefa05a32f..fb1c5ef9495d141f6a712ff6a163041f55118738 100644 --- a/lm_eval/tasks/kobest/utils.py +++ b/lm_eval/tasks/kobest/utils.py @@ -1,5 +1,4 @@ from datasets import Dataset -from sklearn.metrics import f1_score def copa_doc_to_text(doc: dict) -> str: @@ -41,6 +40,8 @@ def hellaswag_process_doc(doc: Dataset) -> Dataset: def macro_f1_score(items): + from sklearn.metrics import f1_score + unzipped_list = list(zip(*items)) golds = unzipped_list[0] preds = unzipped_list[1] diff --git a/lm_eval/tasks/kormedmcqa/_kormedmcqa.yaml b/lm_eval/tasks/kormedmcqa/_kormedmcqa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d6548334fe19609dc358d0dfb697d5888c10e351 --- /dev/null +++ b/lm_eval/tasks/kormedmcqa/_kormedmcqa.yaml @@ -0,0 +1,11 @@ +group: kormedmcqa +task: + - kormedmcqa_doctor + - kormedmcqa_nurse + - kormedmcqa_pharm +aggregate_metric_list: + - metric: exact_match + aggregation: mean + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm_eval/tasks/kormedmcqa/kormedmcqa_doctor.yaml b/lm_eval/tasks/kormedmcqa/kormedmcqa_doctor.yaml index 3977569f67d998e27e88f24ac294411e79c6ffdc..d130dbe8114a7028d647a83229b75813988296d1 100644 --- a/lm_eval/tasks/kormedmcqa/kormedmcqa_doctor.yaml +++ b/lm_eval/tasks/kormedmcqa/kormedmcqa_doctor.yaml @@ -1,4 +1,3 @@ -group: kormedmcqa task : kormedmcqa_doctor dataset_path : sean0042/KorMedMCQA dataset_name : doctor diff --git a/lm_eval/tasks/kormedmcqa/kormedmcqa_nurse.yaml b/lm_eval/tasks/kormedmcqa/kormedmcqa_nurse.yaml index 264fb9a765381e97bd1dd60de48b45cbc275f299..026b6217addc6e5d537f389e89a0f95a5dc0dd09 100644 --- a/lm_eval/tasks/kormedmcqa/kormedmcqa_nurse.yaml +++ b/lm_eval/tasks/kormedmcqa/kormedmcqa_nurse.yaml @@ -1,4 +1,3 @@ -group: kormedmcqa task : kormedmcqa_nurse dataset_path : sean0042/KorMedMCQA dataset_name : nurse diff --git a/lm_eval/tasks/kormedmcqa/kormedmcqa_pharm.yaml b/lm_eval/tasks/kormedmcqa/kormedmcqa_pharm.yaml index bd3e6a3103a1afcd9fe54f16817ed5fefa6c2e5e..91279dd7057d25ee94c7b99529f3521960a29265 100644 --- a/lm_eval/tasks/kormedmcqa/kormedmcqa_pharm.yaml +++ b/lm_eval/tasks/kormedmcqa/kormedmcqa_pharm.yaml @@ -1,4 +1,3 @@ -group: kormedmcqa task : kormedmcqa_pharm dataset_path : sean0042/KorMedMCQA dataset_name : pharm diff --git a/lm_eval/tasks/lambada/lambada_openai.yaml b/lm_eval/tasks/lambada/lambada_openai.yaml index e9fd3a90d514a8650b6c87608cca40e409f60438..8808b5fdf278fe6ad541129455c6c8d4cd1b52bc 100644 --- a/lm_eval/tasks/lambada/lambada_openai.yaml +++ b/lm_eval/tasks/lambada/lambada_openai.yaml @@ -1,4 +1,4 @@ -group: +tag: - lambada task: lambada_openai dataset_path: EleutherAI/lambada_openai diff --git a/lm_eval/tasks/lambada/lambada_standard.yaml b/lm_eval/tasks/lambada/lambada_standard.yaml index 900e18116309391779684eb8c4ebe2903400b784..f76c8dfe3e234abee806219011f9bc5ac30c893f 100644 --- a/lm_eval/tasks/lambada/lambada_standard.yaml +++ b/lm_eval/tasks/lambada/lambada_standard.yaml @@ -1,4 +1,4 @@ -group: +tag: - lambada task: lambada_standard dataset_path: lambada diff --git a/lm_eval/tasks/lambada_cloze/lambada_openai_cloze.yaml b/lm_eval/tasks/lambada_cloze/lambada_openai_cloze.yaml index d25e26d9efd926e79745c251cab1953dde1986bf..602a3ff44dc6383f47ed4799c4bd3f619f8ff136 100644 --- a/lm_eval/tasks/lambada_cloze/lambada_openai_cloze.yaml +++ b/lm_eval/tasks/lambada_cloze/lambada_openai_cloze.yaml @@ -1,4 +1,4 @@ -group: +tag: - lambada_cloze task: lambada_openai_cloze_yaml dataset_path: EleutherAI/lambada_openai diff --git a/lm_eval/tasks/lambada_cloze/lambada_standard_cloze.yaml b/lm_eval/tasks/lambada_cloze/lambada_standard_cloze.yaml index 7cde8fdebc6f85113c3f3548fcc6a0cbe71aaa7b..9c1568b3394f03b22bb9a4ab20bf35bdda3b9440 100644 --- a/lm_eval/tasks/lambada_cloze/lambada_standard_cloze.yaml +++ b/lm_eval/tasks/lambada_cloze/lambada_standard_cloze.yaml @@ -1,4 +1,4 @@ -group: +tag: - lambada_cloze task: lambada_standard_cloze_yaml dataset_path: lambada diff --git a/lm_eval/tasks/lambada_multilingual/lambada_mt_en.yaml b/lm_eval/tasks/lambada_multilingual/lambada_mt_en.yaml index 7e63a6d1bc025afaf862692b110c027008902274..8af05cdb00d322d40991d97af111695b840e6860 100644 --- a/lm_eval/tasks/lambada_multilingual/lambada_mt_en.yaml +++ b/lm_eval/tasks/lambada_multilingual/lambada_mt_en.yaml @@ -1,4 +1,4 @@ -group: +tag: - lambada_multilingual task: lambada_openai_mt_en dataset_path: EleutherAI/lambada_openai diff --git a/lm_eval/tasks/leaderboard/README.md b/lm_eval/tasks/leaderboard/README.md new file mode 100644 index 0000000000000000000000000000000000000000..1bd7bd3e6af099bea9c9a372b0218dfa3b158841 --- /dev/null +++ b/lm_eval/tasks/leaderboard/README.md @@ -0,0 +1,334 @@ +# Leaderboard evaluations +Our goal with this group is to create an unchanging through time version of +evaluations that will power the Open LLM Leaderboard on HuggingFace. + +As we want to evaluate models across capabilities, the list currently contains: +- BBH (3-shots, multichoice) +- GPQA (0-shot, multichoice) +- mmlu-pro (5-shots, multichoice) +- Musr (0-shot, multichoice) +- ifeval (0-shot, generative) +- Math-lvl-5 (4-shots, generative, minerva version) + + +Details on the choice of those evals can be found [here](https://huggingface.co/spaces/open-llm-leaderboard/blog) ! + +## Install +To install the `lm-eval` package with support for leaderboard evaluations, run: + +```bash +git clone --depth 1 https://github.com/EleutherAI/lm-evaluation-harness +cd lm-evaluation-harness +pip install -e ".[math,ifeval,sentencepiece]" +``` + +## BigBenchHard (BBH) + +A suite of 23 challenging BIG-Bench tasks which we call BIG-Bench Hard (BBH). +These are the task for which prior language model evaluations did not +outperform the average human-rater. + +### Paper + +Title: Challenging BIG-Bench Tasks and Whether Chain-of-Thought Can Solve Them + +BIG-Bench (Srivastava et al., 2022) is a diverse evaluation suite that focuses on tasks believed to be beyond the capabilities of current language models. Language models have already made good progress on this benchmark, with the best model in the BIG-Bench paper outperforming average reported human-rater results on 65% of the BIG-Bench tasks via few-shot prompting. But on what tasks do language models fall short of average human-rater performance, and are those tasks actually unsolvable by current language models? +In this work, we focus on a suite of 23 challenging BIG-Bench tasks which we call BIG-Bench Hard (BBH). These are the task for which prior language model evaluations did not outperform the average human-rater. We find that applying chain-of-thought (CoT) prompting to BBH tasks enables PaLM to surpass the average human-rater performance on 10 of the 23 tasks, and Codex (code-davinci-002) to surpass the average human-rater performance on 17 of the 23 tasks. Since many tasks in BBH require multi-step reasoning, few-shot prompting without CoT, as done in the BIG-Bench evaluations (Srivastava et al., 2022), substantially underestimates the best performance and capabilities of language models, which is better captured via CoT prompting. As further analysis, we explore the interaction between CoT and model scale on BBH, finding that CoT enables emergent task performance on several BBH tasks with otherwise flat scaling curves. + + +- paper: https://huggingface.co/papers/2210.09261 +- Homepage: https://github.com/suzgunmirac/BIG-Bench-Hard + +### Citation + +``` +@article{suzgun2022challenging, + title={Challenging BIG-Bench Tasks and Whether Chain-of-Thought Can Solve Them}, + author={Suzgun, Mirac and Scales, Nathan and Sch{\"a}rli, Nathanael and Gehrmann, Sebastian and Tay, Yi and Chung, Hyung Won and Chowdhery, Aakanksha and Le, Quoc V and Chi, Ed H and Zhou, Denny and and Wei, Jason}, + journal={arXiv preprint arXiv:2210.09261}, + year={2022} +} +``` + +### Groups + +- `leaderboard_bbh` + +### Tasks + +- `leaderboard_bbh_boolean_expressions` +- `leaderboard_bbh_causal_judgement` +- `leaderboard_bbh_date_understanding` +- `leaderboard_bbh_disambiguation_qa` +- `leaderboard_bbh_formal_fallacies` +- `leaderboard_bbh_geometric_shapes` +- `leaderboard_bbh_hyperbaton` +- `leaderboard_bbh_logical_deduction_five_objects` +- `leaderboard_bbh_logical_deduction_seven_objects` +- `leaderboard_bbh_logical_deduction_three_objects` +- `leaderboard_bbh_movie_recommendation` +- `leaderboard_bbh_navigate` +- `leaderboard_bbh_object_counting` +- `leaderboard_bbh_penguins_in_a_table` +- `leaderboard_bbh_reasoning_about_colored_objects` +- `leaderboard_bbh_ruin_names` +- `leaderboard_bbh_salient_translation_error_detection` +- `leaderboard_bbh_snarks` +- `leaderboard_bbh_sports_understanding` +- `leaderboard_bbh_temporal_sequences` +- `leaderboard_bbh_tracking_shuffled_objects_five_objects` +- `leaderboard_bbh_tracking_shuffled_objects_seven_objects` +- `leaderboard_bbh_tracking_shuffled_objects_three_objects` +- `leaderboard_bbh_web_of_lies` + +## GPQA + +### Paper + +Title: GPQA: A Graduate-Level Google-Proof Q&A Benchmark + +We present GPQA, a challenging dataset of 448 multiple-choice questions written +by domain experts in biology, physics, and chemistry. We ensure that the +questions are high-quality and extremely difficult: experts who have or are +pursuing PhDs in the corresponding domains reach 65% accuracy (74% when +discounting clear mistakes the experts identified in retrospect), while highly +skilled non-expert validators only reach 34% accuracy, despite spending on +average over 30 minutes with unrestricted access to the web (i.e., the +questions are “Google-proof”). The questions are also difficult for +state-of-the-art AI systems, with our strongest GPT-4–based baseline achieving +39% accuracy. If we are to use future AI systems to help us answer very hard +questions—for example, when developing new scientific knowledge—we need to +develop scalable oversight methods that enable humans to supervise their +outputs, which may be difficult even if the supervisors are themselves skilled +and knowledgeable. The difficulty of GPQA both for skilled non-experts and +frontier AI systems should enable realistic scalable oversight experiments, +which we hope can help devise ways for human experts to reliably get truthful +information from AI systems that surpass human capabilities. + +- Paper: https://huggingface.co/papers/2311.12022 +- Homepage: https://github.com/idavidrein/gpqa/tree/main + +### Citation + +``` +@misc{rein2023gpqa, + title={GPQA: A Graduate-Level Google-Proof Q&A Benchmark}, + author={David Rein and Betty Li Hou and Asa Cooper Stickland and Jackson Petty and Richard Yuanzhe Pang and Julien Dirani and Julian Michael and Samuel R. Bowman}, + year={2023}, + eprint={2311.12022}, + archivePrefix={arXiv}, + primaryClass={cs.AI} +} +``` + +### Groups + +- `leaderboard_gpqa` + +### Tasks + +- `leaderboard_gpqa_extended` +- `leaderboard_gpqa_diamond` +- `leaderboard_gpqa_main` + +## IFEval + +### Paper + +Title: Instruction-Following Evaluation for Large Language Models + +One core capability of Large Language Models (LLMs) is to follow natural +language instructions. However, the evaluation of such abilities is not +standardized: Human evaluations are expensive, slow, and not objectively +reproducible, while LLM-based auto-evaluation is potentially biased or limited +by the ability of the evaluator LLM. To overcome these issues, we introduce +Instruction-Following Eval (IFEval) for large language models. IFEval is a +straightforward and easy-to-reproduce evaluation benchmark. It focuses on a set +of "verifiable instructions" such as "write in more than 400 words" and +"mention the keyword of AI at least 3 times". We identified 25 types of those +verifiable instructions and constructed around 500 prompts, with each prompt +containing one or more verifiable instructions. We show evaluation results of +two widely available LLMs on the market. + +- Paper: https://huggingface.co/papers/2210.09261 +- Homepage: https://github.com/google-research/google-research/tree/master/instruction_following_eval + +### Citation + +``` +@article{zhou2023instructionfollowing, + title={Instruction-Following Evaluation for Large Language Models}, + author={Jeffrey Zhou and Tianjian Lu and Swaroop Mishra and Siddhartha Brahma and Sujoy Basu and Yi Luan and Denny Zhou and Le Hou}, + journal={arXiv preprint arXiv:2311.07911}, + year={2023}, +} +``` + +### Tasks + +- `leaderboard_ifeval` + +## MATH-hard + +This is the 4 shots variant of minerva math but only keeping the level 5 questions. + +### Paper + +Title: Measuring Mathematical Problem Solving With the MATH Dataset + +Many intellectual endeavors require mathematical problem solving, but this +skill remains beyond the capabilities of computers. To measure this ability in +machine learning models, we introduce MATH, a new dataset of 12,500 challenging +competition mathematics problems. Each problem in MATH has a full step-by-step +solution which can be used to teach models to generate answer derivations and +explanations. + +NOTE: The few-shot and the generated answer extraction is based on the +[Minerva](https://arxiv.org/abs/2206.14858) and exact match equivalence is +calculated using the `sympy` library. This requires additional dependencies, +which can be installed via the `lm-eval[math]` extra. + +- Paper: https://huggingface.co/papers/2103.03874 +- Homepage: https://github.com/hendrycks/math + + +### Citation + +``` +@article{hendrycksmath2021, + title={Measuring Mathematical Problem Solving With the MATH Dataset}, + author={Dan Hendrycks and Collin Burns and Saurav Kadavath and Akul Arora and Steven Basart and Eric Tang and Dawn Song and Jacob Steinhardt}, + journal={NeurIPS}, + year={2021} +} +@misc{2206.14858, +Author = {Aitor Lewkowycz and Anders Andreassen and David Dohan and Ethan Dye and Henryk Michalewski and Vinay Ramasesh and Ambrose Slone and Cem Anil and Imanol Schlag and Theo Gutman-Solo and Yuhuai Wu and Behnam Neyshabur and Guy Gur-Ari and Vedant Misra}, +Title = {Solving Quantitative Reasoning Problems with Language Models}, +Year = {2022}, +Eprint = {arXiv:2206.14858}, +} +``` + +### Groups + +- `leaderboard_math_hard` + +### Tasks + +- `leaderboard_math_algebra_hard` +- `leaderboard_math_counting_and_prob_hard` +- `leaderboard_math_geometry_hard` +- `leaderboard_math_intermediate_algebra_hard` +- `leaderboard_math_num_theory_hard` +- `leaderboard_math_prealgebra_hard` +- `leaderboard_math_precalculus_hard` + + +## MMLU-Pro + +### Paper + +Title: MMLU-Pro: A More Robust and Challenging Multi-Task Language +Understanding Benchmark + +In the age of large-scale language models, benchmarks like the Massive +Multitask Language Understanding (MMLU) have been pivotal in pushing the +boundaries of what AI can achieve in language comprehension and reasoning +across diverse domains. However, as models continue to improve, their +performance on these benchmarks has begun to plateau, making it increasingly +difficult to discern differences in model capabilities. This paper introduces +MMLU-Pro, an enhanced dataset designed to extend the mostly knowledge-driven +MMLU benchmark by integrating more challenging, reasoning-focused questions and +expanding the choice set from four to ten options. Additionally, MMLU-Pro +eliminates the trivial and noisy questions in MMLU. Our experimental results +show that MMLU-Pro not only raises the challenge, causing a significant drop in +accuracy by 16% to 33% compared to MMLU but also demonstrates greater stability +under varying prompts. With 24 different prompt styles tested, the sensitivity +of model scores to prompt variations decreased from 4-5% in MMLU to just 2% in +MMLU-Pro. Additionally, we found that models utilizing Chain of Thought (CoT) +reasoning achieved better performance on MMLU-Pro compared to direct answering, +which is in stark contrast to the findings on the original MMLU, indicating +that MMLU-Pro includes more complex reasoning questions. Our assessments +confirm that MMLU-Pro is a more discriminative benchmark to better track +progress in the field. + +- Paper: https://huggingface.co/papers/2406.01574 +- Homepage: https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro + +### Citation + +``` +@misc{wang2024mmluprorobustchallengingmultitask, + title={MMLU-Pro: A More Robust and Challenging Multi-Task Language + Understanding Benchmark}, + author={Yubo Wang and Xueguang Ma and Ge Zhang and Yuansheng Ni and Abhranil Chandra and Shiguang Guo and Weiming Ren and Aaran Arulraj and Xuan He and Ziyan Jiang and Tianle Li and Max Ku and Kai Wang and Alex Zhuang and Rongqi Fan and Xiang Yue and Wenhu Chen}, + year={2024}, + eprint={2406.01574}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2406.01574}, +} +``` + +### Groups + +- `leaderboard_mmlu_pro` + +### Tasks + +- `leaderboard_mmlu_pro` + + +## Musr + +### Paper + +Title: MuSR: Testing the Limits of Chain-of-thought with Multistep Soft +Reasoning + +While large language models (LLMs) equipped with techniques like +chain-of-thought prompting have demonstrated impressive capabilities, they +still fall short in their ability to reason robustly in complex settings. +However, evaluating LLM reasoning is challenging because system capabilities +continue to grow while benchmark datasets for tasks like logical deduction have +remained static. We introduce MuSR, a dataset for evaluating language models on +multistep soft reasoning tasks specified in a natural language narrative. This +dataset has two crucial features. First, it is created through a novel +neurosymbolic synthetic-to-natural generation algorithm, enabling the +construction of complex reasoning instances that challenge GPT-4 (e.g., murder +mysteries roughly 1000 words in length) and which can be scaled further as more +capable LLMs are released. Second, our dataset instances are free text +narratives corresponding to real-world domains of reasoning; this makes it +simultaneously much more challenging than other synthetically-crafted +benchmarks while remaining realistic and tractable for human annotators to +solve with high accuracy. We evaluate a range of LLMs and prompting techniques +on this dataset and characterize the gaps that remain for techniques like +chain-of-thought to perform robust reasoning. + +- Paper: https://huggingface.co/papers/2310.16049 +- Homepage: https://zayne-sprague.github.io/MuSR/ + +### Citation + +``` +@misc{sprague2024musrtestinglimitschainofthought, + title={MuSR: Testing the Limits of Chain-of-thought with Multistep Soft + Reasoning}, + author={Zayne Sprague and Xi Ye and Kaj Bostrom and Swarat Chaudhuri and Greg Durrett}, + year={2024}, + eprint={2310.16049}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2310.16049}, +} +``` + +### Groups + +- `leaderboard_musr` + +### Tasks + +- `leaderboard_musr_murder_mysteries` +- `leaderboard_musr_object_placements` +- `leaderboard_musr_team_allocation` diff --git a/lm_eval/tasks/leaderboard/bbh_mc/_fewshot_template_yaml b/lm_eval/tasks/leaderboard/bbh_mc/_fewshot_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..2b09a7c43c351cdafdafeb9fd6169bb5f1cf8a97 --- /dev/null +++ b/lm_eval/tasks/leaderboard/bbh_mc/_fewshot_template_yaml @@ -0,0 +1,16 @@ +dataset_path: SaylorTwift/bbh +output_type: multiple_choice +test_split: test +doc_to_text: 'Q: {{input}} + + A:' +doc_to_target: "{{target}}" +metric_list: + - metric: acc_norm + aggregation: mean + higher_is_better: true +num_fewshot: 3 +fewshot_config: + sampler: first_n +metadata: + version: 1.0 diff --git a/lm_eval/tasks/leaderboard/bbh_mc/_leaderboard_bbh.yaml b/lm_eval/tasks/leaderboard/bbh_mc/_leaderboard_bbh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9cc10d396824d06bb6e58eeadf6d89cb6e507f16 --- /dev/null +++ b/lm_eval/tasks/leaderboard/bbh_mc/_leaderboard_bbh.yaml @@ -0,0 +1,26 @@ +group: leaderboard_bbh +task: + - leaderboard_bbh_boolean_expressions + - leaderboard_bbh_causal_judgement + - leaderboard_bbh_date_understanding + - leaderboard_bbh_disambiguation_qa + - leaderboard_bbh_formal_fallacies + - leaderboard_bbh_geometric_shapes + - leaderboard_bbh_hyperbaton + - leaderboard_bbh_logical_deduction_five_objects + - leaderboard_bbh_logical_deduction_seven_objects + - leaderboard_bbh_logical_deduction_three_objects + - leaderboard_bbh_movie_recommendation + - leaderboard_bbh_navigate + - leaderboard_bbh_object_counting + - leaderboard_bbh_penguins_in_a_table + - leaderboard_bbh_reasoning_about_colored_objects + - leaderboard_bbh_ruin_names + - leaderboard_bbh_salient_translation_error_detection + - leaderboard_bbh_snarks + - leaderboard_bbh_sports_understanding + - leaderboard_bbh_temporal_sequences + - leaderboard_bbh_tracking_shuffled_objects_five_objects + - leaderboard_bbh_tracking_shuffled_objects_seven_objects + - leaderboard_bbh_tracking_shuffled_objects_three_objects + - leaderboard_bbh_web_of_lies diff --git a/lm_eval/tasks/leaderboard/bbh_mc/boolean_expressions.yaml b/lm_eval/tasks/leaderboard/bbh_mc/boolean_expressions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0660740a836a2f428478376c73c977ae305b215e --- /dev/null +++ b/lm_eval/tasks/leaderboard/bbh_mc/boolean_expressions.yaml @@ -0,0 +1,14 @@ +dataset_name: boolean_expressions +description: 'Evaluate the result of a random Boolean expression.' +doc_to_choice: ["False", "True"] +fewshot_config: + sampler: first_n + samples: + - input: not ( ( not not True ) ) is + target: 'False' + - input: True and False and not True and True is + target: 'False' + - input: not not ( not ( False ) ) is + target: 'True' +include: _fewshot_template_yaml +task: leaderboard_bbh_boolean_expressions diff --git a/lm_eval/tasks/leaderboard/bbh_mc/causal_judgement.yaml b/lm_eval/tasks/leaderboard/bbh_mc/causal_judgement.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c3b13360ec9863b0ce431d7ac7ab7383da553f4b --- /dev/null +++ b/lm_eval/tasks/leaderboard/bbh_mc/causal_judgement.yaml @@ -0,0 +1,62 @@ +dataset_name: causal_judgement +description: 'Answer questions about causal attribution.' +doc_to_choice: ["Yes", "No"] +fewshot_config: + sampler: first_n + samples: + - input: 'How would a typical person answer each of the following questions about + causation? + + Frank T., had an ongoing dispute with his neighbor over a stretch of land and + one day decided to shoot his neighbor in the body. Frank T. had no experience + with guns, his hand slipped on the barrel of the gun, and the shot went wild. + Nonetheless, the bullet bounced off a large boulder several feet away and hit + the neighbor''s body, causing significant injury. Did Frank T. intentionally + shoot his neighbor in the body? + + Options: + + - Yes + + - No' + target: 'No' + - input: 'How would a typical person answer each of the following questions about + causation? + + Suzy and Billy are working on a project that is very important for our nation''s + security. The boss tells them both: "Be sure that you are here at exactly 9 + am. It is absolutely essential that you arrive at that time." Both Billy and + Suzy arrive at 9 am. As it happens, there was a motion detector installed in + the room where they arrived. The motion detector was set up to be triggered + if at least one person appeared in the room at the same time. So the motion + detector went off. Did Billy cause the motion detector to go off? + + Options: + + - Yes + + - No' + target: 'Yes' + - input: 'How would a typical person answer each of the following questions about + causation? + + George and his sister Lena reunite at their parents'' house for Thanksgiving. + Whereas George just got into medical school, Lena is unhappy in her marriage + and recently lost her job. Over the course of the day, George and Lena get into + a number of heated arguments. Later in the afternoon they play a game of darts. + They split the first two games, and the third game is close until the end. Who + will win comes down to George''s last shot. If he hits a high point region, + he wins; if he hits a low point region, Lena wins. George thinks of the difficult + time Lena is having, and he really wants to let her win. He aims the dart at + the low point region. He sets up his shot and the dart lands in the low point + region. After his shot, Lena wins the game and is very happy. Did George hit + the low point region intentionally? + + Options: + + - Yes + + - No' + target: 'Yes' +include: _fewshot_template_yaml +task: leaderboard_bbh_causal_judgement diff --git a/lm_eval/tasks/leaderboard/bbh_mc/date_understanding.yaml b/lm_eval/tasks/leaderboard/bbh_mc/date_understanding.yaml new file mode 100644 index 0000000000000000000000000000000000000000..92f35785d7ff4da93d372a87eaf5f0fdb3579187 --- /dev/null +++ b/lm_eval/tasks/leaderboard/bbh_mc/date_understanding.yaml @@ -0,0 +1,55 @@ +dataset_name: date_understanding +description: 'Infer the date from context.' +doc_to_choice: ["(A)", "(B)", "(C)", "(D)", "(E)", "(F)"] +fewshot_config: + sampler: first_n + samples: + - input: 'Today is Christmas Eve of 1937. What is the date 10 days ago in MM/DD/YYYY? + + Options: + + (A) 12/14/2026 + + (B) 12/14/1950 + + (C) 12/14/2007 + + (D) 12/14/1937 + + (E) 07/14/1938 + + (F) 12/14/1988' + target: (D) + - input: 'Tomorrow is 11/12/2019. What is the date one year ago from today in MM/DD/YYYY? + + Options: + + (A) 09/04/2018 + + (B) 11/11/2018 + + (C) 08/25/2018 + + (D) 11/02/2018 + + (E) 11/04/2018' + target: (B) + - input: 'Jane and John married on Jan 2, 1958. It is their 5-year anniversary today. + What is the date tomorrow in MM/DD/YYYY? + + Options: + + (A) 01/11/1961 + + (B) 01/03/1963 + + (C) 01/18/1961 + + (D) 10/14/1960 + + (E) 01/03/1982 + + (F) 12/03/1960' + target: (B) +include: _fewshot_template_yaml +task: leaderboard_bbh_date_understanding diff --git a/lm_eval/tasks/leaderboard/bbh_mc/disambiguation_qa.yaml b/lm_eval/tasks/leaderboard/bbh_mc/disambiguation_qa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..91b1ed99fae8eebddef3b023e66c09c3d3eb9861 --- /dev/null +++ b/lm_eval/tasks/leaderboard/bbh_mc/disambiguation_qa.yaml @@ -0,0 +1,48 @@ +dataset_name: disambiguation_qa +description: 'Clarify the meaning of sentences with ambiguous pronouns.' +doc_to_choice: ["(A)", "(B)", "(C)"] +fewshot_config: + sampler: first_n + samples: + - input: 'In the following sentences, explain the antecedent of the pronoun (which + thing the pronoun refers to), or state that it is ambiguous. + + Sentence: The chief told the counselor that they took the day off. + + Options: + + (A) The chief took the day off + + (B) The counselor took the day off + + (C) Ambiguous' + target: (A) + - input: 'In the following sentences, explain the antecedent of the pronoun (which + thing the pronoun refers to), or state that it is ambiguous. + + Sentence: The manager sent a message to the secretary, but he didn''t reply + yet. + + Options: + + (A) The secretary didn''t reply yet + + (B) The manager didn''t reply yet + + (C) Ambiguous' + target: (A) + - input: 'In the following sentences, explain the antecedent of the pronoun (which + thing the pronoun refers to), or state that it is ambiguous. + + Sentence: Bailey will plan to meet the director at his office + + Options: + + (A) It will be Bailey''s office + + (B) It will be the director''s office + + (C) Ambiguous' + target: (C) +include: _fewshot_template_yaml +task: leaderboard_bbh_disambiguation_qa diff --git a/lm_eval/tasks/leaderboard/bbh_mc/formal_fallacies.yaml b/lm_eval/tasks/leaderboard/bbh_mc/formal_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eebff0a4df84f196cbd35f0a1c7859af4be722ad --- /dev/null +++ b/lm_eval/tasks/leaderboard/bbh_mc/formal_fallacies.yaml @@ -0,0 +1,55 @@ +dataset_name: formal_fallacies +description: 'Distinguish deductively valid arguments from formal fallacies.' +doc_to_choice: ["valid", "invalid"] +fewshot_config: + sampler: first_n + samples: + - input: '"It is not always easy to see who is related to whom -- and in which ways. + The following argument pertains to this question: To begin with, Lesley is a + close friend of Fernando. Moreover, being a close friend of Fernando or a schoolmate + of Lowell is sufficient for being a great-grandfather of Leroy. It follows that + Lesley is a great-grandfather of Leroy." + + Is the argument, given the explicitly stated premises, deductively valid or + invalid? + + Options: + + - valid + + - invalid' + target: valid + - input: '"It is not always easy to see who is related to whom -- and in which ways. + The following argument pertains to this question: Whoever is not a great-grandfather + of Clyde is a stepbrother of Brian. Being an ancestor of Dana is sufficient + for not being a great-grandfather of Clyde. We may conclude: Everyone who is + an ancestor of Dana is a stepbrother of Brian, too." + + Is the argument, given the explicitly stated premises, deductively valid or + invalid? + + Options: + + - valid + + - invalid' + target: valid + - input: '"It is not always easy to grasp who is consuming which products. The following + argument pertains to this question: Every infrequent user of Paul Mitchell shampoo + is either a rare consumer of Nioxin shampoo or a loyal buyer of Caress soap, + or both. No regular consumer of Lush soap is a rare consumer of Nioxin shampoo + and, in the same time, a loyal buyer of Caress soap. It follows that whoever + is an infrequent user of Paul Mitchell shampoo is not a regular consumer of + Lush soap." + + Is the argument, given the explicitly stated premises, deductively valid or + invalid? + + Options: + + - valid + + - invalid' + target: invalid +include: _fewshot_template_yaml +task: leaderboard_bbh_formal_fallacies diff --git a/lm_eval/tasks/leaderboard/bbh_mc/geometric_shapes.yaml b/lm_eval/tasks/leaderboard/bbh_mc/geometric_shapes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a32fcecac132ed2b537d4fcf4f2812e99726dfbc --- /dev/null +++ b/lm_eval/tasks/leaderboard/bbh_mc/geometric_shapes.yaml @@ -0,0 +1,84 @@ +dataset_name: geometric_shapes +description: 'Name geometric shapes from their SVG paths.' +doc_to_choice: ["(A)","(B)","(C)","(D)","(E)","(F)","(G)","(H)","(I)","(J)","(K)"] +fewshot_config: + sampler: first_n + samples: + - input: 'This SVG path element + draws a + + Options: + + (A) circle + + (B) heptagon + + (C) hexagon + + (D) kite + + (E) line + + (F) octagon + + (G) pentagon + + (H) rectangle + + (I) sector + + (J) triangle' + target: (F) + - input: 'This SVG path element draws a + + Options: + + (A) circle + + (B) heptagon + + (C) hexagon + + (D) kite + + (E) line + + (F) octagon + + (G) pentagon + + (H) rectangle + + (I) sector + + (J) triangle' + target: (G) + - input: 'This SVG path element draws a + + Options: + + (A) circle + + (B) heptagon + + (C) hexagon + + (D) kite + + (E) line + + (F) octagon + + (G) pentagon + + (H) rectangle + + (I) sector + + (J) triangle' + target: (D) +include: _fewshot_template_yaml +task: leaderboard_bbh_geometric_shapes diff --git a/lm_eval/tasks/leaderboard/bbh_mc/hyperbaton.yaml b/lm_eval/tasks/leaderboard/bbh_mc/hyperbaton.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d1eece64c067623487092b845d65246cc78037e --- /dev/null +++ b/lm_eval/tasks/leaderboard/bbh_mc/hyperbaton.yaml @@ -0,0 +1,32 @@ +dataset_name: hyperbaton +description: 'Order adjectives correctly in English sentences.' +doc_to_choice: ["(A)", "(B)"] +fewshot_config: + sampler: first_n + samples: + - input: 'Which sentence has the correct adjective order: + + Options: + + (A) rubber terrible ship + + (B) terrible rubber ship' + target: (B) + - input: 'Which sentence has the correct adjective order: + + Options: + + (A) repulsive small Brazilian exercise ship + + (B) Brazilian repulsive exercise small ship' + target: (A) + - input: 'Which sentence has the correct adjective order: + + Options: + + (A) blue gold wonderful square shoe + + (B) wonderful square blue gold shoe' + target: (B) +include: _fewshot_template_yaml +task: leaderboard_bbh_hyperbaton diff --git a/lm_eval/tasks/leaderboard/bbh_mc/logical_deduction_five_objects.yaml b/lm_eval/tasks/leaderboard/bbh_mc/logical_deduction_five_objects.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9f81dddacbc6902f292375ea4f5318cb814ee60f --- /dev/null +++ b/lm_eval/tasks/leaderboard/bbh_mc/logical_deduction_five_objects.yaml @@ -0,0 +1,50 @@ +dataset_name: logical_deduction_five_objects +description: 'A logical deduction task which requires deducing the order of a sequence + of objects.' +doc_to_choice: ["(A)","(B)","(C)","(D)","(E)"] +fewshot_config: + sampler: first_n + samples: + - input: 'The following paragraphs each describe a set of three objects arranged + in a fixed order. The statements are logically consistent within each paragraph. + In a golf tournament, there were three golfers: Amy, Eli, and Eve. Eve finished + above Amy. Eli finished below Amy. + + Options: + + (A) Amy finished last + + (B) Eli finished last + + (C) Eve finished last' + target: (B) + - input: 'The following paragraphs each describe a set of three objects arranged + in a fixed order. The statements are logically consistent within each paragraph. + On a shelf, there are three books: a white book, a green book, and an orange + book. The green book is to the right of the white book. The orange book is the + rightmost. + + Options: + + (A) The white book is the leftmost + + (B) The green book is the leftmost + + (C) The orange book is the leftmost' + target: (A) + - input: 'The following paragraphs each describe a set of three objects arranged + in a fixed order. The statements are logically consistent within each paragraph. + On a shelf, there are three books: a red book, a gray book, and a white book. + The white book is to the left of the gray book. The red book is the second from + the left. + + Options: + + (A) The red book is the leftmost + + (B) The gray book is the leftmost + + (C) The white book is the leftmost' + target: (C) +include: _fewshot_template_yaml +task: leaderboard_bbh_logical_deduction_five_objects diff --git a/lm_eval/tasks/leaderboard/bbh_mc/logical_deduction_seven_objects.yaml b/lm_eval/tasks/leaderboard/bbh_mc/logical_deduction_seven_objects.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ff426ce7ac517a4a3fe352dda8001bc23ad36e07 --- /dev/null +++ b/lm_eval/tasks/leaderboard/bbh_mc/logical_deduction_seven_objects.yaml @@ -0,0 +1,50 @@ +dataset_name: logical_deduction_seven_objects +description: 'A logical deduction task which requires deducing the order of a sequence + of objects.' +doc_to_choice: ["(A)","(B)","(C)","(D)","(E)","(F)","(G)"] +fewshot_config: + sampler: first_n + samples: + - input: 'The following paragraphs each describe a set of three objects arranged + in a fixed order. The statements are logically consistent within each paragraph. + In a golf tournament, there were three golfers: Amy, Eli, and Eve. Eve finished + above Amy. Eli finished below Amy. + + Options: + + (A) Amy finished last + + (B) Eli finished last + + (C) Eve finished last' + target: (B) + - input: 'The following paragraphs each describe a set of three objects arranged + in a fixed order. The statements are logically consistent within each paragraph. + On a shelf, there are three books: a white book, a green book, and an orange + book. The green book is to the right of the white book. The orange book is the + rightmost. + + Options: + + (A) The white book is the leftmost + + (B) The green book is the leftmost + + (C) The orange book is the leftmost' + target: (A) + - input: 'The following paragraphs each describe a set of three objects arranged + in a fixed order. The statements are logically consistent within each paragraph. + On a shelf, there are three books: a red book, a gray book, and a white book. + The white book is to the left of the gray book. The red book is the second from + the left. + + Options: + + (A) The red book is the leftmost + + (B) The gray book is the leftmost + + (C) The white book is the leftmost' + target: (C) +include: _fewshot_template_yaml +task: leaderboard_bbh_logical_deduction_seven_objects diff --git a/lm_eval/tasks/leaderboard/bbh_mc/logical_deduction_three_objects.yaml b/lm_eval/tasks/leaderboard/bbh_mc/logical_deduction_three_objects.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e62fe2f412bb276dcee713450e4cdab9a565aca5 --- /dev/null +++ b/lm_eval/tasks/leaderboard/bbh_mc/logical_deduction_three_objects.yaml @@ -0,0 +1,50 @@ +dataset_name: logical_deduction_three_objects +description: 'A logical deduction task which requires deducing the order of a sequence + of objects.' +doc_to_choice: ["(A)","(B)","(C)"] +fewshot_config: + sampler: first_n + samples: + - input: 'The following paragraphs each describe a set of three objects arranged + in a fixed order. The statements are logically consistent within each paragraph. + In a golf tournament, there were three golfers: Amy, Eli, and Eve. Eve finished + above Amy. Eli finished below Amy. + + Options: + + (A) Amy finished last + + (B) Eli finished last + + (C) Eve finished last' + target: (B) + - input: 'The following paragraphs each describe a set of three objects arranged + in a fixed order. The statements are logically consistent within each paragraph. + On a shelf, there are three books: a white book, a green book, and an orange + book. The green book is to the right of the white book. The orange book is the + rightmost. + + Options: + + (A) The white book is the leftmost + + (B) The green book is the leftmost + + (C) The orange book is the leftmost' + target: (A) + - input: 'The following paragraphs each describe a set of three objects arranged + in a fixed order. The statements are logically consistent within each paragraph. + On a shelf, there are three books: a red book, a gray book, and a white book. + The white book is to the left of the gray book. The red book is the second from + the left. + + Options: + + (A) The red book is the leftmost + + (B) The gray book is the leftmost + + (C) The white book is the leftmost' + target: (C) +include: _fewshot_template_yaml +task: leaderboard_bbh_logical_deduction_three_objects diff --git a/lm_eval/tasks/leaderboard/bbh_mc/movie_recommendation.yaml b/lm_eval/tasks/leaderboard/bbh_mc/movie_recommendation.yaml new file mode 100644 index 0000000000000000000000000000000000000000..61f3bd8f8b5e85392bef2f6541b6a096c3fff1dd --- /dev/null +++ b/lm_eval/tasks/leaderboard/bbh_mc/movie_recommendation.yaml @@ -0,0 +1,52 @@ +dataset_name: movie_recommendation +description: 'Recommend movies similar to the given list of movies.' +doc_to_choice: ["(A)","(B)","(C)","(D)","(E)","(F)"] +fewshot_config: + sampler: first_n + samples: + - input: 'Find a movie similar to Star Wars Episode IV - A New Hope, Indiana Jones + and the Last Crusade, Star Wars Episode V - The Empire Strikes Back, The Big + Lebowski: + + Options: + + (A) Tetsuo + + (B) the Ironman + + (C) The Princess Bride + + (D) The Barkley Marathons The Race That Eats Its Young + + (E) Bug' + target: (C) + - input: 'Find a movie similar to Twister, The Silence of the Lambs, Independence + Day, Braveheart: + + Options: + + (A) They Shoot Horses + + (B) Don''t They + + (C) Forrest Gump + + (D) The Salton Sea + + (E) Extreme Days' + target: (C) + - input: 'Find a movie similar to Minority Report, Total Recall, Inside Out, Forrest + Gump: + + Options: + + (A) Phenomena + + (B) Lilting + + (C) Catwoman + + (D) Edge of Tomorrow' + target: (D) +include: _fewshot_template_yaml +task: leaderboard_bbh_movie_recommendation diff --git a/lm_eval/tasks/leaderboard/bbh_mc/navigate.yaml b/lm_eval/tasks/leaderboard/bbh_mc/navigate.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a0f67261307eb8c7c68fa5e3d1d58177fb53d2f2 --- /dev/null +++ b/lm_eval/tasks/leaderboard/bbh_mc/navigate.yaml @@ -0,0 +1,38 @@ +dataset_name: navigate +description: 'Given a series of navigation instructions, determine whether one would + end up back at the starting point.' +doc_to_choice: ["Yes","No"] +fewshot_config: + sampler: first_n + samples: + - input: 'If you follow these instructions, do you return to the starting point? + Turn left. Turn around. Turn left. Take 7 steps. Take 2 steps. Take 4 steps. + Take 8 steps. + + Options: + + - Yes + + - No' + target: 'No' + - input: 'If you follow these instructions, do you return to the starting point? + Turn around. Take 1 step. Take 6 steps. Turn around. Take 6 steps. Take 9 steps. + Take 1 step. + + Options: + + - Yes + + - No' + target: 'No' + - input: 'If you follow these instructions, do you return to the starting point? + Always face forward. Take 2 steps right. Take 9 steps left. Take 7 steps right. + + Options: + + - Yes + + - No' + target: 'Yes' +include: _fewshot_template_yaml +task: leaderboard_bbh_navigate diff --git a/lm_eval/tasks/leaderboard/bbh_mc/object_counting.yaml b/lm_eval/tasks/leaderboard/bbh_mc/object_counting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..88168743ee3841a4daef5979452799e9e50aae1b --- /dev/null +++ b/lm_eval/tasks/leaderboard/bbh_mc/object_counting.yaml @@ -0,0 +1,18 @@ +dataset_name: object_counting +description: 'Questions that involve enumerating objects and asking the model to count + them.' +doc_to_choice: ["0","1","2","3","4","5","6","7","8","9","10", "11", "12", "13", "14", "15", "16", "17", "18"] +fewshot_config: + sampler: first_n + samples: + - input: I have a blackberry, a clarinet, a nectarine, a plum, a strawberry, a banana, + a flute, an orange, and a violin. How many fruits do I have? + target: '6' + - input: I have an orange, a raspberry, two peaches, a blackberry, an apple, a grape, + a nectarine, and three plums. How many fruits do I have? + target: '11' + - input: I have a lettuce head, a head of broccoli, an onion, a stalk of celery, + two carrots, a garlic, and a yam. How many vegetables do I have? + target: '8' +include: _fewshot_template_yaml +task: leaderboard_bbh_object_counting diff --git a/lm_eval/tasks/leaderboard/bbh_mc/penguins_in_a_table.yaml b/lm_eval/tasks/leaderboard/bbh_mc/penguins_in_a_table.yaml new file mode 100644 index 0000000000000000000000000000000000000000..206d6752e54c9ee1c47235eb5e33879c48c2429f --- /dev/null +++ b/lm_eval/tasks/leaderboard/bbh_mc/penguins_in_a_table.yaml @@ -0,0 +1,66 @@ +dataset_name: penguins_in_a_table +description: 'Answer questions about a table of penguins and their attributes.' +doc_to_choice: ["(A)","(B)","(C)","(D)","(E)"] +fewshot_config: + sampler: first_n + samples: + - input: 'Here is a table where the first line is a header and each subsequent line + is a penguin: name, age, height (cm), weight (kg) Louis, 7, 50, 11 Bernard, + 5, 80, 13 Vincent, 9, 60, 11 Gwen, 8, 70, 15 For example: the age of Louis + is 7, the weight of Gwen is 15 kg, the height of Bernard is 80 cm. We now add + a penguin to the table: + + James, 12, 90, 12 + + How many penguins are less than 8 years old? + + Options: + + (A) 1 + + (B) 2 + + (C) 3 + + (D) 4 + + (E) 5' + target: (B) + - input: 'Here is a table where the first line is a header and each subsequent line + is a penguin: name, age, height (cm), weight (kg) Louis, 7, 50, 11 Bernard, + 5, 80, 13 Vincent, 9, 60, 11 Gwen, 8, 70, 15 For example: the age of Louis + is 7, the weight of Gwen is 15 kg, the height of Bernard is 80 cm. Which is + the youngest penguin? + + Options: + + (A) Louis + + (B) Bernard + + (C) Vincent + + (D) Gwen + + (E) James' + target: (B) + - input: 'Here is a table where the first line is a header and each subsequent line + is a penguin: name, age, height (cm), weight (kg) Louis, 7, 50, 11 Bernard, + 5, 80, 13 Vincent, 9, 60, 11 Gwen, 8, 70, 15 For example: the age of Louis + is 7, the weight of Gwen is 15 kg, the height of Bernard is 80 cm. What is + the name of the second penguin sorted by alphabetic order? + + Options: + + (A) Louis + + (B) Bernard + + (C) Vincent + + (D) Gwen + + (E) James' + target: (D) +include: _fewshot_template_yaml +task: leaderboard_bbh_penguins_in_a_table diff --git a/lm_eval/tasks/leaderboard/bbh_mc/reasoning_about_colored_objects.yaml b/lm_eval/tasks/leaderboard/bbh_mc/reasoning_about_colored_objects.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6ca071a9c8562182050b92f1060ae63f3af33777 --- /dev/null +++ b/lm_eval/tasks/leaderboard/bbh_mc/reasoning_about_colored_objects.yaml @@ -0,0 +1,114 @@ +dataset_name: reasoning_about_colored_objects +description: 'Answer extremely simple questions about the colors of objects on a surface.' +doc_to_choice: ["(A)","(B)","(C)","(D)","(E)","(F)","(G)","(H)","(I)","(J)","(K)","(L)","(M)","(N)","(O)","(P)","(Q)","(R)"] +fewshot_config: + sampler: first_n + samples: + - input: 'On the nightstand, there is a red pencil, a purple mug, a burgundy keychain, + a fuchsia teddy bear, a black plate, and a blue stress ball. What color is the + stress ball? + + Options: + + (A) red + + (B) orange + + (C) yellow + + (D) green + + (E) blue + + (F) brown + + (G) magenta + + (H) fuchsia + + (I) mauve + + (J) teal + + (K) turquoise + + (L) burgundy + + (M) silver + + (N) gold + + (O) black + + (P) grey + + (Q) purple + + (R) pink' + target: (E) + - input: 'On the table, you see a bunch of objects arranged in a row: a purple paperclip, + a pink stress ball, a brown keychain, a green scrunchiephone charger, a mauve + fidget spinner, and a burgundy pen. What is the color of the object directly + to the right of the stress ball? + + Options: + + (A) red + + (B) orange + + (C) yellow + + (D) green + + (E) blue + + (F) brown + + (G) magenta + + (H) fuchsia + + (I) mauve + + (J) teal + + (K) turquoise + + (L) burgundy + + (M) silver + + (N) gold + + (O) black + + (P) grey + + (Q) purple + + (R) pink' + target: (F) + - input: 'On the nightstand, you see the following items arranged in a row: a teal + plate, a burgundy keychain, a yellow scrunchiephone charger, an orange mug, + a pink notebook, and a grey cup. How many non-orange items do you see to the + left of the teal item? + + Options: + + (A) zero + + (B) one + + (C) two + + (D) three + + (E) four + + (F) five + + (G) six' + target: (A) +include: _fewshot_template_yaml +task: leaderboard_bbh_reasoning_about_colored_objects diff --git a/lm_eval/tasks/leaderboard/bbh_mc/ruin_names.yaml b/lm_eval/tasks/leaderboard/bbh_mc/ruin_names.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f3c12367a6e8a67a195ef22d94dcafb9a62e98ba --- /dev/null +++ b/lm_eval/tasks/leaderboard/bbh_mc/ruin_names.yaml @@ -0,0 +1,48 @@ +dataset_name: ruin_names +description: 'Select the humorous edit that ''ruins'' the input movie or musical artist + name.' +doc_to_choice: ["(A)","(B)","(C)","(D)","(E)","(F)"] +fewshot_config: + sampler: first_n + samples: + - input: 'Which of the following is a humorous edit of this artist or movie name: + ''whitesnake''? + + Options: + + (A) whitesnape + + (B) whitesnapke + + (C) whitesnuake + + (D) mwhitesnake' + target: (A) + - input: 'Which of the following is a humorous edit of this artist or movie name: + ''one of our dinosaurs is missing''? + + Options: + + (A) ofne of our dinosaurs is missing + + (B) one af our dinosaurs is missing + + (C) one of our dinosaurs is pissing + + (D) one of our dinosaur is missing' + target: (C) + - input: 'Which of the following is a humorous edit of this artist or movie name: + ''counting crows''? + + Options: + + (A) countingy crows + + (B) counting cows + + (C) courting crows + + (D) coutnting crows' + target: (B) +include: _fewshot_template_yaml +task: leaderboard_bbh_ruin_names diff --git a/lm_eval/tasks/leaderboard/bbh_mc/salient_translation_error_detection.yaml b/lm_eval/tasks/leaderboard/bbh_mc/salient_translation_error_detection.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2ef54c6fc577363c4281d719bedb7779a45fe383 --- /dev/null +++ b/lm_eval/tasks/leaderboard/bbh_mc/salient_translation_error_detection.yaml @@ -0,0 +1,101 @@ +dataset_name: salient_translation_error_detection +description: 'Detect the type of error in an English translation of a German source + sentence.' +doc_to_choice: ["(A)","(B)","(C)","(D)","(E)","(F)"] +fewshot_config: + sampler: first_n + samples: + - input: "The following translations from German to English contain a particular + error. That error will be one of the following types: Named Entities: An entity + (names, places, locations, etc.) is changed to a different entity. Numerical + Values: Numerical values (ordinals or cardinals), dates, and/or units are changed. + Modifiers or Adjectives: The modifiers and adjectives pertaining to a noun are + changed. Negation or Antonyms: Introduce or remove a negation or change comparatives + to their antonyms. Facts: Trivial factual errors not pertaining to the above + classes are introduced in the translations. Dropped Content: A significant clause + in the translation is removed. Please identify that error. Source: In der Liste + der Baudenkmale in Lenzen (Elbe) sind alle Baudenkmale der brandenburgischen + Stadt Lenzen (Elbe) und ihrer Ortsteile aufgelistet. + + Translation: In the list of architectural monuments in Lenzen all architectural + monuments of the Brandenburg city of Lenzen and its districts are listed. + + The translation contains an error pertaining to + + Options: + + (A) Modifiers or Adjectives + + (B) Numerical Values + + (C) Negation or Antonyms + + (D) Named Entities + + (E) Dropped Content + + (F) Facts" + target: (D) + - input: "The following translations from German to English contain a particular + error. That error will be one of the following types: Named Entities: An entity + (names, places, locations, etc.) is changed to a different entity. Numerical + Values: Numerical values (ordinals or cardinals), dates, and/or units are changed. + Modifiers or Adjectives: The modifiers and adjectives pertaining to a noun are + changed. Negation or Antonyms: Introduce or remove a negation or change comparatives + to their antonyms. Facts: Trivial factual errors not pertaining to the above + classes are introduced in the translations. Dropped Content: A significant clause + in the translation is removed. Please identify that error. Source: Auf dieser + Seite sind die Baudenkm\xE4ler der oberbayerischen Gro\xDFen Kreisstadt Landsberg + am Lech zusammengestellt. + + Translation: On this page are compiled the architectural monuments of the town of + Landsberg am Lech. + + The translation contains an error pertaining to + + Options: + + (A) Modifiers or Adjectives + + (B) Numerical Values + + (C) Negation or Antonyms + + (D) Named Entities + + (E) Dropped Content + + (F) Facts" + target: (E) + - input: "The following translations from German to English contain a particular + error. That error will be one of the following types: Named Entities: An entity + (names, places, locations, etc.) is changed to a different entity. Numerical + Values: Numerical values (ordinals or cardinals), dates, and/or units are changed. + Modifiers or Adjectives: The modifiers and adjectives pertaining to a noun are + changed. Negation or Antonyms: Introduce or remove a negation or change comparatives + to their antonyms. Facts: Trivial factual errors not pertaining to the above + classes are introduced in the translations. Dropped Content: A significant clause + in the translation is removed. Please identify that error. Source: \u0141eba ist + eine Kleinstadt und ein Badeort im Powiat L\u0119borski der polnischen Woiwodschaft + Pommern. + + Translation: Eba is not a small town and seaside resort in the Powiat L\xE9borski + county of the Pomeranian Voivodeship of Poland.\nThe translation contains an error + pertaining to + + Options: + + (A) Modifiers or Adjectives + + (B) Numerical Values + + (C) Negation or Antonyms + + (D) Named Entities + + (E) Dropped Content + + (F) Facts" + target: (C) +include: _fewshot_template_yaml +task: leaderboard_bbh_salient_translation_error_detection diff --git a/lm_eval/tasks/leaderboard/bbh_mc/snarks.yaml b/lm_eval/tasks/leaderboard/bbh_mc/snarks.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a4a02e581bb475a80cc61063f257bc2e59b2dd1d --- /dev/null +++ b/lm_eval/tasks/leaderboard/bbh_mc/snarks.yaml @@ -0,0 +1,42 @@ +dataset_name: snarks +description: 'Determine which of two sentences is sarcastic. + + + According to Cambridge University Dictionary, sarcasm is "the use of remarks that + clearly mean the opposite of what they say, made in order to hurt someone''s feelings + or to criticize something in a humorous way." Sarcastic sentences often contain + satirical or ironic utterances, hyperboles, ambivalent or witty remarks.' +doc_to_choice: ["(A)","(B)"] +fewshot_config: + sampler: first_n + samples: + - input: 'Which statement is sarcastic? + + Options: + + (A) Yes, because having interests and actively researching them is a huge waste + + (B) Yes, because having interests and actively researching them is a huge deal' + target: (A) + - input: 'Which statement is sarcastic? + + Options: + + (A) No one is going to disagree with you on this. Avoiding ad hominem attacks + really help your case + + (B) No one is going to disagree with you on this. Ad hominem attacks really + help your case' + target: (B) + - input: 'Which statement is sarcastic? + + Options: + + (A) Consistency in the league''s punishments? What do you think this is supposed + to be, politics? + + (B) Consistency in the league''s punishments? What do you think this is supposed + to be, moral?' + target: (A) +include: _fewshot_template_yaml +task: leaderboard_bbh_snarks diff --git a/lm_eval/tasks/leaderboard/bbh_mc/sports_understanding.yaml b/lm_eval/tasks/leaderboard/bbh_mc/sports_understanding.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e3f3ea2290344b2df8349a627899d509155153de --- /dev/null +++ b/lm_eval/tasks/leaderboard/bbh_mc/sports_understanding.yaml @@ -0,0 +1,17 @@ +dataset_name: sports_understanding +description: 'Determine whether an artificially constructed sentence relating to sports + is plausible or not.' +doc_to_choice: ["yes","no"] +fewshot_config: + sampler: first_n + samples: + - input: Is the following sentence plausible? "Bam Adebayo scored a reverse layup + in the Western Conference Finals." + target: 'yes' + - input: Is the following sentence plausible? "Santi Cazorla scored a touchdown." + target: 'no' + - input: Is the following sentence plausible? "DeMar DeRozan was called for the + goal tend." + target: 'yes' +include: _fewshot_template_yaml +task: leaderboard_bbh_sports_understanding diff --git a/lm_eval/tasks/leaderboard/bbh_mc/temporal_sequences.yaml b/lm_eval/tasks/leaderboard/bbh_mc/temporal_sequences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..543e711c04c1d3b8e143c748f4f87844bed3371f --- /dev/null +++ b/lm_eval/tasks/leaderboard/bbh_mc/temporal_sequences.yaml @@ -0,0 +1,99 @@ +dataset_name: temporal_sequences +description: 'Task description: Answer questions about which times certain events + could have occurred.' +doc_to_choice: ["(A)","(B)","(C)","(D)"] +fewshot_config: + sampler: first_n + samples: + - input: 'Today, Emily went to the museum. Between what times could they have gone? + + We know that: + + Emily woke up at 1pm. + + Elizabeth saw Emily reading at the library from 2pm to 4pm. + + Jessica saw Emily watching a movie at the theater from 4pm to 5pm. + + Leslie saw Emily waiting at the airport from 5pm to 6pm. + + William saw Emily buying clothes at the mall from 6pm to 7pm. + + The museum was closed after 7pm. + + Between what times could Emily have gone to the museum? + + Options: + + (A) 1pm to 2pm + + (B) 6pm to 7pm + + (C) 5pm to 6pm + + (D) 2pm to 4pm' + target: (A) + - input: 'Today, Elizabeth went to the amusement park. Between what times could + they have gone? + + We know that: + + Elizabeth woke up at 7am. + + David saw Elizabeth fixing their computer at the electronic store from 1pm to + 2pm. + + Sarah saw Elizabeth playing tennis at the tennis court from 2pm to 3pm. + + Susan saw Elizabeth walking towards the Statue of Liberty from 3pm to 6pm. + + Andrew saw Elizabeth taking photos near the Eiffel Tower from 6pm to 9pm. + + Emily saw Elizabeth getting a coffee at the cafe from 9pm to 10pm. + + The amusement park was closed after 10pm. + + Between what times could Elizabeth have gone to the amusement park? + + Options: + + (A) 7am to 1pm + + (B) 9pm to 10pm + + (C) 1pm to 2pm + + (D) 3pm to 6pm' + target: (A) + - input: 'Today, Tiffany went to the beach. Between what times could they have gone? + + We know that: + + Tiffany woke up at 5am. + + Betty saw Tiffany getting a coffee at the cafe from 5am to 6am. + + Jessica saw Tiffany working at the office from 6am to 9am. + + John saw Tiffany stretching at a yoga studio from 9am to 12pm. + + Sean saw Tiffany sitting on a rooftop from 12pm to 2pm. + + Sarah saw Tiffany playing tennis at the tennis court from 2pm to 3pm. + + The beach was closed after 4pm. + + Between what times could Tiffany have gone to the beach? + + Options: + + (A) 9am to 12pm + + (B) 12pm to 2pm + + (C) 5am to 6am + + (D) 3pm to 4pm' + target: (D) +include: _fewshot_template_yaml +task: leaderboard_bbh_temporal_sequences diff --git a/lm_eval/tasks/leaderboard/bbh_mc/tracking_shuffled_objects_five_objects.yaml b/lm_eval/tasks/leaderboard/bbh_mc/tracking_shuffled_objects_five_objects.yaml new file mode 100644 index 0000000000000000000000000000000000000000..75a23a141198fb33161a0bba0db89292db3e418e --- /dev/null +++ b/lm_eval/tasks/leaderboard/bbh_mc/tracking_shuffled_objects_five_objects.yaml @@ -0,0 +1,57 @@ +dataset_name: tracking_shuffled_objects_five_objects +description: 'A task requiring determining the final positions of a set of objects + given their initial positions and a description of a sequence of swaps.' +doc_to_choice: ["(A)","(B)","(C)","(D)","(E)"] +fewshot_config: + sampler: first_n + samples: + - input: 'Alice, Bob, and Claire are playing a game. At the start of the game, they + are each holding a ball: Alice has a yellow ball, Bob has a blue ball, and Claire + has a pink ball. + + As the game progresses, pairs of players trade balls. First, Claire and Alice + swap balls. Then, Alice and Bob swap balls. Finally, Claire and Bob swap balls. + At the end of the game, Bob has the + + Options: + + (A) yellow ball + + (B) blue ball + + (C) pink ball' + target: (A) + - input: 'Alice, Bob, and Claire are playing a game. At the start of the game, they + are each holding a ball: Alice has a white ball, Bob has a purple ball, and + Claire has a pink ball. + + As the game progresses, pairs of players trade balls. First, Bob and Alice swap + balls. Then, Bob and Claire swap balls. Finally, Bob and Alice swap balls. At + the end of the game, Alice has the + + Options: + + (A) white ball + + (B) purple ball + + (C) pink ball' + target: (C) + - input: 'Alice, Bob, and Claire are dancers at a square dance. At the start of + a song, they each have a partner: Alice is dancing with Lola, Bob is dancing + with Rodrigo, and Claire is dancing with Patrick. + + Throughout the song, the dancers often trade partners. First, Alice and Bob + switch partners. Then, Claire and Bob switch partners. Finally, Bob and Alice + switch partners. At the end of the dance, Alice is dancing with + + Options: + + (A) Lola + + (B) Rodrigo + + (C) Patrick' + target: (C) +include: _fewshot_template_yaml +task: leaderboard_bbh_tracking_shuffled_objects_five_objects diff --git a/lm_eval/tasks/leaderboard/bbh_mc/tracking_shuffled_objects_seven_objects.yaml b/lm_eval/tasks/leaderboard/bbh_mc/tracking_shuffled_objects_seven_objects.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d3ea87ccfce90fc2d8ee96d2c62731711399c726 --- /dev/null +++ b/lm_eval/tasks/leaderboard/bbh_mc/tracking_shuffled_objects_seven_objects.yaml @@ -0,0 +1,57 @@ +dataset_name: tracking_shuffled_objects_seven_objects +description: 'A task requiring determining the final positions of a set of objects + given their initial positions and a description of a sequence of swaps.' +doc_to_choice: ["(A)","(B)","(C)","(D)","(E)","(F)","(G)"] +fewshot_config: + sampler: first_n + samples: + - input: 'Alice, Bob, and Claire are playing a game. At the start of the game, they + are each holding a ball: Alice has a yellow ball, Bob has a blue ball, and Claire + has a pink ball. + + As the game progresses, pairs of players trade balls. First, Claire and Alice + swap balls. Then, Alice and Bob swap balls. Finally, Claire and Bob swap balls. + At the end of the game, Bob has the + + Options: + + (A) yellow ball + + (B) blue ball + + (C) pink ball' + target: (A) + - input: 'Alice, Bob, and Claire are playing a game. At the start of the game, they + are each holding a ball: Alice has a white ball, Bob has a purple ball, and + Claire has a pink ball. + + As the game progresses, pairs of players trade balls. First, Bob and Alice swap + balls. Then, Bob and Claire swap balls. Finally, Bob and Alice swap balls. At + the end of the game, Alice has the + + Options: + + (A) white ball + + (B) purple ball + + (C) pink ball' + target: (C) + - input: 'Alice, Bob, and Claire are dancers at a square dance. At the start of + a song, they each have a partner: Alice is dancing with Lola, Bob is dancing + with Rodrigo, and Claire is dancing with Patrick. + + Throughout the song, the dancers often trade partners. First, Alice and Bob + switch partners. Then, Claire and Bob switch partners. Finally, Bob and Alice + switch partners. At the end of the dance, Alice is dancing with + + Options: + + (A) Lola + + (B) Rodrigo + + (C) Patrick' + target: (C) +include: _fewshot_template_yaml +task: leaderboard_bbh_tracking_shuffled_objects_seven_objects diff --git a/lm_eval/tasks/leaderboard/bbh_mc/tracking_shuffled_objects_three_objects.yaml b/lm_eval/tasks/leaderboard/bbh_mc/tracking_shuffled_objects_three_objects.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7ab88a3c7d21fdceb1964d236bbc9c3d21f3c0b8 --- /dev/null +++ b/lm_eval/tasks/leaderboard/bbh_mc/tracking_shuffled_objects_three_objects.yaml @@ -0,0 +1,57 @@ +dataset_name: tracking_shuffled_objects_three_objects +description: 'A task requiring determining the final positions of a set of objects + given their initial positions and a description of a sequence of swaps.' +doc_to_choice: ["(A)","(B)","(C)"] +fewshot_config: + sampler: first_n + samples: + - input: 'Alice, Bob, and Claire are playing a game. At the start of the game, they + are each holding a ball: Alice has a yellow ball, Bob has a blue ball, and Claire + has a pink ball. + + As the game progresses, pairs of players trade balls. First, Claire and Alice + swap balls. Then, Alice and Bob swap balls. Finally, Claire and Bob swap balls. + At the end of the game, Bob has the + + Options: + + (A) yellow ball + + (B) blue ball + + (C) pink ball' + target: (A) + - input: 'Alice, Bob, and Claire are playing a game. At the start of the game, they + are each holding a ball: Alice has a white ball, Bob has a purple ball, and + Claire has a pink ball. + + As the game progresses, pairs of players trade balls. First, Bob and Alice swap + balls. Then, Bob and Claire swap balls. Finally, Bob and Alice swap balls. At + the end of the game, Alice has the + + Options: + + (A) white ball + + (B) purple ball + + (C) pink ball' + target: (C) + - input: 'Alice, Bob, and Claire are dancers at a square dance. At the start of + a song, they each have a partner: Alice is dancing with Lola, Bob is dancing + with Rodrigo, and Claire is dancing with Patrick. + + Throughout the song, the dancers often trade partners. First, Alice and Bob + switch partners. Then, Claire and Bob switch partners. Finally, Bob and Alice + switch partners. At the end of the dance, Alice is dancing with + + Options: + + (A) Lola + + (B) Rodrigo + + (C) Patrick' + target: (C) +include: _fewshot_template_yaml +task: leaderboard_bbh_tracking_shuffled_objects_three_objects diff --git a/lm_eval/tasks/leaderboard/bbh_mc/web_of_lies.yaml b/lm_eval/tasks/leaderboard/bbh_mc/web_of_lies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e25e5f731abd96459d169dec2d1b9916f5cebe9e --- /dev/null +++ b/lm_eval/tasks/leaderboard/bbh_mc/web_of_lies.yaml @@ -0,0 +1,20 @@ +dataset_name: web_of_lies +description: 'Evaluate a random boolean function expressed as a word problem.' +doc_to_choice: ["Yes","No"] +fewshot_config: + sampler: first_n + samples: + - input: 'Question: Fidel tells the truth. Jerry says Fidel tells the truth. Vina + says Jerry tells the truth. Millicent says Vina lies. Raymond says Millicent + lies. Does Raymond tell the truth?' + target: 'Yes' + - input: 'Question: Kristian lies. Millie says Kristian lies. Maybelle says Millie + tells the truth. Fidel says Maybelle lies. Leda says Fidel lies. Does Leda tell + the truth?' + target: 'Yes' + - input: 'Question: Kristian tells the truth. Michaela says Kristian lies. Raymond + says Michaela tells the truth. Osvaldo says Raymond tells the truth. Jamey says + Osvaldo tells the truth. Does Jamey tell the truth?' + target: 'No' +include: _fewshot_template_yaml +task: leaderboard_bbh_web_of_lies diff --git a/lm_eval/tasks/leaderboard/gpqa/_leaderboard_gpqa.yaml b/lm_eval/tasks/leaderboard/gpqa/_leaderboard_gpqa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..448c14ac484bb6a1d7daeb15a5c19ec495f9a658 --- /dev/null +++ b/lm_eval/tasks/leaderboard/gpqa/_leaderboard_gpqa.yaml @@ -0,0 +1,5 @@ +group: leaderboard_gpqa +task: + - leaderboard_gpqa_diamond + - leaderboard_gpqa_extended + - leaderboard_gpqa_main diff --git a/lm_eval/tasks/leaderboard/gpqa/_template_yaml b/lm_eval/tasks/leaderboard/gpqa/_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..f4047292343005b69e259d0bb384126020ba16bc --- /dev/null +++ b/lm_eval/tasks/leaderboard/gpqa/_template_yaml @@ -0,0 +1,19 @@ +dataset_path: Idavidrein/gpqa +output_type: multiple_choice +process_docs: !function utils.process_docs +training_split: train +# Because huggingface dataset only has train split +validation_split: train +test_split: null +doc_to_text: "What is the correct answer to this question:{{Question}}\nChoices:\n(A) {{choice1}}\n(B) {{choice2}}\n(C) {{choice3}}\n(D) {{choice4}}\nAnswer: " +doc_to_target: answer +doc_to_choice: ["(A)", "(B)", "(C)", "(D)"] +num_fewshot: 0 +metric_list: + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 +fewshot_config: + sampler: first_n diff --git a/lm_eval/tasks/leaderboard/gpqa/gpqa_diamond_zeroshot.yaml b/lm_eval/tasks/leaderboard/gpqa/gpqa_diamond_zeroshot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a38921657d611da44b841a1707ff01d6eba071d5 --- /dev/null +++ b/lm_eval/tasks/leaderboard/gpqa/gpqa_diamond_zeroshot.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: gpqa_diamond +include: _template_yaml +task: leaderboard_gpqa_diamond diff --git a/lm_eval/tasks/leaderboard/gpqa/gpqa_extended_zeroshot.yaml b/lm_eval/tasks/leaderboard/gpqa/gpqa_extended_zeroshot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b481d960c8a186a99a72aaea1839f5efe2cdfdc4 --- /dev/null +++ b/lm_eval/tasks/leaderboard/gpqa/gpqa_extended_zeroshot.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: gpqa_extended +include: _template_yaml +task: leaderboard_gpqa_extended diff --git a/lm_eval/tasks/leaderboard/gpqa/gpqa_main_zeroshot.yaml b/lm_eval/tasks/leaderboard/gpqa/gpqa_main_zeroshot.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a15e45b63af0840f41ae97e15419f33ea61f52d4 --- /dev/null +++ b/lm_eval/tasks/leaderboard/gpqa/gpqa_main_zeroshot.yaml @@ -0,0 +1,4 @@ +# Generated by _generate_configs.py +dataset_name: gpqa_main +include: _template_yaml +task: leaderboard_gpqa_main diff --git a/lm_eval/tasks/leaderboard/gpqa/utils.py b/lm_eval/tasks/leaderboard/gpqa/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..c2317e02efd132aea27ec8c8fad284df55ccd382 --- /dev/null +++ b/lm_eval/tasks/leaderboard/gpqa/utils.py @@ -0,0 +1,38 @@ +import random +import re + +import datasets + + +def preprocess(text): + if text is None: + return " " + text = text.strip() + text = text.replace(" [title]", ". ") + text = re.sub("\\[.*?\\]", "", text) + text = text.replace(" ", " ") + return text + + +def process_docs(dataset: datasets.Dataset) -> datasets.Dataset: + def _process_doc(doc): + choices = [ + preprocess(doc["Incorrect Answer 1"]), + preprocess(doc["Incorrect Answer 2"]), + preprocess(doc["Incorrect Answer 3"]), + preprocess(doc["Correct Answer"]), + ] + + random.shuffle(choices) + correct_answer_index = choices.index(preprocess(doc["Correct Answer"])) + + out_doc = { + "choice1": choices[0], + "choice2": choices[1], + "choice3": choices[2], + "choice4": choices[3], + "answer": f"({chr(65 + correct_answer_index)})", + } + return out_doc + + return dataset.map(_process_doc) diff --git a/lm_eval/tasks/leaderboard/ifeval/_leaderboard_instruction_following.yaml b/lm_eval/tasks/leaderboard/ifeval/_leaderboard_instruction_following.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a1203b0069cf3e01003ac7f3b4cc2a351c90214c --- /dev/null +++ b/lm_eval/tasks/leaderboard/ifeval/_leaderboard_instruction_following.yaml @@ -0,0 +1,3 @@ +group: leaderboard_instruction_following +task: + - leaderboard_ifeval diff --git a/lm_eval/tasks/leaderboard/ifeval/ifeval.yaml b/lm_eval/tasks/leaderboard/ifeval/ifeval.yaml new file mode 100644 index 0000000000000000000000000000000000000000..72c3ba02b8b5c1d7464a7c9361b144ba1d201268 --- /dev/null +++ b/lm_eval/tasks/leaderboard/ifeval/ifeval.yaml @@ -0,0 +1,31 @@ +task: leaderboard_ifeval +dataset_path: wis-k/instruction-following-eval +dataset_name: null +output_type: generate_until +test_split: train +num_fewshot: 0 +doc_to_text: prompt +doc_to_target: 0 +generation_kwargs: + until: [] + do_sample: false + temperature: 0.0 + max_gen_toks: 1280 +process_results: !function utils.process_results +metric_list: + - metric: prompt_level_strict_acc + aggregation: mean + higher_is_better: true + - metric: inst_level_strict_acc + aggregation: !function utils.agg_inst_level_acc + higher_is_better: true + - metric: prompt_level_loose_acc + aggregation: mean + higher_is_better: true + - metric: inst_level_loose_acc + aggregation: !function utils.agg_inst_level_acc + higher_is_better: true +metadata: + version: 3.0 +fewshot_config: + sampler: first_n diff --git a/lm_eval/tasks/leaderboard/ifeval/instructions.py b/lm_eval/tasks/leaderboard/ifeval/instructions.py new file mode 100644 index 0000000000000000000000000000000000000000..a79cbba4f5e2fec10b8c0525a215b5e967c6fcfc --- /dev/null +++ b/lm_eval/tasks/leaderboard/ifeval/instructions.py @@ -0,0 +1,1612 @@ +# Copyright 2023 The Google Research Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Library of instructions.""" + +import collections +import json +import logging +import random +import re +import string +from typing import Dict, Optional, Sequence, Union + +import langdetect + +from lm_eval.tasks.ifeval import instructions_util + + +logger = logging.getLogger(__name__) + +_InstructionArgsDtype = Optional[Dict[str, Union[int, str, Sequence[str]]]] + +_LANGUAGES = instructions_util.LANGUAGE_CODES + +# The relational operation for comparison. +_COMPARISON_RELATION = ("less than", "at least") + +# The maximum number of sentences. +_MAX_NUM_SENTENCES = 20 + +# The number of placeholders. +_NUM_PLACEHOLDERS = 4 + +# The number of bullet lists. +_NUM_BULLETS = 5 + +# The options of constrained response. +_CONSTRAINED_RESPONSE_OPTIONS = ( + "My answer is yes.", + "My answer is no.", + "My answer is maybe.", +) + +# The options of starter keywords. +_STARTER_OPTIONS = ( + "I would say", + "My answer is", + "I believe", + "In my opinion", + "I think", + "I reckon", + "I feel", + "From my perspective", + "As I see it", + "According to me", + "As far as I'm concerned", + "To my understanding", + "In my view", + "My take on it is", + "As per my perception", +) + +# The options of ending keywords. +# TODO(jeffreyzhou) add more ending options +_ENDING_OPTIONS = ("Any other questions?", "Is there anything else I can help with?") + +# The number of highlighted sections. +_NUM_HIGHLIGHTED_SECTIONS = 4 + +# The section splitter. +_SECTION_SPLITER = ("Section", "SECTION") + +# The number of sections. +_NUM_SECTIONS = 5 + +# The number of paragraphs. +_NUM_PARAGRAPHS = 5 + +# The postscript marker. +_POSTSCRIPT_MARKER = ("P.S.", "P.P.S") + +# The number of keywords. +_NUM_KEYWORDS = 2 + +# The occurrences of a single keyword. +_KEYWORD_FREQUENCY = 3 + +# The occurrences of a single letter. +_LETTER_FREQUENCY = 10 + +# The occurrences of words with all capital letters. +_ALL_CAPITAL_WORD_FREQUENCY = 20 + +# The number of words in the response. +_NUM_WORDS_LOWER_LIMIT = 100 +_NUM_WORDS_UPPER_LIMIT = 500 + + +class Instruction: + """An instruction template.""" + + def __init__(self, instruction_id): + self.id = instruction_id + + def build_description(self, **kwargs): + raise NotImplementedError("`build_description` not implemented.") + + def get_instruction_args(self): + raise NotImplementedError("`get_instruction_args` not implemented.") + + def get_instruction_args_keys(self): + raise NotImplementedError("`get_instruction_args_keys` not implemented.") + + def check_following(self, value): + raise NotImplementedError("`check_following` not implemented.") + + +class ResponseLanguageChecker(Instruction): + """Check the language of the entire response.""" + + def build_description(self, *, language=None): + """Build the instruction description. + + Args: + language: A string representing the expected language of the response. The + language has to comply to the 97 types defined in + `langid.py` (https://pypi.org/project/langid/1.1.5/), which follows + ISO 639-1 codes (https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes); + for example, `en` for English, `zh` for Chinese, `fr` for French. + + Returns: + A string representing the instruction description. + """ + self._language = language + if self._language is None: + self._language = random.choice(list(_LANGUAGES.keys())) + # TODO(tianjianlu): opens the description generation to more choices. + self._description_pattern = ( + "Your ENTIRE response should be in {language} language, no other " + + "language is allowed." + ) + return self._description_pattern.format(language=_LANGUAGES[self._language]) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"language": self._language} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["language"] + + def check_following(self, value): + """Check if the language of the entire response follows the instruction. + + Args: + value: A string representing the response. + + Returns: + True if the language of `value` follows instruction; otherwise False. + """ + assert isinstance(value, str) + + try: + return langdetect.detect(value) == self._language + except langdetect.LangDetectException as e: + # Count as instruction is followed. + logging.error( + "Unable to detect language for text %s due to %s", value, e + ) # refex: disable=pytotw.037 + return True + + +class NumberOfSentences(Instruction): + """Check the number of sentences.""" + + def build_description(self, *, num_sentences=None, relation=None): + """Build the instruction description. + + Args: + num_sentences: An integer specifying the number of sentences as a + threshold. + relation: A string in (`less than`, `at least`), defining the relational + operator for comparison. + Two relational comparisons are supported for now: + if 'less than', the actual number of sentences < the threshold; + if 'at least', the actual number of sentences >= the threshold. + + Returns: + A string representing the instruction description. + """ + # The number of sentences as a threshold for comparison. + self._num_sentences_threshold = num_sentences + if self._num_sentences_threshold is None or self._num_sentences_threshold < 0: + self._num_sentences_threshold = random.randint(1, _MAX_NUM_SENTENCES) + + if relation is None: + self._comparison_relation = random.choice(_COMPARISON_RELATION) + elif relation not in _COMPARISON_RELATION: + raise ValueError( + "The supported relation for comparison must be in " + f"{_COMPARISON_RELATION}, but {relation} is given." + ) + else: + self._comparison_relation = relation + + self._description_pattern = ( + "Your response should contain {relation} {num_sentences} sentences." + ) + return self._description_pattern.format( + relation=self._comparison_relation, + num_sentences=self._num_sentences_threshold, + ) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return { + "num_sentences": self._num_sentences_threshold, + "relation": self._comparison_relation, + } + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["num_sentences", "relation"] + + def check_following(self, value): + """Check if the number of sentences follows the instruction. + + Args: + value: A string representing the response. + + Returns: + True if the response follows the instruction. + + Raise: + ValueError if the string in `instruction_args` is not in + [`less_than`, `at_least`]. + """ + num_sentences = instructions_util.count_sentences(value) + if self._comparison_relation == _COMPARISON_RELATION[0]: + return num_sentences < self._num_sentences_threshold + elif self._comparison_relation == _COMPARISON_RELATION[1]: + return num_sentences >= self._num_sentences_threshold + + +class PlaceholderChecker(Instruction): + """Check the placeholders in template writing.""" + + def build_description(self, *, num_placeholders=None): + """Build the instruction description. + + Args: + num_placeholders: An integer denoting the minimum number of + placeholders required in the response. + + Returns: + A string representing the instruction description. + """ + self._num_placeholders = num_placeholders + if self._num_placeholders is None or self._num_placeholders < 0: + self._num_placeholders = random.randint(1, _NUM_PLACEHOLDERS) + self._description_pattern = ( + "The response must contain at least {num_placeholders} placeholders " + + "represented by square brackets, such as [address]." + ) + return self._description_pattern.format(num_placeholders=self._num_placeholders) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"num_placeholders": self._num_placeholders} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["num_placeholders"] + + def check_following(self, value): + """Check if the number of placeholders follows the instruction. + + Args: + value: A string representing the response. + + Returns: + True if the actual number of placeholders in the response is greater than + or equal to `num_placeholders`; otherwise, False. + """ + placeholders = re.findall(r"\[.*?\]", value) + num_placeholders = len(placeholders) + return num_placeholders >= self._num_placeholders + + +class BulletListChecker(Instruction): + """Checks the bullet list in the prompt.""" + + def build_description(self, *, num_bullets=None): + """Build the instruction description. + + Args: + num_bullets: An integer specifying the exact number of bullet lists + that is required to appear in the response. + + Returns: + A string representing the instruction description. + """ + self._num_bullets = num_bullets + if self._num_bullets is None or self._num_bullets < 0: + self._num_bullets = random.randint(1, _NUM_BULLETS) + self._description_pattern = ( + "Your answer must contain exactly {num_bullets} bullet points. " + + "Use the markdown bullet points such as:\n" + + "* This is point 1. \n" + + "* This is point 2" + ) + return self._description_pattern.format(num_bullets=self._num_bullets) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"num_bullets": self._num_bullets} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["num_bullets"] + + def check_following(self, value): + r"""Check if the number of bullet lists meets the requirement. + + Args: + value: A string representing the response. The response is expected to + contain some bullet lists that start with `\*`. + + Returns: + True if the actual number of bullet lists in the response meets the + requirement. + """ + bullet_lists = re.findall(r"^\s*\*[^\*].*$", value, flags=re.MULTILINE) + bullet_lists_2 = re.findall(r"^\s*-.*$", value, flags=re.MULTILINE) + num_bullet_lists = len(bullet_lists) + len(bullet_lists_2) + return num_bullet_lists == self._num_bullets + + +class ConstrainedResponseChecker(Instruction): + """Checks the constrained response.""" + + def build_description(self): + """Build the instruction description.""" + # A sequence of string(s) representing the options of the expected response. + self._constrained_responses = _CONSTRAINED_RESPONSE_OPTIONS + self._description_pattern = ( + "Answer with one of the following options: {response_options}" + ) + return self._description_pattern.format( + response_options=self._constrained_responses + ) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response matches the constrained options. + + Args: + value: A string representing the response. + + Returns: + True if the actual response contains one of the options in the constrained + responses; otherwise False. + """ + value = value.strip() + for constrained_response in self._constrained_responses: + if constrained_response in value: + return True + return False + + +class ConstrainedStartChecker(Instruction): + """Checks the response start.""" + + def build_description(self, *, starter=None): + """Build the instruction description. + + Args: + starter: A string representing the keyword that the response should start + with. + + Returns: + A string representing the instruction description. + """ + self._starter = starter.strip() if isinstance(starter, str) else starter + if self._starter is None: + self._starter = random.choice(_STARTER_OPTIONS) + self._description_pattern = ( + "During the conversation, when it is your turn, " + + "please always start with {starter}" + ) + return self._description_pattern.format(starter=self._starter) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"starter": self._starter} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["starter"] + + def check_following(self, value): + """Checks if the response starts with the constrained keyword or phrase. + + Args: + value: A string representing the response. + + Returns: + True if the response starts with the given phrase or keyword that is + contained in `instruction_args`; otherwise, False. + """ + response_pattern = r"^\s*" + self._starter + r".*$" + response_with_constrained_start = re.search( + response_pattern, value, flags=re.MULTILINE + ) + return True if response_with_constrained_start else False + + +class HighlightSectionChecker(Instruction): + """Checks the highlighted section.""" + + def build_description(self, *, num_highlights=None): + """Build the instruction description. + + Args: + num_highlights: An integer specifying the minimum number of highlighted + sections. + + Returns: + A string representing the instruction description. + """ + self._num_highlights = num_highlights + if self._num_highlights is None or self._num_highlights < 0: + self._num_highlights = random.randint(1, _NUM_HIGHLIGHTED_SECTIONS) + + self._description_pattern = ( + "Highlight at least {num_highlights} sections in your answer with " + + "markdown, i.e. *highlighted section*." + ) + + return self._description_pattern.format(num_highlights=self._num_highlights) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"num_highlights": self._num_highlights} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["num_highlights"] + + def check_following(self, value): + """Checks if the number of highlighted sections meets the requirement. + + Args: + value: a string representing the response. The response is expected to + contain highlighted sections in the format of *highlighted*. + + Returns: + True if the actual number of highlighted sections in the format of + *highlighted sections* meets the minimum requirement; otherwise False. + """ + num_highlights = 0 + highlights = re.findall(r"\*[^\n\*]*\*", value) + double_highlights = re.findall(r"\*\*[^\n\*]*\*\*", value) + for highlight in highlights: + if highlight.strip("*").strip(): + num_highlights += 1 + for highlight in double_highlights: + if highlight.removeprefix("**").removesuffix("**").strip(): + num_highlights += 1 + + return num_highlights >= self._num_highlights + + +class SectionChecker(Instruction): + """Checks the sections.""" + + def build_description(self, *, section_spliter=None, num_sections=None): + """Build the instruction description. + + Args: + section_spliter: A string represents the section spliter keyword that + marks a new section, i.e., `Section` or `SECTION`. + num_sections: An integer specifying the number of sections. + + Returns: + A string representing the instruction description. + """ + self._section_spliter = ( + section_spliter.strip() + if isinstance(section_spliter, str) + else section_spliter + ) + if self._section_spliter is None: + self._section_spliter = random.choice(_SECTION_SPLITER) + + self._num_sections = num_sections + if self._num_sections is None or self._num_sections < 0: + self._num_sections = random.randint(1, _NUM_SECTIONS) + + self._description_pattern = ( + "Your response must have {num_sections} sections. Mark the beginning " + + "of each section with {section_spliter} X, such as:\n" + + "{section_spliter} 1\n" + + "[content of section 1]\n" + + "{section_spliter} 2\n" + + "[content of section 2]" + ) + + return self._description_pattern.format( + num_sections=self._num_sections, section_spliter=self._section_spliter + ) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return { + "section_spliter": self._section_spliter, + "num_sections": self._num_sections, + } + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["section_spliter", "num_sections"] + + def check_following(self, value): + """Checks the response contains multiple sections. + + Args: + value: A string representing the response. The response is expected + to contain multiple sections (number of sections is greater than 1). + A new section starts with `Section 1`, where the number denotes the + section index. + + Returns: + True if the number of sections in the response is greater than or equal to + the minimum number of sections; otherwise, False. + """ + section_splitter_patten = r"\s?" + self._section_spliter + r"\s?\d+\s?" + sections = re.split(section_splitter_patten, value) + num_sections = len(sections) - 1 + return num_sections >= self._num_sections + + +class ParagraphChecker(Instruction): + """Checks the paragraphs.""" + + def build_description(self, *, num_paragraphs=None): + """Build the instruction description. + + Args: + num_paragraphs: An integer specifying the number of paragraphs. + + Returns: + A string representing the instruction description. + """ + self._num_paragraphs = num_paragraphs + if self._num_paragraphs is None or self._num_paragraphs < 0: + self._num_paragraphs = random.randint(1, _NUM_PARAGRAPHS) + + self._description_pattern = ( + "There should be {num_paragraphs} paragraphs. " + + "Paragraphs are separated with the markdown divider: ***" + ) + + return self._description_pattern.format(num_paragraphs=self._num_paragraphs) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"num_paragraphs": self._num_paragraphs} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["num_paragraphs"] + + def check_following(self, value): + """Checks the response contains required number of paragraphs. + + Args: + value: A string representing the response. The response may contain + paragraphs that are separated by the markdown divider: `***`. + + Returns: + True if the actual number of paragraphs is the same as required; + otherwise, False. + """ + paragraphs = re.split(r"\s?\*\*\*\s?", value) + num_paragraphs = len(paragraphs) + + for index, paragraph in enumerate(paragraphs): + if not paragraph.strip(): + if index == 0 or index == len(paragraphs) - 1: + num_paragraphs -= 1 + else: + return False + + return num_paragraphs == self._num_paragraphs + + +class PostscriptChecker(Instruction): + """Checks the postscript.""" + + def build_description(self, *, postscript_marker=None): + """Build the instruction description. + + Args: + postscript_marker: A string containing the keyword that marks the start + of the postscript section. + + Returns: + A string representing the instruction description. + """ + self._postscript_marker = ( + postscript_marker.strip() + if isinstance(postscript_marker, str) + else postscript_marker + ) + if self._postscript_marker is None: + self._postscript_marker = random.choice(_POSTSCRIPT_MARKER) + + self._description_pattern = ( + "At the end of your response, please explicitly add a postscript " + + "starting with {postscript}" + ) + + return self._description_pattern.format(postscript=self._postscript_marker) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"postscript_marker": self._postscript_marker} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["postscript_marker"] + + def check_following(self, value): + """Checks if the response follows the postscript format. + + Args: + value: a string representing the response. The response is expected to + contain a postscript section. + + Returns: + True if the response contains a postscript section starting with + the keyword containing in the `instruction_args`; otherwise False. + """ + value = value.lower() + if self._postscript_marker == "P.P.S": + postscript_pattern = r"\s*p\.\s?p\.\s?s.*$" + elif self._postscript_marker == "P.S.": + postscript_pattern = r"\s*p\.\s?s\..*$" + else: + postscript_pattern = r"\s*" + self._postscript_marker.lower() + r".*$" + postscript = re.findall(postscript_pattern, value, flags=re.MULTILINE) + return True if postscript else False + + +class RephraseChecker(Instruction): + """Checks the rephrase.""" + + def build_description(self, *, original_message): + """Build the instruction description. + + Args: + original_message: A string representing the original message. The + rephrased response should only change its words/sentences in between + its two asterisks, for example, *change me*. Both original and rephrased + messages should contain the changes in the form of *change me*. + + Returns: + A string representing the instruction description. + """ + if not self.is_change(original_message): + raise ValueError( + f"Message {original_message} does not contain changes " + "in the form of *change me*." + ) + + self._reference_without_change = original_message + self._description = ( + "Rephrasing: Your rephrased response should only" + + "change the words/sentences in between two asterisks" + + "such as *change me*." + ) + return self._description + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"original_message": self._reference_without_change} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["original_message"] + + def check_following(self, value): + r"""Checks if the rephrasing follows the instruction. + + Args: + value: A string representing the response, which is expected to rephras + the string of `instruction_args`. + + Returns: + True if `value` and `instruction_args` only differ by the words/sentences + in between two asterisks such as *change me*; otherwise, False. + """ + + if not self.is_change(value): + raise ValueError( + f"value {value} does not contain " "changes in the form of *change me*." + ) + + response_without_changes = self.strip_changes(value) + reference_without_changes = self.strip_changes(self._reference_without_change) + + return response_without_changes == reference_without_changes + + def is_change(self, response): + """Check if there is change in the response in the form of *change me*.""" + return re.search(r"\*.*\*", response) + + def strip_changes(self, response): + """Strips off the changes.""" + return re.sub(r"\*.*\*", "", response) + + +class KeywordChecker(Instruction): + """Check the exisitence of certain keywords.""" + + def build_description(self, *, keywords=None): + """Build the instruction description. + + Args: + keywords: A sequence of strings representing the keywords that are + expected in the response. + + Returns: + A string representing the instruction description. + """ + + if not keywords: + self._keywords = instructions_util.generate_keywords( + num_keywords=_NUM_KEYWORDS + ) + else: + self._keywords = keywords + self._keywords = sorted(self._keywords) + + self._description_pattern = "Include keywords {keywords} in the response." + + return self._description_pattern.format(keywords=self._keywords) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"keywords": self._keywords} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["keywords"] + + def check_following(self, value): + """Check if the response contain the expected keywords.""" + for keyword in self._keywords: + if not re.search(keyword, value, flags=re.IGNORECASE): + return False + return True + + +class KeywordFrequencyChecker(Instruction): + """Check the keyword frequency.""" + + def build_description(self, *, keyword=None, frequency=None, relation=None): + """Build the instruction description. + + Args: + keyword: A string representing a keyword that is expected in the response. + frequency: An integer specifying the number of times `keyword` is expected + to appear in the response. + relation: A string in (`less than`, `at least`), defining the relational + operator for comparison. + Two relational comparisons are supported for now: + if 'less than', the actual number of occurrences < frequency; + if 'at least', the actual number of occurrences >= frequency. + + Returns: + A string representing the instruction description. + """ + if not keyword: + self._keyword = instructions_util.generate_keywords(num_keywords=1)[0] + else: + self._keyword = keyword.strip() + + self._frequency = frequency + if self._frequency is None or self._frequency < 0: + self._frequency = random.randint(1, _KEYWORD_FREQUENCY) + + if relation is None: + self._comparison_relation = random.choice(_COMPARISON_RELATION) + elif relation not in _COMPARISON_RELATION: + raise ValueError( + "The supported relation for comparison must be in " + f"{_COMPARISON_RELATION}, but {relation} is given." + ) + else: + self._comparison_relation = relation + + self._description_pattern = ( + "In your response, the word {keyword} should appear {relation} " + + "{frequency} times." + ) + + return self._description_pattern.format( + keyword=self._keyword, + relation=self._comparison_relation, + frequency=self._frequency, + ) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return { + "keyword": self._keyword, + "frequency": self._frequency, + "relation": self._comparison_relation, + } + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["keyword", "frequency", "relation"] + + def check_following(self, value): + """Checks if the response contain the keyword with required frequency.""" + actual_occurrences = len(re.findall(self._keyword, value, flags=re.IGNORECASE)) + + if self._comparison_relation == _COMPARISON_RELATION[0]: + return actual_occurrences < self._frequency + elif self._comparison_relation == _COMPARISON_RELATION[1]: + return actual_occurrences >= self._frequency + + +class NumberOfWords(Instruction): + """Checks the number of words.""" + + def build_description(self, *, num_words=None, relation=None): + """Build the instruction description. + + Args: + num_words: An integer specifying the number of words contained in the + response. + relation: A string in (`less than`, `at least`), defining the relational + operator for comparison. + Two relational comparisons are supported for now: + if 'less than', the actual number of words < num_words; + if 'at least', the actual number of words >= num_words. + + Returns: + A string representing the instruction description. + """ + + self._num_words = num_words + if self._num_words is None or self._num_words < 0: + self._num_words = random.randint( + _NUM_WORDS_LOWER_LIMIT, _NUM_WORDS_UPPER_LIMIT + ) + + if relation is None: + self._comparison_relation = random.choice(_COMPARISON_RELATION) + elif relation not in _COMPARISON_RELATION: + raise ValueError( + "The supported relation for comparison must be in " + f"{_COMPARISON_RELATION}, but {relation} is given." + ) + else: + self._comparison_relation = relation + + self._description_pattern = "Answer with {relation} {num_words} words." + + return self._description_pattern.format( + relation=self._comparison_relation, num_words=self._num_words + ) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"num_words": self._num_words, "relation": self._comparison_relation} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["num_words", "relation"] + + def check_following(self, value): + """Checks if the response contains the expected number of words.""" + num_words = instructions_util.count_words(value) + + if self._comparison_relation == _COMPARISON_RELATION[0]: + return num_words < self._num_words + elif self._comparison_relation == _COMPARISON_RELATION[1]: + return num_words >= self._num_words + + +class JsonFormat(Instruction): + """Check the Json format.""" + + def build_description(self): + self._description_pattern = ( + "Entire output should be wrapped in JSON format. You can use markdown" + " ticks such as ```." + ) + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + value = ( + value.strip() + .removeprefix("```json") + .removeprefix("```Json") + .removeprefix("```JSON") + .removeprefix("```") + .removesuffix("```") + .strip() + ) + try: + json.loads(value) + except ValueError: + return False + return True + + +class ParagraphFirstWordCheck(Instruction): + """Check the paragraph and the first word of the nth paragraph.""" + + def build_description( + self, num_paragraphs=None, nth_paragraph=None, first_word=None + ): + r"""Build the instruction description. + + Args: + num_paragraphs: An integer indicating the number of paragraphs expected + in the response. A paragraph is a subset of the string that is + expected to be separated by '\n\n'. + nth_paragraph: An integer indicating the paragraph number that we look at. + Note that n starts from 1. + first_word: A string that represent the first word of the bth paragraph. + + Returns: + A string representing the instruction description. + """ + self._num_paragraphs = num_paragraphs + if self._num_paragraphs is None or self._num_paragraphs < 0: + self._num_paragraphs = random.randint(1, _NUM_PARAGRAPHS) + + self._nth_paragraph = nth_paragraph + if ( + self._nth_paragraph is None + or self._nth_paragraph <= 0 + or self._nth_paragraph > self._num_paragraphs + ): + self._nth_paragraph = random.randint(1, self._num_paragraphs + 1) + + self._first_word = first_word + if self._first_word is None: + self._first_word = instructions_util.generate_keywords(num_keywords=1)[0] + self._first_word = self._first_word.lower() + + self._description_pattern = ( + "There should be {num_paragraphs} paragraphs. " + + "Paragraphs and only paragraphs are separated with each other by two " + + "new lines as if it was '\\n\\n' in python. " + + "Paragraph {nth_paragraph} must start with word {first_word}." + ) + + return self._description_pattern.format( + num_paragraphs=self._num_paragraphs, + nth_paragraph=self._nth_paragraph, + first_word=self._first_word, + ) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return { + "num_paragraphs": self._num_paragraphs, + "nth_paragraph": self._nth_paragraph, + "first_word": self._first_word, + } + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["num_paragraphs", "nth_paragraph", "first_word"] + + def check_following(self, value): + """Checks for required number of paragraphs and correct first word. + + Args: + value: a string representing the response. The response may contain + paragraphs that are separated by two new lines and the first word of + the nth paragraph will have to match a specified word. + + Returns: + True if the number of paragraphs is the same as required and the first + word of the specified paragraph is the same as required. Otherwise, false. + """ + + paragraphs = re.split(r"\n\n", value) + num_paragraphs = len(paragraphs) + + for paragraph in paragraphs: + if not paragraph.strip(): + num_paragraphs -= 1 + + # check that index doesn't go out of bounds + if self._nth_paragraph <= num_paragraphs: + paragraph = paragraphs[self._nth_paragraph - 1].strip() + if not paragraph: + return False + else: + return False + + first_word = "" + punctuation = {".", ",", "?", "!", "'", '"'} + + # get first word and remove punctuation + word = paragraph.split()[0].strip() + # TODO(jeffrey): make more complex? + word = word.lstrip("'") + word = word.lstrip('"') + + for letter in word: + if letter in punctuation: + break + first_word += letter.lower() + + return num_paragraphs == self._num_paragraphs and first_word == self._first_word + + +# TODO(jeffrey) add relation - at least/at most? +class KeySentenceChecker(Instruction): + """Check the existence of certain key sentences.""" + + def build_description(self, key_sentences=None, num_sentences=None): + """Build the instruction description. + + Args: + key_sentences: A sequences of strings representing the key sentences that + are expected in the response. + num_sentences: The number of key sentences that are expected to be seen in + the response. + + Returns: + A string representing the instruction description. + """ + + if not key_sentences: + # TODO(jeffrey) make a generate sentences function? wonderwords package + self._key_sentences = set(["For now, this is fine."]) + else: + self._key_sentences = key_sentences + + if not num_sentences: + self._num_sentences = random.randint(1, len(self._key_sentences)) + else: + self._num_sentences = num_sentences + + self._description_pattern = ( + "Include {num_sentences} of the following sentences {key_sentences}" + ) + + return self._description_pattern.format( + num_sentences=self._num_sentences, key_sentences=self._key_sentences + ) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return { + "num_sentences": self._num_sentences, + "key_sentences": list(self._key_sentences), + } + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["num_sentences", "key_sentences"] + + def check_following(self, value): + """Checks if the response contains the expected key sentences.""" + count = 0 + sentences = instructions_util.split_into_sentences(value) + for sentence in self._key_sentences: + if sentence in sentences: + count += 1 + + return count == self._num_sentences + + +class ForbiddenWords(Instruction): + """Checks that specified words are not used in response.""" + + def build_description(self, forbidden_words=None): + """Build the instruction description. + + Args: + forbidden_words: A sequences of strings representing words that are not + allowed in the response. + + Returns: + A string representing the instruction description. + """ + + if not forbidden_words: + self._forbidden_words = instructions_util.generate_keywords( + num_keywords=_NUM_KEYWORDS + ) + else: + self._forbidden_words = list(set(forbidden_words)) + self._forbidden_words = sorted(self._forbidden_words) + self._description_pattern = ( + "Do not include keywords {forbidden_words} in the response." + ) + + return self._description_pattern.format(forbidden_words=self._forbidden_words) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return {"forbidden_words": self._forbidden_words} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["forbidden_words"] + + def check_following(self, value): + """Check if the response does not contain the expected keywords.""" + for word in self._forbidden_words: + if re.search(r"\b" + word + r"\b", value, flags=re.IGNORECASE): + return False + return True + + +class RephraseParagraph(Instruction): + """Checks that the paragraph is rephrased.""" + + def build_description(self, *, original_paragraph, low, high): + """Builds the instruction description. + + Args: + original_paragraph: A string presenting the original paragraph. The + rephrases response should have betweeb low-high words in common. + low: An integer presenting the lower bound of similar words. + high: An integer representing the upper bound of similar words. + + Returns: + A string representing the instruction description. + """ + # TODO(jeffrey) make more encompassing + self._original_paragraph = original_paragraph + self._low = low + self._high = high + + self._description = ( + "Rephrase the following paragraph: " + + "{original_paragraph}\nYour response should have " + + "between {low} and {high} of the same words. " + + "Words are the same if and only if all of the " + + "letters, ignoring cases, are the same. For " + + "example, 'run' is the same as 'Run' but different " + + "to 'ran'." + ) + + return self._description.format( + original_paragraph=original_paragraph, low=self._low, high=self._high + ) + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return { + "original_paragraph": self._original_paragraph, + "low": self._low, + "high": self._high, + } + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["original_paragraph", "low", "high"] + + def check_following(self, value): + val_words = re.findall(r"\w+", value.lower()) + original_words = re.findall(r"\w+", self._original_paragraph.lower()) + similar_words = 0 + + dict_val = collections.Counter(val_words) + dict_original = collections.Counter(original_words) + + for word in dict_original: + similar_words += min(dict_original[word], dict_val[word]) + + return similar_words >= self._low and similar_words <= self._high + + +class TwoResponsesChecker(Instruction): + """Check that two responses were given.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = ( + "Give two different responses. Responses and only responses should" + " be separated by 6 asterisk symbols: ******." + ) + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of `build_description`.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response has two different answers. + + Args: + value: A string representing the response. + + Returns: + True if two responses are detected and false otherwise. + """ + valid_responses = list() + responses = value.split("******") + for index, response in enumerate(responses): + if not response.strip(): + if index != 0 and index != len(responses) - 1: + return False + else: + valid_responses.append(response) + return ( + len(valid_responses) == 2 + and valid_responses[0].strip() != valid_responses[1].strip() + ) + + +class RepeatPromptThenAnswer(Instruction): + """Checks that Prompt is first repeated then answered.""" + + def build_description(self, *, prompt_to_repeat=None): + """Build the instruction description. + + Args: + prompt_to_repeat: The prompt that is meant to be repeated. + + Returns: + A string representing the instruction description. + """ + if not prompt_to_repeat: + raise ValueError("prompt_to_repeat must be set.") + else: + self._prompt_to_repeat = prompt_to_repeat + self._description_pattern = ( + "First repeat the request word for word without change," + " then give your answer (1. do not say any words or characters" + " before repeating the request; 2. the request you need to repeat" + " does not include this sentence)" + ) + return self._description_pattern + + def get_instruction_args(self): + return {"prompt_to_repeat": self._prompt_to_repeat} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["prompt_to_repeat"] + + def check_following(self, value): + if value.strip().lower().startswith(self._prompt_to_repeat.strip().lower()): + return True + return False + + +class EndChecker(Instruction): + """Checks that the prompt ends with a given phrase.""" + + def build_description(self, *, end_phrase=None): + """Build the instruction description. + + Args: + end_phrase: A string representing the phrase the response should end with. + + Returns: + A string representing the instruction description. + """ + self._end_phrase = ( + end_phrase.strip() if isinstance(end_phrase, str) else end_phrase + ) + if self._end_phrase is None: + self._end_phrase = random.choice(_ENDING_OPTIONS) + self._description_pattern = ( + "Finish your response with this exact phrase {ender}. " + "No other words should follow this phrase." + ) + return self._description_pattern.format(ender=self._end_phrase) + + def get_instruction_args(self): + return {"end_phrase": self._end_phrase} + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["end_phrase"] + + def check_following(self, value): + """Checks if the response ends with the expected phrase.""" + value = value.strip().strip('"').lower() + self._end_phrase = self._end_phrase.strip().lower() + return value.endswith(self._end_phrase) + + +class TitleChecker(Instruction): + """Checks the response for a title.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = ( + "Your answer must contain a title, wrapped in double angular brackets," + " such as <>." + ) + return self._description_pattern + + def get_instruction_args(self): + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response contains a title.""" + pattern = r"<<[^\n]+>>" + re_pattern = re.compile(pattern) + titles = re.findall(re_pattern, value) + + for title in titles: + if title.lstrip("<").rstrip(">").strip(): + return True + return False + + +class LetterFrequencyChecker(Instruction): + """Checks letter frequency.""" + + def build_description(self, *, letter=None, let_frequency=None, let_relation=None): + """Build the instruction description. + + Args: + letter: A string representing a letter that is expected in the response. + let_frequency: An integer specifying the number of times `keyword` is + expected to appear in the response. + let_relation: A string in (`less than`, `at least`), defining the + relational operator for comparison. Two relational comparisons are + supported for now; if 'less than', the actual number of + occurrences < frequency; if 'at least', the actual number of + occurrences >= frequency. + + Returns: + A string representing the instruction description. + """ + if ( + not letter + or len(letter) > 1 + or ord(letter.lower()) < 97 + or ord(letter.lower()) > 122 + ): + self._letter = random.choice(list(string.ascii_letters)) + else: + self._letter = letter.strip() + self._letter = self._letter.lower() + + self._frequency = let_frequency + if self._frequency is None or self._frequency < 0: + self._frequency = random.randint(1, _LETTER_FREQUENCY) + + if let_relation is None: + self._comparison_relation = random.choice(_COMPARISON_RELATION) + elif let_relation not in _COMPARISON_RELATION: + raise ValueError( + "The supported relation for comparison must be in " + f"{_COMPARISON_RELATION}, but {let_relation} is given." + ) + else: + self._comparison_relation = let_relation + + self._description_pattern = ( + "In your response, the letter {letter} should appear {let_relation}" + " {let_frequency} times." + ) + + return self._description_pattern.format( + letter=self._letter, + let_frequency=self._frequency, + let_relation=self._comparison_relation, + ) + + def get_instruction_args(self): + """Returns the keyword args of build description.""" + return { + "letter": self._letter, + "let_frequency": self._frequency, + "let_relation": self._comparison_relation, + } + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["letter", "let_frequency", "let_relation"] + + def check_following(self, value): + """Checks that the response contains the letter at the right frequency.""" + value = value.lower() + letters = collections.Counter(value) + + if self._comparison_relation == _COMPARISON_RELATION[0]: + return letters[self._letter] < self._frequency + else: + return letters[self._letter] >= self._frequency + + +class CapitalLettersEnglishChecker(Instruction): + """Checks that the response is in english and is in all capital letters.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = ( + "Your entire response should be in English, and in all capital letters." + ) + return self._description_pattern + + def get_instruction_args(self): + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks that the response is in English and in all capital letters.""" + assert isinstance(value, str) + + try: + return value.isupper() and langdetect.detect(value) == "en" + except langdetect.LangDetectException as e: + # Count as instruction is followed. + logging.error( + "Unable to detect language for text %s due to %s", value, e + ) # refex: disable=pytotw.037 + return True + + +class LowercaseLettersEnglishChecker(Instruction): + """Checks that the response is in english and is in all lowercase letters.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = ( + "Your entire response should be in English, and in all lowercase" + " letters. No capital letters are allowed." + ) + return self._description_pattern + + def get_instruction_args(self): + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks that the response is in English and in all lowercase letters.""" + assert isinstance(value, str) + + try: + return value.islower() and langdetect.detect(value) == "en" + except langdetect.LangDetectException as e: + # Count as instruction is followed. + logging.error( + "Unable to detect language for text %s due to %s", value, e + ) # refex: disable=pytotw.037 + return True + + +class CommaChecker(Instruction): + """Checks the response for no commas.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = ( + "In your entire response, refrain from the use of any commas." + ) + return self._description_pattern + + def get_instruction_args(self): + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks that the response does not contain commas.""" + return not re.search(r"\,", value) + + +class CapitalWordFrequencyChecker(Instruction): + """Checks frequency of words with all capital letters.""" + + def build_description( + self, + capital_frequency=None, + capital_relation=None, + ): + """Build the instruction description. + + Args: + capital_frequency: An integer that represents the number of words that + should be in all capital letters. + capital_relation: A string that is 'at least' or 'at most' that refers to + the frequency. + + Returns: + A string representing the instruction description. + """ + self._frequency = capital_frequency + if self._frequency is None: + self._frequency = random.randint(1, _ALL_CAPITAL_WORD_FREQUENCY) + + self._comparison_relation = capital_relation + if capital_relation is None: + self._comparison_relation = random.choice(_COMPARISON_RELATION) + elif capital_relation not in _COMPARISON_RELATION: + raise ValueError( + "The supported relation for comparison must be in " + f"{_COMPARISON_RELATION}, but {capital_relation} is given." + ) + + self._description_pattern = ( + "In your response, words with all capital letters should appear" + " {relation} {frequency} times." + ) + + return self._description_pattern.format( + frequency=self._frequency, relation=self._comparison_relation + ) + + def get_instruction_args(self): + """Returns the keyword args of build description.""" + return { + "capital_frequency": self._frequency, + "capital_relation": self._comparison_relation, + } + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return ["capital_frequency", "capital_relation"] + + def check_following(self, value): + """Checks the frequency of words with all capital letters.""" + # Hyphenated words will count as one word + words = instructions_util.nltk.word_tokenize(value) + capital_words = [word for word in words if word.isupper()] + + capital_words = len(capital_words) + + if self._comparison_relation == _COMPARISON_RELATION[0]: + return capital_words < self._frequency + else: + return capital_words >= self._frequency + + +class QuotationChecker(Instruction): + """Checks response is wrapped with double quotation marks.""" + + def build_description(self): + """Build the instruction description.""" + self._description_pattern = ( + "Wrap your entire response with double quotation marks." + ) + return self._description_pattern + + def get_instruction_args(self): + """Returns the keyword args of build description.""" + return None + + def get_instruction_args_keys(self): + """Returns the args keys of `build_description`.""" + return [] + + def check_following(self, value): + """Checks if the response is wrapped with double quotation marks.""" + value = value.strip() + return len(value) > 1 and value[0] == '"' and value[-1] == '"' diff --git a/lm_eval/tasks/leaderboard/ifeval/instructions_registry.py b/lm_eval/tasks/leaderboard/ifeval/instructions_registry.py new file mode 100644 index 0000000000000000000000000000000000000000..00d9a1de1985beacead34215952ecf4642d1ea35 --- /dev/null +++ b/lm_eval/tasks/leaderboard/ifeval/instructions_registry.py @@ -0,0 +1,168 @@ +# Copyright 2023 The Google Research Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Registry of all instructions.""" + +from lm_eval.tasks.ifeval import instructions + + +_KEYWORD = "keywords:" + +_LANGUAGE = "language:" + +_LENGTH = "length_constraints:" + +_CONTENT = "detectable_content:" + +_FORMAT = "detectable_format:" + +_MULTITURN = "multi-turn:" + +_COMBINATION = "combination:" + +_STARTEND = "startend:" + +_CHANGE_CASES = "change_case:" + +_PUNCTUATION = "punctuation:" + +INSTRUCTION_DICT = { + _KEYWORD + "existence": instructions.KeywordChecker, + _KEYWORD + "frequency": instructions.KeywordFrequencyChecker, + # TODO(jeffreyzhou): make a proper set of sentences to choose from + # _KEYWORD + "key_sentences": instructions.KeySentenceChecker, + _KEYWORD + "forbidden_words": instructions.ForbiddenWords, + _KEYWORD + "letter_frequency": instructions.LetterFrequencyChecker, + _LANGUAGE + "response_language": instructions.ResponseLanguageChecker, + _LENGTH + "number_sentences": instructions.NumberOfSentences, + _LENGTH + "number_paragraphs": instructions.ParagraphChecker, + _LENGTH + "number_words": instructions.NumberOfWords, + _LENGTH + "nth_paragraph_first_word": instructions.ParagraphFirstWordCheck, + _CONTENT + "number_placeholders": instructions.PlaceholderChecker, + _CONTENT + "postscript": instructions.PostscriptChecker, + _FORMAT + "number_bullet_lists": instructions.BulletListChecker, + # TODO(jeffreyzhou): Pre-create paragraph or use prompt to replace + # _CONTENT + "rephrase_paragraph": instructions.RephraseParagraph, + _FORMAT + "constrained_response": instructions.ConstrainedResponseChecker, + _FORMAT + "number_highlighted_sections": (instructions.HighlightSectionChecker), + _FORMAT + "multiple_sections": instructions.SectionChecker, + # TODO(tianjianlu): Re-enable rephrasing with preprocessing the message. + # _FORMAT + "rephrase": instructions.RephraseChecker, + _FORMAT + "json_format": instructions.JsonFormat, + _FORMAT + "title": instructions.TitleChecker, + # TODO(tianjianlu): Re-enable with specific prompts. + # _MULTITURN + "constrained_start": instructions.ConstrainedStartChecker, + _COMBINATION + "two_responses": instructions.TwoResponsesChecker, + _COMBINATION + "repeat_prompt": instructions.RepeatPromptThenAnswer, + _STARTEND + "end_checker": instructions.EndChecker, + _CHANGE_CASES + "capital_word_frequency": instructions.CapitalWordFrequencyChecker, + _CHANGE_CASES + "english_capital": instructions.CapitalLettersEnglishChecker, + _CHANGE_CASES + "english_lowercase": instructions.LowercaseLettersEnglishChecker, + _PUNCTUATION + "no_comma": instructions.CommaChecker, + _STARTEND + "quotation": instructions.QuotationChecker, +} + +INSTRUCTION_CONFLICTS = { + _KEYWORD + "existence": {_KEYWORD + "existence"}, + _KEYWORD + "frequency": {_KEYWORD + "frequency"}, + # TODO(jeffreyzhou): make a proper set of sentences to choose from + # _KEYWORD + "key_sentences": instructions.KeySentenceChecker, + _KEYWORD + "forbidden_words": {_KEYWORD + "forbidden_words"}, + _KEYWORD + "letter_frequency": {_KEYWORD + "letter_frequency"}, + _LANGUAGE + "response_language": { + _LANGUAGE + "response_language", + _FORMAT + "multiple_sections", + _KEYWORD + "existence", + _KEYWORD + "frequency", + _KEYWORD + "forbidden_words", + _STARTEND + "end_checker", + _CHANGE_CASES + "english_capital", + _CHANGE_CASES + "english_lowercase", + }, + _LENGTH + "number_sentences": {_LENGTH + "number_sentences"}, + _LENGTH + "number_paragraphs": { + _LENGTH + "number_paragraphs", + _LENGTH + "nth_paragraph_first_word", + _LENGTH + "number_sentences", + _LENGTH + "nth_paragraph_first_word", + }, + _LENGTH + "number_words": {_LENGTH + "number_words"}, + _LENGTH + "nth_paragraph_first_word": { + _LENGTH + "nth_paragraph_first_word", + _LENGTH + "number_paragraphs", + }, + _CONTENT + "number_placeholders": {_CONTENT + "number_placeholders"}, + _CONTENT + "postscript": {_CONTENT + "postscript"}, + _FORMAT + "number_bullet_lists": {_FORMAT + "number_bullet_lists"}, + # TODO(jeffreyzhou): Pre-create paragraph or use prompt to replace + # _CONTENT + "rephrase_paragraph": instructions.RephraseParagraph, + _FORMAT + "constrained_response": set(INSTRUCTION_DICT.keys()), + _FORMAT + "number_highlighted_sections": {_FORMAT + "number_highlighted_sections"}, + _FORMAT + "multiple_sections": { + _FORMAT + "multiple_sections", + _LANGUAGE + "response_language", + _FORMAT + "number_highlighted_sections", + }, + # TODO(tianjianlu): Re-enable rephrasing with preprocessing the message. + # _FORMAT + "rephrase": instructions.RephraseChecker, + _FORMAT + "json_format": set(INSTRUCTION_DICT.keys()).difference( + {_KEYWORD + "forbidden_words", _KEYWORD + "existence"} + ), + _FORMAT + "title": {_FORMAT + "title"}, + # TODO(tianjianlu): Re-enable with specific prompts. + # _MULTITURN + "constrained_start": instructions.ConstrainedStartChecker, + _COMBINATION + "two_responses": set(INSTRUCTION_DICT.keys()).difference( + { + _KEYWORD + "forbidden_words", + _KEYWORD + "existence", + _LANGUAGE + "response_language", + _FORMAT + "title", + _PUNCTUATION + "no_comma", + } + ), + _COMBINATION + "repeat_prompt": set(INSTRUCTION_DICT.keys()).difference( + {_KEYWORD + "existence", _FORMAT + "title", _PUNCTUATION + "no_comma"} + ), + _STARTEND + "end_checker": {_STARTEND + "end_checker"}, + _CHANGE_CASES + "capital_word_frequency": { + _CHANGE_CASES + "capital_word_frequency", + _CHANGE_CASES + "english_lowercase", + _CHANGE_CASES + "english_capital", + }, + _CHANGE_CASES + "english_capital": {_CHANGE_CASES + "english_capital"}, + _CHANGE_CASES + "english_lowercase": { + _CHANGE_CASES + "english_lowercase", + _CHANGE_CASES + "english_capital", + }, + _PUNCTUATION + "no_comma": {_PUNCTUATION + "no_comma"}, + _STARTEND + "quotation": {_STARTEND + "quotation", _FORMAT + "title"}, +} + + +def conflict_make(conflicts): + """Makes sure if A conflicts with B, B will conflict with A. + + Args: + conflicts: Dictionary of potential conflicts where key is instruction id + and value is set of instruction ids that it conflicts with. + + Returns: + Revised version of the dictionary. All instructions conflict with + themselves. If A conflicts with B, B will conflict with A. + """ + for key in conflicts: + for k in conflicts[key]: + conflicts[k].add(key) + conflicts[key].add(key) + return conflicts diff --git a/lm_eval/tasks/leaderboard/ifeval/instructions_util.py b/lm_eval/tasks/leaderboard/ifeval/instructions_util.py new file mode 100644 index 0000000000000000000000000000000000000000..9ca2d4de4dd013f9766eb877a780ca40061887d6 --- /dev/null +++ b/lm_eval/tasks/leaderboard/ifeval/instructions_util.py @@ -0,0 +1,1696 @@ +# Copyright 2023 The Google Research Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Utility library of instructions.""" + +import functools +import random +import re + +import immutabledict +import nltk +import pkg_resources +from packaging import version + + +# Downloading 'punkt' with nltk<3.9 has a remote code vuln. +# see https://github.com/EleutherAI/lm-evaluation-harness/issues/2210 +# and https://github.com/nltk/nltk/issues/3266 +# for more information. +NLTK_MIN_VERSION = "3.9.1" + + +def download_nltk_resources(): + """Download 'punkt' if not already installed""" + nltk_version = pkg_resources.get_distribution("nltk").version + assert ( + version.parse(nltk_version) >= version.parse(NLTK_MIN_VERSION) + ), f"`nltk` version {nltk_version} is not >= {NLTK_MIN_VERSION}. Please update `nltk` before proceeding--older versions are vulnerable to a remote code execution vulnerability." + + try: + nltk.data.find("tokenizers/punkt_tab") + except LookupError: + nltk.download("punkt_tab") + + +download_nltk_resources() + +WORD_LIST = [ + "western", + "sentence", + "signal", + "dump", + "spot", + "opposite", + "bottom", + "potato", + "administration", + "working", + "welcome", + "morning", + "good", + "agency", + "primary", + "wish", + "responsibility", + "press", + "problem", + "president", + "steal", + "brush", + "read", + "type", + "beat", + "trainer", + "growth", + "lock", + "bone", + "case", + "equal", + "comfortable", + "region", + "replacement", + "performance", + "mate", + "walk", + "medicine", + "film", + "thing", + "rock", + "tap", + "total", + "competition", + "ease", + "south", + "establishment", + "gather", + "parking", + "world", + "plenty", + "breath", + "claim", + "alcohol", + "trade", + "dear", + "highlight", + "street", + "matter", + "decision", + "mess", + "agreement", + "studio", + "coach", + "assist", + "brain", + "wing", + "style", + "private", + "top", + "brown", + "leg", + "buy", + "procedure", + "method", + "speed", + "high", + "company", + "valuable", + "pie", + "analyst", + "session", + "pattern", + "district", + "pleasure", + "dinner", + "swimming", + "joke", + "order", + "plate", + "department", + "motor", + "cell", + "spend", + "cabinet", + "difference", + "power", + "examination", + "engine", + "horse", + "dimension", + "pay", + "toe", + "curve", + "literature", + "bother", + "fire", + "possibility", + "debate", + "activity", + "passage", + "hello", + "cycle", + "background", + "quiet", + "author", + "effect", + "actor", + "page", + "bicycle", + "error", + "throat", + "attack", + "character", + "phone", + "tea", + "increase", + "outcome", + "file", + "specific", + "inspector", + "internal", + "potential", + "staff", + "building", + "employer", + "shoe", + "hand", + "direction", + "garden", + "purchase", + "interview", + "study", + "recognition", + "member", + "spiritual", + "oven", + "sandwich", + "weird", + "passenger", + "particular", + "response", + "reaction", + "size", + "variation", + "a", + "cancel", + "candy", + "exit", + "guest", + "condition", + "fly", + "price", + "weakness", + "convert", + "hotel", + "great", + "mouth", + "mind", + "song", + "sugar", + "suspect", + "telephone", + "ear", + "roof", + "paint", + "refrigerator", + "organization", + "jury", + "reward", + "engineering", + "day", + "possession", + "crew", + "bar", + "road", + "description", + "celebration", + "score", + "mark", + "letter", + "shower", + "suggestion", + "sir", + "luck", + "national", + "progress", + "hall", + "stroke", + "theory", + "offer", + "story", + "tax", + "definition", + "history", + "ride", + "medium", + "opening", + "glass", + "elevator", + "stomach", + "question", + "ability", + "leading", + "village", + "computer", + "city", + "grand", + "confidence", + "candle", + "priest", + "recommendation", + "point", + "necessary", + "body", + "desk", + "secret", + "horror", + "noise", + "culture", + "warning", + "water", + "round", + "diet", + "flower", + "bus", + "tough", + "permission", + "week", + "prompt", + "connection", + "abuse", + "height", + "save", + "corner", + "border", + "stress", + "drive", + "stop", + "rip", + "meal", + "listen", + "confusion", + "girlfriend", + "living", + "relation", + "significance", + "plan", + "creative", + "atmosphere", + "blame", + "invite", + "housing", + "paper", + "drink", + "roll", + "silver", + "drunk", + "age", + "damage", + "smoke", + "environment", + "pack", + "savings", + "influence", + "tourist", + "rain", + "post", + "sign", + "grandmother", + "run", + "profit", + "push", + "clerk", + "final", + "wine", + "swim", + "pause", + "stuff", + "singer", + "funeral", + "average", + "source", + "scene", + "tradition", + "personal", + "snow", + "nobody", + "distance", + "sort", + "sensitive", + "animal", + "major", + "negotiation", + "click", + "mood", + "period", + "arrival", + "expression", + "holiday", + "repeat", + "dust", + "closet", + "gold", + "bad", + "sail", + "combination", + "clothes", + "emphasis", + "duty", + "black", + "step", + "school", + "jump", + "document", + "professional", + "lip", + "chemical", + "front", + "wake", + "while", + "inside", + "watch", + "row", + "subject", + "penalty", + "balance", + "possible", + "adult", + "aside", + "sample", + "appeal", + "wedding", + "depth", + "king", + "award", + "wife", + "blow", + "site", + "camp", + "music", + "safe", + "gift", + "fault", + "guess", + "act", + "shame", + "drama", + "capital", + "exam", + "stupid", + "record", + "sound", + "swing", + "novel", + "minimum", + "ratio", + "machine", + "shape", + "lead", + "operation", + "salary", + "cloud", + "affair", + "hit", + "chapter", + "stage", + "quantity", + "access", + "army", + "chain", + "traffic", + "kick", + "analysis", + "airport", + "time", + "vacation", + "philosophy", + "ball", + "chest", + "thanks", + "place", + "mountain", + "advertising", + "red", + "past", + "rent", + "return", + "tour", + "house", + "construction", + "net", + "native", + "war", + "figure", + "fee", + "spray", + "user", + "dirt", + "shot", + "task", + "stick", + "friend", + "software", + "promotion", + "interaction", + "surround", + "block", + "purpose", + "practice", + "conflict", + "routine", + "requirement", + "bonus", + "hole", + "state", + "junior", + "sweet", + "catch", + "tear", + "fold", + "wall", + "editor", + "life", + "position", + "pound", + "respect", + "bathroom", + "coat", + "script", + "job", + "teach", + "birth", + "view", + "resolve", + "theme", + "employee", + "doubt", + "market", + "education", + "serve", + "recover", + "tone", + "harm", + "miss", + "union", + "understanding", + "cow", + "river", + "association", + "concept", + "training", + "recipe", + "relationship", + "reserve", + "depression", + "proof", + "hair", + "revenue", + "independent", + "lift", + "assignment", + "temporary", + "amount", + "loss", + "edge", + "track", + "check", + "rope", + "estimate", + "pollution", + "stable", + "message", + "delivery", + "perspective", + "mirror", + "assistant", + "representative", + "witness", + "nature", + "judge", + "fruit", + "tip", + "devil", + "town", + "emergency", + "upper", + "drop", + "stay", + "human", + "neck", + "speaker", + "network", + "sing", + "resist", + "league", + "trip", + "signature", + "lawyer", + "importance", + "gas", + "choice", + "engineer", + "success", + "part", + "external", + "worker", + "simple", + "quarter", + "student", + "heart", + "pass", + "spite", + "shift", + "rough", + "lady", + "grass", + "community", + "garage", + "youth", + "standard", + "skirt", + "promise", + "blind", + "television", + "disease", + "commission", + "positive", + "energy", + "calm", + "presence", + "tune", + "basis", + "preference", + "head", + "common", + "cut", + "somewhere", + "presentation", + "current", + "thought", + "revolution", + "effort", + "master", + "implement", + "republic", + "floor", + "principle", + "stranger", + "shoulder", + "grade", + "button", + "tennis", + "police", + "collection", + "account", + "register", + "glove", + "divide", + "professor", + "chair", + "priority", + "combine", + "peace", + "extension", + "maybe", + "evening", + "frame", + "sister", + "wave", + "code", + "application", + "mouse", + "match", + "counter", + "bottle", + "half", + "cheek", + "resolution", + "back", + "knowledge", + "make", + "discussion", + "screw", + "length", + "accident", + "battle", + "dress", + "knee", + "log", + "package", + "it", + "turn", + "hearing", + "newspaper", + "layer", + "wealth", + "profile", + "imagination", + "answer", + "weekend", + "teacher", + "appearance", + "meet", + "bike", + "rise", + "belt", + "crash", + "bowl", + "equivalent", + "support", + "image", + "poem", + "risk", + "excitement", + "remote", + "secretary", + "public", + "produce", + "plane", + "display", + "money", + "sand", + "situation", + "punch", + "customer", + "title", + "shake", + "mortgage", + "option", + "number", + "pop", + "window", + "extent", + "nothing", + "experience", + "opinion", + "departure", + "dance", + "indication", + "boy", + "material", + "band", + "leader", + "sun", + "beautiful", + "muscle", + "farmer", + "variety", + "fat", + "handle", + "director", + "opportunity", + "calendar", + "outside", + "pace", + "bath", + "fish", + "consequence", + "put", + "owner", + "go", + "doctor", + "information", + "share", + "hurt", + "protection", + "career", + "finance", + "force", + "golf", + "garbage", + "aspect", + "kid", + "food", + "boot", + "milk", + "respond", + "objective", + "reality", + "raw", + "ring", + "mall", + "one", + "impact", + "area", + "news", + "international", + "series", + "impress", + "mother", + "shelter", + "strike", + "loan", + "month", + "seat", + "anything", + "entertainment", + "familiar", + "clue", + "year", + "glad", + "supermarket", + "natural", + "god", + "cost", + "conversation", + "tie", + "ruin", + "comfort", + "earth", + "storm", + "percentage", + "assistance", + "budget", + "strength", + "beginning", + "sleep", + "other", + "young", + "unit", + "fill", + "store", + "desire", + "hide", + "value", + "cup", + "maintenance", + "nurse", + "function", + "tower", + "role", + "class", + "camera", + "database", + "panic", + "nation", + "basket", + "ice", + "art", + "spirit", + "chart", + "exchange", + "feedback", + "statement", + "reputation", + "search", + "hunt", + "exercise", + "nasty", + "notice", + "male", + "yard", + "annual", + "collar", + "date", + "platform", + "plant", + "fortune", + "passion", + "friendship", + "spread", + "cancer", + "ticket", + "attitude", + "island", + "active", + "object", + "service", + "buyer", + "bite", + "card", + "face", + "steak", + "proposal", + "patient", + "heat", + "rule", + "resident", + "broad", + "politics", + "west", + "knife", + "expert", + "girl", + "design", + "salt", + "baseball", + "grab", + "inspection", + "cousin", + "couple", + "magazine", + "cook", + "dependent", + "security", + "chicken", + "version", + "currency", + "ladder", + "scheme", + "kitchen", + "employment", + "local", + "attention", + "manager", + "fact", + "cover", + "sad", + "guard", + "relative", + "county", + "rate", + "lunch", + "program", + "initiative", + "gear", + "bridge", + "breast", + "talk", + "dish", + "guarantee", + "beer", + "vehicle", + "reception", + "woman", + "substance", + "copy", + "lecture", + "advantage", + "park", + "cold", + "death", + "mix", + "hold", + "scale", + "tomorrow", + "blood", + "request", + "green", + "cookie", + "church", + "strip", + "forever", + "beyond", + "debt", + "tackle", + "wash", + "following", + "feel", + "maximum", + "sector", + "sea", + "property", + "economics", + "menu", + "bench", + "try", + "language", + "start", + "call", + "solid", + "address", + "income", + "foot", + "senior", + "honey", + "few", + "mixture", + "cash", + "grocery", + "link", + "map", + "form", + "factor", + "pot", + "model", + "writer", + "farm", + "winter", + "skill", + "anywhere", + "birthday", + "policy", + "release", + "husband", + "lab", + "hurry", + "mail", + "equipment", + "sink", + "pair", + "driver", + "consideration", + "leather", + "skin", + "blue", + "boat", + "sale", + "brick", + "two", + "feed", + "square", + "dot", + "rush", + "dream", + "location", + "afternoon", + "manufacturer", + "control", + "occasion", + "trouble", + "introduction", + "advice", + "bet", + "eat", + "kill", + "category", + "manner", + "office", + "estate", + "pride", + "awareness", + "slip", + "crack", + "client", + "nail", + "shoot", + "membership", + "soft", + "anybody", + "web", + "official", + "individual", + "pizza", + "interest", + "bag", + "spell", + "profession", + "queen", + "deal", + "resource", + "ship", + "guy", + "chocolate", + "joint", + "formal", + "upstairs", + "car", + "resort", + "abroad", + "dealer", + "associate", + "finger", + "surgery", + "comment", + "team", + "detail", + "crazy", + "path", + "tale", + "initial", + "arm", + "radio", + "demand", + "single", + "draw", + "yellow", + "contest", + "piece", + "quote", + "pull", + "commercial", + "shirt", + "contribution", + "cream", + "channel", + "suit", + "discipline", + "instruction", + "concert", + "speech", + "low", + "effective", + "hang", + "scratch", + "industry", + "breakfast", + "lay", + "join", + "metal", + "bedroom", + "minute", + "product", + "rest", + "temperature", + "many", + "give", + "argument", + "print", + "purple", + "laugh", + "health", + "credit", + "investment", + "sell", + "setting", + "lesson", + "egg", + "middle", + "marriage", + "level", + "evidence", + "phrase", + "love", + "self", + "benefit", + "guidance", + "affect", + "you", + "dad", + "anxiety", + "special", + "boyfriend", + "test", + "blank", + "payment", + "soup", + "obligation", + "reply", + "smile", + "deep", + "complaint", + "addition", + "review", + "box", + "towel", + "minor", + "fun", + "soil", + "issue", + "cigarette", + "internet", + "gain", + "tell", + "entry", + "spare", + "incident", + "family", + "refuse", + "branch", + "can", + "pen", + "grandfather", + "constant", + "tank", + "uncle", + "climate", + "ground", + "volume", + "communication", + "kind", + "poet", + "child", + "screen", + "mine", + "quit", + "gene", + "lack", + "charity", + "memory", + "tooth", + "fear", + "mention", + "marketing", + "reveal", + "reason", + "court", + "season", + "freedom", + "land", + "sport", + "audience", + "classroom", + "law", + "hook", + "win", + "carry", + "eye", + "smell", + "distribution", + "research", + "country", + "dare", + "hope", + "whereas", + "stretch", + "library", + "if", + "delay", + "college", + "plastic", + "book", + "present", + "use", + "worry", + "champion", + "goal", + "economy", + "march", + "election", + "reflection", + "midnight", + "slide", + "inflation", + "action", + "challenge", + "guitar", + "coast", + "apple", + "campaign", + "field", + "jacket", + "sense", + "way", + "visual", + "remove", + "weather", + "trash", + "cable", + "regret", + "buddy", + "beach", + "historian", + "courage", + "sympathy", + "truck", + "tension", + "permit", + "nose", + "bed", + "son", + "person", + "base", + "meat", + "usual", + "air", + "meeting", + "worth", + "game", + "independence", + "physical", + "brief", + "play", + "raise", + "board", + "she", + "key", + "writing", + "pick", + "command", + "party", + "yesterday", + "spring", + "candidate", + "physics", + "university", + "concern", + "development", + "change", + "string", + "target", + "instance", + "room", + "bitter", + "bird", + "football", + "normal", + "split", + "impression", + "wood", + "long", + "meaning", + "stock", + "cap", + "leadership", + "media", + "ambition", + "fishing", + "essay", + "salad", + "repair", + "today", + "designer", + "night", + "bank", + "drawing", + "inevitable", + "phase", + "vast", + "chip", + "anger", + "switch", + "cry", + "twist", + "personality", + "attempt", + "storage", + "being", + "preparation", + "bat", + "selection", + "white", + "technology", + "contract", + "side", + "section", + "station", + "till", + "structure", + "tongue", + "taste", + "truth", + "difficulty", + "group", + "limit", + "main", + "move", + "feeling", + "light", + "example", + "mission", + "might", + "wait", + "wheel", + "shop", + "host", + "classic", + "alternative", + "cause", + "agent", + "consist", + "table", + "airline", + "text", + "pool", + "craft", + "range", + "fuel", + "tool", + "partner", + "load", + "entrance", + "deposit", + "hate", + "article", + "video", + "summer", + "feature", + "extreme", + "mobile", + "hospital", + "flight", + "fall", + "pension", + "piano", + "fail", + "result", + "rub", + "gap", + "system", + "report", + "suck", + "ordinary", + "wind", + "nerve", + "ask", + "shine", + "note", + "line", + "mom", + "perception", + "brother", + "reference", + "bend", + "charge", + "treat", + "trick", + "term", + "homework", + "bake", + "bid", + "status", + "project", + "strategy", + "orange", + "let", + "enthusiasm", + "parent", + "concentrate", + "device", + "travel", + "poetry", + "business", + "society", + "kiss", + "end", + "vegetable", + "employ", + "schedule", + "hour", + "brave", + "focus", + "process", + "movie", + "illegal", + "general", + "coffee", + "ad", + "highway", + "chemistry", + "psychology", + "hire", + "bell", + "conference", + "relief", + "show", + "neat", + "funny", + "weight", + "quality", + "club", + "daughter", + "zone", + "touch", + "tonight", + "shock", + "burn", + "excuse", + "name", + "survey", + "landscape", + "advance", + "satisfaction", + "bread", + "disaster", + "item", + "hat", + "prior", + "shopping", + "visit", + "east", + "photo", + "home", + "idea", + "father", + "comparison", + "cat", + "pipe", + "winner", + "count", + "lake", + "fight", + "prize", + "foundation", + "dog", + "keep", + "ideal", + "fan", + "struggle", + "peak", + "safety", + "solution", + "hell", + "conclusion", + "population", + "strain", + "alarm", + "measurement", + "second", + "train", + "race", + "due", + "insurance", + "boss", + "tree", + "monitor", + "sick", + "course", + "drag", + "appointment", + "slice", + "still", + "care", + "patience", + "rich", + "escape", + "emotion", + "royal", + "female", + "childhood", + "government", + "picture", + "will", + "sock", + "big", + "gate", + "oil", + "cross", + "pin", + "improvement", + "championship", + "silly", + "help", + "sky", + "pitch", + "man", + "diamond", + "most", + "transition", + "work", + "science", + "committee", + "moment", + "fix", + "teaching", + "dig", + "specialist", + "complex", + "guide", + "people", + "dead", + "voice", + "original", + "break", + "topic", + "data", + "degree", + "reading", + "recording", + "bunch", + "reach", + "judgment", + "lie", + "regular", + "set", + "painting", + "mode", + "list", + "player", + "bear", + "north", + "wonder", + "carpet", + "heavy", + "officer", + "negative", + "clock", + "unique", + "baby", + "pain", + "assumption", + "disk", + "iron", + "bill", + "drawer", + "look", + "double", + "mistake", + "finish", + "future", + "brilliant", + "contact", + "math", + "rice", + "leave", + "restaurant", + "discount", + "sex", + "virus", + "bit", + "trust", + "event", + "wear", + "juice", + "failure", + "bug", + "context", + "mud", + "whole", + "wrap", + "intention", + "draft", + "pressure", + "cake", + "dark", + "explanation", + "space", + "angle", + "word", + "efficiency", + "management", + "habit", + "star", + "chance", + "finding", + "transportation", + "stand", + "criticism", + "flow", + "door", + "injury", + "insect", + "surprise", + "apartment", +] # pylint: disable=line-too-long + +# ISO 639-1 codes to language names. +LANGUAGE_CODES = immutabledict.immutabledict( + { + "en": "English", + "es": "Spanish", + "pt": "Portuguese", + "ar": "Arabic", + "hi": "Hindi", + "fr": "French", + "ru": "Russian", + "de": "German", + "ja": "Japanese", + "it": "Italian", + "bn": "Bengali", + "uk": "Ukrainian", + "th": "Thai", + "ur": "Urdu", + "ta": "Tamil", + "te": "Telugu", + "bg": "Bulgarian", + "ko": "Korean", + "pl": "Polish", + "he": "Hebrew", + "fa": "Persian", + "vi": "Vietnamese", + "ne": "Nepali", + "sw": "Swahili", + "kn": "Kannada", + "mr": "Marathi", + "gu": "Gujarati", + "pa": "Punjabi", + "ml": "Malayalam", + "fi": "Finnish", + } +) + +_ALPHABETS = "([A-Za-z])" +_PREFIXES = "(Mr|St|Mrs|Ms|Dr)[.]" +_SUFFIXES = "(Inc|Ltd|Jr|Sr|Co)" +_STARTERS = r"(Mr|Mrs|Ms|Dr|Prof|Capt|Cpt|Lt|He\s|She\s|It\s|They\s|Their\s|Our\s|We\s|But\s|However\s|That\s|This\s|Wherever)" +_ACRONYMS = "([A-Z][.][A-Z][.](?:[A-Z][.])?)" +_WEBSITES = "[.](com|net|org|io|gov|edu|me)" +_DIGITS = "([0-9])" +_MULTIPLE_DOTS = r"\.{2,}" + + +def split_into_sentences(text): + """Split the text into sentences. + + Args: + text: A string that consists of more than or equal to one sentences. + + Returns: + A list of strings where each string is a sentence. + """ + text = " " + text + " " + text = text.replace("\n", " ") + text = re.sub(_PREFIXES, "\\1", text) + text = re.sub(_WEBSITES, "\\1", text) + text = re.sub(_DIGITS + "[.]" + _DIGITS, "\\1\\2", text) + text = re.sub( + _MULTIPLE_DOTS, + lambda match: "" * len(match.group(0)) + "", + text, + ) + if "Ph.D" in text: + text = text.replace("Ph.D.", "PhD") + text = re.sub(r"\s" + _ALPHABETS + "[.] ", " \\1 ", text) + text = re.sub(_ACRONYMS + " " + _STARTERS, "\\1 \\2", text) + text = re.sub( + _ALPHABETS + "[.]" + _ALPHABETS + "[.]" + _ALPHABETS + "[.]", + "\\1\\2\\3", + text, + ) + text = re.sub(_ALPHABETS + "[.]" + _ALPHABETS + "[.]", "\\1\\2", text) + text = re.sub(" " + _SUFFIXES + "[.] " + _STARTERS, " \\1 \\2", text) + text = re.sub(" " + _SUFFIXES + "[.]", " \\1", text) + text = re.sub(" " + _ALPHABETS + "[.]", " \\1", text) + if "”" in text: + text = text.replace(".”", "”.") + if '"' in text: + text = text.replace('."', '".') + if "!" in text: + text = text.replace('!"', '"!') + if "?" in text: + text = text.replace('?"', '"?') + text = text.replace(".", ".") + text = text.replace("?", "?") + text = text.replace("!", "!") + text = text.replace("", ".") + sentences = text.split("") + sentences = [s.strip() for s in sentences] + if sentences and not sentences[-1]: + sentences = sentences[:-1] + return sentences + + +def count_words(text): + """Counts the number of words.""" + tokenizer = nltk.tokenize.RegexpTokenizer(r"\w+") + tokens = tokenizer.tokenize(text) + num_words = len(tokens) + return num_words + + +@functools.lru_cache(maxsize=None) +def _get_sentence_tokenizer(): + return nltk.data.load("nltk:tokenizers/punkt/english.pickle") + + +def count_sentences(text): + """Count the number of sentences.""" + tokenizer = _get_sentence_tokenizer() + tokenized_sentences = tokenizer.tokenize(text) + return len(tokenized_sentences) + + +def generate_keywords(num_keywords): + """Randomly generates a few keywords.""" + return random.sample(WORD_LIST, k=num_keywords) diff --git a/lm_eval/tasks/leaderboard/ifeval/utils.py b/lm_eval/tasks/leaderboard/ifeval/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..985e8d5ae578c484267c7c2d90ee7c896028941f --- /dev/null +++ b/lm_eval/tasks/leaderboard/ifeval/utils.py @@ -0,0 +1,134 @@ +import dataclasses +from typing import Dict, Optional, Union + +from lm_eval.tasks.ifeval import instructions_registry + + +@dataclasses.dataclass +class InputExample: + key: int + instruction_id_list: list[str] + prompt: str + kwargs: list[Dict[str, Optional[Union[str, int]]]] + + +@dataclasses.dataclass +class OutputExample: + instruction_id_list: list[str] + prompt: str + response: str + follow_all_instructions: bool + follow_instruction_list: list[bool] + + +def test_instruction_following_strict( + inp, + response, +): + """Tests response to see if instructions are followed.""" + instruction_list = inp.instruction_id_list + is_following_list = [] + + for index, instruction_id in enumerate(instruction_list): + instruction_cls = instructions_registry.INSTRUCTION_DICT[instruction_id] + instruction = instruction_cls(instruction_id) + + # Remove None values from kwargs to avoid unexpected keyword argument errors in build_description method. + kwargs = {k: v for k, v in inp.kwargs[index].items() if v} + instruction.build_description(**kwargs) + args = instruction.get_instruction_args() + if args and "prompt" in args: + instruction.build_description(prompt=inp.prompt) + + if response.strip() and instruction.check_following(response): + is_following_list.append(True) + else: + is_following_list.append(False) + + return OutputExample( + instruction_id_list=inp.instruction_id_list, + prompt=inp.prompt, + response=response, + follow_all_instructions=all(is_following_list), + follow_instruction_list=is_following_list, + ) + + +def test_instruction_following_loose( + inp, + response, +): + """Tests response for an upper bound for following instructions.""" + r = response.split("\n") + response_remove_first = "\n".join(r[1:]).strip() + response_remove_last = "\n".join(r[:-1]).strip() + response_remove_both = "\n".join(r[1:-1]).strip() + revised_response = response.replace("*", "") + revised_response_remove_first = response_remove_first.replace("*", "") + revised_response_remove_last = response_remove_last.replace("*", "") + revised_response_remove_both = response_remove_both.replace("*", "") + all_responses = [ + response, + revised_response, + response_remove_first, + response_remove_last, + response_remove_both, + revised_response_remove_first, + revised_response_remove_last, + revised_response_remove_both, + ] + instruction_list = inp.instruction_id_list + is_following_list = [] + + for index, instruction_id in enumerate(instruction_list): + instruction_cls = instructions_registry.INSTRUCTION_DICT[instruction_id] + instruction = instruction_cls(instruction_id) + + # Remove None values from kwargs to avoid unexpected keyword argument errors in build_description method. + kwargs = {k: v for k, v in inp.kwargs[index].items() if v} + instruction.build_description(**kwargs) + args = instruction.get_instruction_args() + if args and "prompt" in args: + instruction.build_description(prompt=inp.prompt) + + is_following = False + for r in all_responses: + if r.strip() and instruction.check_following(r): + is_following = True + break + + is_following_list.append(is_following) + + return OutputExample( + instruction_id_list=inp.instruction_id_list, + prompt=inp.prompt, + response=response, + follow_all_instructions=all(is_following_list), + follow_instruction_list=is_following_list, + ) + + +def process_results(doc, results): + inp = InputExample( + key=doc["key"], + instruction_id_list=doc["instruction_id_list"], + prompt=doc["prompt"], + kwargs=doc["kwargs"], + ) + response = results[0] + + out_strict = test_instruction_following_strict(inp, response) + out_loose = test_instruction_following_loose(inp, response) + + return { + "prompt_level_strict_acc": out_strict.follow_all_instructions, + "inst_level_strict_acc": out_strict.follow_instruction_list, + "prompt_level_loose_acc": out_loose.follow_all_instructions, + "inst_level_loose_acc": out_loose.follow_instruction_list, + } + + +def agg_inst_level_acc(items): + flat_items = [item for sublist in items for item in sublist] + inst_level_acc = sum(flat_items) / len(flat_items) + return inst_level_acc diff --git a/lm_eval/tasks/leaderboard/leaderboard.yaml b/lm_eval/tasks/leaderboard/leaderboard.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d9c5aaac175df34bf0f5ba91684eac6c4db195fc --- /dev/null +++ b/lm_eval/tasks/leaderboard/leaderboard.yaml @@ -0,0 +1,8 @@ +group: leaderboard +task: + - leaderboard_mmlu_pro + - leaderboard_bbh + - leaderboard_gpqa + - leaderboard_math_hard + - leaderboard_ifeval + - leaderboard_musr diff --git a/lm_eval/tasks/leaderboard/math/_leaderboard_math.yaml b/lm_eval/tasks/leaderboard/math/_leaderboard_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..45cf8f13408cdae93860e78c260b92a162c93fb3 --- /dev/null +++ b/lm_eval/tasks/leaderboard/math/_leaderboard_math.yaml @@ -0,0 +1,9 @@ +group: leaderboard_math_hard +task: + - leaderboard_math_algebra_hard + - leaderboard_math_counting_and_prob_hard + - leaderboard_math_geometry_hard + - leaderboard_math_intermediate_algebra_hard + - leaderboard_math_num_theory_hard + - leaderboard_math_prealgebra_hard + - leaderboard_math_precalculus_hard diff --git a/lm_eval/tasks/leaderboard/math/_template_yaml b/lm_eval/tasks/leaderboard/math/_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..9c404b0c5e350fa67bd694d6094ebd84e7594c06 --- /dev/null +++ b/lm_eval/tasks/leaderboard/math/_template_yaml @@ -0,0 +1,26 @@ +dataset_path: lighteval/MATH-Hard +process_docs: !function utils.process_docs +output_type: generate_until +training_split: train +test_split: test +doc_to_text: !function utils.doc_to_text +process_results: !function utils.process_results +doc_to_target: "{{answer if few_shot is undefined else solution}}" +generation_kwargs: + until: + - "Problem:" + do_sample: false + temperature: 0 + max_gen_toks: 1024 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true +num_fewshot: 4 +metadata: + version: 1.0 +dataset_kwargs: + trust_remote_code: true +fewshot_config: + sampler: first_n + samples: !function utils.list_fewshot_samples diff --git a/lm_eval/tasks/leaderboard/math/math_algebra.yaml b/lm_eval/tasks/leaderboard/math/math_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9e1d4753b0343f7df3f4f9625b14aaf117bf2571 --- /dev/null +++ b/lm_eval/tasks/leaderboard/math/math_algebra.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: algebra +task: leaderboard_math_algebra_hard diff --git a/lm_eval/tasks/leaderboard/math/math_counting_and_prob.yaml b/lm_eval/tasks/leaderboard/math/math_counting_and_prob.yaml new file mode 100644 index 0000000000000000000000000000000000000000..41767099ef9b26a721904b97516b1dba49a0e991 --- /dev/null +++ b/lm_eval/tasks/leaderboard/math/math_counting_and_prob.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: counting_and_probability +task: leaderboard_math_counting_and_prob_hard diff --git a/lm_eval/tasks/leaderboard/math/math_geometry.yaml b/lm_eval/tasks/leaderboard/math/math_geometry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..189633c304d7e80a119d35f8f096b296a9c251c9 --- /dev/null +++ b/lm_eval/tasks/leaderboard/math/math_geometry.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: geometry +task: leaderboard_math_geometry_hard diff --git a/lm_eval/tasks/leaderboard/math/math_intermediate_algebra.yaml b/lm_eval/tasks/leaderboard/math/math_intermediate_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9959bd18595827c5411ed115f19a549b033c473e --- /dev/null +++ b/lm_eval/tasks/leaderboard/math/math_intermediate_algebra.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: intermediate_algebra +task: leaderboard_math_intermediate_algebra_hard diff --git a/lm_eval/tasks/leaderboard/math/math_num_theory.yaml b/lm_eval/tasks/leaderboard/math/math_num_theory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a055d94dcbac940ab8af1f0e58a7dc4580696aab --- /dev/null +++ b/lm_eval/tasks/leaderboard/math/math_num_theory.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: number_theory +task: leaderboard_math_num_theory_hard diff --git a/lm_eval/tasks/leaderboard/math/math_prealgebra.yaml b/lm_eval/tasks/leaderboard/math/math_prealgebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..474d7258f461ff77933e3f10e9facfce5a484bb3 --- /dev/null +++ b/lm_eval/tasks/leaderboard/math/math_prealgebra.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: prealgebra +task: leaderboard_math_prealgebra_hard diff --git a/lm_eval/tasks/leaderboard/math/math_precalculus.yaml b/lm_eval/tasks/leaderboard/math/math_precalculus.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4d63541e54cfd1b7510214450591404f8da4dc18 --- /dev/null +++ b/lm_eval/tasks/leaderboard/math/math_precalculus.yaml @@ -0,0 +1,3 @@ +include: _template_yaml +dataset_name: precalculus +task: leaderboard_math_precalculus_hard diff --git a/lm_eval/tasks/leaderboard/math/utils.py b/lm_eval/tasks/leaderboard/math/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..e3ebcf991bac2a3727098a437586faf0d2ce3a62 --- /dev/null +++ b/lm_eval/tasks/leaderboard/math/utils.py @@ -0,0 +1,294 @@ +import re +import signal +from typing import Dict, List, Optional + +import datasets + +from lm_eval.utils import eval_logger + + +try: + import sympy + from sympy.parsing.latex import parse_latex +except ModuleNotFoundError: + raise ModuleNotFoundError( + "`sympy` is required for generating translation task prompt templates. \ +please install sympy via pip install lm-eval[math] or pip install -e .[math]", + ) + + +# taken from +# https://github.com/wellecks/lm-evaluation-harness/blob/master/lm_eval/tasks/minerva_math.py +def doc_to_text(doc: dict) -> str: + return "Problem:" + "\n" + doc["problem"] + "\n\n" + "Solution:" + + +def process_docs(dataset: datasets.Dataset) -> datasets.Dataset: + def _process_doc(doc: dict) -> dict: + out_doc = { + "problem": doc["problem"], + "solution": doc["solution"], + "answer": normalize_final_answer( + remove_boxed(last_boxed_only_string(doc["solution"])) + ), + } + if getattr(doc, "few_shot", None) is not None: + out_doc["few_shot"] = True + return out_doc + + return dataset.map(_process_doc) + + +def list_fewshot_samples() -> list[dict]: + return [ + { + "problem": "Find the domain of the expression $\\frac{\\sqrt{x-2}}{\\sqrt{5-x}}$.}", + "solution": "The expressions inside each square root must be non-negative. Therefore, $x-2 \\ge 0$, so $x\\ge2$, and $5 - x \\ge 0$, so $x \\le 5$. Also, the denominator cannot be equal to zero, so $5-x>0$, which gives $x<5$. Therefore, the domain of the expression is $\\boxed{[2,5)}$.\nFinal Answer: The final answer is $[2,5)$. I hope it is correct.", + "few_shot": "1", + }, + { + "problem": "If $\\det \\mathbf{A} = 2$ and $\\det \\mathbf{B} = 12,$ then find $\\det (\\mathbf{A} \\mathbf{B}).$", + "solution": "We have that $\\det (\\mathbf{A} \\mathbf{B}) = (\\det \\mathbf{A})(\\det \\mathbf{B}) = (2)(12) = \\boxed{24}.$\nFinal Answer: The final answer is $24$. I hope it is correct.", + "few_shot": "1", + }, + { + "problem": "Terrell usually lifts two 20-pound weights 12 times. If he uses two 15-pound weights instead, how many times must Terrell lift them in order to lift the same total weight?", + "solution": "If Terrell lifts two 20-pound weights 12 times, he lifts a total of $2\\cdot 12\\cdot20=480$ pounds of weight. If he lifts two 15-pound weights instead for $n$ times, he will lift a total of $2\\cdot15\\cdot n=30n$ pounds of weight. Equating this to 480 pounds, we can solve for $n$:\n\\begin{align*}\n30n&=480\\\n\\Rightarrow\\qquad n&=480/30=\\boxed{16}\n\\end{align*}\nFinal Answer: The final answer is $16$. I hope it is correct.", + "few_shot": "1", + }, + { + "problem": "If the system of equations\n\n\\begin{align*}\n6x-4y&=a,\\\n6y-9x &=b.\n\\end{align*}has a solution $(x, y)$ where $x$ and $y$ are both nonzero,\nfind $\\frac{a}{b},$ assuming $b$ is nonzero.", + "solution": "If we multiply the first equation by $-\\frac{3}{2}$, we obtain\n\n$$6y-9x=-\\frac{3}{2}a.$$Since we also know that $6y-9x=b$, we have\n\n$$-\\frac{3}{2}a=b\\Rightarrow\\frac{a}{b}=\\boxed{-\\frac{2}{3}}.$$\nFinal Answer: The final answer is $-\\frac{2}{3}$. I hope it is correct.", + "few_shot": "1", + }, + ] + + +def process_results(doc: dict, results: List[str]) -> Dict[str, int]: + candidates = results[0] + + unnormalized_answer = get_unnormalized_answer(candidates) + answer = normalize_final_answer(unnormalized_answer) + + if is_equiv(answer, doc["answer"]): + retval = 1 + else: + retval = 0 + + results = { + "exact_match": retval, + } + return results + + +def last_boxed_only_string(string: str) -> Optional[str]: + idx = string.rfind("\\boxed") + if "\\boxed " in string: + return "\\boxed " + string.split("\\boxed ")[-1].split("$")[0] + if idx < 0: + idx = string.rfind("\\fbox") + if idx < 0: + return None + + i = idx + right_brace_idx = None + num_left_braces_open = 0 + while i < len(string): + if string[i] == "{": + num_left_braces_open += 1 + if string[i] == "}": + num_left_braces_open -= 1 + if num_left_braces_open == 0: + right_brace_idx = i + break + i += 1 + + if right_brace_idx is None: + retval = None + else: + retval = string[idx : right_brace_idx + 1] + + return retval + + +def remove_boxed(s: str) -> str: + if "\\boxed " in s: + left = "\\boxed " + assert s[: len(left)] == left + return s[len(left) :] + + left = "\\boxed{" + + assert s[: len(left)] == left + assert s[-1] == "}" + + return s[len(left) : -1] + + +class timeout: + def __init__(self, seconds=1, error_message="Timeout"): + self.seconds = seconds + self.error_message = error_message + + def handle_timeout(self, signum, frame): + raise TimeoutError(self.error_message) + + def __enter__(self): + signal.signal(signal.SIGALRM, self.handle_timeout) + signal.alarm(self.seconds) + + def __exit__(self, type, value, traceback): + signal.alarm(0) + + +def is_equiv(x1: str, x2: str) -> bool: + """ + x1 and x2 are normalized latex string + """ + try: + with timeout(seconds=5): + try: + parsed_x1 = parse_latex(x1) + parsed_x2 = parse_latex(x2) + except ( + sympy.parsing.latex.errors.LaTeXParsingError, + sympy.SympifyError, + TypeError, + ): + eval_logger.debug(f"couldn't parse one of {x1} or {x2}") + return False + + try: + diff = parsed_x1 - parsed_x2 + except TypeError: + eval_logger.debug(f"couldn't subtract {x1} and {x2}") + return False + + try: + if sympy.simplify(diff) == 0: + return True + else: + return False + except ValueError: + eval_logger.debug( + f"Had some trouble simplifying when comparing {x1} and {x2}" + ) + except TimeoutError: + eval_logger.debug(f"Timed out comparing {x1} and {x2}") + return False + except ImportError as e: + eval_logger.error(e) + raise + except Exception as e: + eval_logger.debug(f"Failed comparing {x1} and {x2} with {e}") + return False + + +def get_unnormalized_answer(text: str) -> str: + INVALID_ANSWER = "[invalidanswer]" + end_seq = "I hope it is correct." + text += end_seq + match = re.search( + r"Final Answer: The final answer is(.*?). I hope it is correct.", + text, + ) + if match: + return match.group(1).strip() + else: + return INVALID_ANSWER + + +SUBSTITUTIONS = [ + ("an ", ""), + ("a ", ""), + (".$", "$"), + ("\\$", ""), + (r"\ ", ""), + (" ", ""), + ("mbox", "text"), + (",\\text{and}", ","), + ("\\text{and}", ","), + ("\\text{m}", "\\text{}"), +] +REMOVED_EXPRESSIONS = [ + "square", + "ways", + "integers", + "dollars", + "mph", + "inches", + "ft", + "hours", + "km", + "units", + "\\ldots", + "sue", + "points", + "feet", + "minutes", + "digits", + "cents", + "degrees", + "cm", + "gm", + "pounds", + "meters", + "meals", + "edges", + "students", + "childrentickets", + "multiples", + "\\text{s}", + "\\text{.}", + "\\text{\ns}", + "\\text{}^2", + "\\text{}^3", + "\\text{\n}", + "\\text{}", + r"\mathrm{th}", + r"^\circ", + r"^{\circ}", + r"\;", + r",\!", + "{,}", + '"', + "\\dots", +] + + +def normalize_final_answer(final_answer: str) -> str: + """ + Normalize a final answer to a quantitative reasoning question. + + Copied character for character from appendix D of Lewkowycz et al. (2022) + """ + final_answer = final_answer.split("=")[-1] + + for before, after in SUBSTITUTIONS: + final_answer = final_answer.replace(before, after) + for expr in REMOVED_EXPRESSIONS: + final_answer = final_answer.replace(expr, "") + + # Extract answer that is in LaTeX math, is bold, + # is surrounded by a box, etc. + final_answer = re.sub(r"(.*?)(\$)(.*?)(\$)(.*)", "$\\3$", final_answer) + final_answer = re.sub(r"(\\text\{)(.*?)(\})", "\\2", final_answer) + final_answer = re.sub(r"(\\textbf\{)(.*?)(\})", "\\2", final_answer) + final_answer = re.sub(r"(\\overline\{)(.*?)(\})", "\\2", final_answer) + final_answer = re.sub(r"(\\boxed\{)(.*)(\})", "\\2", final_answer) + + # Normalize shorthand TeX: + # \fracab -> \frac{a}{b} + # \frac{abc}{bef} -> \frac{abc}{bef} + # \fracabc -> \frac{a}{b}c + # \sqrta -> \sqrt{a} + # \sqrtab -> sqrt{a}b + final_answer = re.sub(r"(frac)([^{])(.)", "frac{\\2}{\\3}", final_answer) + final_answer = re.sub(r"(sqrt)([^{])", "sqrt{\\2}", final_answer) + final_answer = final_answer.replace("$", "") + + # Normalize 100,000 -> 100000 + if final_answer.replace(",", "").isdigit(): + final_answer = final_answer.replace(",", "") + + return final_answer diff --git a/lm_eval/tasks/leaderboard/mmlu_pro/mmlu_pro.yaml b/lm_eval/tasks/leaderboard/mmlu_pro/mmlu_pro.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8ddabf148e767c63d320229c26f6f530d41026f3 --- /dev/null +++ b/lm_eval/tasks/leaderboard/mmlu_pro/mmlu_pro.yaml @@ -0,0 +1,17 @@ +dataset_path: TIGER-Lab/MMLU-Pro # a copy of `cais/leaderboard_mmlu` with no auxiliary_train split +task: leaderboard_mmlu_pro +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: !function utils.doc_to_text +doc_to_choice: !function utils.doc_to_choice +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +num_fewshot: 5 +metadata: + version: 0.1 diff --git a/lm_eval/tasks/leaderboard/mmlu_pro/utils.py b/lm_eval/tasks/leaderboard/mmlu_pro/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..bb398262ad66393d0abdd79c1dff7a92c2d67c8d --- /dev/null +++ b/lm_eval/tasks/leaderboard/mmlu_pro/utils.py @@ -0,0 +1,15 @@ +import string + + +def doc_to_text(doc): + doc_to_text = f"{doc['question']}\n" + + for i in range(len(doc["options"])): + doc_to_text += f"{string.ascii_uppercase[i]}. {doc['options'][i]}\n" + + doc_to_text += "Answer:" + return doc_to_text + + +def doc_to_choice(doc): + return [string.ascii_uppercase[i] for i in range(len(doc["options"]))] diff --git a/lm_eval/tasks/leaderboard/musr/_musr.yaml b/lm_eval/tasks/leaderboard/musr/_musr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..060d231aae9a11c3078999364b23de802caf5638 --- /dev/null +++ b/lm_eval/tasks/leaderboard/musr/_musr.yaml @@ -0,0 +1,5 @@ +group: leaderboard_musr +task: + - leaderboard_musr_murder_mysteries + - leaderboard_musr_object_placements + - leaderboard_musr_team_allocation diff --git a/lm_eval/tasks/leaderboard/musr/_template_yaml b/lm_eval/tasks/leaderboard/musr/_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..d14081247271fb8f3860e7f636b9265e9ef418b5 --- /dev/null +++ b/lm_eval/tasks/leaderboard/musr/_template_yaml @@ -0,0 +1,11 @@ +dataset_path: TAUR-Lab/MuSR +output_type: multiple_choice +doc_to_text: !function utils.doc_to_text +doc_to_target: "{{answer_choice}}" +doc_to_choice: "{{choices}}" +metric_list: + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/leaderboard/musr/musr_murder_mysteries.yaml b/lm_eval/tasks/leaderboard/musr/musr_murder_mysteries.yaml new file mode 100644 index 0000000000000000000000000000000000000000..88aac33eef3d9eee2479f8f0b3c391d6ffdda0a2 --- /dev/null +++ b/lm_eval/tasks/leaderboard/musr/musr_murder_mysteries.yaml @@ -0,0 +1,3 @@ +include: "_template_yaml" +task: leaderboard_musr_murder_mysteries +test_split: murder_mysteries diff --git a/lm_eval/tasks/leaderboard/musr/musr_object_placements.yaml b/lm_eval/tasks/leaderboard/musr/musr_object_placements.yaml new file mode 100644 index 0000000000000000000000000000000000000000..89b29fb732422e2ae0bed278307dc7f633d39e34 --- /dev/null +++ b/lm_eval/tasks/leaderboard/musr/musr_object_placements.yaml @@ -0,0 +1,3 @@ +include: "_template_yaml" +task: leaderboard_musr_object_placements +test_split: object_placements diff --git a/lm_eval/tasks/leaderboard/musr/musr_team_allocation.yaml b/lm_eval/tasks/leaderboard/musr/musr_team_allocation.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8ede2ba9e05bffa93b6f0a5bc82005a842bbce02 --- /dev/null +++ b/lm_eval/tasks/leaderboard/musr/musr_team_allocation.yaml @@ -0,0 +1,3 @@ +include: "_template_yaml" +task: leaderboard_musr_team_allocation +test_split: team_allocation diff --git a/lm_eval/tasks/leaderboard/musr/utils.py b/lm_eval/tasks/leaderboard/musr/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..1d0a7d1ca0c98a4ae0641b0520e693826090b7b9 --- /dev/null +++ b/lm_eval/tasks/leaderboard/musr/utils.py @@ -0,0 +1,26 @@ +import ast + + +def doc_to_choice(doc): + """ + Convert a doc to a choice. + """ + return ast.literal_eval(doc["choices"]) + + +DOC_TO_TEXT = "{narrative}\n\n" "{question}\n\n" "{choices}\n" "Answer:" + + +def doc_to_text(doc): + """ + Convert a doc to text. + """ + choices = "" + for i, choice in enumerate(ast.literal_eval(doc["choices"])): + choices += f"{i+1} - {choice}\n" + + text = DOC_TO_TEXT.format( + narrative=doc["narrative"], question=doc["question"], choices=choices + ) + + return text diff --git a/lm_eval/tasks/lingoly/README.md b/lm_eval/tasks/lingoly/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3ce114e844c9f73d1ee2e9f082b6cab9ac3edd54 --- /dev/null +++ b/lm_eval/tasks/lingoly/README.md @@ -0,0 +1,46 @@ +# LingOly + + +### Paper + +Title: `LINGOLY: A Benchmark of Olympiad-Level Linguistic Reasoning Puzzles in Low-Resource and Extinct Languages` + +Abstract: `https://arxiv.org/abs/2406.06196` + +`In this paper, we present the LingOly benchmark, a novel benchmark for advanced reasoning abilities in large language models. Using challenging Linguistic Olympiad puzzles, we evaluate (i) capabilities for in-context identification and generalisation of linguistic patterns in very low-resource or extinct languages, and (ii) abilities to follow complex task instructions. The LingOly benchmark covers more than 90 mostly low-resource languages, minimising issues of data contamination, and contains 1,133 problems across 6 formats and 5 levels of human difficulty. We assess performance with both direct accuracy and comparison to a no-context baseline to penalise memorisation. Scores from 11 state-of-the-art LLMs demonstrate the benchmark to be challenging, and models perform poorly on the higher difficulty problems. On harder problems, even the top model only achieved 38.7% accuracy, 24.7% improvement over the no-context baseline. Large closed models typically outperform open models, and in general, the higher resource the language, the better the scores. These results indicate, in absence of memorisation, true multi-step out-of-domain reasoning remains a challenge for current language models.` + +Homepage: `https://github.com/am-bean/lingOly` + + +### Citation + +``` +@article{beanLINGOLYBenchmarkOlympiadLevel2024, + title = {{LINGOLY}: A Benchmark of Olympiad-Level Linguistic Reasoning Puzzles in Low-Resource and Extinct Languages}, + shorttitle = {{LINGOLY}}, + url = {http://arxiv.org/abs/2406.06196}, + author = {Bean, Andrew M. and Hellsten, Simi and Mayne, Harry and Magomere, Jabez and Chi, Ethan A. and Chi, Ryan and Hale, Scott A. and Kirk, Hannah Rose}, + month = jun, + year = {2024}, + keywords = {Computer Science - Computation and Language} +} +``` + +### Tasks + +* `lingoly`: `runs both _context and _nocontext and computes the difference` +* `lingoly_context`: `exact match of generations to reference answers` +* `lingoly_nocontext`: `exact match of generations to reference answers, but with context removed` + +### Checklist + +For adding novel benchmarks/datasets to the library: +* [x] Is the task an existing benchmark in the literature? + * [x] Have you referenced the original paper that introduced the task? + * [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + + +If other tasks on this dataset are already supported: +* [ ] Is the "Main" variant of this task clearly denoted? +* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [ ] Have you noted which, if any, published evaluation setups are matched by this variant? diff --git a/lm_eval/tasks/lingoly/lingoly_context.yaml b/lm_eval/tasks/lingoly/lingoly_context.yaml new file mode 100644 index 0000000000000000000000000000000000000000..76dfe41a136087d6b0043758b26309a14a0c3763 --- /dev/null +++ b/lm_eval/tasks/lingoly/lingoly_context.yaml @@ -0,0 +1,30 @@ +task: lingoly_context + +dataset_path: ambean/lingOly # the name of the dataset on the HF Hub. +dataset_name: null # the dataset configuration to use. Leave `null` if your dataset does not require a config to be passed. See https://huggingface.co/docs/datasets/load_hub#configurations for more info. +dataset_kwargs: null # any extra keyword arguments that should be passed to the dataset constructor, e.g. `data_dir`. + +training_split: null +validation_split: test +test_split: test +fewshot_split: null + +generation_kwargs: + until: + - "}\n" + max_gen_toks: 512 + do_sample: false + temperature: 0.0 + +process_docs: !function utils.load_all_questions + +doc_to_text: prompt +doc_to_target: answers + +metric_list: + - metric: !function script.exact_match + aggregation: !function script.aggregate_scores + higher_is_better: true + +metadata: + version: 0 diff --git a/lm_eval/tasks/lingoly/lingoly_group.yaml b/lm_eval/tasks/lingoly/lingoly_group.yaml new file mode 100644 index 0000000000000000000000000000000000000000..261dff426c6721d1669d064ea8deb268def9834e --- /dev/null +++ b/lm_eval/tasks/lingoly/lingoly_group.yaml @@ -0,0 +1,12 @@ +group: lingoly +task: + - group: delta_nc + task: + - lingoly_context + - lingoly_nocontext + aggregate_metric_list: + - metric: exact_match + aggregation: !function script.aggregate_metrics + weight_by_size: false +metadata: + version: 1.0 diff --git a/lm_eval/tasks/lingoly/lingoly_nocontext.yaml b/lm_eval/tasks/lingoly/lingoly_nocontext.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fd6f4dedbca5acf73208d21062c3907456e0f3c9 --- /dev/null +++ b/lm_eval/tasks/lingoly/lingoly_nocontext.yaml @@ -0,0 +1,30 @@ +task: lingoly_nocontext + +dataset_path: ambean/lingOly # the name of the dataset on the HF Hub. +dataset_name: null # the dataset configuration to use. Leave `null` if your dataset does not require a config to be passed. See https://huggingface.co/docs/datasets/load_hub#configurations for more info. +dataset_kwargs: null # any extra keyword arguments that should be passed to the dataset constructor, e.g. `data_dir`. + +training_split: null +validation_split: test +test_split: test +fewshot_split: null + +generation_kwargs: + until: + - "}\n" + max_gen_toks: 512 + do_sample: false + temperature: 0.0 + +process_docs: !function utils.load_all_questions + +doc_to_text: nc_prompt +doc_to_target: answers + +metric_list: + - metric: !function script.exact_match + aggregation: !function script.aggregate_scores + higher_is_better: false + +metadata: + version: 0 diff --git a/lm_eval/tasks/lingoly/script.py b/lm_eval/tasks/lingoly/script.py new file mode 100644 index 0000000000000000000000000000000000000000..455ed2cd70f4f3628452198a6402d6142b2be2d1 --- /dev/null +++ b/lm_eval/tasks/lingoly/script.py @@ -0,0 +1,131 @@ +import ast +import re +import unicodedata as ud + + +def clean_answer(answer: str): + # remove whitespace and final stop + clean = answer.strip().strip(".") + + # reduce multiple spaces to a single space + clean = re.sub(r"[ ]+", " ", clean) + + # reduce to lower case + clean = clean.lower() + + # remove internal + (can't currently handle for marking) + clean = re.sub("\\+", "", clean) + + # make quotes consistent + quotes_map = {"‘": "'", "’": "'", "“": '"', "”": '"'} + + for k, v in quotes_map.items(): + clean = re.sub(k, v, clean) + + # make unicode consistent + clean = ud.normalize("NFKD", clean) + + return clean + + +def safe_exact(references: list[str], predictions: list[str]): + if len(references[0]) == 0: + return 1.0 + if len(predictions[0]) == 0: + return 0.0 + + score = float(references[0] == predictions[0]) + + return score + + +def parse_str_list_score(model, correct, scoring_func): + model = str(model) + if len(correct) == 0: + return 1.0 + if len(model) == 0: + return 0.0 + if ("[" in correct) and (("'" in correct) or ('"' in correct)): + readstr = ast.literal_eval(correct) + if isinstance(readstr, list): + correct = readstr + if isinstance(correct, list): + if all(isinstance(c, str) for c in correct): + max_score = 0.0 + if ( + len(correct) > 24 + ): # bleu and rouge are expensive and don't make sense for any order problems + return clean_answer(model) in [clean_answer(c) for c in correct] + for c in correct: + score = scoring_func( + references=[clean_answer(c)], + predictions=[clean_answer(model)], + ) + if score > max_score: + max_score = score + return max_score + else: + max_score = 0.0 + for c in correct: + if isinstance(c, list): + c = ", ".join(c) + score = scoring_func( + references=[clean_answer(c)], + predictions=[clean_answer(model)], + ) + else: + score = scoring_func( + references=[clean_answer(c)], + predictions=[clean_answer(model)], + ) + if score > max_score: + max_score = score + return max_score + else: + return scoring_func( + references=[clean_answer(correct)], + predictions=[clean_answer(model)], + ) + + +def exact_match(references: list[str], predictions: list[str]): + ref_dict = ast.literal_eval(references[0]) + try: + assert "{" in predictions[0] + if predictions[0][-1] == "}": + pred_dict = ast.literal_eval(predictions[0][predictions[0].index("{") :]) + else: + pred_dict = ast.literal_eval( + predictions[0][predictions[0].index("{") :] + "}" + ) + except (SyntaxError, ValueError, AssertionError): + pred_dict = {} + for k in ref_dict.keys(): + m = re.search(re.escape(str(k)) + """': ([^']+)'[,\\}]""", predictions[0]) + n = re.search(re.escape(str(k)) + """": ([^"]+)"[,\\}]""", predictions[0]) + if m: + pred_dict[k] = m.group()[:-1] + elif n: + pred_dict[k] = n.group()[:-1] + else: + pred_dict[k] = "" + pred_dict_full = { + k: pred_dict[k] if k in pred_dict else "" for k in ref_dict.keys() + } + + scores = [ + parse_str_list_score(pred_dict_full[k], v, safe_exact) + for k, v in ref_dict.items() + ] + + return scores + + +def aggregate_scores(input): + return sum([sum(i) for i in input]) / sum([len(j) for j in input]) + + +def aggregate_metrics( + metrics_scores: list[int], dataset_size: list[int], weight_by_size: bool +): + return metrics_scores[0] - metrics_scores[1] diff --git a/lm_eval/tasks/lingoly/utils.py b/lm_eval/tasks/lingoly/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..21051d7798ba94bfab6f8867496b3977f7a60424 --- /dev/null +++ b/lm_eval/tasks/lingoly/utils.py @@ -0,0 +1,100 @@ +import json + +import datasets + + +def load_questionsheet(qsheet: dict, no_context: bool = False): + subquestions = json.loads(qsheet["questions"]) + + all_subquestions = "" + for sq in subquestions: + all_subquestions += f"\n{sq['prompt']}\n" + for sp in sq["subprompts"]: + all_subquestions += f"{sp['questionpart_n']} {sp['question']}" + all_subquestions += "\n" + + if no_context: + prompt = f"""{qsheet['preamble']} + + {all_subquestions} + """ + else: + prompt = f"""{qsheet['preamble']} + {qsheet['context']} + + {all_subquestions} + """ + + return prompt + + +def format_answers(questionpart_ns: list[str], answers: list[str]): + formatted_output = {} + formatted_answers = {} + for i, qn in enumerate(questionpart_ns): + formatted_output[qn] = "" + formatted_answers[qn] = answers[i] + + formatted_output = json.dumps(formatted_output) + + return formatted_output, formatted_answers + + +def load_question( + qsheet: dict, + question_index: int, + no_context: bool = False, +): + subquestions = json.loads(qsheet["questions"]) + sq = subquestions[question_index] + + all_subquestions = "" + questionpart_ns = [] + answers = [] + all_subquestions += f"\n{sq['prompt']}\n" + for sp in sq["subprompts"]: + all_subquestions += f"{sp['questionpart_n']} {sp['question']}" + questionpart_ns.append(sp["questionpart_n"]) + answers.append(sp["answer"]) + all_subquestions += "\n" + + formatted_output, formatted_answers = format_answers(questionpart_ns, answers) + + question_body = load_questionsheet(qsheet, no_context) + + prompt = f"""Below is a problem sheet from a lingusitics exam. You will first see the entire sheet, then be asked to respond to specific questions from the sheet. Your answers to the questions should rely only on reasoning about the information provided in the sheet. + {question_body} + + Now respond to the following questions: + {all_subquestions} + + Format your response as a json file with the keys as provided below: + {formatted_output} + """ + return prompt, formatted_answers + + +def load_all_questions( + question_sheets: list[dict], +): + prompts = [] + nc_prompts = [] + answers = [] + indices = [] + for qsheet in question_sheets: + for i in range(len(json.loads(qsheet["questions"]))): + prompt, answer = load_question(qsheet, i, no_context=False) + nc_prompt, _ = load_question(qsheet, i, no_context=True) + nc_prompts.append(nc_prompt) + prompts.append(prompt) + answers.append(str(answer)) + indices.append(qsheet["overall_question_n"]) + + qsheets = { + "prompt": prompts, + "nc_prompt": nc_prompts, + "answers": answers, + "index": indices, + } + dataset = datasets.Dataset.from_dict(qsheets) + return dataset diff --git a/lm_eval/tasks/mathqa/mathqa.yaml b/lm_eval/tasks/mathqa/mathqa.yaml index e37ba118071ecd0c1a1098a148e047232ad9bea7..23fc01d2e9d27c95fb7f742084aa4b6a938ae7ca 100644 --- a/lm_eval/tasks/mathqa/mathqa.yaml +++ b/lm_eval/tasks/mathqa/mathqa.yaml @@ -1,4 +1,4 @@ -group: +tag: - math_word_problems task: mathqa dataset_path: math_qa diff --git a/lm_eval/tasks/med_concepts_qa/README.md b/lm_eval/tasks/med_concepts_qa/README.md new file mode 100644 index 0000000000000000000000000000000000000000..666d6446b108afaa2991ec8b3d6921ecc2b704d7 --- /dev/null +++ b/lm_eval/tasks/med_concepts_qa/README.md @@ -0,0 +1,49 @@ +# MedConceptsQA + +### Paper + +Title: `MedConceptsQA: Open Source Medical Concepts QA Benchmark` + +Abstract: https://arxiv.org/abs/2405.07348 + +MedConceptsQA is a dedicated open source benchmark for medical concepts question answering. The benchmark comprises of questions of various medical concepts across different vocabularies: diagnoses, procedures, and drugs. + +The questions are categorized into three levels of difficulty: easy, medium, and hard. + +Our benchmark serves as a valuable resource for evaluating the +abilities of Large Language Models to interpret medical codes and distinguish +between medical concepts. + +### Citation + +``` +@article{shoham2024medconceptsqa, + title={MedConceptsQA--Open Source Medical Concepts QA Benchmark}, + author={Shoham, Ofir Ben and Rappoport, Nadav}, + journal={arXiv preprint arXiv:2405.07348}, + year={2024} +} +``` + +### Groups and Tasks + +#### Groups + +* `med_concepts_qa`: Contains all the QA tasks (diagnosis, procedures ,and drugs). + +#### Tasks + + +* `med_concepts_qa_icd9cm` - ICD9-CM (diagnosis codes, ICD9 format) question-answering. This involves providing information, clarifications, and answering questions related to ICD-9-CM (International Classification of Diseases, 9th Revision, Clinical Modification) diagnosis codes. + + +* `med_concepts_qa_icd10cm` - ICD10-CM (diagnosis codes, ICD10 format) question-answering. This involves providing information, clarifications, and answering questions related to ICD-10-CM (International Classification of Diseases, 10th Revision, Clinical Modification) diagnosis codes. + + +* `med_concepts_qa_icd9proc` - ICD9-Proc (procedure codes, ICD9 format) question-answering. This involves providing information, clarifications, and answering questions related to ICD-9-PCS (International Classification of Diseases, 9th Revision, Procedure Coding System) procedure codes. + + +* `med_concepts_qa_icd10proc` - ICD10-Proc (procedure codes, ICD10 format) question-answering. This involves providing information, clarifications, and answering questions related to ICD-10-PCS (International Classification of Diseases, 10th Revision, Procedure Coding System) procedure codes. + + +* `med_concepts_qa_atc` - ATC (Anatomical Therapeutic Chemical Classification System) question-answering. This involves providing information, clarifications, and answering questions related to the ATC classification system, which is used for the classification of drugs and other medical products according to the organ or system on which they act and their therapeutic, pharmacological, and chemical properties. diff --git a/lm_eval/tasks/med_concepts_qa/_default_template_yaml b/lm_eval/tasks/med_concepts_qa/_default_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..0fb527557dd3906b1ec51a83d0ddbfa48da815e2 --- /dev/null +++ b/lm_eval/tasks/med_concepts_qa/_default_template_yaml @@ -0,0 +1,17 @@ +dataset_path: ofir408/MedConceptsQA +output_type: multiple_choice +description: "Answer A,B,C,D according to the answer to this multiple choice question.\n" +fewshot_split: dev +fewshot_config: + sampler: first_n +num_fewshot: 4 +test_split: test +doc_to_text: "{{question}}\nAnswer:" +doc_to_target: answer_id +doc_to_choice: ['A', 'B', 'C', 'D'] +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/med_concepts_qa/_generate_configs.py b/lm_eval/tasks/med_concepts_qa/_generate_configs.py new file mode 100644 index 0000000000000000000000000000000000000000..fd81edbbd0a6d74efe2ea68f04210bfaaef72661 --- /dev/null +++ b/lm_eval/tasks/med_concepts_qa/_generate_configs.py @@ -0,0 +1,34 @@ +from typing import List + +import yaml + + +def generate_yaml_content(vocab_name: str, level: str): + content = { + "dataset_name": f"{vocab_name}_{level}", + "tag": f"med_concepts_qa_{vocab_name}_tasks", + "include": "_default_template_yaml", + "task": f"med_concepts_qa_{vocab_name}_{level}", + "task_alias": f"{vocab_name}_{level}", + } + return content + + +def generate_yaml_files( + vocab_names: List[str], levels: List[str], file_name_prefix: str +): + for vocab_name in vocab_names: + for level in levels: + yaml_content = generate_yaml_content(vocab_name, level) + filename = f"{file_name_prefix}_{vocab_name}_{level}.yaml" + with open(filename, "w") as yaml_file: + yaml.dump(yaml_content, yaml_file, default_flow_style=False) + print(f"Done to generated {filename}") + + +if __name__ == "__main__": + generate_yaml_files( + vocab_names=["icd9cm", "icd10cm", "icd9proc", "icd10proc", "atc"], + levels=["easy", "medium", "hard"], + file_name_prefix="med_concepts_qa", + ) diff --git a/lm_eval/tasks/med_concepts_qa/_med_concepts_qa.yaml b/lm_eval/tasks/med_concepts_qa/_med_concepts_qa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c737127f986f3fb2a85fcbfd7e167587563de0f9 --- /dev/null +++ b/lm_eval/tasks/med_concepts_qa/_med_concepts_qa.yaml @@ -0,0 +1,10 @@ +group: med_concepts_qa +task: + - med_concepts_qa_icd9cm + - med_concepts_qa_icd10cm + - med_concepts_qa_icd9proc + - med_concepts_qa_icd10proc + - med_concepts_qa_atc +aggregate_metric_list: + - metric: acc + aggregation: mean diff --git a/lm_eval/tasks/med_concepts_qa/_med_concepts_qa_atc.yaml b/lm_eval/tasks/med_concepts_qa/_med_concepts_qa_atc.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2e34ad36c296e9b1e39b5445dc9da8bc640245a4 --- /dev/null +++ b/lm_eval/tasks/med_concepts_qa/_med_concepts_qa_atc.yaml @@ -0,0 +1,6 @@ +group: med_concepts_qa_atc +task: + - med_concepts_qa_atc_tasks +aggregate_metric_list: + - metric: acc + aggregation: mean diff --git a/lm_eval/tasks/med_concepts_qa/_med_concepts_qa_icd10cm.yaml b/lm_eval/tasks/med_concepts_qa/_med_concepts_qa_icd10cm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c4afc1bdfe7c9bf453ebed1fa38f21f94e6906ad --- /dev/null +++ b/lm_eval/tasks/med_concepts_qa/_med_concepts_qa_icd10cm.yaml @@ -0,0 +1,6 @@ +group: med_concepts_qa_icd10cm +task: + - med_concepts_qa_icd10cm_tasks +aggregate_metric_list: + - metric: acc + aggregation: mean diff --git a/lm_eval/tasks/med_concepts_qa/_med_concepts_qa_icd10proc.yaml b/lm_eval/tasks/med_concepts_qa/_med_concepts_qa_icd10proc.yaml new file mode 100644 index 0000000000000000000000000000000000000000..407ea4088df28c4370f2797adb47c01b440caebc --- /dev/null +++ b/lm_eval/tasks/med_concepts_qa/_med_concepts_qa_icd10proc.yaml @@ -0,0 +1,6 @@ +group: med_concepts_qa_icd10proc +task: + - med_concepts_qa_icd10proc_tasks +aggregate_metric_list: + - metric: acc + aggregation: mean diff --git a/lm_eval/tasks/med_concepts_qa/_med_concepts_qa_icd9cm.yaml b/lm_eval/tasks/med_concepts_qa/_med_concepts_qa_icd9cm.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b12ea811ff26631e65c9dd9cb42b56b1c0c7dba1 --- /dev/null +++ b/lm_eval/tasks/med_concepts_qa/_med_concepts_qa_icd9cm.yaml @@ -0,0 +1,6 @@ +group: med_concepts_qa_icd9cm +task: + - med_concepts_qa_icd9cm_tasks +aggregate_metric_list: + - metric: acc + aggregation: mean diff --git a/lm_eval/tasks/med_concepts_qa/_med_concepts_qa_icd9proc.yaml b/lm_eval/tasks/med_concepts_qa/_med_concepts_qa_icd9proc.yaml new file mode 100644 index 0000000000000000000000000000000000000000..94fc034eb20a8db04d2caf4d027e81be947ebf46 --- /dev/null +++ b/lm_eval/tasks/med_concepts_qa/_med_concepts_qa_icd9proc.yaml @@ -0,0 +1,6 @@ +group: med_concepts_qa_icd9proc +task: + - med_concepts_qa_icd9proc_tasks +aggregate_metric_list: + - metric: acc + aggregation: mean diff --git a/lm_eval/tasks/med_concepts_qa/med_concepts_qa_atc_easy.yaml b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_atc_easy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..be2f62a1dc4e3106b1ea6f33a1286b1358f5df5e --- /dev/null +++ b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_atc_easy.yaml @@ -0,0 +1,5 @@ +dataset_name: atc_easy +include: _default_template_yaml +tag: med_concepts_qa_atc_tasks +task: med_concepts_qa_atc_easy +task_alias: atc_easy diff --git a/lm_eval/tasks/med_concepts_qa/med_concepts_qa_atc_hard.yaml b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_atc_hard.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a54d27d97690886c2d0fb716b48dd0daf1a4bbab --- /dev/null +++ b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_atc_hard.yaml @@ -0,0 +1,5 @@ +dataset_name: atc_hard +include: _default_template_yaml +tag: med_concepts_qa_atc_tasks +task: med_concepts_qa_atc_hard +task_alias: atc_hard diff --git a/lm_eval/tasks/med_concepts_qa/med_concepts_qa_atc_medium.yaml b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_atc_medium.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e0b64ac85ba3435b9c7f3b4912225a2703358e6b --- /dev/null +++ b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_atc_medium.yaml @@ -0,0 +1,5 @@ +dataset_name: atc_medium +include: _default_template_yaml +tag: med_concepts_qa_atc_tasks +task: med_concepts_qa_atc_medium +task_alias: atc_medium diff --git a/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd10cm_easy.yaml b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd10cm_easy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0fe8da7ddd08f2b8a51777f06c359ef32f7c2106 --- /dev/null +++ b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd10cm_easy.yaml @@ -0,0 +1,5 @@ +dataset_name: icd10cm_easy +include: _default_template_yaml +tag: med_concepts_qa_icd10cm_tasks +task: med_concepts_qa_icd10cm_easy +task_alias: icd10cm_easy diff --git a/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd10cm_hard.yaml b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd10cm_hard.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2b291b0eb42fad609ea3e49a384f8fd55a48ff71 --- /dev/null +++ b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd10cm_hard.yaml @@ -0,0 +1,5 @@ +dataset_name: icd10cm_hard +include: _default_template_yaml +tag: med_concepts_qa_icd10cm_tasks +task: med_concepts_qa_icd10cm_hard +task_alias: icd10cm_hard diff --git a/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd10cm_medium.yaml b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd10cm_medium.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b1678fa311b550bc04430235059ad54327ec0ecf --- /dev/null +++ b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd10cm_medium.yaml @@ -0,0 +1,5 @@ +dataset_name: icd10cm_medium +include: _default_template_yaml +tag: med_concepts_qa_icd10cm_tasks +task: med_concepts_qa_icd10cm_medium +task_alias: icd10cm_medium diff --git a/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd10proc_easy.yaml b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd10proc_easy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..301f559d7c1cef808e880a809f2a0a91ad9ca227 --- /dev/null +++ b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd10proc_easy.yaml @@ -0,0 +1,5 @@ +dataset_name: icd10proc_easy +include: _default_template_yaml +tag: med_concepts_qa_icd10proc_tasks +task: med_concepts_qa_icd10proc_easy +task_alias: icd10proc_easy diff --git a/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd10proc_hard.yaml b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd10proc_hard.yaml new file mode 100644 index 0000000000000000000000000000000000000000..15efafba09f82bd8e987356511443329538ac594 --- /dev/null +++ b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd10proc_hard.yaml @@ -0,0 +1,5 @@ +dataset_name: icd10proc_hard +include: _default_template_yaml +tag: med_concepts_qa_icd10proc_tasks +task: med_concepts_qa_icd10proc_hard +task_alias: icd10proc_hard diff --git a/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd10proc_medium.yaml b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd10proc_medium.yaml new file mode 100644 index 0000000000000000000000000000000000000000..648bd46075de41e4ccf7b34882990772928123f7 --- /dev/null +++ b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd10proc_medium.yaml @@ -0,0 +1,5 @@ +dataset_name: icd10proc_medium +include: _default_template_yaml +tag: med_concepts_qa_icd10proc_tasks +task: med_concepts_qa_icd10proc_medium +task_alias: icd10proc_medium diff --git a/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9cm_easy.yaml b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9cm_easy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..514a9e258614aaa29e32f2c5bc3d54318175d837 --- /dev/null +++ b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9cm_easy.yaml @@ -0,0 +1,5 @@ +dataset_name: icd9cm_easy +include: _default_template_yaml +tag: med_concepts_qa_icd9cm_tasks +task: med_concepts_qa_icd9cm_easy +task_alias: icd9cm_easy diff --git a/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9cm_hard.yaml b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9cm_hard.yaml new file mode 100644 index 0000000000000000000000000000000000000000..41d0fd5534743ba4a8919c0622e119c383b05588 --- /dev/null +++ b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9cm_hard.yaml @@ -0,0 +1,5 @@ +dataset_name: icd9cm_hard +include: _default_template_yaml +tag: med_concepts_qa_icd9cm_tasks +task: med_concepts_qa_icd9cm_hard +task_alias: icd9cm_hard diff --git a/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9cm_medium.yaml b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9cm_medium.yaml new file mode 100644 index 0000000000000000000000000000000000000000..90f93f9bc2b520186a98c923dd7cf49d58a5293b --- /dev/null +++ b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9cm_medium.yaml @@ -0,0 +1,5 @@ +dataset_name: icd9cm_medium +include: _default_template_yaml +tag: med_concepts_qa_icd9cm_tasks +task: med_concepts_qa_icd9cm_medium +task_alias: icd9cm_medium diff --git a/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9proc_easy.yaml b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9proc_easy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..459fa464ffead9d78810394484ca8cc49cf67d18 --- /dev/null +++ b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9proc_easy.yaml @@ -0,0 +1,5 @@ +dataset_name: icd9proc_easy +include: _default_template_yaml +tag: med_concepts_qa_icd9proc_tasks +task: med_concepts_qa_icd9proc_easy +task_alias: icd9proc_easy diff --git a/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9proc_hard.yaml b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9proc_hard.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5424623c308011cf0891a83ca7bbd9a971e26a0d --- /dev/null +++ b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9proc_hard.yaml @@ -0,0 +1,5 @@ +dataset_name: icd9proc_hard +include: _default_template_yaml +tag: med_concepts_qa_icd9proc_tasks +task: med_concepts_qa_icd9proc_hard +task_alias: icd9proc_hard diff --git a/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9proc_medium.yaml b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9proc_medium.yaml new file mode 100644 index 0000000000000000000000000000000000000000..843029209916822a306e11e2795568a757cc4b2a --- /dev/null +++ b/lm_eval/tasks/med_concepts_qa/med_concepts_qa_icd9proc_medium.yaml @@ -0,0 +1,5 @@ +dataset_name: icd9proc_medium +include: _default_template_yaml +tag: med_concepts_qa_icd9proc_tasks +task: med_concepts_qa_icd9proc_medium +task_alias: icd9proc_medium diff --git a/lm_eval/tasks/mela/README.md b/lm_eval/tasks/mela/README.md new file mode 100644 index 0000000000000000000000000000000000000000..de008e494dc0fa46d06d5208305bdadde60b6441 --- /dev/null +++ b/lm_eval/tasks/mela/README.md @@ -0,0 +1,60 @@ +# Task-name + +### Paper + +Title: [MELA: Multilingual Evaluation of Linguistic Acceptability](https://arxiv.org/abs/2311.09033) + +**Abstract**: In this work, we present the largest benchmark to date on linguistic acceptability: Multilingual Evaluation of Linguistic Acceptability -- MELA, with 46K samples covering 10 languages from a diverse set of language families. We establish LLM baselines on this benchmark, and investigate cross-lingual transfer in acceptability judgements with XLM-R. In pursuit of multilingual interpretability, we conduct probing experiments with fine-tuned XLM-R to explore the process of syntax capability acquisition. Our results show that GPT-4o exhibits a strong multilingual ability, outperforming fine-tuned XLM-R, while open-source multilingual models lag behind by a noticeable gap. Cross-lingual transfer experiments show that transfer in acceptability judgment is non-trivial: 500 Icelandic fine-tuning examples lead to 23 MCC performance in a completely unrelated language -- Chinese. Results of our probing experiments indicate that training on MELA improves the performance of XLM-R on syntax-related tasks. + +Homepage: https://github.com/sjtu-compling/MELA + +### Citation + +``` +@inproceedings{zhang2023mela, + author = {Ziyin Zhang and + Yikang Liu and + Weifang Huang and + Junyu Mao and + Rui Wang and + Hai Hu}, + title = {{MELA:} Multilingual Evaluation of Linguistic Acceptability}, + booktitle = {Proceedings of the 62nd Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers), {ACL} 2024, Bangkok, Thailand}, + publisher = {Association for Computational Linguistics}, + year = {2024}, + url = {https://doi.org/10.48550/arXiv.2311.09033} +} +``` + +### Groups and Tasks + +#### Groups + +- `mela`: multilingual evaluation of linguistic acceptability + +#### Tasks + +- `mela_en`: English +- `mela_zh`: Chinese +- `mela_it`: Italian +- `mela_ru`: Russian +- `mela_de`: Germany +- `mela_fr`: French +- `mela_es`: Spanish +- `mela_ja`: Japanese +- `mela_ar`: Arabic +- `mela_ar`: Icelandic + +### Checklist + +For adding novel benchmarks/datasets to the library: + +- [x] Is the task an existing benchmark in the literature? + - [x] Have you referenced the original paper that introduced the task? + - [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + +If other tasks on this dataset are already supported: + +- [ ] Is the "Main" variant of this task clearly denoted? +- [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates? +- [ ] Have you noted which, if any, published evaluation setups are matched by this variant? diff --git a/lm_eval/tasks/mela/_mela.yaml b/lm_eval/tasks/mela/_mela.yaml new file mode 100644 index 0000000000000000000000000000000000000000..107498c0b23a233690b6591078c5e65fd20b37d2 --- /dev/null +++ b/lm_eval/tasks/mela/_mela.yaml @@ -0,0 +1,17 @@ +group: mela +task: + - mela_en + - mela_zh + - mela_it + - mela_ru + - mela_de + - mela_fr + - mela_es + - mela_ja + - mela_ar + - mela_ar +aggregate_metric_list: + - metric: mcc + weight_by_size: False +metadata: + version: 1 diff --git a/lm_eval/tasks/mela/mela_ar.yaml b/lm_eval/tasks/mela/mela_ar.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eea6026a6dec0baf3cb0dcb764b1334399f1311a --- /dev/null +++ b/lm_eval/tasks/mela/mela_ar.yaml @@ -0,0 +1,4 @@ +include: mela_en.yaml +task: mela_ar +dataset_name: ar +training_split: null diff --git a/lm_eval/tasks/mela/mela_de.yaml b/lm_eval/tasks/mela/mela_de.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3108142d36db4efdc00d254ae4e21fd7170bd334 --- /dev/null +++ b/lm_eval/tasks/mela/mela_de.yaml @@ -0,0 +1,4 @@ +include: mela_en.yaml +task: mela_de +dataset_name: de +training_split: null diff --git a/lm_eval/tasks/mela/mela_en.yaml b/lm_eval/tasks/mela/mela_en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5727b785993c98fad6b2bd630c8dbd076ff60c45 --- /dev/null +++ b/lm_eval/tasks/mela/mela_en.yaml @@ -0,0 +1,17 @@ +task: mela_en +dataset_path: Geralt-Targaryen/MELA +dataset_name: en +training_split: train +validation_split: dev +test_split: test +output_type: multiple_choice +doc_to_text: "Sentence: {{sentence}}\nDetermine whether this sentence is acceptable or unacceptable?\nA. Acceptable\nB. Unacceptable\nAnswer:" +doc_to_choice: ["A", "B"] +doc_to_target: "{{['B', 'A'][label]}}" +description: "Determine whether the following sentence(s) violate certain linguistic constraints. If yes, then it is \"unacceptable\"; otherwise, \"acceptable\".\n\n" +fewshot_split: dev +fewshot_config: + sampler: first_n +metric_list: + - metric: mcc + higher_is_better: true diff --git a/lm_eval/tasks/mela/mela_es.yaml b/lm_eval/tasks/mela/mela_es.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c8c6716243055222e4c07a37ee90d9bd0d33b058 --- /dev/null +++ b/lm_eval/tasks/mela/mela_es.yaml @@ -0,0 +1,4 @@ +include: mela_en.yaml +task: mela_es +dataset_name: es +training_split: null diff --git a/lm_eval/tasks/mela/mela_fr.yaml b/lm_eval/tasks/mela/mela_fr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7bf68ad6a43ba7866bf836663b6f58b558db8e68 --- /dev/null +++ b/lm_eval/tasks/mela/mela_fr.yaml @@ -0,0 +1,4 @@ +include: mela_en.yaml +task: mela_fr +dataset_name: fr +training_split: null diff --git a/lm_eval/tasks/mela/mela_is.yaml b/lm_eval/tasks/mela/mela_is.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cd312f227e35b62bd8438368c741295a2a4bdddd --- /dev/null +++ b/lm_eval/tasks/mela/mela_is.yaml @@ -0,0 +1,4 @@ +include: mela_en.yaml +task: mela_is +dataset_name: is +training_split: null diff --git a/lm_eval/tasks/mela/mela_it.yaml b/lm_eval/tasks/mela/mela_it.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1751d65a7d60921446db78c001b41a51739d9249 --- /dev/null +++ b/lm_eval/tasks/mela/mela_it.yaml @@ -0,0 +1,3 @@ +include: mela_en.yaml +task: mela_it +dataset_name: it diff --git a/lm_eval/tasks/mela/mela_ja.yaml b/lm_eval/tasks/mela/mela_ja.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fb8477e06028d32741816b2a0deac04975fa3ffc --- /dev/null +++ b/lm_eval/tasks/mela/mela_ja.yaml @@ -0,0 +1,4 @@ +include: mela_en.yaml +task: mela_ja +dataset_name: ja +training_split: null diff --git a/lm_eval/tasks/mela/mela_ru.yaml b/lm_eval/tasks/mela/mela_ru.yaml new file mode 100644 index 0000000000000000000000000000000000000000..191a9a1c881c665ca779e624bde1bd21d65d0062 --- /dev/null +++ b/lm_eval/tasks/mela/mela_ru.yaml @@ -0,0 +1,3 @@ +include: mela_en.yaml +task: mela_ru +dataset_name: ru diff --git a/lm_eval/tasks/mela/mela_zh.yaml b/lm_eval/tasks/mela/mela_zh.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e3456abcdda3dac1529c429159a36366509cab13 --- /dev/null +++ b/lm_eval/tasks/mela/mela_zh.yaml @@ -0,0 +1,3 @@ +include: mela_en.yaml +task: mela_zh +dataset_name: zh diff --git a/lm_eval/tasks/mgsm/direct/direct_yaml b/lm_eval/tasks/mgsm/direct/direct_yaml index 3a265cb025916a00807fefd7c3f39466a4ce80ae..d2e301ba27abf703bff771bd4b79fa2448d188c1 100644 --- a/lm_eval/tasks/mgsm/direct/direct_yaml +++ b/lm_eval/tasks/mgsm/direct/direct_yaml @@ -1,7 +1,7 @@ # This file will be included in the generated language-specific task configs. # It doesn't have a yaml file extension as it is not meant to be imported directly # by the harness. -group: mgsm_direct +tag: mgsm_direct dataset_path: juletxara/mgsm dataset_name: null # Overridden by language-specific config. output_type: generate_until diff --git a/lm_eval/tasks/mgsm/en_cot/cot_yaml b/lm_eval/tasks/mgsm/en_cot/cot_yaml index f4d502ee52f4389d4331be7dcde287d1c47c3f59..b53ae970800e31ff95815f0410407e90ea6afe30 100644 --- a/lm_eval/tasks/mgsm/en_cot/cot_yaml +++ b/lm_eval/tasks/mgsm/en_cot/cot_yaml @@ -1,7 +1,7 @@ # This file will be included in the generated language-specific task configs. # It doesn't have a yaml file extension as it is not meant to be imported directly # by the harness. -group: mgsm_cot_native +tag: mgsm_cot_native dataset_path: juletxara/mgsm dataset_name: null # Overridden by language-specific config. output_type: generate_until diff --git a/lm_eval/tasks/mgsm/native_cot/cot_yaml b/lm_eval/tasks/mgsm/native_cot/cot_yaml index dbba882225b1d7c9fbe10352c64a381c97a547c7..eb058ca42aca61df8d180e25879d39d44985f9eb 100644 --- a/lm_eval/tasks/mgsm/native_cot/cot_yaml +++ b/lm_eval/tasks/mgsm/native_cot/cot_yaml @@ -1,7 +1,7 @@ # This file will be included in the generated language-specific task configs. # It doesn't have a yaml file extension as it is not meant to be imported directly # by the harness. -group: mgsm_cot_native +tag: mgsm_cot_native dataset_path: juletxara/mgsm dataset_name: null # Overridden by language-specific config. output_type: generate_until diff --git a/lm_eval/tasks/minerva_math/minerva_math_algebra.yaml b/lm_eval/tasks/minerva_math/minerva_math_algebra.yaml index ac06b7ac6e00580ad7918d2000e82bff486733a5..7eab43c6bcf86c3a75db49ac60e2ad7b19c07625 100644 --- a/lm_eval/tasks/minerva_math/minerva_math_algebra.yaml +++ b/lm_eval/tasks/minerva_math/minerva_math_algebra.yaml @@ -1,4 +1,4 @@ -group: +tag: - math_word_problems task: minerva_math_algebra dataset_path: EleutherAI/hendrycks_math diff --git a/lm_eval/tasks/mmlu/README.md b/lm_eval/tasks/mmlu/README.md new file mode 100644 index 0000000000000000000000000000000000000000..a3425d517654a6b93e03ee1bb681e07de18c4016 --- /dev/null +++ b/lm_eval/tasks/mmlu/README.md @@ -0,0 +1,73 @@ +# Task-name + +### Paper + +Title: `Measuring Massive Multitask Language Understanding` + +Abstract: `https://arxiv.org/abs/2009.03300` + +`The test covers 57 tasks including elementary mathematics, US history, computer science, law, and more.` + +Homepage: `https://github.com/hendrycks/test` + +Note: The `Flan` variants are derived from [here](https://github.com/jasonwei20/flan-2), and as described in Appendix D.1 of [Scaling Instruction-Finetuned Language Models](https://arxiv.org/abs/2210.11416). + +### Citation + +``` +@article{hendryckstest2021, + title={Measuring Massive Multitask Language Understanding}, + author={Dan Hendrycks and Collin Burns and Steven Basart and Andy Zou and Mantas Mazeika and Dawn Song and Jacob Steinhardt}, + journal={Proceedings of the International Conference on Learning Representations (ICLR)}, + year={2021} +} + +@article{hendrycks2021ethics, + title={Aligning AI With Shared Human Values}, + author={Dan Hendrycks and Collin Burns and Steven Basart and Andrew Critch and Jerry Li and Dawn Song and Jacob Steinhardt}, + journal={Proceedings of the International Conference on Learning Representations (ICLR)}, + year={2021} +} +``` + +### Groups, Tags, and Tasks + +#### Groups + +* `mmlu`: `Original multiple-choice MMLU benchmark` +* `mmlu_continuation`: `MMLU but with continuation prompts` +* `mmlu_generation`: `MMLU generation` + +MMLU is the original benchmark as implemented by Hendrycks et al. with the choices in context and the answer letters (e.g `A`, `B`, `C`, `D`) in the continuation. +`mmlu_continuation` is a cloze-style variant without the choices in context and the full answer choice in the continuation. +`mmlu_generation` is a generation variant, similar to the original but the LLM is asked to generate the correct answer letter. + + +#### Subgroups + +* `mmlu_stem' +* `mmlu_humanities' +* `mmlu_social_sciences' +* `mmlu_other' + +Subgroup variants are prefixed with the subgroup name, e.g. `mmlu_stem_continuation`. + +### Checklist + +For adding novel benchmarks/datasets to the library: +* [x] Is the task an existing benchmark in the literature? + * [x] Have you referenced the original paper that introduced the task? + * [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + + +If other tasks on this dataset are already supported: +* [x] Is the "Main" variant of this task clearly denoted? +* [x] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [x] Have you noted which, if any, published evaluation setups are matched by this variant? + +# changelog +ver 1: PR #497 +switch to original implementation + +ver 2: PR #2116 +add missing newline in description. diff --git a/lm_eval/tasks/mmlu/_generate_configs.py b/lm_eval/tasks/mmlu/_generate_configs.py index cb4a076d438be52bcbf4a18ec796d655c541cc01..28b94616ddec7d0690be3d20c830990462f0dde6 100644 --- a/lm_eval/tasks/mmlu/_generate_configs.py +++ b/lm_eval/tasks/mmlu/_generate_configs.py @@ -110,10 +110,9 @@ if __name__ == "__main__": yaml_dict = { "include": base_yaml_name, - "group": f"mmlu_{args.task_prefix}_{category}" + "tag": f"mmlu_{args.task_prefix}_{category}" if args.task_prefix != "" else f"mmlu_{category}", - "group_alias": category.replace("_", " "), "task": f"mmlu_{args.task_prefix}_{subject}" if args.task_prefix != "" else f"mmlu_{subject}", diff --git a/lm_eval/tasks/mmlu/continuation/_continuation_template_yaml b/lm_eval/tasks/mmlu/continuation/_continuation_template_yaml index 45e8928a0cdaa2f347d880d1536324439c3ac4d8..264e27a5e8ebde9a203094c7cc9735ecf8ef3993 100644 --- a/lm_eval/tasks/mmlu/continuation/_continuation_template_yaml +++ b/lm_eval/tasks/mmlu/continuation/_continuation_template_yaml @@ -8,6 +8,6 @@ doc_to_text: "Question: {{question.strip()}}\nAnswer:" doc_to_choice: "{{choices}}" doc_to_target: "{{answer}}" metadata: - version: 0.0 + version: 1.0 dataset_kwargs: trust_remote_code: true diff --git a/lm_eval/tasks/mmlu/continuation/_mmlu.yaml b/lm_eval/tasks/mmlu/continuation/_mmlu.yaml index 77ef1b1d41496ea8715a8197977809f12c1e0aae..c0cabf04b8ac1e1f9c809600214c589cfefbba79 100644 --- a/lm_eval/tasks/mmlu/continuation/_mmlu.yaml +++ b/lm_eval/tasks/mmlu/continuation/_mmlu.yaml @@ -1,6 +1,32 @@ group: mmlu_continuation +group_alias: mmlu (continuation) task: - - mmlu_continuation_stem - - mmlu_continuation_other - - mmlu_continuation_social_sciences - - mmlu_continuation_humanities + - group: stem + task: + - mmlu_continuation_stem + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: other + task: + - mmlu_continuation_other + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: social sciences + task: + - mmlu_continuation_social_sciences + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: humanities + task: + - mmlu_continuation_humanities + aggregate_metric_list: + - metric: acc + weight_by_size: True +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 2 diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_abstract_algebra.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_abstract_algebra.yaml index 962ee9c81fe5cede17ab202103483cba26936498..6f4e29c0fb5147d883ee993d95822dde10b69d4e 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_abstract_algebra.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_abstract_algebra.yaml @@ -1,6 +1,6 @@ "dataset_name": "abstract_algebra" "description": "The following are questions (with answers) about abstract\ \ algebra.\n\n" -"group": "mmlu_continuation_stem" +"tag": "mmlu_continuation_stem" "include": "_continuation_template_yaml" "task": "mmlu_continuation_abstract_algebra" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_anatomy.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_anatomy.yaml index efb8ab56ad6333bd88dc5f58ea50cb3f768bf4c8..bc3de9c4e6679ba4c9f66494c908d99781adf5bb 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_anatomy.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_anatomy.yaml @@ -1,6 +1,6 @@ "dataset_name": "anatomy" "description": "The following are questions (with answers) about anatomy.\n\ \n" -"group": "mmlu_continuation_stem" +"tag": "mmlu_continuation_stem" "include": "_continuation_template_yaml" "task": "mmlu_continuation_anatomy" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_astronomy.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_astronomy.yaml index 7cebe62c87c4163eace5c780b14e0b626afb73f1..76aabcbfcf13a12e66e1af1daae2811b9b388fc8 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_astronomy.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_astronomy.yaml @@ -1,6 +1,6 @@ "dataset_name": "astronomy" "description": "The following are questions (with answers) about astronomy.\n\ \n" -"group": "mmlu_continuation_stem" +"tag": "mmlu_continuation_stem" "include": "_continuation_template_yaml" "task": "mmlu_continuation_astronomy" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_business_ethics.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_business_ethics.yaml index 52e47605bd6d288d7d8fba8f0bd6202ece2e47f9..e64d0920b9d1ac151712aac84a9e9c3f522c3c9f 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_business_ethics.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_business_ethics.yaml @@ -1,6 +1,6 @@ "dataset_name": "business_ethics" "description": "The following are questions (with answers) about business\ \ ethics.\n\n" -"group": "mmlu_continuation_other" +"tag": "mmlu_continuation_other" "include": "_continuation_template_yaml" "task": "mmlu_continuation_business_ethics" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_clinical_knowledge.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_clinical_knowledge.yaml index a0abd04905b0c0179a7b745f3719cc1fab8b5167..e79805df6f73782f25be4a302c738b73ecd2f2a2 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_clinical_knowledge.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_clinical_knowledge.yaml @@ -1,6 +1,6 @@ "dataset_name": "clinical_knowledge" "description": "The following are questions (with answers) about clinical\ \ knowledge.\n\n" -"group": "mmlu_continuation_other" +"tag": "mmlu_continuation_other" "include": "_continuation_template_yaml" "task": "mmlu_continuation_clinical_knowledge" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_college_biology.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_college_biology.yaml index a2c8570c4b7c0d8da385d7b67e5206ed407cec05..936f6ffe49245d558c0ef8fdf04b600dc177c375 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_college_biology.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_college_biology.yaml @@ -1,6 +1,6 @@ "dataset_name": "college_biology" "description": "The following are questions (with answers) about college\ \ biology.\n\n" -"group": "mmlu_continuation_stem" +"tag": "mmlu_continuation_stem" "include": "_continuation_template_yaml" "task": "mmlu_continuation_college_biology" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_college_chemistry.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_college_chemistry.yaml index cfb11c87b17aa2143bca0fa372b261aefa6c5c64..289364ee44351c3d1bcee1193563babe6abe2a63 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_college_chemistry.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_college_chemistry.yaml @@ -1,6 +1,6 @@ "dataset_name": "college_chemistry" "description": "The following are questions (with answers) about college\ \ chemistry.\n\n" -"group": "mmlu_continuation_stem" +"tag": "mmlu_continuation_stem" "include": "_continuation_template_yaml" "task": "mmlu_continuation_college_chemistry" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_college_computer_science.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_college_computer_science.yaml index 67c6548e539a86fb167fb5ebada17826ac6c380f..c7d3c5696067f09f9a68fdd9c3f7a1002d264128 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_college_computer_science.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_college_computer_science.yaml @@ -1,6 +1,6 @@ "dataset_name": "college_computer_science" "description": "The following are questions (with answers) about college\ \ computer science.\n\n" -"group": "mmlu_continuation_stem" +"tag": "mmlu_continuation_stem" "include": "_continuation_template_yaml" "task": "mmlu_continuation_college_computer_science" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_college_mathematics.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_college_mathematics.yaml index f694391b2cb92dcd0c7828c376b02ac62cf26215..2dbc0932f63c0782e106db5fc27e96da9d816dec 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_college_mathematics.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_college_mathematics.yaml @@ -1,6 +1,6 @@ "dataset_name": "college_mathematics" "description": "The following are questions (with answers) about college\ \ mathematics.\n\n" -"group": "mmlu_continuation_stem" +"tag": "mmlu_continuation_stem" "include": "_continuation_template_yaml" "task": "mmlu_continuation_college_mathematics" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_college_medicine.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_college_medicine.yaml index 97b83a3feb4daba7335f88eb9d5d54ed1f1b555c..38abd2426f844916087795c4cc04355d8d6c2776 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_college_medicine.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_college_medicine.yaml @@ -1,6 +1,6 @@ "dataset_name": "college_medicine" "description": "The following are questions (with answers) about college\ \ medicine.\n\n" -"group": "mmlu_continuation_other" +"tag": "mmlu_continuation_other" "include": "_continuation_template_yaml" "task": "mmlu_continuation_college_medicine" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_college_physics.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_college_physics.yaml index b01e3ff19175ab0a826959511f1bb1d939caf710..ee6b42584c834a5e92506650ee3aba58ed1cfd66 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_college_physics.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_college_physics.yaml @@ -1,6 +1,6 @@ "dataset_name": "college_physics" "description": "The following are questions (with answers) about college\ \ physics.\n\n" -"group": "mmlu_continuation_stem" +"tag": "mmlu_continuation_stem" "include": "_continuation_template_yaml" "task": "mmlu_continuation_college_physics" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_computer_security.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_computer_security.yaml index 8ba7f5b0ab2fd736220ada787836da763a928866..7ebb487dfbf634d390d2b2f9aa0e31e5a2f68fc6 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_computer_security.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_computer_security.yaml @@ -1,6 +1,6 @@ "dataset_name": "computer_security" "description": "The following are questions (with answers) about computer\ \ security.\n\n" -"group": "mmlu_continuation_stem" +"tag": "mmlu_continuation_stem" "include": "_continuation_template_yaml" "task": "mmlu_continuation_computer_security" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_conceptual_physics.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_conceptual_physics.yaml index 8a4fd2968ddbf4edd04c5c1efbba1c5ac974a558..7c554caf07da77e4a9bb0bea9672dfcee4777b91 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_conceptual_physics.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_conceptual_physics.yaml @@ -1,6 +1,6 @@ "dataset_name": "conceptual_physics" "description": "The following are questions (with answers) about conceptual\ \ physics.\n\n" -"group": "mmlu_continuation_stem" +"tag": "mmlu_continuation_stem" "include": "_continuation_template_yaml" "task": "mmlu_continuation_conceptual_physics" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_econometrics.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_econometrics.yaml index 2bd6526549bf1fee2249062a63f28080311750d4..848ce4e1f0dbff32d304c28f3d60d453e591a30f 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_econometrics.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_econometrics.yaml @@ -1,6 +1,6 @@ "dataset_name": "econometrics" "description": "The following are questions (with answers) about econometrics.\n\ \n" -"group": "mmlu_continuation_social_sciences" +"tag": "mmlu_continuation_social_sciences" "include": "_continuation_template_yaml" "task": "mmlu_continuation_econometrics" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_electrical_engineering.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_electrical_engineering.yaml index b7e9cbdb471091d8eaf191f384cfcc50a0fdccdf..d71dd16481a2bb5289ef5b713218dae0292bb11a 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_electrical_engineering.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_electrical_engineering.yaml @@ -1,6 +1,6 @@ "dataset_name": "electrical_engineering" "description": "The following are questions (with answers) about electrical\ \ engineering.\n\n" -"group": "mmlu_continuation_stem" +"tag": "mmlu_continuation_stem" "include": "_continuation_template_yaml" "task": "mmlu_continuation_electrical_engineering" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_elementary_mathematics.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_elementary_mathematics.yaml index 689dc17c9cc3b4a4609ad1969f2128067562ff66..fe8aa09718cb8aef0dad48c21926f7dacc7b8ee9 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_elementary_mathematics.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_elementary_mathematics.yaml @@ -1,6 +1,6 @@ "dataset_name": "elementary_mathematics" "description": "The following are questions (with answers) about elementary\ \ mathematics.\n\n" -"group": "mmlu_continuation_stem" +"tag": "mmlu_continuation_stem" "include": "_continuation_template_yaml" "task": "mmlu_continuation_elementary_mathematics" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_formal_logic.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_formal_logic.yaml index 149b6c844963cd93141dfa0e9f5efa3310d46e48..eb5dbd2e505e3fb4604dd75f2d5fe1a35fce3391 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_formal_logic.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_formal_logic.yaml @@ -1,6 +1,6 @@ "dataset_name": "formal_logic" "description": "The following are questions (with answers) about formal\ \ logic.\n\n" -"group": "mmlu_continuation_humanities" +"tag": "mmlu_continuation_humanities" "include": "_continuation_template_yaml" "task": "mmlu_continuation_formal_logic" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_global_facts.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_global_facts.yaml index e02fba5f68f955191fdfa68852d33c9ec70af9f8..280a50d2ee229b5f047a02024298474225203e54 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_global_facts.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_global_facts.yaml @@ -1,6 +1,6 @@ "dataset_name": "global_facts" "description": "The following are questions (with answers) about global\ \ facts.\n\n" -"group": "mmlu_continuation_other" +"tag": "mmlu_continuation_other" "include": "_continuation_template_yaml" "task": "mmlu_continuation_global_facts" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_high_school_biology.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_high_school_biology.yaml index f774f95868f8dc102ca7f840ee966990529008b4..e518a5239a6da013ad31bfca284a3b7096bce840 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_high_school_biology.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_high_school_biology.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_biology" "description": "The following are questions (with answers) about high\ \ school biology.\n\n" -"group": "mmlu_continuation_stem" +"tag": "mmlu_continuation_stem" "include": "_continuation_template_yaml" "task": "mmlu_continuation_high_school_biology" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_high_school_chemistry.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_high_school_chemistry.yaml index c6a338a72cbf4e4e11e348d728f326ecb34f1b9b..c38d60a7706306b215e156d4c27f05585945f7b4 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_high_school_chemistry.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_high_school_chemistry.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_chemistry" "description": "The following are questions (with answers) about high\ \ school chemistry.\n\n" -"group": "mmlu_continuation_stem" +"tag": "mmlu_continuation_stem" "include": "_continuation_template_yaml" "task": "mmlu_continuation_high_school_chemistry" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_high_school_computer_science.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_high_school_computer_science.yaml index 4038906350e8bbd42a48d6493628835781e57313..5fe34f7af35456657c1acf40e05b3aaabc7893e8 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_high_school_computer_science.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_high_school_computer_science.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_computer_science" "description": "The following are questions (with answers) about high\ \ school computer science.\n\n" -"group": "mmlu_continuation_stem" +"tag": "mmlu_continuation_stem" "include": "_continuation_template_yaml" "task": "mmlu_continuation_high_school_computer_science" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_high_school_european_history.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_high_school_european_history.yaml index 0ef0cbd2b9dc2471b27798fe95dba20c432ec920..666c2742d1b762c103bbd02ff121676a047fb3e5 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_high_school_european_history.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_high_school_european_history.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_european_history" "description": "The following are questions (with answers) about high\ \ school european history.\n\n" -"group": "mmlu_continuation_humanities" +"tag": "mmlu_continuation_humanities" "include": "_continuation_template_yaml" "task": "mmlu_continuation_high_school_european_history" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_high_school_geography.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_high_school_geography.yaml index 301280b89b1960df33f8601a5d4ef9e4eab9cb11..41f6caf3e7f3b762af7c0350ca9a73d39bede2b8 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_high_school_geography.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_high_school_geography.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_geography" "description": "The following are questions (with answers) about high\ \ school geography.\n\n" -"group": "mmlu_continuation_social_sciences" +"tag": "mmlu_continuation_social_sciences" "include": "_continuation_template_yaml" "task": "mmlu_continuation_high_school_geography" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_high_school_government_and_politics.yaml index 792ed584c77571e2459d23c521f51e4bc884142f..e80233dc891e6890a5dec384ed2fbe5b82aca094 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_high_school_government_and_politics.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_high_school_government_and_politics.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_government_and_politics" "description": "The following are questions (with answers) about high\ \ school government and politics.\n\n" -"group": "mmlu_continuation_social_sciences" +"tag": "mmlu_continuation_social_sciences" "include": "_continuation_template_yaml" "task": "mmlu_continuation_high_school_government_and_politics" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_high_school_macroeconomics.yaml index 858d91f676b75768e2915338aceffd7df8b2f49d..ce7fa9d5e3caa8dd3ec8e25172afda5f997b6c0c 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_high_school_macroeconomics.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_high_school_macroeconomics.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_macroeconomics" "description": "The following are questions (with answers) about high\ \ school macroeconomics.\n\n" -"group": "mmlu_continuation_social_sciences" +"tag": "mmlu_continuation_social_sciences" "include": "_continuation_template_yaml" "task": "mmlu_continuation_high_school_macroeconomics" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_high_school_mathematics.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_high_school_mathematics.yaml index 9c64bb05b1b3b1d9953c396eac8540664847cbde..2598dcb38eb9f8fdacced20c57d62c83dacb8a40 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_high_school_mathematics.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_high_school_mathematics.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_mathematics" "description": "The following are questions (with answers) about high\ \ school mathematics.\n\n" -"group": "mmlu_continuation_stem" +"tag": "mmlu_continuation_stem" "include": "_continuation_template_yaml" "task": "mmlu_continuation_high_school_mathematics" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_high_school_microeconomics.yaml index df33f528de4016f1ea4be413b942c6d23a1b4489..96c414d3c411c6380cf83dca3b7aedc325598220 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_high_school_microeconomics.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_high_school_microeconomics.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_microeconomics" "description": "The following are questions (with answers) about high\ \ school microeconomics.\n\n" -"group": "mmlu_continuation_social_sciences" +"tag": "mmlu_continuation_social_sciences" "include": "_continuation_template_yaml" "task": "mmlu_continuation_high_school_microeconomics" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_high_school_physics.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_high_school_physics.yaml index 562993b6bd0bf0b2f7911a991ac6a1a8067b070c..45ab0a539a02ae322f66db689d8eddf13c8b856a 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_high_school_physics.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_high_school_physics.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_physics" "description": "The following are questions (with answers) about high\ \ school physics.\n\n" -"group": "mmlu_continuation_stem" +"tag": "mmlu_continuation_stem" "include": "_continuation_template_yaml" "task": "mmlu_continuation_high_school_physics" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_high_school_psychology.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_high_school_psychology.yaml index b3b4a9349599d9dc9b850f2604ef79109fffb003..48dedf5c5ed94a836e0d802398ab05d7ab7db6ce 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_high_school_psychology.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_high_school_psychology.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_psychology" "description": "The following are questions (with answers) about high\ \ school psychology.\n\n" -"group": "mmlu_continuation_social_sciences" +"tag": "mmlu_continuation_social_sciences" "include": "_continuation_template_yaml" "task": "mmlu_continuation_high_school_psychology" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_high_school_statistics.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_high_school_statistics.yaml index be899e1d4e88ff2f3c27d88b6ea3f5bd292217d7..2ee2418c7ff5235c1e31cf381502f5b21db60230 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_high_school_statistics.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_high_school_statistics.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_statistics" "description": "The following are questions (with answers) about high\ \ school statistics.\n\n" -"group": "mmlu_continuation_stem" +"tag": "mmlu_continuation_stem" "include": "_continuation_template_yaml" "task": "mmlu_continuation_high_school_statistics" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_high_school_us_history.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_high_school_us_history.yaml index ad16ce7db1defbaae71a1d5c4974c415680d4134..a00f16ceba2cfd3f313c8fe0d2df4a43e4bbe23d 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_high_school_us_history.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_high_school_us_history.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_us_history" "description": "The following are questions (with answers) about high\ \ school us history.\n\n" -"group": "mmlu_continuation_humanities" +"tag": "mmlu_continuation_humanities" "include": "_continuation_template_yaml" "task": "mmlu_continuation_high_school_us_history" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_high_school_world_history.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_high_school_world_history.yaml index dc2bcbfbbdd4edaa84a28087e242de4ee4dbb950..dc4cddf553bf0144b5d4ecc5eabe8efef0cf0367 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_high_school_world_history.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_high_school_world_history.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_world_history" "description": "The following are questions (with answers) about high\ \ school world history.\n\n" -"group": "mmlu_continuation_humanities" +"tag": "mmlu_continuation_humanities" "include": "_continuation_template_yaml" "task": "mmlu_continuation_high_school_world_history" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_human_aging.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_human_aging.yaml index 1ad19e9aae5f8c94f966876ad15ca118ef9983cd..314edeb6c26c6a6be2d819b7c66e047cd48f8933 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_human_aging.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_human_aging.yaml @@ -1,6 +1,6 @@ "dataset_name": "human_aging" "description": "The following are questions (with answers) about human\ \ aging.\n\n" -"group": "mmlu_continuation_other" +"tag": "mmlu_continuation_other" "include": "_continuation_template_yaml" "task": "mmlu_continuation_human_aging" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_human_sexuality.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_human_sexuality.yaml index 607a951d35711f7385a4a999ba226a8f6b467827..a1473819ab4307f1e02024a0828ad9803710a59b 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_human_sexuality.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_human_sexuality.yaml @@ -1,6 +1,6 @@ "dataset_name": "human_sexuality" "description": "The following are questions (with answers) about human\ \ sexuality.\n\n" -"group": "mmlu_continuation_social_sciences" +"tag": "mmlu_continuation_social_sciences" "include": "_continuation_template_yaml" "task": "mmlu_continuation_human_sexuality" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_international_law.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_international_law.yaml index 0783ce18b5792ed66371dbe746e05f6997f29ac3..5ea8944bcc109000525b90f26f1d0da914d17437 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_international_law.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_international_law.yaml @@ -1,6 +1,6 @@ "dataset_name": "international_law" "description": "The following are questions (with answers) about international\ \ law.\n\n" -"group": "mmlu_continuation_humanities" +"tag": "mmlu_continuation_humanities" "include": "_continuation_template_yaml" "task": "mmlu_continuation_international_law" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_jurisprudence.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_jurisprudence.yaml index 37de68e1d24c7be97dbc4066a2408ee753ca8183..fca1dda86cc382604ca1bcbc308e0062e08dfa80 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_jurisprudence.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_jurisprudence.yaml @@ -1,6 +1,6 @@ "dataset_name": "jurisprudence" "description": "The following are questions (with answers) about jurisprudence.\n\ \n" -"group": "mmlu_continuation_humanities" +"tag": "mmlu_continuation_humanities" "include": "_continuation_template_yaml" "task": "mmlu_continuation_jurisprudence" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_logical_fallacies.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_logical_fallacies.yaml index d5b37d4a955e545490bbd63173049a9452813b88..1b576f9fb3d0ce1d21e8d7543b56a539300be36a 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_logical_fallacies.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_logical_fallacies.yaml @@ -1,6 +1,6 @@ "dataset_name": "logical_fallacies" "description": "The following are questions (with answers) about logical\ \ fallacies.\n\n" -"group": "mmlu_continuation_humanities" +"tag": "mmlu_continuation_humanities" "include": "_continuation_template_yaml" "task": "mmlu_continuation_logical_fallacies" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_machine_learning.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_machine_learning.yaml index 7fc0ffd41d23bae6181d9ce0bdf75310b6f3ef27..15fc3f4bdf0f34e96149ca2f8dddc90d037e8483 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_machine_learning.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_machine_learning.yaml @@ -1,6 +1,6 @@ "dataset_name": "machine_learning" "description": "The following are questions (with answers) about machine\ \ learning.\n\n" -"group": "mmlu_continuation_stem" +"tag": "mmlu_continuation_stem" "include": "_continuation_template_yaml" "task": "mmlu_continuation_machine_learning" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_management.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_management.yaml index fe25108ffd84f869670799ea5ed9da7ac494f5b5..575604e0acf52132d9e489a070d28fd761e739eb 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_management.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_management.yaml @@ -1,6 +1,6 @@ "dataset_name": "management" "description": "The following are questions (with answers) about management.\n\ \n" -"group": "mmlu_continuation_other" +"tag": "mmlu_continuation_other" "include": "_continuation_template_yaml" "task": "mmlu_continuation_management" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_marketing.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_marketing.yaml index 2beebd29827f47a0398370638ec8cc32ecf961e5..af715bee02cfe813b5f045670c8e46dda258e77d 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_marketing.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_marketing.yaml @@ -1,6 +1,6 @@ "dataset_name": "marketing" "description": "The following are questions (with answers) about marketing.\n\ \n" -"group": "mmlu_continuation_other" +"tag": "mmlu_continuation_other" "include": "_continuation_template_yaml" "task": "mmlu_continuation_marketing" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_medical_genetics.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_medical_genetics.yaml index 04d63864d3ffa8671678cad613c96611d4a96554..3bf63614168f648497d046f015472497a2ac7553 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_medical_genetics.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_medical_genetics.yaml @@ -1,6 +1,6 @@ "dataset_name": "medical_genetics" "description": "The following are questions (with answers) about medical\ \ genetics.\n\n" -"group": "mmlu_continuation_other" +"tag": "mmlu_continuation_other" "include": "_continuation_template_yaml" "task": "mmlu_continuation_medical_genetics" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_miscellaneous.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_miscellaneous.yaml index 95367c773c24ad3a2d389ae21245be628d46686e..f457800932ec2fba831a1d81e6ca4495816f981f 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_miscellaneous.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_miscellaneous.yaml @@ -1,6 +1,6 @@ "dataset_name": "miscellaneous" "description": "The following are questions (with answers) about miscellaneous.\n\ \n" -"group": "mmlu_continuation_other" +"tag": "mmlu_continuation_other" "include": "_continuation_template_yaml" "task": "mmlu_continuation_miscellaneous" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_moral_disputes.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_moral_disputes.yaml index 82780df7a1cc91dd840f1e0f3938e922d6f47969..0df1392d5baceb1a3dda1464acbb0b025a8428e8 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_moral_disputes.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_moral_disputes.yaml @@ -1,6 +1,6 @@ "dataset_name": "moral_disputes" "description": "The following are questions (with answers) about moral\ \ disputes.\n\n" -"group": "mmlu_continuation_humanities" +"tag": "mmlu_continuation_humanities" "include": "_continuation_template_yaml" "task": "mmlu_continuation_moral_disputes" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_moral_scenarios.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_moral_scenarios.yaml index d60487e5087281d6f69db06cff9c3d947f25557a..bea5e514b85a6ed83026a6fe9d399f92eb59ea99 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_moral_scenarios.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_moral_scenarios.yaml @@ -1,6 +1,6 @@ "dataset_name": "moral_scenarios" "description": "The following are questions (with answers) about moral\ \ scenarios.\n\n" -"group": "mmlu_continuation_humanities" +"tag": "mmlu_continuation_humanities" "include": "_continuation_template_yaml" "task": "mmlu_continuation_moral_scenarios" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_nutrition.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_nutrition.yaml index b502fcf74d468596587e1e72914f13a0260c4503..8db80340b2a9984cb8c3e41766e3f0e89af8f252 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_nutrition.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_nutrition.yaml @@ -1,6 +1,6 @@ "dataset_name": "nutrition" "description": "The following are questions (with answers) about nutrition.\n\ \n" -"group": "mmlu_continuation_other" +"tag": "mmlu_continuation_other" "include": "_continuation_template_yaml" "task": "mmlu_continuation_nutrition" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_philosophy.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_philosophy.yaml index e3cc3d45fde7907ca8ff281514f115b053f74bd2..165de6c90ba1d4756c39e2f5605226dbeb86e314 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_philosophy.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_philosophy.yaml @@ -1,6 +1,6 @@ "dataset_name": "philosophy" "description": "The following are questions (with answers) about philosophy.\n\ \n" -"group": "mmlu_continuation_humanities" +"tag": "mmlu_continuation_humanities" "include": "_continuation_template_yaml" "task": "mmlu_continuation_philosophy" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_prehistory.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_prehistory.yaml index 2c29ee75cd78e410c9cf2a9392a83dfe49078e87..02c4ee7f8af1856f498b7a55c83e085782e36666 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_prehistory.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_prehistory.yaml @@ -1,6 +1,6 @@ "dataset_name": "prehistory" "description": "The following are questions (with answers) about prehistory.\n\ \n" -"group": "mmlu_continuation_humanities" +"tag": "mmlu_continuation_humanities" "include": "_continuation_template_yaml" "task": "mmlu_continuation_prehistory" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_professional_accounting.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_professional_accounting.yaml index 588a4d431236cdf8abbe35c91f1843f20dd2a181..bb36a82b9c043b519379626f2d3618efdda9907b 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_professional_accounting.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_professional_accounting.yaml @@ -1,6 +1,6 @@ "dataset_name": "professional_accounting" "description": "The following are questions (with answers) about professional\ \ accounting.\n\n" -"group": "mmlu_continuation_other" +"tag": "mmlu_continuation_other" "include": "_continuation_template_yaml" "task": "mmlu_continuation_professional_accounting" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_professional_law.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_professional_law.yaml index ea8b35a0dc48cee447d72f3b67c7866b079029cb..ac9f2592f41a2bcae43da174d2eb969cf1805251 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_professional_law.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_professional_law.yaml @@ -1,6 +1,6 @@ "dataset_name": "professional_law" "description": "The following are questions (with answers) about professional\ \ law.\n\n" -"group": "mmlu_continuation_humanities" +"tag": "mmlu_continuation_humanities" "include": "_continuation_template_yaml" "task": "mmlu_continuation_professional_law" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_professional_medicine.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_professional_medicine.yaml index 73bdc706d831f01dbf23982ebbb1258e0958e332..328c128377609327abe0460e2d4ab6af716d02c3 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_professional_medicine.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_professional_medicine.yaml @@ -1,6 +1,6 @@ "dataset_name": "professional_medicine" "description": "The following are questions (with answers) about professional\ \ medicine.\n\n" -"group": "mmlu_continuation_other" +"tag": "mmlu_continuation_other" "include": "_continuation_template_yaml" "task": "mmlu_continuation_professional_medicine" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_professional_psychology.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_professional_psychology.yaml index dab85998051d6b98424a391e921b983146539643..0cca5bde048a23367aa2ccebc893e9fa71996d98 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_professional_psychology.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_professional_psychology.yaml @@ -1,6 +1,6 @@ "dataset_name": "professional_psychology" "description": "The following are questions (with answers) about professional\ \ psychology.\n\n" -"group": "mmlu_continuation_social_sciences" +"tag": "mmlu_continuation_social_sciences" "include": "_continuation_template_yaml" "task": "mmlu_continuation_professional_psychology" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_public_relations.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_public_relations.yaml index d4cffea20bf7082596f3de570068432f7bbbc817..700c407c2377d8d4d83bbf88d8f7a003a2e2900d 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_public_relations.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_public_relations.yaml @@ -1,6 +1,6 @@ "dataset_name": "public_relations" "description": "The following are questions (with answers) about public\ \ relations.\n\n" -"group": "mmlu_continuation_social_sciences" +"tag": "mmlu_continuation_social_sciences" "include": "_continuation_template_yaml" "task": "mmlu_continuation_public_relations" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_security_studies.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_security_studies.yaml index df0adc7834b2ac2198b7d38f7d256d63583dad60..4f5ef99e0f8fe8c98bc9994757d9cc6617e3550e 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_security_studies.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_security_studies.yaml @@ -1,6 +1,6 @@ "dataset_name": "security_studies" "description": "The following are questions (with answers) about security\ \ studies.\n\n" -"group": "mmlu_continuation_social_sciences" +"tag": "mmlu_continuation_social_sciences" "include": "_continuation_template_yaml" "task": "mmlu_continuation_security_studies" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_sociology.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_sociology.yaml index 5da671e1dee80168ac3bdf2dcc3efb2465f479e8..e78621aaa547b419f4133b94ce8dcba00c407f5c 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_sociology.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_sociology.yaml @@ -1,6 +1,6 @@ "dataset_name": "sociology" "description": "The following are questions (with answers) about sociology.\n\ \n" -"group": "mmlu_continuation_social_sciences" +"tag": "mmlu_continuation_social_sciences" "include": "_continuation_template_yaml" "task": "mmlu_continuation_sociology" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_us_foreign_policy.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_us_foreign_policy.yaml index a426a5ec8464b5bfe838962ad8b9273d6e23fda1..989bb29aa095e83c2744011775864ef27258ca28 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_us_foreign_policy.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_us_foreign_policy.yaml @@ -1,6 +1,6 @@ "dataset_name": "us_foreign_policy" "description": "The following are questions (with answers) about us\ \ foreign policy.\n\n" -"group": "mmlu_continuation_social_sciences" +"tag": "mmlu_continuation_social_sciences" "include": "_continuation_template_yaml" "task": "mmlu_continuation_us_foreign_policy" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_virology.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_virology.yaml index 589abc864ba9814066532617e42debbc3b811406..5c938190bdd755f411914905d5309daa6938f313 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_virology.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_virology.yaml @@ -1,6 +1,6 @@ "dataset_name": "virology" "description": "The following are questions (with answers) about virology.\n\ \n" -"group": "mmlu_continuation_other" +"tag": "mmlu_continuation_other" "include": "_continuation_template_yaml" "task": "mmlu_continuation_virology" diff --git a/lm_eval/tasks/mmlu/continuation/mmlu_world_religions.yaml b/lm_eval/tasks/mmlu/continuation/mmlu_world_religions.yaml index 532442ef0dde542a39d1c490681956bee7adbc6f..f707670066d3f2db4554221a12a3983e2d8febf5 100644 --- a/lm_eval/tasks/mmlu/continuation/mmlu_world_religions.yaml +++ b/lm_eval/tasks/mmlu/continuation/mmlu_world_religions.yaml @@ -1,6 +1,6 @@ "dataset_name": "world_religions" "description": "The following are questions (with answers) about world\ \ religions.\n\n" -"group": "mmlu_continuation_humanities" +"tag": "mmlu_continuation_humanities" "include": "_continuation_template_yaml" "task": "mmlu_continuation_world_religions" diff --git a/lm_eval/tasks/mmlu/default/_default_template_yaml b/lm_eval/tasks/mmlu/default/_default_template_yaml index 0f22d02c86f57898658a5fff72484766777be8bc..ed0e70536b94b9d2127c2e02999d34cd6d0c3943 100644 --- a/lm_eval/tasks/mmlu/default/_default_template_yaml +++ b/lm_eval/tasks/mmlu/default/_default_template_yaml @@ -12,6 +12,6 @@ metric_list: aggregation: mean higher_is_better: true metadata: - version: 0.0 + version: 1.0 dataset_kwargs: trust_remote_code: true diff --git a/lm_eval/tasks/mmlu/default/_mmlu.yaml b/lm_eval/tasks/mmlu/default/_mmlu.yaml index 584de02993acedc37b6d508967334f9a55675f6b..55099c6f16febd89270ad022abe181bf8ccd708e 100644 --- a/lm_eval/tasks/mmlu/default/_mmlu.yaml +++ b/lm_eval/tasks/mmlu/default/_mmlu.yaml @@ -4,3 +4,8 @@ task: - mmlu_other - mmlu_social_sciences - mmlu_humanities +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 2 diff --git a/lm_eval/tasks/mmlu/default/_mmlu_humanities.yaml b/lm_eval/tasks/mmlu/default/_mmlu_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7156e2230f09b461b8e783db323b9ee2d8023192 --- /dev/null +++ b/lm_eval/tasks/mmlu/default/_mmlu_humanities.yaml @@ -0,0 +1,9 @@ +group: mmlu_humanities +group_alias: humanities +task: + - mmlu_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 2 diff --git a/lm_eval/tasks/mmlu/default/_mmlu_other.yaml b/lm_eval/tasks/mmlu/default/_mmlu_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..79025cec0c639a37872287ecb5ae5c444dce7478 --- /dev/null +++ b/lm_eval/tasks/mmlu/default/_mmlu_other.yaml @@ -0,0 +1,9 @@ +group: mmlu_other +group_alias: other +task: + - mmlu_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 2 diff --git a/lm_eval/tasks/mmlu/default/_mmlu_social_sciences.yaml b/lm_eval/tasks/mmlu/default/_mmlu_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fab1ec2c1416bc644c8723bdb18905dff9c00040 --- /dev/null +++ b/lm_eval/tasks/mmlu/default/_mmlu_social_sciences.yaml @@ -0,0 +1,9 @@ +group: mmlu_social_sciences +group_alias: social sciences +task: + - mmlu_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 2 diff --git a/lm_eval/tasks/mmlu/default/_mmlu_stem.yaml b/lm_eval/tasks/mmlu/default/_mmlu_stem.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cda82eff10a03afe1a05fd8a1368cf3a7c63dcd8 --- /dev/null +++ b/lm_eval/tasks/mmlu/default/_mmlu_stem.yaml @@ -0,0 +1,9 @@ +group: mmlu_stem +group_alias: stem +task: + - mmlu_stem_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 2 diff --git a/lm_eval/tasks/mmlu/default/mmlu_abstract_algebra.yaml b/lm_eval/tasks/mmlu/default/mmlu_abstract_algebra.yaml index 90f3cc50b3f8e54b584237edfa50fcdc96d0e625..dcde12cb4c5566567482e095c87860f1c6179473 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_abstract_algebra.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_abstract_algebra.yaml @@ -1,8 +1,7 @@ "dataset_name": "abstract_algebra" "description": "The following are multiple choice questions (with answers) about abstract\ \ algebra.\n\n" -"group": "mmlu_stem" -"group_alias": "stem" +"tag": "mmlu_stem_tasks" "include": "_default_template_yaml" "task": "mmlu_abstract_algebra" "task_alias": "abstract_algebra" diff --git a/lm_eval/tasks/mmlu/default/mmlu_anatomy.yaml b/lm_eval/tasks/mmlu/default/mmlu_anatomy.yaml index 0e9e09b27fa40e222d6284c58826a089391e1bb4..5fef7490dd31872f2ed9dcde5c1e817e910b5e39 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_anatomy.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_anatomy.yaml @@ -1,8 +1,7 @@ "dataset_name": "anatomy" "description": "The following are multiple choice questions (with answers) about anatomy.\n\ \n" -"group": "mmlu_stem" -"group_alias": "stem" +"tag": "mmlu_stem_tasks" "include": "_default_template_yaml" "task": "mmlu_anatomy" "task_alias": "anatomy" diff --git a/lm_eval/tasks/mmlu/default/mmlu_astronomy.yaml b/lm_eval/tasks/mmlu/default/mmlu_astronomy.yaml index e3bdfc9521f3b98bf684a1c022f69e3c14c9d406..660f07476dfdd115fc0b8d5f04c685b23857cc33 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_astronomy.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_astronomy.yaml @@ -1,8 +1,7 @@ "dataset_name": "astronomy" "description": "The following are multiple choice questions (with answers) about astronomy.\n\ \n" -"group": "mmlu_stem" -"group_alias": "stem" +"tag": "mmlu_stem_tasks" "include": "_default_template_yaml" "task": "mmlu_astronomy" "task_alias": "astronomy" diff --git a/lm_eval/tasks/mmlu/default/mmlu_business_ethics.yaml b/lm_eval/tasks/mmlu/default/mmlu_business_ethics.yaml index ea0d1fe2a68cb893759f3c550775ff3554f4fc13..a0f1b1c2dcd802effdf589d4f85b412593dfb622 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_business_ethics.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_business_ethics.yaml @@ -1,8 +1,7 @@ "dataset_name": "business_ethics" "description": "The following are multiple choice questions (with answers) about business\ \ ethics.\n\n" -"group": "mmlu_other" -"group_alias": "other" +"tag": "mmlu_other_tasks" "include": "_default_template_yaml" "task": "mmlu_business_ethics" "task_alias": "business_ethics" diff --git a/lm_eval/tasks/mmlu/default/mmlu_clinical_knowledge.yaml b/lm_eval/tasks/mmlu/default/mmlu_clinical_knowledge.yaml index 20bab1470e8b58b9e56652edf8128a4e7959bd33..1c72b71648df5a690963c95180a76f7ad0a495d4 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_clinical_knowledge.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_clinical_knowledge.yaml @@ -1,8 +1,7 @@ "dataset_name": "clinical_knowledge" "description": "The following are multiple choice questions (with answers) about clinical\ \ knowledge.\n\n" -"group": "mmlu_other" -"group_alias": "other" +"tag": "mmlu_other_tasks" "include": "_default_template_yaml" "task": "mmlu_clinical_knowledge" "task_alias": "clinical_knowledge" diff --git a/lm_eval/tasks/mmlu/default/mmlu_college_biology.yaml b/lm_eval/tasks/mmlu/default/mmlu_college_biology.yaml index afb4d9c6c0d209644806ca8376a25cf28d6fa2ed..ddfd713aa0581b36fdad44da4f80e5b500c47154 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_college_biology.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_college_biology.yaml @@ -1,8 +1,7 @@ "dataset_name": "college_biology" "description": "The following are multiple choice questions (with answers) about college\ \ biology.\n\n" -"group": "mmlu_stem" -"group_alias": "stem" +"tag": "mmlu_stem_tasks" "include": "_default_template_yaml" "task": "mmlu_college_biology" "task_alias": "college_biology" diff --git a/lm_eval/tasks/mmlu/default/mmlu_college_chemistry.yaml b/lm_eval/tasks/mmlu/default/mmlu_college_chemistry.yaml index a7de3532359a418dc0d1ba8f9096700e2a888907..388c3a91bed8ffb7645e0e7f23fb0a81117503cc 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_college_chemistry.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_college_chemistry.yaml @@ -1,8 +1,7 @@ "dataset_name": "college_chemistry" "description": "The following are multiple choice questions (with answers) about college\ \ chemistry.\n\n" -"group": "mmlu_stem" -"group_alias": "stem" +"tag": "mmlu_stem_tasks" "include": "_default_template_yaml" "task": "mmlu_college_chemistry" "task_alias": "college_chemistry" diff --git a/lm_eval/tasks/mmlu/default/mmlu_college_computer_science.yaml b/lm_eval/tasks/mmlu/default/mmlu_college_computer_science.yaml index 9786cc6ebd0d14936bb0138d322161b01a4574dd..a3f692423abfbf036fc0347fdfbb2642a6d16c39 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_college_computer_science.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_college_computer_science.yaml @@ -1,8 +1,7 @@ "dataset_name": "college_computer_science" "description": "The following are multiple choice questions (with answers) about college\ \ computer science.\n\n" -"group": "mmlu_stem" -"group_alias": "stem" +"tag": "mmlu_stem_tasks" "include": "_default_template_yaml" "task": "mmlu_college_computer_science" "task_alias": "college_computer_science" diff --git a/lm_eval/tasks/mmlu/default/mmlu_college_mathematics.yaml b/lm_eval/tasks/mmlu/default/mmlu_college_mathematics.yaml index e7699f8bfcedffdcc83984f96c6d337f41135b54..08a9628af175edb897c7f6d88b96d4969fccad29 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_college_mathematics.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_college_mathematics.yaml @@ -1,8 +1,7 @@ "dataset_name": "college_mathematics" "description": "The following are multiple choice questions (with answers) about college\ \ mathematics.\n\n" -"group": "mmlu_stem" -"group_alias": "stem" +"tag": "mmlu_stem_tasks" "include": "_default_template_yaml" "task": "mmlu_college_mathematics" "task_alias": "college_mathematics" diff --git a/lm_eval/tasks/mmlu/default/mmlu_college_medicine.yaml b/lm_eval/tasks/mmlu/default/mmlu_college_medicine.yaml index df9e890136328d4caf3ca3c4715919358dcf92dc..35197a2a1885f7daf30209d4309dd059243260a8 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_college_medicine.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_college_medicine.yaml @@ -1,8 +1,7 @@ "dataset_name": "college_medicine" "description": "The following are multiple choice questions (with answers) about college\ \ medicine.\n\n" -"group": "mmlu_other" -"group_alias": "other" +"tag": "mmlu_other_tasks" "include": "_default_template_yaml" "task": "mmlu_college_medicine" "task_alias": "college_medicine" diff --git a/lm_eval/tasks/mmlu/default/mmlu_college_physics.yaml b/lm_eval/tasks/mmlu/default/mmlu_college_physics.yaml index 3c5e7462e3fefe122e9df4bb5ff9458f8ae9dfc8..9b5017afac65e0acf080a9df84098a1f21681833 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_college_physics.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_college_physics.yaml @@ -1,8 +1,7 @@ "dataset_name": "college_physics" "description": "The following are multiple choice questions (with answers) about college\ \ physics.\n\n" -"group": "mmlu_stem" -"group_alias": "stem" +"tag": "mmlu_stem_tasks" "include": "_default_template_yaml" "task": "mmlu_college_physics" "task_alias": "college_physics" diff --git a/lm_eval/tasks/mmlu/default/mmlu_computer_security.yaml b/lm_eval/tasks/mmlu/default/mmlu_computer_security.yaml index df9c4a51b6ca2715b921f2a33ef4755ebfacbc46..8f9b42820f7f7196c6d02922337eaedb7ede5388 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_computer_security.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_computer_security.yaml @@ -1,8 +1,7 @@ "dataset_name": "computer_security" "description": "The following are multiple choice questions (with answers) about computer\ \ security.\n\n" -"group": "mmlu_stem" -"group_alias": "stem" +"tag": "mmlu_stem_tasks" "include": "_default_template_yaml" "task": "mmlu_computer_security" "task_alias": "computer_security" diff --git a/lm_eval/tasks/mmlu/default/mmlu_conceptual_physics.yaml b/lm_eval/tasks/mmlu/default/mmlu_conceptual_physics.yaml index 8ab59ed10f1cf02f47eacf7930a9efcbfc230fcf..af61a7e1579ac8613b5535e15a57adc629e2d571 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_conceptual_physics.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_conceptual_physics.yaml @@ -1,8 +1,7 @@ "dataset_name": "conceptual_physics" "description": "The following are multiple choice questions (with answers) about conceptual\ \ physics.\n\n" -"group": "mmlu_stem" -"group_alias": "stem" +"tag": "mmlu_stem_tasks" "include": "_default_template_yaml" "task": "mmlu_conceptual_physics" "task_alias": "conceptual_physics" diff --git a/lm_eval/tasks/mmlu/default/mmlu_econometrics.yaml b/lm_eval/tasks/mmlu/default/mmlu_econometrics.yaml index a974fc8462d649c27ac09192d3cf84d9b162f82d..609c20af2acbdd7ef36104dc97db97a40bfca6a5 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_econometrics.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_econometrics.yaml @@ -1,8 +1,7 @@ "dataset_name": "econometrics" "description": "The following are multiple choice questions (with answers) about econometrics.\n\ \n" -"group": "mmlu_social_sciences" -"group_alias": "social_sciences" +"tag": "mmlu_social_sciences_tasks" "include": "_default_template_yaml" "task": "mmlu_econometrics" "task_alias": "econometrics" diff --git a/lm_eval/tasks/mmlu/default/mmlu_electrical_engineering.yaml b/lm_eval/tasks/mmlu/default/mmlu_electrical_engineering.yaml index 9c45cc6109d2b53322b2303efcb1f57e99d8d60c..8fa2137ad05a14e32d5d7e8973d6bc9c18d1a555 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_electrical_engineering.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_electrical_engineering.yaml @@ -1,8 +1,7 @@ "dataset_name": "electrical_engineering" "description": "The following are multiple choice questions (with answers) about electrical\ \ engineering.\n\n" -"group": "mmlu_stem" -"group_alias": "stem" +"tag": "mmlu_stem_tasks" "include": "_default_template_yaml" "task": "mmlu_electrical_engineering" "task_alias": "electrical_engineering" diff --git a/lm_eval/tasks/mmlu/default/mmlu_elementary_mathematics.yaml b/lm_eval/tasks/mmlu/default/mmlu_elementary_mathematics.yaml index 2154ab65454c9234b57d76782315c6559b0e7e0f..d15f6d5ae88b6edf0bba2298ffaacbd4d103aedd 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_elementary_mathematics.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_elementary_mathematics.yaml @@ -1,8 +1,7 @@ "dataset_name": "elementary_mathematics" "description": "The following are multiple choice questions (with answers) about elementary\ \ mathematics.\n\n" -"group": "mmlu_stem" -"group_alias": "stem" +"tag": "mmlu_stem_tasks" "include": "_default_template_yaml" "task": "mmlu_elementary_mathematics" "task_alias": "elementary_mathematics" diff --git a/lm_eval/tasks/mmlu/default/mmlu_formal_logic.yaml b/lm_eval/tasks/mmlu/default/mmlu_formal_logic.yaml index 689d3d1f87503953be03cd54973d240b64f846f4..ee2fc2f61073dc11f6f745eaf8927ab70aadad3f 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_formal_logic.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_formal_logic.yaml @@ -1,8 +1,7 @@ "dataset_name": "formal_logic" "description": "The following are multiple choice questions (with answers) about formal\ \ logic.\n\n" -"group": "mmlu_humanities" -"group_alias": "humanities" +"tag": "mmlu_humanities_tasks" "include": "_default_template_yaml" "task": "mmlu_formal_logic" "task_alias": "formal_logic" diff --git a/lm_eval/tasks/mmlu/default/mmlu_global_facts.yaml b/lm_eval/tasks/mmlu/default/mmlu_global_facts.yaml index 60b5c12921ee2ad1e2200392ecc3ffaef44f4105..b27ddefd25be9c6695900ce6d290a811b68356df 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_global_facts.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_global_facts.yaml @@ -1,8 +1,7 @@ "dataset_name": "global_facts" "description": "The following are multiple choice questions (with answers) about global\ \ facts.\n\n" -"group": "mmlu_other" -"group_alias": "other" +"tag": "mmlu_other_tasks" "include": "_default_template_yaml" "task": "mmlu_global_facts" "task_alias": "global_facts" diff --git a/lm_eval/tasks/mmlu/default/mmlu_high_school_biology.yaml b/lm_eval/tasks/mmlu/default/mmlu_high_school_biology.yaml index c7e055dcc8dcfffb2572a8f4ab1770479e84daeb..22bc47943f0f66614f79cd0de5e7614afa1f08d5 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_high_school_biology.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_high_school_biology.yaml @@ -1,8 +1,7 @@ "dataset_name": "high_school_biology" "description": "The following are multiple choice questions (with answers) about high\ \ school biology.\n\n" -"group": "mmlu_stem" -"group_alias": "stem" +"tag": "mmlu_stem_tasks" "include": "_default_template_yaml" "task": "mmlu_high_school_biology" "task_alias": "high_school_biology" diff --git a/lm_eval/tasks/mmlu/default/mmlu_high_school_chemistry.yaml b/lm_eval/tasks/mmlu/default/mmlu_high_school_chemistry.yaml index 8e9421c1a6f7271ddfc87ad099a914f81bc031e9..5a25617cbd821411e6f0ca9fac853c76b7adb319 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_high_school_chemistry.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_high_school_chemistry.yaml @@ -1,8 +1,7 @@ "dataset_name": "high_school_chemistry" "description": "The following are multiple choice questions (with answers) about high\ \ school chemistry.\n\n" -"group": "mmlu_stem" -"group_alias": "stem" +"tag": "mmlu_stem_tasks" "include": "_default_template_yaml" "task": "mmlu_high_school_chemistry" "task_alias": "high_school_chemistry" diff --git a/lm_eval/tasks/mmlu/default/mmlu_high_school_computer_science.yaml b/lm_eval/tasks/mmlu/default/mmlu_high_school_computer_science.yaml index 87ec15cc3ad2bcd2d9d1d802389cb9e3d26e5abf..ad4c7d312c7e8f6517d308e6ffeb635a354b843e 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_high_school_computer_science.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_high_school_computer_science.yaml @@ -1,8 +1,7 @@ "dataset_name": "high_school_computer_science" "description": "The following are multiple choice questions (with answers) about high\ \ school computer science.\n\n" -"group": "mmlu_stem" -"group_alias": "stem" +"tag": "mmlu_stem_tasks" "include": "_default_template_yaml" "task": "mmlu_high_school_computer_science" "task_alias": "high_school_computer_science" diff --git a/lm_eval/tasks/mmlu/default/mmlu_high_school_european_history.yaml b/lm_eval/tasks/mmlu/default/mmlu_high_school_european_history.yaml index be0d696a962c38a9ffb5404ea75cb2d402f10ec0..7c51bbdd7aa87b39da8145f8ea45f6fe13d17623 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_high_school_european_history.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_high_school_european_history.yaml @@ -1,8 +1,7 @@ "dataset_name": "high_school_european_history" "description": "The following are multiple choice questions (with answers) about high\ \ school european history.\n\n" -"group": "mmlu_humanities" -"group_alias": "humanities" +"tag": "mmlu_humanities_tasks" "include": "_default_template_yaml" "task": "mmlu_high_school_european_history" "task_alias": "high_school_european_history" diff --git a/lm_eval/tasks/mmlu/default/mmlu_high_school_geography.yaml b/lm_eval/tasks/mmlu/default/mmlu_high_school_geography.yaml index 57c5261a8be0e498111d834b8787341104f24775..aad87f1ad57a48102d7807a7a3fd75af86755912 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_high_school_geography.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_high_school_geography.yaml @@ -1,8 +1,7 @@ "dataset_name": "high_school_geography" "description": "The following are multiple choice questions (with answers) about high\ \ school geography.\n\n" -"group": "mmlu_social_sciences" -"group_alias": "social_sciences" +"tag": "mmlu_social_sciences_tasks" "include": "_default_template_yaml" "task": "mmlu_high_school_geography" "task_alias": "high_school_geography" diff --git a/lm_eval/tasks/mmlu/default/mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/mmlu/default/mmlu_high_school_government_and_politics.yaml index 2e92f152081e102b07feb394038aa0b30ca827a2..2b93b363d658357619eaf907f8d04af339c22a12 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_high_school_government_and_politics.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_high_school_government_and_politics.yaml @@ -1,8 +1,7 @@ "dataset_name": "high_school_government_and_politics" "description": "The following are multiple choice questions (with answers) about high\ \ school government and politics.\n\n" -"group": "mmlu_social_sciences" -"group_alias": "social_sciences" +"tag": "mmlu_social_sciences_tasks" "include": "_default_template_yaml" "task": "mmlu_high_school_government_and_politics" "task_alias": "high_school_government_and_politics" diff --git a/lm_eval/tasks/mmlu/default/mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/mmlu/default/mmlu_high_school_macroeconomics.yaml index 988d132aad3309d086e8f8549bd6710f2bd2b817..a08c579d1480ab592917d5a6673e63cf09198417 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_high_school_macroeconomics.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_high_school_macroeconomics.yaml @@ -1,8 +1,7 @@ "dataset_name": "high_school_macroeconomics" "description": "The following are multiple choice questions (with answers) about high\ \ school macroeconomics.\n\n" -"group": "mmlu_social_sciences" -"group_alias": "social_sciences" +"tag": "mmlu_social_sciences_tasks" "include": "_default_template_yaml" "task": "mmlu_high_school_macroeconomics" "task_alias": "high_school_macroeconomics" diff --git a/lm_eval/tasks/mmlu/default/mmlu_high_school_mathematics.yaml b/lm_eval/tasks/mmlu/default/mmlu_high_school_mathematics.yaml index f7c07a608592709c08d4e2130e9f99abd5058880..3f1b6d70e022414b7d370635daa49e3a9a8649c2 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_high_school_mathematics.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_high_school_mathematics.yaml @@ -1,8 +1,7 @@ "dataset_name": "high_school_mathematics" "description": "The following are multiple choice questions (with answers) about high\ \ school mathematics.\n\n" -"group": "mmlu_stem" -"group_alias": "stem" +"tag": "mmlu_stem_tasks" "include": "_default_template_yaml" "task": "mmlu_high_school_mathematics" "task_alias": "high_school_mathematics" diff --git a/lm_eval/tasks/mmlu/default/mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/mmlu/default/mmlu_high_school_microeconomics.yaml index 5339a0235cd8e27e51c7292a7cd2201c46cbc2f1..ac4f65dad5783bf23c50d9a39e912fe797a047e6 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_high_school_microeconomics.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_high_school_microeconomics.yaml @@ -1,8 +1,7 @@ "dataset_name": "high_school_microeconomics" "description": "The following are multiple choice questions (with answers) about high\ \ school microeconomics.\n\n" -"group": "mmlu_social_sciences" -"group_alias": "social_sciences" +"tag": "mmlu_social_sciences_tasks" "include": "_default_template_yaml" "task": "mmlu_high_school_microeconomics" "task_alias": "high_school_microeconomics" diff --git a/lm_eval/tasks/mmlu/default/mmlu_high_school_physics.yaml b/lm_eval/tasks/mmlu/default/mmlu_high_school_physics.yaml index 0fae0405aa093a0fb2eaea8fa8a48d0e4e220c9a..b8c449aa1b5bb48d6899c328c82c44ee3ae3ef24 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_high_school_physics.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_high_school_physics.yaml @@ -1,8 +1,7 @@ "dataset_name": "high_school_physics" "description": "The following are multiple choice questions (with answers) about high\ \ school physics.\n\n" -"group": "mmlu_stem" -"group_alias": "stem" +"tag": "mmlu_stem_tasks" "include": "_default_template_yaml" "task": "mmlu_high_school_physics" "task_alias": "high_school_physics" diff --git a/lm_eval/tasks/mmlu/default/mmlu_high_school_psychology.yaml b/lm_eval/tasks/mmlu/default/mmlu_high_school_psychology.yaml index 31ecb18eb91397d2bad070d11d141f3375c66eaf..47ba836c71b2be9759bd9fe48dd0cb687ef08636 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_high_school_psychology.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_high_school_psychology.yaml @@ -1,8 +1,7 @@ "dataset_name": "high_school_psychology" "description": "The following are multiple choice questions (with answers) about high\ \ school psychology.\n\n" -"group": "mmlu_social_sciences" -"group_alias": "social_sciences" +"tag": "mmlu_social_sciences_tasks" "include": "_default_template_yaml" "task": "mmlu_high_school_psychology" "task_alias": "high_school_psychology" diff --git a/lm_eval/tasks/mmlu/default/mmlu_high_school_statistics.yaml b/lm_eval/tasks/mmlu/default/mmlu_high_school_statistics.yaml index 54d70880eff3e5aa6ab8f8b233ee45b9f30ca25c..ef5bdd7cf1577a7ba9f3365643c5e56b21c8a77e 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_high_school_statistics.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_high_school_statistics.yaml @@ -1,8 +1,7 @@ "dataset_name": "high_school_statistics" "description": "The following are multiple choice questions (with answers) about high\ \ school statistics.\n\n" -"group": "mmlu_stem" -"group_alias": "stem" +"tag": "mmlu_stem_tasks" "include": "_default_template_yaml" "task": "mmlu_high_school_statistics" "task_alias": "high_school_statistics" diff --git a/lm_eval/tasks/mmlu/default/mmlu_high_school_us_history.yaml b/lm_eval/tasks/mmlu/default/mmlu_high_school_us_history.yaml index e4432fe4459fcaa3c1ccf311c094ceb9122d0637..ececdb0ab921bdc24b8aac41979a93d35670d0c6 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_high_school_us_history.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_high_school_us_history.yaml @@ -1,8 +1,7 @@ "dataset_name": "high_school_us_history" "description": "The following are multiple choice questions (with answers) about high\ \ school us history.\n\n" -"group": "mmlu_humanities" -"group_alias": "humanities" +"tag": "mmlu_humanities_tasks" "include": "_default_template_yaml" "task": "mmlu_high_school_us_history" "task_alias": "high_school_us_history" diff --git a/lm_eval/tasks/mmlu/default/mmlu_high_school_world_history.yaml b/lm_eval/tasks/mmlu/default/mmlu_high_school_world_history.yaml index 08773a20089a5551710cb82447d6ce23d65c367d..af34c8ddbe51abc0f44baff2bf8087b4c749825f 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_high_school_world_history.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_high_school_world_history.yaml @@ -1,8 +1,7 @@ "dataset_name": "high_school_world_history" "description": "The following are multiple choice questions (with answers) about high\ \ school world history.\n\n" -"group": "mmlu_humanities" -"group_alias": "humanities" +"tag": "mmlu_humanities_tasks" "include": "_default_template_yaml" "task": "mmlu_high_school_world_history" "task_alias": "high_school_world_history" diff --git a/lm_eval/tasks/mmlu/default/mmlu_human_aging.yaml b/lm_eval/tasks/mmlu/default/mmlu_human_aging.yaml index c9e1feb154293d0a75b25837f365ec892756a465..3ca720be7c7d757c579e4563cb805dc36a6dcc6d 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_human_aging.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_human_aging.yaml @@ -1,8 +1,7 @@ "dataset_name": "human_aging" "description": "The following are multiple choice questions (with answers) about human\ \ aging.\n\n" -"group": "mmlu_other" -"group_alias": "other" +"tag": "mmlu_other_tasks" "include": "_default_template_yaml" "task": "mmlu_human_aging" "task_alias": "human_aging" diff --git a/lm_eval/tasks/mmlu/default/mmlu_human_sexuality.yaml b/lm_eval/tasks/mmlu/default/mmlu_human_sexuality.yaml index 715859a1e8ccd52ac406b9f9dcaf4dc80dbaa42f..2acddd1e4ec1d85a7475202d43f5917abb085684 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_human_sexuality.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_human_sexuality.yaml @@ -1,8 +1,7 @@ "dataset_name": "human_sexuality" "description": "The following are multiple choice questions (with answers) about human\ \ sexuality.\n\n" -"group": "mmlu_social_sciences" -"group_alias": "social_sciences" +"tag": "mmlu_social_sciences_tasks" "include": "_default_template_yaml" "task": "mmlu_human_sexuality" "task_alias": "human_sexuality" diff --git a/lm_eval/tasks/mmlu/default/mmlu_international_law.yaml b/lm_eval/tasks/mmlu/default/mmlu_international_law.yaml index 68765225bbee355a00a2d0d3e9a8931624fc45dd..9fb2a162aab92931f8b560ce0e76155fbc9bb675 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_international_law.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_international_law.yaml @@ -1,8 +1,7 @@ "dataset_name": "international_law" "description": "The following are multiple choice questions (with answers) about international\ \ law.\n\n" -"group": "mmlu_humanities" -"group_alias": "humanities" +"tag": "mmlu_humanities_tasks" "include": "_default_template_yaml" "task": "mmlu_international_law" "task_alias": "international_law" diff --git a/lm_eval/tasks/mmlu/default/mmlu_jurisprudence.yaml b/lm_eval/tasks/mmlu/default/mmlu_jurisprudence.yaml index e16de5c40b2a29cfc0cadeaad5ed11df9b7503e8..3c62a911ff5d849651d8c9e09feb34847846d147 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_jurisprudence.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_jurisprudence.yaml @@ -1,8 +1,7 @@ "dataset_name": "jurisprudence" "description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ \n" -"group": "mmlu_humanities" -"group_alias": "humanities" +"tag": "mmlu_humanities_tasks" "include": "_default_template_yaml" "task": "mmlu_jurisprudence" "task_alias": "jurisprudence" diff --git a/lm_eval/tasks/mmlu/default/mmlu_logical_fallacies.yaml b/lm_eval/tasks/mmlu/default/mmlu_logical_fallacies.yaml index 8b12057bc78ffc195b4d016bfdc1c931f08e6267..adf8821e9a8ac9d80f1cfb5c6af5b74a63efda27 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_logical_fallacies.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_logical_fallacies.yaml @@ -1,8 +1,7 @@ "dataset_name": "logical_fallacies" "description": "The following are multiple choice questions (with answers) about logical\ \ fallacies.\n\n" -"group": "mmlu_humanities" -"group_alias": "humanities" +"tag": "mmlu_humanities_tasks" "include": "_default_template_yaml" "task": "mmlu_logical_fallacies" "task_alias": "logical_fallacies" diff --git a/lm_eval/tasks/mmlu/default/mmlu_machine_learning.yaml b/lm_eval/tasks/mmlu/default/mmlu_machine_learning.yaml index 2387d680b2d57d2d409ecc1bf744f512dd4fcef3..d846f96084a8cba059348a90d800a86b92ba09c2 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_machine_learning.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_machine_learning.yaml @@ -1,8 +1,7 @@ "dataset_name": "machine_learning" "description": "The following are multiple choice questions (with answers) about machine\ \ learning.\n\n" -"group": "mmlu_stem" -"group_alias": "stem" +"tag": "mmlu_stem_tasks" "include": "_default_template_yaml" "task": "mmlu_machine_learning" "task_alias": "machine_learning" diff --git a/lm_eval/tasks/mmlu/default/mmlu_management.yaml b/lm_eval/tasks/mmlu/default/mmlu_management.yaml index d0cdc812d7e94eb51a9f1d3148b3d51253c1c92a..7dff834ef804039858b6955155a8338dd11b30b3 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_management.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_management.yaml @@ -1,8 +1,7 @@ "dataset_name": "management" "description": "The following are multiple choice questions (with answers) about management.\n\ \n" -"group": "mmlu_other" -"group_alias": "other" +"tag": "mmlu_other_tasks" "include": "_default_template_yaml" "task": "mmlu_management" "task_alias": "management" diff --git a/lm_eval/tasks/mmlu/default/mmlu_marketing.yaml b/lm_eval/tasks/mmlu/default/mmlu_marketing.yaml index a614db2977421ad88dcf4cd963647534f0d23b88..4ef004988965c41ff075f2f976b98dca4657ca04 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_marketing.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_marketing.yaml @@ -1,8 +1,7 @@ "dataset_name": "marketing" "description": "The following are multiple choice questions (with answers) about marketing.\n\ \n" -"group": "mmlu_other" -"group_alias": "other" +"tag": "mmlu_other_tasks" "include": "_default_template_yaml" "task": "mmlu_marketing" "task_alias": "marketing" diff --git a/lm_eval/tasks/mmlu/default/mmlu_medical_genetics.yaml b/lm_eval/tasks/mmlu/default/mmlu_medical_genetics.yaml index 5d7ce7083bc62f5859988fb76f84c4dc102175e8..989fb2c1aea91035421e49c7a11293c48ffec0bc 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_medical_genetics.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_medical_genetics.yaml @@ -1,8 +1,7 @@ "dataset_name": "medical_genetics" "description": "The following are multiple choice questions (with answers) about medical\ \ genetics.\n\n" -"group": "mmlu_other" -"group_alias": "other" +"tag": "mmlu_other_tasks" "include": "_default_template_yaml" "task": "mmlu_medical_genetics" "task_alias": "medical_genetics" diff --git a/lm_eval/tasks/mmlu/default/mmlu_miscellaneous.yaml b/lm_eval/tasks/mmlu/default/mmlu_miscellaneous.yaml index 77e819cf831fb69381d75a1c039ef9ffaee3028f..e7bb68bc2eb0f55b784943bd18296aabe3b86a31 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_miscellaneous.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_miscellaneous.yaml @@ -1,8 +1,7 @@ "dataset_name": "miscellaneous" "description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ \n" -"group": "mmlu_other" -"group_alias": "other" +"tag": "mmlu_other_tasks" "include": "_default_template_yaml" "task": "mmlu_miscellaneous" "task_alias": "miscellaneous" diff --git a/lm_eval/tasks/mmlu/default/mmlu_moral_disputes.yaml b/lm_eval/tasks/mmlu/default/mmlu_moral_disputes.yaml index 2df1a1ddb6302f7e563852fdd56cdb0d1668dad1..348d21403f06669e198146286b83e227fbde5a16 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_moral_disputes.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_moral_disputes.yaml @@ -1,8 +1,7 @@ "dataset_name": "moral_disputes" "description": "The following are multiple choice questions (with answers) about moral\ \ disputes.\n\n" -"group": "mmlu_humanities" -"group_alias": "humanities" +"tag": "mmlu_humanities_tasks" "include": "_default_template_yaml" "task": "mmlu_moral_disputes" "task_alias": "moral_disputes" diff --git a/lm_eval/tasks/mmlu/default/mmlu_moral_scenarios.yaml b/lm_eval/tasks/mmlu/default/mmlu_moral_scenarios.yaml index 6da63cb24847aeaf580fc54f701326a187836517..3762ee1200848439f08a3c69703af4cffb3a9d74 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_moral_scenarios.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_moral_scenarios.yaml @@ -1,8 +1,7 @@ "dataset_name": "moral_scenarios" "description": "The following are multiple choice questions (with answers) about moral\ \ scenarios.\n\n" -"group": "mmlu_humanities" -"group_alias": "humanities" +"tag": "mmlu_humanities_tasks" "include": "_default_template_yaml" "task": "mmlu_moral_scenarios" "task_alias": "moral_scenarios" diff --git a/lm_eval/tasks/mmlu/default/mmlu_nutrition.yaml b/lm_eval/tasks/mmlu/default/mmlu_nutrition.yaml index df70fbb2de2f40a5cb10a9093d4a0243e45c6ffc..55f8ca01ff42a296c07d8fd2e2ccda373d91775b 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_nutrition.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_nutrition.yaml @@ -1,8 +1,7 @@ "dataset_name": "nutrition" "description": "The following are multiple choice questions (with answers) about nutrition.\n\ \n" -"group": "mmlu_other" -"group_alias": "other" +"tag": "mmlu_other_tasks" "include": "_default_template_yaml" "task": "mmlu_nutrition" "task_alias": "nutrition" diff --git a/lm_eval/tasks/mmlu/default/mmlu_philosophy.yaml b/lm_eval/tasks/mmlu/default/mmlu_philosophy.yaml index 9dba09c10ebe140068f8054ee4d391f00682745d..5331c812ef70cb0123d754835fabde16ce330245 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_philosophy.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_philosophy.yaml @@ -1,8 +1,7 @@ "dataset_name": "philosophy" "description": "The following are multiple choice questions (with answers) about philosophy.\n\ \n" -"group": "mmlu_humanities" -"group_alias": "humanities" +"tag": "mmlu_humanities_tasks" "include": "_default_template_yaml" "task": "mmlu_philosophy" "task_alias": "philosophy" diff --git a/lm_eval/tasks/mmlu/default/mmlu_prehistory.yaml b/lm_eval/tasks/mmlu/default/mmlu_prehistory.yaml index d787898c62a174ddfd3895da3b164651f30d266b..0b4ff970a10b7be9ab08527124ea236227b60428 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_prehistory.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_prehistory.yaml @@ -1,8 +1,7 @@ "dataset_name": "prehistory" "description": "The following are multiple choice questions (with answers) about prehistory.\n\ \n" -"group": "mmlu_humanities" -"group_alias": "humanities" +"tag": "mmlu_humanities_tasks" "include": "_default_template_yaml" "task": "mmlu_prehistory" "task_alias": "prehistory" diff --git a/lm_eval/tasks/mmlu/default/mmlu_professional_accounting.yaml b/lm_eval/tasks/mmlu/default/mmlu_professional_accounting.yaml index 3443c336f2bc8dcf33e03bffa438a0e60a98ad3e..27b2ec9b9b70e00616d2560c3a8b1259781e8cfb 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_professional_accounting.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_professional_accounting.yaml @@ -1,8 +1,7 @@ "dataset_name": "professional_accounting" "description": "The following are multiple choice questions (with answers) about professional\ \ accounting.\n\n" -"group": "mmlu_other" -"group_alias": "other" +"tag": "mmlu_other_tasks" "include": "_default_template_yaml" "task": "mmlu_professional_accounting" "task_alias": "professional_accounting" diff --git a/lm_eval/tasks/mmlu/default/mmlu_professional_law.yaml b/lm_eval/tasks/mmlu/default/mmlu_professional_law.yaml index f3a02631e28703dc8d3a6091175c49b9eb5676eb..07c36f1c38d46a513359d80284ead794dd72b7bd 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_professional_law.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_professional_law.yaml @@ -1,8 +1,7 @@ "dataset_name": "professional_law" "description": "The following are multiple choice questions (with answers) about professional\ \ law.\n\n" -"group": "mmlu_humanities" -"group_alias": "humanities" +"tag": "mmlu_humanities_tasks" "include": "_default_template_yaml" "task": "mmlu_professional_law" "task_alias": "professional_law" diff --git a/lm_eval/tasks/mmlu/default/mmlu_professional_medicine.yaml b/lm_eval/tasks/mmlu/default/mmlu_professional_medicine.yaml index e8c49b5eb084d9cb2133c333f573ccbe3b8d9824..2c5754bf379cfd884ad837243105a49e3e28d386 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_professional_medicine.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_professional_medicine.yaml @@ -1,8 +1,7 @@ "dataset_name": "professional_medicine" "description": "The following are multiple choice questions (with answers) about professional\ \ medicine.\n\n" -"group": "mmlu_other" -"group_alias": "other" +"tag": "mmlu_other_tasks" "include": "_default_template_yaml" "task": "mmlu_professional_medicine" "task_alias": "professional_medicine" diff --git a/lm_eval/tasks/mmlu/default/mmlu_professional_psychology.yaml b/lm_eval/tasks/mmlu/default/mmlu_professional_psychology.yaml index ec48a06fe11d98facbf5c4d24e03595ceb2fb2bd..e0c0608ef6860edb4b8492402c674a7efda2070f 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_professional_psychology.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_professional_psychology.yaml @@ -1,8 +1,7 @@ "dataset_name": "professional_psychology" "description": "The following are multiple choice questions (with answers) about professional\ \ psychology.\n\n" -"group": "mmlu_social_sciences" -"group_alias": "social_sciences" +"tag": "mmlu_social_sciences_tasks" "include": "_default_template_yaml" "task": "mmlu_professional_psychology" "task_alias": "professional_psychology" diff --git a/lm_eval/tasks/mmlu/default/mmlu_public_relations.yaml b/lm_eval/tasks/mmlu/default/mmlu_public_relations.yaml index db36fb498f6886549b420eaee34fb68dddfb126c..43b675bdfd088bb7e651eece031198b5c0fb8ab3 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_public_relations.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_public_relations.yaml @@ -1,8 +1,7 @@ "dataset_name": "public_relations" "description": "The following are multiple choice questions (with answers) about public\ \ relations.\n\n" -"group": "mmlu_social_sciences" -"group_alias": "social_sciences" +"tag": "mmlu_social_sciences_tasks" "include": "_default_template_yaml" "task": "mmlu_public_relations" "task_alias": "public_relations" diff --git a/lm_eval/tasks/mmlu/default/mmlu_security_studies.yaml b/lm_eval/tasks/mmlu/default/mmlu_security_studies.yaml index 072dfd704815409d0a21576fa9620b1b5cb1a15b..b02342d95ede5148ee8b0aeb9e4ad4fb7dd05938 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_security_studies.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_security_studies.yaml @@ -1,8 +1,7 @@ "dataset_name": "security_studies" "description": "The following are multiple choice questions (with answers) about security\ \ studies.\n\n" -"group": "mmlu_social_sciences" -"group_alias": "social_sciences" +"tag": "mmlu_social_sciences_tasks" "include": "_default_template_yaml" "task": "mmlu_security_studies" "task_alias": "security_studies" diff --git a/lm_eval/tasks/mmlu/default/mmlu_sociology.yaml b/lm_eval/tasks/mmlu/default/mmlu_sociology.yaml index efcbd27b8d653f2df89f52880d47fe5a461e1246..49fa11620fb7147752328a484d56f8ead64c4387 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_sociology.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_sociology.yaml @@ -1,8 +1,7 @@ "dataset_name": "sociology" "description": "The following are multiple choice questions (with answers) about sociology.\n\ \n" -"group": "mmlu_social_sciences" -"group_alias": "social_sciences" +"tag": "mmlu_social_sciences_tasks" "include": "_default_template_yaml" "task": "mmlu_sociology" "task_alias": "sociology" diff --git a/lm_eval/tasks/mmlu/default/mmlu_us_foreign_policy.yaml b/lm_eval/tasks/mmlu/default/mmlu_us_foreign_policy.yaml index d80ee94adf4fd7b74dc368696924065862b9ce9e..bc4335e9eace7816ba112e4f55912223444d4c1f 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_us_foreign_policy.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_us_foreign_policy.yaml @@ -1,8 +1,7 @@ "dataset_name": "us_foreign_policy" "description": "The following are multiple choice questions (with answers) about us\ \ foreign policy.\n\n" -"group": "mmlu_social_sciences" -"group_alias": "social_sciences" +"tag": "mmlu_social_sciences_tasks" "include": "_default_template_yaml" "task": "mmlu_us_foreign_policy" "task_alias": "us_foreign_policy" diff --git a/lm_eval/tasks/mmlu/default/mmlu_virology.yaml b/lm_eval/tasks/mmlu/default/mmlu_virology.yaml index d935f92aa80e30f773960e6127466a4d7f768448..8f8bc114c3ce7437ad0fb413a69a859f69bcbf99 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_virology.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_virology.yaml @@ -1,8 +1,7 @@ "dataset_name": "virology" "description": "The following are multiple choice questions (with answers) about virology.\n\ \n" -"group": "mmlu_other" -"group_alias": "other" +"tag": "mmlu_other_tasks" "include": "_default_template_yaml" "task": "mmlu_virology" "task_alias": "virology" diff --git a/lm_eval/tasks/mmlu/default/mmlu_world_religions.yaml b/lm_eval/tasks/mmlu/default/mmlu_world_religions.yaml index 8681354f7057d2d3ff2d68a98b3a5e6f23c960cc..b90972c7031c30d89beea835f70aab7cf45cce81 100644 --- a/lm_eval/tasks/mmlu/default/mmlu_world_religions.yaml +++ b/lm_eval/tasks/mmlu/default/mmlu_world_religions.yaml @@ -1,8 +1,7 @@ "dataset_name": "world_religions" "description": "The following are multiple choice questions (with answers) about world\ \ religions.\n\n" -"group": "mmlu_humanities" -"group_alias": "humanities" +"tag": "mmlu_humanities_tasks" "include": "_default_template_yaml" "task": "mmlu_world_religions" "task_alias": "world_religions" diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/_mmlu.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/_mmlu.yaml index cb43b0481fb94e877f232a26bd5176dff0e82ab9..c6c1c6a19dc7638adfa630ce80b58294f5b351b8 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/_mmlu.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/_mmlu.yaml @@ -1,6 +1,34 @@ group: mmlu_flan_cot_fewshot +group_alias: mmlu (flan style, fewshot cot) task: - - mmlu_flan_cot_fewshot_stem - - mmlu_flan_cot_fewshot_other - - mmlu_flan_cot_fewshot_social_sciences - - mmlu_flan_cot_fewshot_humanities + - group: stem + task: + - mmlu_flan_cot_fewshot_stem + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: other + task: + - mmlu_flan_cot_fewshot_other + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: social sciences + task: + - mmlu_flan_cot_fewshot_social_sciences + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: humanities + task: + - mmlu_flan_cot_fewshot_humanities + aggregate_metric_list: + - metric: acc + weight_by_size: True +aggregate_metric_list: + - aggregation: mean + metric: exact_match + weight_by_size: True + filter_list: get-answer +metadata: + version: 2 diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/_mmlu_flan_cot_fewshot_template_yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/_mmlu_flan_cot_fewshot_template_yaml index 11850950ab04f59db07903265f4c24792f6883ef..cfbf222e5ba5892a5e26113d382ea86ec1300ce0 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/_mmlu_flan_cot_fewshot_template_yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/_mmlu_flan_cot_fewshot_template_yaml @@ -1,12 +1,11 @@ dataset_path: hails/mmlu_no_train # a copy of `cais/mmlu` with no auxiliary_train split validation_split: validation test_split: test -fewshot_split: dev fewshot_config: sampler: first_n output_type: generate_until -doc_to_text: "Q: {{question.strip()}}\n(A) {{choices[0]}} (B) {{choices[1]}} (C) {{choices[2]}} (D) {{choices[3]}}\nA: Let's think step by step." -doc_to_target: "{{['(A)', '(B)', '(C)', '(D)'][answer]}}" +doc_to_text: "{% if choices is defined%}Q: {{question.strip()}}\n(A) {{choices[0]}} (B) {{choices[1]}} (C) {{choices[2]}} (D) {{choices[3]}}\nA: Let's think step by step.{% else %}Q: {{ question.strip() }}\nA:{% endif %}" +doc_to_target: "{{['(A)', '(B)', '(C)', '(D)'][answer] if answer is defined else target}}" filter_list: - name: "get-answer" filter: @@ -18,7 +17,7 @@ generation_kwargs: - "" do_sample: false temperature: 0.0 -num_fewshot: 0 +num_fewshot: 4 metric_list: - metric: exact_match aggregation: mean @@ -26,6 +25,6 @@ metric_list: ignore_case: true ignore_punctuation: true metadata: - version: 1.0 + version: 2.0 dataset_kwargs: trust_remote_code: true diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_abstract_algebra.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_abstract_algebra.yaml index 3c51ab8a79efca310683852b86be1d15c1051b8b..6235d5c0997558a123258cba3dfdb4b844a2fb60 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_abstract_algebra.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_abstract_algebra.yaml @@ -54,6 +54,6 @@ fewshot_config: not have any roots. For c = 2 the polynomial x^2 + 2 has two roots at x = 1 and x = 2. Hence Z_3[x]/(x^2 + c) is a field if and only if c = 1. The answer is (B).' -group: mmlu_flan_cot_fewshot_stem +tag: mmlu_flan_cot_fewshot_stem include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_abstract_algebra diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_anatomy.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_anatomy.yaml index fd886e09e1e76f3d3fcae0d1369ffec5c8370586..e6521bdebf5efa653c7bc798fa7c4ecb985e4166 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_anatomy.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_anatomy.yaml @@ -70,6 +70,6 @@ fewshot_config: \ origin of the hyoid bone are the second and the third pharyngeal arches\u2014\ this information is covered in the last option (D). Therefore, we conclude that\ \ (D) must be the correct answer. The answer is (D).\n\n" -group: mmlu_flan_cot_fewshot_stem +tag: mmlu_flan_cot_fewshot_stem include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_anatomy diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_astronomy.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_astronomy.yaml index 2b21444ee39e5f7c09266a801a2a2b9febd5cb88..b89974588e8c83db8aedc80b607e25212a676592 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_astronomy.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_astronomy.yaml @@ -65,6 +65,6 @@ fewshot_config: because it explains that the surface is red due to the rusted materials on the surface and the red color comes from the rust. So the correct option is (A). The answer is (A).' -group: mmlu_flan_cot_fewshot_stem +tag: mmlu_flan_cot_fewshot_stem include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_astronomy diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_business_ethics.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_business_ethics.yaml index f43fd420e3c41014876e46883abdf5e7e205fae5..6c83d4bc8c06531a9375f52d8688f8b4a7cdb974 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_business_ethics.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_business_ethics.yaml @@ -70,6 +70,6 @@ fewshot_config: \ moral arguments relating to: negative *externalities*, the *power* that corporations\ \ possess and the *mutual independence* of business and society. The answer\ \ is (D).\n\n" -group: mmlu_flan_cot_fewshot_other +tag: mmlu_flan_cot_fewshot_other include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_business_ethics diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_clinical_knowledge.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_clinical_knowledge.yaml index df627d1cee8f94490e0f16f936f8a38dbc5c4f8f..008d2f870ad82e3ebee126511418c67c787c7b3b 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_clinical_knowledge.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_clinical_knowledge.yaml @@ -43,6 +43,6 @@ fewshot_config: target: 'Let''s think step by step. We refer to Wikipedia articles on clinical knowledge for help. The energy for muscular contraction is provided by ATP (adenosine triphosphate), which is the powerhouse of the cell. The answer is (A).' -group: mmlu_flan_cot_fewshot_other +tag: mmlu_flan_cot_fewshot_other include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_clinical_knowledge diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_biology.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_biology.yaml index 4a976498e5a30bfed2ec4c032a186782506a7203..28f7f989b86b28cccc5e9128268e8ea6bc80e662 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_biology.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_biology.yaml @@ -70,6 +70,6 @@ fewshot_config: that have different origins, which is not the case for the human and bird forearms, which rules out (D). Humans and birds do belong to the same clade - a group of organisms composed of a common ancestor. The answer is (C).' -group: mmlu_flan_cot_fewshot_stem +tag: mmlu_flan_cot_fewshot_stem include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_college_biology diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_chemistry.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_chemistry.yaml index b337ade0858b19d6f876f35a0ef41e52bd09d31d..4a8cfc9e4436f1dc50b75efdf84a0b6f0625f2bf 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_chemistry.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_chemistry.yaml @@ -44,6 +44,6 @@ fewshot_config: \ into 2 lines. This will be further split into 4 lines by the interaction with\ \ three equivalent 1H nuclei. The total number of lines is therefore $2 \\cdot\ \ 4 = 8$. The answer is (E).\n\n" -group: mmlu_flan_cot_fewshot_stem +tag: mmlu_flan_cot_fewshot_stem include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_college_chemistry diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_computer_science.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_computer_science.yaml index 6e19ffed8e46c3e2fd8057fdb9ca2510e53ab5fa..5eccde7c6be9620f63bbf3b1de42f32c8e121539 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_computer_science.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_computer_science.yaml @@ -175,6 +175,6 @@ fewshot_config: (1000 nanoseconds / cache miss) * (1 cache miss / 50 instructions) * (50 instructions / 27000 nanoseconds) = 1000 * (1/50) * (50/27000) = 1000/27000 = 1/27. The answer is (B).' -group: mmlu_flan_cot_fewshot_stem +tag: mmlu_flan_cot_fewshot_stem include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_college_computer_science diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_mathematics.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_mathematics.yaml index bac8310909a166b07d11707d065c14d289b57a9c..5552cc35bc55ffab1b53538ee2605778e6f215d6 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_mathematics.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_mathematics.yaml @@ -68,6 +68,6 @@ fewshot_config: \ Then, for all $t \\in \\mathbb{R}$, we have $(s(t))-2=K e^{-t / 25}$, and\ \ so $s(t)=2+K e^{-t / 25}$. Then $3=s(0)=2+K e^{0}=2+K$, so $K=1$. Then $s(100)=2+K\ \ e^{-100 / 25}=2+1 \\cdot e^{-4}=2+e^{-4}$. The answer is (D).\n\n" -group: mmlu_flan_cot_fewshot_stem +tag: mmlu_flan_cot_fewshot_stem include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_college_mathematics diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_medicine.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_medicine.yaml index deee7daf6913ae28230a0789ccb11c4c91575f6c..7eac0bab3f9286469b44e815c8a2090fc5ff0832 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_medicine.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_medicine.yaml @@ -63,6 +63,6 @@ fewshot_config: for help. Glucose (also known as the blood sugar) is the main sugar found in the human body. It is transported into the muscle cell via diffusion through protein transporters called GLUT4. The answer is (A).' -group: mmlu_flan_cot_fewshot_other +tag: mmlu_flan_cot_fewshot_other include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_college_medicine diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_physics.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_physics.yaml index c935b4889fbcff0cfd901b2603d02e4a022dfce3..aa158a9f3c3e9e420608970a3ea91e744a042a6a 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_physics.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_college_physics.yaml @@ -56,6 +56,6 @@ fewshot_config: of the gas container is constant, no work will be done (since work is pressure times change in volume). So, at constant volume, all of the heat goes into the internal energy. The answer is (B).' -group: mmlu_flan_cot_fewshot_stem +tag: mmlu_flan_cot_fewshot_stem include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_college_physics diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_computer_security.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_computer_security.yaml index d4d2d24ca92f64db6e566d0c6cb6732081499196..6a7c5cb2d5d1a2943b5b1ad6c17bf5ce0a324f5a 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_computer_security.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_computer_security.yaml @@ -45,6 +45,6 @@ fewshot_config: of the TLS heartbeat extension. The vulnerability was classified as a buffer over-read, a situation where more data can be read than should be allowed. The answer is (C).' -group: mmlu_flan_cot_fewshot_stem +tag: mmlu_flan_cot_fewshot_stem include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_computer_security diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_conceptual_physics.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_conceptual_physics.yaml index 85054dd39b05d11e66c4c0486d4520b63740afe7..a4757faf1d490e312ea70a3d4dbe291459366471 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_conceptual_physics.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_conceptual_physics.yaml @@ -44,6 +44,6 @@ fewshot_config: \ orthogonal to the wind is the same as it would be in the absence of the wind.\ \ The total speed, which is these two components added in quadrature, is thus\ \ greater than the speed in still air. The answer is (B).\n\n" -group: mmlu_flan_cot_fewshot_stem +tag: mmlu_flan_cot_fewshot_stem include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_conceptual_physics diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_econometrics.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_econometrics.yaml index 11100c8afb3616038b1fc7e2863eca35d185af6a..e4282345ce0f2e8bab97a80413fbd2b796a7fe3e 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_econometrics.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_econometrics.yaml @@ -82,6 +82,6 @@ fewshot_config: target: 'Let''s think step by step. We refer to Wikipedia articles on econometrics for help. This is a formal logic problem about stationally process. For a stationary autoregressive process, shocks will eventually die away. The answer is (A).' -group: mmlu_flan_cot_fewshot_social_sciences +tag: mmlu_flan_cot_fewshot_social_sciences include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_econometrics diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_electrical_engineering.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_electrical_engineering.yaml index 5f2c597487871fbd4b6135c28608667ba95029aa..305d2340c5ffa69761ef8dc2ab128849e571bbf8 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_electrical_engineering.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_electrical_engineering.yaml @@ -42,6 +42,6 @@ fewshot_config: target: 'Let''s think step by step. In lap winding, effectively two resistors are connected in parallel, so the actual resistance of each pair is 1 Ohm. Since we have 50 pairs, we get a total resistance of 50 Ohms. The answer is (C).' -group: mmlu_flan_cot_fewshot_stem +tag: mmlu_flan_cot_fewshot_stem include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_electrical_engineering diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_elementary_mathematics.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_elementary_mathematics.yaml index 3da11ec1b41dca67beb44bb853059aa61a218275..1fb9eef860424b49adf108566edadaf1500dbf5f 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_elementary_mathematics.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_elementary_mathematics.yaml @@ -72,6 +72,6 @@ fewshot_config: (D) (5 x 9) x (6 x 9)' target: 'Let''s think step by step. We know that 9 = (5 + 4), so 5 x 9 = 5 x (5 + 4) = (5 x 5) + (5 x 4). The answer is (B).' -group: mmlu_flan_cot_fewshot_stem +tag: mmlu_flan_cot_fewshot_stem include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_elementary_mathematics diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_formal_logic.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_formal_logic.yaml index 743deb85252cc963d03c4eb7c9fd93b53a5646c9..3fbc73c3d24f9cef06a41bbcfddea55aec1b424a 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_formal_logic.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_formal_logic.yaml @@ -65,6 +65,6 @@ fewshot_config: \ p do not drive on Mars.\nOf all these options, Option (C) appears to be the\ \ best and most meaningful interpretation of the argument \u201CNo people drive\ \ on Mars.\u201D The answer is (C).\n\n" -group: mmlu_flan_cot_fewshot_humanities +tag: mmlu_flan_cot_fewshot_humanities include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_formal_logic diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_global_facts.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_global_facts.yaml index 34e8c4b7f496f55441f8dcd45c476756b1640548..739c00106536c8df64213d55b83c053db1314124 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_global_facts.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_global_facts.yaml @@ -44,6 +44,6 @@ fewshot_config: for help. As of 2019, most people tend to be optimistic about their own future but pessimistic about the future of their nation or the world. The answer is (B).' -group: mmlu_flan_cot_fewshot_other +tag: mmlu_flan_cot_fewshot_other include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_global_facts diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_biology.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_biology.yaml index b780fd29e937be86c98ecc97c9076b1d085cd121..0dfb19f924761c6dd56cc6b3b9ada38b5bf473e0 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_biology.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_biology.yaml @@ -64,6 +64,6 @@ fewshot_config: core cell cycle regulators inside the cell. The most common regulators are cyclins and cyclin-dependent kinases. Fibroblast cells do not play any role in cell division. The answer is (D).' -group: mmlu_flan_cot_fewshot_stem +tag: mmlu_flan_cot_fewshot_stem include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_high_school_biology diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_chemistry.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_chemistry.yaml index b10f03d45c15df87a27589eb4b65e46f736ad464..948c375824506cc188ed821c4512725bfdc92d06 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_chemistry.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_chemistry.yaml @@ -61,6 +61,6 @@ fewshot_config: \ strong acid, Nitric acid, will react with the conjugate base. Therefore the\ \ maximum amount of acid that can be added will be equal to the amount of acetate\ \ ion, or 2 moles. The answer is (C).\n\n" -group: mmlu_flan_cot_fewshot_stem +tag: mmlu_flan_cot_fewshot_stem include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_high_school_chemistry diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_computer_science.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_computer_science.yaml index 5dee647a04b222cad2f3f8f27ceadf27a9f6a8ae..6f21030ae880ec0d6d42f0e2618c312b55b82549 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_computer_science.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_computer_science.yaml @@ -79,6 +79,6 @@ fewshot_config: its value is greater than 100, regardless of the elements in the list. Choice D is incorrect because its step 3 does not increment the value of position, so it will repeat forever. The answer is (B).' -group: mmlu_flan_cot_fewshot_stem +tag: mmlu_flan_cot_fewshot_stem include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_high_school_computer_science diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_european_history.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_european_history.yaml index 5a7cfb5cf4caeaf1f764d98e9358acf659b1f88c..4707a1857f44fad0ef67313b5f1901b5b6c869b6 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_european_history.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_european_history.yaml @@ -194,6 +194,6 @@ fewshot_config: wrote extensively against the monoplization of power and advocated for a system of checks and balances in government to prevent the rise of despotism. The answer is (B).' -group: mmlu_flan_cot_fewshot_humanities +tag: mmlu_flan_cot_fewshot_humanities include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_high_school_european_history diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_geography.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_geography.yaml index 66f146b69de1eac409fc6313b5941c162f36db61..96f4b365af04a3dc5c754def2899a5291ada1072 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_geography.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_geography.yaml @@ -48,6 +48,6 @@ fewshot_config: target: 'Let''s think step by step. We refer to Wikipedia articles on geography for help. The difference between number of births and deaths gives the population increase at any given time. The answer is (A).' -group: mmlu_flan_cot_fewshot_social_sciences +tag: mmlu_flan_cot_fewshot_social_sciences include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_high_school_geography diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_government_and_politics.yaml index a22f99de7670f92155ffe79cf98659b9edce44d3..4c11772183c0266288f95f4d273ebbdf32c0dba1 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_government_and_politics.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_government_and_politics.yaml @@ -56,6 +56,6 @@ fewshot_config: target: 'Let''s think step by step. We refer to Wikipedia articles on government and politics for help. The US Constitution is not very specific about the powers of the president, leading to uncertainty over its limits. The answer is (A).' -group: mmlu_flan_cot_fewshot_social_sciences +tag: mmlu_flan_cot_fewshot_social_sciences include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_high_school_government_and_politics diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_macroeconomics.yaml index 5cecd65fc79af66451132283c76de3fe2ffcf003..5700d5df946539608bbf10555e0f06ca07672b09 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_macroeconomics.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_macroeconomics.yaml @@ -48,6 +48,6 @@ fewshot_config: for help. The economic transactions related to the performance of the American pop-singer in Paris happens entirely outside the U.S. and hence is not included in the GDP numbers. The answer is (C).' -group: mmlu_flan_cot_fewshot_social_sciences +tag: mmlu_flan_cot_fewshot_social_sciences include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_high_school_macroeconomics diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_mathematics.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_mathematics.yaml index c86bca44abebaa12c79cd6c517fc59493a0cc422..e3e05795561a11dca35dd342bc88b4794584809c 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_mathematics.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_mathematics.yaml @@ -46,6 +46,6 @@ fewshot_config: target: 'Let''s think step by step. The least common multiple of 2, 3 and 5 is 30, so during a 7 minute dance, all the three lights will come on at the same time $2*7+1=15$ times. The answer is (B).' -group: mmlu_flan_cot_fewshot_stem +tag: mmlu_flan_cot_fewshot_stem include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_high_school_mathematics diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_microeconomics.yaml index c43e5511b2f047c4906f2f0d7efa5e9fd44499a6..ead78be898187a7c08c292eaeebfe6b067ef4413 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_microeconomics.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_microeconomics.yaml @@ -51,6 +51,6 @@ fewshot_config: for help. An increase in the construction of new houses means an increase demand of in-house painting, thus increases the demand for housepainters. The answer is (C).' -group: mmlu_flan_cot_fewshot_social_sciences +tag: mmlu_flan_cot_fewshot_social_sciences include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_high_school_microeconomics diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_physics.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_physics.yaml index 277329dcd11f734c787ab14b64e216b9f41c4ffe..5d2166b8595529213797cf12efd67f90ca4e9e61 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_physics.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_physics.yaml @@ -45,6 +45,6 @@ fewshot_config: cannot have any net displacement because the pipe closure stops them. So the particle displacement is at a node. This closure also causes the pressure to be maximal, i.e. an antinode. The answer is (B).' -group: mmlu_flan_cot_fewshot_stem +tag: mmlu_flan_cot_fewshot_stem include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_high_school_physics diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_psychology.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_psychology.yaml index 7600dcbb9475077d3ed8e3d64b2c054e091700b4..553fe18de24f14765d14d0fb2dbdfaf72449af32 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_psychology.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_psychology.yaml @@ -59,6 +59,6 @@ fewshot_config: for help. People with an external locus of control believes fate and luck play an important role in their lives, while people with an internal locus of control believes they control their lives. The answer is (D).' -group: mmlu_flan_cot_fewshot_social_sciences +tag: mmlu_flan_cot_fewshot_social_sciences include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_high_school_psychology diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_statistics.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_statistics.yaml index 8822c4bf053c1037d4afdfee9a1538d0d8683771..66ed702a6b9b57dc69719a13923e04ebc8218ea9 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_statistics.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_statistics.yaml @@ -76,6 +76,6 @@ fewshot_config: not perfectly correlated. Statement B is false because uncorrelated variables regression lines can have slope zero. Statement C is false because correlation is symmetric in the two random variables. The answer is (D).' -group: mmlu_flan_cot_fewshot_stem +tag: mmlu_flan_cot_fewshot_stem include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_high_school_statistics diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_us_history.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_us_history.yaml index 54246224a350cb3180303cad65e60925000fa9f5..8cea5109f6570086dce3cf1815dc50f1889d80ad 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_us_history.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_us_history.yaml @@ -151,6 +151,6 @@ fewshot_config: suspect Washington''s military response to Whiskey Rebellion. Bacon''s Rebellion and Pontiac''s Rebellion happen before the Revolution and they can be ruled out. The answer is (C).' -group: mmlu_flan_cot_fewshot_humanities +tag: mmlu_flan_cot_fewshot_humanities include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_high_school_us_history diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_world_history.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_world_history.yaml index e39c5cb346e6932dd8fa20f5acb3540a74e93f37..2fbdaf05c137270f4ff4207e7c6ce81c2a34d30c 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_world_history.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_high_school_world_history.yaml @@ -95,6 +95,6 @@ fewshot_config: for help. Brahman refers to the ultimate reality of all things in the Hindu religion. In contrast, Buddhism does not have a concept of supreme God. The answer is (A).' -group: mmlu_flan_cot_fewshot_humanities +tag: mmlu_flan_cot_fewshot_humanities include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_high_school_world_history diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_human_aging.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_human_aging.yaml index 1cb9c6404f297205edee34d0432dfdc543919073..3eec010845fa68ad974bdb7cd922a0028365d96e 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_human_aging.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_human_aging.yaml @@ -37,6 +37,6 @@ fewshot_config: target: 'Let''s think step by step. We refer to Wikipedia articles on human aging for help. Texas does not have state tax, and has low cost of living compared with the other three options. The answer is (A).' -group: mmlu_flan_cot_fewshot_other +tag: mmlu_flan_cot_fewshot_other include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_human_aging diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_human_sexuality.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_human_sexuality.yaml index 870c3c1651b6b8bc6f0e7ddbd4f7beac88ea65ff..dab78f0c5ec1042d23240bb71f59b212885585aa 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_human_sexuality.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_human_sexuality.yaml @@ -45,6 +45,6 @@ fewshot_config: target: 'Let''s think step by step. We refer to Wikipedia articles on human sexuality for help. Morning sickness usually begins by nine weeks after conception, corresponding to the first trimester. The answer is (A).' -group: mmlu_flan_cot_fewshot_social_sciences +tag: mmlu_flan_cot_fewshot_social_sciences include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_human_sexuality diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_international_law.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_international_law.yaml index 89589dcdfcdb41bf473791eb5bbe6e8cdfdf3963..99341f395352d7b6a9a8d1a71005ca821ac9b723 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_international_law.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_international_law.yaml @@ -65,6 +65,6 @@ fewshot_config: target: 'Let''s think step by step. We refer to Wikipedia articles on international law for help. Article 2(4) of the UN Charter prohibits states from using armed forces in their international relations. The answer is (A).' -group: mmlu_flan_cot_fewshot_humanities +tag: mmlu_flan_cot_fewshot_humanities include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_international_law diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_jurisprudence.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_jurisprudence.yaml index 383f3d3a958c15bd1db3659476f854c20160fbed..3c714f7e595a48d28b750479a46183a02fc24dc0 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_jurisprudence.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_jurisprudence.yaml @@ -54,6 +54,6 @@ fewshot_config: principle'', and reject the ''system of natural liberty'', but the POP would not choose equality above liberty, since the POP assume both equal and free citizens. The answer is (A).' -group: mmlu_flan_cot_fewshot_humanities +tag: mmlu_flan_cot_fewshot_humanities include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_jurisprudence diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_logical_fallacies.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_logical_fallacies.yaml index f09e4924201a8ceb9d0e6766827e39de994e2740..dd72cab8cf8ea0553a633810e59c0ba4059c2176 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_logical_fallacies.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_logical_fallacies.yaml @@ -56,6 +56,6 @@ fewshot_config: fallacies for help. The argument against the person fallacy occurs when someone irrelevantly attacks the character of an opposing arguer, instead of addressing that opponent''s arguments. The answer is (C).' -group: mmlu_flan_cot_fewshot_humanities +tag: mmlu_flan_cot_fewshot_humanities include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_logical_fallacies diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_machine_learning.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_machine_learning.yaml index 8cc48107c686d00ab1881b5b5732c7e31705c531..33622ac4e7291eb380b6e58382e4fe84052a5bdc 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_machine_learning.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_machine_learning.yaml @@ -69,6 +69,6 @@ fewshot_config: \ epsilon when we have N samples if 2 exp(-2 epsilon^2 N)<0.05, this implies\ \ that N > -1/(2*epsilon**2) log ( 0.05/2 )= log (40)*5000. Since log(40)>1,\ \ we have that one needs more than 1000 examples. The answer is (D).\n\n" -group: mmlu_flan_cot_fewshot_stem +tag: mmlu_flan_cot_fewshot_stem include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_machine_learning diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_management.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_management.yaml index 63f6168bdacb962c3540c5103d9e81c65f2c8a89..87d9ba8c9aa31733a5849695213d97deff9c2ded 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_management.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_management.yaml @@ -41,6 +41,6 @@ fewshot_config: for help. The Ohio State Leadership Studies conducted in the 1940s identified initiating structure and consideration as the two main dimensions of leader behavior. The answer is (D).' -group: mmlu_flan_cot_fewshot_other +tag: mmlu_flan_cot_fewshot_other include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_management diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_marketing.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_marketing.yaml index ec5caa40b32e9596416c7fb4f3c506181c296f32..182eb52ec509c34c225a176774be653d747e120e 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_marketing.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_marketing.yaml @@ -51,6 +51,6 @@ fewshot_config: target: 'Let''s think step by step. We refer to Wikipedia articles on marketing for help. Geodemographics is a natural outcome when combining demographic and geographic variables. The answer is (A).' -group: mmlu_flan_cot_fewshot_other +tag: mmlu_flan_cot_fewshot_other include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_marketing diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_medical_genetics.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_medical_genetics.yaml index 7a7bbe753612eedb55693dc0ed24667545e335cc..757010bfed0f08f882995eed787d4f68e0f8121c 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_medical_genetics.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_medical_genetics.yaml @@ -46,6 +46,6 @@ fewshot_config: genetics for help. A Southern blot is a method in molecular biology for detecting specific DNA sequences in a sample. Large triplet repeat expansions are usually detected with this method. The answer is (C).' -group: mmlu_flan_cot_fewshot_other +tag: mmlu_flan_cot_fewshot_other include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_medical_genetics diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_miscellaneous.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_miscellaneous.yaml index eb86cf4cd4145d0e5a6a358d268e701563fadc66..2fe892eb06f522df6e99606013d26e0df1517cf3 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_miscellaneous.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_miscellaneous.yaml @@ -38,6 +38,6 @@ fewshot_config: (A) one (B) two (C) four (D) eight' target: 'Let''s think step by step. We refer to Wikipedia for help. Most cars have two axles to rotate the wheels.. The answer is (B).' -group: mmlu_flan_cot_fewshot_other +tag: mmlu_flan_cot_fewshot_other include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_miscellaneous diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_moral_disputes.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_moral_disputes.yaml index e24621a9e11bf27b8e0e3ae8d1e7c22325e58b5e..45a92e075582d6c0e2eb11c0310f87fc5debb4bb 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_moral_disputes.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_moral_disputes.yaml @@ -59,6 +59,6 @@ fewshot_config: as it treats individuals as incapable of communal relations. It is unclear that capital punishment is to the benefit of, or a deterrent of harm to the community. The answer is (A).' -group: mmlu_flan_cot_fewshot_humanities +tag: mmlu_flan_cot_fewshot_humanities include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_moral_disputes diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_moral_scenarios.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_moral_scenarios.yaml index f2217ef8b43e82b474b6460b26fee0c4432391ca..b4a0231930d62fe73a1cb84b58420baf0182365c 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_moral_scenarios.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_moral_scenarios.yaml @@ -57,6 +57,6 @@ fewshot_config: target: 'Let''s think step by step. We refer to Wikipedia articles on moral scenarios for help. Loving someone is not wrong. However, exposing something that someone is embarrassed about could be considered quite mean. The answer is (C).' -group: mmlu_flan_cot_fewshot_humanities +tag: mmlu_flan_cot_fewshot_humanities include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_moral_scenarios diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_nutrition.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_nutrition.yaml index f6a09b77dff66b0b73bc8b17ca47a1d63b394787..66498dc564350f893e7bd45078528b5750bce0ca 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_nutrition.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_nutrition.yaml @@ -58,6 +58,6 @@ fewshot_config: target: 'Let''s think step by step. We refer to Wikipedia articles on nutrition for help. The risk ratio is not sufficiently reduced that it could not be explained by random chance given the studies sample size. The answer is (C).' -group: mmlu_flan_cot_fewshot_other +tag: mmlu_flan_cot_fewshot_other include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_nutrition diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_philosophy.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_philosophy.yaml index 2877c760248f1f0acc0310468afa3630d2f734bd..b5116be01d6a463f23f9b7422de1b0b48bbdb55e 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_philosophy.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_philosophy.yaml @@ -39,6 +39,6 @@ fewshot_config: for help. Psychological egoism suggests that one behaves based on what makes one feels good, hence it is a claim about human nature and how humans are capable of behaving. The answer is (C).' -group: mmlu_flan_cot_fewshot_humanities +tag: mmlu_flan_cot_fewshot_humanities include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_philosophy diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_prehistory.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_prehistory.yaml index 49cbb91f722b51d4fb26999416592e97e37d52c3..6f9e5d81664445497a13a9adf7ca818ed6d2c7ef 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_prehistory.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_prehistory.yaml @@ -54,6 +54,6 @@ fewshot_config: target: 'Let''s think step by step. We refer to Wikipedia articles on prehistory for help. Pacal built the temples as the funerary monument to legitimize his kingship. The answer is (D).' -group: mmlu_flan_cot_fewshot_humanities +tag: mmlu_flan_cot_fewshot_humanities include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_prehistory diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_accounting.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_accounting.yaml index 81073c1c5db05a8749a8850045dda2023123757c..8d18fc22626b952d18491b849cabc720706c17c9 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_accounting.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_accounting.yaml @@ -58,6 +58,6 @@ fewshot_config: for help. Among the four transactions, only Proceeds from long-term debt belongs to the financing activities section of cashflow, hence the amount reported should be $100000. The answer is (D).' -group: mmlu_flan_cot_fewshot_other +tag: mmlu_flan_cot_fewshot_other include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_professional_accounting diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_law.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_law.yaml index 9a4a17d3ab2c80ef329ab1d68b7b82f4f9d51a2b..307f8940bc445305fdbf00e89910cd5237a41312 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_law.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_law.yaml @@ -117,6 +117,6 @@ fewshot_config: a due process clause. Hence the strongest argument should be the statute is overbroad and consequently invalid under the First and Fourteenth Amendments. The answer is (D).' -group: mmlu_flan_cot_fewshot_humanities +tag: mmlu_flan_cot_fewshot_humanities include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_professional_law diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_medicine.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_medicine.yaml index 9033f07e3c79ea581fcb79de4fda73dc34d58709..4a5b0d995ae7d0897eaea3a6cfd0719562cdc3c9 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_medicine.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_medicine.yaml @@ -77,6 +77,6 @@ fewshot_config: for help. The symptoms and the adrenal mass suggested pheochromocytoma, and the blood pressure indicates hypertension. Phenoxybenzamine is used to treat hypertension caused by pheochromocytoma. The answer is (D).' -group: mmlu_flan_cot_fewshot_other +tag: mmlu_flan_cot_fewshot_other include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_professional_medicine diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_psychology.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_psychology.yaml index 772c8f47063fadb87121cabc82939d5793c846bb..60b5da683ff87d207304b894a5138a6c439a1c86 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_psychology.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_professional_psychology.yaml @@ -57,6 +57,6 @@ fewshot_config: for help. Based on the circumstances, you should tell your client about the pros and cons of each program, but it would be inappropriate to receive the bonus, so you should not claim the $50 bonus. The answer is (D).' -group: mmlu_flan_cot_fewshot_social_sciences +tag: mmlu_flan_cot_fewshot_social_sciences include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_professional_psychology diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_public_relations.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_public_relations.yaml index f7fbabcf8e0ce718cfcc2f10a0c462233e5698a8..fe384b1e2b7d19c216f8344d5c249f2c16dc723b 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_public_relations.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_public_relations.yaml @@ -50,6 +50,6 @@ fewshot_config: for help. If a public relations media practitioner does not know the answer to a reporter''s question, they should say ''I don''t know'' and offer to provide the information later. The answer is (C).' -group: mmlu_flan_cot_fewshot_social_sciences +tag: mmlu_flan_cot_fewshot_social_sciences include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_public_relations diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_security_studies.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_security_studies.yaml index d1b3092d5e040f3e8ef3195b4568aaa1d3e2ddb2..b37e35b3bd4fefa0ca040f0d59ff2fcae156fb45 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_security_studies.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_security_studies.yaml @@ -99,6 +99,6 @@ fewshot_config: target: 'Let''s think step by step. We refer to Wikipedia articles on security studies for help. Coercive diplomacy uses the threat of force to induce the opponent to comply with demands. The answer is (B).' -group: mmlu_flan_cot_fewshot_social_sciences +tag: mmlu_flan_cot_fewshot_social_sciences include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_security_studies diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_sociology.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_sociology.yaml index 0b7e1c05b9a9d6f572ec57b4b10cbb9cee903c1a..4229d64785ded8673d421a9fb1571d0cce705a93 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_sociology.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_sociology.yaml @@ -53,6 +53,6 @@ fewshot_config: for help. The post-war welfare state of 1948 aimed to provide free healthcare and education, full employment, and universal welfare. But it did not aim to provide a minimum wage. The answer is (B).' -group: mmlu_flan_cot_fewshot_social_sciences +tag: mmlu_flan_cot_fewshot_social_sciences include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_sociology diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_us_foreign_policy.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_us_foreign_policy.yaml index 3421476c4dfb248f9eb6643691ab3c6ad2166cb1..bc86b7c88fa1b62d2f12deaa16394f43fc722225 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_us_foreign_policy.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_us_foreign_policy.yaml @@ -51,6 +51,6 @@ fewshot_config: target: 'Let''s think step by step. We refer to Wikipedia articles on us foreign policy for help. The 2008 financial crisis damanged the international reputation of the American model of political economy and capitalism. The answer is (A).' -group: mmlu_flan_cot_fewshot_social_sciences +tag: mmlu_flan_cot_fewshot_social_sciences include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_us_foreign_policy diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_virology.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_virology.yaml index 700ce10d8ba4e6784c60ce428a907cef897d8c7e..0e70f0ee24cc946a1cfbc51cb87d4fde20d8171c 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_virology.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_virology.yaml @@ -40,6 +40,6 @@ fewshot_config: target: 'Let''s think step by step. We refer to Wikipedia articles on virology for help. Paroviruses are highly impactful because they do not have nucleic acid. The answer is (A).' -group: mmlu_flan_cot_fewshot_other +tag: mmlu_flan_cot_fewshot_other include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_virology diff --git a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_world_religions.yaml b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_world_religions.yaml index d939723598163f21b430d54201652a5b9a840728..41502cc7a3a1318b7c6a0f2ac16cda86dda08486 100644 --- a/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_world_religions.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_fewshot/mmlu_world_religions.yaml @@ -37,6 +37,6 @@ fewshot_config: target: 'Let''s think step by step. We refer to Wikipedia articles on world religions for help. In Judaism, the most distinctive sign of the covenant is circumcision (brit milah). The answer is (B).' -group: mmlu_flan_cot_fewshot_humanities +tag: mmlu_flan_cot_fewshot_humanities include: _mmlu_flan_cot_fewshot_template_yaml task: mmlu_flan_cot_fewshot_world_religions diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/_mmlu.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/_mmlu.yaml index 390425c7d248a99e385c73f19d2cfa9e6d27747c..745a892568bd84b38252e20bbc9a0bea73ddb1db 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/_mmlu.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/_mmlu.yaml @@ -1,6 +1,32 @@ group: mmlu_flan_cot_zeroshot +group_alias: mmlu (flan style, zeroshot cot) task: - - mmlu_flan_cot_zeroshot_stem - - mmlu_flan_cot_zeroshot_other - - mmlu_flan_cot_zeroshot_social_sciences - - mmlu_flan_cot_zeroshot_humanities + - group: stem + task: + - mmlu_flan_cot_zeroshot_stem + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: other + task: + - mmlu_flan_cot_zeroshot_other + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: social sciences + task: + - mmlu_flan_cot_zeroshot_social_sciences + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: humanities + task: + - mmlu_flan_cot_zeroshot_humanities + aggregate_metric_list: + - metric: acc + weight_by_size: True +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 2 diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/_mmlu_flan_cot_zeroshot_template_yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/_mmlu_flan_cot_zeroshot_template_yaml index 113d20b77115900898d3fb8d1996a79a34abc2c3..7588b67e1905dc7eae3790c33a16de460c465f67 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/_mmlu_flan_cot_zeroshot_template_yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/_mmlu_flan_cot_zeroshot_template_yaml @@ -12,7 +12,7 @@ filter_list: - function: "take_first" - name: "flexible-extract" filter: - - function: !function utils.MultiChoiceRegexFilter + - function: "multi_choice_regex" group_select: -1 ignore_case: true ignore_punctuation: true @@ -33,6 +33,6 @@ metric_list: ignore_case: true ignore_punctuation: true metadata: - version: 2.0 + version: 3.0 dataset_kwargs: trust_remote_code: true diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_abstract_algebra.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_abstract_algebra.yaml index 8609f626a25d5f37d41ab8a312dfe226e44dbbd8..5e885b818eae4bbc87374c756b68ecd11e44bd69 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_abstract_algebra.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_abstract_algebra.yaml @@ -1,6 +1,6 @@ "dataset_name": "abstract_algebra" "description": "The following are multiple choice questions (with answers) about abstract\ \ algebra.\n\n" -"group": "mmlu_flan_cot_zeroshot_stem" +"tag": "mmlu_flan_cot_zeroshot_stem" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_abstract_algebra" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_anatomy.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_anatomy.yaml index 2923349d0fb02f219844b192fb38880537cb9c3a..7f17410a7cc0869223730328f55803d8d424e930 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_anatomy.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_anatomy.yaml @@ -1,6 +1,6 @@ "dataset_name": "anatomy" "description": "The following are multiple choice questions (with answers) about anatomy.\n\ \n" -"group": "mmlu_flan_cot_zeroshot_stem" +"tag": "mmlu_flan_cot_zeroshot_stem" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_anatomy" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_astronomy.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_astronomy.yaml index e5ffd8ffe302442af98c246e9d7bac54c063d81f..b5b821f97642ad5987244a0ac4c9988c2fca3857 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_astronomy.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_astronomy.yaml @@ -1,6 +1,6 @@ "dataset_name": "astronomy" "description": "The following are multiple choice questions (with answers) about astronomy.\n\ \n" -"group": "mmlu_flan_cot_zeroshot_stem" +"tag": "mmlu_flan_cot_zeroshot_stem" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_astronomy" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_business_ethics.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_business_ethics.yaml index a64285711f2f23775ddc37431b5c39f5a589f9ec..b26c679e26b6bd04d77eb5e0bb2ebaddcc515561 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_business_ethics.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_business_ethics.yaml @@ -1,6 +1,6 @@ "dataset_name": "business_ethics" "description": "The following are multiple choice questions (with answers) about business\ \ ethics.\n\n" -"group": "mmlu_flan_cot_zeroshot_other" +"tag": "mmlu_flan_cot_zeroshot_other" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_business_ethics" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_clinical_knowledge.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_clinical_knowledge.yaml index e3655230cf9703b3d02bfa457f667d02676a3aa7..3c0e9d17db10f4e69d1c44d5a127f2bbe1f4e279 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_clinical_knowledge.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_clinical_knowledge.yaml @@ -1,6 +1,6 @@ "dataset_name": "clinical_knowledge" "description": "The following are multiple choice questions (with answers) about clinical\ \ knowledge.\n\n" -"group": "mmlu_flan_cot_zeroshot_other" +"tag": "mmlu_flan_cot_zeroshot_other" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_clinical_knowledge" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_biology.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_biology.yaml index 736bb6decd83da6ac642049a9bcf3c9af0f562f2..de020f4eaca7fdeb650688f034ee3b5d89490ddc 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_biology.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_biology.yaml @@ -1,6 +1,6 @@ "dataset_name": "college_biology" "description": "The following are multiple choice questions (with answers) about college\ \ biology.\n\n" -"group": "mmlu_flan_cot_zeroshot_stem" +"tag": "mmlu_flan_cot_zeroshot_stem" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_college_biology" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_chemistry.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_chemistry.yaml index 7b7199664bb086da78f2314bd69f69177c6116a4..b8e5bbcf76b9fb3ad012511b213ffbbd554cd58d 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_chemistry.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_chemistry.yaml @@ -1,6 +1,6 @@ "dataset_name": "college_chemistry" "description": "The following are multiple choice questions (with answers) about college\ \ chemistry.\n\n" -"group": "mmlu_flan_cot_zeroshot_stem" +"tag": "mmlu_flan_cot_zeroshot_stem" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_college_chemistry" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_computer_science.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_computer_science.yaml index 185f2a6614dad536c1a16d1b1017599a18427a67..04b5e750949984abcd7889be80485e52c97dba9f 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_computer_science.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_computer_science.yaml @@ -1,6 +1,6 @@ "dataset_name": "college_computer_science" "description": "The following are multiple choice questions (with answers) about college\ \ computer science.\n\n" -"group": "mmlu_flan_cot_zeroshot_stem" +"tag": "mmlu_flan_cot_zeroshot_stem" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_college_computer_science" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_mathematics.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_mathematics.yaml index 210eb127c831b310084a9d2df11f1dc29cb62d87..81c59cc2c20f340a76ed3d945e976ce3c832815c 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_mathematics.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_mathematics.yaml @@ -1,6 +1,6 @@ "dataset_name": "college_mathematics" "description": "The following are multiple choice questions (with answers) about college\ \ mathematics.\n\n" -"group": "mmlu_flan_cot_zeroshot_stem" +"tag": "mmlu_flan_cot_zeroshot_stem" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_college_mathematics" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_medicine.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_medicine.yaml index 51c8a3c09ef78f6d7d8249c4f244a5c63c249cd5..0450a068f4b763629e463d9882e4a3e99f86d726 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_medicine.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_medicine.yaml @@ -1,6 +1,6 @@ "dataset_name": "college_medicine" "description": "The following are multiple choice questions (with answers) about college\ \ medicine.\n\n" -"group": "mmlu_flan_cot_zeroshot_other" +"tag": "mmlu_flan_cot_zeroshot_other" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_college_medicine" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_physics.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_physics.yaml index 319c72142454df340260c1a6dd3f59f244a57eab..82c2bb2ab586be2346237a6aa8b2ea9fd9170c97 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_physics.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_college_physics.yaml @@ -1,6 +1,6 @@ "dataset_name": "college_physics" "description": "The following are multiple choice questions (with answers) about college\ \ physics.\n\n" -"group": "mmlu_flan_cot_zeroshot_stem" +"tag": "mmlu_flan_cot_zeroshot_stem" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_college_physics" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_computer_security.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_computer_security.yaml index ae4bda965ef4dd839b400959b391a71f1fcddcd3..78216a44778fa0f9f1e057d5dc45b998fd5e87fc 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_computer_security.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_computer_security.yaml @@ -1,6 +1,6 @@ "dataset_name": "computer_security" "description": "The following are multiple choice questions (with answers) about computer\ \ security.\n\n" -"group": "mmlu_flan_cot_zeroshot_stem" +"tag": "mmlu_flan_cot_zeroshot_stem" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_computer_security" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_conceptual_physics.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_conceptual_physics.yaml index 2e1e43dbad9432de41c580779108843761280313..52304bdf8eeac624c63331b259255a98866dc2ac 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_conceptual_physics.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_conceptual_physics.yaml @@ -1,6 +1,6 @@ "dataset_name": "conceptual_physics" "description": "The following are multiple choice questions (with answers) about conceptual\ \ physics.\n\n" -"group": "mmlu_flan_cot_zeroshot_stem" +"tag": "mmlu_flan_cot_zeroshot_stem" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_conceptual_physics" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_econometrics.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_econometrics.yaml index 9ff25bba4657133ff33a491c641589aed6476114..c5be81c442710f91ad3e1ca6a0651105b2f14e24 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_econometrics.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_econometrics.yaml @@ -1,6 +1,6 @@ "dataset_name": "econometrics" "description": "The following are multiple choice questions (with answers) about econometrics.\n\ \n" -"group": "mmlu_flan_cot_zeroshot_social_sciences" +"tag": "mmlu_flan_cot_zeroshot_social_sciences" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_econometrics" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_electrical_engineering.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_electrical_engineering.yaml index ca10a43e910d6fe090af53ffaf90e645e1ad69a1..934a1a20a69d987904fe9c8b605c93e4ed149309 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_electrical_engineering.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_electrical_engineering.yaml @@ -1,6 +1,6 @@ "dataset_name": "electrical_engineering" "description": "The following are multiple choice questions (with answers) about electrical\ \ engineering.\n\n" -"group": "mmlu_flan_cot_zeroshot_stem" +"tag": "mmlu_flan_cot_zeroshot_stem" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_electrical_engineering" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_elementary_mathematics.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_elementary_mathematics.yaml index 065c92d200e4df5a303cc1f4269ada803524cd89..96ec81d6a8716ad60a4b3215faa42f3c3b1396d7 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_elementary_mathematics.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_elementary_mathematics.yaml @@ -1,6 +1,6 @@ "dataset_name": "elementary_mathematics" "description": "The following are multiple choice questions (with answers) about elementary\ \ mathematics.\n\n" -"group": "mmlu_flan_cot_zeroshot_stem" +"tag": "mmlu_flan_cot_zeroshot_stem" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_elementary_mathematics" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_formal_logic.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_formal_logic.yaml index ec2d323cae468b5efd9739929a0822dfb853e233..915c96de78b68bdd2b8b8cbb26f2f8ec0ae24167 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_formal_logic.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_formal_logic.yaml @@ -1,6 +1,6 @@ "dataset_name": "formal_logic" "description": "The following are multiple choice questions (with answers) about formal\ \ logic.\n\n" -"group": "mmlu_flan_cot_zeroshot_humanities" +"tag": "mmlu_flan_cot_zeroshot_humanities" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_formal_logic" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_global_facts.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_global_facts.yaml index b1e29a3e1379f3e09c72a4c25e92855c8ba42bd3..8a1f7491590b80e784360ceb72619efe4d9568f1 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_global_facts.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_global_facts.yaml @@ -1,6 +1,6 @@ "dataset_name": "global_facts" "description": "The following are multiple choice questions (with answers) about global\ \ facts.\n\n" -"group": "mmlu_flan_cot_zeroshot_other" +"tag": "mmlu_flan_cot_zeroshot_other" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_global_facts" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_biology.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_biology.yaml index 0e5794db64588edef17d3c396f96ef870383cfa3..5c4043d9bd7e6a38d702afa7ccb4028e98001445 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_biology.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_biology.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_biology" "description": "The following are multiple choice questions (with answers) about high\ \ school biology.\n\n" -"group": "mmlu_flan_cot_zeroshot_stem" +"tag": "mmlu_flan_cot_zeroshot_stem" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_high_school_biology" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_chemistry.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_chemistry.yaml index eba398b0393383621f3d688ea5356409eb56b215..5aee89159d40e4f7c788cf670d9fa2e405d32c75 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_chemistry.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_chemistry.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_chemistry" "description": "The following are multiple choice questions (with answers) about high\ \ school chemistry.\n\n" -"group": "mmlu_flan_cot_zeroshot_stem" +"tag": "mmlu_flan_cot_zeroshot_stem" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_high_school_chemistry" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_computer_science.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_computer_science.yaml index 4a69dbb3b9ca30313c8d333763a10796d1692bca..eb3eb2134bf8e3e8b8e81f29432db3e81b5f2fcf 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_computer_science.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_computer_science.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_computer_science" "description": "The following are multiple choice questions (with answers) about high\ \ school computer science.\n\n" -"group": "mmlu_flan_cot_zeroshot_stem" +"tag": "mmlu_flan_cot_zeroshot_stem" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_high_school_computer_science" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_european_history.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_european_history.yaml index 54eafb51d385f7afd35a78d2ed8098565d1c5297..6fc261e8fe114ffc9d7be99110d659704018f159 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_european_history.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_european_history.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_european_history" "description": "The following are multiple choice questions (with answers) about high\ \ school european history.\n\n" -"group": "mmlu_flan_cot_zeroshot_humanities" +"tag": "mmlu_flan_cot_zeroshot_humanities" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_high_school_european_history" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_geography.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_geography.yaml index 0898c87664e5250530d6998337c8fc601e1b876d..baabc83a9e25b700600fe516d9a84833c32f4f29 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_geography.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_geography.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_geography" "description": "The following are multiple choice questions (with answers) about high\ \ school geography.\n\n" -"group": "mmlu_flan_cot_zeroshot_social_sciences" +"tag": "mmlu_flan_cot_zeroshot_social_sciences" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_high_school_geography" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_government_and_politics.yaml index d82fb6b01612fa74173989d6a7297d4f7521d3dd..41365c509da451280527720e651d5793d1b83960 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_government_and_politics.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_government_and_politics.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_government_and_politics" "description": "The following are multiple choice questions (with answers) about high\ \ school government and politics.\n\n" -"group": "mmlu_flan_cot_zeroshot_social_sciences" +"tag": "mmlu_flan_cot_zeroshot_social_sciences" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_high_school_government_and_politics" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_macroeconomics.yaml index b94fc2a6e356b8099d0b7a28f2ba07395d6b8599..05e62fa85cb3fdf871ec246de43d32c7a5209db1 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_macroeconomics.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_macroeconomics.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_macroeconomics" "description": "The following are multiple choice questions (with answers) about high\ \ school macroeconomics.\n\n" -"group": "mmlu_flan_cot_zeroshot_social_sciences" +"tag": "mmlu_flan_cot_zeroshot_social_sciences" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_high_school_macroeconomics" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_mathematics.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_mathematics.yaml index dff0960afbb669c59b6159b45a5a474110c0d770..c9a9ca3b3840ee7169b59a53cec4c595c783cd4e 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_mathematics.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_mathematics.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_mathematics" "description": "The following are multiple choice questions (with answers) about high\ \ school mathematics.\n\n" -"group": "mmlu_flan_cot_zeroshot_stem" +"tag": "mmlu_flan_cot_zeroshot_stem" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_high_school_mathematics" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_microeconomics.yaml index 75a08c48bd9b4c7b44df1b07064bf920b7c4f8af..2fb8639003555bdca712f3dc49ed6e463158be42 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_microeconomics.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_microeconomics.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_microeconomics" "description": "The following are multiple choice questions (with answers) about high\ \ school microeconomics.\n\n" -"group": "mmlu_flan_cot_zeroshot_social_sciences" +"tag": "mmlu_flan_cot_zeroshot_social_sciences" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_high_school_microeconomics" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_physics.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_physics.yaml index 177d42da511dde59f1baf00f1212834b483e3426..c149ef083a87f6d3eb412f9e3fb2fbd131ec4c0e 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_physics.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_physics.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_physics" "description": "The following are multiple choice questions (with answers) about high\ \ school physics.\n\n" -"group": "mmlu_flan_cot_zeroshot_stem" +"tag": "mmlu_flan_cot_zeroshot_stem" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_high_school_physics" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_psychology.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_psychology.yaml index d5d477233122391157ebe6ce3b817902c5a39712..999f9be74e2bc278a068c344030ae27f3b2c3006 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_psychology.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_psychology.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_psychology" "description": "The following are multiple choice questions (with answers) about high\ \ school psychology.\n\n" -"group": "mmlu_flan_cot_zeroshot_social_sciences" +"tag": "mmlu_flan_cot_zeroshot_social_sciences" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_high_school_psychology" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_statistics.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_statistics.yaml index b245cf9e51337a10a12ae8c9a5df6f92e0144ec0..a0f905569c82f31ec76a75505bfae64c28d72640 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_statistics.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_statistics.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_statistics" "description": "The following are multiple choice questions (with answers) about high\ \ school statistics.\n\n" -"group": "mmlu_flan_cot_zeroshot_stem" +"tag": "mmlu_flan_cot_zeroshot_stem" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_high_school_statistics" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_us_history.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_us_history.yaml index 2e187da2980912702558bab3098b363f3e68efcd..1d09cdcaa3b268d599e055f82c92779d4ecd2bcb 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_us_history.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_us_history.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_us_history" "description": "The following are multiple choice questions (with answers) about high\ \ school us history.\n\n" -"group": "mmlu_flan_cot_zeroshot_humanities" +"tag": "mmlu_flan_cot_zeroshot_humanities" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_high_school_us_history" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_world_history.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_world_history.yaml index c89dd0faa47730d507b7337abbf38e00879389b5..28a63b1b9106219486b5487b24396baf44179276 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_world_history.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_high_school_world_history.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_world_history" "description": "The following are multiple choice questions (with answers) about high\ \ school world history.\n\n" -"group": "mmlu_flan_cot_zeroshot_humanities" +"tag": "mmlu_flan_cot_zeroshot_humanities" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_high_school_world_history" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_human_aging.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_human_aging.yaml index 230781b4a5d4293f555f8fe1f6395d818de5eca9..5a71bfc38aab72f17a01e3da11fc037ce28ef033 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_human_aging.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_human_aging.yaml @@ -1,6 +1,6 @@ "dataset_name": "human_aging" "description": "The following are multiple choice questions (with answers) about human\ \ aging.\n\n" -"group": "mmlu_flan_cot_zeroshot_other" +"tag": "mmlu_flan_cot_zeroshot_other" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_human_aging" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_human_sexuality.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_human_sexuality.yaml index ed2116dd24e9804e2ae4def9fd0cd9518618cf3a..fa9b895b7331b051385a31165c725c2ef976db69 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_human_sexuality.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_human_sexuality.yaml @@ -1,6 +1,6 @@ "dataset_name": "human_sexuality" "description": "The following are multiple choice questions (with answers) about human\ \ sexuality.\n\n" -"group": "mmlu_flan_cot_zeroshot_social_sciences" +"tag": "mmlu_flan_cot_zeroshot_social_sciences" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_human_sexuality" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_international_law.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_international_law.yaml index d777e9fc81262b0ff745fb2f9c82376c669c15df..33766a464fa475a012d229c194c93fffb84942b6 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_international_law.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_international_law.yaml @@ -1,6 +1,6 @@ "dataset_name": "international_law" "description": "The following are multiple choice questions (with answers) about international\ \ law.\n\n" -"group": "mmlu_flan_cot_zeroshot_humanities" +"tag": "mmlu_flan_cot_zeroshot_humanities" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_international_law" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_jurisprudence.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_jurisprudence.yaml index 62b86dd01f37099fa29c1bdfbba5f1beb97fb509..642e6ce4f34992cb5be8b840ea481c7a389d9ce8 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_jurisprudence.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_jurisprudence.yaml @@ -1,6 +1,6 @@ "dataset_name": "jurisprudence" "description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ \n" -"group": "mmlu_flan_cot_zeroshot_humanities" +"tag": "mmlu_flan_cot_zeroshot_humanities" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_jurisprudence" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_logical_fallacies.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_logical_fallacies.yaml index 07ae843871ca58cdce30815b0088cd7f7027e0d3..12594895469fbf0644e1908e4299f93f417703e8 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_logical_fallacies.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_logical_fallacies.yaml @@ -1,6 +1,6 @@ "dataset_name": "logical_fallacies" "description": "The following are multiple choice questions (with answers) about logical\ \ fallacies.\n\n" -"group": "mmlu_flan_cot_zeroshot_humanities" +"tag": "mmlu_flan_cot_zeroshot_humanities" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_logical_fallacies" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_machine_learning.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_machine_learning.yaml index cd4813effe182c7e3fc8270d23fd1f3e7d882836..0c27feea94ce017e35bcd453d6cbf5c4db5b3334 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_machine_learning.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_machine_learning.yaml @@ -1,6 +1,6 @@ "dataset_name": "machine_learning" "description": "The following are multiple choice questions (with answers) about machine\ \ learning.\n\n" -"group": "mmlu_flan_cot_zeroshot_stem" +"tag": "mmlu_flan_cot_zeroshot_stem" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_machine_learning" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_management.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_management.yaml index b7164c1cfcb00b9359809173da72dad15383143c..f1a13763a2bd796821efa251071359ce0acbf1cf 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_management.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_management.yaml @@ -1,6 +1,6 @@ "dataset_name": "management" "description": "The following are multiple choice questions (with answers) about management.\n\ \n" -"group": "mmlu_flan_cot_zeroshot_other" +"tag": "mmlu_flan_cot_zeroshot_other" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_management" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_marketing.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_marketing.yaml index 0827f78df766331ccdf6235a4b4120a089cdd9c9..0fe6e44b7fe464396e85a53f70831bbb48ff8ece 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_marketing.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_marketing.yaml @@ -1,6 +1,6 @@ "dataset_name": "marketing" "description": "The following are multiple choice questions (with answers) about marketing.\n\ \n" -"group": "mmlu_flan_cot_zeroshot_other" +"tag": "mmlu_flan_cot_zeroshot_other" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_marketing" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_medical_genetics.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_medical_genetics.yaml index 1706ee5b4ac151a36e7a75b638d75e8194696f8d..813b6a3fe90413bd35a11f82624df600d8bf682b 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_medical_genetics.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_medical_genetics.yaml @@ -1,6 +1,6 @@ "dataset_name": "medical_genetics" "description": "The following are multiple choice questions (with answers) about medical\ \ genetics.\n\n" -"group": "mmlu_flan_cot_zeroshot_other" +"tag": "mmlu_flan_cot_zeroshot_other" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_medical_genetics" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_miscellaneous.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_miscellaneous.yaml index 295d801a470ac7671d6f75c0527b6d508a70a6f8..c2a95e892a8e6d357e6a9f771272d06422b14d1a 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_miscellaneous.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_miscellaneous.yaml @@ -1,6 +1,6 @@ "dataset_name": "miscellaneous" "description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ \n" -"group": "mmlu_flan_cot_zeroshot_other" +"tag": "mmlu_flan_cot_zeroshot_other" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_miscellaneous" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_moral_disputes.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_moral_disputes.yaml index a4595f06991b1096e49325c897fbe6f0b3eea6c2..a6a76a2a7930589f3603fa070e974116b4996e96 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_moral_disputes.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_moral_disputes.yaml @@ -1,6 +1,6 @@ "dataset_name": "moral_disputes" "description": "The following are multiple choice questions (with answers) about moral\ \ disputes.\n\n" -"group": "mmlu_flan_cot_zeroshot_humanities" +"tag": "mmlu_flan_cot_zeroshot_humanities" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_moral_disputes" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_moral_scenarios.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_moral_scenarios.yaml index a0e41ae4ea1346ac76c56bcae3682b86610fe04f..a5f8c4e6f144dcb4c0eb6881b095434c76105bb6 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_moral_scenarios.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_moral_scenarios.yaml @@ -1,6 +1,6 @@ "dataset_name": "moral_scenarios" "description": "The following are multiple choice questions (with answers) about moral\ \ scenarios.\n\n" -"group": "mmlu_flan_cot_zeroshot_humanities" +"tag": "mmlu_flan_cot_zeroshot_humanities" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_moral_scenarios" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_nutrition.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_nutrition.yaml index 4c87be43317b9a9854208d7278bf51ceb922a2cd..f0f144cb44e5218d3a70193fddca2a2883e6b1b8 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_nutrition.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_nutrition.yaml @@ -1,6 +1,6 @@ "dataset_name": "nutrition" "description": "The following are multiple choice questions (with answers) about nutrition.\n\ \n" -"group": "mmlu_flan_cot_zeroshot_other" +"tag": "mmlu_flan_cot_zeroshot_other" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_nutrition" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_philosophy.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_philosophy.yaml index 534707cb2b08c1605f0bfeeabcbe8ab0bd372038..a4e4c0c4b6ccd34ebf4ff1133d0e26ddd8dc90d9 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_philosophy.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_philosophy.yaml @@ -1,6 +1,6 @@ "dataset_name": "philosophy" "description": "The following are multiple choice questions (with answers) about philosophy.\n\ \n" -"group": "mmlu_flan_cot_zeroshot_humanities" +"tag": "mmlu_flan_cot_zeroshot_humanities" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_philosophy" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_prehistory.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_prehistory.yaml index 3233ba4e3b7396b96ea2f3e4788d2e70b670abf4..9db801a6a9f2d911e2bdbbe0084fd235c7572776 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_prehistory.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_prehistory.yaml @@ -1,6 +1,6 @@ "dataset_name": "prehistory" "description": "The following are multiple choice questions (with answers) about prehistory.\n\ \n" -"group": "mmlu_flan_cot_zeroshot_humanities" +"tag": "mmlu_flan_cot_zeroshot_humanities" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_prehistory" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_professional_accounting.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_professional_accounting.yaml index 021090c6a0671c8a56dc60254590add42e94f917..e94bef0581e5290ff4790b5d48863a198a904879 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_professional_accounting.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_professional_accounting.yaml @@ -1,6 +1,6 @@ "dataset_name": "professional_accounting" "description": "The following are multiple choice questions (with answers) about professional\ \ accounting.\n\n" -"group": "mmlu_flan_cot_zeroshot_other" +"tag": "mmlu_flan_cot_zeroshot_other" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_professional_accounting" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_professional_law.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_professional_law.yaml index 73d115d73c03e608117f7d3f6bf17efd688041ba..25239d9a35941d49797c15986cc43213b0ec74d6 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_professional_law.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_professional_law.yaml @@ -1,6 +1,6 @@ "dataset_name": "professional_law" "description": "The following are multiple choice questions (with answers) about professional\ \ law.\n\n" -"group": "mmlu_flan_cot_zeroshot_humanities" +"tag": "mmlu_flan_cot_zeroshot_humanities" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_professional_law" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_professional_medicine.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_professional_medicine.yaml index 47cf9573d08e1f911af7fbfbd984358fecd275d9..4f961bff89745dd8999c2ee497bdf9a7df88e04f 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_professional_medicine.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_professional_medicine.yaml @@ -1,6 +1,6 @@ "dataset_name": "professional_medicine" "description": "The following are multiple choice questions (with answers) about professional\ \ medicine.\n\n" -"group": "mmlu_flan_cot_zeroshot_other" +"tag": "mmlu_flan_cot_zeroshot_other" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_professional_medicine" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_professional_psychology.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_professional_psychology.yaml index cc055d5bacacf77b1dd5f70b68bbaa81d1aad2ff..48758ef76eaf72e4236a8569e041ea03e6626e67 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_professional_psychology.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_professional_psychology.yaml @@ -1,6 +1,6 @@ "dataset_name": "professional_psychology" "description": "The following are multiple choice questions (with answers) about professional\ \ psychology.\n\n" -"group": "mmlu_flan_cot_zeroshot_social_sciences" +"tag": "mmlu_flan_cot_zeroshot_social_sciences" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_professional_psychology" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_public_relations.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_public_relations.yaml index 14d02c3a3c015e78cb780c646461fb7ac70a5ce4..62a56a4478bf9eafbcf1a8034abfeea6240e99ca 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_public_relations.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_public_relations.yaml @@ -1,6 +1,6 @@ "dataset_name": "public_relations" "description": "The following are multiple choice questions (with answers) about public\ \ relations.\n\n" -"group": "mmlu_flan_cot_zeroshot_social_sciences" +"tag": "mmlu_flan_cot_zeroshot_social_sciences" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_public_relations" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_security_studies.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_security_studies.yaml index cae551e2adf6e28153c612f7059f527c116f3d10..062f49630e82b66be1ea0e75ed9fe73c8d635215 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_security_studies.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_security_studies.yaml @@ -1,6 +1,6 @@ "dataset_name": "security_studies" "description": "The following are multiple choice questions (with answers) about security\ \ studies.\n\n" -"group": "mmlu_flan_cot_zeroshot_social_sciences" +"tag": "mmlu_flan_cot_zeroshot_social_sciences" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_security_studies" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_sociology.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_sociology.yaml index 45b94193c55ac43e7ee6dc33462e128748a68c21..36b4711831ef6fafde0915178e28513692f9c8d5 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_sociology.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_sociology.yaml @@ -1,6 +1,6 @@ "dataset_name": "sociology" "description": "The following are multiple choice questions (with answers) about sociology.\n\ \n" -"group": "mmlu_flan_cot_zeroshot_social_sciences" +"tag": "mmlu_flan_cot_zeroshot_social_sciences" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_sociology" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_us_foreign_policy.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_us_foreign_policy.yaml index 52e482775f7f5873ccf19628a1e4bb85baf4b2b0..c4afb8f84a193442cd98a856ada7e43f1515cbce 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_us_foreign_policy.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_us_foreign_policy.yaml @@ -1,6 +1,6 @@ "dataset_name": "us_foreign_policy" "description": "The following are multiple choice questions (with answers) about us\ \ foreign policy.\n\n" -"group": "mmlu_flan_cot_zeroshot_social_sciences" +"tag": "mmlu_flan_cot_zeroshot_social_sciences" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_us_foreign_policy" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_virology.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_virology.yaml index fda1af06360f4f753dee2f3ca6aa4f2720558965..a8e427612f45461a5d873edbafb3d6e0eba4e9f1 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_virology.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_virology.yaml @@ -1,6 +1,6 @@ "dataset_name": "virology" "description": "The following are multiple choice questions (with answers) about virology.\n\ \n" -"group": "mmlu_flan_cot_zeroshot_other" +"tag": "mmlu_flan_cot_zeroshot_other" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_virology" diff --git a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_world_religions.yaml b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_world_religions.yaml index 4051828232c392e69408f8ad71d4bbbbc41a260f..0eb04f31f0baaf6ac0f358de2897d5267e1a4357 100644 --- a/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_world_religions.yaml +++ b/lm_eval/tasks/mmlu/flan_cot_zeroshot/mmlu_world_religions.yaml @@ -1,6 +1,6 @@ "dataset_name": "world_religions" "description": "The following are multiple choice questions (with answers) about world\ \ religions.\n\n" -"group": "mmlu_flan_cot_zeroshot_humanities" +"tag": "mmlu_flan_cot_zeroshot_humanities" "include": "_mmlu_flan_cot_zeroshot_template_yaml" "task": "mmlu_flan_cot_zeroshot_world_religions" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/_mmlu.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/_mmlu.yaml index 7705a171c32e73a31693cca354d0baeaf98b64b8..14465ad6e5c5434974832399ea95903b59e4eaf5 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/_mmlu.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/_mmlu.yaml @@ -1,6 +1,32 @@ group: mmlu_flan_n_shot_generative +group_alias: mmlu (flan style, generative) task: - - mmlu_flan_n_shot_generative_stem - - mmlu_flan_n_shot_generative_other - - mmlu_flan_n_shot_generative_social_sciences - - mmlu_flan_n_shot_generative_humanities + - group: stem + task: + - mmlu_flan_n_shot_generative_stem + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: other + task: + - mmlu_flan_n_shot_generative_other + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: social sciences + task: + - mmlu_flan_n_shot_generative_social_sciences + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: humanities + task: + - mmlu_flan_n_shot_generative_humanities + aggregate_metric_list: + - metric: acc + weight_by_size: True +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 2 diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/_mmlu_flan_generative_template_yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/_mmlu_flan_generative_template_yaml index 28be9cfccabdf314db5bdcfdf6346172420b986b..a38a06969e2649d2fc0cf8e2be3efc60d91b3076 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/_mmlu_flan_generative_template_yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/_mmlu_flan_generative_template_yaml @@ -1,4 +1,3 @@ -group: mmlu_flan_n_shot_generative dataset_path: hails/mmlu_no_train # a copy of `cais/mmlu` with no auxiliary_train split test_split: test fewshot_split: dev @@ -13,7 +12,7 @@ filter_list: - function: "take_first" - name: "flexible-extract" filter: - - function: !function utils.MultiChoiceRegexFilter + - function: "multi_choice_regex" group_select: 0 regex_pattern: "(\\([A-Z]\\))" ignore_case: true @@ -30,6 +29,6 @@ metric_list: aggregation: mean higher_is_better: true metadata: - version: 2.0 + version: 3.0 dataset_kwargs: trust_remote_code: true diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_abstract_algebra.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_abstract_algebra.yaml index 40cced2c583fd71fdf6f425b4637a04daad7b52b..3e602ee8100ed612d89385532ea30004c3033c35 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_abstract_algebra.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_abstract_algebra.yaml @@ -1,6 +1,6 @@ "dataset_name": "abstract_algebra" "description": "The following are multiple choice questions (with answers) about abstract\ \ algebra.\n\n" -"group": "mmlu_flan_n_shot_generative_stem" +"tag": "mmlu_flan_n_shot_generative_stem" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_abstract_algebra" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_anatomy.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_anatomy.yaml index 606049a506ac86fa8f3950216e05b34f1f1bc612..fa12cc8ef35b19f3b81dcc58a0107d424a3580cc 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_anatomy.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_anatomy.yaml @@ -1,6 +1,6 @@ "dataset_name": "anatomy" "description": "The following are multiple choice questions (with answers) about anatomy.\n\ \n" -"group": "mmlu_flan_n_shot_generative_stem" +"tag": "mmlu_flan_n_shot_generative_stem" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_anatomy" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_astronomy.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_astronomy.yaml index db5faa222dc34b2db9b56333dda2c2326a394365..a4178654e0e6e7a053839319c7936967133cf756 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_astronomy.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_astronomy.yaml @@ -1,6 +1,6 @@ "dataset_name": "astronomy" "description": "The following are multiple choice questions (with answers) about astronomy.\n\ \n" -"group": "mmlu_flan_n_shot_generative_stem" +"tag": "mmlu_flan_n_shot_generative_stem" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_astronomy" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_business_ethics.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_business_ethics.yaml index add2ffb46b10712593c4c28b622f32d6db3bcaf8..4447d276b066ddec93b8f7efcf2d74d13810f458 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_business_ethics.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_business_ethics.yaml @@ -1,6 +1,6 @@ "dataset_name": "business_ethics" "description": "The following are multiple choice questions (with answers) about business\ \ ethics.\n\n" -"group": "mmlu_flan_n_shot_generative_other" +"tag": "mmlu_flan_n_shot_generative_other" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_business_ethics" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_clinical_knowledge.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_clinical_knowledge.yaml index e3f24569c2d1bf4a99f21417deec009d51cffdb1..38f799060fa6901b890d3a87d8aa9b9444d34b57 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_clinical_knowledge.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_clinical_knowledge.yaml @@ -1,6 +1,6 @@ "dataset_name": "clinical_knowledge" "description": "The following are multiple choice questions (with answers) about clinical\ \ knowledge.\n\n" -"group": "mmlu_flan_n_shot_generative_other" +"tag": "mmlu_flan_n_shot_generative_other" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_clinical_knowledge" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_biology.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_biology.yaml index 3772b0e6411ec48f7ee0b68fe15ba7288cfde811..f36eb1f598f754154c2b15b24bbb650358c707c5 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_biology.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_biology.yaml @@ -1,6 +1,6 @@ "dataset_name": "college_biology" "description": "The following are multiple choice questions (with answers) about college\ \ biology.\n\n" -"group": "mmlu_flan_n_shot_generative_stem" +"tag": "mmlu_flan_n_shot_generative_stem" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_college_biology" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_chemistry.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_chemistry.yaml index cedcf0cc2ffc855be657dc7fa2869b6c3a4e9c76..0eccce652fade13a319af78e06a7528b11814302 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_chemistry.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_chemistry.yaml @@ -1,6 +1,6 @@ "dataset_name": "college_chemistry" "description": "The following are multiple choice questions (with answers) about college\ \ chemistry.\n\n" -"group": "mmlu_flan_n_shot_generative_stem" +"tag": "mmlu_flan_n_shot_generative_stem" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_college_chemistry" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_computer_science.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_computer_science.yaml index a060903a97c31f701fb9a2158df072120d38e46a..fd415aa10efaf96331d9fef82c5b6a2bb538263a 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_computer_science.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_computer_science.yaml @@ -1,6 +1,6 @@ "dataset_name": "college_computer_science" "description": "The following are multiple choice questions (with answers) about college\ \ computer science.\n\n" -"group": "mmlu_flan_n_shot_generative_stem" +"tag": "mmlu_flan_n_shot_generative_stem" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_college_computer_science" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_mathematics.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_mathematics.yaml index 1899ce6540bcacdc29922d8152cbc72da1a058da..2d062721102c0f6e6c09574398a60db74c26b593 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_mathematics.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_mathematics.yaml @@ -1,6 +1,6 @@ "dataset_name": "college_mathematics" "description": "The following are multiple choice questions (with answers) about college\ \ mathematics.\n\n" -"group": "mmlu_flan_n_shot_generative_stem" +"tag": "mmlu_flan_n_shot_generative_stem" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_college_mathematics" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_medicine.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_medicine.yaml index ab052dd69e6d0f10a35fbc21cf7289f142066a21..edc660d9c30dfad6666f5e1b4c679489f62c5991 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_medicine.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_medicine.yaml @@ -1,6 +1,6 @@ "dataset_name": "college_medicine" "description": "The following are multiple choice questions (with answers) about college\ \ medicine.\n\n" -"group": "mmlu_flan_n_shot_generative_other" +"tag": "mmlu_flan_n_shot_generative_other" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_college_medicine" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_physics.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_physics.yaml index 3b1e64e640ba8ab8625f02d3636b24770caac001..aac8f400d1d9005376bfe3354753e87700a7bda8 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_physics.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_college_physics.yaml @@ -1,6 +1,6 @@ "dataset_name": "college_physics" "description": "The following are multiple choice questions (with answers) about college\ \ physics.\n\n" -"group": "mmlu_flan_n_shot_generative_stem" +"tag": "mmlu_flan_n_shot_generative_stem" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_college_physics" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_computer_security.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_computer_security.yaml index cd312a93a35d32ff5df23896d55106f80315ca96..178c468346a5022a5d0031fd27c6b9a07ab24150 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_computer_security.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_computer_security.yaml @@ -1,6 +1,6 @@ "dataset_name": "computer_security" "description": "The following are multiple choice questions (with answers) about computer\ \ security.\n\n" -"group": "mmlu_flan_n_shot_generative_stem" +"tag": "mmlu_flan_n_shot_generative_stem" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_computer_security" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_conceptual_physics.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_conceptual_physics.yaml index 49e6b38c5b82a0afa099fcca02941e743bd3ce60..e3cfbe6250d19aab6e60c9089f0feb91eed37423 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_conceptual_physics.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_conceptual_physics.yaml @@ -1,6 +1,6 @@ "dataset_name": "conceptual_physics" "description": "The following are multiple choice questions (with answers) about conceptual\ \ physics.\n\n" -"group": "mmlu_flan_n_shot_generative_stem" +"tag": "mmlu_flan_n_shot_generative_stem" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_conceptual_physics" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_econometrics.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_econometrics.yaml index d9b4ebfcdd5d23fc606e50bce368c0b4db545428..ad8704e4f8e3a60ee2ff7e370cf7394c0359aeb7 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_econometrics.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_econometrics.yaml @@ -1,6 +1,6 @@ "dataset_name": "econometrics" "description": "The following are multiple choice questions (with answers) about econometrics.\n\ \n" -"group": "mmlu_flan_n_shot_generative_social_sciences" +"tag": "mmlu_flan_n_shot_generative_social_sciences" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_econometrics" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_electrical_engineering.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_electrical_engineering.yaml index e6efb0ef202bc6a5df3d041f02a0abaa10a6c957..56eeae0183ca0c087b0a16aa317f2b93d5f1b87b 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_electrical_engineering.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_electrical_engineering.yaml @@ -1,6 +1,6 @@ "dataset_name": "electrical_engineering" "description": "The following are multiple choice questions (with answers) about electrical\ \ engineering.\n\n" -"group": "mmlu_flan_n_shot_generative_stem" +"tag": "mmlu_flan_n_shot_generative_stem" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_electrical_engineering" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_elementary_mathematics.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_elementary_mathematics.yaml index b33cf3188391c328495536e671c58924578d4049..da3b3af2b5f310232cbd9c9ee63081acbb571638 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_elementary_mathematics.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_elementary_mathematics.yaml @@ -1,6 +1,6 @@ "dataset_name": "elementary_mathematics" "description": "The following are multiple choice questions (with answers) about elementary\ \ mathematics.\n\n" -"group": "mmlu_flan_n_shot_generative_stem" +"tag": "mmlu_flan_n_shot_generative_stem" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_elementary_mathematics" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_formal_logic.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_formal_logic.yaml index 1c2ad3a1b6fc4f5f407e13645b77cb4da61d403d..2d3f4edc644842cbc3fae865c96f99322daaafbf 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_formal_logic.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_formal_logic.yaml @@ -1,6 +1,6 @@ "dataset_name": "formal_logic" "description": "The following are multiple choice questions (with answers) about formal\ \ logic.\n\n" -"group": "mmlu_flan_n_shot_generative_humanities" +"tag": "mmlu_flan_n_shot_generative_humanities" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_formal_logic" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_global_facts.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_global_facts.yaml index a2352ff7ecb45b1724b7588582fc65db9a4f1e75..4feef1895254438bde19ebfc3d7a36aee87e61de 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_global_facts.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_global_facts.yaml @@ -1,6 +1,6 @@ "dataset_name": "global_facts" "description": "The following are multiple choice questions (with answers) about global\ \ facts.\n\n" -"group": "mmlu_flan_n_shot_generative_other" +"tag": "mmlu_flan_n_shot_generative_other" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_global_facts" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_biology.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_biology.yaml index 0b51f34a90b4dff6465b37d39d5f5dab7b2866c6..245d9be815c3644bf3298a0d093a76410b7487b6 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_biology.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_biology.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_biology" "description": "The following are multiple choice questions (with answers) about high\ \ school biology.\n\n" -"group": "mmlu_flan_n_shot_generative_stem" +"tag": "mmlu_flan_n_shot_generative_stem" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_high_school_biology" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_chemistry.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_chemistry.yaml index 0066ba7716c6316d911d273cdffd0a21fb019bdf..34eb30d32d5b6927d44d59a63f5a549587f414f1 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_chemistry.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_chemistry.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_chemistry" "description": "The following are multiple choice questions (with answers) about high\ \ school chemistry.\n\n" -"group": "mmlu_flan_n_shot_generative_stem" +"tag": "mmlu_flan_n_shot_generative_stem" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_high_school_chemistry" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_computer_science.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_computer_science.yaml index b0d4ef155befae06523140e408ebbe6d2e88f4b3..34250a6c61cb5e29acbb99f8a080d45f74a91d45 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_computer_science.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_computer_science.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_computer_science" "description": "The following are multiple choice questions (with answers) about high\ \ school computer science.\n\n" -"group": "mmlu_flan_n_shot_generative_stem" +"tag": "mmlu_flan_n_shot_generative_stem" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_high_school_computer_science" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_european_history.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_european_history.yaml index 4b17db63997a395704060e1390180526a44b1ecc..42b7dd4d5aa2ab541b7f269c84845d262db452c5 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_european_history.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_european_history.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_european_history" "description": "The following are multiple choice questions (with answers) about high\ \ school european history.\n\n" -"group": "mmlu_flan_n_shot_generative_humanities" +"tag": "mmlu_flan_n_shot_generative_humanities" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_high_school_european_history" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_geography.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_geography.yaml index 93f8de208581eef0c20dd612c3d5925fa9963dbf..e67277aa5480e1a9465169112755c3da70e12e6e 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_geography.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_geography.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_geography" "description": "The following are multiple choice questions (with answers) about high\ \ school geography.\n\n" -"group": "mmlu_flan_n_shot_generative_social_sciences" +"tag": "mmlu_flan_n_shot_generative_social_sciences" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_high_school_geography" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_government_and_politics.yaml index 7ae12c1744008b00d8b786a5075295b2d4ce8f81..84643a74239db620816f0d8a67575d0c8268e58f 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_government_and_politics.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_government_and_politics.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_government_and_politics" "description": "The following are multiple choice questions (with answers) about high\ \ school government and politics.\n\n" -"group": "mmlu_flan_n_shot_generative_social_sciences" +"tag": "mmlu_flan_n_shot_generative_social_sciences" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_high_school_government_and_politics" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_macroeconomics.yaml index 71d822595f4699348f1aca2d47cb0bd9529d5e6f..eb08333804237ac3e0584db637d5c91477a6a93d 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_macroeconomics.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_macroeconomics.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_macroeconomics" "description": "The following are multiple choice questions (with answers) about high\ \ school macroeconomics.\n\n" -"group": "mmlu_flan_n_shot_generative_social_sciences" +"tag": "mmlu_flan_n_shot_generative_social_sciences" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_high_school_macroeconomics" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_mathematics.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_mathematics.yaml index 20d31e12cce9944cc53e914301dcb44984b7674a..f1ca028d8262f22807eb591c3e498fecabd9887b 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_mathematics.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_mathematics.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_mathematics" "description": "The following are multiple choice questions (with answers) about high\ \ school mathematics.\n\n" -"group": "mmlu_flan_n_shot_generative_stem" +"tag": "mmlu_flan_n_shot_generative_stem" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_high_school_mathematics" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_microeconomics.yaml index 5c6d6ef9f6ab6faf79e324652ea22a7bce2f84e7..c60982b78dab4866a6827fe5b1bf9f2b710ed8d3 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_microeconomics.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_microeconomics.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_microeconomics" "description": "The following are multiple choice questions (with answers) about high\ \ school microeconomics.\n\n" -"group": "mmlu_flan_n_shot_generative_social_sciences" +"tag": "mmlu_flan_n_shot_generative_social_sciences" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_high_school_microeconomics" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_physics.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_physics.yaml index 5b016778ac9333281b7e061b1a8413515df814a6..33b8d16739c9faf352ad242bd76b2bc33bc21aa6 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_physics.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_physics.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_physics" "description": "The following are multiple choice questions (with answers) about high\ \ school physics.\n\n" -"group": "mmlu_flan_n_shot_generative_stem" +"tag": "mmlu_flan_n_shot_generative_stem" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_high_school_physics" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_psychology.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_psychology.yaml index 1abf244c102484600de19995725959641b73b383..f47bbbb68c02a417e60e5b0a19f4f85c5723b41b 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_psychology.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_psychology.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_psychology" "description": "The following are multiple choice questions (with answers) about high\ \ school psychology.\n\n" -"group": "mmlu_flan_n_shot_generative_social_sciences" +"tag": "mmlu_flan_n_shot_generative_social_sciences" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_high_school_psychology" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_statistics.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_statistics.yaml index df3e8d9300002f309b7cdc792e5583d3c2769da7..741971895ba27ad6651ac456def204a078ac5d3e 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_statistics.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_statistics.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_statistics" "description": "The following are multiple choice questions (with answers) about high\ \ school statistics.\n\n" -"group": "mmlu_flan_n_shot_generative_stem" +"tag": "mmlu_flan_n_shot_generative_stem" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_high_school_statistics" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_us_history.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_us_history.yaml index 68e3f0a97e14bdad7ae7456e28db627b8495797b..48696971c9e850a18baadd6c3e9f958851cc2a3e 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_us_history.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_us_history.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_us_history" "description": "The following are multiple choice questions (with answers) about high\ \ school us history.\n\n" -"group": "mmlu_flan_n_shot_generative_humanities" +"tag": "mmlu_flan_n_shot_generative_humanities" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_high_school_us_history" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_world_history.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_world_history.yaml index dfb839c314246b34607c3a68038ce71a64b1ff96..ae6cfcbba3f86dc0339edc3a361c898e6c8716fd 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_world_history.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_high_school_world_history.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_world_history" "description": "The following are multiple choice questions (with answers) about high\ \ school world history.\n\n" -"group": "mmlu_flan_n_shot_generative_humanities" +"tag": "mmlu_flan_n_shot_generative_humanities" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_high_school_world_history" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_human_aging.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_human_aging.yaml index a857698f7770454d7ad9e7bd402a5a2f2a921af6..677f119a754f0c671fae0f2285bb8ff29f2af85e 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_human_aging.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_human_aging.yaml @@ -1,6 +1,6 @@ "dataset_name": "human_aging" "description": "The following are multiple choice questions (with answers) about human\ \ aging.\n\n" -"group": "mmlu_flan_n_shot_generative_other" +"tag": "mmlu_flan_n_shot_generative_other" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_human_aging" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_human_sexuality.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_human_sexuality.yaml index 3dde3af5a4f718cfbbc1d76e963fd568f3084279..d4e33d7d607ef2f07ea0fdb67305b8f88a45d13a 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_human_sexuality.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_human_sexuality.yaml @@ -1,6 +1,6 @@ "dataset_name": "human_sexuality" "description": "The following are multiple choice questions (with answers) about human\ \ sexuality.\n\n" -"group": "mmlu_flan_n_shot_generative_social_sciences" +"tag": "mmlu_flan_n_shot_generative_social_sciences" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_human_sexuality" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_international_law.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_international_law.yaml index be9018cfdb5023350aa78a30c576c42feb0627a9..ac5d9d5a46b7f4f1daafb7c7f0feb66933c4829d 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_international_law.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_international_law.yaml @@ -1,6 +1,6 @@ "dataset_name": "international_law" "description": "The following are multiple choice questions (with answers) about international\ \ law.\n\n" -"group": "mmlu_flan_n_shot_generative_humanities" +"tag": "mmlu_flan_n_shot_generative_humanities" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_international_law" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_jurisprudence.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_jurisprudence.yaml index f6b3c7aeabe7cec4b984d966971d03872d04cef2..c2f135869aca516492cd9dc8ce210838173a1d7a 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_jurisprudence.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_jurisprudence.yaml @@ -1,6 +1,6 @@ "dataset_name": "jurisprudence" "description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ \n" -"group": "mmlu_flan_n_shot_generative_humanities" +"tag": "mmlu_flan_n_shot_generative_humanities" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_jurisprudence" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_logical_fallacies.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_logical_fallacies.yaml index c6363390211d1ca822c3299b2bbd2e7fcb36fff4..6624e07743a432cc354ccff7af2363db2ec1ae11 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_logical_fallacies.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_logical_fallacies.yaml @@ -1,6 +1,6 @@ "dataset_name": "logical_fallacies" "description": "The following are multiple choice questions (with answers) about logical\ \ fallacies.\n\n" -"group": "mmlu_flan_n_shot_generative_humanities" +"tag": "mmlu_flan_n_shot_generative_humanities" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_logical_fallacies" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_machine_learning.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_machine_learning.yaml index 64496cfb07007054b60a746a1ae3196f2d53abfc..ab6c459ae50e7311dc9d8819ec753c69f6d9583b 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_machine_learning.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_machine_learning.yaml @@ -1,6 +1,6 @@ "dataset_name": "machine_learning" "description": "The following are multiple choice questions (with answers) about machine\ \ learning.\n\n" -"group": "mmlu_flan_n_shot_generative_stem" +"tag": "mmlu_flan_n_shot_generative_stem" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_machine_learning" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_management.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_management.yaml index 63292cc18cc62dbb1dfe4c6f16b051c23447f583..4af9ded012e921feeb38d31cde98fef9888aba95 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_management.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_management.yaml @@ -1,6 +1,6 @@ "dataset_name": "management" "description": "The following are multiple choice questions (with answers) about management.\n\ \n" -"group": "mmlu_flan_n_shot_generative_other" +"tag": "mmlu_flan_n_shot_generative_other" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_management" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_marketing.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_marketing.yaml index 0716dc144e3d01f2991c7126a2d47c53deb52bc9..22ef9d3fd49556afd4578685099abc0bb9b64c9e 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_marketing.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_marketing.yaml @@ -1,6 +1,6 @@ "dataset_name": "marketing" "description": "The following are multiple choice questions (with answers) about marketing.\n\ \n" -"group": "mmlu_flan_n_shot_generative_other" +"tag": "mmlu_flan_n_shot_generative_other" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_marketing" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_medical_genetics.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_medical_genetics.yaml index 92115979b24e64d2314808276bd2772247570dd6..c24da7938b431acdd991830424777e6645cf9bbb 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_medical_genetics.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_medical_genetics.yaml @@ -1,6 +1,6 @@ "dataset_name": "medical_genetics" "description": "The following are multiple choice questions (with answers) about medical\ \ genetics.\n\n" -"group": "mmlu_flan_n_shot_generative_other" +"tag": "mmlu_flan_n_shot_generative_other" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_medical_genetics" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_miscellaneous.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_miscellaneous.yaml index 74e889444a157feadfb0a9f183639452e8caaf74..c5b90845321c954cc2e7875fdc084e5935444af7 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_miscellaneous.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_miscellaneous.yaml @@ -1,6 +1,6 @@ "dataset_name": "miscellaneous" "description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ \n" -"group": "mmlu_flan_n_shot_generative_other" +"tag": "mmlu_flan_n_shot_generative_other" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_miscellaneous" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_moral_disputes.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_moral_disputes.yaml index 58bf43dfa819e83af629cc17b33ac6b49a5404dd..295c39a6efce509983b01b18c20375866b08d3bc 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_moral_disputes.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_moral_disputes.yaml @@ -1,6 +1,6 @@ "dataset_name": "moral_disputes" "description": "The following are multiple choice questions (with answers) about moral\ \ disputes.\n\n" -"group": "mmlu_flan_n_shot_generative_humanities" +"tag": "mmlu_flan_n_shot_generative_humanities" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_moral_disputes" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_moral_scenarios.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_moral_scenarios.yaml index 9630e517b03d2695214f95c5c33501b4a48ba6d5..f09f982f26462304a20420e9b61bf3ef941448a0 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_moral_scenarios.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_moral_scenarios.yaml @@ -1,6 +1,6 @@ "dataset_name": "moral_scenarios" "description": "The following are multiple choice questions (with answers) about moral\ \ scenarios.\n\n" -"group": "mmlu_flan_n_shot_generative_humanities" +"tag": "mmlu_flan_n_shot_generative_humanities" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_moral_scenarios" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_nutrition.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_nutrition.yaml index df14da9daf3bae25c95f685d06eba25fe6e740c5..cf633f270a6d9fbbaa0a793bc5d5e48731a31d57 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_nutrition.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_nutrition.yaml @@ -1,6 +1,6 @@ "dataset_name": "nutrition" "description": "The following are multiple choice questions (with answers) about nutrition.\n\ \n" -"group": "mmlu_flan_n_shot_generative_other" +"tag": "mmlu_flan_n_shot_generative_other" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_nutrition" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_philosophy.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_philosophy.yaml index 20f5d60bd040853a7de055ef9804238944b16807..6a5fe27eefb47badf4c13e87ad0fbac96b08283e 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_philosophy.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_philosophy.yaml @@ -1,6 +1,6 @@ "dataset_name": "philosophy" "description": "The following are multiple choice questions (with answers) about philosophy.\n\ \n" -"group": "mmlu_flan_n_shot_generative_humanities" +"tag": "mmlu_flan_n_shot_generative_humanities" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_philosophy" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_prehistory.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_prehistory.yaml index 3695e77055f40403a4aa384ddb20084bb5302359..60788fc6c201bf316398f48adc9575dcb806b649 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_prehistory.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_prehistory.yaml @@ -1,6 +1,6 @@ "dataset_name": "prehistory" "description": "The following are multiple choice questions (with answers) about prehistory.\n\ \n" -"group": "mmlu_flan_n_shot_generative_humanities" +"tag": "mmlu_flan_n_shot_generative_humanities" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_prehistory" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_professional_accounting.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_professional_accounting.yaml index 222642ac43c9c896249e7975a3a79dc70f689c78..f312af231f28d9343f7a0e2353cec110fda1f9a4 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_professional_accounting.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_professional_accounting.yaml @@ -1,6 +1,6 @@ "dataset_name": "professional_accounting" "description": "The following are multiple choice questions (with answers) about professional\ \ accounting.\n\n" -"group": "mmlu_flan_n_shot_generative_other" +"tag": "mmlu_flan_n_shot_generative_other" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_professional_accounting" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_professional_law.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_professional_law.yaml index b4d39e4934d39ce0b926808cfaf8ffcff3694c1a..be0533f0d8b90fc9f82226579ec849ac3f24be15 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_professional_law.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_professional_law.yaml @@ -1,6 +1,6 @@ "dataset_name": "professional_law" "description": "The following are multiple choice questions (with answers) about professional\ \ law.\n\n" -"group": "mmlu_flan_n_shot_generative_humanities" +"tag": "mmlu_flan_n_shot_generative_humanities" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_professional_law" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_professional_medicine.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_professional_medicine.yaml index c420d0d3607a30e7edf13bfa2afb36dd3bce364b..9cae6f8a5ec27d73bcf9b57e8597b377aee62835 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_professional_medicine.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_professional_medicine.yaml @@ -1,6 +1,6 @@ "dataset_name": "professional_medicine" "description": "The following are multiple choice questions (with answers) about professional\ \ medicine.\n\n" -"group": "mmlu_flan_n_shot_generative_other" +"tag": "mmlu_flan_n_shot_generative_other" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_professional_medicine" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_professional_psychology.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_professional_psychology.yaml index c5ba7495324a3194032983f9bf7430abfb429b69..21a39c51b7d246c3dd49e47ee0f5dd1865059c36 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_professional_psychology.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_professional_psychology.yaml @@ -1,6 +1,6 @@ "dataset_name": "professional_psychology" "description": "The following are multiple choice questions (with answers) about professional\ \ psychology.\n\n" -"group": "mmlu_flan_n_shot_generative_social_sciences" +"tag": "mmlu_flan_n_shot_generative_social_sciences" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_professional_psychology" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_public_relations.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_public_relations.yaml index 9aa7d686f0a0d918838b59c7b4408374d1bce1c4..b2687d99a279caac3f322ff178a1ea1ac7ea44f8 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_public_relations.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_public_relations.yaml @@ -1,6 +1,6 @@ "dataset_name": "public_relations" "description": "The following are multiple choice questions (with answers) about public\ \ relations.\n\n" -"group": "mmlu_flan_n_shot_generative_social_sciences" +"tag": "mmlu_flan_n_shot_generative_social_sciences" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_public_relations" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_security_studies.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_security_studies.yaml index 6d2e0cdaba7b02415346ac8ff0e0dee4626c6e0a..6c36a5522d3c0d6f165dbd5eaac9f5208822fb9d 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_security_studies.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_security_studies.yaml @@ -1,6 +1,6 @@ "dataset_name": "security_studies" "description": "The following are multiple choice questions (with answers) about security\ \ studies.\n\n" -"group": "mmlu_flan_n_shot_generative_social_sciences" +"tag": "mmlu_flan_n_shot_generative_social_sciences" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_security_studies" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_sociology.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_sociology.yaml index 3c42d0b9bfef8c4f7c021712155249fa05a13200..7ce0809907575855a8680ec1db533688ad42de46 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_sociology.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_sociology.yaml @@ -1,6 +1,6 @@ "dataset_name": "sociology" "description": "The following are multiple choice questions (with answers) about sociology.\n\ \n" -"group": "mmlu_flan_n_shot_generative_social_sciences" +"tag": "mmlu_flan_n_shot_generative_social_sciences" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_sociology" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_us_foreign_policy.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_us_foreign_policy.yaml index 5c51472532d22ee6706847d028cb711c92356431..56ed5e16281b6aca3720868538c93d2877d438b6 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_us_foreign_policy.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_us_foreign_policy.yaml @@ -1,6 +1,6 @@ "dataset_name": "us_foreign_policy" "description": "The following are multiple choice questions (with answers) about us\ \ foreign policy.\n\n" -"group": "mmlu_flan_n_shot_generative_social_sciences" +"tag": "mmlu_flan_n_shot_generative_social_sciences" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_us_foreign_policy" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_virology.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_virology.yaml index fb083b62b3b7d4889f0c22cf56ec43edc9732293..257dcfbf8a18c96d836d6db1214e8ff69ec63278 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_virology.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_virology.yaml @@ -1,6 +1,6 @@ "dataset_name": "virology" "description": "The following are multiple choice questions (with answers) about virology.\n\ \n" -"group": "mmlu_flan_n_shot_generative_other" +"tag": "mmlu_flan_n_shot_generative_other" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_virology" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_world_religions.yaml b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_world_religions.yaml index 0f2c199a09beb9287a6e4b6c7e671f6be522e06d..39b64d03d3983f5c692a1a762c8457175dbf5408 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_world_religions.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/generative/mmlu_world_religions.yaml @@ -1,6 +1,6 @@ "dataset_name": "world_religions" "description": "The following are multiple choice questions (with answers) about world\ \ religions.\n\n" -"group": "mmlu_flan_n_shot_generative_humanities" +"tag": "mmlu_flan_n_shot_generative_humanities" "include": "_mmlu_flan_generative_template_yaml" "task": "mmlu_flan_n_shot_generative_world_religions" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/_mmlu.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/_mmlu.yaml index 43c5f0076a69e39f138040880fde5c107d4535f6..2cfa0fb9c30451fa79f6b8b038a01692c830f1a7 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/_mmlu.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/_mmlu.yaml @@ -1,6 +1,32 @@ group: mmlu_flan_n_shot_loglikelihood +group_alias: mmlu (flan style, loglikelihood) task: - - mmlu_flan_n_shot_loglikelihood_stem - - mmlu_flan_n_shot_loglikelihood_other - - mmlu_flan_n_shot_loglikelihood_social_sciences - - mmlu_flan_n_shot_loglikelihood_humanities + - group: stem + task: + - mmlu_flan_n_shot_loglikelihood_stem + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: other + task: + - mmlu_flan_n_shot_loglikelihood_other + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: social sciences + task: + - mmlu_flan_n_shot_loglikelihood_social_sciences + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: humanities + task: + - mmlu_flan_n_shot_loglikelihood_humanities + aggregate_metric_list: + - metric: acc + weight_by_size: True +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 2 diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/_mmlu_flan_loglikelihood_template_yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/_mmlu_flan_loglikelihood_template_yaml index 8f223281eb1c121c4d5a8402e6fc5365a53d1cb1..4605a4a15f2e84c4572388192fc1e51d717f70b1 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/_mmlu_flan_loglikelihood_template_yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/_mmlu_flan_loglikelihood_template_yaml @@ -12,6 +12,6 @@ metric_list: aggregation: mean higher_is_better: true metadata: - version: 1.0 + version: 2.0 dataset_kwargs: trust_remote_code: true diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_abstract_algebra.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_abstract_algebra.yaml index 068898e4c3a03c1b895f0cf63e2279193fabfd86..f5dfa65ded384d6e1299b8e5564f5a655f2ced79 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_abstract_algebra.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_abstract_algebra.yaml @@ -1,6 +1,6 @@ "dataset_name": "abstract_algebra" "description": "The following are multiple choice questions (with answers) about abstract\ \ algebra.\n\n" -"group": "mmlu_flan_n_shot_loglikelihood_stem" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_abstract_algebra" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_anatomy.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_anatomy.yaml index db5fa24e5c51723fd77f010298319d8b3f915ce2..e837e5d8fd3e1577af4d23d2120d1b55029f052f 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_anatomy.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_anatomy.yaml @@ -1,6 +1,6 @@ "dataset_name": "anatomy" "description": "The following are multiple choice questions (with answers) about anatomy.\n\ \n" -"group": "mmlu_flan_n_shot_loglikelihood_stem" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_anatomy" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_astronomy.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_astronomy.yaml index 5f71dbcfa10f829b6514d652933b2cc94eb77bd2..43b9bc7ed89429c2d08cc74cc4472ebea28f67a2 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_astronomy.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_astronomy.yaml @@ -1,6 +1,6 @@ "dataset_name": "astronomy" "description": "The following are multiple choice questions (with answers) about astronomy.\n\ \n" -"group": "mmlu_flan_n_shot_loglikelihood_stem" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_astronomy" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_business_ethics.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_business_ethics.yaml index 54dc204d2431face1dcd41235b8bb3679dff3496..2438e6678be07c008922d83ea5016efab56ebc78 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_business_ethics.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_business_ethics.yaml @@ -1,6 +1,6 @@ "dataset_name": "business_ethics" "description": "The following are multiple choice questions (with answers) about business\ \ ethics.\n\n" -"group": "mmlu_flan_n_shot_loglikelihood_other" +"tag": "mmlu_flan_n_shot_loglikelihood_other" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_business_ethics" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_clinical_knowledge.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_clinical_knowledge.yaml index 121b3c22efeebfcb3fe06839f8fec1d43a6a9831..82d66adda5d600a94d5f6e36544dd63d2de3fece 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_clinical_knowledge.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_clinical_knowledge.yaml @@ -1,6 +1,6 @@ "dataset_name": "clinical_knowledge" "description": "The following are multiple choice questions (with answers) about clinical\ \ knowledge.\n\n" -"group": "mmlu_flan_n_shot_loglikelihood_other" +"tag": "mmlu_flan_n_shot_loglikelihood_other" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_clinical_knowledge" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_biology.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_biology.yaml index cadb6fb75d42815018b3f7fe241d25e6ee8021ac..15e6e75d3491dfd034df789a3481fb3a39dcaa02 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_biology.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_biology.yaml @@ -1,6 +1,6 @@ "dataset_name": "college_biology" "description": "The following are multiple choice questions (with answers) about college\ \ biology.\n\n" -"group": "mmlu_flan_n_shot_loglikelihood_stem" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_college_biology" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_chemistry.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_chemistry.yaml index 8989ed40c62c05a0ebd089cd89139bc810ea55ba..2b8c1bd3a8de310698082f738d287743d3731c23 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_chemistry.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_chemistry.yaml @@ -1,6 +1,6 @@ "dataset_name": "college_chemistry" "description": "The following are multiple choice questions (with answers) about college\ \ chemistry.\n\n" -"group": "mmlu_flan_n_shot_loglikelihood_stem" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_college_chemistry" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_computer_science.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_computer_science.yaml index e4bdbdd67206016451228bbfa7b318279f9b43dc..1178c7b072f82bebdd4281a371d6105514a686e8 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_computer_science.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_computer_science.yaml @@ -1,6 +1,6 @@ "dataset_name": "college_computer_science" "description": "The following are multiple choice questions (with answers) about college\ \ computer science.\n\n" -"group": "mmlu_flan_n_shot_loglikelihood_stem" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_college_computer_science" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_mathematics.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_mathematics.yaml index 17d0cb541babb66924e32c17b92a13679116f851..9776889b514c04c6c93aeedfd0ced7c620d11493 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_mathematics.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_mathematics.yaml @@ -1,6 +1,6 @@ "dataset_name": "college_mathematics" "description": "The following are multiple choice questions (with answers) about college\ \ mathematics.\n\n" -"group": "mmlu_flan_n_shot_loglikelihood_stem" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_college_mathematics" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_medicine.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_medicine.yaml index c45a6c9c138cbe906aa7a9207f472b1b92d8522d..c8fdad90bd103ff616b4b14c2a3e9024208e149a 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_medicine.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_medicine.yaml @@ -1,6 +1,6 @@ "dataset_name": "college_medicine" "description": "The following are multiple choice questions (with answers) about college\ \ medicine.\n\n" -"group": "mmlu_flan_n_shot_loglikelihood_other" +"tag": "mmlu_flan_n_shot_loglikelihood_other" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_college_medicine" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_physics.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_physics.yaml index d325f9796f9826f38dfae5c796d58cffef18ad2e..77a89689127b4ca129b9434653198b051324fc0a 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_physics.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_college_physics.yaml @@ -1,6 +1,6 @@ "dataset_name": "college_physics" "description": "The following are multiple choice questions (with answers) about college\ \ physics.\n\n" -"group": "mmlu_flan_n_shot_loglikelihood_stem" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_college_physics" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_computer_security.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_computer_security.yaml index 5b0a75ff1e2981e4741a2dde05f0663cd10aea1d..e787e51745218e2465b739ee82b51c456bd228ab 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_computer_security.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_computer_security.yaml @@ -1,6 +1,6 @@ "dataset_name": "computer_security" "description": "The following are multiple choice questions (with answers) about computer\ \ security.\n\n" -"group": "mmlu_flan_n_shot_loglikelihood_stem" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_computer_security" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_conceptual_physics.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_conceptual_physics.yaml index 94b7eaf62fecff26bf515294f5d36c2887304baa..859e88e48a5cea7114b85c31c594f832520bacb0 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_conceptual_physics.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_conceptual_physics.yaml @@ -1,6 +1,6 @@ "dataset_name": "conceptual_physics" "description": "The following are multiple choice questions (with answers) about conceptual\ \ physics.\n\n" -"group": "mmlu_flan_n_shot_loglikelihood_stem" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_conceptual_physics" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_econometrics.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_econometrics.yaml index 146d4847d816e546c0ec817cd2e236f453d6bae1..0455a515eab5e3102a659d917758b942c00b952d 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_econometrics.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_econometrics.yaml @@ -1,6 +1,6 @@ "dataset_name": "econometrics" "description": "The following are multiple choice questions (with answers) about econometrics.\n\ \n" -"group": "mmlu_flan_n_shot_loglikelihood_social_sciences" +"tag": "mmlu_flan_n_shot_loglikelihood_social_sciences" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_econometrics" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_electrical_engineering.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_electrical_engineering.yaml index 61cb27e22855e169d30f7fdcd71a18f42afc1ce3..b63e06172ec302a916f3be4b0a2ea0f1efa86674 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_electrical_engineering.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_electrical_engineering.yaml @@ -1,6 +1,6 @@ "dataset_name": "electrical_engineering" "description": "The following are multiple choice questions (with answers) about electrical\ \ engineering.\n\n" -"group": "mmlu_flan_n_shot_loglikelihood_stem" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_electrical_engineering" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_elementary_mathematics.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_elementary_mathematics.yaml index 39e10f856c003cc0c382a088278c2a44ee0ad92a..79771d21543868dd73bf6ff84201ef07d79c89a2 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_elementary_mathematics.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_elementary_mathematics.yaml @@ -1,6 +1,6 @@ "dataset_name": "elementary_mathematics" "description": "The following are multiple choice questions (with answers) about elementary\ \ mathematics.\n\n" -"group": "mmlu_flan_n_shot_loglikelihood_stem" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_elementary_mathematics" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_formal_logic.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_formal_logic.yaml index 7fb8aa923735d58bc22d32f93f556cfe54cd66af..3e46d8e21c62ff03a6f47bbbc7a6d085840049a4 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_formal_logic.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_formal_logic.yaml @@ -1,6 +1,6 @@ "dataset_name": "formal_logic" "description": "The following are multiple choice questions (with answers) about formal\ \ logic.\n\n" -"group": "mmlu_flan_n_shot_loglikelihood_humanities" +"tag": "mmlu_flan_n_shot_loglikelihood_humanities" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_formal_logic" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_global_facts.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_global_facts.yaml index 5ffc9069ac1e7ee42eada962ee7c7a5146b05be6..9e7aff59325d7dab9a02c4eda3a886d062fe3b4a 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_global_facts.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_global_facts.yaml @@ -1,6 +1,6 @@ "dataset_name": "global_facts" "description": "The following are multiple choice questions (with answers) about global\ \ facts.\n\n" -"group": "mmlu_flan_n_shot_loglikelihood_other" +"tag": "mmlu_flan_n_shot_loglikelihood_other" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_global_facts" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_biology.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_biology.yaml index 328b47f8bc141d6f465f38ebd634c1dece5d0269..dfe33de2be1d2f821c92fc46111150e1ac366b7e 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_biology.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_biology.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_biology" "description": "The following are multiple choice questions (with answers) about high\ \ school biology.\n\n" -"group": "mmlu_flan_n_shot_loglikelihood_stem" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_high_school_biology" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_chemistry.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_chemistry.yaml index 350583752e02786d1f21eccb8118ca9c8f0e1af8..661ea0ca2f72242eb4daf520f6683a9de3a7c32c 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_chemistry.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_chemistry.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_chemistry" "description": "The following are multiple choice questions (with answers) about high\ \ school chemistry.\n\n" -"group": "mmlu_flan_n_shot_loglikelihood_stem" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_high_school_chemistry" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_computer_science.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_computer_science.yaml index cd2e1285a9b53895c766e23ea33a859d8fa81218..b271a661f943fdd6d364833c9f994c19ee10cd22 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_computer_science.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_computer_science.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_computer_science" "description": "The following are multiple choice questions (with answers) about high\ \ school computer science.\n\n" -"group": "mmlu_flan_n_shot_loglikelihood_stem" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_high_school_computer_science" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_european_history.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_european_history.yaml index 62f9465f9253dc46df99a6fa3495c9987de4ce85..f1a329ebb24804c92690b5210cb27f6ec47be93d 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_european_history.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_european_history.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_european_history" "description": "The following are multiple choice questions (with answers) about high\ \ school european history.\n\n" -"group": "mmlu_flan_n_shot_loglikelihood_humanities" +"tag": "mmlu_flan_n_shot_loglikelihood_humanities" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_high_school_european_history" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_geography.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_geography.yaml index c2e8d83f450497a1c2abf1dbc520ce44e31ff199..fe681101f6704e7f058e27350b37838ba63fcd07 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_geography.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_geography.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_geography" "description": "The following are multiple choice questions (with answers) about high\ \ school geography.\n\n" -"group": "mmlu_flan_n_shot_loglikelihood_social_sciences" +"tag": "mmlu_flan_n_shot_loglikelihood_social_sciences" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_high_school_geography" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_government_and_politics.yaml index 9b72fb19b7148583e017f51d6df5e66bb45eeb53..d8a8f279fc5bfaa8b610f3ff5dcd1c2be0c88e07 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_government_and_politics.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_government_and_politics.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_government_and_politics" "description": "The following are multiple choice questions (with answers) about high\ \ school government and politics.\n\n" -"group": "mmlu_flan_n_shot_loglikelihood_social_sciences" +"tag": "mmlu_flan_n_shot_loglikelihood_social_sciences" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_high_school_government_and_politics" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_macroeconomics.yaml index abc4f87a3c03e2d1753ff7fbaff7c9195cdd8ed3..45664135facb151e9b6f91347bbc135297880acb 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_macroeconomics.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_macroeconomics.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_macroeconomics" "description": "The following are multiple choice questions (with answers) about high\ \ school macroeconomics.\n\n" -"group": "mmlu_flan_n_shot_loglikelihood_social_sciences" +"tag": "mmlu_flan_n_shot_loglikelihood_social_sciences" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_high_school_macroeconomics" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_mathematics.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_mathematics.yaml index 1ddd6df3de4895cc83be706567ab92b10351c053..49903260ceff13c03070606e04beb45d99d660f7 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_mathematics.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_mathematics.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_mathematics" "description": "The following are multiple choice questions (with answers) about high\ \ school mathematics.\n\n" -"group": "mmlu_flan_n_shot_loglikelihood_stem" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_high_school_mathematics" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_microeconomics.yaml index d72fc2ac23689e308b41670d5acefb5222509616..394c1d77e553a24820ba5db934bfa8fd95a8a269 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_microeconomics.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_microeconomics.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_microeconomics" "description": "The following are multiple choice questions (with answers) about high\ \ school microeconomics.\n\n" -"group": "mmlu_flan_n_shot_loglikelihood_social_sciences" +"tag": "mmlu_flan_n_shot_loglikelihood_social_sciences" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_high_school_microeconomics" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_physics.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_physics.yaml index c09b2c1d6255ec21a677d1a3dbf10b71302fda29..7f32ef2fcc4bf03e34b43c5a3d1135431742db71 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_physics.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_physics.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_physics" "description": "The following are multiple choice questions (with answers) about high\ \ school physics.\n\n" -"group": "mmlu_flan_n_shot_loglikelihood_stem" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_high_school_physics" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_psychology.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_psychology.yaml index 8d4bfdd476fa13bc838d39e101c7d4cd6c837da4..9a9aac0736a9610469c70b925b70b3f384ca9777 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_psychology.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_psychology.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_psychology" "description": "The following are multiple choice questions (with answers) about high\ \ school psychology.\n\n" -"group": "mmlu_flan_n_shot_loglikelihood_social_sciences" +"tag": "mmlu_flan_n_shot_loglikelihood_social_sciences" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_high_school_psychology" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_statistics.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_statistics.yaml index bc0c9d1a04c4290e0b283e6c2a53be7212f4f276..5e7e02afb94aebb1676c5c395c51e37d4f149a39 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_statistics.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_statistics.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_statistics" "description": "The following are multiple choice questions (with answers) about high\ \ school statistics.\n\n" -"group": "mmlu_flan_n_shot_loglikelihood_stem" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_high_school_statistics" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_us_history.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_us_history.yaml index 0524e7675be12322aeb336f95ce805b7e322a66b..7bc84ea9dd78e87166f1e7b67c248d242cb98d83 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_us_history.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_us_history.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_us_history" "description": "The following are multiple choice questions (with answers) about high\ \ school us history.\n\n" -"group": "mmlu_flan_n_shot_loglikelihood_humanities" +"tag": "mmlu_flan_n_shot_loglikelihood_humanities" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_high_school_us_history" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_world_history.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_world_history.yaml index 8664736255b9f7d55945cb40fd285a49b6a626f3..f25cf646bebbca23ed23ea421473e6c2461dda8a 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_world_history.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_high_school_world_history.yaml @@ -1,6 +1,6 @@ "dataset_name": "high_school_world_history" "description": "The following are multiple choice questions (with answers) about high\ \ school world history.\n\n" -"group": "mmlu_flan_n_shot_loglikelihood_humanities" +"tag": "mmlu_flan_n_shot_loglikelihood_humanities" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_high_school_world_history" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_human_aging.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_human_aging.yaml index 9d0a0179e675716b33feb140c268b4926ac6b46d..c258f919041775e1d2bf1226264a10b1133802db 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_human_aging.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_human_aging.yaml @@ -1,6 +1,6 @@ "dataset_name": "human_aging" "description": "The following are multiple choice questions (with answers) about human\ \ aging.\n\n" -"group": "mmlu_flan_n_shot_loglikelihood_other" +"tag": "mmlu_flan_n_shot_loglikelihood_other" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_human_aging" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_human_sexuality.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_human_sexuality.yaml index 96979867c1ceba5b5a4bf952b70dd37d2d36b233..1e192a78b48bd37e4dc37efc5783b527f84c3e55 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_human_sexuality.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_human_sexuality.yaml @@ -1,6 +1,6 @@ "dataset_name": "human_sexuality" "description": "The following are multiple choice questions (with answers) about human\ \ sexuality.\n\n" -"group": "mmlu_flan_n_shot_loglikelihood_social_sciences" +"tag": "mmlu_flan_n_shot_loglikelihood_social_sciences" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_human_sexuality" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_international_law.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_international_law.yaml index 3d6eb6b28fadb9ed0b28487880eda0b6fd9bf4c3..662bf6eb35157889356a6be7ded31d5f6f2a39ac 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_international_law.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_international_law.yaml @@ -1,6 +1,6 @@ "dataset_name": "international_law" "description": "The following are multiple choice questions (with answers) about international\ \ law.\n\n" -"group": "mmlu_flan_n_shot_loglikelihood_humanities" +"tag": "mmlu_flan_n_shot_loglikelihood_humanities" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_international_law" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_jurisprudence.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_jurisprudence.yaml index 0ef1cb184e345dae777028b123717f38ac3c8c63..82036dc1da79c464f21f90b46e4681b061fe5ea1 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_jurisprudence.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_jurisprudence.yaml @@ -1,6 +1,6 @@ "dataset_name": "jurisprudence" "description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ \n" -"group": "mmlu_flan_n_shot_loglikelihood_humanities" +"tag": "mmlu_flan_n_shot_loglikelihood_humanities" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_jurisprudence" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_logical_fallacies.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_logical_fallacies.yaml index b735d0738561226e6922687d7d103a5da4446055..346e4b669771f23d7a3a805b329e96e711cd367e 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_logical_fallacies.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_logical_fallacies.yaml @@ -1,6 +1,6 @@ "dataset_name": "logical_fallacies" "description": "The following are multiple choice questions (with answers) about logical\ \ fallacies.\n\n" -"group": "mmlu_flan_n_shot_loglikelihood_humanities" +"tag": "mmlu_flan_n_shot_loglikelihood_humanities" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_logical_fallacies" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_machine_learning.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_machine_learning.yaml index fccc7058b5b5598dbe7efebf9f04c484bc071388..3d7c280155ae7302b0bed56715c5ea92191e3faf 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_machine_learning.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_machine_learning.yaml @@ -1,6 +1,6 @@ "dataset_name": "machine_learning" "description": "The following are multiple choice questions (with answers) about machine\ \ learning.\n\n" -"group": "mmlu_flan_n_shot_loglikelihood_stem" +"tag": "mmlu_flan_n_shot_loglikelihood_stem" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_machine_learning" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_management.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_management.yaml index a40da661e36759dd9206b7c8eda6217b693ecf22..7a732a778fb85eac5467fe2744e51340bce0c302 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_management.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_management.yaml @@ -1,6 +1,6 @@ "dataset_name": "management" "description": "The following are multiple choice questions (with answers) about management.\n\ \n" -"group": "mmlu_flan_n_shot_loglikelihood_other" +"tag": "mmlu_flan_n_shot_loglikelihood_other" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_management" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_marketing.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_marketing.yaml index 3537a86b933669b1f2cce6362184eb6bf61988e7..56760226dba043ba37a110cf7065bbd52c3e9c93 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_marketing.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_marketing.yaml @@ -1,6 +1,6 @@ "dataset_name": "marketing" "description": "The following are multiple choice questions (with answers) about marketing.\n\ \n" -"group": "mmlu_flan_n_shot_loglikelihood_other" +"tag": "mmlu_flan_n_shot_loglikelihood_other" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_marketing" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_medical_genetics.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_medical_genetics.yaml index 49247525eadd9ca9e8368d8f2d0606a97d837e0d..6635c9613155a7c23bf67329b4be950e57fe2d30 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_medical_genetics.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_medical_genetics.yaml @@ -1,6 +1,6 @@ "dataset_name": "medical_genetics" "description": "The following are multiple choice questions (with answers) about medical\ \ genetics.\n\n" -"group": "mmlu_flan_n_shot_loglikelihood_other" +"tag": "mmlu_flan_n_shot_loglikelihood_other" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_medical_genetics" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_miscellaneous.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_miscellaneous.yaml index c6aa9bafad16dfc8abcee5a83b701f2e7934a037..ce8dff42a80057d6557f81e5aead49b4e93e4ef3 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_miscellaneous.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_miscellaneous.yaml @@ -1,6 +1,6 @@ "dataset_name": "miscellaneous" "description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ \n" -"group": "mmlu_flan_n_shot_loglikelihood_other" +"tag": "mmlu_flan_n_shot_loglikelihood_other" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_miscellaneous" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_moral_disputes.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_moral_disputes.yaml index 4ff46f425aed8740eb3fe349fd5afebb6b079a06..62460e82f3386022679443efe3c989c2ffb59abf 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_moral_disputes.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_moral_disputes.yaml @@ -1,6 +1,6 @@ "dataset_name": "moral_disputes" "description": "The following are multiple choice questions (with answers) about moral\ \ disputes.\n\n" -"group": "mmlu_flan_n_shot_loglikelihood_humanities" +"tag": "mmlu_flan_n_shot_loglikelihood_humanities" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_moral_disputes" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_moral_scenarios.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_moral_scenarios.yaml index cdcc3b01048ca4451f9b554d69b0fe51e549d76b..408c69f11630d4c237079e327b1b4c9fe3971dc9 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_moral_scenarios.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_moral_scenarios.yaml @@ -1,6 +1,6 @@ "dataset_name": "moral_scenarios" "description": "The following are multiple choice questions (with answers) about moral\ \ scenarios.\n\n" -"group": "mmlu_flan_n_shot_loglikelihood_humanities" +"tag": "mmlu_flan_n_shot_loglikelihood_humanities" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_moral_scenarios" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_nutrition.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_nutrition.yaml index 5773ca1f70b7e6bab5e5c328d62f483ee6ae74be..5494f9dc462494e198dfc7ad86d63a186637bf5c 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_nutrition.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_nutrition.yaml @@ -1,6 +1,6 @@ "dataset_name": "nutrition" "description": "The following are multiple choice questions (with answers) about nutrition.\n\ \n" -"group": "mmlu_flan_n_shot_loglikelihood_other" +"tag": "mmlu_flan_n_shot_loglikelihood_other" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_nutrition" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_philosophy.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_philosophy.yaml index 944b44a14477a42d7662075521a6edeafa778685..4fd1f01a1707126eaf93e6a668d681408c8c7fe6 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_philosophy.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_philosophy.yaml @@ -1,6 +1,6 @@ "dataset_name": "philosophy" "description": "The following are multiple choice questions (with answers) about philosophy.\n\ \n" -"group": "mmlu_flan_n_shot_loglikelihood_humanities" +"tag": "mmlu_flan_n_shot_loglikelihood_humanities" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_philosophy" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_prehistory.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_prehistory.yaml index 184a9584176eb47d55ad9fd323e6409a853ee315..1eb08bfbeb44dc8279ab6796e673b3b271517548 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_prehistory.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_prehistory.yaml @@ -1,6 +1,6 @@ "dataset_name": "prehistory" "description": "The following are multiple choice questions (with answers) about prehistory.\n\ \n" -"group": "mmlu_flan_n_shot_loglikelihood_humanities" +"tag": "mmlu_flan_n_shot_loglikelihood_humanities" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_prehistory" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_professional_accounting.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_professional_accounting.yaml index 129b7a50df6ebb26178d41dc845e4189476023e7..5a23a990afe0abf5b354a15dfec3b5bbd2775fc9 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_professional_accounting.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_professional_accounting.yaml @@ -1,6 +1,6 @@ "dataset_name": "professional_accounting" "description": "The following are multiple choice questions (with answers) about professional\ \ accounting.\n\n" -"group": "mmlu_flan_n_shot_loglikelihood_other" +"tag": "mmlu_flan_n_shot_loglikelihood_other" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_professional_accounting" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_professional_law.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_professional_law.yaml index d28f19355341230399d390d57449e0333d104c83..0a80f2baacbe17445f4a1ea564c7a72174b1c445 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_professional_law.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_professional_law.yaml @@ -1,6 +1,6 @@ "dataset_name": "professional_law" "description": "The following are multiple choice questions (with answers) about professional\ \ law.\n\n" -"group": "mmlu_flan_n_shot_loglikelihood_humanities" +"tag": "mmlu_flan_n_shot_loglikelihood_humanities" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_professional_law" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_professional_medicine.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_professional_medicine.yaml index 68c8dc46da23e0fdff8f8eb57c467849f15e567a..da9e30e118445c2a796eb4145f5c308e9e33215f 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_professional_medicine.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_professional_medicine.yaml @@ -1,6 +1,6 @@ "dataset_name": "professional_medicine" "description": "The following are multiple choice questions (with answers) about professional\ \ medicine.\n\n" -"group": "mmlu_flan_n_shot_loglikelihood_other" +"tag": "mmlu_flan_n_shot_loglikelihood_other" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_professional_medicine" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_professional_psychology.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_professional_psychology.yaml index 72481ae0490e88f2b4074a4ae0eb84af5f403bf0..ce7043a07273fe781cb56733ab02b0b1cb4bf059 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_professional_psychology.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_professional_psychology.yaml @@ -1,6 +1,6 @@ "dataset_name": "professional_psychology" "description": "The following are multiple choice questions (with answers) about professional\ \ psychology.\n\n" -"group": "mmlu_flan_n_shot_loglikelihood_social_sciences" +"tag": "mmlu_flan_n_shot_loglikelihood_social_sciences" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_professional_psychology" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_public_relations.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_public_relations.yaml index 73280df59bcfcdb7c06acc69ef616805eccb8ad9..debace7ca0d0c1dbdfad6ad1621dcc5d9a1469eb 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_public_relations.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_public_relations.yaml @@ -1,6 +1,6 @@ "dataset_name": "public_relations" "description": "The following are multiple choice questions (with answers) about public\ \ relations.\n\n" -"group": "mmlu_flan_n_shot_loglikelihood_social_sciences" +"tag": "mmlu_flan_n_shot_loglikelihood_social_sciences" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_public_relations" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_security_studies.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_security_studies.yaml index 03b27c90cd85d4288145e05ad0f1c219ebda34ed..eb1f585ce890b5a4fcccc72c3066691509330c49 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_security_studies.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_security_studies.yaml @@ -1,6 +1,6 @@ "dataset_name": "security_studies" "description": "The following are multiple choice questions (with answers) about security\ \ studies.\n\n" -"group": "mmlu_flan_n_shot_loglikelihood_social_sciences" +"tag": "mmlu_flan_n_shot_loglikelihood_social_sciences" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_security_studies" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_sociology.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_sociology.yaml index 630d16925febdb8ef943ea5d7b5faa649f487a9d..0580f7ae31687590598a77fe950d282020d9be16 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_sociology.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_sociology.yaml @@ -1,6 +1,6 @@ "dataset_name": "sociology" "description": "The following are multiple choice questions (with answers) about sociology.\n\ \n" -"group": "mmlu_flan_n_shot_loglikelihood_social_sciences" +"tag": "mmlu_flan_n_shot_loglikelihood_social_sciences" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_sociology" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_us_foreign_policy.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_us_foreign_policy.yaml index 2274d0670dbe7cd43ab747d95638c000fe3b877f..3ff2d9ea791abf7bad56784b804bcadd5c82c077 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_us_foreign_policy.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_us_foreign_policy.yaml @@ -1,6 +1,6 @@ "dataset_name": "us_foreign_policy" "description": "The following are multiple choice questions (with answers) about us\ \ foreign policy.\n\n" -"group": "mmlu_flan_n_shot_loglikelihood_social_sciences" +"tag": "mmlu_flan_n_shot_loglikelihood_social_sciences" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_us_foreign_policy" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_virology.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_virology.yaml index 8bd36105dfe6381f3f8d289502a716326bdba650..c3edfd9528eed5d4199ebb0ba06a328ed8c50dd8 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_virology.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_virology.yaml @@ -1,6 +1,6 @@ "dataset_name": "virology" "description": "The following are multiple choice questions (with answers) about virology.\n\ \n" -"group": "mmlu_flan_n_shot_loglikelihood_other" +"tag": "mmlu_flan_n_shot_loglikelihood_other" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_virology" diff --git a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_world_religions.yaml b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_world_religions.yaml index e59c28498f3103940076b7846ba68f169c10484e..765e70c8fc22dbd75ba495a6490ec788d4e44b7e 100644 --- a/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_world_religions.yaml +++ b/lm_eval/tasks/mmlu/flan_n_shot/loglikelihood/mmlu_world_religions.yaml @@ -1,6 +1,6 @@ "dataset_name": "world_religions" "description": "The following are multiple choice questions (with answers) about world\ \ religions.\n\n" -"group": "mmlu_flan_n_shot_loglikelihood_humanities" +"tag": "mmlu_flan_n_shot_loglikelihood_humanities" "include": "_mmlu_flan_loglikelihood_template_yaml" "task": "mmlu_flan_n_shot_loglikelihood_world_religions" diff --git a/lm_eval/tasks/mmlu/generative/_default_template_yaml b/lm_eval/tasks/mmlu/generative/_default_template_yaml index bc1ad0db076b54bfb83cc5918c617b7e0fe27773..1452e0f5b34a899e537ad8889bf012c403ff58cc 100644 --- a/lm_eval/tasks/mmlu/generative/_default_template_yaml +++ b/lm_eval/tasks/mmlu/generative/_default_template_yaml @@ -15,6 +15,6 @@ metric_list: aggregation: mean higher_is_better: true metadata: - version: 1.0 + version: 2.0 dataset_kwargs: trust_remote_code: true diff --git a/lm_eval/tasks/mmlu/generative/_mmlu.yaml b/lm_eval/tasks/mmlu/generative/_mmlu.yaml index 15ead35ddcff01582a956040108ab0e4f10a5f20..1a63611bdb2d35ef4fb358a5258187c1cab99a65 100644 --- a/lm_eval/tasks/mmlu/generative/_mmlu.yaml +++ b/lm_eval/tasks/mmlu/generative/_mmlu.yaml @@ -1,6 +1,33 @@ group: mmlu_generative +group_alias: mmlu (generative) task: - - mmlu_stem_generative - - mmlu_other_generative - - mmlu_social_sciences_generative - - mmlu_humanities_generative + - group: stem + task: + - mmlu_stem_generative + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: other + task: + - mmlu_other_generative + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: social sciences + task: + - mmlu_social_sciences_generative + aggregate_metric_list: + - metric: acc + weight_by_size: True + - group: humanities + task: + - mmlu_humanities_generative + aggregate_metric_list: + - metric: acc + weight_by_size: True +aggregate_metric_list: + - aggregation: mean + metric: exact_match + weight_by_size: True +metadata: + version: 2 diff --git a/lm_eval/tasks/mmlu/generative/mmlu_abstract_algebra.yaml b/lm_eval/tasks/mmlu/generative/mmlu_abstract_algebra.yaml index bbdfee5497508a1a78ae454687bee69cd8a64c84..17bfcafb79b113cffe93f6e90c68562b7eae7c95 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_abstract_algebra.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_abstract_algebra.yaml @@ -1,8 +1,7 @@ "dataset_name": "abstract_algebra" "description": "The following are multiple choice questions (with answers) about abstract\ \ algebra.\n\n" -"group": "mmlu_stem_generative" -"group_alias": "stem" +"tag": "mmlu_stem_generative" "include": "_default_template_yaml" "task": "mmlu_abstract_algebra_generative" "task_alias": "abstract_algebra" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_anatomy.yaml b/lm_eval/tasks/mmlu/generative/mmlu_anatomy.yaml index c8870b45e78f6ef4f369677f09229cf34235527a..72afc359a495af12d3dcb2b062c6442d92d45c88 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_anatomy.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_anatomy.yaml @@ -1,8 +1,7 @@ "dataset_name": "anatomy" "description": "The following are multiple choice questions (with answers) about anatomy.\n\ \n" -"group": "mmlu_stem_generative" -"group_alias": "stem" +"tag": "mmlu_stem_generative" "include": "_default_template_yaml" "task": "mmlu_anatomy_generative" "task_alias": "anatomy" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_astronomy.yaml b/lm_eval/tasks/mmlu/generative/mmlu_astronomy.yaml index 53aa3803e66c069f5b005fb82f8e75d2cd6ec0be..0b41447e74a2b95732b102bfe5ed642d3d208d2b 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_astronomy.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_astronomy.yaml @@ -1,8 +1,7 @@ "dataset_name": "astronomy" "description": "The following are multiple choice questions (with answers) about astronomy.\n\ \n" -"group": "mmlu_stem_generative" -"group_alias": "stem" +"tag": "mmlu_stem_generative" "include": "_default_template_yaml" "task": "mmlu_astronomy_generative" "task_alias": "astronomy" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_business_ethics.yaml b/lm_eval/tasks/mmlu/generative/mmlu_business_ethics.yaml index 9a4f183bfecee32992f2e5a1cea759c33caff7a4..e7c15d443691af36dcdc761eb41b8673f3782d0b 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_business_ethics.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_business_ethics.yaml @@ -1,8 +1,7 @@ "dataset_name": "business_ethics" "description": "The following are multiple choice questions (with answers) about business\ \ ethics.\n\n" -"group": "mmlu_other_generative" -"group_alias": "other" +"tag": "mmlu_other_generative" "include": "_default_template_yaml" "task": "mmlu_business_ethics_generative" "task_alias": "business_ethics" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_clinical_knowledge.yaml b/lm_eval/tasks/mmlu/generative/mmlu_clinical_knowledge.yaml index c55259ad859f424fad6ecd4d4393433962141d58..24cd0b72d3f68fb00da90397979816b85ea1c76c 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_clinical_knowledge.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_clinical_knowledge.yaml @@ -1,8 +1,7 @@ "dataset_name": "clinical_knowledge" "description": "The following are multiple choice questions (with answers) about clinical\ \ knowledge.\n\n" -"group": "mmlu_other_generative" -"group_alias": "other" +"tag": "mmlu_other_generative" "include": "_default_template_yaml" "task": "mmlu_clinical_knowledge_generative" "task_alias": "clinical_knowledge" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_college_biology.yaml b/lm_eval/tasks/mmlu/generative/mmlu_college_biology.yaml index 62306d795a1b0eec1f1fae56ba75b2ae858f450e..2ff9cc284007337e30369dd4864b2b723e8e6768 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_college_biology.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_college_biology.yaml @@ -1,8 +1,7 @@ "dataset_name": "college_biology" "description": "The following are multiple choice questions (with answers) about college\ \ biology.\n\n" -"group": "mmlu_stem_generative" -"group_alias": "stem" +"tag": "mmlu_stem_generative" "include": "_default_template_yaml" "task": "mmlu_college_biology_generative" "task_alias": "college_biology" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_college_chemistry.yaml b/lm_eval/tasks/mmlu/generative/mmlu_college_chemistry.yaml index 5da9c0412d9755ea2b8bafba87ede1e0d5be89f7..12d9ce3eab1332fa202cf6f99a52785865aed1a7 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_college_chemistry.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_college_chemistry.yaml @@ -1,8 +1,7 @@ "dataset_name": "college_chemistry" "description": "The following are multiple choice questions (with answers) about college\ \ chemistry.\n\n" -"group": "mmlu_stem_generative" -"group_alias": "stem" +"tag": "mmlu_stem_generative" "include": "_default_template_yaml" "task": "mmlu_college_chemistry_generative" "task_alias": "college_chemistry" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_college_computer_science.yaml b/lm_eval/tasks/mmlu/generative/mmlu_college_computer_science.yaml index af57ed39e6a184117efa6d9960842647914e89fe..73d91c52acd76bf99ce1869296257d25143ad149 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_college_computer_science.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_college_computer_science.yaml @@ -1,8 +1,7 @@ "dataset_name": "college_computer_science" "description": "The following are multiple choice questions (with answers) about college\ \ computer science.\n\n" -"group": "mmlu_stem_generative" -"group_alias": "stem" +"tag": "mmlu_stem_generative" "include": "_default_template_yaml" "task": "mmlu_college_computer_science_generative" "task_alias": "college_computer_science" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_college_mathematics.yaml b/lm_eval/tasks/mmlu/generative/mmlu_college_mathematics.yaml index 4cf22a4f9aeecfa952281539a9a9d03dec967564..15ae9dded855610af45a15bab8aa56596bfaddd4 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_college_mathematics.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_college_mathematics.yaml @@ -1,8 +1,7 @@ "dataset_name": "college_mathematics" "description": "The following are multiple choice questions (with answers) about college\ \ mathematics.\n\n" -"group": "mmlu_stem_generative" -"group_alias": "stem" +"tag": "mmlu_stem_generative" "include": "_default_template_yaml" "task": "mmlu_college_mathematics_generative" "task_alias": "college_mathematics" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_college_medicine.yaml b/lm_eval/tasks/mmlu/generative/mmlu_college_medicine.yaml index 579085df012a2587c499bf3aba51ae5a65b66355..0461ab7ae7dab9df6b10591fd14791a2cc3eff0f 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_college_medicine.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_college_medicine.yaml @@ -1,8 +1,7 @@ "dataset_name": "college_medicine" "description": "The following are multiple choice questions (with answers) about college\ \ medicine.\n\n" -"group": "mmlu_other_generative" -"group_alias": "other" +"tag": "mmlu_other_generative" "include": "_default_template_yaml" "task": "mmlu_college_medicine_generative" "task_alias": "college_medicine" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_college_physics.yaml b/lm_eval/tasks/mmlu/generative/mmlu_college_physics.yaml index b281f7cd4a7c778f682490adbfd49ef74c507bea..0d997d8974c99a549a2216a9bd9237f05a619e21 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_college_physics.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_college_physics.yaml @@ -1,8 +1,7 @@ "dataset_name": "college_physics" "description": "The following are multiple choice questions (with answers) about college\ \ physics.\n\n" -"group": "mmlu_stem_generative" -"group_alias": "stem" +"tag": "mmlu_stem_generative" "include": "_default_template_yaml" "task": "mmlu_college_physics_generative" "task_alias": "college_physics" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_computer_security.yaml b/lm_eval/tasks/mmlu/generative/mmlu_computer_security.yaml index 0b75f49e77803677dd2cb550df9207884a363c2e..ee64d20100e25fc4bcf7f446b1e98acf042c4ab8 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_computer_security.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_computer_security.yaml @@ -1,8 +1,7 @@ "dataset_name": "computer_security" "description": "The following are multiple choice questions (with answers) about computer\ \ security.\n\n" -"group": "mmlu_stem_generative" -"group_alias": "stem" +"tag": "mmlu_stem_generative" "include": "_default_template_yaml" "task": "mmlu_computer_security_generative" "task_alias": "computer_security" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_conceptual_physics.yaml b/lm_eval/tasks/mmlu/generative/mmlu_conceptual_physics.yaml index 44b9580709b49755206736c91c0ee43c3c3f815c..75764a2cbf542ba09a99ae252c76a103bf534a9f 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_conceptual_physics.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_conceptual_physics.yaml @@ -1,8 +1,7 @@ "dataset_name": "conceptual_physics" "description": "The following are multiple choice questions (with answers) about conceptual\ \ physics.\n\n" -"group": "mmlu_stem_generative" -"group_alias": "stem" +"tag": "mmlu_stem_generative" "include": "_default_template_yaml" "task": "mmlu_conceptual_physics_generative" "task_alias": "conceptual_physics" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_econometrics.yaml b/lm_eval/tasks/mmlu/generative/mmlu_econometrics.yaml index 3aa007265e19dda6819c7bcd863f7578e0b31f5d..43fec80ad3f505bedb810df609a8c6e8d2c2c0ed 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_econometrics.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_econometrics.yaml @@ -1,8 +1,7 @@ "dataset_name": "econometrics" "description": "The following are multiple choice questions (with answers) about econometrics.\n\ \n" -"group": "mmlu_social_sciences_generative" -"group_alias": "social_sciences" +"tag": "mmlu_social_sciences_generative" "include": "_default_template_yaml" "task": "mmlu_econometrics_generative" "task_alias": "econometrics" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_electrical_engineering.yaml b/lm_eval/tasks/mmlu/generative/mmlu_electrical_engineering.yaml index b73932900b9a7d9e288e1391d24fdc9dc8ff8a0a..130ec2b2aa2210322c1e2f86cdf6be31dd72bffc 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_electrical_engineering.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_electrical_engineering.yaml @@ -1,8 +1,7 @@ "dataset_name": "electrical_engineering" "description": "The following are multiple choice questions (with answers) about electrical\ \ engineering.\n\n" -"group": "mmlu_stem_generative" -"group_alias": "stem" +"tag": "mmlu_stem_generative" "include": "_default_template_yaml" "task": "mmlu_electrical_engineering_generative" "task_alias": "electrical_engineering" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_elementary_mathematics.yaml b/lm_eval/tasks/mmlu/generative/mmlu_elementary_mathematics.yaml index fa364774220c6e77b39224d121ca47a4aacae235..4afd087dc47f27653b54ff48a27a187bc9af07bc 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_elementary_mathematics.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_elementary_mathematics.yaml @@ -1,8 +1,7 @@ "dataset_name": "elementary_mathematics" "description": "The following are multiple choice questions (with answers) about elementary\ \ mathematics.\n\n" -"group": "mmlu_stem_generative" -"group_alias": "stem" +"tag": "mmlu_stem_generative" "include": "_default_template_yaml" "task": "mmlu_elementary_mathematics_generative" "task_alias": "elementary_mathematics" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_formal_logic.yaml b/lm_eval/tasks/mmlu/generative/mmlu_formal_logic.yaml index cf8d3ce699f59ce27b1bc1dba57f83818d8fce85..72c28c0b188b8b8fd69ba9ed79595f0d173f71cf 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_formal_logic.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_formal_logic.yaml @@ -1,8 +1,7 @@ "dataset_name": "formal_logic" "description": "The following are multiple choice questions (with answers) about formal\ \ logic.\n\n" -"group": "mmlu_humanities_generative" -"group_alias": "humanities" +"tag": "mmlu_humanities_generative" "include": "_default_template_yaml" "task": "mmlu_formal_logic_generative" "task_alias": "formal_logic" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_global_facts.yaml b/lm_eval/tasks/mmlu/generative/mmlu_global_facts.yaml index 6e49d4f29d1500faf50119b6ffe88f46cb69e3a7..b788025ad5ddf0d859fc12a0d0f139c0975b16ba 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_global_facts.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_global_facts.yaml @@ -1,8 +1,7 @@ "dataset_name": "global_facts" "description": "The following are multiple choice questions (with answers) about global\ \ facts.\n\n" -"group": "mmlu_other_generative" -"group_alias": "other" +"tag": "mmlu_other_generative" "include": "_default_template_yaml" "task": "mmlu_global_facts_generative" "task_alias": "global_facts" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_high_school_biology.yaml b/lm_eval/tasks/mmlu/generative/mmlu_high_school_biology.yaml index fd88cb764b609a9a025cbb0820960d5215c88025..3677842dcfc091bb28525889479a48096cbb854d 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_high_school_biology.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_high_school_biology.yaml @@ -1,8 +1,7 @@ "dataset_name": "high_school_biology" "description": "The following are multiple choice questions (with answers) about high\ \ school biology.\n\n" -"group": "mmlu_stem_generative" -"group_alias": "stem" +"tag": "mmlu_stem_generative" "include": "_default_template_yaml" "task": "mmlu_high_school_biology_generative" "task_alias": "high_school_biology" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_high_school_chemistry.yaml b/lm_eval/tasks/mmlu/generative/mmlu_high_school_chemistry.yaml index 91cc4bda15962eaeacdef3419611a60fd55908ce..2df93cab2a999a7d6d8e78d3ac9c3ce9aeddcf12 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_high_school_chemistry.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_high_school_chemistry.yaml @@ -1,8 +1,7 @@ "dataset_name": "high_school_chemistry" "description": "The following are multiple choice questions (with answers) about high\ \ school chemistry.\n\n" -"group": "mmlu_stem_generative" -"group_alias": "stem" +"tag": "mmlu_stem_generative" "include": "_default_template_yaml" "task": "mmlu_high_school_chemistry_generative" "task_alias": "high_school_chemistry" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_high_school_computer_science.yaml b/lm_eval/tasks/mmlu/generative/mmlu_high_school_computer_science.yaml index d5b8d52969eb0c49e99e06c8d29d72c53c5a53a0..ec5dc7f89abd7ddc57438c71e0502fce1ac47279 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_high_school_computer_science.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_high_school_computer_science.yaml @@ -1,8 +1,7 @@ "dataset_name": "high_school_computer_science" "description": "The following are multiple choice questions (with answers) about high\ \ school computer science.\n\n" -"group": "mmlu_stem_generative" -"group_alias": "stem" +"tag": "mmlu_stem_generative" "include": "_default_template_yaml" "task": "mmlu_high_school_computer_science_generative" "task_alias": "high_school_computer_science" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_high_school_european_history.yaml b/lm_eval/tasks/mmlu/generative/mmlu_high_school_european_history.yaml index 1ad133fc7a9dcadf6db593aa5d066d9020e3dd34..9732754bbd7352957dbe299494083e17b960c1bc 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_high_school_european_history.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_high_school_european_history.yaml @@ -1,8 +1,7 @@ "dataset_name": "high_school_european_history" "description": "The following are multiple choice questions (with answers) about high\ \ school european history.\n\n" -"group": "mmlu_humanities_generative" -"group_alias": "humanities" +"tag": "mmlu_humanities_generative" "include": "_default_template_yaml" "task": "mmlu_high_school_european_history_generative" "task_alias": "high_school_european_history" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_high_school_geography.yaml b/lm_eval/tasks/mmlu/generative/mmlu_high_school_geography.yaml index 56d6449f81d67c3724e420b96d65f87c8ea22125..66b1a3c97a64f9ee7db414ab13d3146efba5612d 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_high_school_geography.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_high_school_geography.yaml @@ -1,8 +1,7 @@ "dataset_name": "high_school_geography" "description": "The following are multiple choice questions (with answers) about high\ \ school geography.\n\n" -"group": "mmlu_social_sciences_generative" -"group_alias": "social_sciences" +"tag": "mmlu_social_sciences_generative" "include": "_default_template_yaml" "task": "mmlu_high_school_geography_generative" "task_alias": "high_school_geography" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_high_school_government_and_politics.yaml b/lm_eval/tasks/mmlu/generative/mmlu_high_school_government_and_politics.yaml index a4cc73ca5bdd9d0d9eb6f72dd9151bac20b95fd1..46861fdc1149b72d4ac3f347c0e09f679f6c6e54 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_high_school_government_and_politics.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_high_school_government_and_politics.yaml @@ -1,8 +1,7 @@ "dataset_name": "high_school_government_and_politics" "description": "The following are multiple choice questions (with answers) about high\ \ school government and politics.\n\n" -"group": "mmlu_social_sciences_generative" -"group_alias": "social_sciences" +"tag": "mmlu_social_sciences_generative" "include": "_default_template_yaml" "task": "mmlu_high_school_government_and_politics_generative" "task_alias": "high_school_government_and_politics" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_high_school_macroeconomics.yaml b/lm_eval/tasks/mmlu/generative/mmlu_high_school_macroeconomics.yaml index 57496b746e94a632d2dcd2a711f415c3065b4f9d..ada415922b2b777f153cf387f9095cce9c75304b 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_high_school_macroeconomics.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_high_school_macroeconomics.yaml @@ -1,8 +1,7 @@ "dataset_name": "high_school_macroeconomics" "description": "The following are multiple choice questions (with answers) about high\ \ school macroeconomics.\n\n" -"group": "mmlu_social_sciences_generative" -"group_alias": "social_sciences" +"tag": "mmlu_social_sciences_generative" "include": "_default_template_yaml" "task": "mmlu_high_school_macroeconomics_generative" "task_alias": "high_school_macroeconomics" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_high_school_mathematics.yaml b/lm_eval/tasks/mmlu/generative/mmlu_high_school_mathematics.yaml index 26dbffe00f48ac99f3980dd7b69aa723b02a3039..8b22a5888e61be187f5bbbca1e38171eecd6252d 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_high_school_mathematics.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_high_school_mathematics.yaml @@ -1,8 +1,7 @@ "dataset_name": "high_school_mathematics" "description": "The following are multiple choice questions (with answers) about high\ \ school mathematics.\n\n" -"group": "mmlu_stem_generative" -"group_alias": "stem" +"tag": "mmlu_stem_generative" "include": "_default_template_yaml" "task": "mmlu_high_school_mathematics_generative" "task_alias": "high_school_mathematics" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_high_school_microeconomics.yaml b/lm_eval/tasks/mmlu/generative/mmlu_high_school_microeconomics.yaml index ed9e0dcb0a6bf5a6fbdd2cd60c3ee4d82d1fca2b..c59ff16270084981614d6f01065851c005039413 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_high_school_microeconomics.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_high_school_microeconomics.yaml @@ -1,8 +1,7 @@ "dataset_name": "high_school_microeconomics" "description": "The following are multiple choice questions (with answers) about high\ \ school microeconomics.\n\n" -"group": "mmlu_social_sciences_generative" -"group_alias": "social_sciences" +"tag": "mmlu_social_sciences_generative" "include": "_default_template_yaml" "task": "mmlu_high_school_microeconomics_generative" "task_alias": "high_school_microeconomics" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_high_school_physics.yaml b/lm_eval/tasks/mmlu/generative/mmlu_high_school_physics.yaml index 080aed1a792c49dddeb8d3182fce1110a7423991..21d846afb9c8c6b372d59ee462561bb8f67ae83e 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_high_school_physics.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_high_school_physics.yaml @@ -1,8 +1,7 @@ "dataset_name": "high_school_physics" "description": "The following are multiple choice questions (with answers) about high\ \ school physics.\n\n" -"group": "mmlu_stem_generative" -"group_alias": "stem" +"tag": "mmlu_stem_generative" "include": "_default_template_yaml" "task": "mmlu_high_school_physics_generative" "task_alias": "high_school_physics" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_high_school_psychology.yaml b/lm_eval/tasks/mmlu/generative/mmlu_high_school_psychology.yaml index 131c43658ebd21d54088c1c34ceba0eb0e2b71ee..cd1321a5f17efca463edbc6711c197fb18c3a81d 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_high_school_psychology.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_high_school_psychology.yaml @@ -1,8 +1,7 @@ "dataset_name": "high_school_psychology" "description": "The following are multiple choice questions (with answers) about high\ \ school psychology.\n\n" -"group": "mmlu_social_sciences_generative" -"group_alias": "social_sciences" +"tag": "mmlu_social_sciences_generative" "include": "_default_template_yaml" "task": "mmlu_high_school_psychology_generative" "task_alias": "high_school_psychology" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_high_school_statistics.yaml b/lm_eval/tasks/mmlu/generative/mmlu_high_school_statistics.yaml index 9470e01a3520ae5ae8ff7cd31953a6430abf86e5..f1442fb8df4168606151af5cc1dfd769bb2e70e3 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_high_school_statistics.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_high_school_statistics.yaml @@ -1,8 +1,7 @@ "dataset_name": "high_school_statistics" "description": "The following are multiple choice questions (with answers) about high\ \ school statistics.\n\n" -"group": "mmlu_stem_generative" -"group_alias": "stem" +"tag": "mmlu_stem_generative" "include": "_default_template_yaml" "task": "mmlu_high_school_statistics_generative" "task_alias": "high_school_statistics" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_high_school_us_history.yaml b/lm_eval/tasks/mmlu/generative/mmlu_high_school_us_history.yaml index 2d5b1c18672486f5f550158b807e5e2d91805487..4552a560f38e3ed5db503fa677548a11766873c2 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_high_school_us_history.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_high_school_us_history.yaml @@ -1,8 +1,7 @@ "dataset_name": "high_school_us_history" "description": "The following are multiple choice questions (with answers) about high\ \ school us history.\n\n" -"group": "mmlu_humanities_generative" -"group_alias": "humanities" +"tag": "mmlu_humanities_generative" "include": "_default_template_yaml" "task": "mmlu_high_school_us_history_generative" "task_alias": "high_school_us_history" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_high_school_world_history.yaml b/lm_eval/tasks/mmlu/generative/mmlu_high_school_world_history.yaml index 0ee7f993780dec476083502fd8e62c425e82779a..d510f22ff39219829e6a9030cb39dc2c43062ca4 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_high_school_world_history.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_high_school_world_history.yaml @@ -1,8 +1,7 @@ "dataset_name": "high_school_world_history" "description": "The following are multiple choice questions (with answers) about high\ \ school world history.\n\n" -"group": "mmlu_humanities_generative" -"group_alias": "humanities" +"tag": "mmlu_humanities_generative" "include": "_default_template_yaml" "task": "mmlu_high_school_world_history_generative" "task_alias": "high_school_world_history" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_human_aging.yaml b/lm_eval/tasks/mmlu/generative/mmlu_human_aging.yaml index ac19aaed6c6a07ebc592ed14b2b7071a01918d5a..56352f4a8c86966853cdbafd68453d1ee85dbabb 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_human_aging.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_human_aging.yaml @@ -1,8 +1,7 @@ "dataset_name": "human_aging" "description": "The following are multiple choice questions (with answers) about human\ \ aging.\n\n" -"group": "mmlu_other_generative" -"group_alias": "other" +"tag": "mmlu_other_generative" "include": "_default_template_yaml" "task": "mmlu_human_aging_generative" "task_alias": "human_aging" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_human_sexuality.yaml b/lm_eval/tasks/mmlu/generative/mmlu_human_sexuality.yaml index 360075f2ac666e750d72d522f479abd7f1b2e8ef..a23559cfb36a380131573f46b30bbdb5f4656b42 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_human_sexuality.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_human_sexuality.yaml @@ -1,8 +1,7 @@ "dataset_name": "human_sexuality" "description": "The following are multiple choice questions (with answers) about human\ \ sexuality.\n\n" -"group": "mmlu_social_sciences_generative" -"group_alias": "social_sciences" +"tag": "mmlu_social_sciences_generative" "include": "_default_template_yaml" "task": "mmlu_human_sexuality_generative" "task_alias": "human_sexuality" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_international_law.yaml b/lm_eval/tasks/mmlu/generative/mmlu_international_law.yaml index 94df336808a97531df4b76977c3112f3049328af..878df6f3cacb299a51afacca461204fdc4e3a782 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_international_law.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_international_law.yaml @@ -1,8 +1,7 @@ "dataset_name": "international_law" "description": "The following are multiple choice questions (with answers) about international\ \ law.\n\n" -"group": "mmlu_humanities_generative" -"group_alias": "humanities" +"tag": "mmlu_humanities_generative" "include": "_default_template_yaml" "task": "mmlu_international_law_generative" "task_alias": "international_law" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_jurisprudence.yaml b/lm_eval/tasks/mmlu/generative/mmlu_jurisprudence.yaml index c4e2f91cd3d421f543a763419b95b693ce9b5bf0..c5782d81551072a0ff03d79c930f02edb64488f3 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_jurisprudence.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_jurisprudence.yaml @@ -1,8 +1,7 @@ "dataset_name": "jurisprudence" "description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ \n" -"group": "mmlu_humanities_generative" -"group_alias": "humanities" +"tag": "mmlu_humanities_generative" "include": "_default_template_yaml" "task": "mmlu_jurisprudence_generative" "task_alias": "jurisprudence" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_logical_fallacies.yaml b/lm_eval/tasks/mmlu/generative/mmlu_logical_fallacies.yaml index 72197f51c5a4b899e325a61b8025bb5e16a7d7c4..43e8e0168b9f4638cc80b76ff1a4edc8893212b4 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_logical_fallacies.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_logical_fallacies.yaml @@ -1,8 +1,7 @@ "dataset_name": "logical_fallacies" "description": "The following are multiple choice questions (with answers) about logical\ \ fallacies.\n\n" -"group": "mmlu_humanities_generative" -"group_alias": "humanities" +"tag": "mmlu_humanities_generative" "include": "_default_template_yaml" "task": "mmlu_logical_fallacies_generative" "task_alias": "logical_fallacies" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_machine_learning.yaml b/lm_eval/tasks/mmlu/generative/mmlu_machine_learning.yaml index b1b336780ff5cf2a90166123f679f98c82f7fcb9..8d39a4b53164ce8bb641c99fa50f24ace308d3f4 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_machine_learning.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_machine_learning.yaml @@ -1,8 +1,7 @@ "dataset_name": "machine_learning" "description": "The following are multiple choice questions (with answers) about machine\ \ learning.\n\n" -"group": "mmlu_stem_generative" -"group_alias": "stem" +"tag": "mmlu_stem_generative" "include": "_default_template_yaml" "task": "mmlu_machine_learning_generative" "task_alias": "machine_learning" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_management.yaml b/lm_eval/tasks/mmlu/generative/mmlu_management.yaml index 9e1f704027b6ea07fc641bceda55e1db6bc9aecd..6d51ea0d0aa41fb4b2579162111aa8ebd8ce8f6d 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_management.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_management.yaml @@ -1,8 +1,7 @@ "dataset_name": "management" "description": "The following are multiple choice questions (with answers) about management.\n\ \n" -"group": "mmlu_other_generative" -"group_alias": "other" +"tag": "mmlu_other_generative" "include": "_default_template_yaml" "task": "mmlu_management_generative" "task_alias": "management" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_marketing.yaml b/lm_eval/tasks/mmlu/generative/mmlu_marketing.yaml index 05b568773c547c12ca7fff82d25741a135b3fea3..744385a2ea524d6f651851856e15aaf190eb847e 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_marketing.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_marketing.yaml @@ -1,8 +1,7 @@ "dataset_name": "marketing" "description": "The following are multiple choice questions (with answers) about marketing.\n\ \n" -"group": "mmlu_other_generative" -"group_alias": "other" +"tag": "mmlu_other_generative" "include": "_default_template_yaml" "task": "mmlu_marketing_generative" "task_alias": "marketing" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_medical_genetics.yaml b/lm_eval/tasks/mmlu/generative/mmlu_medical_genetics.yaml index a8208145f366dac8274bd9a146567b2903ceefb3..7fea57959818525acdada5bf8a327b0ce96fefb0 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_medical_genetics.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_medical_genetics.yaml @@ -1,8 +1,7 @@ "dataset_name": "medical_genetics" "description": "The following are multiple choice questions (with answers) about medical\ \ genetics.\n\n" -"group": "mmlu_other_generative" -"group_alias": "other" +"tag": "mmlu_other_generative" "include": "_default_template_yaml" "task": "mmlu_medical_genetics_generative" "task_alias": "medical_genetics" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_miscellaneous.yaml b/lm_eval/tasks/mmlu/generative/mmlu_miscellaneous.yaml index 86952e80b9bd149ccb24b57941ee58c30570b4d3..e7e0fabc2536d4894526b680deba9a382ff9c3ff 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_miscellaneous.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_miscellaneous.yaml @@ -1,8 +1,7 @@ "dataset_name": "miscellaneous" "description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ \n" -"group": "mmlu_other_generative" -"group_alias": "other" +"tag": "mmlu_other_generative" "include": "_default_template_yaml" "task": "mmlu_miscellaneous_generative" "task_alias": "miscellaneous" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_moral_disputes.yaml b/lm_eval/tasks/mmlu/generative/mmlu_moral_disputes.yaml index 3e2396d0d5f9acd45bf599e67a1cd052f9f8f482..61d2feee6a9cf4ed4d71b7c2f9aa68f5219c270a 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_moral_disputes.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_moral_disputes.yaml @@ -1,8 +1,7 @@ "dataset_name": "moral_disputes" "description": "The following are multiple choice questions (with answers) about moral\ \ disputes.\n\n" -"group": "mmlu_humanities_generative" -"group_alias": "humanities" +"tag": "mmlu_humanities_generative" "include": "_default_template_yaml" "task": "mmlu_moral_disputes_generative" "task_alias": "moral_disputes" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_moral_scenarios.yaml b/lm_eval/tasks/mmlu/generative/mmlu_moral_scenarios.yaml index 694c2cfee511b5daf26f31723a74dd6145ef1d96..2aeb93f967f0811d3a2f1d886aedfb334a96714e 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_moral_scenarios.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_moral_scenarios.yaml @@ -1,8 +1,7 @@ "dataset_name": "moral_scenarios" "description": "The following are multiple choice questions (with answers) about moral\ \ scenarios.\n\n" -"group": "mmlu_humanities_generative" -"group_alias": "humanities" +"tag": "mmlu_humanities_generative" "include": "_default_template_yaml" "task": "mmlu_moral_scenarios_generative" "task_alias": "moral_scenarios" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_nutrition.yaml b/lm_eval/tasks/mmlu/generative/mmlu_nutrition.yaml index 835df556d55c598da0a8c628ae026427e2f09c59..638ac8100b6f918ccaa0a3dc13946512d3c97b33 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_nutrition.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_nutrition.yaml @@ -1,8 +1,7 @@ "dataset_name": "nutrition" "description": "The following are multiple choice questions (with answers) about nutrition.\n\ \n" -"group": "mmlu_other_generative" -"group_alias": "other" +"tag": "mmlu_other_generative" "include": "_default_template_yaml" "task": "mmlu_nutrition_generative" "task_alias": "nutrition" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_philosophy.yaml b/lm_eval/tasks/mmlu/generative/mmlu_philosophy.yaml index 8423ea121a654bf5032cad570984bea98c908e29..149894b8484cb1fad9ddad1fc5cb2c07a659aea1 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_philosophy.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_philosophy.yaml @@ -1,8 +1,7 @@ "dataset_name": "philosophy" "description": "The following are multiple choice questions (with answers) about philosophy.\n\ \n" -"group": "mmlu_humanities_generative" -"group_alias": "humanities" +"tag": "mmlu_humanities_generative" "include": "_default_template_yaml" "task": "mmlu_philosophy_generative" "task_alias": "philosophy" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_prehistory.yaml b/lm_eval/tasks/mmlu/generative/mmlu_prehistory.yaml index e9ef7f57391706a8c6f36d96d756f78e48676eae..e130e1baacc3f8a8f558b568336896668e84dd4f 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_prehistory.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_prehistory.yaml @@ -1,8 +1,7 @@ "dataset_name": "prehistory" "description": "The following are multiple choice questions (with answers) about prehistory.\n\ \n" -"group": "mmlu_humanities_generative" -"group_alias": "humanities" +"tag": "mmlu_humanities_generative" "include": "_default_template_yaml" "task": "mmlu_prehistory_generative" "task_alias": "prehistory" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_professional_accounting.yaml b/lm_eval/tasks/mmlu/generative/mmlu_professional_accounting.yaml index d809d830dc9718e7816da8f6a6b3192f190bfc58..a46792ec22d84ee3193996653f536084b9ab7861 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_professional_accounting.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_professional_accounting.yaml @@ -1,8 +1,7 @@ "dataset_name": "professional_accounting" "description": "The following are multiple choice questions (with answers) about professional\ \ accounting.\n\n" -"group": "mmlu_other_generative" -"group_alias": "other" +"tag": "mmlu_other_generative" "include": "_default_template_yaml" "task": "mmlu_professional_accounting_generative" "task_alias": "professional_accounting" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_professional_law.yaml b/lm_eval/tasks/mmlu/generative/mmlu_professional_law.yaml index c6e16833661591894ee1473d3784c94f695dfe3f..f087657e579524b35bf7de4c0f81cb5b697caed4 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_professional_law.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_professional_law.yaml @@ -1,8 +1,7 @@ "dataset_name": "professional_law" "description": "The following are multiple choice questions (with answers) about professional\ \ law.\n\n" -"group": "mmlu_humanities_generative" -"group_alias": "humanities" +"tag": "mmlu_humanities_generative" "include": "_default_template_yaml" "task": "mmlu_professional_law_generative" "task_alias": "professional_law" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_professional_medicine.yaml b/lm_eval/tasks/mmlu/generative/mmlu_professional_medicine.yaml index 8f69a42f0e4a4f532f123839b52765f668e650ec..bc80878980195f58ac5ae26a0a70589a47b325d5 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_professional_medicine.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_professional_medicine.yaml @@ -1,8 +1,7 @@ "dataset_name": "professional_medicine" "description": "The following are multiple choice questions (with answers) about professional\ \ medicine.\n\n" -"group": "mmlu_other_generative" -"group_alias": "other" +"tag": "mmlu_other_generative" "include": "_default_template_yaml" "task": "mmlu_professional_medicine_generative" "task_alias": "professional_medicine" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_professional_psychology.yaml b/lm_eval/tasks/mmlu/generative/mmlu_professional_psychology.yaml index c63f20b4f2ef010a748eb204b7996fc27b7e1f41..d0b36ccde61e7edc33464a676d4fe0fcc25f3304 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_professional_psychology.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_professional_psychology.yaml @@ -1,8 +1,7 @@ "dataset_name": "professional_psychology" "description": "The following are multiple choice questions (with answers) about professional\ \ psychology.\n\n" -"group": "mmlu_social_sciences_generative" -"group_alias": "social_sciences" +"tag": "mmlu_social_sciences_generative" "include": "_default_template_yaml" "task": "mmlu_professional_psychology_generative" "task_alias": "professional_psychology" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_public_relations.yaml b/lm_eval/tasks/mmlu/generative/mmlu_public_relations.yaml index 5a8e9173bd5346ddc75350fdbbfb2326aa797c90..37cdccba9b7cebbaa34c5f1e9da01655367477f6 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_public_relations.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_public_relations.yaml @@ -1,8 +1,7 @@ "dataset_name": "public_relations" "description": "The following are multiple choice questions (with answers) about public\ \ relations.\n\n" -"group": "mmlu_social_sciences_generative" -"group_alias": "social_sciences" +"tag": "mmlu_social_sciences_generative" "include": "_default_template_yaml" "task": "mmlu_public_relations_generative" "task_alias": "public_relations" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_security_studies.yaml b/lm_eval/tasks/mmlu/generative/mmlu_security_studies.yaml index 67208998748e55780645b2765e96e90fe83951e7..36c235feefd1548320400e7e8d9f3e03f2d478d0 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_security_studies.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_security_studies.yaml @@ -1,8 +1,7 @@ "dataset_name": "security_studies" "description": "The following are multiple choice questions (with answers) about security\ \ studies.\n\n" -"group": "mmlu_social_sciences_generative" -"group_alias": "social_sciences" +"tag": "mmlu_social_sciences_generative" "include": "_default_template_yaml" "task": "mmlu_security_studies_generative" "task_alias": "security_studies" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_sociology.yaml b/lm_eval/tasks/mmlu/generative/mmlu_sociology.yaml index 2d4474a89ecf888704d30e16d60afd5e966c8b1d..b7e2e592e4457118c9458ccb757b823f9adbb193 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_sociology.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_sociology.yaml @@ -1,8 +1,7 @@ "dataset_name": "sociology" "description": "The following are multiple choice questions (with answers) about sociology.\n\ \n" -"group": "mmlu_social_sciences_generative" -"group_alias": "social_sciences" +"tag": "mmlu_social_sciences_generative" "include": "_default_template_yaml" "task": "mmlu_sociology_generative" "task_alias": "sociology" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_us_foreign_policy.yaml b/lm_eval/tasks/mmlu/generative/mmlu_us_foreign_policy.yaml index db5cf05fff7ce12200dc7bd6d05ac885c294f597..d5fb95366245eae638918270bff4353024195d5f 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_us_foreign_policy.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_us_foreign_policy.yaml @@ -1,8 +1,7 @@ "dataset_name": "us_foreign_policy" "description": "The following are multiple choice questions (with answers) about us\ \ foreign policy.\n\n" -"group": "mmlu_social_sciences_generative" -"group_alias": "social_sciences" +"tag": "mmlu_social_sciences_generative" "include": "_default_template_yaml" "task": "mmlu_us_foreign_policy_generative" "task_alias": "us_foreign_policy" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_virology.yaml b/lm_eval/tasks/mmlu/generative/mmlu_virology.yaml index 470ed66e459b0fb26ab22021c17d3760d5a42552..9954dc182f1bbd5030b94d2a08b2ddf4a135a6cf 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_virology.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_virology.yaml @@ -1,8 +1,7 @@ "dataset_name": "virology" "description": "The following are multiple choice questions (with answers) about virology.\n\ \n" -"group": "mmlu_other_generative" -"group_alias": "other" +"tag": "mmlu_other_generative" "include": "_default_template_yaml" "task": "mmlu_virology_generative" "task_alias": "virology" diff --git a/lm_eval/tasks/mmlu/generative/mmlu_world_religions.yaml b/lm_eval/tasks/mmlu/generative/mmlu_world_religions.yaml index bbaf81025c429e8591c470aa7e94e66820a8e45f..1db5128b43e615d0fc41f9c7448db3b5ea39942c 100644 --- a/lm_eval/tasks/mmlu/generative/mmlu_world_religions.yaml +++ b/lm_eval/tasks/mmlu/generative/mmlu_world_religions.yaml @@ -1,8 +1,7 @@ "dataset_name": "world_religions" "description": "The following are multiple choice questions (with answers) about world\ \ religions.\n\n" -"group": "mmlu_humanities_generative" -"group_alias": "humanities" +"tag": "mmlu_humanities_generative" "include": "_default_template_yaml" "task": "mmlu_world_religions_generative" "task_alias": "world_religions" diff --git a/lm_eval/tasks/mmlu_pro/README.md b/lm_eval/tasks/mmlu_pro/README.md new file mode 100644 index 0000000000000000000000000000000000000000..8fddbef266f755ae611c38b6ec2aea05ff6aa033 --- /dev/null +++ b/lm_eval/tasks/mmlu_pro/README.md @@ -0,0 +1,64 @@ +# mmlu_pro + +### Paper + +Title: `MMLU-Pro: A More Robust and Challenging Multi-Task Language Understanding Benchmark` + +Abstract: `In the age of large-scale language models, benchmarks like the Massive Multitask Language Understanding (MMLU) have been pivotal in pushing the boundaries of what AI can achieve in language comprehension and reasoning across diverse domains. However, as models continue to improve, their performance on these benchmarks has begun to plateau, making it increasingly difficult to discern differences in model capabilities. This paper introduces MMLU-Pro, an enhanced dataset designed to extend the mostly knowledge-driven MMLU benchmark by integrating more challenging, reasoning-focused questions and expanding the choice set from four to ten options. Additionally, MMLU-Pro eliminates the trivial and noisy questions in MMLU. Our experimental results show that MMLU-Pro not only raises the challenge, causing a significant drop in accuracy by 16% to 33% compared to MMLU but also demonstrates greater stability under varying prompts. With 24 different prompt styles tested, the sensitivity of model scores to prompt variations decreased from 4-5% in MMLU to just 2% in MMLU-Pro. Additionally, we found that models utilizing Chain of Thought (CoT) reasoning achieved better performance on MMLU-Pro compared to direct answering, which is in stark contrast to the findings on the original MMLU, indicating that MMLU-Pro includes more complex reasoning questions. Our assessments confirm that MMLU-Pro is a more discriminative benchmark to better track progress in the field.` + +Homepage: https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro + +### Citation + +```bibtex +@misc{wang2024mmlupro, + title={MMLU-Pro: A More Robust and Challenging Multi-Task Language Understanding Benchmark}, + author={Yubo Wang and Xueguang Ma and Ge Zhang and Yuansheng Ni and Abhranil Chandra and Shiguang Guo and Weiming Ren and Aaran Arulraj and Xuan He and Ziyan Jiang and Tianle Li and Max Ku and Kai Wang and Alex Zhuang and Rongqi Fan and Xiang Yue and Wenhu Chen}, + year={2024}, + eprint={2406.01574}, + archivePrefix={arXiv}, + primaryClass={id='cs.CL' full_name='Computation and Language' is_active=True alt_name='cmp-lg' in_archive='cs' is_general=False description='Covers natural language processing. Roughly includes material in ACM Subject Class I.2.7. Note that work on artificial languages (programming languages, logics, formal systems) that does not explicitly address natural-language issues broadly construed (natural-language processing, computational linguistics, speech, text retrieval, etc.) is not appropriate for this area.'} +} +``` + +### Groups and Tasks + +#### Groups + +* `mmlu_pro`: 'All 14 subjects of the mmlu_pro dataset, evaluated following the methodology in mmlu's original implementation' + +#### Tasks + +The following tasks evaluate subjects in the mmlu_pro dataset +- `mmlu_pro_biology` +- `mmlu_pro_business` +- `mmlu_pro_chemistry` +- `mmlu_pro_computer_science` +- `mmlu_pro_economics` +- `mmlu_pro_engineering` +- `mmlu_pro_health` +- `mmlu_pro_history` +- `mmlu_pro_law` +- `mmlu_pro_math` +- `mmlu_pro_other` +- `mmlu_pro_philosophy` +- `mmlu_pro_physics` +- `mmlu_pro_psychology` + +### Checklist + +For adding novel benchmarks/datasets to the library: +* [x] Is the task an existing benchmark in the literature? + * [x] Have you referenced the original paper that introduced the task? + * [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + + +If other tasks on this dataset are already supported: +* [ ] Is the "Main" variant of this task clearly denoted? +* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [ ] Have you noted which, if any, published evaluation setups are matched by this variant? + +### Changelog + +* (tasks, group) 2024-09-23 -- (version 1 --> version 2) + * Added one newline to task description(s) as per [reference implementation](https://github.com/TIGER-AI-Lab/MMLU-Pro/blob/47b9891aacb8bd7cda29d5c5ba17b9434dd333bc/evaluate_from_local.py#L93) diff --git a/lm_eval/tasks/mmlu_pro/_default_template_yaml b/lm_eval/tasks/mmlu_pro/_default_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..9c4f44b5c96769d70215403b06bccab33f1bbfb7 --- /dev/null +++ b/lm_eval/tasks/mmlu_pro/_default_template_yaml @@ -0,0 +1,33 @@ +dataset_path: TIGER-Lab/MMLU-Pro +test_split: test +fewshot_split: validation +fewshot_config: + sampler: first_n + doc_to_text: !function utils.fewshot_to_text + doc_to_target: "" +output_type: generate_until +doc_to_text: !function utils.doc_to_text +doc_to_target: answer +filter_list: + - name: "custom-extract" + filter: + - function: "regex" + regex_pattern: 'answer is \(?([ABCDEFGHIJ])\)?' + # regex_pattern: r".*[aA]nswer:\s*([A-J])", + - function: "take_first" +generation_kwargs: + until: + - "" + - "Q:" + - "<|im_end|>" + do_sample: false + temperature: 0.0 +num_fewshot: 5 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/mmlu_pro/_mmlu_pro.yaml b/lm_eval/tasks/mmlu_pro/_mmlu_pro.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fc3204127604d6eac759299f77d63ce9ef49d24e --- /dev/null +++ b/lm_eval/tasks/mmlu_pro/_mmlu_pro.yaml @@ -0,0 +1,23 @@ +group: mmlu_pro +task: + - mmlu_pro_biology + - mmlu_pro_business + - mmlu_pro_chemistry + - mmlu_pro_computer_science + - mmlu_pro_economics + - mmlu_pro_engineering + - mmlu_pro_health + - mmlu_pro_history + - mmlu_pro_law + - mmlu_pro_math + - mmlu_pro_other + - mmlu_pro_philosophy + - mmlu_pro_physics + - mmlu_pro_psychology +aggregate_metric_list: + - aggregation: mean + metric: exact_match + weight_by_size: true + filter_list: custom-extract +metadata: + version: 2.0 diff --git a/lm_eval/tasks/mmlu_pro/mmlu_pro_biology.yaml b/lm_eval/tasks/mmlu_pro/mmlu_pro_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..80aee85108ce8ac9452c0d0d07341e05b11b0b7a --- /dev/null +++ b/lm_eval/tasks/mmlu_pro/mmlu_pro_biology.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about biology. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_biology" +task_alias: "biology" +process_docs: !function utils.process_biology diff --git a/lm_eval/tasks/mmlu_pro/mmlu_pro_business.yaml b/lm_eval/tasks/mmlu_pro/mmlu_pro_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..daf871f6bb5abd614c2058c0552389d41cbced50 --- /dev/null +++ b/lm_eval/tasks/mmlu_pro/mmlu_pro_business.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about business. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_business" +task_alias: "business" +process_docs: !function utils.process_business diff --git a/lm_eval/tasks/mmlu_pro/mmlu_pro_chemistry.yaml b/lm_eval/tasks/mmlu_pro/mmlu_pro_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5baf354ec202e66647bacdc9e9008617cd2d4244 --- /dev/null +++ b/lm_eval/tasks/mmlu_pro/mmlu_pro_chemistry.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about chemistry. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_chemistry" +task_alias: "chemistry" +process_docs: !function utils.process_chemistry diff --git a/lm_eval/tasks/mmlu_pro/mmlu_pro_computer_science.yaml b/lm_eval/tasks/mmlu_pro/mmlu_pro_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7de347373e7b92d8eb9ba33bbf8ec4fb2a3dbc49 --- /dev/null +++ b/lm_eval/tasks/mmlu_pro/mmlu_pro_computer_science.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about computer science. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_computer_science" +task_alias: "computer_science" +process_docs: !function utils.process_computer_science diff --git a/lm_eval/tasks/mmlu_pro/mmlu_pro_economics.yaml b/lm_eval/tasks/mmlu_pro/mmlu_pro_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..274612783fb749aad10ec67698c3abf5dce13ebf --- /dev/null +++ b/lm_eval/tasks/mmlu_pro/mmlu_pro_economics.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about economics. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_economics" +task_alias: "economics" +process_docs: !function utils.process_economics diff --git a/lm_eval/tasks/mmlu_pro/mmlu_pro_engineering.yaml b/lm_eval/tasks/mmlu_pro/mmlu_pro_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dcf02f5029823ab83c1cebfbbf9819ac5ec4f1d0 --- /dev/null +++ b/lm_eval/tasks/mmlu_pro/mmlu_pro_engineering.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about engineering. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_engineering" +task_alias: "engineering" +process_docs: !function utils.process_engineering diff --git a/lm_eval/tasks/mmlu_pro/mmlu_pro_health.yaml b/lm_eval/tasks/mmlu_pro/mmlu_pro_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d161d1d81a29a06a9425bafc9e86211a2669b9be --- /dev/null +++ b/lm_eval/tasks/mmlu_pro/mmlu_pro_health.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about health. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_health" +task_alias: "health" +process_docs: !function utils.process_health diff --git a/lm_eval/tasks/mmlu_pro/mmlu_pro_history.yaml b/lm_eval/tasks/mmlu_pro/mmlu_pro_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d28efd3e77ef31dc2df96847c90e7c76920007bc --- /dev/null +++ b/lm_eval/tasks/mmlu_pro/mmlu_pro_history.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about history. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_history" +task_alias: "history" +process_docs: !function utils.process_history diff --git a/lm_eval/tasks/mmlu_pro/mmlu_pro_law.yaml b/lm_eval/tasks/mmlu_pro/mmlu_pro_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ba99f16dbb8df07ab9272d8b47d23839a919bc9c --- /dev/null +++ b/lm_eval/tasks/mmlu_pro/mmlu_pro_law.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about law. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_law" +task_alias: "law" +process_docs: !function utils.process_law diff --git a/lm_eval/tasks/mmlu_pro/mmlu_pro_math.yaml b/lm_eval/tasks/mmlu_pro/mmlu_pro_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0526f11f975f556f94c21acbc7074c0b4b76dd42 --- /dev/null +++ b/lm_eval/tasks/mmlu_pro/mmlu_pro_math.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about math. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_math" +task_alias: "math" +process_docs: !function utils.process_math diff --git a/lm_eval/tasks/mmlu_pro/mmlu_pro_other.yaml b/lm_eval/tasks/mmlu_pro/mmlu_pro_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..beb2ec9da7011dd1437828fc442ad21733cdf1a8 --- /dev/null +++ b/lm_eval/tasks/mmlu_pro/mmlu_pro_other.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about other. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_other" +task_alias: "other" +process_docs: !function utils.process_other diff --git a/lm_eval/tasks/mmlu_pro/mmlu_pro_philosophy.yaml b/lm_eval/tasks/mmlu_pro/mmlu_pro_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..99e5d65b4c0af9a2a35589ab104cb07be100cabc --- /dev/null +++ b/lm_eval/tasks/mmlu_pro/mmlu_pro_philosophy.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about philosophy. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_philosophy" +task_alias: "philosophy" +process_docs: !function utils.process_philosophy diff --git a/lm_eval/tasks/mmlu_pro/mmlu_pro_physics.yaml b/lm_eval/tasks/mmlu_pro/mmlu_pro_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7e7fa740bd58ebd55382bd8464abd9abaee3d96e --- /dev/null +++ b/lm_eval/tasks/mmlu_pro/mmlu_pro_physics.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about physics. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_physics" +task_alias: "physics" +process_docs: !function utils.process_physics diff --git a/lm_eval/tasks/mmlu_pro/mmlu_pro_psychology.yaml b/lm_eval/tasks/mmlu_pro/mmlu_pro_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b28fb72d329d4d9755e45201275604d257851754 --- /dev/null +++ b/lm_eval/tasks/mmlu_pro/mmlu_pro_psychology.yaml @@ -0,0 +1,5 @@ +description: "The following are multiple choice questions (with answers) about psychology. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n" +include: "_default_template_yaml" +task: "mmlu_pro_psychology" +task_alias: "psychology" +process_docs: !function utils.process_psychology diff --git a/lm_eval/tasks/mmlu_pro/utils.py b/lm_eval/tasks/mmlu_pro/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..03117be5f165fd7edf40404bf9934b3753039f1d --- /dev/null +++ b/lm_eval/tasks/mmlu_pro/utils.py @@ -0,0 +1,63 @@ +from functools import partial + + +choices = [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", +] + + +def format_cot_example(example, including_answer=True): + prompt = "Question:\n" + question = example["question"] + options = example["options"] + prompt += question + "\n" + prompt += "Options:\n" + for i, opt in enumerate(options): + prompt += "{}. {}\n".format(choices[i], opt) + if including_answer: + cot_content = example["cot_content"].replace( + "A: Let's think step by step.", "Answer: Let's think step by step." + ) + prompt += cot_content + "\n\n" + else: + prompt += "Answer: Let's think step by step." + return prompt + + +doc_to_text = partial(format_cot_example, including_answer=False) +fewshot_to_text = partial(format_cot_example, including_answer=True) + + +def process_docs(dataset, subject): + return dataset.filter(lambda x: x["category"] == subject) + + +process_biology = partial(process_docs, subject="biology") +process_business = partial(process_docs, subject="business") +process_chemistry = partial(process_docs, subject="chemistry") +process_computer_science = partial(process_docs, subject="computer science") +process_economics = partial(process_docs, subject="economics") +process_engineering = partial(process_docs, subject="engineering") +process_health = partial(process_docs, subject="health") +process_history = partial(process_docs, subject="history") +process_law = partial(process_docs, subject="law") +process_math = partial(process_docs, subject="math") +process_other = partial(process_docs, subject="other") +process_philosophy = partial(process_docs, subject="philosophy") +process_physics = partial(process_docs, subject="physics") +process_psychology = partial(process_docs, subject="psychology") diff --git a/lm_eval/tasks/mmlusr/README.md b/lm_eval/tasks/mmlusr/README.md new file mode 100644 index 0000000000000000000000000000000000000000..6d8a79fbab7bf6373b75481267791553332536a5 --- /dev/null +++ b/lm_eval/tasks/mmlusr/README.md @@ -0,0 +1,64 @@ +# MMLU-SR + +## Paper +Title: [Reasoning or Simply Next Token Prediction? A Benchmark for Stress-Testing Large Language Models](https://arxiv.org/abs/2406.15468v1) + + +We propose MMLU-SR, a novel dataset designed to measure the true comprehension abilities of Large Language Models (LLMs) by challenging their performance in question-answering tasks with modified terms. We reasoned that an agent that ``truly'' understands a concept can still evaluate it when key terms are replaced by suitably defined alternate terms, and sought to differentiate such comprehension from mere text replacement. In our study, we modified standardized test questions by replacing a key term with a dummy word along with its definition. The key term could be in the context of questions, answers, or both questions and answers. +Notwithstanding the high scores achieved by recent popular LLMs on the MMLU leaderboard, we found a substantial reduction in model performance after such replacement, suggesting poor comprehension. This new benchmark provides a rigorous benchmark for testing true model comprehension, and poses a challenge to the broader scientific community. + +Github Homepage: [https://github.com/Wang-ML-Lab/MMLU-SR](https://github.com/Wang-ML-Lab/MMLU-SR) +Huggingface Dataset: [https://huggingface.co/datasets/NiniCat/MMLU-SR]([https://huggingface.co/datasets/NiniCat/MMLU-SR) + + +## Citation +```bib +@misc{wang2024reasoningsimplytokenprediction, + title={Reasoning or Simply Next Token Prediction? A Benchmark for Stress-Testing Large Language Models}, + author={Wentian Wang and Paul Kantor and Jacob Feldman and Lazaros Gallos and Hao Wang}, + year={2024}, + eprint={2406.15468}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2406.15468}, +} +``` + +### Groups and Tasks + +#### Groups + +- `mmlusr`: MMLU variant where the terminology in the question and answers are modified. +- `mmlusr_answer_only`: MMLU variant where the terminology in the answers are modified. +- `mmlusr_question_only`: MMLU variant where the terminology in the question is modified. + +#### Tasks + +There are 57 symbol replaced subjects in each group. You can run a single task by: + +* `mmlusr_question_only_abstract_algebra` + +Or by categories: + +* `mmlusr_question_only_stem_tasks ` + + +### Checklist + +The checklist is the following: + +For adding novel benchmarks/datasets to the library: +* [x] Is the task an existing benchmark in the literature? + * [x] Have you referenced the original paper that introduced the task? + * [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + * The implementation in the original paper is one where the model is first fine-tuned on the data. They do have a few-shot evaluation for GPT-3, however the few-shot context used here is sourced from [Lewkowycz et al](https://arxiv.org/abs/2206.14858). The achieved accuracy on Llama-2 models is comparable to that provided in the paper, though not identical. + + +If other tasks on this dataset are already supported: +* [x] Is the "Main" variant of this task clearly denoted? +* [x] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [x] Have you noted which, if any, published evaluation setups are matched by this variant? + +### Variant Wishlist + +- [ ] zero-shot variant diff --git a/lm_eval/tasks/mmlusr/answer_only/_answer_only.yaml b/lm_eval/tasks/mmlusr/answer_only/_answer_only.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2473a2af3ade70dd004a3756a61d9b83bba36179 --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/_answer_only.yaml @@ -0,0 +1,44 @@ +group: mmlusr_answer_only +group_alias: MMLU-SR (Answer Only) +task: + - group: mmlusr_ao_stem + group_alias: STEM (Answer Only) + task: + - mmlusr_answer_only_stem_tasks + aggregate_metric_list: + - metric: acc + weight_by_size: True + metadata: + version: 2 + - group: mmlusr_ao_other + group_alias: Other (Answer Only) + task: + - mmlusr_answer_only_other_tasks + aggregate_metric_list: + - metric: acc + weight_by_size: True + metadata: + version: 2 + - group: mmlusr_ao_social_sciences + group_alias: Social Sciences (Answer Only) + task: + - mmlusr_answer_only_social_sciences_tasks + aggregate_metric_list: + - metric: acc + weight_by_size: True + metadata: + version: 2 + - group: mmlusr_ao_humanities + group_alias: Humanities (Answer Only) + task: + - mmlusr_answer_only_humanities_tasks + aggregate_metric_list: + - metric: acc + weight_by_size: True + metadata: + version: 2 +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 2 diff --git a/lm_eval/tasks/mmlusr/answer_only/_mmlusr_a_yml b/lm_eval/tasks/mmlusr/answer_only/_mmlusr_a_yml new file mode 100644 index 0000000000000000000000000000000000000000..7489f544ba29aee29332dd11197461a025ef1494 --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/_mmlusr_a_yml @@ -0,0 +1,16 @@ +dataset_path: NiniCat/MMLU-SR +test_split: test +fewshot_split: train +fewshot_config: + sampler: first_n +output_type: multiple_choice +process_docs: !function utils.process_docs +doc_to_text: "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_abstract_algebra.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..527bc9cc1b3ebe1004e26a68e5fffcee7158d4ef --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_abstract_algebra.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_abstract_algebra" +"description": "The following are multiple choice questions (with answers) about abstract\ + \ algebra.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_abstract_algebra" +"task_alias": "abstract algebra" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_anatomy.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1e4acc8c3810cc6f8bbe32216b8d0cdfa9463135 --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_anatomy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_anatomy" +"description": "The following are multiple choice questions (with answers) about anatomy.\n\ + \n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_anatomy" +"task_alias": "anatomy" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_astronomy.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..068072de60489d7c26b576314862c73492af92c8 --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_astronomy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_astronomy" +"description": "The following are multiple choice questions (with answers) about astronomy.\n\ + \n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_astronomy" +"task_alias": "astronomy" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_business_ethics.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1e836e31c55e8cb6b203049f75c13aaf57870a6f --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_business_ethics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_business_ethics" +"description": "The following are multiple choice questions (with answers) about business\ + \ ethics.\n\n" +"tag": "mmlusr_answer_only_other_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_business_ethics" +"task_alias": "business ethics" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_clinical_knowledge.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1ef709675c2c7016d2aabbcde3bb3ff3dcafcbcc --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_clinical_knowledge.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_clinical_knowledge" +"description": "The following are multiple choice questions (with answers) about clinical\ + \ knowledge.\n\n" +"tag": "mmlusr_answer_only_other_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_clinical_knowledge" +"task_alias": "clinical knowledge" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_college_biology.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b967895a7017094a9b5f78ad9ddbeff7a56ab6f9 --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_college_biology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_college_biology" +"description": "The following are multiple choice questions (with answers) about college\ + \ biology.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_college_biology" +"task_alias": "college biology" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_college_chemistry.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8dd100e7bc82bcc44d7f121e0dab7852dbc69074 --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_college_chemistry.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_college_chemistry" +"description": "The following are multiple choice questions (with answers) about college\ + \ chemistry.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_college_chemistry" +"task_alias": "college chemistry" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_college_computer_science.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bbd7e4c1582a780028ddaf3cb2f1824dd4b3ebb2 --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_college_computer_science.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_college_computer_science" +"description": "The following are multiple choice questions (with answers) about college\ + \ computer science.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_college_computer_science" +"task_alias": "college computer science" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_college_mathematics.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8d85c49dc1c76b880ea1bc4ae255c2d152d6eaa8 --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_college_mathematics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_college_mathematics" +"description": "The following are multiple choice questions (with answers) about college\ + \ mathematics.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_college_mathematics" +"task_alias": "college mathematics" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_college_medicine.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..132e0b6041126c058b61a3ddcb27e51383134c77 --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_college_medicine.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_college_medicine" +"description": "The following are multiple choice questions (with answers) about college\ + \ medicine.\n\n" +"tag": "mmlusr_answer_only_other_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_college_medicine" +"task_alias": "college medicine" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_college_physics.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..77b47241d095b8a62ebdfe2d995ad438c69de4ec --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_college_physics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_college_physics" +"description": "The following are multiple choice questions (with answers) about college\ + \ physics.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_college_physics" +"task_alias": "college physics" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_computer_security.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ba3d60d51b852e27637d1c1ab43af1d4da7fc057 --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_computer_security.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_computer_security" +"description": "The following are multiple choice questions (with answers) about computer\ + \ security.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_computer_security" +"task_alias": "computer security" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_conceptual_physics.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e0a84ecc3c5015d2338cfffc88877e39b9be7c75 --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_conceptual_physics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_conceptual_physics" +"description": "The following are multiple choice questions (with answers) about conceptual\ + \ physics.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_conceptual_physics" +"task_alias": "conceptual physics" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_econometrics.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..996d44f46eab624e768c92553659847682e2f2cc --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_econometrics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_econometrics" +"description": "The following are multiple choice questions (with answers) about econometrics.\n\ + \n" +"tag": "mmlusr_answer_only_social_sciences_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_econometrics" +"task_alias": "econometrics" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_electrical_engineering.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ab695e6ab46ea036cdccd91487c47aca35f1a99a --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_electrical_engineering.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_electrical_engineering" +"description": "The following are multiple choice questions (with answers) about electrical\ + \ engineering.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_electrical_engineering" +"task_alias": "electrical engineering" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_elementary_mathematics.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dff9fbf25b49d3b59d0ae716cd4bf3a210547742 --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_elementary_mathematics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_elementary_mathematics" +"description": "The following are multiple choice questions (with answers) about elementary\ + \ mathematics.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_elementary_mathematics" +"task_alias": "elementary mathematics" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_formal_logic.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e26ed865bc94d0923f7f1103d89f556a914df8aa --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_formal_logic.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_formal_logic" +"description": "The following are multiple choice questions (with answers) about formal\ + \ logic.\n\n" +"tag": "mmlusr_answer_only_humanities_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_formal_logic" +"task_alias": "formal logic" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_global_facts.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ec9c0f42b35eda761c005ddb94670e649a0a9ee8 --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_global_facts.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_global_facts" +"description": "The following are multiple choice questions (with answers) about global\ + \ facts.\n\n" +"tag": "mmlusr_answer_only_other_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_global_facts" +"task_alias": "global facts" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_biology.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..41ed53cb9a75af634f1b241da2345985f72b65e5 --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_biology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_high_school_biology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school biology.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_high_school_biology" +"task_alias": "high school biology" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_chemistry.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..95a3303f3de284b7759a08dba7440398d009296f --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_chemistry.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_high_school_chemistry" +"description": "The following are multiple choice questions (with answers) about high\ + \ school chemistry.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_high_school_chemistry" +"task_alias": "high school chemistry" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_computer_science.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e665fb3400855e8eefc2b8d8dc8054bb0b48279e --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_computer_science.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_high_school_computer_science" +"description": "The following are multiple choice questions (with answers) about high\ + \ school computer science.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_high_school_computer_science" +"task_alias": "high school computer science" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_european_history.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9d7c1cb8da0a7de39a28efbb39f943b0d230d93f --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_european_history.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_high_school_european_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school european history.\n\n" +"tag": "mmlusr_answer_only_humanities_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_high_school_european_history" +"task_alias": "high school european history" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_geography.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8a4980060190c2d9870a7da722d8a28aa3d2f3af --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_geography.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_high_school_geography" +"description": "The following are multiple choice questions (with answers) about high\ + \ school geography.\n\n" +"tag": "mmlusr_answer_only_social_sciences_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_high_school_geography" +"task_alias": "high school geography" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_government_and_politics.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bf66e3a3a760c5cf5b81d4b83323230c3aafe1ed --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_government_and_politics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_high_school_government_and_politics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school government and politics.\n\n" +"tag": "mmlusr_answer_only_social_sciences_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_high_school_government_and_politics" +"task_alias": "high school government and politics" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_macroeconomics.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..95e35cd8b177c7aab73d90f85f29812d4c2d69fb --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_macroeconomics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_high_school_macroeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school macroeconomics.\n\n" +"tag": "mmlusr_answer_only_social_sciences_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_high_school_macroeconomics" +"task_alias": "high school macroeconomics" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_mathematics.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7da2d1859ad5c85fa25d3579156e66d9ec2e5e45 --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_mathematics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_high_school_mathematics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school mathematics.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_high_school_mathematics" +"task_alias": "high school mathematics" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_microeconomics.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e3af9a2c79bf4f5c090a309138139350d40581be --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_microeconomics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_high_school_microeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school microeconomics.\n\n" +"tag": "mmlusr_answer_only_social_sciences_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_high_school_microeconomics" +"task_alias": "high school microeconomics" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_physics.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..52fb7377927412590328af0a2ad21c845ba21322 --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_physics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_high_school_physics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school physics.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_high_school_physics" +"task_alias": "high school physics" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_psychology.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..df77619cbb8d2b8ccc42083beac223920833e3ec --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_psychology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_high_school_psychology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school psychology.\n\n" +"tag": "mmlusr_answer_only_social_sciences_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_high_school_psychology" +"task_alias": "high school psychology" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_statistics.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2119fb39d1cf03028a61f6f57cac66343ba308ba --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_statistics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_high_school_statistics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school statistics.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_high_school_statistics" +"task_alias": "high school statistics" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_us_history.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2287ae457a1f4013ac607955f0050d7ce7f1caad --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_us_history.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_high_school_us_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school us history.\n\n" +"tag": "mmlusr_answer_only_humanities_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_high_school_us_history" +"task_alias": "high school us history" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_world_history.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5b8f4f37e2e23019f522ecd0c39406485943d808 --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_high_school_world_history.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_high_school_world_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school world history.\n\n" +"tag": "mmlusr_answer_only_humanities_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_high_school_world_history" +"task_alias": "high school world history" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_human_aging.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6a188ddb6517a367d83b3791c0875d9fd01ad56f --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_human_aging.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_human_aging" +"description": "The following are multiple choice questions (with answers) about human\ + \ aging.\n\n" +"tag": "mmlusr_answer_only_other_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_human_aging" +"task_alias": "human aging" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_human_sexuality.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..18c45333c5ba7b3a5cc80fa340c7f58481f4a4b3 --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_human_sexuality.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_human_sexuality" +"description": "The following are multiple choice questions (with answers) about human\ + \ sexuality.\n\n" +"tag": "mmlusr_answer_only_social_sciences_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_human_sexuality" +"task_alias": "human sexuality" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_international_law.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..05e482d1683ab518d7b7258ee99b8cc532af0f00 --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_international_law.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_international_law" +"description": "The following are multiple choice questions (with answers) about international\ + \ law.\n\n" +"tag": "mmlusr_answer_only_humanities_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_international_law" +"task_alias": "international law" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_jurisprudence.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..73edd6cb293f97e2ae112c159602fc6f57caeb24 --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_jurisprudence.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_jurisprudence" +"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ + \n" +"tag": "mmlusr_answer_only_humanities_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_jurisprudence" +"task_alias": "jurisprudence" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_logical_fallacies.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ab18c9270e07ff3101fcb25e219f94e1ff80aef0 --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_logical_fallacies.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_logical_fallacies" +"description": "The following are multiple choice questions (with answers) about logical\ + \ fallacies.\n\n" +"tag": "mmlusr_answer_only_humanities_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_logical_fallacies" +"task_alias": "logical fallacies" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_machine_learning.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1b833c706f24750d113100414d7b8857c936899e --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_machine_learning.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_machine_learning" +"description": "The following are multiple choice questions (with answers) about machine\ + \ learning.\n\n" +"tag": "mmlusr_answer_only_stem_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_machine_learning" +"task_alias": "machine learning" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_management.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..26ec67401daf8f5e3af5f6c4b1ca8937927a6850 --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_management.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_management" +"description": "The following are multiple choice questions (with answers) about management.\n\ + \n" +"tag": "mmlusr_answer_only_other_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_management" +"task_alias": "management" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_marketing.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..23fe03659ba9df8e12873d9a3065de6093b8834e --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_marketing.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_marketing" +"description": "The following are multiple choice questions (with answers) about marketing.\n\ + \n" +"tag": "mmlusr_answer_only_other_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_marketing" +"task_alias": "marketing" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_medical_genetics.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..63355c88aa30126955f5fc14be0b8b54f47a5169 --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_medical_genetics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_medical_genetics" +"description": "The following are multiple choice questions (with answers) about medical\ + \ genetics.\n\n" +"tag": "mmlusr_answer_only_other_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_medical_genetics" +"task_alias": "medical genetics" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_miscellaneous.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..12153929983be65dcbcbc59b195414607dd1ecae --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_miscellaneous.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_miscellaneous" +"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ + \n" +"tag": "mmlusr_answer_only_other_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_miscellaneous" +"task_alias": "miscellaneous" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_moral_disputes.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2f09854fbaa531d5a6e694f578dcf18ac1947395 --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_moral_disputes.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_moral_disputes" +"description": "The following are multiple choice questions (with answers) about moral\ + \ disputes.\n\n" +"tag": "mmlusr_answer_only_humanities_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_moral_disputes" +"task_alias": "moral disputes" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_moral_scenarios.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dee1c01eb1b9ca1cb85ae2c0ba8d0638a78c9515 --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_moral_scenarios.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_moral_scenarios" +"description": "The following are multiple choice questions (with answers) about moral\ + \ scenarios.\n\n" +"tag": "mmlusr_answer_only_humanities_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_moral_scenarios" +"task_alias": "moral scenarios" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_nutrition.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a890f9331b76cfcfd706cec97c87ae9b358f9b33 --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_nutrition.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_nutrition" +"description": "The following are multiple choice questions (with answers) about nutrition.\n\ + \n" +"tag": "mmlusr_answer_only_other_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_nutrition" +"task_alias": "nutrition" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_philosophy.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..538dea756ccc6b97d3071922fa1a2f67d5daede8 --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_philosophy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_philosophy" +"description": "The following are multiple choice questions (with answers) about philosophy.\n\ + \n" +"tag": "mmlusr_answer_only_humanities_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_philosophy" +"task_alias": "philosophy" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_prehistory.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a93b5c4ff7034736e17af2b48e876f2ccdadbff9 --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_prehistory.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_prehistory" +"description": "The following are multiple choice questions (with answers) about prehistory.\n\ + \n" +"tag": "mmlusr_answer_only_humanities_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_prehistory" +"task_alias": "prehistory" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_professional_accounting.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b9f45995cbf30cf4453f30ed47a644718349c695 --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_professional_accounting.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_professional_accounting" +"description": "The following are multiple choice questions (with answers) about professional\ + \ accounting.\n\n" +"tag": "mmlusr_answer_only_other_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_professional_accounting" +"task_alias": "professional accounting" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_professional_law.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..caccccf0ded8e25f7c92e4788885583f5f014168 --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_professional_law.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_professional_law" +"description": "The following are multiple choice questions (with answers) about professional\ + \ law.\n\n" +"tag": "mmlusr_answer_only_humanities_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_professional_law" +"task_alias": "professional law" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_professional_medicine.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..374b239c3fbda26af4691d302350cf60216e7ede --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_professional_medicine.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_professional_medicine" +"description": "The following are multiple choice questions (with answers) about professional\ + \ medicine.\n\n" +"tag": "mmlusr_answer_only_other_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_professional_medicine" +"task_alias": "professional medicine" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_professional_psychology.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..58a9fc2d31f1c9758e0fbb9b90f13922cfb20247 --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_professional_psychology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_professional_psychology" +"description": "The following are multiple choice questions (with answers) about professional\ + \ psychology.\n\n" +"tag": "mmlusr_answer_only_social_sciences_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_professional_psychology" +"task_alias": "professional psychology" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_public_relations.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..86cc337b0642aa2043df46db35aa26b2c77e8ccc --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_public_relations.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_public_relations" +"description": "The following are multiple choice questions (with answers) about public\ + \ relations.\n\n" +"tag": "mmlusr_answer_only_social_sciences_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_public_relations" +"task_alias": "public relations" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_security_studies.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5e72f02f5595c91afbb06a97414bc78866c47203 --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_security_studies.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_security_studies" +"description": "The following are multiple choice questions (with answers) about security\ + \ studies.\n\n" +"tag": "mmlusr_answer_only_social_sciences_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_security_studies" +"task_alias": "security studies" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_sociology.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..58fa3d8de1b736f727119945ec4c59470f550d6e --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_sociology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_sociology" +"description": "The following are multiple choice questions (with answers) about sociology.\n\ + \n" +"tag": "mmlusr_answer_only_social_sciences_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_sociology" +"task_alias": "sociology" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_us_foreign_policy.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..91a6d66340278bc0ff53085d88c910779d76d7e8 --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_us_foreign_policy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_us_foreign_policy" +"description": "The following are multiple choice questions (with answers) about us\ + \ foreign policy.\n\n" +"tag": "mmlusr_answer_only_social_sciences_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_us_foreign_policy" +"task_alias": "us foreign policy" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_virology.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1400fb8421512daebaf5640a8385e69ce645e59d --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_virology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_virology" +"description": "The following are multiple choice questions (with answers) about virology.\n\ + \n" +"tag": "mmlusr_answer_only_other_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_virology" +"task_alias": "virology" diff --git a/lm_eval/tasks/mmlusr/answer_only/answer_only_world_religions.yaml b/lm_eval/tasks/mmlusr/answer_only/answer_only_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6014213538672bf733088ea63b9d1262f47a1ff4 --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/answer_only_world_religions.yaml @@ -0,0 +1,7 @@ +"dataset_name": "answer_only_world_religions" +"description": "The following are multiple choice questions (with answers) about world\ + \ religions.\n\n" +"tag": "mmlusr_answer_only_humanities_tasks" +"include": "_mmlusr_a_yml" +"task": "mmlusr_answer_only_world_religions" +"task_alias": "world religions" diff --git a/lm_eval/tasks/mmlusr/answer_only/utils.py b/lm_eval/tasks/mmlusr/answer_only/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..f1341bd59050caa11c56a9a36210428417e6c9f4 --- /dev/null +++ b/lm_eval/tasks/mmlusr/answer_only/utils.py @@ -0,0 +1,19 @@ +import datasets + + +def process_docs(dataset: datasets.Dataset) -> datasets.Dataset: + def _helper(doc): + # Assuming that the 'answer' field in the dataset now contains numbers 0-3 instead of 'A', 'B', 'C', 'D' + answer_list = ["A", "B", "C", "D"] + # Convert numeric index to corresponding letter + answer_index = int(doc["answer"]) # Make sure the answer is an integer + answer_letter = answer_list[answer_index] + + out_doc = { + "questions": doc["question"], + "choices": [doc["choice1"], doc["choice2"], doc["choice3"], doc["choice4"]], + "answer": answer_letter, # Include the letter for clarity + } + return out_doc + + return dataset.map(_helper) diff --git a/lm_eval/tasks/mmlusr/config.py b/lm_eval/tasks/mmlusr/config.py new file mode 100644 index 0000000000000000000000000000000000000000..527ebf1f2fcafe0524dc2e1ec143e8c8c064086e --- /dev/null +++ b/lm_eval/tasks/mmlusr/config.py @@ -0,0 +1,154 @@ +""" +Take in a YAML, and output all "other" splits with this YAML +""" + +import argparse +import logging +import os + +import yaml +from tqdm import tqdm + + +eval_logger = logging.getLogger("lm-eval") + + +SUBJECTS = { + "abstract_algebra": "stem", + "anatomy": "stem", + "astronomy": "stem", + "business_ethics": "other", + "clinical_knowledge": "other", + "college_biology": "stem", + "college_chemistry": "stem", + "college_computer_science": "stem", + "college_mathematics": "stem", + "college_medicine": "other", + "college_physics": "stem", + "computer_security": "stem", + "conceptual_physics": "stem", + "econometrics": "social_sciences", + "electrical_engineering": "stem", + "elementary_mathematics": "stem", + "formal_logic": "humanities", + "global_facts": "other", + "high_school_biology": "stem", + "high_school_chemistry": "stem", + "high_school_computer_science": "stem", + "high_school_european_history": "humanities", + "high_school_geography": "social_sciences", + "high_school_government_and_politics": "social_sciences", + "high_school_macroeconomics": "social_sciences", + "high_school_mathematics": "stem", + "high_school_microeconomics": "social_sciences", + "high_school_physics": "stem", + "high_school_psychology": "social_sciences", + "high_school_statistics": "stem", + "high_school_us_history": "humanities", + "high_school_world_history": "humanities", + "human_aging": "other", + "human_sexuality": "social_sciences", + "international_law": "humanities", + "jurisprudence": "humanities", + "logical_fallacies": "humanities", + "machine_learning": "stem", + "management": "other", + "marketing": "other", + "medical_genetics": "other", + "miscellaneous": "other", + "moral_disputes": "humanities", + "moral_scenarios": "humanities", + "nutrition": "other", + "philosophy": "humanities", + "prehistory": "humanities", + "professional_accounting": "other", + "professional_law": "humanities", + "professional_medicine": "other", + "professional_psychology": "social_sciences", + "public_relations": "social_sciences", + "security_studies": "social_sciences", + "sociology": "social_sciences", + "us_foreign_policy": "social_sciences", + "virology": "other", + "world_religions": "humanities", +} + +GROUPS = ["question_and_answer"] + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Generate configuration YAML files for LM Evaluation Harness." + ) + # Path to the base YAML file from which to inherit settings + parser.add_argument( + "--base_yaml_path", + required=True, + help="Path to the base YAML configuration file.", + ) + + # Directory where the generated YAML files will be saved + parser.add_argument( + "--save_dir", + default="/data/local/cat/lm-evaluation-harness/lm_eval/tasks/mmlusr/question_and_answer", + ) + + # Optional prefix to add to task names in the YAML files + parser.add_argument("--task_prefix", default="") + + parser.add_argument("--cot_prompt_path", default=None) + + # Optional prefix to add to group names in the YAML files + parser.add_argument("--group_prefix", default="") + + return parser.parse_args() + + +if __name__ == "__main__": + args = parse_args() + + # Load base YAML configuration + base_yaml_name = os.path.basename(args.base_yaml_path) + with open(args.base_yaml_path, "r", encoding="utf-8") as f: + base_yaml = yaml.full_load(f) + + if args.cot_prompt_path is not None: + import json + + with open(args.cot_prompt_path, encoding="utf-8") as f: + cot_file = json.load(f) + + for group in GROUPS: + for subject, category in tqdm(SUBJECTS.items()): + if args.cot_prompt_path is not None: + description = cot_file[subject] + else: + description = f"The following are multiple choice questions (with answers) about {' '.join(subject.split('_'))}.\n\n" + + yaml_dict = { + "include": base_yaml_name, + "tag": f"mmlusr_{args.group_prefix}{group}_{category}" + if args.group_prefix + else f"mmlusr_{group}_{category}", + "task": f"mmlusr_{args.task_prefix}{group}_{subject}" + if args.task_prefix + else f"mmlusr_{group}_{subject}", + "task_alias": subject.replace("_", " "), + "description": description, + "dataset_name": f"{group}_{subject}", + } + + # File path for saving the generated YAML file + file_save_path = os.path.join(args.save_dir, f"{group}_{subject}.yaml") + with open(file_save_path, "w", encoding="utf-8") as yaml_file: + yaml.dump(yaml_dict, yaml_file, allow_unicode=True, default_style='"') + eval_logger.info(f"Saved YAML for {group} {subject} to {file_save_path}") + + # Save group configuration if specified + if args.group_prefix: + file_save_path = os.path.join( + args.save_prefix_path, args.group_prefix + ".yaml" + ) + eval_logger.info(f"Saving benchmark config to {file_save_path}") + with open(file_save_path, "w", encoding="utf-8") as yaml_file: + yaml.dump(yaml_dict, yaml_file, indent=4, default_flow_style=False) diff --git a/lm_eval/tasks/mmlusr/question_and_answer/_mmlusr_qna_yml b/lm_eval/tasks/mmlusr/question_and_answer/_mmlusr_qna_yml new file mode 100644 index 0000000000000000000000000000000000000000..7489f544ba29aee29332dd11197461a025ef1494 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/_mmlusr_qna_yml @@ -0,0 +1,16 @@ +dataset_path: NiniCat/MMLU-SR +test_split: test +fewshot_split: train +fewshot_config: + sampler: first_n +output_type: multiple_choice +process_docs: !function utils.process_docs +doc_to_text: "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/mmlusr/question_and_answer/_question_and_answer.yaml b/lm_eval/tasks/mmlusr/question_and_answer/_question_and_answer.yaml new file mode 100644 index 0000000000000000000000000000000000000000..390cbc02770f8834623caa7aaadc61673b8b1316 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/_question_and_answer.yaml @@ -0,0 +1,44 @@ +group: mmlusr +group_alias: MMLU-SR (Question & Answer) +task: + - group: mmlusr_qa_stem + group_alias: STEM (Question & Answer) + task: + - mmlusr_question_and_answer_stem_tasks + aggregate_metric_list: + - metric: acc + weight_by_size: True + metadata: + version: 2 + - group: mmlusr_qa_other + group_alias: Other (Question & Answer) + task: + - mmlusr_question_and_answer_other_tasks + aggregate_metric_list: + - metric: acc + weight_by_size: True + metadata: + version: 2 + - group: mmlusr_qa_social_sciences + group_alias: Social Sciences (Question & Answer) + task: + - mmlusr_question_and_answer_social_sciences_tasks + aggregate_metric_list: + - metric: acc + weight_by_size: True + metadata: + version: 2 + - group: mmlusr_qa_humanities + group_alias: Humanities (Question & Answer) + task: + - mmlusr_question_and_answer_humanities_tasks + aggregate_metric_list: + - metric: acc + weight_by_size: True + metadata: + version: 2 +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 2 diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_abstract_algebra.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bfdd80e6422675dd71ba5b9c50d1a4f7d70f1094 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_abstract_algebra.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_abstract_algebra" +"description": "The following are multiple choice questions (with answers) about abstract\ + \ algebra.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_abstract_algebra" +"task_alias": "abstract algebra" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_anatomy.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..316bede4232b218f1c9beda91c9b7b3069aebc9b --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_anatomy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_anatomy" +"description": "The following are multiple choice questions (with answers) about anatomy.\n\ + \n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_anatomy" +"task_alias": "anatomy" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_astronomy.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e9f89e1c9700d86611e1f0ec6079ce31cf8723d0 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_astronomy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_astronomy" +"description": "The following are multiple choice questions (with answers) about astronomy.\n\ + \n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_astronomy" +"task_alias": "astronomy" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_business_ethics.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4a46298259c80432c775638a847511c8159e9b75 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_business_ethics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_business_ethics" +"description": "The following are multiple choice questions (with answers) about business\ + \ ethics.\n\n" +"tag": "mmlusr_question_and_answer_other_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_business_ethics" +"task_alias": "business ethics" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_clinical_knowledge.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c43c9a3d54e1a757f1653f7f9d4027fe0052db4e --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_clinical_knowledge.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_clinical_knowledge" +"description": "The following are multiple choice questions (with answers) about clinical\ + \ knowledge.\n\n" +"tag": "mmlusr_question_and_answer_other_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_clinical_knowledge" +"task_alias": "clinical knowledge" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_biology.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4f00615bdf93f1e4f86e71cc070e825b098482d7 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_biology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_college_biology" +"description": "The following are multiple choice questions (with answers) about college\ + \ biology.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_college_biology" +"task_alias": "college biology" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_chemistry.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..837bc2253831748957f28911fe56fdec68791e6d --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_chemistry.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_college_chemistry" +"description": "The following are multiple choice questions (with answers) about college\ + \ chemistry.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_college_chemistry" +"task_alias": "college chemistry" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_computer_science.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bb1c76395f154edcea570342f130f5afdfd1eeac --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_computer_science.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_college_computer_science" +"description": "The following are multiple choice questions (with answers) about college\ + \ computer science.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_college_computer_science" +"task_alias": "college computer science" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_mathematics.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..08c6e334c23f6f7f34a9f23c5e64acc7648dd3a7 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_mathematics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_college_mathematics" +"description": "The following are multiple choice questions (with answers) about college\ + \ mathematics.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_college_mathematics" +"task_alias": "college mathematics" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_medicine.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5c44360ab90f4fb3678dd7f6837b503cdbb47ec3 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_medicine.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_college_medicine" +"description": "The following are multiple choice questions (with answers) about college\ + \ medicine.\n\n" +"tag": "mmlusr_question_and_answer_other_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_college_medicine" +"task_alias": "college medicine" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_physics.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..372d6b9f20bc2b4eb9a8ddc6266a330dc6ea3b7a --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_college_physics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_college_physics" +"description": "The following are multiple choice questions (with answers) about college\ + \ physics.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_college_physics" +"task_alias": "college physics" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_computer_security.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8f85146a96059161990172672a688c3ac7922cea --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_computer_security.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_computer_security" +"description": "The following are multiple choice questions (with answers) about computer\ + \ security.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_computer_security" +"task_alias": "computer security" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_conceptual_physics.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7b37e1927f26b2963cbe86ca3a3f4f7c9a56337b --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_conceptual_physics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_conceptual_physics" +"description": "The following are multiple choice questions (with answers) about conceptual\ + \ physics.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_conceptual_physics" +"task_alias": "conceptual physics" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_econometrics.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d83b0ba9da42029ea88b4b80916f272841cba6b3 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_econometrics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_econometrics" +"description": "The following are multiple choice questions (with answers) about econometrics.\n\ + \n" +"tag": "mmlusr_question_and_answer_social_sciences_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_econometrics" +"task_alias": "econometrics" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_electrical_engineering.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1898c3a97e949d8b41a0b57d64f6d7cca2ff0795 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_electrical_engineering.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_electrical_engineering" +"description": "The following are multiple choice questions (with answers) about electrical\ + \ engineering.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_electrical_engineering" +"task_alias": "electrical engineering" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_elementary_mathematics.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c828feb107020806f43489f46791d2d7ba255662 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_elementary_mathematics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_elementary_mathematics" +"description": "The following are multiple choice questions (with answers) about elementary\ + \ mathematics.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_elementary_mathematics" +"task_alias": "elementary mathematics" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_formal_logic.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..294e99a46b2b81da8daf02a143f35bcaab85d6ad --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_formal_logic.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_formal_logic" +"description": "The following are multiple choice questions (with answers) about formal\ + \ logic.\n\n" +"tag": "mmlusr_question_and_answer_humanities_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_formal_logic" +"task_alias": "formal logic" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_global_facts.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..79c1a879a63d4805f5c74eacc12924bbb768df4c --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_global_facts.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_global_facts" +"description": "The following are multiple choice questions (with answers) about global\ + \ facts.\n\n" +"tag": "mmlusr_question_and_answer_other_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_global_facts" +"task_alias": "global facts" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_biology.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..90fe29fa46a24551b133e2c6882e7b2d41c0ac00 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_biology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_high_school_biology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school biology.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_high_school_biology" +"task_alias": "high school biology" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_chemistry.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3e4423ae0eebe7b638f73cbc7729d570975ed2c9 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_chemistry.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_high_school_chemistry" +"description": "The following are multiple choice questions (with answers) about high\ + \ school chemistry.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_high_school_chemistry" +"task_alias": "high school chemistry" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_computer_science.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fe1e3d49548039351950964207f0f4e884a5ac2c --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_computer_science.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_high_school_computer_science" +"description": "The following are multiple choice questions (with answers) about high\ + \ school computer science.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_high_school_computer_science" +"task_alias": "high school computer science" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_european_history.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..933d46f021948758a5ddb11fd1d44e29996ef146 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_european_history.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_high_school_european_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school european history.\n\n" +"tag": "mmlusr_question_and_answer_humanities_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_high_school_european_history" +"task_alias": "high school european history" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_geography.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fa99ad15d1f345e1fe7f7d07f50a723340d40e79 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_geography.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_high_school_geography" +"description": "The following are multiple choice questions (with answers) about high\ + \ school geography.\n\n" +"tag": "mmlusr_question_and_answer_social_sciences_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_high_school_geography" +"task_alias": "high school geography" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_government_and_politics.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b4835f4e09cfffc60344332a66f724c4838b4668 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_government_and_politics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_high_school_government_and_politics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school government and politics.\n\n" +"tag": "mmlusr_question_and_answer_social_sciences_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_high_school_government_and_politics" +"task_alias": "high school government and politics" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_macroeconomics.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..252ba9ceaf7cce7fa9636565a945cdbe53470494 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_macroeconomics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_high_school_macroeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school macroeconomics.\n\n" +"tag": "mmlusr_question_and_answer_social_sciences_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_high_school_macroeconomics" +"task_alias": "high school macroeconomics" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_mathematics.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f88bf56047e124b4349efbbcce7018de8649e7c5 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_mathematics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_high_school_mathematics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school mathematics.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_high_school_mathematics" +"task_alias": "high school mathematics" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_microeconomics.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bef2656aff6d404e69fa3fd4655969801d1fad76 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_microeconomics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_high_school_microeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school microeconomics.\n\n" +"tag": "mmlusr_question_and_answer_social_sciences_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_high_school_microeconomics" +"task_alias": "high school microeconomics" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_physics.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f02cc7fa7847f1564958ac7e3ccf7e15771b05bd --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_physics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_high_school_physics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school physics.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_high_school_physics" +"task_alias": "high school physics" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_psychology.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..df87039f7b5ab3118a1c6bd3530dad5642bc167f --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_psychology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_high_school_psychology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school psychology.\n\n" +"tag": "mmlusr_question_and_answer_social_sciences_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_high_school_psychology" +"task_alias": "high school psychology" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_statistics.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cbd9244a673ec274df54065fd325193eb2ac6128 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_statistics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_high_school_statistics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school statistics.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_high_school_statistics" +"task_alias": "high school statistics" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_us_history.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..efcf6898d2233c2f2e48dee853f02faae0fe468b --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_us_history.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_high_school_us_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school us history.\n\n" +"tag": "mmlusr_question_and_answer_humanities_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_high_school_us_history" +"task_alias": "high school us history" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_world_history.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a6488f284c0b8b535ad19a49a3999c06c88be079 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_high_school_world_history.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_high_school_world_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school world history.\n\n" +"tag": "mmlusr_question_and_answer_humanities_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_high_school_world_history" +"task_alias": "high school world history" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_human_aging.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b103d60f713daf41524ab25f47af17cf589f552a --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_human_aging.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_human_aging" +"description": "The following are multiple choice questions (with answers) about human\ + \ aging.\n\n" +"tag": "mmlusr_question_and_answer_other_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_human_aging" +"task_alias": "human aging" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_human_sexuality.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..eac93d56265e4bf731a524a0ac7270dea84f4ef0 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_human_sexuality.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_human_sexuality" +"description": "The following are multiple choice questions (with answers) about human\ + \ sexuality.\n\n" +"tag": "mmlusr_question_and_answer_social_sciences_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_human_sexuality" +"task_alias": "human sexuality" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_international_law.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5f7d5403cfc101bbc6f1ccbf2f833491b38fc03a --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_international_law.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_international_law" +"description": "The following are multiple choice questions (with answers) about international\ + \ law.\n\n" +"tag": "mmlusr_question_and_answer_humanities_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_international_law" +"task_alias": "international law" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_jurisprudence.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..775bee63388fb53adaa0f9ecd34ef36e2c82d272 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_jurisprudence.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_jurisprudence" +"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ + \n" +"tag": "mmlusr_question_and_answer_humanities_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_jurisprudence" +"task_alias": "jurisprudence" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_logical_fallacies.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1f2706a93f5d88e32c366ba4dee4ac18e87a24e1 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_logical_fallacies.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_logical_fallacies" +"description": "The following are multiple choice questions (with answers) about logical\ + \ fallacies.\n\n" +"tag": "mmlusr_question_and_answer_humanities_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_logical_fallacies" +"task_alias": "logical fallacies" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_machine_learning.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6299c4d06d84c7d3360771c6ca633b3230196b10 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_machine_learning.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_machine_learning" +"description": "The following are multiple choice questions (with answers) about machine\ + \ learning.\n\n" +"tag": "mmlusr_question_and_answer_stem_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_machine_learning" +"task_alias": "machine learning" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_management.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..60ae89e2891e290653289a4c4467924e3fed67f8 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_management.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_management" +"description": "The following are multiple choice questions (with answers) about management.\n\ + \n" +"tag": "mmlusr_question_and_answer_other_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_management" +"task_alias": "management" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_marketing.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4399b96ea620efaf67af9d5a5f28bea4353543a2 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_marketing.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_marketing" +"description": "The following are multiple choice questions (with answers) about marketing.\n\ + \n" +"tag": "mmlusr_question_and_answer_other_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_marketing" +"task_alias": "marketing" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_medical_genetics.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..477b6cf9cfcb367942135c16d59f0e59c805d14c --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_medical_genetics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_medical_genetics" +"description": "The following are multiple choice questions (with answers) about medical\ + \ genetics.\n\n" +"tag": "mmlusr_question_and_answer_other_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_medical_genetics" +"task_alias": "medical genetics" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_miscellaneous.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..204ea3ae360632adb105ec1b54494803c5325638 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_miscellaneous.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_miscellaneous" +"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ + \n" +"tag": "mmlusr_question_and_answer_other_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_miscellaneous" +"task_alias": "miscellaneous" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_moral_disputes.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4ceb216f67caf3f9bcdf421450e6f786dc130b0c --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_moral_disputes.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_moral_disputes" +"description": "The following are multiple choice questions (with answers) about moral\ + \ disputes.\n\n" +"tag": "mmlusr_question_and_answer_humanities_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_moral_disputes" +"task_alias": "moral disputes" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_moral_scenarios.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d434fb1cc1696b9b002be317631ad0288771d4e9 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_moral_scenarios.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_moral_scenarios" +"description": "The following are multiple choice questions (with answers) about moral\ + \ scenarios.\n\n" +"tag": "mmlusr_question_and_answer_humanities_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_moral_scenarios" +"task_alias": "moral scenarios" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_nutrition.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e564410f9632cd39de634f071d5438f00ad83a56 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_nutrition.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_nutrition" +"description": "The following are multiple choice questions (with answers) about nutrition.\n\ + \n" +"tag": "mmlusr_question_and_answer_other_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_nutrition" +"task_alias": "nutrition" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_philosophy.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bf9c19bc01416c529a67cd497eaf3d96539e1a09 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_philosophy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_philosophy" +"description": "The following are multiple choice questions (with answers) about philosophy.\n\ + \n" +"tag": "mmlusr_question_and_answer_humanities_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_philosophy" +"task_alias": "philosophy" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_prehistory.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a966669031fa998e252a49ed2d742ae99e635883 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_prehistory.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_prehistory" +"description": "The following are multiple choice questions (with answers) about prehistory.\n\ + \n" +"tag": "mmlusr_question_and_answer_humanities_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_prehistory" +"task_alias": "prehistory" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_professional_accounting.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..68973e37619fb764a0d73a97218f184fd49e4a5a --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_professional_accounting.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_professional_accounting" +"description": "The following are multiple choice questions (with answers) about professional\ + \ accounting.\n\n" +"tag": "mmlusr_question_and_answer_other_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_professional_accounting" +"task_alias": "professional accounting" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_professional_law.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a158fd123b8f1c45821d728dae7b70f3a8c2ab59 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_professional_law.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_professional_law" +"description": "The following are multiple choice questions (with answers) about professional\ + \ law.\n\n" +"tag": "mmlusr_question_and_answer_humanities_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_professional_law" +"task_alias": "professional law" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_professional_medicine.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..738e24e91d01c91776951dc991388a0e426d17c8 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_professional_medicine.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_professional_medicine" +"description": "The following are multiple choice questions (with answers) about professional\ + \ medicine.\n\n" +"tag": "mmlusr_question_and_answer_other_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_professional_medicine" +"task_alias": "professional medicine" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_professional_psychology.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..26f42c50f80254a5e390cb3996035c60005d1e8b --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_professional_psychology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_professional_psychology" +"description": "The following are multiple choice questions (with answers) about professional\ + \ psychology.\n\n" +"tag": "mmlusr_question_and_answer_social_sciences_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_professional_psychology" +"task_alias": "professional psychology" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_public_relations.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c92e67290e0eb97b728f9b76083c00c425b1ab1f --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_public_relations.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_public_relations" +"description": "The following are multiple choice questions (with answers) about public\ + \ relations.\n\n" +"tag": "mmlusr_question_and_answer_social_sciences_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_public_relations" +"task_alias": "public relations" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_security_studies.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9c5ba3c9c3801df6335b1f41f2a70b7a4819bcc3 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_security_studies.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_security_studies" +"description": "The following are multiple choice questions (with answers) about security\ + \ studies.\n\n" +"tag": "mmlusr_question_and_answer_social_sciences_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_security_studies" +"task_alias": "security studies" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_sociology.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d41098618f71fdad4c838c65a3a310d56624c99 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_sociology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_sociology" +"description": "The following are multiple choice questions (with answers) about sociology.\n\ + \n" +"tag": "mmlusr_question_and_answer_social_sciences_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_sociology" +"task_alias": "sociology" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_us_foreign_policy.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ced65cb6f0dd15963c4af54d2a7ee49bd6b62b62 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_us_foreign_policy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_us_foreign_policy" +"description": "The following are multiple choice questions (with answers) about us\ + \ foreign policy.\n\n" +"tag": "mmlusr_question_and_answer_social_sciences_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_us_foreign_policy" +"task_alias": "us foreign policy" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_virology.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..da7c0ca54e9f5a3fa91275cff8960f2a81e0aa4e --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_virology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_virology" +"description": "The following are multiple choice questions (with answers) about virology.\n\ + \n" +"tag": "mmlusr_question_and_answer_other_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_virology" +"task_alias": "virology" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_world_religions.yaml b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e44bd345d5900826296561acde1900985984b3db --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/question_and_answer_world_religions.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_and_answer_world_religions" +"description": "The following are multiple choice questions (with answers) about world\ + \ religions.\n\n" +"tag": "mmlusr_question_and_answer_humanities_tasks" +"include": "_mmlusr_qna_yml" +"task": "mmlusr_question_and_answer_world_religions" +"task_alias": "world religions" diff --git a/lm_eval/tasks/mmlusr/question_and_answer/utils.py b/lm_eval/tasks/mmlusr/question_and_answer/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..f1341bd59050caa11c56a9a36210428417e6c9f4 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_and_answer/utils.py @@ -0,0 +1,19 @@ +import datasets + + +def process_docs(dataset: datasets.Dataset) -> datasets.Dataset: + def _helper(doc): + # Assuming that the 'answer' field in the dataset now contains numbers 0-3 instead of 'A', 'B', 'C', 'D' + answer_list = ["A", "B", "C", "D"] + # Convert numeric index to corresponding letter + answer_index = int(doc["answer"]) # Make sure the answer is an integer + answer_letter = answer_list[answer_index] + + out_doc = { + "questions": doc["question"], + "choices": [doc["choice1"], doc["choice2"], doc["choice3"], doc["choice4"]], + "answer": answer_letter, # Include the letter for clarity + } + return out_doc + + return dataset.map(_helper) diff --git a/lm_eval/tasks/mmlusr/question_only/_mmlusr_q_yml b/lm_eval/tasks/mmlusr/question_only/_mmlusr_q_yml new file mode 100644 index 0000000000000000000000000000000000000000..7489f544ba29aee29332dd11197461a025ef1494 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/_mmlusr_q_yml @@ -0,0 +1,16 @@ +dataset_path: NiniCat/MMLU-SR +test_split: test +fewshot_split: train +fewshot_config: + sampler: first_n +output_type: multiple_choice +process_docs: !function utils.process_docs +doc_to_text: "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:" +doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/mmlusr/question_only/_question_only.yaml b/lm_eval/tasks/mmlusr/question_only/_question_only.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b8c5c972107dd625b685a58810f10b33814a418f --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/_question_only.yaml @@ -0,0 +1,44 @@ +group: mmlusr_question_only +group_alias: MMLU-SR (Question Only) +task: + - group: mmlusr_qo_stem + group_alias: STEM (Question Only) + task: + - mmlusr_question_only_stem_tasks + aggregate_metric_list: + - metric: acc + weight_by_size: True + metadata: + version: 2 + - group: mmlusr_qo_other + group_alias: Other (Question Only) + task: + - mmlusr_question_only_other_tasks + aggregate_metric_list: + - metric: acc + weight_by_size: True + metadata: + version: 2 + - group: mmlusr_qo_social_sciences + group_alias: Social Sciences (Question Only) + task: + - mmlusr_question_only_social_sciences_tasks + aggregate_metric_list: + - metric: acc + weight_by_size: True + metadata: + version: 2 + - group: mmlusr_qo_humanities + group_alias: Humanities (Question Only) + task: + - mmlusr_question_only_humanities_tasks + aggregate_metric_list: + - metric: acc + weight_by_size: True + metadata: + version: 2 +aggregate_metric_list: + - metric: acc + weight_by_size: True +metadata: + version: 2 diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_abstract_algebra.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_abstract_algebra.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3ae764f7b5f193d68d7e41f61654b42c4617306c --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_abstract_algebra.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_abstract_algebra" +"description": "The following are multiple choice questions (with answers) about abstract\ + \ algebra.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_abstract_algebra" +"task_alias": "abstract algebra" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_anatomy.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_anatomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..85fe75793d52621f6bba23d662ee3b18673ab5f0 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_anatomy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_anatomy" +"description": "The following are multiple choice questions (with answers) about anatomy.\n\ + \n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_anatomy" +"task_alias": "anatomy" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_astronomy.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_astronomy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e32ddfed16226fbb600a846121d7b1aa8d89758f --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_astronomy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_astronomy" +"description": "The following are multiple choice questions (with answers) about astronomy.\n\ + \n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_astronomy" +"task_alias": "astronomy" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_business_ethics.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_business_ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2d6404156f5114268f12d064d1073ba83621ea9d --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_business_ethics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_business_ethics" +"description": "The following are multiple choice questions (with answers) about business\ + \ ethics.\n\n" +"tag": "mmlusr_question_only_other_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_business_ethics" +"task_alias": "business ethics" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_clinical_knowledge.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_clinical_knowledge.yaml new file mode 100644 index 0000000000000000000000000000000000000000..33398345525ca0f7aa2f7deded01380e5123e526 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_clinical_knowledge.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_clinical_knowledge" +"description": "The following are multiple choice questions (with answers) about clinical\ + \ knowledge.\n\n" +"tag": "mmlusr_question_only_other_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_clinical_knowledge" +"task_alias": "clinical knowledge" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_college_biology.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_college_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..940bddc28febc92801af615859429a4ce77e45f7 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_college_biology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_college_biology" +"description": "The following are multiple choice questions (with answers) about college\ + \ biology.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_college_biology" +"task_alias": "college biology" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_college_chemistry.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_college_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dc7b6cdae3751fa357451a2e29a7bead8c5e67b1 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_college_chemistry.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_college_chemistry" +"description": "The following are multiple choice questions (with answers) about college\ + \ chemistry.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_college_chemistry" +"task_alias": "college chemistry" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_college_computer_science.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_college_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7feae9f0b1418bb514afe7c773fccf6eb379d1e5 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_college_computer_science.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_college_computer_science" +"description": "The following are multiple choice questions (with answers) about college\ + \ computer science.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_college_computer_science" +"task_alias": "college computer science" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_college_mathematics.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_college_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3c379c5f5f6f13a3d3b80ded80685a211dafc05e --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_college_mathematics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_college_mathematics" +"description": "The following are multiple choice questions (with answers) about college\ + \ mathematics.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_college_mathematics" +"task_alias": "college mathematics" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_college_medicine.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_college_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3f035787e33118323fd2c91698c4a6282fe858b6 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_college_medicine.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_college_medicine" +"description": "The following are multiple choice questions (with answers) about college\ + \ medicine.\n\n" +"tag": "mmlusr_question_only_other_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_college_medicine" +"task_alias": "college medicine" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_college_physics.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_college_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..84e9599e5c0036574ac0387e6056e15023b8f648 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_college_physics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_college_physics" +"description": "The following are multiple choice questions (with answers) about college\ + \ physics.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_college_physics" +"task_alias": "college physics" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_computer_security.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_computer_security.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7ac0de044f7eec2d8fcce8687b9de7d68d3bc2d7 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_computer_security.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_computer_security" +"description": "The following are multiple choice questions (with answers) about computer\ + \ security.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_computer_security" +"task_alias": "computer security" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_conceptual_physics.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_conceptual_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..75d50b14ca1204b8b6b9d770c60ff41c5b8c37f4 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_conceptual_physics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_conceptual_physics" +"description": "The following are multiple choice questions (with answers) about conceptual\ + \ physics.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_conceptual_physics" +"task_alias": "conceptual physics" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_econometrics.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_econometrics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..edd501fa06848966e1546f9fd1e3c78d4d237223 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_econometrics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_econometrics" +"description": "The following are multiple choice questions (with answers) about econometrics.\n\ + \n" +"tag": "mmlusr_question_only_social_sciences_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_econometrics" +"task_alias": "econometrics" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_electrical_engineering.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_electrical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8be2f268be83fbdb8b49d7da39b0533a50f6bf5a --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_electrical_engineering.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_electrical_engineering" +"description": "The following are multiple choice questions (with answers) about electrical\ + \ engineering.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_electrical_engineering" +"task_alias": "electrical engineering" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_elementary_mathematics.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_elementary_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0681dbc1dfbaa88506c770001cee4543ed5fb0cb --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_elementary_mathematics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_elementary_mathematics" +"description": "The following are multiple choice questions (with answers) about elementary\ + \ mathematics.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_elementary_mathematics" +"task_alias": "elementary mathematics" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_formal_logic.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_formal_logic.yaml new file mode 100644 index 0000000000000000000000000000000000000000..51ae64f4d656c6228707174d2cde036a1b2a0376 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_formal_logic.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_formal_logic" +"description": "The following are multiple choice questions (with answers) about formal\ + \ logic.\n\n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_formal_logic" +"task_alias": "formal logic" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_global_facts.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_global_facts.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4fe24005f69a858196c9c5727850333e765ae151 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_global_facts.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_global_facts" +"description": "The following are multiple choice questions (with answers) about global\ + \ facts.\n\n" +"tag": "mmlusr_question_only_other_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_global_facts" +"task_alias": "global facts" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_high_school_biology.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_high_school_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..030fd2e090ab8dabac6abc249f7577160b6b8ac8 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_high_school_biology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_high_school_biology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school biology.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_high_school_biology" +"task_alias": "high school biology" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_high_school_chemistry.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_high_school_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0f7b38e0e9be0150471694c84d62abdc2c5d6dcd --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_high_school_chemistry.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_high_school_chemistry" +"description": "The following are multiple choice questions (with answers) about high\ + \ school chemistry.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_high_school_chemistry" +"task_alias": "high school chemistry" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_high_school_computer_science.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_high_school_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..12f9d626c335be093730798bdb4116cf930030fd --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_high_school_computer_science.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_high_school_computer_science" +"description": "The following are multiple choice questions (with answers) about high\ + \ school computer science.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_high_school_computer_science" +"task_alias": "high school computer science" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_high_school_european_history.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_high_school_european_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..746d125e5456e50e180f81d1e5beb47bd59d5840 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_high_school_european_history.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_high_school_european_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school european history.\n\n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_high_school_european_history" +"task_alias": "high school european history" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_high_school_geography.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_high_school_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..abe2d6f5ac68b7f53a86d6147cf8664001830444 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_high_school_geography.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_high_school_geography" +"description": "The following are multiple choice questions (with answers) about high\ + \ school geography.\n\n" +"tag": "mmlusr_question_only_social_sciences_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_high_school_geography" +"task_alias": "high school geography" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_high_school_government_and_politics.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_high_school_government_and_politics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5a7fb24eeda5201a12669824e340fe73ea65ffba --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_high_school_government_and_politics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_high_school_government_and_politics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school government and politics.\n\n" +"tag": "mmlusr_question_only_social_sciences_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_high_school_government_and_politics" +"task_alias": "high school government and politics" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_high_school_macroeconomics.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_high_school_macroeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ecb0772234251072aeb4fa156a10bbb81f0707ad --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_high_school_macroeconomics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_high_school_macroeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school macroeconomics.\n\n" +"tag": "mmlusr_question_only_social_sciences_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_high_school_macroeconomics" +"task_alias": "high school macroeconomics" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_high_school_mathematics.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_high_school_mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aacf362d2f2a8f30f5e972406359479d74980bba --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_high_school_mathematics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_high_school_mathematics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school mathematics.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_high_school_mathematics" +"task_alias": "high school mathematics" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_high_school_microeconomics.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_high_school_microeconomics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dc288c976b09afec24cdff337ce3e648d600519a --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_high_school_microeconomics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_high_school_microeconomics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school microeconomics.\n\n" +"tag": "mmlusr_question_only_social_sciences_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_high_school_microeconomics" +"task_alias": "high school microeconomics" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_high_school_physics.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_high_school_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aaa42363325004e19bf462adee55bb94ac1aebe5 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_high_school_physics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_high_school_physics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school physics.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_high_school_physics" +"task_alias": "high school physics" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_high_school_psychology.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_high_school_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..33085c5c2a6cf6054dd82879381365a66cc770cc --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_high_school_psychology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_high_school_psychology" +"description": "The following are multiple choice questions (with answers) about high\ + \ school psychology.\n\n" +"tag": "mmlusr_question_only_social_sciences_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_high_school_psychology" +"task_alias": "high school psychology" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_high_school_statistics.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_high_school_statistics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ae69628a60bb02fefd9320c651ae6e9df35dc181 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_high_school_statistics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_high_school_statistics" +"description": "The following are multiple choice questions (with answers) about high\ + \ school statistics.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_high_school_statistics" +"task_alias": "high school statistics" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_high_school_us_history.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_high_school_us_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cf226b5a437c2de1a554e7f997d7604fe1c5acf9 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_high_school_us_history.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_high_school_us_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school us history.\n\n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_high_school_us_history" +"task_alias": "high school us history" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_high_school_world_history.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_high_school_world_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..37b67158f4559b6a23072fca79405db8f49c2ee1 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_high_school_world_history.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_high_school_world_history" +"description": "The following are multiple choice questions (with answers) about high\ + \ school world history.\n\n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_high_school_world_history" +"task_alias": "high school world history" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_human_aging.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_human_aging.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2dd67daf3f0c9f1691974b85f2a86c8b8eadcb97 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_human_aging.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_human_aging" +"description": "The following are multiple choice questions (with answers) about human\ + \ aging.\n\n" +"tag": "mmlusr_question_only_other_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_human_aging" +"task_alias": "human aging" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_human_sexuality.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_human_sexuality.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bfaee537e7e91e95a276f0dd2214ad0291ace25e --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_human_sexuality.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_human_sexuality" +"description": "The following are multiple choice questions (with answers) about human\ + \ sexuality.\n\n" +"tag": "mmlusr_question_only_social_sciences_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_human_sexuality" +"task_alias": "human sexuality" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_international_law.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_international_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fde605633bd9f8c4146a7a213ac69bdc2a100680 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_international_law.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_international_law" +"description": "The following are multiple choice questions (with answers) about international\ + \ law.\n\n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_international_law" +"task_alias": "international law" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_jurisprudence.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_jurisprudence.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e2f95fd2b12870ea7e6a4cebddbe8c4d6ac0eeba --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_jurisprudence.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_jurisprudence" +"description": "The following are multiple choice questions (with answers) about jurisprudence.\n\ + \n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_jurisprudence" +"task_alias": "jurisprudence" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_logical_fallacies.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_logical_fallacies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8e07150c7fdecd13a80a995badaca082fdf9d0d1 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_logical_fallacies.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_logical_fallacies" +"description": "The following are multiple choice questions (with answers) about logical\ + \ fallacies.\n\n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_logical_fallacies" +"task_alias": "logical fallacies" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_machine_learning.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_machine_learning.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5bccaf4a4164b9a471395fddc9b01ff8bb838108 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_machine_learning.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_machine_learning" +"description": "The following are multiple choice questions (with answers) about machine\ + \ learning.\n\n" +"tag": "mmlusr_question_only_stem_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_machine_learning" +"task_alias": "machine learning" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_management.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_management.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ca72f214c40c18a7688d76ef7dcc12c69aebb11b --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_management.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_management" +"description": "The following are multiple choice questions (with answers) about management.\n\ + \n" +"tag": "mmlusr_question_only_other_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_management" +"task_alias": "management" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_marketing.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a47f15b6b44872250e000ff3e65529fdfd317e19 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_marketing.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_marketing" +"description": "The following are multiple choice questions (with answers) about marketing.\n\ + \n" +"tag": "mmlusr_question_only_other_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_marketing" +"task_alias": "marketing" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_medical_genetics.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_medical_genetics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..88829f61c1f0c087808364b3eb11871ea8af2302 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_medical_genetics.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_medical_genetics" +"description": "The following are multiple choice questions (with answers) about medical\ + \ genetics.\n\n" +"tag": "mmlusr_question_only_other_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_medical_genetics" +"task_alias": "medical genetics" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_miscellaneous.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_miscellaneous.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ad3de69466630145c31bc474ce399f6067f72cff --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_miscellaneous.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_miscellaneous" +"description": "The following are multiple choice questions (with answers) about miscellaneous.\n\ + \n" +"tag": "mmlusr_question_only_other_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_miscellaneous" +"task_alias": "miscellaneous" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_moral_disputes.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_moral_disputes.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4a84f610571a1eb1ed58dda6988628c4cc83f761 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_moral_disputes.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_moral_disputes" +"description": "The following are multiple choice questions (with answers) about moral\ + \ disputes.\n\n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_moral_disputes" +"task_alias": "moral disputes" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_moral_scenarios.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_moral_scenarios.yaml new file mode 100644 index 0000000000000000000000000000000000000000..56ef60495f02f99e8090737056d582e7f962047f --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_moral_scenarios.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_moral_scenarios" +"description": "The following are multiple choice questions (with answers) about moral\ + \ scenarios.\n\n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_moral_scenarios" +"task_alias": "moral scenarios" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_nutrition.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_nutrition.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2518b48dc991d272263f01c67908bb703b277139 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_nutrition.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_nutrition" +"description": "The following are multiple choice questions (with answers) about nutrition.\n\ + \n" +"tag": "mmlusr_question_only_other_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_nutrition" +"task_alias": "nutrition" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_philosophy.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e7c17c5dd8758471f9d6485e1c92414d9b16bee5 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_philosophy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_philosophy" +"description": "The following are multiple choice questions (with answers) about philosophy.\n\ + \n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_philosophy" +"task_alias": "philosophy" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_prehistory.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_prehistory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2297b0f122817d57a14b59eecfc5d70a7fa02f05 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_prehistory.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_prehistory" +"description": "The following are multiple choice questions (with answers) about prehistory.\n\ + \n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_prehistory" +"task_alias": "prehistory" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_professional_accounting.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_professional_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a04374117fd545b719ad12fa838eae269efad696 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_professional_accounting.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_professional_accounting" +"description": "The following are multiple choice questions (with answers) about professional\ + \ accounting.\n\n" +"tag": "mmlusr_question_only_other_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_professional_accounting" +"task_alias": "professional accounting" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_professional_law.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_professional_law.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8b8e572b9ef7b84e1baa098b57e3bc7e1d6bcb25 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_professional_law.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_professional_law" +"description": "The following are multiple choice questions (with answers) about professional\ + \ law.\n\n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_professional_law" +"task_alias": "professional law" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_professional_medicine.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_professional_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c25aa01755421c42b70335f7c1a8cf9ccea659a5 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_professional_medicine.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_professional_medicine" +"description": "The following are multiple choice questions (with answers) about professional\ + \ medicine.\n\n" +"tag": "mmlusr_question_only_other_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_professional_medicine" +"task_alias": "professional medicine" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_professional_psychology.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_professional_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..89ebc81c7f0d9999811da446610df2b5e2a7e316 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_professional_psychology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_professional_psychology" +"description": "The following are multiple choice questions (with answers) about professional\ + \ psychology.\n\n" +"tag": "mmlusr_question_only_social_sciences_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_professional_psychology" +"task_alias": "professional psychology" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_public_relations.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_public_relations.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d23cb2b93d3391944e7aa30222381051e593b8d2 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_public_relations.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_public_relations" +"description": "The following are multiple choice questions (with answers) about public\ + \ relations.\n\n" +"tag": "mmlusr_question_only_social_sciences_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_public_relations" +"task_alias": "public relations" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_security_studies.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_security_studies.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0ff913d961a5f17bdc987659e1fdc36658b5b06e --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_security_studies.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_security_studies" +"description": "The following are multiple choice questions (with answers) about security\ + \ studies.\n\n" +"tag": "mmlusr_question_only_social_sciences_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_security_studies" +"task_alias": "security studies" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_sociology.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d705e8485c85f3aa3493fad724aca687998155b8 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_sociology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_sociology" +"description": "The following are multiple choice questions (with answers) about sociology.\n\ + \n" +"tag": "mmlusr_question_only_social_sciences_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_sociology" +"task_alias": "sociology" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_us_foreign_policy.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_us_foreign_policy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7a9a7b8743e5d5f6cfc133f10e0c00125b87d962 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_us_foreign_policy.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_us_foreign_policy" +"description": "The following are multiple choice questions (with answers) about us\ + \ foreign policy.\n\n" +"tag": "mmlusr_question_only_social_sciences_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_us_foreign_policy" +"task_alias": "us foreign policy" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_virology.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_virology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..034cfa8bdbe78b7d9d0054da332e9b4702f01b18 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_virology.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_virology" +"description": "The following are multiple choice questions (with answers) about virology.\n\ + \n" +"tag": "mmlusr_question_only_other_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_virology" +"task_alias": "virology" diff --git a/lm_eval/tasks/mmlusr/question_only/question_only_world_religions.yaml b/lm_eval/tasks/mmlusr/question_only/question_only_world_religions.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4e66514c8a27b899b6d9545ac812d309cc85d62a --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/question_only_world_religions.yaml @@ -0,0 +1,7 @@ +"dataset_name": "question_only_world_religions" +"description": "The following are multiple choice questions (with answers) about world\ + \ religions.\n\n" +"tag": "mmlusr_question_only_humanities_tasks" +"include": "_mmlusr_q_yml" +"task": "mmlusr_question_only_world_religions" +"task_alias": "world religions" diff --git a/lm_eval/tasks/mmlusr/question_only/utils.py b/lm_eval/tasks/mmlusr/question_only/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..f1341bd59050caa11c56a9a36210428417e6c9f4 --- /dev/null +++ b/lm_eval/tasks/mmlusr/question_only/utils.py @@ -0,0 +1,19 @@ +import datasets + + +def process_docs(dataset: datasets.Dataset) -> datasets.Dataset: + def _helper(doc): + # Assuming that the 'answer' field in the dataset now contains numbers 0-3 instead of 'A', 'B', 'C', 'D' + answer_list = ["A", "B", "C", "D"] + # Convert numeric index to corresponding letter + answer_index = int(doc["answer"]) # Make sure the answer is an integer + answer_letter = answer_list[answer_index] + + out_doc = { + "questions": doc["question"], + "choices": [doc["choice1"], doc["choice2"], doc["choice3"], doc["choice4"]], + "answer": answer_letter, # Include the letter for clarity + } + return out_doc + + return dataset.map(_helper) diff --git a/lm_eval/tasks/mmmu/README.md b/lm_eval/tasks/mmmu/README.md new file mode 100644 index 0000000000000000000000000000000000000000..e9d0da12f667975456235581e3e95e27e5114a84 --- /dev/null +++ b/lm_eval/tasks/mmmu/README.md @@ -0,0 +1,150 @@ +# MMMU Benchmark + +### Paper + +Title: `MMMU: A Massive Multi-discipline MultimodalUnderstanding and Reasoning Benchmark for Expert AGI` + +Abstract: `MMMU is a new benchmark designed to evaluate multimodal models on massive multi-discipline tasks demanding college-level subject knowledge and deliberate reasoning.` + +`The benchmark is composed of 30 tasks, for a total of 900 mixed image+text examples (some with multiple images in context)` + +Homepage: `https://github.com/MMMU-Benchmark/MMMU/tree/main/mmmu` + +Note: Some questions have multiple images in context. To control for this use `max_images=N` in model init. + +### Citation + +``` +@inproceedings{yue2023mmmu, + title={MMMU: A Massive Multi-discipline Multimodal Understanding and Reasoning Benchmark for Expert AGI}, + author={Xiang Yue and Yuansheng Ni and Kai Zhang and Tianyu Zheng and Ruoqi Liu and Ge Zhang and Samuel Stevens and Dongfu Jiang and Weiming Ren and Yuxuan Sun and Cong Wei and Botao Yu and Ruibin Yuan and Renliang Sun and Ming Yin and Boyuan Zheng and Zhenzhu Yang and Yibo Liu and Wenhao Huang and Huan Sun and Yu Su and Wenhu Chen}, + booktitle={Proceedings of CVPR}, + year={2024}, + } +``` + +### Groups, Tags, and Tasks + +#### Groups + +* `mmmu_val` +* `mmmu_val_art_and_design` +* `mmmu_val_business` +* `mmmu_val_health_and_medicine` +* `mmmu_val_humanities_and_social_science` +* `mmmu_val_science` +* `mmmu_val_tech_and_engineering` + +#### Tags + + +#### Tasks + +* `mmmu_val_accounting` +* `mmmu_val_agriculture` +* `mmmu_val_architecture_and_engineering.yaml` +* `mmmu_val_art` +* `mmmu_val_art_theory` +* `mmmu_val_basic_medical_science` +* `mmmu_val_biology` +* `mmmu_val_chemistry` +* `mmmu_val_computer_science` +* `mmmu_val_clinical_medicine` +* `mmmu_val_design` +* `mmmu_val_diagnostics_and_laboratory_medicine` +* `mmmu_val_electronics` +* `mmmu_val_energy_and_power` +* `mmmu_val_economics` +* `mmmu_val_finance` +* `mmmu_val_geography` +* `mmmu_val_history` +* ... + +### Variants + +The `mmmu_val` group implements MMMU using processing code [from the original MMMU authors](https://github.com/MMMU-Benchmark/MMMU/tree/main/mmmu) and uses the prompt format found in [the MMMU repository for Llava-1.5](https://github.com/MMMU-Benchmark/MMMU/blob/main/mmmu/configs/llava1.5.yaml). This implementation should give scores on par with or slightly higher than those reported by [lmms-eval](https://github.com/EvolvingLMMs-Lab/lmms-eval/tree/main/lmms_eval/tasks/mmmu) for `mmmu_val` and the MMMU repository code. + +Scores on several tested models (**all with `--apply_chat_template`**) are: + +Qwen2-VL-2B: +``` +hf-multimodal (pretrained=Qwen/Qwen2-VL-2B-Instruct,attn_implementation=flash_attention_2,dtype=bfloat16,convert_img_format=True), gen_kwargs: (None), limit: None, num_fewshot: None, batch_size: 2 +``` +``` +| Groups |Version|Filter|n-shot|Metric| |Value | |Stderr| +|--------------------------------|------:|------|------|------|---|-----:|---|-----:| +|mmmu_val | 0|none | |acc |↑ |0.3778|± |0.0155| +| - Art and Design | 0|none | |acc |↑ |0.5500|± |0.0415| +| - Business | 0|none | |acc |↑ |0.3600|± |0.0389| +| - Health and Medicine | 0|none | |acc |↑ |0.3667|± |0.0394| +| - Humanities and Social Science| 0|none | |acc |↑ |0.5167|± |0.0438| +| - Science | 0|none | |acc |↑ |0.2467|± |0.0352| +| - Tech and Engineering | 0|none | |acc |↑ |0.3143|± |0.0317| +``` +Author-reported score: 41.1% + + +Qwen2-VL-7B: +``` +hf-multimodal (pretrained=Qwen/Qwen2-VL-7B-Instruct,attn_implementation=flash_attention_2,dtype=bfloat16,convert_img_format=True), gen_kwargs: (None), limit: None, num_fewshot: None, batch_size: 2 +``` +``` +| Groups |Version|Filter|n-shot|Metric| |Value | |Stderr| +|--------------------------------|------:|------|------|------|---|-----:|---|-----:| +|mmmu_val | 0|none | |acc |↑ |0.5056|± |0.0160| +| - Art and Design | 0|none | |acc |↑ |0.6917|± |0.0398| +| - Business | 0|none | |acc |↑ |0.4333|± |0.0406| +| - Health and Medicine | 0|none | |acc |↑ |0.5667|± |0.0401| +| - Humanities and Social Science| 0|none | |acc |↑ |0.6750|± |0.0426| +| - Science | 0|none | |acc |↑ |0.3800|± |0.0392| +| - Tech and Engineering | 0|none | |acc |↑ |0.4000|± |0.0341| +``` +Author-reported score: 54.1% + +Idefics2-8B: +``` +hf-multimodal (pretrained=HuggingFaceM4/idefics2-8b,attn_implementation=flash_attention_2,dtype=bfloat16,convert_img_format=True,max_images=2), gen_kwargs: (None), limit: None, num_fewshot: None, batch_size: 2 +``` +``` +| Groups |Version|Filter|n-shot|Metric| |Value | |Stderr| +|--------------------------------|------:|------|------|------|---|-----:|---|-----:| +|mmmu_val | 0|none | |acc |↑ |0.4011|± |0.0154| +| - Art and Design | 0|none | |acc |↑ |0.6167|± |0.0436| +| - Business | 0|none | |acc |↑ |0.3200|± |0.0373| +| - Health and Medicine | 0|none | |acc |↑ |0.4000|± |0.0401| +| - Humanities and Social Science| 0|none | |acc |↑ |0.5750|± |0.0424| +| - Science | 0|none | |acc |↑ |0.2600|± |0.0358| +| - Tech and Engineering | 0|none | |acc |↑ |0.3381|± |0.0312| +``` +Author-reported score: ~43% + +Llava-v1.6-Mistral-7B: +``` +hf-multimodal (pretrained=llava-hf/llava-v1.6-mistral-7b-hf,attn_implementation=flash_attention_2,dtype=bfloat16,convert_img_format=True), gen_kwargs: (None), limit: None, num_fewshot: None, batch_size: 2 +``` +``` +| Groups |Version|Filter|n-shot|Metric| |Value | |Stderr| +|--------------------------------|------:|------|------|------|---|-----:|---|-----:| +|mmmu_val | 0|none | |acc |↑ |0.3522|± |0.0151| +| - Art and Design | 0|none | |acc |↑ |0.5167|± |0.0440| +| - Business | 0|none | |acc |↑ |0.2667|± |0.0362| +| - Health and Medicine | 0|none | |acc |↑ |0.3867|± |0.0397| +| - Humanities and Social Science| 0|none | |acc |↑ |0.5917|± |0.0433| +| - Science | 0|none | |acc |↑ |0.2200|± |0.0342| +| - Tech and Engineering | 0|none | |acc |↑ |0.2524|± |0.0299| +``` +Author-reported score: 35.3% + + +### Checklist + +For adding novel benchmarks/datasets to the library: +* [x] Is the task an existing benchmark in the literature? + * [x] Have you referenced the original paper that introduced the task? + * [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + + +If other tasks on this dataset are already supported: +* [x] Is the "Main" variant of this task clearly denoted? +* [x] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [x] Have you noted which, if any, published evaluation setups are matched by this variant? diff --git a/lm_eval/tasks/mmmu/_art_and_design.yaml b/lm_eval/tasks/mmmu/_art_and_design.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b0dda1876fea6415c442957b5bafb24499789276 --- /dev/null +++ b/lm_eval/tasks/mmmu/_art_and_design.yaml @@ -0,0 +1,13 @@ +group: mmmu_val_art_and_design +group_alias: Art and Design +task: + - mmmu_val_art + - mmmu_val_art_theory + - mmmu_val_design + - mmmu_val_music +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm_eval/tasks/mmmu/_business.yaml b/lm_eval/tasks/mmmu/_business.yaml new file mode 100644 index 0000000000000000000000000000000000000000..497948e5e5c0f5ea7a8c72a5624f0a925eedc7d6 --- /dev/null +++ b/lm_eval/tasks/mmmu/_business.yaml @@ -0,0 +1,14 @@ +group: mmmu_val_business +group_alias: Business +task: + - mmmu_val_accounting + - mmmu_val_economics + - mmmu_val_finance + - mmmu_val_manage + - mmmu_val_marketing +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm_eval/tasks/mmmu/_health_and_medicine.yaml b/lm_eval/tasks/mmmu/_health_and_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..04709d3d4653b4a1bf912eaad2ee7dde27330822 --- /dev/null +++ b/lm_eval/tasks/mmmu/_health_and_medicine.yaml @@ -0,0 +1,14 @@ +group: mmmu_val_health_and_medicine +group_alias: Health and Medicine +task: + - mmmu_val_basic_medical_science + - mmmu_val_clinical_medicine + - mmmu_val_diagnostics_and_laboratory_medicine + - mmmu_val_pharmacy + - mmmu_val_public_health +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm_eval/tasks/mmmu/_humanities_and_social_sciences.yaml b/lm_eval/tasks/mmmu/_humanities_and_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b8d70a9ea3f4e802c6eae7c8a6ceeac2f295a9dc --- /dev/null +++ b/lm_eval/tasks/mmmu/_humanities_and_social_sciences.yaml @@ -0,0 +1,13 @@ +group: mmmu_val_humanities_and_social_science +group_alias: Humanities and Social Science +task: + - mmmu_val_history + - mmmu_val_literature + - mmmu_val_sociology + - mmmu_val_psychology +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm_eval/tasks/mmmu/_mmmu.yaml b/lm_eval/tasks/mmmu/_mmmu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6bcd234bbc065f7a70a79fad3deba1e00bf2bc45 --- /dev/null +++ b/lm_eval/tasks/mmmu/_mmmu.yaml @@ -0,0 +1,14 @@ +group: mmmu_val +task: + - mmmu_val_art_and_design + - mmmu_val_business + - mmmu_val_health_and_medicine + - mmmu_val_humanities_and_social_science + - mmmu_val_science + - mmmu_val_tech_and_engineering +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm_eval/tasks/mmmu/_science.yaml b/lm_eval/tasks/mmmu/_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9b41585a6563b42e1706e29cceb96b11db152dd2 --- /dev/null +++ b/lm_eval/tasks/mmmu/_science.yaml @@ -0,0 +1,14 @@ +group: mmmu_val_science +group_alias: Science +task: + - mmmu_val_biology + - mmmu_val_chemistry + - mmmu_val_geography + - mmmu_val_math + - mmmu_val_physics +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm_eval/tasks/mmmu/_tech_and_engineering.yaml b/lm_eval/tasks/mmmu/_tech_and_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..956f9906b3668df97880c121dd95b0b8fd12b850 --- /dev/null +++ b/lm_eval/tasks/mmmu/_tech_and_engineering.yaml @@ -0,0 +1,16 @@ +group: mmmu_val_tech_and_engineering +group_alias: Tech and Engineering +task: + - mmmu_val_agriculture + - mmmu_val_architecture_and_engineering + - mmmu_val_computer_science + - mmmu_val_electronics + - mmmu_val_energy_and_power + - mmmu_val_materials + - mmmu_val_mechanical_engineering +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm_eval/tasks/mmmu/_template_yaml b/lm_eval/tasks/mmmu/_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..f92ce60d6becd951f7aff35fb82b5345bf0fdeeb --- /dev/null +++ b/lm_eval/tasks/mmmu/_template_yaml @@ -0,0 +1,19 @@ +dataset_path: MMMU/MMMU +validation_split: validation +output_type: generate_until +doc_to_image: !function utils.doc_to_image +doc_to_text: !function utils.doc_to_text +doc_to_target: "answer" +process_results: !function utils.process_results +generation_kwargs: + until: + - "<|endoftext|>" + temperature: 0.0 + do_sample: false + max_gen_toks: 512 +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/lm_eval/tasks/mmmu/mmmu_accounting.yaml b/lm_eval/tasks/mmmu/mmmu_accounting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3c4aa41d9c7b4f29ac73e33ae8268ae2f7aa611a --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_accounting.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_accounting +include: _template_yaml +task_alias: Accounting +dataset_name: Accounting diff --git a/lm_eval/tasks/mmmu/mmmu_agriculture.yaml b/lm_eval/tasks/mmmu/mmmu_agriculture.yaml new file mode 100644 index 0000000000000000000000000000000000000000..387f33f7a985a38f0790b45db64b77a02ee222ee --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_agriculture.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_agriculture +include: _template_yaml +task_alias: Agriculture +dataset_name: Agriculture diff --git a/lm_eval/tasks/mmmu/mmmu_architecture_and_engineering.yaml b/lm_eval/tasks/mmmu/mmmu_architecture_and_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..829ee3a8f8b47d0ab9b499e4fe810d8b3c30f2b5 --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_architecture_and_engineering.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_architecture_and_engineering +include: _template_yaml +task_alias: Architecture and Engineering +dataset_name: Architecture_and_Engineering diff --git a/lm_eval/tasks/mmmu/mmmu_art.yaml b/lm_eval/tasks/mmmu/mmmu_art.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ff812ab3c1d02ddd0e0e0152155a97499cfa9411 --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_art.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_art +include: _template_yaml +task_alias: Art +dataset_name: Art diff --git a/lm_eval/tasks/mmmu/mmmu_art_theory.yaml b/lm_eval/tasks/mmmu/mmmu_art_theory.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d6331871a56f2fe103179a31575dd049e94ca602 --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_art_theory.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_art_theory +include: _template_yaml +task_alias: Art Theory +dataset_name: Art_Theory diff --git a/lm_eval/tasks/mmmu/mmmu_basic_medical_science.yaml b/lm_eval/tasks/mmmu/mmmu_basic_medical_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d7486f1ee341a5a407f8fb3f5d93af5d45c6360b --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_basic_medical_science.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_basic_medical_science +include: _template_yaml +task_alias: Basic Medical Science +dataset_name: Basic_Medical_Science diff --git a/lm_eval/tasks/mmmu/mmmu_biology.yaml b/lm_eval/tasks/mmmu/mmmu_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..49d36f380f9f6a12912522388cb8cc5d93b1360a --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_biology.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_biology +include: _template_yaml +task_alias: Biology +dataset_name: Biology diff --git a/lm_eval/tasks/mmmu/mmmu_chemistry.yaml b/lm_eval/tasks/mmmu/mmmu_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cb096531c2e33b4dd754da03687a0778062a30cf --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_chemistry.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_chemistry +include: _template_yaml +task_alias: Chemistry +dataset_name: Chemistry diff --git a/lm_eval/tasks/mmmu/mmmu_clinical_medicine.yaml b/lm_eval/tasks/mmmu/mmmu_clinical_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a0833d165f9e54b12cf5eb748c96112bcb6e152b --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_clinical_medicine.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_clinical_medicine +include: _template_yaml +task_alias: Clinical Medicine +dataset_name: Clinical_Medicine diff --git a/lm_eval/tasks/mmmu/mmmu_computer_science.yaml b/lm_eval/tasks/mmmu/mmmu_computer_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..53c91a61b33c60ac21f657dac44a91b16a37ce87 --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_computer_science.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_computer_science +include: _template_yaml +task_alias: Computer Science +dataset_name: Computer_Science diff --git a/lm_eval/tasks/mmmu/mmmu_design.yaml b/lm_eval/tasks/mmmu/mmmu_design.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0367dd65ee0d1c6c5446baa2fac17da50778c9b6 --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_design.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_design +include: _template_yaml +task_alias: Design +dataset_name: Design diff --git a/lm_eval/tasks/mmmu/mmmu_diagnostics_and_laboratory_medicine.yaml b/lm_eval/tasks/mmmu/mmmu_diagnostics_and_laboratory_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2ec1b7dca693ad3c3c5030aacdaa72fbc0551323 --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_diagnostics_and_laboratory_medicine.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_diagnostics_and_laboratory_medicine +include: _template_yaml +task_alias: Diagnostics and Laboratory Medicine +dataset_name: Diagnostics_and_Laboratory_Medicine diff --git a/lm_eval/tasks/mmmu/mmmu_economics.yaml b/lm_eval/tasks/mmmu/mmmu_economics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3d6465da7416b7bc3c23ef44482d125e0e6fb9ea --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_economics.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_economics +include: _template_yaml +task_alias: Economics +dataset_name: Economics diff --git a/lm_eval/tasks/mmmu/mmmu_electronics.yaml b/lm_eval/tasks/mmmu/mmmu_electronics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..02df22f8a39f63a7a8c0c318c73b2135c8233275 --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_electronics.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_electronics +include: _template_yaml +task_alias: Electronics +dataset_name: Electronics diff --git a/lm_eval/tasks/mmmu/mmmu_energy_and_power.yaml b/lm_eval/tasks/mmmu/mmmu_energy_and_power.yaml new file mode 100644 index 0000000000000000000000000000000000000000..add24ebed72c43fa49beadd8a75f5f6aca1b6122 --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_energy_and_power.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_energy_and_power +include: _template_yaml +task_alias: Energy and Power +dataset_name: Energy_and_Power diff --git a/lm_eval/tasks/mmmu/mmmu_finance.yaml b/lm_eval/tasks/mmmu/mmmu_finance.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c6b6f042c1a1d53e43d21f42e8ac50854c429bd4 --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_finance.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_finance +include: _template_yaml +task_alias: Finance +dataset_name: Finance diff --git a/lm_eval/tasks/mmmu/mmmu_geography.yaml b/lm_eval/tasks/mmmu/mmmu_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b5f90221366966cc24d981e128f7ff8a172b41e8 --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_geography.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_geography +include: _template_yaml +task_alias: Geography +dataset_name: Geography diff --git a/lm_eval/tasks/mmmu/mmmu_history.yaml b/lm_eval/tasks/mmmu/mmmu_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..435388fd74b86392fe4b4124056f34590528d726 --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_history.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_history +include: _template_yaml +task_alias: History +dataset_name: History diff --git a/lm_eval/tasks/mmmu/mmmu_literature.yaml b/lm_eval/tasks/mmmu/mmmu_literature.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9e0a1ba50372ed8014d407e72d946b07322e08d9 --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_literature.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_literature +include: _template_yaml +task_alias: Literature +dataset_name: Literature diff --git a/lm_eval/tasks/mmmu/mmmu_manage.yaml b/lm_eval/tasks/mmmu/mmmu_manage.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bd1992411458354a650165e3872e28da4374c101 --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_manage.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_manage +include: _template_yaml +task_alias: Manage +dataset_name: Manage diff --git a/lm_eval/tasks/mmmu/mmmu_marketing.yaml b/lm_eval/tasks/mmmu/mmmu_marketing.yaml new file mode 100644 index 0000000000000000000000000000000000000000..53848171be0e0b92c540da15bd3a9ffc60d00e9b --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_marketing.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_marketing +include: _template_yaml +task_alias: Marketing +dataset_name: Marketing diff --git a/lm_eval/tasks/mmmu/mmmu_materials.yaml b/lm_eval/tasks/mmmu/mmmu_materials.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ee469443163c792073cda7b1b485d1e7fca6044b --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_materials.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_materials +include: _template_yaml +task_alias: Materials +dataset_name: Materials diff --git a/lm_eval/tasks/mmmu/mmmu_math.yaml b/lm_eval/tasks/mmmu/mmmu_math.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e6817689ab1f8a7232ebdd367592dc782893bea4 --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_math.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_math +include: _template_yaml +task_alias: Math +dataset_name: Math diff --git a/lm_eval/tasks/mmmu/mmmu_mechanical_engineering.yaml b/lm_eval/tasks/mmmu/mmmu_mechanical_engineering.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4b9d95be85b7181514e4918439690e4ce5f84eea --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_mechanical_engineering.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_mechanical_engineering +include: _template_yaml +task_alias: Mechanical Engineering +dataset_name: Mechanical_Engineering diff --git a/lm_eval/tasks/mmmu/mmmu_music.yaml b/lm_eval/tasks/mmmu/mmmu_music.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cf4032dea3e9425a01cd924f57a487a8de9b647e --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_music.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_music +include: _template_yaml +task_alias: Music +dataset_name: Music diff --git a/lm_eval/tasks/mmmu/mmmu_pharmacy.yaml b/lm_eval/tasks/mmmu/mmmu_pharmacy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..04a4f1ff67a160da03ca99ebcd66047b48524fbb --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_pharmacy.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_pharmacy +include: _template_yaml +task_alias: Pharmacy +dataset_name: Pharmacy diff --git a/lm_eval/tasks/mmmu/mmmu_physics.yaml b/lm_eval/tasks/mmmu/mmmu_physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..53371cad73c7ff9e5b340a18a53eb94c452720c6 --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_physics.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_physics +include: _template_yaml +task_alias: Physics +dataset_name: Physics diff --git a/lm_eval/tasks/mmmu/mmmu_psychology.yaml b/lm_eval/tasks/mmmu/mmmu_psychology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8471d4a18f4bf566186d2c9938d14a2f7eccd769 --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_psychology.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_psychology +include: _template_yaml +task_alias: Psychology +dataset_name: Psychology diff --git a/lm_eval/tasks/mmmu/mmmu_public_health.yaml b/lm_eval/tasks/mmmu/mmmu_public_health.yaml new file mode 100644 index 0000000000000000000000000000000000000000..aefc0590a3878e182494633463279e1b3ec40021 --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_public_health.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_public_health +include: _template_yaml +task_alias: Public Health +dataset_name: Public_Health diff --git a/lm_eval/tasks/mmmu/mmmu_sociology.yaml b/lm_eval/tasks/mmmu/mmmu_sociology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5da664a8e7a20f038086829478945f9bb4cdbe53 --- /dev/null +++ b/lm_eval/tasks/mmmu/mmmu_sociology.yaml @@ -0,0 +1,4 @@ +task: mmmu_val_sociology +include: _template_yaml +task_alias: Sociology +dataset_name: Sociology diff --git a/lm_eval/tasks/mmmu/utils.py b/lm_eval/tasks/mmmu/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..87a3022098b29233615b404fdc91da2403154c3d --- /dev/null +++ b/lm_eval/tasks/mmmu/utils.py @@ -0,0 +1,341 @@ +import ast +import random +import re + +import numpy as np + + +random.seed(42) + + +# source for prompt fstrings: https://github.com/MMMU-Benchmark/MMMU/blob/7787d60648c82a9d40acd656fa541a6c74f58995/eval/configs/llava1.5.yaml#L3 +MULTI_CHOICE_EXAMPLE_FORMAT = """{} + +{} + +Answer with the option's letter from the given choices directly.""" + + +SHORT_ANS_EXAMPLE_FORMAT = """{} + +Answer the question using a single word or phrase.""" + +START_CHR = "A" + + +def doc_to_image(doc): + # get formatted prompt (incl. multi-choice options) pre- reformatting + input_text = _doc_to_text(doc) + # locate instances in input + image_placeholders = [ + img.replace(" ", "_").replace("<", "").replace(">", "") + for img in re.findall("", input_text) + ] + + # collect visuals (can have dupes of a given image or be out of order) + # E.g. validation_Math_19 contains and but seen as [, , , , ] + visuals = [doc[img] for img in image_placeholders] + + return visuals + + +def doc_to_text(doc): + """Get the prompt for a given document.""" + + prompt = _doc_to_text(doc) + + for i in range(1, 8): + # replace with . TODO: check this is always the right decision incl. for non-HF models + prompt = prompt.replace(f"", "") + + return prompt + + +def _doc_to_text(doc): + """Helper--get the prompt for a given document but DO NOT yet replace with .""" + + if doc["question_type"] == "multiple-choice": + choices_str = "" + + for i, choice in enumerate(ast.literal_eval(doc["options"])): + # add (A) {choice1}\n , (B) {choice2}\n , and so on + # to create the list of formatted choices in the prompt + choices_str += f"\n({chr(ord(START_CHR) + i)}) {choice}" + + choices_str = ( + choices_str.lstrip() + ) # remove the extraneous prepended \n that we added + + prompt = MULTI_CHOICE_EXAMPLE_FORMAT.format(doc["question"], choices_str) + else: + prompt = SHORT_ANS_EXAMPLE_FORMAT.format(doc["question"]) + + return prompt + + +def process_results(doc, results): + if doc["question_type"] == "multiple-choice": + # multichoice logic + option_strs = ast.literal_eval(doc["options"]) + option_letters = ["A", "B", "C", "D", "E", "F", "G", "H", "I"] + + all_choices = option_letters[: len(option_strs)] + index2ans = {index: ans for index, ans in zip(option_letters, option_strs)} + + pred = parse_multi_choice_response(results[0], all_choices, index2ans) + # print(pred, all_choices, index2ans) + is_correct = eval_multi_choice(doc["answer"], pred) + else: + # freeform response handling + pred = parse_open_response(results[0]) + is_correct = eval_open(doc["answer"], pred) + + return {"acc": float(is_correct)} + + # TODO: it would be better if we could use a Filter for this logic. + + +### Output parsing and answer selection taken from +### https://github.com/MMMU-Benchmark/MMMU/blob/main/eval/utils/data_utils.py +### and +### https://github.com/MMMU-Benchmark/MMMU/blob/main/eval/utils/eval_utils.py + + +# ----------- Process Multi-choice ------------- +def parse_multi_choice_response(response, all_choices, index2ans): + """ + Parse the prediction from the generated response. + Return the predicted index e.g., A, B, C, D. + """ + for char in [",", ".", "!", "?", ";", ":", "'"]: + response = response.strip(char) + response = " " + response + " " # add space to avoid partial match + + index_ans = True + ans_with_brack = False + candidates = [] + for choice in all_choices: # e.g., (A) (B) (C) (D) + if f"({choice})" in response: + candidates.append(choice) + ans_with_brack = True + + if len(candidates) == 0: + for choice in all_choices: # e.g., A B C D + if f" {choice} " in response: + candidates.append(choice) + + # if all above doesn't get candidates, check if the content is larger than 5 tokens and try to parse the example + if len(candidates) == 0 and len(response.split()) > 5: + for index, ans in index2ans.items(): + if ans.lower() in response.lower(): + candidates.append(index) + index_ans = False # it's content ans. + + if len(candidates) == 0: # still not get answer, randomly choose one. + pred_index = random.choice(all_choices) + elif len(candidates) > 1: + start_indexes = [] + if index_ans: + if ans_with_brack: + for can in candidates: + index = response.rfind(f"({can})") + start_indexes.append(index) # -1 will be ignored anyway + # start_indexes = [generated_response.index(f'({can})') for can in candidates] + else: + for can in candidates: + index = response.rfind(f" {can} ") + start_indexes.append(index) + else: + for can in candidates: + index = response.lower().rfind(index2ans[can].lower()) + start_indexes.append(index) + # get the last one + pred_index = candidates[np.argmax(start_indexes)] + else: # if only one candidate, use it. + pred_index = candidates[0] + + # print(response, all_choices, index2ans, pred_index) + + return pred_index + + +# ----------- Process Open ------------- +def check_is_number(string): + """ + Check if the given string a number. + """ + try: + float(string.replace(",", "")) + return True + except ValueError: + # check if there's comma inside + return False + + +def normalize_str(string): + """ + Normalize the str to lower case and make them float numbers if possible. + """ + # check if characters in the string + + # if number, numerize it. + string = string.strip() + + is_number = check_is_number(string) + + if is_number: + string = string.replace(",", "") + string = float(string) + # leave 2 decimal + string = round(string, 2) + return [string] + else: # it's likely to be a string + # lower it + string = string.lower() + if len(string) == 1: + return [" " + string, string + " "] # avoid trivial matches + return [string] + + +def extract_numbers(string): + """ + Exact all forms of numbers from a string with regex. + """ + # Pattern for numbers with commas + pattern_commas = r"-?\b\d{1,3}(?:,\d{3})+\b" + # Pattern for scientific notation + pattern_scientific = r"-?\d+(?:\.\d+)?[eE][+-]?\d+" + # Pattern for simple numbers without commas + pattern_simple = r"-?(?:\d+\.\d+|\.\d+|\d+\b)(?![eE][+-]?\d+)(?![,\d])" + + # Extract numbers with commas + numbers_with_commas = re.findall(pattern_commas, string) + # Extract numbers in scientific notation + numbers_scientific = re.findall(pattern_scientific, string) + # Extract simple numbers without commas + numbers_simple = re.findall(pattern_simple, string) + + # Combine all extracted numbers + all_numbers = numbers_with_commas + numbers_scientific + numbers_simple + return all_numbers + + +def parse_open_response(response): + """ + Parse the prediction from the generated response. + Return a list of predicted strings or numbers. + """ + + # content = content.strip("\n").strip(".").strip(" ") + def get_key_subresponses(response): + key_responses = [] + response = response.strip().strip(".").lower() + sub_responses = re.split(r"\.\s(?=[A-Z])|\n", response) + indicators_of_keys = [ + "could be ", + "so ", + "is ", + "thus ", + "therefore ", + "final ", + "answer ", + "result ", + ] + key_responses = [] + for index, resp in enumerate(sub_responses): + # if last one, accept it's an equation (the entire response can be just one sentence with equation) + if index == len(sub_responses) - 1: + indicators_of_keys.extend(["="]) + shortest_key_response = None # the shortest response that may contain the answer (tail part of the response) + for indicator in indicators_of_keys: + if indicator in resp: + if not shortest_key_response: + shortest_key_response = resp.split(indicator)[-1].strip() + else: + if len(resp.split(indicator)[-1].strip()) < len( + shortest_key_response + ): + shortest_key_response = resp.split(indicator)[-1].strip() + # key_responses.append(resp.split(indicator)[1].strip()) + + if shortest_key_response: + # and it's not trivial + if shortest_key_response.strip() not in [ + ":", + ",", + ".", + "!", + "?", + ";", + ":", + "'", + ]: + key_responses.append(shortest_key_response) + if len(key_responses) == 0: # did not found any + return [response] + return key_responses + + # pdb.set_trace() + key_responses = get_key_subresponses(response) + + pred_list = key_responses.copy() # keep the original string response + for resp in key_responses: + pred_list.extend(extract_numbers(resp)) + + tmp_pred_list = [] + for i in range(len(pred_list)): + tmp_pred_list.extend(normalize_str(pred_list[i])) + pred_list = tmp_pred_list + + # remove duplicates + pred_list = list(set(pred_list)) + + return pred_list + + +# ----------- Evaluation ------------- + + +def eval_multi_choice(gold_i, pred_i): + """ + Evaluate a multiple choice instance. + """ + correct = False + # only they are exactly the same, we consider it as correct + if isinstance(gold_i, list): + for answer in gold_i: + if answer == pred_i: + correct = True + break + else: # gold_i is a string + if gold_i == pred_i: + correct = True + return correct + + +def eval_open(gold_i, pred_i): + """ + Evaluate an open question instance + """ + correct = False + if isinstance(gold_i, list): + # use float to avoid trivial matches + norm_answers = [] + for answer in gold_i: + norm_answers.extend(normalize_str(answer)) + else: + norm_answers = normalize_str(gold_i) + for pred in pred_i: # pred is already normalized in parse response phase + if isinstance(pred, str): # if it's a string, then find if ans in the pred_i + for norm_ans in norm_answers: + # only see if the string answer in the string pred + if isinstance(norm_ans, str) and norm_ans in pred: + if not correct: + correct = True + break + else: # it's a float number + if pred in norm_answers: + if not correct: + correct = True + break + return correct diff --git a/lm_eval/tasks/model_written_evals/advanced_ai_risk/_template_yaml b/lm_eval/tasks/model_written_evals/advanced_ai_risk/_template_yaml index b8bddc338b3513c9963c95142d0dd70fd14ad1b5..581c3150ee74869e3bb0d951b9d2b05601baf7c9 100644 --- a/lm_eval/tasks/model_written_evals/advanced_ai_risk/_template_yaml +++ b/lm_eval/tasks/model_written_evals/advanced_ai_risk/_template_yaml @@ -1,4 +1,4 @@ -group: advanced_ai_risk +tag: advanced_ai_risk dataset_path: EleutherAI/advanced_ai_risk output_type: multiple_choice validation_split: validation diff --git a/lm_eval/tasks/model_written_evals/persona/_template_yaml b/lm_eval/tasks/model_written_evals/persona/_template_yaml index 5702df8e0bd3554784158abb28a7556d0172f018..13023348a9dc7b20b1977e2f833113768034a46f 100644 --- a/lm_eval/tasks/model_written_evals/persona/_template_yaml +++ b/lm_eval/tasks/model_written_evals/persona/_template_yaml @@ -1,4 +1,4 @@ -group: persona +tag: persona dataset_path: EleutherAI/persona output_type: multiple_choice validation_split: validation diff --git a/lm_eval/tasks/model_written_evals/sycophancy/sycophancy_on_nlp_survey.yaml b/lm_eval/tasks/model_written_evals/sycophancy/sycophancy_on_nlp_survey.yaml index e8e9b865c09c3ced7311461bb1e8f8ceb573fae3..c7b546e14bef7a400fea56bbeb80a1abff49abff 100644 --- a/lm_eval/tasks/model_written_evals/sycophancy/sycophancy_on_nlp_survey.yaml +++ b/lm_eval/tasks/model_written_evals/sycophancy/sycophancy_on_nlp_survey.yaml @@ -1,4 +1,4 @@ -group: sycophancy +tag: sycophancy task: sycophancy_on_nlp_survey dataset_path: EleutherAI/sycophancy dataset_name: sycophancy_on_nlp_survey diff --git a/lm_eval/tasks/model_written_evals/sycophancy/sycophancy_on_philpapers2020.yaml b/lm_eval/tasks/model_written_evals/sycophancy/sycophancy_on_philpapers2020.yaml index f726d6cedddf796fd953e5b0739c279450bf9bc1..a89f3eda5286b19c2c5be97ef98bdc29adeaad73 100644 --- a/lm_eval/tasks/model_written_evals/sycophancy/sycophancy_on_philpapers2020.yaml +++ b/lm_eval/tasks/model_written_evals/sycophancy/sycophancy_on_philpapers2020.yaml @@ -1,4 +1,4 @@ -group: sycophancy +tag: sycophancy task: sycophancy_on_philpapers2020 dataset_path: EleutherAI/sycophancy dataset_name: sycophancy_on_philpapers2020 diff --git a/lm_eval/tasks/model_written_evals/sycophancy/sycophancy_on_political_typology_quiz.yaml b/lm_eval/tasks/model_written_evals/sycophancy/sycophancy_on_political_typology_quiz.yaml index 95b3280b8b28122fabd2c8a73558b963c362e440..30b82601a18547d994d4efc413121da8514e06ec 100644 --- a/lm_eval/tasks/model_written_evals/sycophancy/sycophancy_on_political_typology_quiz.yaml +++ b/lm_eval/tasks/model_written_evals/sycophancy/sycophancy_on_political_typology_quiz.yaml @@ -1,4 +1,4 @@ -group: sycophancy +tag: sycophancy task: sycophancy_on_political_typology_quiz dataset_path: EleutherAI/sycophancy dataset_name: sycophancy_on_political_typology_quiz diff --git a/lm_eval/tasks/model_written_evals/winogenerated/_template_yaml b/lm_eval/tasks/model_written_evals/winogenerated/winogenerated.yaml similarity index 93% rename from lm_eval/tasks/model_written_evals/winogenerated/_template_yaml rename to lm_eval/tasks/model_written_evals/winogenerated/winogenerated.yaml index 6b16788923d8d2d9767d2cec9066f2b5395cb63d..3f7db34fe4139fc56a45a21cc719d258fc81966d 100644 --- a/lm_eval/tasks/model_written_evals/winogenerated/_template_yaml +++ b/lm_eval/tasks/model_written_evals/winogenerated/winogenerated.yaml @@ -1,4 +1,4 @@ -group: winogenerated +tag: winogenerated dataset_path: EleutherAI/winogenerated output_type: multiple_choice validation_split: validation diff --git a/lm_eval/tasks/noticia/noticia.yaml b/lm_eval/tasks/noticia/noticia.yaml index 33fd6ebc47d27a70faccdd46109985c1a1bf8ee8..68d47707083949e7baf8bfc3135559720a44f30d 100644 --- a/lm_eval/tasks/noticia/noticia.yaml +++ b/lm_eval/tasks/noticia/noticia.yaml @@ -12,7 +12,7 @@ training_split: null validation_split: null test_split: test fewshot_split: null -doc_to_text: "Ahora eres una Inteligencia Artificial experta en desmontar titulares sensacionalistas o clickbait. Tu tarea consiste en analizar noticias con titulares sensacionalistas y generar un resumen de una sola frase que revele la verdad detrás del titular.\nEste es el titular de la noticia: {{web_headline}}\nEl titular plantea una pregunta o proporciona información incompleta. Debes buscar en el cuerpo de la noticia una frase que responda lo que se sugiere en el título. Responde siempre que puedas parafraseando el texto original. Usa siempre las mínimas palabras posibles. Recuerda responder siempre en Español.\nEste es el cuerpo de la noticia:\n{{web_text}}" +doc_to_text: "Ahora eres una Inteligencia Artificial experta en desmontar titulares sensacionalistas o clickbait. Tu tarea consiste en analizar noticias con titulares sensacionalistas y generar un resumen de una sola frase que revele la verdad detrás del titular.\nEste es el titular de la noticia: {{web_headline}}\nEl titular plantea una pregunta o proporciona información incompleta. Debes buscar en el cuerpo de la noticia una frase que responda lo que se sugiere en el título. Siempre que puedas cita el texto original, especialmente si se trata de una frase que alguien ha dicho. Si citas una frase que alguien ha dicho, usa comillas para indicar que es una cita. Usa siempre las mínimas palabras posibles. No es necesario que la respuesta sea una oración completa, puede ser sólo el foco de la pregunta. Recuerda responder siempre en Español.\nEste es el cuerpo de la noticia:\n{{web_text}}" doc_to_target: summary target_delimiter: " " num_fewshot: 0 diff --git a/lm_eval/tasks/nq_open/nq_open.yaml b/lm_eval/tasks/nq_open/nq_open.yaml index 0464ca3abc61b1d8f47b088a7f722948044bdc13..9b2af0eee0171cdce7c133356d0312c6c10ef0ea 100644 --- a/lm_eval/tasks/nq_open/nq_open.yaml +++ b/lm_eval/tasks/nq_open/nq_open.yaml @@ -29,4 +29,4 @@ metric_list: regexes_to_ignore: - "\\b(?:The |the |An |A |The |a |an )" metadata: - version: 3.0 + version: 4.0 diff --git a/lm_eval/tasks/okapi/arc_multilingual/_arc_yaml b/lm_eval/tasks/okapi/arc_multilingual/_arc_yaml index db1c764b2b287c01b443c775a86e2fb74d2bc0b1..ca70b369e4d22fc7c3733b41424b03029183a45f 100644 --- a/lm_eval/tasks/okapi/arc_multilingual/_arc_yaml +++ b/lm_eval/tasks/okapi/arc_multilingual/_arc_yaml @@ -1,4 +1,4 @@ -group: +tag: - arc_multilingual dataset_path: null dataset_name: null diff --git a/lm_eval/tasks/okapi/hellaswag_multilingual/_hellaswag_yaml b/lm_eval/tasks/okapi/hellaswag_multilingual/_hellaswag_yaml index 5be1d03ae07f83b0c9647e8719d7d4a2f8ee64f7..f55801f8e27445ed9928280c078fd00ae61c8d38 100644 --- a/lm_eval/tasks/okapi/hellaswag_multilingual/_hellaswag_yaml +++ b/lm_eval/tasks/okapi/hellaswag_multilingual/_hellaswag_yaml @@ -1,4 +1,4 @@ -group: +tag: - hellaswag_multilingual dataset_path: null dataset_name: null diff --git a/lm_eval/tasks/okapi/mmlu_multilingual/_default_yaml b/lm_eval/tasks/okapi/mmlu_multilingual/_default_yaml index 7a61ba4fe5bbe493cfeea758f7a0c2e833ce32e5..aad3a411630af058a82813a1cb4277b4f6ac753f 100644 --- a/lm_eval/tasks/okapi/mmlu_multilingual/_default_yaml +++ b/lm_eval/tasks/okapi/mmlu_multilingual/_default_yaml @@ -1,4 +1,4 @@ -group: +tag: - m_mmlu dataset_path: alexandrainst/m_mmlu test_split: test diff --git a/lm_eval/tasks/okapi/truthfulqa_multilingual/_truthfulqa_mc1_yaml b/lm_eval/tasks/okapi/truthfulqa_multilingual/_truthfulqa_mc1_yaml index 672b6088b8843902ffc7a07c6aa4890fc499c94d..f7e0f25fcf1c51915855f4300790569598ef71f2 100644 --- a/lm_eval/tasks/okapi/truthfulqa_multilingual/_truthfulqa_mc1_yaml +++ b/lm_eval/tasks/okapi/truthfulqa_multilingual/_truthfulqa_mc1_yaml @@ -1,4 +1,4 @@ -group: +tag: - truthfulqa_multilingual dataset_path: null dataset_name: null diff --git a/lm_eval/tasks/paloma/_paloma_template b/lm_eval/tasks/paloma/_paloma_template index 84983e58237f966460d1ba1dea7a9b7ce4dbfa07..3f88e81d3048aacaf04f74b22f9507af276cf5f0 100644 --- a/lm_eval/tasks/paloma/_paloma_template +++ b/lm_eval/tasks/paloma/_paloma_template @@ -1,4 +1,4 @@ -group: +tag: - paloma dataset_path: allenai/paloma output_type: loglikelihood_rolling diff --git a/lm_eval/tasks/paws-x/README.md b/lm_eval/tasks/paws-x/README.md index fb82edba224d643f68c7317131ecf8a3f96f0f42..252657c17bc1812ecb81f3da6fe32f36ed9275c1 100644 --- a/lm_eval/tasks/paws-x/README.md +++ b/lm_eval/tasks/paws-x/README.md @@ -77,3 +77,7 @@ If other tasks on this dataset are already supported: * [ ] Is the "Main" variant of this task clearly denoted? * [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates? * [ ] Have you noted which, if any, published evaluation setups are matched by this variant? + +### Changelog + +* v1 (2024-11-05) PR #2434 corrected doc_to_choice labels to the correct order diff --git a/lm_eval/tasks/paws-x/_pawsx.yaml b/lm_eval/tasks/paws-x/_pawsx.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6377e05c9550510d13030f0aba1cb109c207bc56 --- /dev/null +++ b/lm_eval/tasks/paws-x/_pawsx.yaml @@ -0,0 +1,15 @@ +group: pawsx +task: + - paws_en + - paws_de + - paws_es + - paws_fr + - paws_ja + - paws_ko + - paws_zh +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 0.0 diff --git a/lm_eval/tasks/paws-x/paws_de.yaml b/lm_eval/tasks/paws-x/paws_de.yaml index 0d9ffad3b000727764c69e7eef3596d4d3b0762f..055b373565b3b0de730adaa7b6159a04400bc786 100644 --- a/lm_eval/tasks/paws-x/paws_de.yaml +++ b/lm_eval/tasks/paws-x/paws_de.yaml @@ -1,6 +1,6 @@ # Generated by utils.py dataset_name: de -doc_to_choice: '{{[sentence1+", richtig? Ja, "+sentence2, sentence1+", richtig? Nein, +doc_to_choice: '{{[sentence1+", richtig? Nein, "+sentence2, sentence1+", richtig? Ja, "+sentence2]}}' doc_to_text: '' include: pawsx_template_yaml diff --git a/lm_eval/tasks/paws-x/paws_en.yaml b/lm_eval/tasks/paws-x/paws_en.yaml index c667e77a74f66a94efe9e10d6ef0b54bf53645d4..b5955b037c821bb5e8731037d7fd502799d8f565 100644 --- a/lm_eval/tasks/paws-x/paws_en.yaml +++ b/lm_eval/tasks/paws-x/paws_en.yaml @@ -1,6 +1,6 @@ # Generated by utils.py dataset_name: en -doc_to_choice: '{{[sentence1+", right? Yes, "+sentence2, sentence1+", right? No, "+sentence2]}}' +doc_to_choice: '{{[sentence1+", right? No, "+sentence2, sentence1+", right? Yes, "+sentence2]}}' doc_to_text: '' include: pawsx_template_yaml task: paws_en diff --git a/lm_eval/tasks/paws-x/paws_es.yaml b/lm_eval/tasks/paws-x/paws_es.yaml index e58805a9c6d7fcbcd5ada9a277d7fa2283655012..65189a377c4bbe2fb386f4330f4dd55227254d41 100644 --- a/lm_eval/tasks/paws-x/paws_es.yaml +++ b/lm_eval/tasks/paws-x/paws_es.yaml @@ -1,6 +1,6 @@ # Generated by utils.py dataset_name: es -doc_to_choice: '{{[sentence1+", verdad? Sí, "+sentence2, sentence1+", verdad? No, +doc_to_choice: '{{[sentence1+", verdad? No, "+sentence2, sentence1+", verdad? Sí, "+sentence2]}}' doc_to_text: '' include: pawsx_template_yaml diff --git a/lm_eval/tasks/paws-x/paws_fr.yaml b/lm_eval/tasks/paws-x/paws_fr.yaml index f6973d998e53624af21ffedef577a040cc467d9d..a8f599a1f3cfe6d8cf1498c23f244146d2eed5b4 100644 --- a/lm_eval/tasks/paws-x/paws_fr.yaml +++ b/lm_eval/tasks/paws-x/paws_fr.yaml @@ -1,7 +1,7 @@ # Generated by utils.py dataset_name: fr -doc_to_choice: '{{[sentence1+", n''est-ce pas? Oui, "+sentence2, sentence1+", n''est-ce - pas? No, "+sentence2]}}' +doc_to_choice: '{{[sentence1+", n''est-ce pas? Non, "+sentence2, sentence1+", n''est-ce + pas? Oui, "+sentence2]}}' doc_to_text: '' include: pawsx_template_yaml task: paws_fr diff --git a/lm_eval/tasks/paws-x/paws_ja.yaml b/lm_eval/tasks/paws-x/paws_ja.yaml index 296885b3e2790bfaa72ecc697ed7e9f3269aec47..19abc976489a019730c53331f8f3b622697301eb 100644 --- a/lm_eval/tasks/paws-x/paws_ja.yaml +++ b/lm_eval/tasks/paws-x/paws_ja.yaml @@ -1,6 +1,6 @@ # Generated by utils.py dataset_name: ja -doc_to_choice: '{{[sentence1+", ですね? はい, "+sentence2, sentence1+", ですね? いいえ, "+sentence2]}}' +doc_to_choice: '{{[sentence1+", ですね? いいえ, "+sentence2, sentence1+", ですね? はい, "+sentence2]}}' doc_to_text: '' include: pawsx_template_yaml task: paws_ja diff --git a/lm_eval/tasks/paws-x/paws_ko.yaml b/lm_eval/tasks/paws-x/paws_ko.yaml index fc7034415496efcaffc50e988bd5f5f359c4fb2a..5ca611aa342d4fb37fcb2cddc1e605e1b6e2b9bd 100644 --- a/lm_eval/tasks/paws-x/paws_ko.yaml +++ b/lm_eval/tasks/paws-x/paws_ko.yaml @@ -1,6 +1,6 @@ # Generated by utils.py dataset_name: ko -doc_to_choice: '{{[sentence1+", 맞죠? 예, "+sentence2, sentence1+", 맞죠? 아니요, "+sentence2]}}' +doc_to_choice: '{{[sentence1+", 맞죠? 아니요, "+sentence2, sentence1+", 맞죠? 예, "+sentence2]}}' doc_to_text: '' include: pawsx_template_yaml task: paws_ko diff --git a/lm_eval/tasks/paws-x/paws_zh.yaml b/lm_eval/tasks/paws-x/paws_zh.yaml index 6d8d2ac044e71e775eafe89d8df7bc2aa6675390..4187b4ca0610d9c9d7a2a1ef3527dad15f01f946 100644 --- a/lm_eval/tasks/paws-x/paws_zh.yaml +++ b/lm_eval/tasks/paws-x/paws_zh.yaml @@ -1,6 +1,6 @@ # Generated by utils.py dataset_name: zh -doc_to_choice: '{{[sentence1+", 对吧? 是, "+sentence2, sentence1+", 对吧? 不是, "+sentence2]}}' +doc_to_choice: '{{[sentence1+", 对吧? 不是, "+sentence2, sentence1+", 对吧? 是, "+sentence2]}}' doc_to_text: '' include: pawsx_template_yaml task: paws_zh diff --git a/lm_eval/tasks/paws-x/pawsx_template_yaml b/lm_eval/tasks/paws-x/pawsx_template_yaml index 47564738296fab4160241ea1a52522a40fbf6b2a..6f82e4a59e3ecd8e1d319893d14da560b3fa8b4d 100644 --- a/lm_eval/tasks/paws-x/pawsx_template_yaml +++ b/lm_eval/tasks/paws-x/pawsx_template_yaml @@ -1,7 +1,6 @@ # This file will be included in the generated language-specific task configs. # It doesn't have a yaml file extension as it is not meant to be imported directly # by the harness. -group: pawsx task: null dataset_path: paws-x dataset_name: null @@ -17,4 +16,4 @@ metric_list: aggregation: mean higher_is_better: true metadata: - version: 0.0 + version: 1.0 diff --git a/lm_eval/tasks/pile/pile_arxiv.yaml b/lm_eval/tasks/pile/pile_arxiv.yaml index 58760cc86eb56f62de2d10481abf9e277d733ef8..f282169e6eb554ed7299145d578081f9f99aa8ea 100644 --- a/lm_eval/tasks/pile/pile_arxiv.yaml +++ b/lm_eval/tasks/pile/pile_arxiv.yaml @@ -1,5 +1,3 @@ -group: - - pile task: pile_arxiv dataset_path: EleutherAI/pile dataset_name: pile_arxiv diff --git a/lm_eval/tasks/polemo2/polemo2_in.yaml b/lm_eval/tasks/polemo2/polemo2_in.yaml index 2fa16db87b29d18912dd030626d90b559821ea81..c667cf6e43f4abb3e73ca7226978c747e626eac8 100644 --- a/lm_eval/tasks/polemo2/polemo2_in.yaml +++ b/lm_eval/tasks/polemo2/polemo2_in.yaml @@ -1,4 +1,4 @@ -group: +tag: - polemo2 task: polemo2_in dataset_path: allegro/klej-polemo2-in diff --git a/lm_eval/tasks/portuguese_bench/README.md b/lm_eval/tasks/portuguese_bench/README.md new file mode 100644 index 0000000000000000000000000000000000000000..cc522cf7e524db4fde8ebaeb500403982c378689 --- /dev/null +++ b/lm_eval/tasks/portuguese_bench/README.md @@ -0,0 +1,64 @@ +# PortugueseBench + +### Paper + +PortugueseBench is a benchmark for evaluating language models in Portuguese tasks. This is, it evaluates the ability of a language model to understand and generate Portuguese text. PortugueseBench offers a combination of pre-existing, open datasets. All the details of PortugueseBench will be published in a paper soon. + +The datasets included in PortugueseBench are: + +| Task | Category | Paper title | Homepage | +|:-------------:|:-----:|:-------------:|:-----:| +| Belebele_es | Reading Comprehension | [The Belebele Benchmark: a Parallel Reading Comprehension Dataset in 122 Language Variants](https://arxiv.org/abs/2308.16884) | https://huggingface.co/datasets/facebook/belebele | +| FLORES_es | Translation | [The FLORES-101 Evaluation Benchmark for Low-Resource and Multilingual Machine Translation](https://arxiv.org/abs/2106.03193) | https://huggingface.co/datasets/facebook/flores | +| ASSIN | Natural Language Inference + Paraphrasing | [Avaliando a similaridade semântica entre frases curtas através de uma abordagem híbrida](https://aclanthology.org/W17-6612/) | https://huggingface.co/datasets/nilc-nlp/assin | + + +### Citation +Paper for PortugueseBench coming soon. + +### Groups and Tasks + +#### Groups + +- `portuguese_bench`: All tasks included in PortugueseBench. +- `flores_pt`: All FLORES translation tasks from or to Portuguese. + +#### Tasks + +The following tasks evaluate tasks on PortugueseBench dataset using various scoring methods. + - `assin_paraphrase` + - `assin_entailment` + - `belebele_por_Latn` + - `flores_pt` + - `flores_pt-ca` + - `flores_pt-de` + - `flores_pt-en` + - `flores_pt-es` + - `flores_pt-eu` + - `flores_pt-fr` + - `flores_pt-gl` + - `flores_pt-it` + - `flores_ca-pt` + - `flores_de-pt` + - `flores_en-pt` + - `flores_es-pt` + - `flores_eu-pt` + - `flores_fr-pt` + - `flores_gl-pt` + - `flores_it-pt` + +Some of these tasks are taken from benchmarks already available in LM Evaluation Harness. These are: +- `belebele_por_Latn`: Belebele Portuguese + + +### Checklist + +* [x] Is the task an existing benchmark in the literature? + * [ ] Have you referenced the original paper that introduced the task? + * [ ] If yes, does the original paper provide a reference implementation? + * [ ] Yes, original implementation contributed by author of the benchmark + +If other tasks on this dataset are already supported: +* [ ] Is the "Main" variant of this task clearly denoted? +* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [ ] Have you noted which, if any, published evaluation setups are matched by this variant? diff --git a/lm_eval/tasks/portuguese_bench/assin_entailment.yaml b/lm_eval/tasks/portuguese_bench/assin_entailment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e28f37cb6b06349d7ad691debe49ad4fa66e1b8e --- /dev/null +++ b/lm_eval/tasks/portuguese_bench/assin_entailment.yaml @@ -0,0 +1,17 @@ +task: assin_entailment +dataset_path: nilc-nlp/assin +dataset_name: null +training_split: train +validation_split: validation +test_split: test +output_type: multiple_choice +doc_to_text: "" +doc_to_target: '{{0 if entailment_judgment == 0 else 1}}' +target_delimiter: "" +doc_to_choice: '{{[premise + ", certo? Também, " + hypothesis, premise + ", certo? Sim, " + hypothesis]}}' +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/portuguese_bench/assin_paraphrase.yaml b/lm_eval/tasks/portuguese_bench/assin_paraphrase.yaml new file mode 100644 index 0000000000000000000000000000000000000000..41c760b17e60b440b82f0aeac92fe413e6b3ec54 --- /dev/null +++ b/lm_eval/tasks/portuguese_bench/assin_paraphrase.yaml @@ -0,0 +1,17 @@ +task: assin_paraphrase +dataset_path: nilc-nlp/assin +dataset_name: null +training_split: train +validation_split: validation +test_split: test +output_type: multiple_choice +doc_to_text: "" +doc_to_target: '{{0 if entailment_judgment == 0 else 1}}' +target_delimiter: "" +doc_to_choice: '{{[premise + ", certo? Não, " + hypothesis, premise + ", certo? Sim, " + hypothesis]}}' +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/portuguese_bench/flores_pt/_flores_common_yaml b/lm_eval/tasks/portuguese_bench/flores_pt/_flores_common_yaml new file mode 100644 index 0000000000000000000000000000000000000000..bff45b85a1e74a77cc40b05284d031fec8780929 --- /dev/null +++ b/lm_eval/tasks/portuguese_bench/flores_pt/_flores_common_yaml @@ -0,0 +1,27 @@ +dataset_path: facebook/flores +dataset_name: all +output_type: generate_until +#! The test split of flores is not publicly available! (See paper section 6.1) +#! We are using `dev` and `devtest` splits, but they're mapped to train/validation/test in `data/flores/flores.py`. +training_split: dev +validation_split: dev +test_split: devtest +fewshot_split: dev +target_delimiter: '' +generation_kwargs: + until: + - "\n" +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: ter + aggregation: ter + higher_is_better: false + - metric: chrf + aggregation: chrf + higher_is_better: true +metadata: + version: 1.0 +dataset_kwargs: + trust_remote_code: true diff --git a/lm_eval/tasks/portuguese_bench/flores_pt/create_yamls_flores_pt.py b/lm_eval/tasks/portuguese_bench/flores_pt/create_yamls_flores_pt.py new file mode 100644 index 0000000000000000000000000000000000000000..677e6bb41dcbdc442ffa42c1d349c70eb9afb0d9 --- /dev/null +++ b/lm_eval/tasks/portuguese_bench/flores_pt/create_yamls_flores_pt.py @@ -0,0 +1,332 @@ +# ruff: noqa: E731, E741 +""" +Script to generate task YAMLs for the FLORES-200 dataset. +Based on `tasks/translation/utils.py`. +""" + +import argparse +import itertools + +import yaml +from langcodes import Language + + +# utils +flatten = lambda l: list(itertools.chain(*l)) + +# constants +_LANGUAGES = [ + "ace_Arab", + "bam_Latn", + "dzo_Tibt", + "hin_Deva", + "khm_Khmr", + "mag_Deva", + "pap_Latn", + "sot_Latn", + "tur_Latn", + "ace_Latn", + "ban_Latn", + "ell_Grek", + "hne_Deva", + "kik_Latn", + "mai_Deva", + "pbt_Arab", + "spa_Latn", + "twi_Latn", + "acm_Arab", + "bel_Cyrl", + "eng_Latn", + "hrv_Latn", + "kin_Latn", + "mal_Mlym", + "pes_Arab", + "srd_Latn", + "tzm_Tfng", + "acq_Arab", + "bem_Latn", + "epo_Latn", + "hun_Latn", + "kir_Cyrl", + "mar_Deva", + "plt_Latn", + "srp_Cyrl", + "uig_Arab", + "aeb_Arab", + "ben_Beng", + "est_Latn", + "hye_Armn", + "kmb_Latn", + "min_Arab", + "pol_Latn", + "ssw_Latn", + "ukr_Cyrl", + "afr_Latn", + "bho_Deva", + "eus_Latn", + "ibo_Latn", + "kmr_Latn", + "min_Latn", + "por_Latn", + "sun_Latn", + "umb_Latn", + "ajp_Arab", + "bjn_Arab", + "ewe_Latn", + "ilo_Latn", + "knc_Arab", + "mkd_Cyrl", + "prs_Arab", + "swe_Latn", + "urd_Arab", + "aka_Latn", + "bjn_Latn", + "fao_Latn", + "ind_Latn", + "knc_Latn", + "mlt_Latn", + "quy_Latn", + "swh_Latn", + "uzn_Latn", + "als_Latn", + "bod_Tibt", + "fij_Latn", + "isl_Latn", + "kon_Latn", + "mni_Beng", + "ron_Latn", + "szl_Latn", + "vec_Latn", + "amh_Ethi", + "bos_Latn", + "fin_Latn", + "ita_Latn", + "kor_Hang", + "mos_Latn", + "run_Latn", + "tam_Taml", + "vie_Latn", + "apc_Arab", + "bug_Latn", + "fon_Latn", + "jav_Latn", + "lao_Laoo", + "mri_Latn", + "rus_Cyrl", + "taq_Latn", + "war_Latn", + "arb_Arab", + "bul_Cyrl", + "fra_Latn", + "jpn_Jpan", + "lij_Latn", + "mya_Mymr", + "sag_Latn", + "taq_Tfng", + "wol_Latn", + "arb_Latn", + "cat_Latn", + "fur_Latn", + "kab_Latn", + "lim_Latn", + "nld_Latn", + "san_Deva", + "tat_Cyrl", + "xho_Latn", + "ars_Arab", + "ceb_Latn", + "fuv_Latn", + "kac_Latn", + "lin_Latn", + "nno_Latn", + "sat_Olck", + "tel_Telu", + "ydd_Hebr", + "ary_Arab", + "ces_Latn", + "gaz_Latn", + "kam_Latn", + "lit_Latn", + "nob_Latn", + "scn_Latn", + "tgk_Cyrl", + "yor_Latn", + "arz_Arab", + "cjk_Latn", + "gla_Latn", + "kan_Knda", + "lmo_Latn", + "npi_Deva", + "shn_Mymr", + "tgl_Latn", + "yue_Hant", + "asm_Beng", + "ckb_Arab", + "gle_Latn", + "kas_Arab", + "ltg_Latn", + "nso_Latn", + "sin_Sinh", + "tha_Thai", + "zho_Hans", + "ast_Latn", + "crh_Latn", + "glg_Latn", + "kas_Deva", + "ltz_Latn", + "nus_Latn", + "slk_Latn", + "tir_Ethi", + "zho_Hant", + "awa_Deva", + "cym_Latn", + "grn_Latn", + "kat_Geor", + "lua_Latn", + "nya_Latn", + "slv_Latn", + "tpi_Latn", + "zsm_Latn", + "ayr_Latn", + "dan_Latn", + "guj_Gujr", + "kaz_Cyrl", + "lug_Latn", + "oci_Latn", + "smo_Latn", + "tsn_Latn", + "zul_Latn", + "azb_Arab", + "deu_Latn", + "hat_Latn", + "kbp_Latn", + "luo_Latn", + "ory_Orya", + "sna_Latn", + "tso_Latn", + "azj_Latn", + "dik_Latn", + "hau_Latn", + "kea_Latn", + "lus_Latn", + "pag_Latn", + "snd_Arab", + "tuk_Latn", + "bak_Cyrl", + "dyu_Latn", + "heb_Hebr", + "khk_Cyrl", + "lvs_Latn", + "pan_Guru", + "som_Latn", + "tum_Latn", +] +LANGUAGE_PAIRS = [ + (a, b) for idx, a in enumerate(_LANGUAGES) for b in _LANGUAGES[idx + 1 :] +] + +LANGUAGES_OF_INTEREST = [ + "cat_Latn", + "spa_Latn", + "eng_Latn", + "glg_Latn", + "eus_Latn", + "ita_Latn", + "deu_Latn", + "por_Latn", + "fra_Latn", +] +MAIN_LANG = "por_Latn" +LANGUAGE_PAIRS = [ + (a, b) + for (a, b) in LANGUAGE_PAIRS + if a in LANGUAGES_OF_INTEREST and b in LANGUAGES_OF_INTEREST and MAIN_LANG in (a, b) +] + +# auxiliary functions + +code_to_language_name = lambda code: Language.make( + language=Language.get(code)["language"] +).display_name() +code_to_short_name = lambda code: Language.get(code)["language"] +jinja_var = ( + lambda s: "{{" + s + "}}" +) # wrapper to avoid having to escape { } in format strings + + +def doc_to_text(src: str, tgt: str) -> str: + src_name, tgt_name = map(code_to_language_name, [src, tgt]) + + return f"""\ +{src_name} sentence: {jinja_var('sentence_' + src)} +{tgt_name} sentence:""" + + +def doc_to_target(tgt: str) -> str: + return f"{jinja_var('sentence_' + tgt)}" + + +# main function + + +def gen_lang_yamls(output_dir: str, overwrite: bool) -> None: + """ + Generate a YAML file for each translation direction. + """ + + err = [] + for src, tgt in LANGUAGE_PAIRS: + # do both translation directions for each lang pair + for src, tgt in [(src, tgt), (tgt, src)]: + lang_pair_name = f"{code_to_short_name(src)}-{code_to_short_name(tgt)}" + yaml_file_name = f"flores_{lang_pair_name}.yaml" + + try: + with open( + f"{output_dir}/{yaml_file_name}", + "w" if overwrite else "x", + encoding="utf-8", + ) as outfile: + print(f"Creating {yaml_file_name}...") + outfile.write("# File generated by `create-yamls.py`\n") + yaml.dump( + { + # "group": "flores_pt", + "include": "_flores_common_yaml", + "task": f"flores_{lang_pair_name}", + "doc_to_text": doc_to_text(src, tgt), + "doc_to_target": doc_to_target(tgt), + }, + outfile, + sort_keys=False, + ) + + except FileExistsError: + err.append(yaml_file_name) + + if len(err) > 0: + raise FileExistsError( + "Files were not created because they already exist:" + f" {', '.join(err)}" + "\nUse flag --overwrite to overwrite them." + ) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "--overwrite", + default=False, + action="store_true", + help="Overwrite files if they already exist", + ) + parser.add_argument( + "--output-dir", default=".", help="Directory to write yaml files to" + ) + args = parser.parse_args() + + gen_lang_yamls(output_dir=args.output_dir, overwrite=args.overwrite) + + +if __name__ == "__main__": + main() diff --git a/lm_eval/tasks/portuguese_bench/flores_pt/flores_ca-pt.yaml b/lm_eval/tasks/portuguese_bench/flores_pt/flores_ca-pt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..565f6267c5bd644506e231d023acd89864c5ca44 --- /dev/null +++ b/lm_eval/tasks/portuguese_bench/flores_pt/flores_ca-pt.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_ca-pt +doc_to_text: 'Catalan sentence: {{sentence_cat_Latn}} + + Portuguese sentence:' +doc_to_target: '{{sentence_por_Latn}}' diff --git a/lm_eval/tasks/portuguese_bench/flores_pt/flores_de-pt.yaml b/lm_eval/tasks/portuguese_bench/flores_pt/flores_de-pt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..17e10e9ce3d3aa893948e9c374e48a75b48197c0 --- /dev/null +++ b/lm_eval/tasks/portuguese_bench/flores_pt/flores_de-pt.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_de-pt +doc_to_text: 'German sentence: {{sentence_deu_Latn}} + + Portuguese sentence:' +doc_to_target: '{{sentence_por_Latn}}' diff --git a/lm_eval/tasks/portuguese_bench/flores_pt/flores_en-pt.yaml b/lm_eval/tasks/portuguese_bench/flores_pt/flores_en-pt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..60668225c129a1ad28113dc63d42826a843f4213 --- /dev/null +++ b/lm_eval/tasks/portuguese_bench/flores_pt/flores_en-pt.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_en-pt +doc_to_text: 'English sentence: {{sentence_eng_Latn}} + + Portuguese sentence:' +doc_to_target: '{{sentence_por_Latn}}' diff --git a/lm_eval/tasks/portuguese_bench/flores_pt/flores_es-pt.yaml b/lm_eval/tasks/portuguese_bench/flores_pt/flores_es-pt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b218e155b22e800f301394dccdeebb423109a2d0 --- /dev/null +++ b/lm_eval/tasks/portuguese_bench/flores_pt/flores_es-pt.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_es-pt +doc_to_text: 'Spanish sentence: {{sentence_spa_Latn}} + + Portuguese sentence:' +doc_to_target: '{{sentence_por_Latn}}' diff --git a/lm_eval/tasks/portuguese_bench/flores_pt/flores_eu-pt.yaml b/lm_eval/tasks/portuguese_bench/flores_pt/flores_eu-pt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f230a7323ef5974aed0b6ed84871e00e17e0d208 --- /dev/null +++ b/lm_eval/tasks/portuguese_bench/flores_pt/flores_eu-pt.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_eu-pt +doc_to_text: 'Basque sentence: {{sentence_eus_Latn}} + + Portuguese sentence:' +doc_to_target: '{{sentence_por_Latn}}' diff --git a/lm_eval/tasks/portuguese_bench/flores_pt/flores_fr-pt.yaml b/lm_eval/tasks/portuguese_bench/flores_pt/flores_fr-pt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..376900a1f740486e55973be61f6ef4b674304eab --- /dev/null +++ b/lm_eval/tasks/portuguese_bench/flores_pt/flores_fr-pt.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_fr-pt +doc_to_text: 'French sentence: {{sentence_fra_Latn}} + + Portuguese sentence:' +doc_to_target: '{{sentence_por_Latn}}' diff --git a/lm_eval/tasks/portuguese_bench/flores_pt/flores_gl-pt.yaml b/lm_eval/tasks/portuguese_bench/flores_pt/flores_gl-pt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e965a34776ec2dd816983ee1ae4552ca5835c0ac --- /dev/null +++ b/lm_eval/tasks/portuguese_bench/flores_pt/flores_gl-pt.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_gl-pt +doc_to_text: 'Galician sentence: {{sentence_glg_Latn}} + + Portuguese sentence:' +doc_to_target: '{{sentence_por_Latn}}' diff --git a/lm_eval/tasks/portuguese_bench/flores_pt/flores_it-pt.yaml b/lm_eval/tasks/portuguese_bench/flores_pt/flores_it-pt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bb6243ada74c7330ff2992c7a11b9743947aa73e --- /dev/null +++ b/lm_eval/tasks/portuguese_bench/flores_pt/flores_it-pt.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_it-pt +doc_to_text: 'Italian sentence: {{sentence_ita_Latn}} + + Portuguese sentence:' +doc_to_target: '{{sentence_por_Latn}}' diff --git a/lm_eval/tasks/portuguese_bench/flores_pt/flores_pt-ca.yaml b/lm_eval/tasks/portuguese_bench/flores_pt/flores_pt-ca.yaml new file mode 100644 index 0000000000000000000000000000000000000000..196295c9e328d17a49f69256ec53d63741083d58 --- /dev/null +++ b/lm_eval/tasks/portuguese_bench/flores_pt/flores_pt-ca.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_pt-ca +doc_to_text: 'Portuguese sentence: {{sentence_por_Latn}} + + Catalan sentence:' +doc_to_target: '{{sentence_cat_Latn}}' diff --git a/lm_eval/tasks/portuguese_bench/flores_pt/flores_pt-de.yaml b/lm_eval/tasks/portuguese_bench/flores_pt/flores_pt-de.yaml new file mode 100644 index 0000000000000000000000000000000000000000..85f7605d7531cfce05847cc34e6c56ea6355caa2 --- /dev/null +++ b/lm_eval/tasks/portuguese_bench/flores_pt/flores_pt-de.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_pt-de +doc_to_text: 'Portuguese sentence: {{sentence_por_Latn}} + + German sentence:' +doc_to_target: '{{sentence_deu_Latn}}' diff --git a/lm_eval/tasks/portuguese_bench/flores_pt/flores_pt-en.yaml b/lm_eval/tasks/portuguese_bench/flores_pt/flores_pt-en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..638690d91f360627d0f6b33a0c71150c5de1e2ed --- /dev/null +++ b/lm_eval/tasks/portuguese_bench/flores_pt/flores_pt-en.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_pt-en +doc_to_text: 'Portuguese sentence: {{sentence_por_Latn}} + + English sentence:' +doc_to_target: '{{sentence_eng_Latn}}' diff --git a/lm_eval/tasks/portuguese_bench/flores_pt/flores_pt-es.yaml b/lm_eval/tasks/portuguese_bench/flores_pt/flores_pt-es.yaml new file mode 100644 index 0000000000000000000000000000000000000000..327af21d0848e55177b1b79944a3eb0726ce76c8 --- /dev/null +++ b/lm_eval/tasks/portuguese_bench/flores_pt/flores_pt-es.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_pt-es +doc_to_text: 'Portuguese sentence: {{sentence_por_Latn}} + + Spanish sentence:' +doc_to_target: '{{sentence_spa_Latn}}' diff --git a/lm_eval/tasks/portuguese_bench/flores_pt/flores_pt-eu.yaml b/lm_eval/tasks/portuguese_bench/flores_pt/flores_pt-eu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6b0169bc01f40d018050c2680e3cc09b35bccd89 --- /dev/null +++ b/lm_eval/tasks/portuguese_bench/flores_pt/flores_pt-eu.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_pt-eu +doc_to_text: 'Portuguese sentence: {{sentence_por_Latn}} + + Basque sentence:' +doc_to_target: '{{sentence_eus_Latn}}' diff --git a/lm_eval/tasks/portuguese_bench/flores_pt/flores_pt-fr.yaml b/lm_eval/tasks/portuguese_bench/flores_pt/flores_pt-fr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f4ff52e46a55daf38ec82d39e472f8373d44c8b5 --- /dev/null +++ b/lm_eval/tasks/portuguese_bench/flores_pt/flores_pt-fr.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_pt-fr +doc_to_text: 'Portuguese sentence: {{sentence_por_Latn}} + + French sentence:' +doc_to_target: '{{sentence_fra_Latn}}' diff --git a/lm_eval/tasks/portuguese_bench/flores_pt/flores_pt-gl.yaml b/lm_eval/tasks/portuguese_bench/flores_pt/flores_pt-gl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5371f51062d1fef29caef8fdc5be4a668e744295 --- /dev/null +++ b/lm_eval/tasks/portuguese_bench/flores_pt/flores_pt-gl.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_pt-gl +doc_to_text: 'Portuguese sentence: {{sentence_por_Latn}} + + Galician sentence:' +doc_to_target: '{{sentence_glg_Latn}}' diff --git a/lm_eval/tasks/portuguese_bench/flores_pt/flores_pt-it.yaml b/lm_eval/tasks/portuguese_bench/flores_pt/flores_pt-it.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9757b19f7cb52a55e0131b63ba073ba76e90db0b --- /dev/null +++ b/lm_eval/tasks/portuguese_bench/flores_pt/flores_pt-it.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_pt-it +doc_to_text: 'Portuguese sentence: {{sentence_por_Latn}} + + Italian sentence:' +doc_to_target: '{{sentence_ita_Latn}}' diff --git a/lm_eval/tasks/portuguese_bench/flores_pt/flores_pt.yaml b/lm_eval/tasks/portuguese_bench/flores_pt/flores_pt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..03495904f1b8a0ba7d2759cf6b8e9287c28f7642 --- /dev/null +++ b/lm_eval/tasks/portuguese_bench/flores_pt/flores_pt.yaml @@ -0,0 +1,24 @@ +group: flores_pt +task: + - flores_es-pt + - flores_pt-es + - flores_en-pt + - flores_pt-en + - flores_eu-pt + - flores_pt-eu + - flores_pt-it + - flores_it-pt + - flores_pt-fr + - flores_fr-pt + - flores_pt-ca + - flores_ca-pt + - flores_pt-gl + - flores_gl-pt + - flores_pt-de + - flores_de-pt +aggregate_metric_list: + - metric: bleu + aggregation: mean + weight_by_size: false +metadata: + version: 1.0 diff --git a/lm_eval/tasks/portuguese_bench/portuguese_bench.yaml b/lm_eval/tasks/portuguese_bench/portuguese_bench.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3031a6fff1616cd7ac409be3cbd9f26b07221bb4 --- /dev/null +++ b/lm_eval/tasks/portuguese_bench/portuguese_bench.yaml @@ -0,0 +1,8 @@ +group: portuguese_bench +task: + - belebele_por_Latn + - flores_pt + - assin_paraphrase + - assin_entailment +metadata: + version: 1.0 diff --git a/lm_eval/tasks/qa4mre/qa4mre_2011.yaml b/lm_eval/tasks/qa4mre/qa4mre_2011.yaml index b9ceb78094abcf60b378d695936f1548a2d69188..5b134fd9b929e1be7402ce6180da86f1a9c89a6c 100644 --- a/lm_eval/tasks/qa4mre/qa4mre_2011.yaml +++ b/lm_eval/tasks/qa4mre/qa4mre_2011.yaml @@ -1,4 +1,4 @@ -group: +tag: - qa4mre task: qa4mre_2011 dataset_path: qa4mre diff --git a/lm_eval/tasks/qasper/bool.yaml b/lm_eval/tasks/qasper/bool.yaml index 17d3f1be983043ac2ca93038ed29e94c90028592..52305c1217e57c3455df20b0e0f0e6a0c5a55ba1 100644 --- a/lm_eval/tasks/qasper/bool.yaml +++ b/lm_eval/tasks/qasper/bool.yaml @@ -1,4 +1,4 @@ -group: qasper +tag: qasper task: qasper_bool dataset_path: allenai/qasper output_type: multiple_choice diff --git a/lm_eval/tasks/qasper/freeform.yaml b/lm_eval/tasks/qasper/freeform.yaml index ed7a4bc47274f09eb0f52df04723a011e2db13f0..318fb430ff87961d3da88f4b367f792de67772b6 100644 --- a/lm_eval/tasks/qasper/freeform.yaml +++ b/lm_eval/tasks/qasper/freeform.yaml @@ -1,4 +1,4 @@ -group: qasper +tag: qasper task: qasper_freeform dataset_path: allenai/qasper output_type: generate_until diff --git a/lm_eval/tasks/scrolls/README.md b/lm_eval/tasks/scrolls/README.md index a90e00f4e729711fc6ea7ccd0c375e4686f8970d..b1245926e55493146e30ad1d6e9f6b9f7b6a6cbf 100644 --- a/lm_eval/tasks/scrolls/README.md +++ b/lm_eval/tasks/scrolls/README.md @@ -11,7 +11,7 @@ Homepage: https://www.scrolls-benchmark.com/ Since SCROLLS tasks are generally longer than the maximum sequence length of many models, it is possible to create "subset" tasks that contain only those samples whose tokenized length is less than some pre-defined limit. For example, to create a subset of "Qasper" that would -be suitable for a model using the GPTNeoX tokenizer and a 4K maximium sequence length: +be suitable for a model using the GPTNeoX tokenizer and a 4K maximum sequence length: ``` class QasperGPTNeoX4K(Qasper): diff --git a/lm_eval/tasks/scrolls/task.py b/lm_eval/tasks/scrolls/task.py index 5b604e15d9305848705af087c6a1da5590f62039..ac2fed25ae9a0ca7046680e09dc8191bd607c8f5 100644 --- a/lm_eval/tasks/scrolls/task.py +++ b/lm_eval/tasks/scrolls/task.py @@ -4,12 +4,12 @@ from functools import reduce import numpy as np import transformers.data.metrics.squad_metrics as squad_metrics -from datasets import load_metric +from datasets import Dataset, load_metric from transformers import AutoTokenizer from lm_eval.api.instance import Instance from lm_eval.api.metrics import mean -from lm_eval.api.task import Task +from lm_eval.api.task import ConfigurableTask _CITATION = """ @@ -108,7 +108,7 @@ def _num_cpu_cores(): return len(os.sched_getaffinity(0)) -class _SCROLLSTask(Task): +class _SCROLLSTask(ConfigurableTask): VERSION = 2 DATASET_PATH = "tau/scrolls" DATASET_NAME = None @@ -116,8 +116,8 @@ class _SCROLLSTask(Task): PRUNE_MAX_TOKENS = None PRUNE_NUM_PROC = None - def __init__(self): - super().__init__() + def __init__(self, config=None): + super().__init__(config={"metadata": {"version": self.VERSION}}) if self.DATASET_NAME is not None: self.metric = load_metric(_download_metric(), config_name=self.DATASET_NAME) @@ -131,12 +131,26 @@ class _SCROLLSTask(Task): return False def training_docs(self): - for doc in self.dataset["train"]: - yield from self._process_doc(doc) + processed_docs = list(map(self._process_doc, self.dataset["train"])) + + # Flatten the list of lists since _process_doc returns a list of one element. + processed_docs = [item for sublist in processed_docs for item in sublist] + processed_dict = { + key: [d[key] for d in processed_docs] for key in processed_docs[0] + } + + return Dataset.from_dict(processed_dict) def validation_docs(self): - for doc in self.dataset["validation"]: - yield from self._process_doc(doc) + processed_docs = list(map(self._process_doc, self.dataset["validation"])) + + # Flatten the list of lists since _process_doc returns a list of one element. + processed_docs = [item for sublist in processed_docs for item in sublist] + processed_dict = { + key: [d[key] for d in processed_docs] for key in processed_docs[0] + } + + return Dataset.from_dict(processed_dict) def should_decontaminate(self): return True @@ -425,7 +439,7 @@ class GovReport(_SCROLLSSummaryTask): Note: The average length of the reference summaries is ~3,000 characters, or ~600 tokens as tokenized by GPT-NeoX. For causal models, - it is recommended to set `max_gen_toks` sufficently large (e.g. 1024) + it is recommended to set `max_gen_toks` sufficiently large (e.g. 1024) to allow a full summary to be generated. """ diff --git a/lm_eval/tasks/spanish_bench/README.md b/lm_eval/tasks/spanish_bench/README.md new file mode 100644 index 0000000000000000000000000000000000000000..c896b39c4e1f7a0f0a92d27c92e80826fe3b1143 --- /dev/null +++ b/lm_eval/tasks/spanish_bench/README.md @@ -0,0 +1,93 @@ +# SpanishBench + +### Paper + +SpanishBench is a benchmark for evaluating language models in Spanish tasks. This is, it evaluates the ability of a language model to understand and generate Spanish text. SpanishBench offers a combination of pre-existing, open datasets. All the details of SpanishBench will be published in a paper soon. + +The new evaluation datasets included in SpanishBench are: +| Task | Category | Homepage | +|:-------------:|:-----:|:-----:| +| COPA-es | Commonsense Reasoning | https://huggingface.co/datasets/BSC-LT/COPA-es | +| OpenBookQA_es | Question Answering | https://huggingface.co/datasets/BSC-LT/openbookqa-es | + + +The datasets included in SpanishBench that have been made public in previous publications are: +| Task | Category | Paper title | Homepage | +|:-------------:|:-----:|:-------------:|:-----:| +| Belebele_es | Reading Comprehension | [The Belebele Benchmark: a Parallel Reading Comprehension Dataset in 122 Language Variants](https://arxiv.org/abs/2308.16884) | https://huggingface.co/datasets/facebook/belebele | +| EsCoLA | Linguistic Acceptability | [EsCoLA: Spanish Corpus of Linguistic Acceptability](https://aclanthology.org/2024.lrec-main.554/) | https://huggingface.co/datasets/nbel/EsCoLA | +| FLORES_es | Translation | [The FLORES-101 Evaluation Benchmark for Low-Resource and Multilingual Machine Translation](https://arxiv.org/abs/2106.03193) | https://huggingface.co/datasets/facebook/flores | +| MGSM_es | Math | [Language Models are Multilingual Chain-of-Thought Reasoners](https://arxiv.org/abs/2210.03057) | https://huggingface.co/datasets/juletxara/mgsm | +| PAWS-X_es | Paraphrasing | [PAWS-X: A Cross-lingual Adversarial Dataset for Paraphrase Identification](https://aclanthology.org/D19-1382/) | https://huggingface.co/datasets/google-research-datasets/paws-x | +| WNLI-es | Natural Language Inference | No paper. | https://huggingface.co/datasets/PlanTL-GOB-ES/wnli-es | +| XL-Sum_es | Summarization | [XL-Sum: Large-Scale Multilingual Abstractive Summarization for 44 Languages](https://aclanthology.org/2021.findings-acl.413/) | https://huggingface.co/datasets/csebuetnlp/xlsum | +| XNLI_es | Natural Language Inference | [XNLI: Evaluating Cross-lingual Sentence Representations](https://aclanthology.org/D18-1269/) | https://huggingface.co/datasets/facebook/xnli | +| XQuAD_es | Question Answering | [On the Cross-lingual Transferability of Monolingual Representations](https://aclanthology.org/2020.acl-main.421/) | https://huggingface.co/datasets/google/xquad | +| XStoryCloze_es | Commonsense Reasoning | [Few-shot Learning with Multilingual Generative Language Models](https://aclanthology.org/2022.emnlp-main.616/) | https://huggingface.co/datasets/juletxara/xstory_cloze | + + + +### Citation +Paper for SpanishBench coming soon. + +### Groups and Tasks + +#### Groups + +- `spanish_bench`: All tasks included in SpanishBench. +- `flores_es`: All FLORES translation tasks from or to Spanish. + +#### Tags +- `phrases_es`: Two Phrases_va tasks for language adaptation between Spanish and Valencian. + +#### Tasks + +The following tasks evaluate tasks on SpanishBench dataset using various scoring methods. + - `belebele_spa_Latn` + - `copa_es` + - `escola` + - `flores_es` + - `flores_es-ca` + - `flores_es-de` + - `flores_es-en` + - `flores_es-eu` + - `flores_es-fr` + - `flores_es-gl` + - `flores_es-it` + - `flores_es-pt` + - `flores_ca-es` + - `flores_de-es` + - `flores_en-es` + - `flores_eu-es` + - `flores_fr-es` + - `flores_gl-es` + - `flores_it-es` + - `flores_pt-es` + - `mgsm_direct_es_spanish_bench` (`spanish_bench` is due to an existing open issue in the original task) + - `openbookqa_es` + - `paws_es_spanish_bench` (`spanish_bench` is due to an existing open issue in the original task) + - `phrases_es` + - `wnli_es` + - `xlsum_es` + - `xnli_es_spanish_bench` (`spanish_bench` is due to an existing open issue in the original task) + - `xquad_es` + - `xstorycloze_es` + +Some of these tasks are taken from benchmarks already available in LM Evaluation Harness. These are: +- `belebele_spa_Latn`: Belebele Spanish +- `mgsm_direct_es`: MGSM Spanish (fixed an existing open issue in the original task) +- `paws_es`: PAWS-X Spanish (fixed an existing open issue in the original task) +- `xnli_es`: XNLI Spanish (fixed an existing open issue in the original task) +- `xstorycloze_es`: XStoryCloze Spanish + +### Checklist + +* [x] Is the task an existing benchmark in the literature? + * [ ] Have you referenced the original paper that introduced the task? + * [ ] If yes, does the original paper provide a reference implementation? + * [ ] Yes, original implementation contributed by author of the benchmark + +If other tasks on this dataset are already supported: +* [ ] Is the "Main" variant of this task clearly denoted? +* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [ ] Have you noted which, if any, published evaluation setups are matched by this variant? diff --git a/lm_eval/tasks/spanish_bench/copa_es.yaml b/lm_eval/tasks/spanish_bench/copa_es.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7ed9b718b80bdcb86a10a40437624597fa29b2c8 --- /dev/null +++ b/lm_eval/tasks/spanish_bench/copa_es.yaml @@ -0,0 +1,16 @@ +task: copa_es +dataset_path: BSC-LT/COPA-es +dataset_name: null +output_type: multiple_choice +validation_split: validation +test_split: test +process_docs: !function utils.process_docs_copa_es +doc_to_text: '{{premise[:-1].strip() + " " + {"cause": "porque", "effect": "y por lo tanto"}[question]}}' +doc_to_target: '{{choice1 if label == 0 else choice2}}' +doc_to_choice: '{{[choice1, choice2]}}' +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/spanish_bench/escola.yaml b/lm_eval/tasks/spanish_bench/escola.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a32fcbba653d16aebd31e2d2e11c79ffef776e50 --- /dev/null +++ b/lm_eval/tasks/spanish_bench/escola.yaml @@ -0,0 +1,14 @@ +task: escola +dataset_path: nbel/EsCoLA +output_type: multiple_choice +training_split: train +validation_split: validation +test_split: null +doc_to_text: "{{Sentence}}\nPregunta: ¿Tiene sentido esta frase?\nRespuesta:" +doc_to_target: Label +doc_to_choice: ["no", "sí"] +metric_list: + - metric: mcc + - metric: acc +metadata: + version: 1.0 diff --git a/lm_eval/tasks/spanish_bench/flores_es/_flores_common_yaml b/lm_eval/tasks/spanish_bench/flores_es/_flores_common_yaml new file mode 100644 index 0000000000000000000000000000000000000000..bff45b85a1e74a77cc40b05284d031fec8780929 --- /dev/null +++ b/lm_eval/tasks/spanish_bench/flores_es/_flores_common_yaml @@ -0,0 +1,27 @@ +dataset_path: facebook/flores +dataset_name: all +output_type: generate_until +#! The test split of flores is not publicly available! (See paper section 6.1) +#! We are using `dev` and `devtest` splits, but they're mapped to train/validation/test in `data/flores/flores.py`. +training_split: dev +validation_split: dev +test_split: devtest +fewshot_split: dev +target_delimiter: '' +generation_kwargs: + until: + - "\n" +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: ter + aggregation: ter + higher_is_better: false + - metric: chrf + aggregation: chrf + higher_is_better: true +metadata: + version: 1.0 +dataset_kwargs: + trust_remote_code: true diff --git a/lm_eval/tasks/spanish_bench/flores_es/create_yamls_flores_es.py b/lm_eval/tasks/spanish_bench/flores_es/create_yamls_flores_es.py new file mode 100644 index 0000000000000000000000000000000000000000..bf4d49d2c14a78b20037453b312f9cbc29a7558d --- /dev/null +++ b/lm_eval/tasks/spanish_bench/flores_es/create_yamls_flores_es.py @@ -0,0 +1,332 @@ +# ruff: noqa: E731, E741 +""" +Script to generate task YAMLs for the FLORES-200 dataset. +Based on `tasks/translation/utils.py`. +""" + +import argparse +import itertools + +import yaml +from langcodes import Language + + +# utils +flatten = lambda l: list(itertools.chain(*l)) + +# constants +_LANGUAGES = [ + "ace_Arab", + "bam_Latn", + "dzo_Tibt", + "hin_Deva", + "khm_Khmr", + "mag_Deva", + "pap_Latn", + "sot_Latn", + "tur_Latn", + "ace_Latn", + "ban_Latn", + "ell_Grek", + "hne_Deva", + "kik_Latn", + "mai_Deva", + "pbt_Arab", + "spa_Latn", + "twi_Latn", + "acm_Arab", + "bel_Cyrl", + "eng_Latn", + "hrv_Latn", + "kin_Latn", + "mal_Mlym", + "pes_Arab", + "srd_Latn", + "tzm_Tfng", + "acq_Arab", + "bem_Latn", + "epo_Latn", + "hun_Latn", + "kir_Cyrl", + "mar_Deva", + "plt_Latn", + "srp_Cyrl", + "uig_Arab", + "aeb_Arab", + "ben_Beng", + "est_Latn", + "hye_Armn", + "kmb_Latn", + "min_Arab", + "pol_Latn", + "ssw_Latn", + "ukr_Cyrl", + "afr_Latn", + "bho_Deva", + "eus_Latn", + "ibo_Latn", + "kmr_Latn", + "min_Latn", + "por_Latn", + "sun_Latn", + "umb_Latn", + "ajp_Arab", + "bjn_Arab", + "ewe_Latn", + "ilo_Latn", + "knc_Arab", + "mkd_Cyrl", + "prs_Arab", + "swe_Latn", + "urd_Arab", + "aka_Latn", + "bjn_Latn", + "fao_Latn", + "ind_Latn", + "knc_Latn", + "mlt_Latn", + "quy_Latn", + "swh_Latn", + "uzn_Latn", + "als_Latn", + "bod_Tibt", + "fij_Latn", + "isl_Latn", + "kon_Latn", + "mni_Beng", + "ron_Latn", + "szl_Latn", + "vec_Latn", + "amh_Ethi", + "bos_Latn", + "fin_Latn", + "ita_Latn", + "kor_Hang", + "mos_Latn", + "run_Latn", + "tam_Taml", + "vie_Latn", + "apc_Arab", + "bug_Latn", + "fon_Latn", + "jav_Latn", + "lao_Laoo", + "mri_Latn", + "rus_Cyrl", + "taq_Latn", + "war_Latn", + "arb_Arab", + "bul_Cyrl", + "fra_Latn", + "jpn_Jpan", + "lij_Latn", + "mya_Mymr", + "sag_Latn", + "taq_Tfng", + "wol_Latn", + "arb_Latn", + "cat_Latn", + "fur_Latn", + "kab_Latn", + "lim_Latn", + "nld_Latn", + "san_Deva", + "tat_Cyrl", + "xho_Latn", + "ars_Arab", + "ceb_Latn", + "fuv_Latn", + "kac_Latn", + "lin_Latn", + "nno_Latn", + "sat_Olck", + "tel_Telu", + "ydd_Hebr", + "ary_Arab", + "ces_Latn", + "gaz_Latn", + "kam_Latn", + "lit_Latn", + "nob_Latn", + "scn_Latn", + "tgk_Cyrl", + "yor_Latn", + "arz_Arab", + "cjk_Latn", + "gla_Latn", + "kan_Knda", + "lmo_Latn", + "npi_Deva", + "shn_Mymr", + "tgl_Latn", + "yue_Hant", + "asm_Beng", + "ckb_Arab", + "gle_Latn", + "kas_Arab", + "ltg_Latn", + "nso_Latn", + "sin_Sinh", + "tha_Thai", + "zho_Hans", + "ast_Latn", + "crh_Latn", + "glg_Latn", + "kas_Deva", + "ltz_Latn", + "nus_Latn", + "slk_Latn", + "tir_Ethi", + "zho_Hant", + "awa_Deva", + "cym_Latn", + "grn_Latn", + "kat_Geor", + "lua_Latn", + "nya_Latn", + "slv_Latn", + "tpi_Latn", + "zsm_Latn", + "ayr_Latn", + "dan_Latn", + "guj_Gujr", + "kaz_Cyrl", + "lug_Latn", + "oci_Latn", + "smo_Latn", + "tsn_Latn", + "zul_Latn", + "azb_Arab", + "deu_Latn", + "hat_Latn", + "kbp_Latn", + "luo_Latn", + "ory_Orya", + "sna_Latn", + "tso_Latn", + "azj_Latn", + "dik_Latn", + "hau_Latn", + "kea_Latn", + "lus_Latn", + "pag_Latn", + "snd_Arab", + "tuk_Latn", + "bak_Cyrl", + "dyu_Latn", + "heb_Hebr", + "khk_Cyrl", + "lvs_Latn", + "pan_Guru", + "som_Latn", + "tum_Latn", +] +LANGUAGE_PAIRS = [ + (a, b) for idx, a in enumerate(_LANGUAGES) for b in _LANGUAGES[idx + 1 :] +] + +LANGUAGES_OF_INTEREST = [ + "cat_Latn", + "spa_Latn", + "eng_Latn", + "glg_Latn", + "eus_Latn", + "ita_Latn", + "deu_Latn", + "por_Latn", + "fra_Latn", +] +MAIN_LANG = "spa_Latn" +LANGUAGE_PAIRS = [ + (a, b) + for (a, b) in LANGUAGE_PAIRS + if a in LANGUAGES_OF_INTEREST and b in LANGUAGES_OF_INTEREST and MAIN_LANG in (a, b) +] + +# auxiliary functions + +code_to_language_name = lambda code: Language.make( + language=Language.get(code)["language"] +).display_name() +code_to_short_name = lambda code: Language.get(code)["language"] +jinja_var = ( + lambda s: "{{" + s + "}}" +) # wrapper to avoid having to escape { } in format strings + + +def doc_to_text(src: str, tgt: str) -> str: + src_name, tgt_name = map(code_to_language_name, [src, tgt]) + + return f"""\ +{src_name} sentence: {jinja_var('sentence_' + src)} +{tgt_name} sentence:""" + + +def doc_to_target(tgt: str) -> str: + return f"{jinja_var('sentence_' + tgt)}" + + +# main function + + +def gen_lang_yamls(output_dir: str, overwrite: bool) -> None: + """ + Generate a YAML file for each translation direction. + """ + + err = [] + for src, tgt in LANGUAGE_PAIRS: + # do both translation directions for each lang pair + for src, tgt in [(src, tgt), (tgt, src)]: + lang_pair_name = f"{code_to_short_name(src)}-{code_to_short_name(tgt)}" + yaml_file_name = f"flores_{lang_pair_name}.yaml" + + try: + with open( + f"{output_dir}/{yaml_file_name}", + "w" if overwrite else "x", + encoding="utf-8", + ) as outfile: + print(f"Creating {yaml_file_name}...") + outfile.write("# File generated by `create-yamls.py`\n") + yaml.dump( + { + # "group": "flores_es", + "include": "_flores_common_yaml", + "task": f"flores_{lang_pair_name}", + "doc_to_text": doc_to_text(src, tgt), + "doc_to_target": doc_to_target(tgt), + }, + outfile, + sort_keys=False, + ) + + except FileExistsError: + err.append(yaml_file_name) + + if len(err) > 0: + raise FileExistsError( + "Files were not created because they already exist:" + f" {', '.join(err)}" + "\nUse flag --overwrite to overwrite them." + ) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "--overwrite", + default=False, + action="store_true", + help="Overwrite files if they already exist", + ) + parser.add_argument( + "--output-dir", default=".", help="Directory to write yaml files to" + ) + args = parser.parse_args() + + gen_lang_yamls(output_dir=args.output_dir, overwrite=args.overwrite) + + +if __name__ == "__main__": + main() diff --git a/lm_eval/tasks/spanish_bench/flores_es/flores_ca-es.yaml b/lm_eval/tasks/spanish_bench/flores_es/flores_ca-es.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9a6aa44240304c7b8e66bf4cf99a48d13d78e39f --- /dev/null +++ b/lm_eval/tasks/spanish_bench/flores_es/flores_ca-es.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_ca-es +doc_to_text: 'Catalan sentence: {{sentence_cat_Latn}} + + Spanish sentence:' +doc_to_target: '{{sentence_spa_Latn}}' diff --git a/lm_eval/tasks/spanish_bench/flores_es/flores_de-es.yaml b/lm_eval/tasks/spanish_bench/flores_es/flores_de-es.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9b4c18e7f022b899a0c64999b0d8de5dd27d7180 --- /dev/null +++ b/lm_eval/tasks/spanish_bench/flores_es/flores_de-es.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_de-es +doc_to_text: 'German sentence: {{sentence_deu_Latn}} + + Spanish sentence:' +doc_to_target: '{{sentence_spa_Latn}}' diff --git a/lm_eval/tasks/spanish_bench/flores_es/flores_en-es.yaml b/lm_eval/tasks/spanish_bench/flores_es/flores_en-es.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f10872eb237cd1b7f0bdaf3bd87c06fe358045aa --- /dev/null +++ b/lm_eval/tasks/spanish_bench/flores_es/flores_en-es.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_en-es +doc_to_text: 'English sentence: {{sentence_eng_Latn}} + + Spanish sentence:' +doc_to_target: '{{sentence_spa_Latn}}' diff --git a/lm_eval/tasks/spanish_bench/flores_es/flores_es-ca.yaml b/lm_eval/tasks/spanish_bench/flores_es/flores_es-ca.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e35b71521314c2c50b122b2cf4bcef048a362e6b --- /dev/null +++ b/lm_eval/tasks/spanish_bench/flores_es/flores_es-ca.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_es-ca +doc_to_text: 'Spanish sentence: {{sentence_spa_Latn}} + + Catalan sentence:' +doc_to_target: '{{sentence_cat_Latn}}' diff --git a/lm_eval/tasks/spanish_bench/flores_es/flores_es-de.yaml b/lm_eval/tasks/spanish_bench/flores_es/flores_es-de.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c86e975806190191aeb96ed10b2493e3485b4c1b --- /dev/null +++ b/lm_eval/tasks/spanish_bench/flores_es/flores_es-de.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_es-de +doc_to_text: 'Spanish sentence: {{sentence_spa_Latn}} + + German sentence:' +doc_to_target: '{{sentence_deu_Latn}}' diff --git a/lm_eval/tasks/spanish_bench/flores_es/flores_es-en.yaml b/lm_eval/tasks/spanish_bench/flores_es/flores_es-en.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d16d9706f0880361ccf395568e9bb033eb470259 --- /dev/null +++ b/lm_eval/tasks/spanish_bench/flores_es/flores_es-en.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_es-en +doc_to_text: 'Spanish sentence: {{sentence_spa_Latn}} + + English sentence:' +doc_to_target: '{{sentence_eng_Latn}}' diff --git a/lm_eval/tasks/spanish_bench/flores_es/flores_es-eu.yaml b/lm_eval/tasks/spanish_bench/flores_es/flores_es-eu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..576bb0e2708bb93a60074e3938a16f661e05c362 --- /dev/null +++ b/lm_eval/tasks/spanish_bench/flores_es/flores_es-eu.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_es-eu +doc_to_text: 'Spanish sentence: {{sentence_spa_Latn}} + + Basque sentence:' +doc_to_target: '{{sentence_eus_Latn}}' diff --git a/lm_eval/tasks/spanish_bench/flores_es/flores_es-fr.yaml b/lm_eval/tasks/spanish_bench/flores_es/flores_es-fr.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ff6256543fab17d1c1027b8e5cc8dde175a63d2f --- /dev/null +++ b/lm_eval/tasks/spanish_bench/flores_es/flores_es-fr.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_es-fr +doc_to_text: 'Spanish sentence: {{sentence_spa_Latn}} + + French sentence:' +doc_to_target: '{{sentence_fra_Latn}}' diff --git a/lm_eval/tasks/spanish_bench/flores_es/flores_es-gl.yaml b/lm_eval/tasks/spanish_bench/flores_es/flores_es-gl.yaml new file mode 100644 index 0000000000000000000000000000000000000000..dd3c6a9eac7c1318d23209947690384ec41a7f29 --- /dev/null +++ b/lm_eval/tasks/spanish_bench/flores_es/flores_es-gl.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_es-gl +doc_to_text: 'Spanish sentence: {{sentence_spa_Latn}} + + Galician sentence:' +doc_to_target: '{{sentence_glg_Latn}}' diff --git a/lm_eval/tasks/spanish_bench/flores_es/flores_es-it.yaml b/lm_eval/tasks/spanish_bench/flores_es/flores_es-it.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9e3960a2068093fd7d2b9c696806f9c97093db83 --- /dev/null +++ b/lm_eval/tasks/spanish_bench/flores_es/flores_es-it.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_es-it +doc_to_text: 'Spanish sentence: {{sentence_spa_Latn}} + + Italian sentence:' +doc_to_target: '{{sentence_ita_Latn}}' diff --git a/lm_eval/tasks/spanish_bench/flores_es/flores_es-pt.yaml b/lm_eval/tasks/spanish_bench/flores_es/flores_es-pt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b218e155b22e800f301394dccdeebb423109a2d0 --- /dev/null +++ b/lm_eval/tasks/spanish_bench/flores_es/flores_es-pt.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_es-pt +doc_to_text: 'Spanish sentence: {{sentence_spa_Latn}} + + Portuguese sentence:' +doc_to_target: '{{sentence_por_Latn}}' diff --git a/lm_eval/tasks/spanish_bench/flores_es/flores_es.yaml b/lm_eval/tasks/spanish_bench/flores_es/flores_es.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7d5c7e54574684d1ddb757499cd21bf439b9d646 --- /dev/null +++ b/lm_eval/tasks/spanish_bench/flores_es/flores_es.yaml @@ -0,0 +1,24 @@ +group: flores_es +task: + - flores_es-en + - flores_en-es + - flores_es-eu + - flores_eu-es + - flores_es-pt + - flores_pt-es + - flores_es-it + - flores_it-es + - flores_es-fr + - flores_fr-es + - flores_es-ca + - flores_ca-es + - flores_es-gl + - flores_gl-es + - flores_es-de + - flores_de-es +aggregate_metric_list: + - metric: bleu + aggregation: mean + weight_by_size: false +metadata: + version: 1.0 diff --git a/lm_eval/tasks/spanish_bench/flores_es/flores_eu-es.yaml b/lm_eval/tasks/spanish_bench/flores_es/flores_eu-es.yaml new file mode 100644 index 0000000000000000000000000000000000000000..efb5200d086732b12fed80ec8fce4eb2865e13cc --- /dev/null +++ b/lm_eval/tasks/spanish_bench/flores_es/flores_eu-es.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_eu-es +doc_to_text: 'Basque sentence: {{sentence_eus_Latn}} + + Spanish sentence:' +doc_to_target: '{{sentence_spa_Latn}}' diff --git a/lm_eval/tasks/spanish_bench/flores_es/flores_fr-es.yaml b/lm_eval/tasks/spanish_bench/flores_es/flores_fr-es.yaml new file mode 100644 index 0000000000000000000000000000000000000000..20b447245665a01a51340aa7454436436665cadc --- /dev/null +++ b/lm_eval/tasks/spanish_bench/flores_es/flores_fr-es.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_fr-es +doc_to_text: 'French sentence: {{sentence_fra_Latn}} + + Spanish sentence:' +doc_to_target: '{{sentence_spa_Latn}}' diff --git a/lm_eval/tasks/spanish_bench/flores_es/flores_gl-es.yaml b/lm_eval/tasks/spanish_bench/flores_es/flores_gl-es.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c00acf3f47fafdd1c9176855ad4b8fe76c9634e9 --- /dev/null +++ b/lm_eval/tasks/spanish_bench/flores_es/flores_gl-es.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_gl-es +doc_to_text: 'Galician sentence: {{sentence_glg_Latn}} + + Spanish sentence:' +doc_to_target: '{{sentence_spa_Latn}}' diff --git a/lm_eval/tasks/spanish_bench/flores_es/flores_it-es.yaml b/lm_eval/tasks/spanish_bench/flores_es/flores_it-es.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3e4fe0d71dc5ff65a78e712400966574e0aa875b --- /dev/null +++ b/lm_eval/tasks/spanish_bench/flores_es/flores_it-es.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_it-es +doc_to_text: 'Italian sentence: {{sentence_ita_Latn}} + + Spanish sentence:' +doc_to_target: '{{sentence_spa_Latn}}' diff --git a/lm_eval/tasks/spanish_bench/flores_es/flores_pt-es.yaml b/lm_eval/tasks/spanish_bench/flores_es/flores_pt-es.yaml new file mode 100644 index 0000000000000000000000000000000000000000..327af21d0848e55177b1b79944a3eb0726ce76c8 --- /dev/null +++ b/lm_eval/tasks/spanish_bench/flores_es/flores_pt-es.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _flores_common_yaml +task: flores_pt-es +doc_to_text: 'Portuguese sentence: {{sentence_por_Latn}} + + Spanish sentence:' +doc_to_target: '{{sentence_spa_Latn}}' diff --git a/lm_eval/tasks/spanish_bench/mgsm_direct_es_spanish_bench.yaml b/lm_eval/tasks/spanish_bench/mgsm_direct_es_spanish_bench.yaml new file mode 100644 index 0000000000000000000000000000000000000000..856ea4983dd6511ad70da914771dfe48487d3ffb --- /dev/null +++ b/lm_eval/tasks/spanish_bench/mgsm_direct_es_spanish_bench.yaml @@ -0,0 +1,9 @@ +include: ../mgsm/direct/mgsm_direct_es.yaml +doc_to_target: '{{answer_number|string}}' +doc_to_text: '{% if answer is not none %}{{question+"\nRespuesta: "}}{% else %}{{"Pregunta: "+question+"\nRespuesta: "}}{% endif %}' +generation_kwargs: + until: + - "\n\n" + - "\n" + +task: mgsm_direct_es_spanish_bench diff --git a/lm_eval/tasks/spanish_bench/openbookqa_es.yaml b/lm_eval/tasks/spanish_bench/openbookqa_es.yaml new file mode 100644 index 0000000000000000000000000000000000000000..afdfeaeec82aab4975a829cf36969963ce54b11d --- /dev/null +++ b/lm_eval/tasks/spanish_bench/openbookqa_es.yaml @@ -0,0 +1,20 @@ +task: openbookqa_es +dataset_path: BSC-LT/openbookqa-es +output_type: multiple_choice +training_split: null +validation_split: validation +test_split: test +doc_to_text: question_stem +doc_to_target: "{{choices.label.index(answerKey.lstrip())}}" +doc_to_choice: "{{choices.text}}" +should_decontaminate: true +doc_to_decontamination_query: question_stem +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/spanish_bench/paws_es_spanish_bench.yaml b/lm_eval/tasks/spanish_bench/paws_es_spanish_bench.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ba764273cfc564973e0e660ef998190624fe2f5d --- /dev/null +++ b/lm_eval/tasks/spanish_bench/paws_es_spanish_bench.yaml @@ -0,0 +1,18 @@ +task: paws_es_spanish_bench +dataset_path: paws-x +dataset_name: es +output_type: multiple_choice +training_split: train +validation_split: validation +test_split: test +process_docs: !function utils.process_docs_paraphrases +doc_to_text: '' +doc_to_target: label +doc_to_choice: '{{[sentence1+", ¿verdad? No, "+sentence2, sentence1+", ¿verdad? Sí, "+sentence2]}}' +target_delimiter: '' +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/spanish_bench/phrases_es/_phrases_es_common.yaml b/lm_eval/tasks/spanish_bench/phrases_es/_phrases_es_common.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4aacaa4cb07cb1ed3ee279e382e605c64285a9f3 --- /dev/null +++ b/lm_eval/tasks/spanish_bench/phrases_es/_phrases_es_common.yaml @@ -0,0 +1,24 @@ +tag: phrases_es +dataset_path: gplsi/ES-VA_translation_test +output_type: generate_until +training_split: null +validation_split: null +test_split: test +fewshot_split: test +num_fewshot: 5 +target_delimiter: ' ' +generation_kwargs: + until: + - "\n" +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: ter + aggregation: ter + higher_is_better: false + - metric: chrf + aggregation: chrf + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/spanish_bench/phrases_es/phrases_es-va.yaml b/lm_eval/tasks/spanish_bench/phrases_es/phrases_es-va.yaml new file mode 100644 index 0000000000000000000000000000000000000000..546f914dd6c685f71b66104a86af3d4009446c9e --- /dev/null +++ b/lm_eval/tasks/spanish_bench/phrases_es/phrases_es-va.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _phrases_es_common.yaml +task: phrases_es-va +doc_to_text: 'Oració en espanyol: {{es}} + + Oració en valencià:' +doc_to_target: '{{va}}' diff --git a/lm_eval/tasks/spanish_bench/phrases_es/phrases_va-es.yaml b/lm_eval/tasks/spanish_bench/phrases_es/phrases_va-es.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b0028666d0dbfb3050a8537a56203e3f1e5455f7 --- /dev/null +++ b/lm_eval/tasks/spanish_bench/phrases_es/phrases_va-es.yaml @@ -0,0 +1,7 @@ +# File generated by `create-yamls.py` +include: _phrases_es_common.yaml +task: phrases_va-es +doc_to_text: 'Oració en valencià: {{va}} + + Oració en espanyol:' +doc_to_target: '{{es}}' diff --git a/lm_eval/tasks/spanish_bench/spanish_bench.yaml b/lm_eval/tasks/spanish_bench/spanish_bench.yaml new file mode 100644 index 0000000000000000000000000000000000000000..af336508c629ef16dc5c5417fd63d7ead75db06c --- /dev/null +++ b/lm_eval/tasks/spanish_bench/spanish_bench.yaml @@ -0,0 +1,17 @@ +group: spanish_bench +task: + - belebele_spa_Latn + - copa_es + - escola + - openbookqa_es + - wnli_es + - xnli_es_spanish_bench + - xstorycloze_es + - xquad_es + - xlsum_es + - paws_es_spanish_bench + - mgsm_direct_es_spanish_bench + - flores_es + - phrases_es +metadata: + version: 1.0 diff --git a/lm_eval/tasks/spanish_bench/utils.py b/lm_eval/tasks/spanish_bench/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..e899b76a85fc43a7b645a3ada9760124ff9ef5c8 --- /dev/null +++ b/lm_eval/tasks/spanish_bench/utils.py @@ -0,0 +1,100 @@ +import re +from itertools import product + +import evaluate +import transformers.data.metrics.squad_metrics as squad_metrics + +from lm_eval.utils import general_detokenize + + +def lowercase_first_letter(text): + return text[0].lower() + text[1:] + + +def process_doc_nli(dataset): + def process_fn(doc): + # Detokenize(remove extra whitespaces) + doc["premise"] = general_detokenize(doc["premise"]).strip() + doc["hypothesis"] = general_detokenize(doc["hypothesis"]).strip() + # Remove last punctuation mark in the premise + doc["premise"] = ( + doc["premise"][:-1] + if doc["premise"].endswith((".", ",", "!", "?")) + else doc["premise"] + ) + # Lowercase the first letter in the hypothesis + doc["hypothesis"] = lowercase_first_letter(doc["hypothesis"]) + # Ensure that the hypothesis ends with a dot + doc["hypothesis"] = ( + (doc["hypothesis"] + ".") + if not doc["hypothesis"].endswith(".") + else doc["hypothesis"] + ) + return doc + + return dataset.map(process_fn) + + +def process_xlsum(dataset): + def _process_doc(doc): + # Remove double spaces + doc["text"] = re.sub(r" +", " ", doc["text"]) + doc["summary"] = re.sub(r" +", " ", doc["summary"]) + return doc + + return dataset.map(_process_doc) + + +def process_docs_paraphrases(dataset): + empty_docs = [] + + def _process_doc(doc): + if doc["sentence1"] not in [None, ""] and doc["sentence2"] not in [None, ""]: + doc["sentence1"] = general_detokenize(doc["sentence1"]).strip() + doc["sentence2"] = general_detokenize(doc["sentence2"]).strip() + # Remove final punctuation mark in the first sentence + if doc["sentence1"].endswith((".", ",", ";")): + doc["sentence1"] = doc["sentence1"][:-1] + # Start the second sentence in lowercase (to be used after "Yes, ...") + doc["sentence2"] = lowercase_first_letter(doc["sentence2"]) + return doc + else: + empty_docs.append(doc) + return doc + + if empty_docs != []: + len_empty_docs = len(empty_docs) + print( + f"Found {len_empty_docs} empty documents out of the {len(dataset)} total docs in the dataset: {empty_docs}" + ) + return dataset.filter( + lambda doc: doc["sentence1"] not in [None, ""] + and doc["sentence2"] not in [None, ""] + ).map(_process_doc) + + +def process_docs_copa_es(dataset): + def _process_doc(doc): + doc["choice1"] = lowercase_first_letter(doc["choice1"]) + doc["choice2"] = lowercase_first_letter(doc["choice2"]) + return doc + + return dataset.map(_process_doc) + + +def rouge1(items): + """ + # passthrough for efficiency + """ + return items + + +def rouge1_agg(items): + """ + Higher is better + """ + refs = list(zip(*items))[0] + preds = list(zip(*items))[1] + rouge_scorer = evaluate.load("rouge") + # import code; code.interact(local=dict(globals(), **locals())) + return rouge_scorer.compute(predictions=preds, references=refs)["rouge1"] diff --git a/lm_eval/tasks/spanish_bench/wnli_es.yaml b/lm_eval/tasks/spanish_bench/wnli_es.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7e9c72dafddd1351f9fafb3ecc60f608ebda4eff --- /dev/null +++ b/lm_eval/tasks/spanish_bench/wnli_es.yaml @@ -0,0 +1,14 @@ +task: wnli_es +dataset_path: PlanTL-GOB-ES/wnli-es +dataset_name: null +output_type: multiple_choice +training_split: train +validation_split: validation +test_split: null +doc_to_text: "{{sentence1}}\nPregunta: {{sentence2}} ¿Verdadero o Falso?\nRespuesta:" +doc_to_target: label +doc_to_choice: ["Falso", "Verdadero"] +metric_list: + - metric: acc +metadata: + version: 1.0 diff --git a/lm_eval/tasks/spanish_bench/xlsum_es.yaml b/lm_eval/tasks/spanish_bench/xlsum_es.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c9a0354efeec772f05bb94fb2e869ada9ceb017a --- /dev/null +++ b/lm_eval/tasks/spanish_bench/xlsum_es.yaml @@ -0,0 +1,22 @@ +task: xlsum_es +dataset_path: csebuetnlp/xlsum +dataset_name: spanish +doc_to_text: 'Texto: {{text}} + + Resumen:' +doc_to_target: '{{summary}}' +output_type: generate_until +test_split: test +training_split: train +validation_split: validation +fewshot_split: train +process_docs: !function utils.process_xlsum +metric_list: + - metric: bleu + aggregation: bleu + higher_is_better: true + - metric: !function utils.rouge1 + aggregation: !function utils.rouge1_agg + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/spanish_bench/xnli_es_spanish_bench.yaml b/lm_eval/tasks/spanish_bench/xnli_es_spanish_bench.yaml new file mode 100644 index 0000000000000000000000000000000000000000..beb898bde1079efd4702faa62d2813b7d5ae2053 --- /dev/null +++ b/lm_eval/tasks/spanish_bench/xnli_es_spanish_bench.yaml @@ -0,0 +1,19 @@ +# Task configuration derived from Eleuther AI's implementation as of March 22, 2024, supplemented with an additional preprocessing function +task: xnli_es_spanish_bench +dataset_path: xnli +dataset_name: es +output_type: multiple_choice +doc_to_choice: '{{[premise+", ¿correcto? Sí, "+hypothesis,premise+", ¿correcto? Así + que, "+hypothesis,premise+", ¿correcto? No, "+hypothesis]}}' +doc_to_text: '' +target_delimiter: '' +process_docs: !function utils.process_doc_nli +training_split: null +validation_split: validation +doc_to_target: label +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/squad_completion/task.py b/lm_eval/tasks/squad_completion/task.py index c2ecef19aa3ac8d7b5cb720655ab012296cf1e2c..7603a9868453875cf2605d08f22e67026f3fb101 100644 --- a/lm_eval/tasks/squad_completion/task.py +++ b/lm_eval/tasks/squad_completion/task.py @@ -12,7 +12,7 @@ class SQUADCompletion(ConfigurableTask): DATASET_PATH = "hazyresearch/based-squad" DATASET_NAME = "default" - def __init__(self): + def __init__(self, **kwargs): super().__init__(config={"metadata": {"version": self.VERSION}}) def has_training_docs(self): diff --git a/lm_eval/tasks/squadv2/task.py b/lm_eval/tasks/squadv2/task.py index 4c3d7f6fa92f619765a42aaa253fc9477274b81f..184a5978e74d25327f0cca4677fecb93b2510309 100644 --- a/lm_eval/tasks/squadv2/task.py +++ b/lm_eval/tasks/squadv2/task.py @@ -37,7 +37,9 @@ _CITATION = """ def _squad_metric(predictions, references): - squad_metric = datasets.load_metric("squad_v2") + import evaluate + + squad_metric = evaluate.load("squad_v2") return squad_metric.compute(predictions=predictions, references=references) @@ -52,7 +54,7 @@ class SQuAD2(ConfigurableTask): DATASET_PATH = "squad_v2" DATASET_NAME = None - def __init__(self): + def __init__(self, config=None): super().__init__(config={"metadata": {"version": self.VERSION}}) # HF changed squad on us so we have to make sure we aren't running the old one diff --git a/lm_eval/tasks/storycloze/storycloze_2016.yaml b/lm_eval/tasks/storycloze/storycloze_2016.yaml index df1c2629cbb15e070bd7e6954d5b528ccc8f030d..c513c68a9cc4b3670cd906ad1093b62b9becdd6f 100644 --- a/lm_eval/tasks/storycloze/storycloze_2016.yaml +++ b/lm_eval/tasks/storycloze/storycloze_2016.yaml @@ -1,7 +1,7 @@ -group: storycloze +tag: storycloze task: storycloze_2016 dataset_path: story_cloze -dataset_name: 2016 +dataset_name: "2016" output_type: multiple_choice validation_split: validation test_split: test diff --git a/lm_eval/tasks/storycloze/storycloze_2018.yaml b/lm_eval/tasks/storycloze/storycloze_2018.yaml index aa630efc73c866366175588a378beb28568d884d..de21aa433568118735f8f75957cad46d9a48dfa0 100644 --- a/lm_eval/tasks/storycloze/storycloze_2018.yaml +++ b/lm_eval/tasks/storycloze/storycloze_2018.yaml @@ -1,7 +1,7 @@ -group: storycloze +tag: storycloze task: storycloze_2018 dataset_path: story_cloze -dataset_name: 2018 +dataset_name: "2018" output_type: multiple_choice validation_split: validation test_split: test diff --git a/lm_eval/tasks/super_glue/README.md b/lm_eval/tasks/super_glue/README.md index c8e807718af5abcec3cbb0ac91af2aab6cb4a3fc..868b3a931d7c1c1d5658baccfe7f9e77e8afaf4a 100644 --- a/lm_eval/tasks/super_glue/README.md +++ b/lm_eval/tasks/super_glue/README.md @@ -26,10 +26,14 @@ Homepage: https://super.gluebenchmark.com/ } ``` -### Groups and Tasks +### Groups, Tags, and Tasks #### Groups +None. + +#### Tags + * `super-glue-lm-eval-v1`: SuperGLUE eval adapted from LM Eval V1 * `super-glue-t5-prompt`: SuperGLUE prompt and evaluation that matches the T5 paper (if using accelerate, will error if record is included.) diff --git a/lm_eval/tasks/super_glue/boolq/default.yaml b/lm_eval/tasks/super_glue/boolq/default.yaml index f26e4682c40ff7c7ba1183fecaadb5718206dbfd..1123761dfcf6ac0388266fa16dc98bfc751a9595 100644 --- a/lm_eval/tasks/super_glue/boolq/default.yaml +++ b/lm_eval/tasks/super_glue/boolq/default.yaml @@ -1,4 +1,4 @@ -group: +tag: - super-glue-lm-eval-v1 task: boolq dataset_path: super_glue diff --git a/lm_eval/tasks/super_glue/boolq/seq2seq.yaml b/lm_eval/tasks/super_glue/boolq/seq2seq.yaml index 569316cb31b909755ba6916dea4e54f80fc95df1..08211ec11f400b40d749e22302dffb013c4b0b6c 100644 --- a/lm_eval/tasks/super_glue/boolq/seq2seq.yaml +++ b/lm_eval/tasks/super_glue/boolq/seq2seq.yaml @@ -1,4 +1,4 @@ -group: +tag: - super-glue-lm-eval-v1-seq2seq task: "boolq-seq2seq" dataset_path: super_glue diff --git a/lm_eval/tasks/super_glue/boolq/t5-prompt.yaml b/lm_eval/tasks/super_glue/boolq/t5-prompt.yaml index 7089381ad86c05913b111d1888878b721a33a222..b3ceb78d912193db6e400cfe9a3b565eca8c16d5 100644 --- a/lm_eval/tasks/super_glue/boolq/t5-prompt.yaml +++ b/lm_eval/tasks/super_glue/boolq/t5-prompt.yaml @@ -1,4 +1,4 @@ -group: +tag: - super-glue-t5-prompt task: super_glue-boolq-t5-prompt dataset_path: super_glue diff --git a/lm_eval/tasks/super_glue/cb/aggregate.py b/lm_eval/tasks/super_glue/cb/aggregate.py index 4b99849f9bfa8307006879666ecf971b17b511b2..d8c1af3580eea906fed64990a317e9cd7766db15 100644 --- a/lm_eval/tasks/super_glue/cb/aggregate.py +++ b/lm_eval/tasks/super_glue/cb/aggregate.py @@ -1,13 +1,14 @@ import numpy as np -import sklearn def cb_multi_fi(items): + from sklearn.metrics import f1_score + preds, golds = zip(*items) preds = np.array(preds) golds = np.array(golds) - f11 = sklearn.metrics.f1_score(y_true=golds == 0, y_pred=preds == 0) - f12 = sklearn.metrics.f1_score(y_true=golds == 1, y_pred=preds == 1) - f13 = sklearn.metrics.f1_score(y_true=golds == 2, y_pred=preds == 2) + f11 = f1_score(y_true=golds == 0, y_pred=preds == 0) + f12 = f1_score(y_true=golds == 1, y_pred=preds == 1) + f13 = f1_score(y_true=golds == 2, y_pred=preds == 2) avg_f1 = np.mean([f11, f12, f13]) return avg_f1 diff --git a/lm_eval/tasks/super_glue/cb/default.yaml b/lm_eval/tasks/super_glue/cb/default.yaml index c575e9872aa712eff69f779a7114d5baed487706..550635ed78bc87b32f8f1a55167faeff5ebddeb2 100644 --- a/lm_eval/tasks/super_glue/cb/default.yaml +++ b/lm_eval/tasks/super_glue/cb/default.yaml @@ -1,4 +1,4 @@ -group: +tag: - super-glue-lm-eval-v1 task: cb dataset_path: super_glue diff --git a/lm_eval/tasks/super_glue/cb/t5-prompt.yaml b/lm_eval/tasks/super_glue/cb/t5-prompt.yaml index 984e17935ad2479fb9d48dabfeb14f14269da2db..8b6f512ca2a246a5b208a616ab6e0df2fc30c5b7 100644 --- a/lm_eval/tasks/super_glue/cb/t5-prompt.yaml +++ b/lm_eval/tasks/super_glue/cb/t5-prompt.yaml @@ -1,4 +1,4 @@ -group: +tag: - super-glue-t5-prompt task: super_glue-cb-t5-prompt dataset_path: super_glue diff --git a/lm_eval/tasks/super_glue/cb/t5_utils.py b/lm_eval/tasks/super_glue/cb/t5_utils.py index ec02e34538e15f71861f354b437060da5390544e..33cbaddf43988a4b7253a647b59885bf91437a23 100644 --- a/lm_eval/tasks/super_glue/cb/t5_utils.py +++ b/lm_eval/tasks/super_glue/cb/t5_utils.py @@ -1,6 +1,3 @@ -import sklearn.metrics - - def mean_3class_f1(predictions, references): # This is a passthrough function string_label = ["entailment", "contradiction", "neutral"] predictions = ( @@ -23,6 +20,8 @@ def agg_mean_3class_f1(items): } def _fn(predictions, references): + import sklearn.metrics + metric_fn = getattr(sklearn.metrics, metric_str) metric_val = metric_fn(references, predictions, **metric_fn_kwargs) return metric_val diff --git a/lm_eval/tasks/super_glue/copa/default.yaml b/lm_eval/tasks/super_glue/copa/default.yaml index 1af5dbf47258e203e7a1b506e7ba6e91351a61e4..9e2d1a2f05bc0b9615e27115881b6eadd5e6eb38 100644 --- a/lm_eval/tasks/super_glue/copa/default.yaml +++ b/lm_eval/tasks/super_glue/copa/default.yaml @@ -1,4 +1,4 @@ -group: +tag: - super-glue-lm-eval-v1 task: copa dataset_path: super_glue diff --git a/lm_eval/tasks/super_glue/copa/t5-prompt.yaml b/lm_eval/tasks/super_glue/copa/t5-prompt.yaml index 20a90db98d28a78307b7e46b99834eaf98cc3f9e..7358b9086c6fc09fa661543155ee100f8147c170 100644 --- a/lm_eval/tasks/super_glue/copa/t5-prompt.yaml +++ b/lm_eval/tasks/super_glue/copa/t5-prompt.yaml @@ -1,4 +1,4 @@ -group: +tag: - super-glue-t5-prompt task: super_glue-copa-t5-prompt dataset_path: super_glue diff --git a/lm_eval/tasks/super_glue/multirc/default.yaml b/lm_eval/tasks/super_glue/multirc/default.yaml index 5a388299f6496673a3edc9c5047fddd1a14302e4..c14f7040d54a7e49854fbcb92e0ce06fc37ffbdd 100644 --- a/lm_eval/tasks/super_glue/multirc/default.yaml +++ b/lm_eval/tasks/super_glue/multirc/default.yaml @@ -1,4 +1,4 @@ -group: +tag: - super-glue-lm-eval-v1 task: multirc dataset_path: super_glue diff --git a/lm_eval/tasks/super_glue/multirc/t5-prompt.yaml b/lm_eval/tasks/super_glue/multirc/t5-prompt.yaml index 927a357158abf96502f955470fcd8afbe0eee49c..566a65ccf9bcac696622b456ef92b9577593d3f7 100644 --- a/lm_eval/tasks/super_glue/multirc/t5-prompt.yaml +++ b/lm_eval/tasks/super_glue/multirc/t5-prompt.yaml @@ -1,4 +1,4 @@ -group: +tag: - super-glue-t5-prompt task: super_glue-multirc-t5-prompt dataset_path: super_glue diff --git a/lm_eval/tasks/super_glue/multirc/t5_utils.py b/lm_eval/tasks/super_glue/multirc/t5_utils.py index d17d498fa25db9a6d7f56e03c43c9e661d66f9f1..04f3652b2193bc562ca4a9a067bd803f4f6bdce1 100644 --- a/lm_eval/tasks/super_glue/multirc/t5_utils.py +++ b/lm_eval/tasks/super_glue/multirc/t5_utils.py @@ -1,7 +1,6 @@ import collections import numpy as np -import sklearn.metrics def f1(predictions, references): # This is a passthrough function @@ -19,10 +18,12 @@ def f1(predictions, references): # This is a passthrough function def agg_f1(items): + from sklearn.metrics import f1_score + predictions, references = zip(*items) references, predictions = np.asarray(references), np.asarray(predictions) - return sklearn.metrics.f1_score(references, predictions) + return f1_score(references, predictions) def em(predictions, references): # This is a passthrough function diff --git a/lm_eval/tasks/super_glue/record/default.yaml b/lm_eval/tasks/super_glue/record/default.yaml index ca978fd2ab4db0661ac12185169bc9b8517d1fe8..9dc924fc798cdb2eeba74c9bdb6ec56f47c6a650 100644 --- a/lm_eval/tasks/super_glue/record/default.yaml +++ b/lm_eval/tasks/super_glue/record/default.yaml @@ -1,4 +1,4 @@ -group: +tag: - super-glue-lm-eval-v1 task: record dataset_path: super_glue diff --git a/lm_eval/tasks/super_glue/record/t5-prompt.yaml b/lm_eval/tasks/super_glue/record/t5-prompt.yaml index c999bc90301ecc92ec36292a9544733a370b5e69..f565171fc22e52c0e5c54b04a2b33f88c21659e9 100644 --- a/lm_eval/tasks/super_glue/record/t5-prompt.yaml +++ b/lm_eval/tasks/super_glue/record/t5-prompt.yaml @@ -1,4 +1,4 @@ -group: +tag: - super-glue-t5-prompt task: super_glue-record-t5-prompt dataset_path: super_glue diff --git a/lm_eval/tasks/super_glue/rte/default.yaml b/lm_eval/tasks/super_glue/rte/default.yaml index 6754af1a1e5688110ab9853e1d53e833ef02dd29..b660f36dd557e406002394c56defce3c032470ec 100644 --- a/lm_eval/tasks/super_glue/rte/default.yaml +++ b/lm_eval/tasks/super_glue/rte/default.yaml @@ -1,4 +1,4 @@ -group: +tag: - super-glue-lm-eval-v1 task: sglue_rte dataset_path: super_glue diff --git a/lm_eval/tasks/super_glue/rte/t5-prompt.yaml b/lm_eval/tasks/super_glue/rte/t5-prompt.yaml index 9e80686e2a36cbe2a3851ba18fe12130894b7ad7..27caab0dde4e42db1d0e9298ea6c0ecf6af21303 100644 --- a/lm_eval/tasks/super_glue/rte/t5-prompt.yaml +++ b/lm_eval/tasks/super_glue/rte/t5-prompt.yaml @@ -1,4 +1,4 @@ -group: +tag: - super-glue-t5-prompt task: super_glue-rte-t5-prompt dataset_path: super_glue diff --git a/lm_eval/tasks/super_glue/wic/default.yaml b/lm_eval/tasks/super_glue/wic/default.yaml index 0f86855a7811ca1e2c11f61201237f8d10ed524c..4bb8ed59a018496d391d85a9e1ac7b4bcc5fca49 100644 --- a/lm_eval/tasks/super_glue/wic/default.yaml +++ b/lm_eval/tasks/super_glue/wic/default.yaml @@ -1,4 +1,4 @@ -group: +tag: - super-glue-lm-eval-v1 task: "wic" dataset_path: super_glue diff --git a/lm_eval/tasks/super_glue/wic/t5-prompt.yaml b/lm_eval/tasks/super_glue/wic/t5-prompt.yaml index 3a0dbb2f7fd64f2ec3ae3e6d58c4dd7e0963edc2..b642a70be0fb83649fe551b2be5b8f62a43a2346 100644 --- a/lm_eval/tasks/super_glue/wic/t5-prompt.yaml +++ b/lm_eval/tasks/super_glue/wic/t5-prompt.yaml @@ -1,4 +1,4 @@ -group: +tag: - super-glue-t5-prompt task: super_glue-wic-t5-prompt dataset_path: super_glue diff --git a/lm_eval/tasks/super_glue/wsc/default.yaml b/lm_eval/tasks/super_glue/wsc/default.yaml index b9c7ec347c2beccb8fdc54ada1082a763c9cfe0d..2cd12679c020f217b39e2c4e4fb6a7a2d7a537df 100644 --- a/lm_eval/tasks/super_glue/wsc/default.yaml +++ b/lm_eval/tasks/super_glue/wsc/default.yaml @@ -1,4 +1,4 @@ -group: +tag: - super-glue-lm-eval-v1 task: wsc dataset_path: super_glue diff --git a/lm_eval/tasks/super_glue/wsc/t5-prompt.yaml b/lm_eval/tasks/super_glue/wsc/t5-prompt.yaml index 6030d1faf210da7b9aab301d059a74978a411a1f..77bfe7d0da7b2206d70a43771e60577c338dd73d 100644 --- a/lm_eval/tasks/super_glue/wsc/t5-prompt.yaml +++ b/lm_eval/tasks/super_glue/wsc/t5-prompt.yaml @@ -1,4 +1,4 @@ -group: +tag: - super-glue-t5-prompt task: super_glue-wsc-t5-prompt dataset_path: super_glue diff --git a/lm_eval/tasks/swde/task.py b/lm_eval/tasks/swde/task.py index 413911621380e65a9345961e71406d1a7a9afcee..7226364b799e30e0a58fc35ef4a9e00970a22c91 100644 --- a/lm_eval/tasks/swde/task.py +++ b/lm_eval/tasks/swde/task.py @@ -12,7 +12,7 @@ class SWDE(ConfigurableTask): DATASET_PATH = "hazyresearch/based-swde-v2" DATASET_NAME = "default" - def __init__(self): + def __init__(self, **kwargs): super().__init__(config={"metadata": {"version": self.VERSION}}) def has_training_docs(self): diff --git a/lm_eval/tasks/tinyBenchmarks/utils_truthfulqa.py b/lm_eval/tasks/tinyBenchmarks/utils_truthfulqa.py index 2f7d92d1b7f0434bbe00c7bcbe636115732182c2..e404d096a2adec920bcf5d0f654a9f8789585a3f 100644 --- a/lm_eval/tasks/tinyBenchmarks/utils_truthfulqa.py +++ b/lm_eval/tasks/tinyBenchmarks/utils_truthfulqa.py @@ -6,6 +6,8 @@ from rouge_score import rouge_scorer, scoring """ This code mirrors the utils of the original truthful_qa task """ +ROUGE_SCORER = None + def process_results_mc2(doc, results): lls, is_greedy = zip(*results) @@ -153,7 +155,11 @@ def rouge(refs, preds): A `list` of predicted `strs`. """ rouge_types = ["rouge1", "rouge2", "rougeLsum"] - scorer = rouge_scorer.RougeScorer(rouge_types) + global ROUGE_SCORER + if ROUGE_SCORER is None: + # init RougeScorer once (https://github.com/EleutherAI/lm-evaluation-harness/issues/1692)--rouge_types are constant + ROUGE_SCORER = rouge_scorer.RougeScorer(rouge_types) + scorer = ROUGE_SCORER # Add newlines between sentences to correctly compute `rougeLsum`. def _prepare_summary(summary): diff --git a/lm_eval/tasks/tmlu/README.md b/lm_eval/tasks/tmlu/README.md new file mode 100644 index 0000000000000000000000000000000000000000..6ce4d93663bbd949cc26ef6b4140cd7dfff471b2 --- /dev/null +++ b/lm_eval/tasks/tmlu/README.md @@ -0,0 +1,59 @@ +# TMLU + +### Paper + +Title: `Measuring Taiwanese Mandarin Language Understanding` + +Abstract: `The evaluation of large language models (LLMs) has drawn substantial attention in the field recently. This work focuses on evaluating LLMs in a Chinese context, specifically, for Traditional Chinese which has been largely underrepresented in existing benchmarks. We present TMLU, a holistic evaluation suit tailored for assessing the advanced knowledge and reasoning capability in LLMs, under the context of Taiwanese Mandarin. TMLU consists of an array of 37 subjects across social science, STEM, humanities, Taiwan-specific content, and others, ranging from middle school to professional levels. In addition, we curate chain-of-thought-like few-shot explanations for each subject to facilitate the evaluation of complex reasoning skills. To establish a comprehensive baseline, we conduct extensive experiments and analysis on 24 advanced LLMs. The results suggest that Chinese open-weight models demonstrate inferior performance comparing to multilingual proprietary ones, and open-weight models tailored for Taiwanese Mandarin lag behind the Simplified-Chinese counterparts. The findings indicate great headrooms for improvement, and emphasize the goal of TMLU to foster the development of localized Taiwanese-Mandarin LLMs. We release the benchmark and evaluation scripts for the community to promote future research.` + + +Homepage: [TMLU Huggingface Dataset](https://huggingface.co/datasets/miulab/tmlu) + + +### Citation + +``` +@article{DBLP:journals/corr/abs-2403-20180, + author = {Po{-}Heng Chen and + Sijia Cheng and + Wei{-}Lin Chen and + Yen{-}Ting Lin and + Yun{-}Nung Chen}, + title = {Measuring Taiwanese Mandarin Language Understanding}, + journal = {CoRR}, + volume = {abs/2403.20180}, + year = {2024}, + url = {https://doi.org/10.48550/arXiv.2403.20180}, + doi = {10.48550/ARXIV.2403.20180}, + eprinttype = {arXiv}, + eprint = {2403.20180}, + timestamp = {Wed, 10 Apr 2024 17:37:45 +0200}, + biburl = {https://dblp.org/rec/journals/corr/abs-2403-20180.bib}, + bibsource = {dblp computer science bibliography, https://dblp.org} +} +``` + +### Groups and Tasks + +#### Groups + +* `tmlu`: `The dataset comprises 2,981 multiple-choice questions from 37 subjects. ` + +#### Tasks + +The following tasks evaluate subjects in the TMLU dataset using loglikelihood-based multiple-choice scoring: + +* `tmlu_{subject_english}` + +### Checklist + +For adding novel benchmarks/datasets to the library: +* [x] Is the task an existing benchmark in the literature? + * [x] Have you referenced the original paper that introduced the task? + * [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + + +If other tasks on this dataset are already supported: +* [x] Is the "Main" variant of this task clearly denoted? +* [x] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [x] Have you noted which, if any, published evaluation setups are matched by this variant? diff --git a/lm_eval/tasks/tmlu/default/_default_template_yaml b/lm_eval/tasks/tmlu/default/_default_template_yaml new file mode 100644 index 0000000000000000000000000000000000000000..aad508cda823191d25e393607a81d5a76c24e9f9 --- /dev/null +++ b/lm_eval/tasks/tmlu/default/_default_template_yaml @@ -0,0 +1,16 @@ +dataset_path: miulab/tmlu +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +process_docs: !function utils.process_docs +# doc_to_text: "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:" +# doc_to_choice: ["A", "B", "C", "D"] +doc_to_target: answer +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true +metadata: + version: 0.1 diff --git a/lm_eval/tasks/tmlu/default/_generate_configs.py b/lm_eval/tasks/tmlu/default/_generate_configs.py new file mode 100644 index 0000000000000000000000000000000000000000..86b176085dc76366db3f6745d21e99a3a40b1b0c --- /dev/null +++ b/lm_eval/tasks/tmlu/default/_generate_configs.py @@ -0,0 +1,198 @@ +""" +Take in a YAML, and output all "other" splits with this YAML +""" + +import argparse +import os + +import pandas as pd +import yaml +from tqdm import tqdm + + +categories = { + "STEM": [ + "biology", + "chemistry", + "mathematics" "physics", + "earth science", + ], + "humanities": ["Chinese", "history", "Tour", "law"], + "social_sciences": [ + "civics", + "geography", + "accounting", + "psychologist", + ], + "Taiwan Specific": [ + "Taiwan Specific", + ], + "other": ["Medicine", "Nutritionist"], # (business, health, misc.) +} + +task_list = [ + "AST civics", + "AST geography", + "CAP civics", + "CAP geography", + "GSAT civics", + "GSAT geography", + "MOEX Accountant", + "MOEX Clinical psychologist", + "AST biology", + "AST chemistry", + "AST mathematics", + "AST physics", + "CAP biology", + "CAP chemistry", + "CAP earth science", + "CAP mathematics", + "CAP physics", + "GSAT biology", + "GSAT chemistry", + "GSAT earth science", + "GSAT mathematics", + "GSAT physics", + "AST Chinese", + "AST history", + "CAP Chinese", + "CAP history", + "GSAT Chinese", + "GSAT history", + "MOEX Tour guide", + "MOEX Tour leader", + "MOEX Lawyer qualification", + "HB Driving Rule", + "MOEX Teacher qualification", + "MOEX Taiwan tourist resources", + "MOEX Basic Traditional Chinese Medicine", + "MOEX Clinical Traditional Chinese Medicine", + "MOEX Nutritionist", +] +subject2name = {} +subject2num_choice = {} +# subject2category = {} +SUBJECTS = {} + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--base_yaml_path", default="_default_template_yaml") + parser.add_argument("--save_prefix_path", default="tmlu") + parser.add_argument("--cot_prompt_path", default=None) + parser.add_argument("--task_prefix", default="") + parser.add_argument("--group_prefix", default="") + parser.add_argument("--subject_file", default="../subject.tsv") + return parser.parse_args() + + +if __name__ == "__main__": + args = parse_args() + from pathlib import Path + + # Initialization + SUBJECT_FILE = Path(__file__).parent / Path(args.subject_file) + + df = pd.read_csv(SUBJECT_FILE, delimiter="\t") + + for _, row in df.iterrows(): + for _c in categories: + if row["subject"] in SUBJECTS: + raise ValueError(f"Duplicate tasks. {row['subject']} already exists.") + if row["category"] in categories[_c]: # append new item into SUBJECTS + SUBJECTS[row["subject"]] = _c + subject2name[row["subject"]] = row["name"] + subject2num_choice[row["subject"]] = row["# Choices"] + break + # End of SUBJECTS initialization + + # get filename of base_yaml so we can `"include": ` it in our "other" YAMLs. + base_yaml_name = os.path.split(args.base_yaml_path)[-1] + with open(args.base_yaml_path) as f: + base_yaml = yaml.full_load(f) + + if args.cot_prompt_path is not None: + import json + + with open(args.cot_prompt_path) as f: + cot_file = json.load(f) + + ALL_CATEGORIES = [] + for subject, category in tqdm(SUBJECTS.items()): + if category not in ALL_CATEGORIES: + ALL_CATEGORIES.append(category) + + if args.cot_prompt_path is not None: + description = cot_file[subject] + else: + name_of_subject = subject2name[subject].replace("＿", " ") + description = f"以下為{name_of_subject}的單選題，請提供正確答案的選項。\n\n" + # description = f"The following are multiple choice questions (with answers) about {' '.join(subject.split('_'))}.\n\n" + + num_choies = subject2num_choice[subject] + # basic_doc_to_text = "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}" + basic_doc_to_choice = ["A", "B", "C", "D"] + if num_choies == 5: + # basic_doc_to_text += "\nE. {{choices[4]}}" + basic_doc_to_choice.append("E") + if num_choies == 6: + # basic_doc_to_text += "\nE. {{choices[4]}}\nF. {{choices[5]}}" + basic_doc_to_choice += ["E", "F"] + # basic_doc_to_text += "\nAnswer:" + # basic_doc_to_text = "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}{% if choices[4] %}\nE. {{choices[4]}}{% endif %}{% if choices[5] %}\nF. {{choices[5]}}{% endif %}\nAnswer:" + basic_doc_to_text = "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}{% if choices is defined and choices|length > 4 %}\nE. {{choices[4]}}{% endif %}{% if choices is defined and choices|length > 5 %}\nF. {{choices[5]}}{% endif %}\nAnswer:" + + yaml_dict = { + "include": base_yaml_name, + "group": f"tmlu_{args.task_prefix}_{category}" + if args.task_prefix != "" + else f"tmlu_{category}", + "group_alias": category.replace("_", " "), + "task": f"tmlu_{args.task_prefix}_{subject}" + if args.task_prefix != "" + else f"tmlu_{subject}", + "task_alias": subject.replace("_", " "), + "dataset_name": subject, + "description": description, + # doc_to_text: "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\nD. {{choices[3]}}\nAnswer:" + "doc_to_text": basic_doc_to_text, + # doc_to_choice: ["A", "B", "C", "D"] + "doc_to_choice": basic_doc_to_choice, + } + + file_save_path = args.save_prefix_path + f"_{subject}.yaml" + # eval_logger.info(f"Saving yaml for subset {subject} to {file_save_path}") + with open(file_save_path, "w") as yaml_file: + yaml.dump( + yaml_dict, + yaml_file, + # width=float("inf"), + allow_unicode=True, + default_style='"', + ) + + if args.task_prefix != "": + mmlu_subcategories = [ + f"tmlu_{args.task_prefix}_{category}" for category in ALL_CATEGORIES + ] + else: + mmlu_subcategories = [f"tmlu_{category}" for category in ALL_CATEGORIES] + + if args.group_prefix != "": + file_save_path = args.group_prefix + ".yaml" + else: + file_save_path = args.save_prefix_path + ".yaml" + + # eval_logger.info(f"Saving benchmark config to {file_save_path}") + with open(file_save_path, "w") as yaml_file: + yaml.dump( + { + "group": f"tmlu_{args.task_prefix}" + if args.task_prefix != "" + else "tmlu", + "task": mmlu_subcategories, + }, + yaml_file, + indent=4, + default_flow_style=False, + ) diff --git a/lm_eval/tasks/tmlu/default/_tmlu.yaml b/lm_eval/tasks/tmlu/default/_tmlu.yaml new file mode 100644 index 0000000000000000000000000000000000000000..08344c85ce6795162d589e4f93beffa8d9f79d8b --- /dev/null +++ b/lm_eval/tasks/tmlu/default/_tmlu.yaml @@ -0,0 +1,37 @@ +group: tmlu +group_alias: TMLU +task: + - group: tmlu_social_sciences + group_alias: Social Sciences + task: + - tmlu_social_sciences_tasks + aggregate_metric_list: + - metric: acc + - group: tmlu_stem + group_alias: STEM + task: + - tmlu_stem_tasks + aggregate_metric_list: + - metric: acc + - group: tmlu_humanities + group_alias: Humanities + task: + - tmlu_humanities_tasks + aggregate_metric_list: + - metric: acc + - group: tmlu_taiwan_specific + group_alias: Taiwan Specific + task: + - tmlu_taiwan_specific_tasks + aggregate_metric_list: + - metric: acc + - group: tmlu_other + group_alias: Other + task: + - tmlu_other_tasks + aggregate_metric_list: + - metric: acc +aggregate_metric_list: + - metric: acc +metadata: + version: 1 diff --git a/lm_eval/tasks/tmlu/default/tmlu_AST_biology.yaml b/lm_eval/tasks/tmlu/default/tmlu_AST_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..68a7f4a34214db3707a1bed8d835b97ad742e8a0 --- /dev/null +++ b/lm_eval/tasks/tmlu/default/tmlu_AST_biology.yaml @@ -0,0 +1,15 @@ +"dataset_name": "AST_biology" +"description": "以下為分科測驗生物的單選題，請提供正確答案的選項。\n\n" +"doc_to_choice": +- "A" +- "B" +- "C" +- "D" +"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\n\ + D. {{choices[3]}}{% if choices is defined and choices|length > 4 %}\nE. {{choices[4]}}{%\ + \ endif %}{% if choices is defined and choices|length > 5 %}\nF. {{choices[5]}}{%\ + \ endif %}\nAnswer:" +"tag": "tmlu_stem_tasks" +"include": "_default_template_yaml" +"task": "tmlu_AST_biology" +"task_alias": "AST biology" diff --git a/lm_eval/tasks/tmlu/default/tmlu_AST_chemistry.yaml b/lm_eval/tasks/tmlu/default/tmlu_AST_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2a9f229183893b144e359c354099d8cfe3e6b375 --- /dev/null +++ b/lm_eval/tasks/tmlu/default/tmlu_AST_chemistry.yaml @@ -0,0 +1,16 @@ +"dataset_name": "AST_chemistry" +"description": "以下為分科測驗化學的單選題，請提供正確答案的選項。\n\n" +"doc_to_choice": +- "A" +- "B" +- "C" +- "D" +- "E" +"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\n\ + D. {{choices[3]}}{% if choices is defined and choices|length > 4 %}\nE. {{choices[4]}}{%\ + \ endif %}{% if choices is defined and choices|length > 5 %}\nF. {{choices[5]}}{%\ + \ endif %}\nAnswer:" +"tag": "tmlu_stem_tasks" +"include": "_default_template_yaml" +"task": "tmlu_AST_chemistry" +"task_alias": "AST chemistry" diff --git a/lm_eval/tasks/tmlu/default/tmlu_AST_chinese.yaml b/lm_eval/tasks/tmlu/default/tmlu_AST_chinese.yaml new file mode 100644 index 0000000000000000000000000000000000000000..216c87122b899177876e20d89fc9b4f1959bec53 --- /dev/null +++ b/lm_eval/tasks/tmlu/default/tmlu_AST_chinese.yaml @@ -0,0 +1,15 @@ +"dataset_name": "AST_chinese" +"description": "以下為分科測驗國文的單選題，請提供正確答案的選項。\n\n" +"doc_to_choice": +- "A" +- "B" +- "C" +- "D" +"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\n\ + D. {{choices[3]}}{% if choices is defined and choices|length > 4 %}\nE. {{choices[4]}}{%\ + \ endif %}{% if choices is defined and choices|length > 5 %}\nF. {{choices[5]}}{%\ + \ endif %}\nAnswer:" +"tag": "tmlu_humanities_tasks" +"include": "_default_template_yaml" +"task": "tmlu_AST_chinese" +"task_alias": "AST chinese" diff --git a/lm_eval/tasks/tmlu/default/tmlu_AST_civics.yaml b/lm_eval/tasks/tmlu/default/tmlu_AST_civics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b7fe538f617164471469018beca8ff087d5c82be --- /dev/null +++ b/lm_eval/tasks/tmlu/default/tmlu_AST_civics.yaml @@ -0,0 +1,15 @@ +"dataset_name": "AST_civics" +"description": "以下為分科測驗公民的單選題，請提供正確答案的選項。\n\n" +"doc_to_choice": +- "A" +- "B" +- "C" +- "D" +"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\n\ + D. {{choices[3]}}{% if choices is defined and choices|length > 4 %}\nE. {{choices[4]}}{%\ + \ endif %}{% if choices is defined and choices|length > 5 %}\nF. {{choices[5]}}{%\ + \ endif %}\nAnswer:" +"tag": "tmlu_social_sciences_tasks" +"include": "_default_template_yaml" +"task": "tmlu_AST_civics" +"task_alias": "AST civics" diff --git a/lm_eval/tasks/tmlu/default/tmlu_AST_geography.yaml b/lm_eval/tasks/tmlu/default/tmlu_AST_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..921765f022935b0f4d73bbc8fcc2a32fac79ef79 --- /dev/null +++ b/lm_eval/tasks/tmlu/default/tmlu_AST_geography.yaml @@ -0,0 +1,15 @@ +"dataset_name": "AST_geography" +"description": "以下為分科測驗地理的單選題，請提供正確答案的選項。\n\n" +"doc_to_choice": +- "A" +- "B" +- "C" +- "D" +"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\n\ + D. {{choices[3]}}{% if choices is defined and choices|length > 4 %}\nE. {{choices[4]}}{%\ + \ endif %}{% if choices is defined and choices|length > 5 %}\nF. {{choices[5]}}{%\ + \ endif %}\nAnswer:" +"tag": "tmlu_social_sciences_tasks" +"include": "_default_template_yaml" +"task": "tmlu_AST_geography" +"task_alias": "AST geography" diff --git a/lm_eval/tasks/tmlu/default/tmlu_AST_history.yaml b/lm_eval/tasks/tmlu/default/tmlu_AST_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..483f46bd39a8a6e839869d2fb0d90b90a98df4d8 --- /dev/null +++ b/lm_eval/tasks/tmlu/default/tmlu_AST_history.yaml @@ -0,0 +1,15 @@ +"dataset_name": "AST_history" +"description": "以下為分科測驗歷史的單選題，請提供正確答案的選項。\n\n" +"doc_to_choice": +- "A" +- "B" +- "C" +- "D" +"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\n\ + D. {{choices[3]}}{% if choices is defined and choices|length > 4 %}\nE. {{choices[4]}}{%\ + \ endif %}{% if choices is defined and choices|length > 5 %}\nF. {{choices[5]}}{%\ + \ endif %}\nAnswer:" +"tag": "tmlu_humanities_tasks" +"include": "_default_template_yaml" +"task": "tmlu_AST_history" +"task_alias": "AST history" diff --git a/lm_eval/tasks/tmlu/default/tmlu_CAP_biology.yaml b/lm_eval/tasks/tmlu/default/tmlu_CAP_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..98454938b6231dcdf2cc2e43b23c07a26792634c --- /dev/null +++ b/lm_eval/tasks/tmlu/default/tmlu_CAP_biology.yaml @@ -0,0 +1,15 @@ +"dataset_name": "CAP_biology" +"description": "以下為會考生物的單選題，請提供正確答案的選項。\n\n" +"doc_to_choice": +- "A" +- "B" +- "C" +- "D" +"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\n\ + D. {{choices[3]}}{% if choices is defined and choices|length > 4 %}\nE. {{choices[4]}}{%\ + \ endif %}{% if choices is defined and choices|length > 5 %}\nF. {{choices[5]}}{%\ + \ endif %}\nAnswer:" +"tag": "tmlu_stem_tasks" +"include": "_default_template_yaml" +"task": "tmlu_CAP_biology" +"task_alias": "CAP biology" diff --git a/lm_eval/tasks/tmlu/default/tmlu_CAP_chemistry.yaml b/lm_eval/tasks/tmlu/default/tmlu_CAP_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8ac15d8434592e231956f5c565d2137d73a7163d --- /dev/null +++ b/lm_eval/tasks/tmlu/default/tmlu_CAP_chemistry.yaml @@ -0,0 +1,15 @@ +"dataset_name": "CAP_chemistry" +"description": "以下為會考化學的單選題，請提供正確答案的選項。\n\n" +"doc_to_choice": +- "A" +- "B" +- "C" +- "D" +"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\n\ + D. {{choices[3]}}{% if choices is defined and choices|length > 4 %}\nE. {{choices[4]}}{%\ + \ endif %}{% if choices is defined and choices|length > 5 %}\nF. {{choices[5]}}{%\ + \ endif %}\nAnswer:" +"tag": "tmlu_stem_tasks" +"include": "_default_template_yaml" +"task": "tmlu_CAP_chemistry" +"task_alias": "CAP chemistry" diff --git a/lm_eval/tasks/tmlu/default/tmlu_CAP_chinese.yaml b/lm_eval/tasks/tmlu/default/tmlu_CAP_chinese.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b71b479caa03f5bea04ad0d4c0a7255203fc0d2d --- /dev/null +++ b/lm_eval/tasks/tmlu/default/tmlu_CAP_chinese.yaml @@ -0,0 +1,15 @@ +"dataset_name": "CAP_chinese" +"description": "以下為會考國文的單選題，請提供正確答案的選項。\n\n" +"doc_to_choice": +- "A" +- "B" +- "C" +- "D" +"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\n\ + D. {{choices[3]}}{% if choices is defined and choices|length > 4 %}\nE. {{choices[4]}}{%\ + \ endif %}{% if choices is defined and choices|length > 5 %}\nF. {{choices[5]}}{%\ + \ endif %}\nAnswer:" +"tag": "tmlu_humanities_tasks" +"include": "_default_template_yaml" +"task": "tmlu_CAP_chinese" +"task_alias": "CAP chinese" diff --git a/lm_eval/tasks/tmlu/default/tmlu_CAP_civics.yaml b/lm_eval/tasks/tmlu/default/tmlu_CAP_civics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..bdf1ea3b2608cd99004e7a809e0827a77deab606 --- /dev/null +++ b/lm_eval/tasks/tmlu/default/tmlu_CAP_civics.yaml @@ -0,0 +1,15 @@ +"dataset_name": "CAP_civics" +"description": "以下為會考公民的單選題，請提供正確答案的選項。\n\n" +"doc_to_choice": +- "A" +- "B" +- "C" +- "D" +"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\n\ + D. {{choices[3]}}{% if choices is defined and choices|length > 4 %}\nE. {{choices[4]}}{%\ + \ endif %}{% if choices is defined and choices|length > 5 %}\nF. {{choices[5]}}{%\ + \ endif %}\nAnswer:" +"tag": "tmlu_social_sciences_tasks" +"include": "_default_template_yaml" +"task": "tmlu_CAP_civics" +"task_alias": "CAP civics" diff --git a/lm_eval/tasks/tmlu/default/tmlu_CAP_earth_science.yaml b/lm_eval/tasks/tmlu/default/tmlu_CAP_earth_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..16c4349619844070006676df1fba1dc9db4fe990 --- /dev/null +++ b/lm_eval/tasks/tmlu/default/tmlu_CAP_earth_science.yaml @@ -0,0 +1,15 @@ +"dataset_name": "CAP_earth_science" +"description": "以下為會考地球科學的單選題，請提供正確答案的選項。\n\n" +"doc_to_choice": +- "A" +- "B" +- "C" +- "D" +"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\n\ + D. {{choices[3]}}{% if choices is defined and choices|length > 4 %}\nE. {{choices[4]}}{%\ + \ endif %}{% if choices is defined and choices|length > 5 %}\nF. {{choices[5]}}{%\ + \ endif %}\nAnswer:" +"tag": "tmlu_stem_tasks" +"include": "_default_template_yaml" +"task": "tmlu_CAP_earth_science" +"task_alias": "CAP earth science" diff --git a/lm_eval/tasks/tmlu/default/tmlu_CAP_geography.yaml b/lm_eval/tasks/tmlu/default/tmlu_CAP_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..82f52c2834598642e1a138102327d23e3bc0ed5b --- /dev/null +++ b/lm_eval/tasks/tmlu/default/tmlu_CAP_geography.yaml @@ -0,0 +1,15 @@ +"dataset_name": "CAP_geography" +"description": "以下為會考地理的單選題，請提供正確答案的選項。\n\n" +"doc_to_choice": +- "A" +- "B" +- "C" +- "D" +"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\n\ + D. {{choices[3]}}{% if choices is defined and choices|length > 4 %}\nE. {{choices[4]}}{%\ + \ endif %}{% if choices is defined and choices|length > 5 %}\nF. {{choices[5]}}{%\ + \ endif %}\nAnswer:" +"tag": "tmlu_social_sciences_tasks" +"include": "_default_template_yaml" +"task": "tmlu_CAP_geography" +"task_alias": "CAP geography" diff --git a/lm_eval/tasks/tmlu/default/tmlu_CAP_history.yaml b/lm_eval/tasks/tmlu/default/tmlu_CAP_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d0ce5fc377f19c3cc1c45427b66fb470429dd537 --- /dev/null +++ b/lm_eval/tasks/tmlu/default/tmlu_CAP_history.yaml @@ -0,0 +1,15 @@ +"dataset_name": "CAP_history" +"description": "以下為會考歷史的單選題，請提供正確答案的選項。\n\n" +"doc_to_choice": +- "A" +- "B" +- "C" +- "D" +"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\n\ + D. {{choices[3]}}{% if choices is defined and choices|length > 4 %}\nE. {{choices[4]}}{%\ + \ endif %}{% if choices is defined and choices|length > 5 %}\nF. {{choices[5]}}{%\ + \ endif %}\nAnswer:" +"tag": "tmlu_humanities_tasks" +"include": "_default_template_yaml" +"task": "tmlu_CAP_history" +"task_alias": "CAP history" diff --git a/lm_eval/tasks/tmlu/default/tmlu_GSAT_biology.yaml b/lm_eval/tasks/tmlu/default/tmlu_GSAT_biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2835e23a6dd4c295c0c60fbad0ad9a30411f356b --- /dev/null +++ b/lm_eval/tasks/tmlu/default/tmlu_GSAT_biology.yaml @@ -0,0 +1,16 @@ +"dataset_name": "GSAT_biology" +"description": "以下為學測生物的單選題，請提供正確答案的選項。\n\n" +"doc_to_choice": +- "A" +- "B" +- "C" +- "D" +- "E" +"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\n\ + D. {{choices[3]}}{% if choices is defined and choices|length > 4 %}\nE. {{choices[4]}}{%\ + \ endif %}{% if choices is defined and choices|length > 5 %}\nF. {{choices[5]}}{%\ + \ endif %}\nAnswer:" +"tag": "tmlu_stem_tasks" +"include": "_default_template_yaml" +"task": "tmlu_GSAT_biology" +"task_alias": "GSAT biology" diff --git a/lm_eval/tasks/tmlu/default/tmlu_GSAT_chemistry.yaml b/lm_eval/tasks/tmlu/default/tmlu_GSAT_chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6baad6da672e9910a4a8dc638903c87fdbf6176a --- /dev/null +++ b/lm_eval/tasks/tmlu/default/tmlu_GSAT_chemistry.yaml @@ -0,0 +1,16 @@ +"dataset_name": "GSAT_chemistry" +"description": "以下為學測化學的單選題，請提供正確答案的選項。\n\n" +"doc_to_choice": +- "A" +- "B" +- "C" +- "D" +- "E" +"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\n\ + D. {{choices[3]}}{% if choices is defined and choices|length > 4 %}\nE. {{choices[4]}}{%\ + \ endif %}{% if choices is defined and choices|length > 5 %}\nF. {{choices[5]}}{%\ + \ endif %}\nAnswer:" +"tag": "tmlu_stem_tasks" +"include": "_default_template_yaml" +"task": "tmlu_GSAT_chemistry" +"task_alias": "GSAT chemistry" diff --git a/lm_eval/tasks/tmlu/default/tmlu_GSAT_chinese.yaml b/lm_eval/tasks/tmlu/default/tmlu_GSAT_chinese.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f1cd7000a41a7e73465475ce639b957fa029a6c2 --- /dev/null +++ b/lm_eval/tasks/tmlu/default/tmlu_GSAT_chinese.yaml @@ -0,0 +1,15 @@ +"dataset_name": "GSAT_chinese" +"description": "以下為學測國文的單選題，請提供正確答案的選項。\n\n" +"doc_to_choice": +- "A" +- "B" +- "C" +- "D" +"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\n\ + D. {{choices[3]}}{% if choices is defined and choices|length > 4 %}\nE. {{choices[4]}}{%\ + \ endif %}{% if choices is defined and choices|length > 5 %}\nF. {{choices[5]}}{%\ + \ endif %}\nAnswer:" +"tag": "tmlu_humanities_tasks" +"include": "_default_template_yaml" +"task": "tmlu_GSAT_chinese" +"task_alias": "GSAT chinese" diff --git a/lm_eval/tasks/tmlu/default/tmlu_GSAT_civics.yaml b/lm_eval/tasks/tmlu/default/tmlu_GSAT_civics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..347c4f13c56dd99c0257eff684bdbfffe5cdb86c --- /dev/null +++ b/lm_eval/tasks/tmlu/default/tmlu_GSAT_civics.yaml @@ -0,0 +1,15 @@ +"dataset_name": "GSAT_civics" +"description": "以下為學測公民的單選題，請提供正確答案的選項。\n\n" +"doc_to_choice": +- "A" +- "B" +- "C" +- "D" +"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\n\ + D. {{choices[3]}}{% if choices is defined and choices|length > 4 %}\nE. {{choices[4]}}{%\ + \ endif %}{% if choices is defined and choices|length > 5 %}\nF. {{choices[5]}}{%\ + \ endif %}\nAnswer:" +"tag": "tmlu_social_sciences_tasks" +"include": "_default_template_yaml" +"task": "tmlu_GSAT_civics" +"task_alias": "GSAT civics" diff --git a/lm_eval/tasks/tmlu/default/tmlu_GSAT_earth_science.yaml b/lm_eval/tasks/tmlu/default/tmlu_GSAT_earth_science.yaml new file mode 100644 index 0000000000000000000000000000000000000000..de0db88544a2ef990b9978cfc61e8fa32e50e61d --- /dev/null +++ b/lm_eval/tasks/tmlu/default/tmlu_GSAT_earth_science.yaml @@ -0,0 +1,16 @@ +"dataset_name": "GSAT_earth_science" +"description": "以下為學測地球科學的單選題，請提供正確答案的選項。\n\n" +"doc_to_choice": +- "A" +- "B" +- "C" +- "D" +- "E" +"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\n\ + D. {{choices[3]}}{% if choices is defined and choices|length > 4 %}\nE. {{choices[4]}}{%\ + \ endif %}{% if choices is defined and choices|length > 5 %}\nF. {{choices[5]}}{%\ + \ endif %}\nAnswer:" +"tag": "tmlu_stem_tasks" +"include": "_default_template_yaml" +"task": "tmlu_GSAT_earth_science" +"task_alias": "GSAT earth science" diff --git a/lm_eval/tasks/tmlu/default/tmlu_GSAT_geography.yaml b/lm_eval/tasks/tmlu/default/tmlu_GSAT_geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..752fc9033931efc5ce7618832d9065c43be7a220 --- /dev/null +++ b/lm_eval/tasks/tmlu/default/tmlu_GSAT_geography.yaml @@ -0,0 +1,15 @@ +"dataset_name": "GSAT_geography" +"description": "以下為學測地理的單選題，請提供正確答案的選項。\n\n" +"doc_to_choice": +- "A" +- "B" +- "C" +- "D" +"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\n\ + D. {{choices[3]}}{% if choices is defined and choices|length > 4 %}\nE. {{choices[4]}}{%\ + \ endif %}{% if choices is defined and choices|length > 5 %}\nF. {{choices[5]}}{%\ + \ endif %}\nAnswer:" +"tag": "tmlu_social_sciences_tasks" +"include": "_default_template_yaml" +"task": "tmlu_GSAT_geography" +"task_alias": "GSAT geography" diff --git a/lm_eval/tasks/tmlu/default/tmlu_GSAT_history.yaml b/lm_eval/tasks/tmlu/default/tmlu_GSAT_history.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e2e2547367db598d148a9772e68f32a05fa49e03 --- /dev/null +++ b/lm_eval/tasks/tmlu/default/tmlu_GSAT_history.yaml @@ -0,0 +1,15 @@ +"dataset_name": "GSAT_history" +"description": "以下為學測歷史的單選題，請提供正確答案的選項。\n\n" +"doc_to_choice": +- "A" +- "B" +- "C" +- "D" +"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\n\ + D. {{choices[3]}}{% if choices is defined and choices|length > 4 %}\nE. {{choices[4]}}{%\ + \ endif %}{% if choices is defined and choices|length > 5 %}\nF. {{choices[5]}}{%\ + \ endif %}\nAnswer:" +"tag": "tmlu_humanities_tasks" +"include": "_default_template_yaml" +"task": "tmlu_GSAT_history" +"task_alias": "GSAT history" diff --git a/lm_eval/tasks/tmlu/default/tmlu_accountant.yaml b/lm_eval/tasks/tmlu/default/tmlu_accountant.yaml new file mode 100644 index 0000000000000000000000000000000000000000..61aa6c8cba4fb755071ef241329dfe5ca86d8483 --- /dev/null +++ b/lm_eval/tasks/tmlu/default/tmlu_accountant.yaml @@ -0,0 +1,15 @@ +"dataset_name": "accountant" +"description": "以下為會計師的單選題，請提供正確答案的選項。\n\n" +"doc_to_choice": +- "A" +- "B" +- "C" +- "D" +"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\n\ + D. {{choices[3]}}{% if choices is defined and choices|length > 4 %}\nE. {{choices[4]}}{%\ + \ endif %}{% if choices is defined and choices|length > 5 %}\nF. {{choices[5]}}{%\ + \ endif %}\nAnswer:" +"tag": "tmlu_social_sciences_tasks" +"include": "_default_template_yaml" +"task": "tmlu_accountant" +"task_alias": "accountant" diff --git a/lm_eval/tasks/tmlu/default/tmlu_basic_traditional_chinese_medicine.yaml b/lm_eval/tasks/tmlu/default/tmlu_basic_traditional_chinese_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..adefcf84948f2a128abbe17a33a4946dbc49b598 --- /dev/null +++ b/lm_eval/tasks/tmlu/default/tmlu_basic_traditional_chinese_medicine.yaml @@ -0,0 +1,15 @@ +"dataset_name": "basic_traditional_chinese_medicine" +"description": "以下為中醫基礎醫學的單選題，請提供正確答案的選項。\n\n" +"doc_to_choice": +- "A" +- "B" +- "C" +- "D" +"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\n\ + D. {{choices[3]}}{% if choices is defined and choices|length > 4 %}\nE. {{choices[4]}}{%\ + \ endif %}{% if choices is defined and choices|length > 5 %}\nF. {{choices[5]}}{%\ + \ endif %}\nAnswer:" +"tag": "tmlu_other_tasks" +"include": "_default_template_yaml" +"task": "tmlu_basic_traditional_chinese_medicine" +"task_alias": "basic traditional chinese medicine" diff --git a/lm_eval/tasks/tmlu/default/tmlu_clinical_psychologist.yaml b/lm_eval/tasks/tmlu/default/tmlu_clinical_psychologist.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1106b9eb4bb1e4d1ce4531c058f52ca7d1e57557 --- /dev/null +++ b/lm_eval/tasks/tmlu/default/tmlu_clinical_psychologist.yaml @@ -0,0 +1,15 @@ +"dataset_name": "clinical_psychologist" +"description": "以下為臨床心理師的單選題，請提供正確答案的選項。\n\n" +"doc_to_choice": +- "A" +- "B" +- "C" +- "D" +"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\n\ + D. {{choices[3]}}{% if choices is defined and choices|length > 4 %}\nE. {{choices[4]}}{%\ + \ endif %}{% if choices is defined and choices|length > 5 %}\nF. {{choices[5]}}{%\ + \ endif %}\nAnswer:" +"tag": "tmlu_social_sciences_tasks" +"include": "_default_template_yaml" +"task": "tmlu_clinical_psychologist" +"task_alias": "clinical psychologist" diff --git a/lm_eval/tasks/tmlu/default/tmlu_clinical_traditional_chinese_medicine.yaml b/lm_eval/tasks/tmlu/default/tmlu_clinical_traditional_chinese_medicine.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9207f40b660d6a48f1ffad78aee320f8644b4977 --- /dev/null +++ b/lm_eval/tasks/tmlu/default/tmlu_clinical_traditional_chinese_medicine.yaml @@ -0,0 +1,15 @@ +"dataset_name": "clinical_traditional_chinese_medicine" +"description": "以下為中醫針灸的單選題，請提供正確答案的選項。\n\n" +"doc_to_choice": +- "A" +- "B" +- "C" +- "D" +"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\n\ + D. {{choices[3]}}{% if choices is defined and choices|length > 4 %}\nE. {{choices[4]}}{%\ + \ endif %}{% if choices is defined and choices|length > 5 %}\nF. {{choices[5]}}{%\ + \ endif %}\nAnswer:" +"tag": "tmlu_other_tasks" +"include": "_default_template_yaml" +"task": "tmlu_clinical_traditional_chinese_medicine" +"task_alias": "clinical traditional chinese medicine" diff --git a/lm_eval/tasks/tmlu/default/tmlu_driving_rule.yaml b/lm_eval/tasks/tmlu/default/tmlu_driving_rule.yaml new file mode 100644 index 0000000000000000000000000000000000000000..965084c8d5d3b1904b724d80665c1f19084c73c2 --- /dev/null +++ b/lm_eval/tasks/tmlu/default/tmlu_driving_rule.yaml @@ -0,0 +1,15 @@ +"dataset_name": "driving_rule" +"description": "以下為台灣駕駛規則的單選題，請提供正確答案的選項。\n\n" +"doc_to_choice": +- "A" +- "B" +- "C" +- "D" +"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\n\ + D. {{choices[3]}}{% if choices is defined and choices|length > 4 %}\nE. {{choices[4]}}{%\ + \ endif %}{% if choices is defined and choices|length > 5 %}\nF. {{choices[5]}}{%\ + \ endif %}\nAnswer:" +"tag": "tmlu_taiwan_specific" +"include": "_default_template_yaml" +"task": "tmlu_driving_rule" +"task_alias": "driving rule" diff --git a/lm_eval/tasks/tmlu/default/tmlu_lawyer_qualification.yaml b/lm_eval/tasks/tmlu/default/tmlu_lawyer_qualification.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0926ebcd08270af62bd8a062b895774b376def6c --- /dev/null +++ b/lm_eval/tasks/tmlu/default/tmlu_lawyer_qualification.yaml @@ -0,0 +1,15 @@ +"dataset_name": "lawyer_qualification" +"description": "以下為律師資格的單選題，請提供正確答案的選項。\n\n" +"doc_to_choice": +- "A" +- "B" +- "C" +- "D" +"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\n\ + D. {{choices[3]}}{% if choices is defined and choices|length > 4 %}\nE. {{choices[4]}}{%\ + \ endif %}{% if choices is defined and choices|length > 5 %}\nF. {{choices[5]}}{%\ + \ endif %}\nAnswer:" +"tag": "tmlu_humanities_tasks" +"include": "_default_template_yaml" +"task": "tmlu_lawyer_qualification" +"task_alias": "lawyer qualification" diff --git a/lm_eval/tasks/tmlu/default/tmlu_nutritionist.yaml b/lm_eval/tasks/tmlu/default/tmlu_nutritionist.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ca0a08fc11799e5608ff3c951dd491123e9a734b --- /dev/null +++ b/lm_eval/tasks/tmlu/default/tmlu_nutritionist.yaml @@ -0,0 +1,15 @@ +"dataset_name": "nutritionist" +"description": "以下為營養師的單選題，請提供正確答案的選項。\n\n" +"doc_to_choice": +- "A" +- "B" +- "C" +- "D" +"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\n\ + D. {{choices[3]}}{% if choices is defined and choices|length > 4 %}\nE. {{choices[4]}}{%\ + \ endif %}{% if choices is defined and choices|length > 5 %}\nF. {{choices[5]}}{%\ + \ endif %}\nAnswer:" +"tag": "tmlu_other_tasks" +"include": "_default_template_yaml" +"task": "tmlu_nutritionist" +"task_alias": "nutritionist" diff --git a/lm_eval/tasks/tmlu/default/tmlu_taiwan_tourist_resources.yaml b/lm_eval/tasks/tmlu/default/tmlu_taiwan_tourist_resources.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6a1fc7b26ace1e95c2e1df92c26cc41c12d4632e --- /dev/null +++ b/lm_eval/tasks/tmlu/default/tmlu_taiwan_tourist_resources.yaml @@ -0,0 +1,15 @@ +"dataset_name": "taiwan_tourist_resources" +"description": "以下為台灣觀光資源的單選題，請提供正確答案的選項。\n\n" +"doc_to_choice": +- "A" +- "B" +- "C" +- "D" +"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\n\ + D. {{choices[3]}}{% if choices is defined and choices|length > 4 %}\nE. {{choices[4]}}{%\ + \ endif %}{% if choices is defined and choices|length > 5 %}\nF. {{choices[5]}}{%\ + \ endif %}\nAnswer:" +"tag": "tmlu_taiwan_specific" +"include": "_default_template_yaml" +"task": "tmlu_taiwan_tourist_resources" +"task_alias": "taiwan tourist resources" diff --git a/lm_eval/tasks/tmlu/default/tmlu_teacher_qualification.yaml b/lm_eval/tasks/tmlu/default/tmlu_teacher_qualification.yaml new file mode 100644 index 0000000000000000000000000000000000000000..987c2d7d92199355c1158111391a24d983353881 --- /dev/null +++ b/lm_eval/tasks/tmlu/default/tmlu_teacher_qualification.yaml @@ -0,0 +1,15 @@ +"dataset_name": "teacher_qualification" +"description": "以下為教師資格的單選題，請提供正確答案的選項。\n\n" +"doc_to_choice": +- "A" +- "B" +- "C" +- "D" +"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\n\ + D. {{choices[3]}}{% if choices is defined and choices|length > 4 %}\nE. {{choices[4]}}{%\ + \ endif %}{% if choices is defined and choices|length > 5 %}\nF. {{choices[5]}}{%\ + \ endif %}\nAnswer:" +"tag": "tmlu_taiwan_specific" +"include": "_default_template_yaml" +"task": "tmlu_teacher_qualification" +"task_alias": "teacher qualification" diff --git a/lm_eval/tasks/tmlu/default/tmlu_tour_guide.yaml b/lm_eval/tasks/tmlu/default/tmlu_tour_guide.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c3a759ca53fd6024852688c12dc123e587e55ff6 --- /dev/null +++ b/lm_eval/tasks/tmlu/default/tmlu_tour_guide.yaml @@ -0,0 +1,15 @@ +"dataset_name": "tour_guide" +"description": "以下為導遊的單選題，請提供正確答案的選項。\n\n" +"doc_to_choice": +- "A" +- "B" +- "C" +- "D" +"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\n\ + D. {{choices[3]}}{% if choices is defined and choices|length > 4 %}\nE. {{choices[4]}}{%\ + \ endif %}{% if choices is defined and choices|length > 5 %}\nF. {{choices[5]}}{%\ + \ endif %}\nAnswer:" +"tag": "tmlu_humanities_tasks" +"include": "_default_template_yaml" +"task": "tmlu_tour_guide" +"task_alias": "tour guide" diff --git a/lm_eval/tasks/tmlu/default/tmlu_tour_leader.yaml b/lm_eval/tasks/tmlu/default/tmlu_tour_leader.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d8a607f656db47647cd4e35338bbf8f78af72240 --- /dev/null +++ b/lm_eval/tasks/tmlu/default/tmlu_tour_leader.yaml @@ -0,0 +1,15 @@ +"dataset_name": "tour_leader" +"description": "以下為領隊的單選題，請提供正確答案的選項。\n\n" +"doc_to_choice": +- "A" +- "B" +- "C" +- "D" +"doc_to_text": "{{question.strip()}}\nA. {{choices[0]}}\nB. {{choices[1]}}\nC. {{choices[2]}}\n\ + D. {{choices[3]}}{% if choices is defined and choices|length > 4 %}\nE. {{choices[4]}}{%\ + \ endif %}{% if choices is defined and choices|length > 5 %}\nF. {{choices[5]}}{%\ + \ endif %}\nAnswer:" +"tag": "tmlu_humanities_tasks" +"include": "_default_template_yaml" +"task": "tmlu_tour_leader" +"task_alias": "tour leader" diff --git a/lm_eval/tasks/tmlu/default/utils.py b/lm_eval/tasks/tmlu/default/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..8bdfd6db5ad9f6870d784309a338f52bf3bec0e6 --- /dev/null +++ b/lm_eval/tasks/tmlu/default/utils.py @@ -0,0 +1,23 @@ +import datasets + + +def process_docs(dataset: datasets.Dataset) -> datasets.Dataset: + def _helper(doc): + # modifies the contents of a single + # document in our dataset. + answer_list = ["A", "B", "C", "D"] + choices = [doc["A"], doc["B"], doc["C"], doc["D"]] + if doc.get("E", None): + answer_list.append("E") + choices.append(doc["E"]) + if doc.get("F", None): + answer_list.append("F") + choices.append(doc["F"]) + out_doc = { + "questions": doc["question"], + "choices": choices, + "goal": answer_list.index(doc["answer"]), + } + return out_doc + + return dataset.map(_helper) # returns back a datasets.Dataset object diff --git a/lm_eval/tasks/tmlu/subject.tsv b/lm_eval/tasks/tmlu/subject.tsv new file mode 100644 index 0000000000000000000000000000000000000000..17c39cfb898c56ebbb1246e35c924c2d192da0e7 --- /dev/null +++ b/lm_eval/tasks/tmlu/subject.tsv @@ -0,0 +1,38 @@ +category subject name # Questions # Choices +civics AST_civics 分科測驗公民 57 4 +geography AST_geography 分科測驗地理 58 4 +civics CAP_civics 會考公民 73 4 +geography CAP_geography 會考地理 45 4 +civics GSAT_civics 學測公民 73 4 +geography GSAT_geography 學測地理 49 4 +accounting accountant 會計師 117 4 +psychologist clinical_psychologist 臨床心理師 117 4 +biology AST_biology 分科測驗生物 40 4 +chemistry AST_chemistry 分科測驗化學 34 5 +mathematics AST_mathematics 分科測驗數學 25 5 +physics AST_physics 分科測驗物理 43 5 +biology CAP_biology 會考生物 27 4 +chemistry CAP_chemistry 會考化學 27 4 +earth science CAP_earth_science 會考地球科學 15 4 +mathematics CAP_mathematics 會考數學 115 4 +physics CAP_physics 會考物理 15 4 +biology GSAT_biology 學測生物 21 5 +chemistry GSAT_chemistry 學測化學 29 5 +earth science GSAT_earth_science 學測地球科學 24 5 +mathematics GSAT_mathematics 學測數學 29 5 +physics GSAT_physics 學測物理 24 5 +Chinese AST_chinese 分科測驗國文 131 4 +history AST_history 分科測驗歷史 56 4 +Chinese CAP_chinese 會考國文 61 4 +history CAP_history 會考歷史 56 4 +Chinese GSAT_chinese 學測國文 97 4 +history GSAT_history 學測歷史 85 4 +Tour tour_guide 導遊 99 4 +Tour tour_leader 領隊 145 4 +law lawyer_qualification 律師資格 279 4 +Taiwan Specific driving_rule 台灣駕駛規則 432 4 +Taiwan Specific teacher_qualification 教師資格 75 4 +Taiwan Specific taiwan_tourist_resources 台灣觀光資源 50 4 +Medicine basic_traditional_chinese_medicine 中醫基礎醫學 159 4 +Medicine clinical_traditional_chinese_medicine 中醫針灸 79 4 +Nutritionist nutritionist 營養師 120 4 diff --git a/lm_eval/tasks/tmmluplus/default/_generate_configs.py b/lm_eval/tasks/tmmluplus/default/_generate_configs.py index a3b1271a576100e9f5022066a8ce67bf1b5b21ce..06ef7a710fc1b0a617494594ecbbb6908f235325 100644 --- a/lm_eval/tasks/tmmluplus/default/_generate_configs.py +++ b/lm_eval/tasks/tmmluplus/default/_generate_configs.py @@ -11,7 +11,7 @@ from tqdm import tqdm # Copy from https://github.com/iKala/ievals/blob/main/ievals/settings.py -# from TMMLU+ offical example +# from TMMLU+ official example categories = { "STEM": [ "physics", diff --git a/lm_eval/tasks/tmmluplus/default/_tmmluplus.yaml b/lm_eval/tasks/tmmluplus/default/_tmmluplus.yaml new file mode 100644 index 0000000000000000000000000000000000000000..45208d4dc21992a90222ae00561dfb71ed7e1fff --- /dev/null +++ b/lm_eval/tasks/tmmluplus/default/_tmmluplus.yaml @@ -0,0 +1,13 @@ +group: tmmluplus +task: +- tmmluplus_other +- tmmluplus_social_sciences +- tmmluplus_humanities +- tmmluplus_STEM +aggregate_metric_list: + - metric: acc + weight_by_size: True + - metric: acc_norm + weight_by_size: True +metadata: + version: 2.0 diff --git a/lm_eval/tasks/tmmluplus/default/_tmmluplus_STEM.yaml b/lm_eval/tasks/tmmluplus/default/_tmmluplus_STEM.yaml new file mode 100644 index 0000000000000000000000000000000000000000..47f81f5085e49cffb750fddc396d5835eca59a55 --- /dev/null +++ b/lm_eval/tasks/tmmluplus/default/_tmmluplus_STEM.yaml @@ -0,0 +1,10 @@ +group: tmmluplus_STEM +task: +- tmmluplus_STEM_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: True + - metric: acc_norm + weight_by_size: True +metadata: + version: 2.0 diff --git a/lm_eval/tasks/tmmluplus/default/_tmmluplus_humanities.yaml b/lm_eval/tasks/tmmluplus/default/_tmmluplus_humanities.yaml new file mode 100644 index 0000000000000000000000000000000000000000..1cd42f88dd18bc96190a1054525f5517e4129659 --- /dev/null +++ b/lm_eval/tasks/tmmluplus/default/_tmmluplus_humanities.yaml @@ -0,0 +1,10 @@ +group: tmmluplus_humanities +task: +- tmmluplus_humanities_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: True + - metric: acc_norm + weight_by_size: True +metadata: + version: 2.0 diff --git a/lm_eval/tasks/tmmluplus/default/_tmmluplus_other.yaml b/lm_eval/tasks/tmmluplus/default/_tmmluplus_other.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2b679ef414696383c56540d5749ea55776351447 --- /dev/null +++ b/lm_eval/tasks/tmmluplus/default/_tmmluplus_other.yaml @@ -0,0 +1,10 @@ +group: tmmluplus_other +task: +- tmmluplus_other_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: True + - metric: acc_norm + weight_by_size: True +metadata: + version: 2.0 diff --git a/lm_eval/tasks/tmmluplus/default/_tmmluplus_social_sciences.yaml b/lm_eval/tasks/tmmluplus/default/_tmmluplus_social_sciences.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a219550bbc94b85a3636fca0a62eff298fc4b34e --- /dev/null +++ b/lm_eval/tasks/tmmluplus/default/_tmmluplus_social_sciences.yaml @@ -0,0 +1,10 @@ +group: tmmluplus_social_sciences +task: +- tmmluplus_social_sciences_tasks +aggregate_metric_list: + - metric: acc + weight_by_size: True + - metric: acc_norm + weight_by_size: True +metadata: + version: 2.0 diff --git a/lm_eval/tasks/tmmluplus/default/_default_template_yaml b/lm_eval/tasks/tmmluplus/default/_tmmluplus_template_yaml similarity index 97% rename from lm_eval/tasks/tmmluplus/default/_default_template_yaml rename to lm_eval/tasks/tmmluplus/default/_tmmluplus_template_yaml index 7ece2e2d84cb43f6e1d7403ae83a73be41e164f7..c03cfb1b37289875d64e0879ede1cf3e6fc35cff 100644 --- a/lm_eval/tasks/tmmluplus/default/_default_template_yaml +++ b/lm_eval/tasks/tmmluplus/default/_tmmluplus_template_yaml @@ -16,4 +16,4 @@ metric_list: aggregation: mean higher_is_better: true metadata: - version: 0.1 + version: 2.0 diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus.yaml deleted file mode 100644 index 105cf98aff37b28535e8166ae685e5fac105eaed..0000000000000000000000000000000000000000 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus.yaml +++ /dev/null @@ -1,6 +0,0 @@ -group: tmmluplus -task: -- tmmluplus_other -- tmmluplus_social_sciences -- tmmluplus_humanities -- tmmluplus_STEM diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_accounting.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_accounting.yaml index 9e27c919efd224866e9ceeb38424e0147a454193..c6ee4a50265190a2c0fd8384b13745d429e19cdc 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_accounting.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_accounting.yaml @@ -1,7 +1,6 @@ "dataset_name": "accounting" "description": "以下為會計學的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_accounting" "task_alias": "accounting" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_administrative_law.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_administrative_law.yaml index 454efec64feb6daa0f4bc91f542b23003b2e62d9..369578366023eb46b63946945e36be0211db1321 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_administrative_law.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_administrative_law.yaml @@ -1,7 +1,6 @@ "dataset_name": "administrative_law" "description": "以下為行政法的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_humanities" -"group_alias": "humanities" -"include": "_default_template_yaml" +"tag": "tmmluplus_humanities_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_administrative_law" "task_alias": "administrative law" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_advance_chemistry.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_advance_chemistry.yaml index d5baa64be2e643521a1f486a4618babca2ca4ef6..549b3cc864e865e01ab9e907685658028dcc04a1 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_advance_chemistry.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_advance_chemistry.yaml @@ -1,7 +1,6 @@ "dataset_name": "advance_chemistry" "description": "以下為化學的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_STEM" -"group_alias": "STEM" -"include": "_default_template_yaml" +"tag": "tmmluplus_STEM_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_advance_chemistry" "task_alias": "advance chemistry" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_agriculture.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_agriculture.yaml index 340369c89024e057f9e73945c32655555b666c29..8ef2912690613d4dcd5fdfb5a2dd4869b9bb1405 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_agriculture.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_agriculture.yaml @@ -1,7 +1,6 @@ "dataset_name": "agriculture" "description": "以下為農業的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_agriculture" "task_alias": "agriculture" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_anti_money_laundering.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_anti_money_laundering.yaml index 95bb0e47861f3c37954e74ae9b1fe17095f3eaa7..7f8d873f49b2a7b3cd6e07111a9f9ae63129b44b 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_anti_money_laundering.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_anti_money_laundering.yaml @@ -1,7 +1,6 @@ "dataset_name": "anti_money_laundering" "description": "以下為洗錢防制的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_humanities" -"group_alias": "humanities" -"include": "_default_template_yaml" +"tag": "tmmluplus_humanities_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_anti_money_laundering" "task_alias": "anti money laundering" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_auditing.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_auditing.yaml index a8168029b29291cab1e6f596acd51e00699e3cf2..e5c598b7e36beec102c5672a235afb8f650918f9 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_auditing.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_auditing.yaml @@ -1,7 +1,6 @@ "dataset_name": "auditing" "description": "以下為審計學的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_auditing" "task_alias": "auditing" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_basic_medical_science.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_basic_medical_science.yaml index d329b78a488839aaa46007bc83db187c5c1cd562..6c6db6aab8638b8e15e29d9c28d44b04115fb0a1 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_basic_medical_science.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_basic_medical_science.yaml @@ -1,7 +1,6 @@ "dataset_name": "basic_medical_science" "description": "以下為基礎醫學的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_STEM" -"group_alias": "STEM" -"include": "_default_template_yaml" +"tag": "tmmluplus_STEM_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_basic_medical_science" "task_alias": "basic medical science" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_business_management.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_business_management.yaml index 9cacf04896d941ec705d1c3774952cbb516236f5..e8377ee3f5dc035017284820968d9842a131ed27 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_business_management.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_business_management.yaml @@ -1,7 +1,6 @@ "dataset_name": "business_management" "description": "以下為企業管理的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_business_management" "task_alias": "business management" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_chinese_language_and_literature.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_chinese_language_and_literature.yaml index 73eba7684bff03a432c85228f7c9a764ca2544ad..53a8d652adec28c21c5aa6c23be04d4aaf72302a 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_chinese_language_and_literature.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_chinese_language_and_literature.yaml @@ -1,7 +1,6 @@ "dataset_name": "chinese_language_and_literature" "description": "以下為國文的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_social_sciences" -"group_alias": "social sciences" -"include": "_default_template_yaml" +"tag": "tmmluplus_social_sciences_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_chinese_language_and_literature" "task_alias": "chinese language and literature" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_clinical_psychology.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_clinical_psychology.yaml index f8194feb7dee9c2100f6ecf50b602235d1ac0a2a..7753e2a0961a9d557c6804aa0a1827052cee50c1 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_clinical_psychology.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_clinical_psychology.yaml @@ -1,7 +1,6 @@ "dataset_name": "clinical_psychology" "description": "以下為臨床心理學的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_social_sciences" -"group_alias": "social sciences" -"include": "_default_template_yaml" +"tag": "tmmluplus_social_sciences_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_clinical_psychology" "task_alias": "clinical psychology" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_computer_science.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_computer_science.yaml index c55f6a4a3ae23aaa1fb4a31941f9d5020df892f4..00949ddc4d9f48d617d28650e81f84072109b55b 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_computer_science.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_computer_science.yaml @@ -1,7 +1,6 @@ "dataset_name": "computer_science" "description": "以下為資訊工程的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_STEM" -"group_alias": "STEM" -"include": "_default_template_yaml" +"tag": "tmmluplus_STEM_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_computer_science" "task_alias": "computer science" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_culinary_skills.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_culinary_skills.yaml index 457eac1d18465a434abfd4916acffb8ac7d30529..92f1109829f694d9166892d405f7e9f5548fb678 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_culinary_skills.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_culinary_skills.yaml @@ -1,7 +1,6 @@ "dataset_name": "culinary_skills" "description": "以下為餐旅的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_culinary_skills" "task_alias": "culinary skills" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_dentistry.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_dentistry.yaml index d6295240fc3a37046d0c8d0038eb58130667a807..836a9c129eb4b682b3f11c6779ea353bdd5fcc67 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_dentistry.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_dentistry.yaml @@ -1,7 +1,6 @@ "dataset_name": "dentistry" "description": "以下為牙醫學的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_dentistry" "task_alias": "dentistry" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_economics.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_economics.yaml index 2ed100fb42d428d0afd0c26f560da9700eb30b04..8665c62f8745cbb534af35b6e55d3c9eaa0e331c 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_economics.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_economics.yaml @@ -1,7 +1,6 @@ "dataset_name": "economics" "description": "以下為經濟學的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_social_sciences" -"group_alias": "social sciences" -"include": "_default_template_yaml" +"tag": "tmmluplus_social_sciences_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_economics" "task_alias": "economics" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_education.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_education.yaml index cc601810008841e7d2bca53680b237c2b52c16ff..46f230cdbac9836b4ed61f7348cbcf94b2310c3c 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_education.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_education.yaml @@ -1,7 +1,6 @@ "dataset_name": "education" "description": "以下為教育常識的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_social_sciences" -"group_alias": "social sciences" -"include": "_default_template_yaml" +"tag": "tmmluplus_social_sciences_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_education" "task_alias": "education" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_education_(profession_level).yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_education_(profession_level).yaml index f986517b66c9f46443655b940c251007ba782c50..281654c506644a96d4b70b58f607e7c799c25e1b 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_education_(profession_level).yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_education_(profession_level).yaml @@ -1,7 +1,6 @@ "dataset_name": "education_(profession_level)" "description": "以下為教育專業的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_social_sciences" -"group_alias": "social sciences" -"include": "_default_template_yaml" +"tag": "tmmluplus_social_sciences_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_education_(profession_level)" "task_alias": "education (profession level)" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_educational_psychology.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_educational_psychology.yaml index bed8f2d41be8aa0349ea47a153212d8ffa7e5bb3..be1c2c8a739296482eb7776ab9731c768f502576 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_educational_psychology.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_educational_psychology.yaml @@ -1,7 +1,6 @@ "dataset_name": "educational_psychology" "description": "以下為教育心理的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_social_sciences" -"group_alias": "social sciences" -"include": "_default_template_yaml" +"tag": "tmmluplus_social_sciences_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_educational_psychology" "task_alias": "educational psychology" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_engineering_math.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_engineering_math.yaml index cf681cd57f2e3b2096080c08a6c2e3de2eaa1443..a8a35e1c74533c99a5bc8c45ac92032fe232a875 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_engineering_math.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_engineering_math.yaml @@ -1,7 +1,6 @@ "dataset_name": "engineering_math" "description": "以下為工程數學的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_STEM" -"group_alias": "STEM" -"include": "_default_template_yaml" +"tag": "tmmluplus_STEM_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_engineering_math" "task_alias": "engineering math" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_finance_banking.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_finance_banking.yaml index e60086e12e5e97ec9df7ff5c616df95f92762ed1..465c1d74d695263852d3d1f3493e47c62b77bff9 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_finance_banking.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_finance_banking.yaml @@ -1,7 +1,6 @@ "dataset_name": "finance_banking" "description": "以下為金融與法規的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_finance_banking" "task_alias": "finance banking" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_financial_analysis.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_financial_analysis.yaml index 9990ab5d0447b969c6f5ae026d5db0d388a00b29..647189c668702739a99dbe5ee8af56098b23a05d 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_financial_analysis.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_financial_analysis.yaml @@ -1,7 +1,6 @@ "dataset_name": "financial_analysis" "description": "以下為財務分析的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_financial_analysis" "task_alias": "financial analysis" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_fire_science.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_fire_science.yaml index 807f6f5f2ea23185bf82734e860f650f96ae8984..9b78539c6adadb6c71b8689ff03ea503aad82c08 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_fire_science.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_fire_science.yaml @@ -1,7 +1,6 @@ "dataset_name": "fire_science" "description": "以下為火災學的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_fire_science" "task_alias": "fire science" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_general_principles_of_law.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_general_principles_of_law.yaml index 30b21caeada339782994aeedb9d92d1c77b683c5..8213106bf0d7457ae699582d56239c6f9aabf709 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_general_principles_of_law.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_general_principles_of_law.yaml @@ -1,7 +1,6 @@ "dataset_name": "general_principles_of_law" "description": "以下為法學大意的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_humanities" -"group_alias": "humanities" -"include": "_default_template_yaml" +"tag": "tmmluplus_humanities_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_general_principles_of_law" "task_alias": "general principles of law" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_geography_of_taiwan.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_geography_of_taiwan.yaml index 80ab36b73d77f58ef11f6a6aa047b51d2ca2cad2..ae24c2e108c07a726075ceefdede64bcbf69144e 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_geography_of_taiwan.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_geography_of_taiwan.yaml @@ -1,7 +1,6 @@ "dataset_name": "geography_of_taiwan" "description": "以下為台灣地理的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_social_sciences" -"group_alias": "social sciences" -"include": "_default_template_yaml" +"tag": "tmmluplus_social_sciences_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_geography_of_taiwan" "task_alias": "geography of taiwan" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_human_behavior.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_human_behavior.yaml index 54aaa80fa3b24df4452b3a5c2c75fdb29bb51cdb..bb2fa7231c074535203632a475b504f58778a5fc 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_human_behavior.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_human_behavior.yaml @@ -1,7 +1,6 @@ "dataset_name": "human_behavior" "description": "以下為人類行為與社會的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_social_sciences" -"group_alias": "social sciences" -"include": "_default_template_yaml" +"tag": "tmmluplus_social_sciences_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_human_behavior" "task_alias": "human behavior" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_insurance_studies.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_insurance_studies.yaml index fa23be46c1af606deb01c860a4703d30edda019d..5d1abf7801cb2d58432a845a589cf78b422de750 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_insurance_studies.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_insurance_studies.yaml @@ -1,7 +1,6 @@ "dataset_name": "insurance_studies" "description": "以下為保險學的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_insurance_studies" "task_alias": "insurance studies" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_introduction_to_law.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_introduction_to_law.yaml index 6434d3eae80048a24afc61a9a042d931fa54fa05..ddbd488952c34b42d4afd6d7fc88551a73c65672 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_introduction_to_law.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_introduction_to_law.yaml @@ -1,7 +1,6 @@ "dataset_name": "introduction_to_law" "description": "以下為法律概論的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_humanities" -"group_alias": "humanities" -"include": "_default_template_yaml" +"tag": "tmmluplus_humanities_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_introduction_to_law" "task_alias": "introduction to law" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_jce_humanities.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_jce_humanities.yaml index 2ff3bed0731b042baaaed575011b1c0ea6a26aff..b70bcdd3db414935ed8e33e6db57d26638182bc6 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_jce_humanities.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_jce_humanities.yaml @@ -1,7 +1,6 @@ "dataset_name": "jce_humanities" "description": "以下為指考人文科目的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_humanities" -"group_alias": "humanities" -"include": "_default_template_yaml" +"tag": "tmmluplus_humanities_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_jce_humanities" "task_alias": "jce humanities" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_junior_chemistry.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_junior_chemistry.yaml index de9c0691cf1e5d3c4773f831c77d8056f355277f..3c43b6b638d1fca8f1384b863a51a09bf59f92ab 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_junior_chemistry.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_junior_chemistry.yaml @@ -1,7 +1,6 @@ "dataset_name": "junior_chemistry" "description": "以下為國中理化的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_STEM" -"group_alias": "STEM" -"include": "_default_template_yaml" +"tag": "tmmluplus_STEM_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_junior_chemistry" "task_alias": "junior chemistry" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_junior_chinese_exam.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_junior_chinese_exam.yaml index 937090520ef5e21d5877a46d4c9b1530b56ecba1..ac68698b68dd8668339567f0232b3b72a4cc9816 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_junior_chinese_exam.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_junior_chinese_exam.yaml @@ -1,7 +1,6 @@ "dataset_name": "junior_chinese_exam" "description": "以下為國中會考基測國文的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_social_sciences" -"group_alias": "social sciences" -"include": "_default_template_yaml" +"tag": "tmmluplus_social_sciences_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_junior_chinese_exam" "task_alias": "junior chinese exam" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_junior_math_exam.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_junior_math_exam.yaml index a869a55f1445123550389b44b718df00d4dd2ef5..bc72e5e09664117197e0ef49cb520c24925509c7 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_junior_math_exam.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_junior_math_exam.yaml @@ -1,7 +1,6 @@ "dataset_name": "junior_math_exam" "description": "以下為國中會考基測數學科的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_STEM" -"group_alias": "STEM" -"include": "_default_template_yaml" +"tag": "tmmluplus_STEM_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_junior_math_exam" "task_alias": "junior math exam" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_junior_science_exam.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_junior_science_exam.yaml index ed4b8d272c94356d6f2de8b600304470b75134ea..740e674e6cab498ca4337e08ead2e1c55bd80eb4 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_junior_science_exam.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_junior_science_exam.yaml @@ -1,7 +1,6 @@ "dataset_name": "junior_science_exam" "description": "以下為國中會考基測自然科的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_STEM" -"group_alias": "STEM" -"include": "_default_template_yaml" +"tag": "tmmluplus_STEM_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_junior_science_exam" "task_alias": "junior science exam" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_junior_social_studies.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_junior_social_studies.yaml index 760ff0e794a401489dc7a7ddd3e258f2a707edde..54472b09e70344e88794933ea1f6e30242707ab6 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_junior_social_studies.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_junior_social_studies.yaml @@ -1,7 +1,6 @@ "dataset_name": "junior_social_studies" "description": "以下為國中會考基測社會科的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_junior_social_studies" "task_alias": "junior social studies" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_linear_algebra.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_linear_algebra.yaml index c2286dbca9b66f3320f8db206b0c1dc279b6069e..762bdfc0da1dfa7d388609c0ae8bfcedf0153cfd 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_linear_algebra.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_linear_algebra.yaml @@ -1,7 +1,6 @@ "dataset_name": "linear_algebra" "description": "以下為線代的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_STEM" -"group_alias": "STEM" -"include": "_default_template_yaml" +"tag": "tmmluplus_STEM_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_linear_algebra" "task_alias": "linear algebra" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_logic_reasoning.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_logic_reasoning.yaml index 059bef3d9687e0f38383f6984c12aedfdfbc5b00..58fdc83f0294d00ceb4e67bad0171f444dbb2622 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_logic_reasoning.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_logic_reasoning.yaml @@ -1,7 +1,6 @@ "dataset_name": "logic_reasoning" "description": "以下為邏輯思維的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_logic_reasoning" "task_alias": "logic reasoning" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_macroeconomics.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_macroeconomics.yaml index 91009abe691ffcc0c910729244620557ccad2d6c..81984475d3d026a6cbfa26d3a1053166ba84849a 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_macroeconomics.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_macroeconomics.yaml @@ -1,7 +1,6 @@ "dataset_name": "macroeconomics" "description": "以下為總經的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_social_sciences" -"group_alias": "social sciences" -"include": "_default_template_yaml" +"tag": "tmmluplus_social_sciences_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_macroeconomics" "task_alias": "macroeconomics" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_management_accounting.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_management_accounting.yaml index 2d071f869736fbe10cfbc400d74371d2f27a8ffc..fd77179d652fdfb15ec7ac6c0d4a56cd5f18ac1d 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_management_accounting.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_management_accounting.yaml @@ -1,7 +1,6 @@ "dataset_name": "management_accounting" "description": "以下為管理會計的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_management_accounting" "task_alias": "management accounting" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_marketing_management.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_marketing_management.yaml index da39f0a879b33956012c8f2fefba88586a9c4b4d..f903c4b6f0c2d8a32dd220b83c9b651964cbab0c 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_marketing_management.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_marketing_management.yaml @@ -1,7 +1,6 @@ "dataset_name": "marketing_management" "description": "以下為行銷管理的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_marketing_management" "task_alias": "marketing management" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_mechanical.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_mechanical.yaml index 81ea0dce68b2a7d0be1733fd94fc37c997bf894f..5300cc1d9164a733501c9b2bc2fa5454e335b1bc 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_mechanical.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_mechanical.yaml @@ -1,7 +1,6 @@ "dataset_name": "mechanical" "description": "以下為機械與機電概論的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_mechanical" "task_alias": "mechanical" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_music.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_music.yaml index 72864c0035da8cb92b491773be7a8a5e8a3b1685..b0e62badf688275692ace02e19c33352069210e7 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_music.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_music.yaml @@ -1,7 +1,6 @@ "dataset_name": "music" "description": "以下為音樂科的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_music" "task_alias": "music" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_national_protection.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_national_protection.yaml index 62e98266d83a247c9e56f119316780dedac1369e..4ae2953b1f1e5b29d2ba1a62ec88f8b456b9ac9f 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_national_protection.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_national_protection.yaml @@ -1,7 +1,6 @@ "dataset_name": "national_protection" "description": "以下為軍事的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_social_sciences" -"group_alias": "social sciences" -"include": "_default_template_yaml" +"tag": "tmmluplus_social_sciences_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_national_protection" "task_alias": "national protection" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_nautical_science.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_nautical_science.yaml index e02c1dbc2a9d78354f71330bacdea52803abf8cb..1be4cf1d6f722dd6afe2530b78a60d9da75c9fc6 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_nautical_science.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_nautical_science.yaml @@ -1,7 +1,6 @@ "dataset_name": "nautical_science" "description": "以下為航海的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_nautical_science" "task_alias": "nautical science" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_occupational_therapy_for_psychological_disorders.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_occupational_therapy_for_psychological_disorders.yaml index ba2bfa827ed9f5edd8b0799fa9ca9127e16f7f4e..5f6f33956e05428d2a19abe1ef333fea0089a94f 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_occupational_therapy_for_psychological_disorders.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_occupational_therapy_for_psychological_disorders.yaml @@ -1,7 +1,6 @@ "dataset_name": "occupational_therapy_for_psychological_disorders" "description": "以下為心理障礙職能治療學的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_social_sciences" -"group_alias": "social sciences" -"include": "_default_template_yaml" +"tag": "tmmluplus_social_sciences_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_occupational_therapy_for_psychological_disorders" "task_alias": "occupational therapy for psychological disorders" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_official_document_management.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_official_document_management.yaml index 77dce1d671784c86e11899e10ba9ec8c2a1d7b54..16617d2f546b4ddb1798c934acccc917bd102958 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_official_document_management.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_official_document_management.yaml @@ -1,7 +1,6 @@ "dataset_name": "official_document_management" "description": "以下為機關文書的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_official_document_management" "task_alias": "official document management" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_optometry.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_optometry.yaml index 7e3b78b7edd3136d3ed8a10d5e959d3fb72bc7bd..ccec870859049fb81b55fe7ed1e8b1db6e4cec09 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_optometry.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_optometry.yaml @@ -1,7 +1,6 @@ "dataset_name": "optometry" "description": "以下為視光學的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_optometry" "task_alias": "optometry" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_organic_chemistry.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_organic_chemistry.yaml index 75a096f9b4a242f12b207c6daa453d0dc2217e1f..8f1a8cce0c194860f69bf0ba4c63184771532162 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_organic_chemistry.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_organic_chemistry.yaml @@ -1,7 +1,6 @@ "dataset_name": "organic_chemistry" "description": "以下為有機化學的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_STEM" -"group_alias": "STEM" -"include": "_default_template_yaml" +"tag": "tmmluplus_STEM_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_organic_chemistry" "task_alias": "organic chemistry" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_pharmacology.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_pharmacology.yaml index 0da7ec9842f7bdedaa266b48f4b88a7518966d80..f42435e585be06b91cc9dc0c017e21df21ca4cce 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_pharmacology.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_pharmacology.yaml @@ -1,7 +1,6 @@ "dataset_name": "pharmacology" "description": "以下為藥理學的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_pharmacology" "task_alias": "pharmacology" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_pharmacy.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_pharmacy.yaml index a87aa4be10228833b31b3c29c9bda9d6f5dcf8bf..4cb8e83f22b9f21f260fe444221d5f9e2feccbac 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_pharmacy.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_pharmacy.yaml @@ -1,7 +1,6 @@ "dataset_name": "pharmacy" "description": "以下為藥劑學的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_STEM" -"group_alias": "STEM" -"include": "_default_template_yaml" +"tag": "tmmluplus_STEM_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_pharmacy" "task_alias": "pharmacy" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_physical_education.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_physical_education.yaml index fb3558e9baa4cf6ee4c1f19a244341a3a484861c..af762327bb64b1b41a1aea9d3d7a780950625bb2 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_physical_education.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_physical_education.yaml @@ -1,7 +1,6 @@ "dataset_name": "physical_education" "description": "以下為體育的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_social_sciences" -"group_alias": "social sciences" -"include": "_default_template_yaml" +"tag": "tmmluplus_social_sciences_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_physical_education" "task_alias": "physical education" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_physics.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_physics.yaml index 3a8443fe19d0e33011093547f6ada042188a5cee..c9d4e16b34c8431cc59908fa9d4ac3c4219b3e84 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_physics.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_physics.yaml @@ -1,7 +1,6 @@ "dataset_name": "physics" "description": "以下為物理的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_STEM" -"group_alias": "STEM" -"include": "_default_template_yaml" +"tag": "tmmluplus_STEM_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_physics" "task_alias": "physics" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_politic_science.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_politic_science.yaml index 67d2068c5d1bd046263992fe4150fc4519e1345a..9ebbdb242b4f8d2717722dd0f23acee5f72a8bf8 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_politic_science.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_politic_science.yaml @@ -1,7 +1,6 @@ "dataset_name": "politic_science" "description": "以下為政治的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_social_sciences" -"group_alias": "social sciences" -"include": "_default_template_yaml" +"tag": "tmmluplus_social_sciences_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_politic_science" "task_alias": "politic science" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_real_estate.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_real_estate.yaml index ba90b7aa565bf0102967508392d286e13c25a747..971557b53173283967197e657fcb10ce54b6ac24 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_real_estate.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_real_estate.yaml @@ -1,7 +1,6 @@ "dataset_name": "real_estate" "description": "以下為房地產的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_real_estate" "task_alias": "real estate" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_secondary_physics.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_secondary_physics.yaml index 6865167cb1f51310ba30d7b4745e62bc878c5d8f..1660fc6d5b544c8f7085be241c3f9fad1d2091f8 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_secondary_physics.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_secondary_physics.yaml @@ -1,7 +1,6 @@ "dataset_name": "secondary_physics" "description": "以下為高中物理的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_STEM" -"group_alias": "STEM" -"include": "_default_template_yaml" +"tag": "tmmluplus_STEM_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_secondary_physics" "task_alias": "secondary physics" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_statistics_and_machine_learning.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_statistics_and_machine_learning.yaml index 9a6e68ba514f60c8c2f6760a80a09a3cd65eb1a0..252496cf6399cbe166f7d9ba8cf64a2bfac9ba2d 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_statistics_and_machine_learning.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_statistics_and_machine_learning.yaml @@ -1,7 +1,6 @@ "dataset_name": "statistics_and_machine_learning" "description": "以下為統計與機器學習的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_STEM" -"group_alias": "STEM" -"include": "_default_template_yaml" +"tag": "tmmluplus_STEM_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_statistics_and_machine_learning" "task_alias": "statistics and machine learning" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_taiwanese_hokkien.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_taiwanese_hokkien.yaml index 89297df3158681f837462d90ead8660b563ee3e0..2977a1674a791406c1ced974cb6b986403ed8279 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_taiwanese_hokkien.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_taiwanese_hokkien.yaml @@ -1,7 +1,6 @@ "dataset_name": "taiwanese_hokkien" "description": "以下為閩南語的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_social_sciences" -"group_alias": "social sciences" -"include": "_default_template_yaml" +"tag": "tmmluplus_social_sciences_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_taiwanese_hokkien" "task_alias": "taiwanese hokkien" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_taxation.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_taxation.yaml index f54520270fb40d6c91b9ef508235a938b87be190..c382b8c84011189911b54330218518907793cdde 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_taxation.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_taxation.yaml @@ -1,7 +1,6 @@ "dataset_name": "taxation" "description": "以下為稅務的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_humanities" -"group_alias": "humanities" -"include": "_default_template_yaml" +"tag": "tmmluplus_humanities_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_taxation" "task_alias": "taxation" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_technical.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_technical.yaml index 6167a8fe0f63000a8d714ef2ed286ed950297d54..b4621a4e491c6875b1c4bbd07de7be82f7760057 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_technical.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_technical.yaml @@ -1,7 +1,6 @@ "dataset_name": "technical" "description": "以下為技術工相關的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_technical" "task_alias": "technical" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_three_principles_of_people.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_three_principles_of_people.yaml index de50db700ba23b2941ece04a5f0d4eb0999ffe10..83a3628ddf3d3ec4d697e00be6ccbe5552726d5b 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_three_principles_of_people.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_three_principles_of_people.yaml @@ -1,7 +1,6 @@ "dataset_name": "three_principles_of_people" "description": "以下為三民主義的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_social_sciences" -"group_alias": "social sciences" -"include": "_default_template_yaml" +"tag": "tmmluplus_social_sciences_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_three_principles_of_people" "task_alias": "three principles of people" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_trade.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_trade.yaml index 1c42e22e034952c8e26ebff70d3b028e5aee643e..7eacf4dca0b88a1bf180fa54f52afb089c083d00 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_trade.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_trade.yaml @@ -1,7 +1,6 @@ "dataset_name": "trade" "description": "以下為貿易的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_trade" "task_alias": "trade" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_traditional_chinese_medicine_clinical_medicine.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_traditional_chinese_medicine_clinical_medicine.yaml index b5a3fdf197c6f64ecda03af7c6119721ae18df11..7d626dd26aab6593ed5aa64bc837cb538703dac0 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_traditional_chinese_medicine_clinical_medicine.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_traditional_chinese_medicine_clinical_medicine.yaml @@ -1,7 +1,6 @@ "dataset_name": "traditional_chinese_medicine_clinical_medicine" "description": "以下為中醫臨床醫學的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_traditional_chinese_medicine_clinical_medicine" "task_alias": "traditional chinese medicine clinical medicine" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_trust_practice.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_trust_practice.yaml index 0c1e7ae3f44ea9c3fe3f11d41cb1353f787359b4..f4b3a010fc1eaf813720b98b9d48c0b769305de0 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_trust_practice.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_trust_practice.yaml @@ -1,7 +1,6 @@ "dataset_name": "trust_practice" "description": "以下為信託實務的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_humanities" -"group_alias": "humanities" -"include": "_default_template_yaml" +"tag": "tmmluplus_humanities_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_trust_practice" "task_alias": "trust practice" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_ttqav2.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_ttqav2.yaml index 2216d4d46ca7331e2eae24a24affcaf51b68225f..95a10411804ba04722fb385d35fc375f42686270 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_ttqav2.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_ttqav2.yaml @@ -1,7 +1,6 @@ "dataset_name": "ttqav2" "description": "以下為台灣在地用語的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_social_sciences" -"group_alias": "social sciences" -"include": "_default_template_yaml" +"tag": "tmmluplus_social_sciences_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_ttqav2" "task_alias": "ttqav2" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_tve_chinese_language.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_tve_chinese_language.yaml index 0c98fedd3725c1bcf7899328bb9b51a3b3ab5e02..a86600882a0f9a164dcb89ec2dfbe78f7c655683 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_tve_chinese_language.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_tve_chinese_language.yaml @@ -1,7 +1,6 @@ "dataset_name": "tve_chinese_language" "description": "以下為統測國文的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_social_sciences" -"group_alias": "social sciences" -"include": "_default_template_yaml" +"tag": "tmmluplus_social_sciences_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_tve_chinese_language" "task_alias": "tve chinese language" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_tve_design.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_tve_design.yaml index e36b1548025bb6a9a9f95cf58833ba94465aab4e..01a27149e5124eeb22ca79e41d97bf0b8aac60f1 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_tve_design.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_tve_design.yaml @@ -1,7 +1,6 @@ "dataset_name": "tve_design" "description": "以下為統測 設計的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_tve_design" "task_alias": "tve design" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_tve_mathematics.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_tve_mathematics.yaml index b4158f17e9cf040bd5c38419493c2c3c50227ff4..6240db295f2e63697de85e0e5e6baa0d7020352d 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_tve_mathematics.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_tve_mathematics.yaml @@ -1,7 +1,6 @@ "dataset_name": "tve_mathematics" "description": "以下為統測數學的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_STEM" -"group_alias": "STEM" -"include": "_default_template_yaml" +"tag": "tmmluplus_STEM_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_tve_mathematics" "task_alias": "tve mathematics" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_tve_natural_sciences.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_tve_natural_sciences.yaml index c2ecf4ad00c56eb7835bbdf2ce8d5abb33457d07..833c47fc85d331f7f8b8efa3a06381697345d367 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_tve_natural_sciences.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_tve_natural_sciences.yaml @@ -1,7 +1,6 @@ "dataset_name": "tve_natural_sciences" "description": "以下為統測自然科的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_STEM" -"group_alias": "STEM" -"include": "_default_template_yaml" +"tag": "tmmluplus_STEM_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_tve_natural_sciences" "task_alias": "tve natural sciences" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_veterinary_pathology.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_veterinary_pathology.yaml index c1fcb43897705ecf140fcbc34c69fb0b74f66331..c5a8edcb4eeadd262a4ee2ab5b41b19c821455bf 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_veterinary_pathology.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_veterinary_pathology.yaml @@ -1,7 +1,6 @@ "dataset_name": "veterinary_pathology" "description": "以下為獸醫病理學的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_veterinary_pathology" "task_alias": "veterinary pathology" diff --git a/lm_eval/tasks/tmmluplus/default/tmmluplus_veterinary_pharmacology.yaml b/lm_eval/tasks/tmmluplus/default/tmmluplus_veterinary_pharmacology.yaml index 45c6553b2985013ae44ddaa401b7e2a10cfa59ee..b75b1f0075e8e9719c409a2f5053f1057d8d0f69 100644 --- a/lm_eval/tasks/tmmluplus/default/tmmluplus_veterinary_pharmacology.yaml +++ b/lm_eval/tasks/tmmluplus/default/tmmluplus_veterinary_pharmacology.yaml @@ -1,7 +1,6 @@ "dataset_name": "veterinary_pharmacology" "description": "以下為獸醫藥理學的單選題，請提供正確答案的選項。\n\n" -"group": "tmmluplus_other" -"group_alias": "other" -"include": "_default_template_yaml" +"tag": "tmmluplus_other_tasks" +"include": "_tmmluplus_template_yaml" "task": "tmmluplus_veterinary_pharmacology" "task_alias": "veterinary pharmacology" diff --git a/lm_eval/tasks/translation/iwslt2017_ar-en.yaml b/lm_eval/tasks/translation/iwslt2017_ar-en.yaml index ea713393c1dfbe9f7e1f6d055dd4768ace31269e..824f4eba6730f57ef5282ec557b884b1dc772db9 100644 --- a/lm_eval/tasks/translation/iwslt2017_ar-en.yaml +++ b/lm_eval/tasks/translation/iwslt2017_ar-en.yaml @@ -5,8 +5,7 @@ doc_to_target: ' {{translation["en"]}}' doc_to_text: 'Arabic phrase: {{translation["ar"]}} English phrase:' -group: -- generate_until +tag: - translation - iwslt2017 include: wmt_common_yaml diff --git a/lm_eval/tasks/translation/iwslt2017_en-ar.yaml b/lm_eval/tasks/translation/iwslt2017_en-ar.yaml index 891ad50fd6fb60fdb8f21f9004857d739a15640f..c4b124b01807c5af10b95c30d6251107c0c95c9c 100644 --- a/lm_eval/tasks/translation/iwslt2017_en-ar.yaml +++ b/lm_eval/tasks/translation/iwslt2017_en-ar.yaml @@ -5,8 +5,7 @@ doc_to_target: ' {{translation["ar"]}}' doc_to_text: 'English phrase: {{translation["en"]}} Arabic phrase:' -group: -- generate_until +tag: - translation - iwslt2017 include: wmt_common_yaml diff --git a/lm_eval/tasks/translation/wmt14_en-fr.yaml b/lm_eval/tasks/translation/wmt14_en-fr.yaml index b7e42dca5acca5036ec8b3b619501557c6a1c36c..dd5f3ce0438746a0a3dbfd0151b178b383191175 100644 --- a/lm_eval/tasks/translation/wmt14_en-fr.yaml +++ b/lm_eval/tasks/translation/wmt14_en-fr.yaml @@ -5,8 +5,7 @@ doc_to_target: ' {{translation["fr"]}}' doc_to_text: 'English phrase: {{translation["en"]}} French phrase:' -group: -- generate_until +tag: - translation - wmt14 - gpt3_translation_benchmarks diff --git a/lm_eval/tasks/translation/wmt14_fr-en.yaml b/lm_eval/tasks/translation/wmt14_fr-en.yaml index 09ddd57d6049c29f35150aa4de94c6db3604a0a4..d91fed417607115a6b3c2a67afdb0c1b4c6410cf 100644 --- a/lm_eval/tasks/translation/wmt14_fr-en.yaml +++ b/lm_eval/tasks/translation/wmt14_fr-en.yaml @@ -5,8 +5,7 @@ doc_to_target: ' {{translation["en"]}}' doc_to_text: 'French phrase: {{translation["fr"]}} English phrase:' -group: -- generate_until +tag: - translation - wmt14 - gpt3_translation_benchmarks diff --git a/lm_eval/tasks/translation/wmt16_de-en.yaml b/lm_eval/tasks/translation/wmt16_de-en.yaml index 23d50e4aacc8c4e19a8b282e4051e80ec18edf29..d5ac1805ee77927129d1d668b455731511874485 100644 --- a/lm_eval/tasks/translation/wmt16_de-en.yaml +++ b/lm_eval/tasks/translation/wmt16_de-en.yaml @@ -5,8 +5,7 @@ doc_to_target: ' {{translation["en"]}}' doc_to_text: 'German phrase: {{translation["de"]}} English phrase:' -group: -- generate_until +tag: - translation - wmt16 - gpt3_translation_benchmarks diff --git a/lm_eval/tasks/translation/wmt16_en-de.yaml b/lm_eval/tasks/translation/wmt16_en-de.yaml index 8d391b6c6b879c15f0c8d63119824647ea6997c3..d52ab498f7d6baa2cba68b1c35e3eb8dcd34a6df 100644 --- a/lm_eval/tasks/translation/wmt16_en-de.yaml +++ b/lm_eval/tasks/translation/wmt16_en-de.yaml @@ -5,8 +5,7 @@ doc_to_target: ' {{translation["de"]}}' doc_to_text: 'English phrase: {{translation["en"]}} German phrase:' -group: -- generate_until +tag: - translation - wmt16 - gpt3_translation_benchmarks diff --git a/lm_eval/tasks/translation/wmt16_en-ro.yaml b/lm_eval/tasks/translation/wmt16_en-ro.yaml index 45a8cae11824bd726064448422f021ec73d7ce87..096f8743b4bfb56b332aea69b0056ade8d200fd7 100644 --- a/lm_eval/tasks/translation/wmt16_en-ro.yaml +++ b/lm_eval/tasks/translation/wmt16_en-ro.yaml @@ -5,8 +5,7 @@ doc_to_target: ' {{translation["ro"]}}' doc_to_text: 'English phrase: {{translation["en"]}} Romanian phrase:' -group: -- generate_until +tag: - translation - wmt16 - gpt3_translation_benchmarks diff --git a/lm_eval/tasks/translation/wmt16_ro-en.yaml b/lm_eval/tasks/translation/wmt16_ro-en.yaml index 39441eac1c8cb2a8ec4d4e9c9b31402607a5ea77..9e338347cc9d885d03e639856976b795a50a0ce2 100644 --- a/lm_eval/tasks/translation/wmt16_ro-en.yaml +++ b/lm_eval/tasks/translation/wmt16_ro-en.yaml @@ -5,8 +5,7 @@ doc_to_target: ' {{translation["en"]}}' doc_to_text: 'Romanian phrase: {{translation["ro"]}} English phrase:' -group: -- generate_until +tag: - translation - wmt16 - gpt3_translation_benchmarks diff --git a/lm_eval/tasks/truthfulqa/README.md b/lm_eval/tasks/truthfulqa/README.md index f29db6ca5406429a4a3c98a884e6154556554f64..c5cb1689a948ce58d3adeb971828dacf640e1933 100644 --- a/lm_eval/tasks/truthfulqa/README.md +++ b/lm_eval/tasks/truthfulqa/README.md @@ -36,8 +36,8 @@ Homepage: `https://github.com/sylinrl/TruthfulQA` #### Tasks * `truthfulqa_mc1`: `Multiple-choice, single answer` -* (MISSING)`truthfulqa_mc2`: `Multiple-choice, multiple answers` -* (MISSING)`truthfulqa_gen`: `Answer generation` +* `truthfulqa_mc2`: `Multiple-choice, multiple answers` +* `truthfulqa_gen`: `Answer generation` ### Checklist diff --git a/lm_eval/tasks/truthfulqa/truthfulqa_gen.yaml b/lm_eval/tasks/truthfulqa/truthfulqa_gen.yaml index e83a68a3f550fad22fbd3fc05462b3911d40f6dd..ead101df9cb2ceb795f131e71a3f716bbcc6ed2f 100644 --- a/lm_eval/tasks/truthfulqa/truthfulqa_gen.yaml +++ b/lm_eval/tasks/truthfulqa/truthfulqa_gen.yaml @@ -1,4 +1,4 @@ -group: +tag: - truthfulqa task: truthfulqa_gen dataset_path: truthful_qa diff --git a/lm_eval/tasks/truthfulqa/truthfulqa_mc1.yaml b/lm_eval/tasks/truthfulqa/truthfulqa_mc1.yaml index d9d3a696aaaf844b51fc060a04daa9d534335f75..5ea8a28696c306b4d17b2eba48cb443571e7267a 100644 --- a/lm_eval/tasks/truthfulqa/truthfulqa_mc1.yaml +++ b/lm_eval/tasks/truthfulqa/truthfulqa_mc1.yaml @@ -1,4 +1,4 @@ -group: +tag: - truthfulqa task: truthfulqa_mc1 dataset_path: truthful_qa diff --git a/lm_eval/tasks/truthfulqa/utils.py b/lm_eval/tasks/truthfulqa/utils.py index 399969ca5cb9d11cd2ff89880355082b495556e8..3f2317551ef18c4c0fb4f5b98b56bbbcc607b1f0 100644 --- a/lm_eval/tasks/truthfulqa/utils.py +++ b/lm_eval/tasks/truthfulqa/utils.py @@ -4,6 +4,9 @@ import sacrebleu from rouge_score import rouge_scorer, scoring +ROUGE_SCORER = None + + def process_results_mc2(doc, results): lls, is_greedy = zip(*results) @@ -149,8 +152,14 @@ def rouge(refs, preds): :param preds: A `list` of predicted `strs`. """ + rouge_types = ["rouge1", "rouge2", "rougeLsum"] - scorer = rouge_scorer.RougeScorer(rouge_types) + + global ROUGE_SCORER + if ROUGE_SCORER is None: + # init RougeScorer once (https://github.com/EleutherAI/lm-evaluation-harness/issues/1692)--rouge_types are constant + ROUGE_SCORER = rouge_scorer.RougeScorer(rouge_types) + scorer = ROUGE_SCORER # Add newlines between sentences to correctly compute `rougeLsum`. def _prepare_summary(summary): diff --git a/lm_eval/tasks/turkishmmlu/README.md b/lm_eval/tasks/turkishmmlu/README.md new file mode 100644 index 0000000000000000000000000000000000000000..fd7d2e0db0d83684511d531505b7ff6f1ac3eb50 --- /dev/null +++ b/lm_eval/tasks/turkishmmlu/README.md @@ -0,0 +1,94 @@ +# TurkishMMLU + +This repository contains configuration files for LM Evaluation Harness for Few-Shot and Chain-of-Thought experiments for TurkishMMLU. Using these configurations with LM Evaluation Harness, the results of this study are obtained. + +TurkishMMLU is a multiple-choice Question-Answering dataset created for the Turkish Natural Language Processing (NLP) community based on Turkish Highschool Curricula across nine subjects. This comprehensive study is conducted to provide Question-Answering benchmark for Turkish language. The questions of the dataset are written by curriculum experts, suitable for the high-school curricula in Turkey, covering subjects ranging from natural sciences and math questions to more culturally representative topics such as Turkish Literature and the history of the Turkish Republic. + +To access this dataset please send an email to: +arda.yueksel@tum.de or akoksal@cis.lmu.de. + +## Abstract + +Multiple choice question answering tasks evaluate the reasoning, comprehension, and mathematical abilities of Large Language Models (LLMs). While existing benchmarks employ automatic translation for multilingual evaluation, this approach is error-prone and potentially introduces culturally biased questions, especially in social sciences. We introduce the first multitask, multiple-choice Turkish QA benchmark, TurkishMMLU, to evaluate LLMs' understanding of the Turkish language. TurkishMMLU includes over 10,000 questions, covering 9 different subjects from Turkish high-school education curricula. These questions are written by curriculum experts, suitable for the high-school curricula in Turkey, covering subjects ranging from natural sciences and math questions to more culturally representative topics such as Turkish Literature and the history of the Turkish Republic. We evaluate over 20 LLMs, including multilingual open-source (e.g., Gemma, Llama, MT5), closed-source (GPT 4o, Claude, Gemini), and Turkish-adapted (e.g., Trendyol) models. We provide an extensive evaluation, including zero-shot and few-shot evaluation of LLMs, chain-of-thought reasoning, and question difficulty analysis along with model performance. We provide an in-depth analysis of the Turkish capabilities and limitations of current LLMs to provide insights for future LLMs for the Turkish language. We publicly release our code for the dataset and evaluation. + +## Dataset + +Dataset is divided into four categories Natural Sciences, Mathematics, Language, and Social Sciences and Humanities with a total of nine subjects in Turkish highschool education. It is available in multiple choice for LLM evaluation. The questions also contain difficulty indicator referred as Correctness ratio. + +## Evaluation + +5-Shot evaluation results from the paper includes open and closed source SOTA LLM with different architectures. For this study, multilingual and Turkish adapted models are tested. + +The evaluation results of this study are obtained using the provided configurations with LM Evaluation Harness. + +| Model | Source | Average | Natural Sciences | Math | Turkish L & L | Social Sciences and Humanities | +| ------------------- | ------ | ------- | ---------------- | ---- | ------------- | ------------------------------ | +| GPT 4o | Closed | 83.1 | 75.3 | 59.0 | 82.0 | 95.3 | +| Claude-3 Opus | Closed | 79.1 | 71.7 | 59.0 | 77.0 | 90.3 | +| GPT 4-turbo | Closed | 75.7 | 70.3 | 57.0 | 67.0 | 86.5 | +| Llama-3 70B-IT | Closed | 67.3 | 56.7 | 42.0 | 57.0 | 84.3 | +| Claude-3 Sonnet | Closed | 67.3 | 67.3 | 44.0 | 58.0 | 75.5 | +| Llama-3 70B | Open | 66.1 | 56.0 | 37.0 | 57.0 | 83.3 | +| Claude-3 Haiku | Closed | 65.4 | 57.0 | 40.0 | 61.0 | 79.3 | +| Gemini 1.0-pro | Closed | 63.2 | 52.7 | 29.0 | 63.0 | 79.8 | +| C4AI Command-r+ | Open | 60.6 | 50.0 | 26.0 | 57.0 | 78.0 | +| Aya-23 35B | Open | 55.6 | 43.3 | 31.0 | 49.0 | 72.5 | +| C4AI Command-r | Open | 54.9 | 44.7 | 29.0 | 49.0 | 70.5 | +| Mixtral 8x22B | Open | 54.8 | 45.3 | 27.0 | 49.0 | 70.3 | +| GPT 3.5-turbo | Closed | 51.0 | 42.7 | 39.0 | 35.0 | 61.8 | +| Llama-3 8B-IT | Open | 46.4 | 36.7 | 29.0 | 39.0 | 60.0 | +| Llama-3 8B | Open | 46.2 | 37.3 | 30.0 | 33.0 | 60.3 | +| Mixtral 8x7B-IT | Open | 45.2 | 41.3 | 28.0 | 39.0 | 54.0 | +| Aya-23 8B | Open | 45.0 | 39.0 | 23.0 | 31.0 | 58.5 | +| Gemma 7B | Open | 43.6 | 34.3 | 22.0 | 47.0 | 55.0 | +| Aya-101 | Open | 40.7 | 31.3 | 24.0 | 38.0 | 55.0 | +| Trendyol-LLM 7B-C-D | Open | 34.1 | 30.3 | 22.0 | 28.0 | 41.5 | +| mT0-xxl | Open | 33.9 | 29.3 | 28.0 | 21.0 | 42.0 | +| Mistral 7B-IT | Open | 32.0 | 34.3 | 26.0 | 38.0 | 30.3 | +| Llama-2 7B | Open | 22.3 | 25.3 | 20.0 | 20.0 | 19.8 | +| mT5-xxl | Open | 18.1 | 19.3 | 24.0 | 14.0 | 16.8 | + +## Citation + +``` +@misc{yüksel2024turkishmmlumeasuringmassivemultitask, +title={TurkishMMLU: Measuring Massive Multitask Language Understanding in Turkish}, +author={Arda Yüksel and Abdullatif Köksal and Lütfi Kerem Şenel and Anna Korhonen and Hinrich Schütze}, +year={2024}, +eprint={2407.12402}, +archivePrefix={arXiv}, +primaryClass={cs.CL}, +url={https://arxiv.org/abs/2407.12402}, +} +``` + +### Groups and Tasks + +#### Groups + +- `turkishmmlu`: 'All 9 Subjects of Turkish MMLU namely: + Biology, Chemistry, Physics, Geography, Philosophy, History, Religion and Ethics, Turkish Language and Literature, and Mathematics + +#### Tasks + +The following tasks evaluate subjects in the TurkishMMLU dataset + +- `turkishmmlu_{subject}` + +The following task evaluate subjects in the TurkishMMLU dataset in Chain-of-Thought (COT) + +- `turkishmmlu_cot_{subject}` + +### Checklist + +For adding novel benchmarks/datasets to the library: + +- [x] Is the task an existing benchmark in the literature? + - [x] Have you referenced the original paper that introduced the task? + - [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + +If other tasks on this dataset are already supported: + +- [ ] Is the "Main" variant of this task clearly denoted? +- [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates? +- [ ] Have you noted which, if any, published evaluation setups are matched by this variant? diff --git a/lm_eval/tasks/turkishmmlu/config/Biology.yaml b/lm_eval/tasks/turkishmmlu/config/Biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cafd341ba89113f943498641e92d7cb241d054b5 --- /dev/null +++ b/lm_eval/tasks/turkishmmlu/config/Biology.yaml @@ -0,0 +1,3 @@ +include: _turkishmmlu_default_yaml +task: turkishmmlu_biology +dataset_name: Biology diff --git a/lm_eval/tasks/turkishmmlu/config/Chemistry.yaml b/lm_eval/tasks/turkishmmlu/config/Chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..42dd1a87d482ed10950a6614d96b2361f57962e4 --- /dev/null +++ b/lm_eval/tasks/turkishmmlu/config/Chemistry.yaml @@ -0,0 +1,3 @@ +include: _turkishmmlu_default_yaml +task: turkishmmlu_chemistry +dataset_name: Chemistry diff --git a/lm_eval/tasks/turkishmmlu/config/Geography.yaml b/lm_eval/tasks/turkishmmlu/config/Geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..110f77ca0e54df0c2653c62a6f461920c7a34476 --- /dev/null +++ b/lm_eval/tasks/turkishmmlu/config/Geography.yaml @@ -0,0 +1,3 @@ +include: _turkishmmlu_default_yaml +task: turkishmmlu_geography +dataset_name: Geography diff --git a/lm_eval/tasks/turkishmmlu/config/History.yaml b/lm_eval/tasks/turkishmmlu/config/History.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d8350f5ef3497b3194614027a72a1e2539d45a19 --- /dev/null +++ b/lm_eval/tasks/turkishmmlu/config/History.yaml @@ -0,0 +1,3 @@ +include: _turkishmmlu_default_yaml +task: turkishmmlu_history +dataset_name: History diff --git a/lm_eval/tasks/turkishmmlu/config/Mathematics.yaml b/lm_eval/tasks/turkishmmlu/config/Mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4318be85929e4f24425c863a21a3e8607956e62a --- /dev/null +++ b/lm_eval/tasks/turkishmmlu/config/Mathematics.yaml @@ -0,0 +1,3 @@ +include: _turkishmmlu_default_yaml +task: turkishmmlu_mathematics +dataset_name: Mathematics diff --git a/lm_eval/tasks/turkishmmlu/config/Philosophy.yaml b/lm_eval/tasks/turkishmmlu/config/Philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4ced0c333fa17ef0b5391103254d130bea9633cd --- /dev/null +++ b/lm_eval/tasks/turkishmmlu/config/Philosophy.yaml @@ -0,0 +1,3 @@ +include: _turkishmmlu_default_yaml +task: turkishmmlu_philosophy +dataset_name: Philosophy diff --git a/lm_eval/tasks/turkishmmlu/config/Physics.yaml b/lm_eval/tasks/turkishmmlu/config/Physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9baeb743a782b7bdd906e8e94297a0f85d9a9099 --- /dev/null +++ b/lm_eval/tasks/turkishmmlu/config/Physics.yaml @@ -0,0 +1,3 @@ +include: _turkishmmlu_default_yaml +task: turkishmmlu_physics +dataset_name: Physics diff --git a/lm_eval/tasks/turkishmmlu/config/Religion_and_Ethics.yaml b/lm_eval/tasks/turkishmmlu/config/Religion_and_Ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7513119d5b7504ced4aa6ad818ca257f76a50794 --- /dev/null +++ b/lm_eval/tasks/turkishmmlu/config/Religion_and_Ethics.yaml @@ -0,0 +1,3 @@ +include: _turkishmmlu_default_yaml +task: turkishmmlu_religion_and_ethics +dataset_name: Religion_and_Ethics diff --git a/lm_eval/tasks/turkishmmlu/config/Turkish_Language_and_Literature.yaml b/lm_eval/tasks/turkishmmlu/config/Turkish_Language_and_Literature.yaml new file mode 100644 index 0000000000000000000000000000000000000000..552e08b7d6bf41870c6ab3d5a5d3d22c1ce90061 --- /dev/null +++ b/lm_eval/tasks/turkishmmlu/config/Turkish_Language_and_Literature.yaml @@ -0,0 +1,3 @@ +include: _turkishmmlu_default_yaml +task: turkishmmlu_turkish_language_and_literature +dataset_name: Turkish_Language_and_Literature diff --git a/lm_eval/tasks/turkishmmlu/config/_turkishmmlu_default_yaml b/lm_eval/tasks/turkishmmlu/config/_turkishmmlu_default_yaml new file mode 100644 index 0000000000000000000000000000000000000000..e1dae1a9773bfcde888a96291c479c472c6b7077 --- /dev/null +++ b/lm_eval/tasks/turkishmmlu/config/_turkishmmlu_default_yaml @@ -0,0 +1,21 @@ +tag: turkishmmlu +task: null +dataset_path: AYueksel/TurkishMMLU +dataset_name: TurkishMMLU +test_split: test +fewshot_split: dev +fewshot_config: + sampler: first_n +output_type: multiple_choice +doc_to_text: "Soru: {{ question.strip() }}\nA. {{ choices[0] }}\nB. {{ choices[1] }}\nC. {{ choices[2] }}\nD. {{ choices[3] }}\nE. {{ choices[4] }}\nCevap:" +doc_to_choice: ["A", "B", "C", "D", "E"] +doc_to_target: "{{['A', 'B', 'C', 'D', 'E'].index(answer)}}" +metric_list: + - metric: acc + aggregation: mean + higher_is_better: true + - metric: acc_norm + aggregation: mean + higher_is_better: true +metadata: + version: 0.0 diff --git a/lm_eval/tasks/turkishmmlu/config_cot/Biology.yaml b/lm_eval/tasks/turkishmmlu/config_cot/Biology.yaml new file mode 100644 index 0000000000000000000000000000000000000000..98d61be7d7e06b637ee9fec20c615136fc673b63 --- /dev/null +++ b/lm_eval/tasks/turkishmmlu/config_cot/Biology.yaml @@ -0,0 +1,54 @@ +include: _turkishmmlu_cot_default_yaml +task: turkishmmlu_cot_biology +description: + "Soru: AaKkDdEeX$^{R}$X$^{r}$ genotipli bireyde AKD alelleri bağlı olup diğerleri bağımsızdır.\ + \ Bu birey bu karakterler açısından kaç çeşit gamet oluşturabilir? (Krossing over gerçekleşmemiştir.)\nA) 2 \nB)\ + \ 4 \nC) 8 \nD) 16 \nE) 32 \nÇözüm: Adım adım düşünelim.\ + \ Bir bireyde A, K ve D genleri bağımlı olduğunda, bu üç gen birlikte hareket eder ve aynı gamet içinde bulunurlar.\ + \ Diğer lokuslardaki alleller bağımsızdır.\ + \ AKD lokusundaki allel kombinasyonları: AKD veya akd (2 seçenek)\ + \ Diğer lokuslardaki allel kombinasyonları: Ee (2 seçenek), X$^{R}$X$^{r}$ (2 seçenek).\ + \ Şimdi, bağımlı olan AKD lokusundaki kombinasyonu diğer bağımsız lokuslardaki kombinasyonlarla çarpacağız:\ + \ 2 x 2 x 2 = 8\ + \ Bu birey 8 farklı gamet oluşturabilir.\ + \ Doğru cevap C şıkkıdır.\n\nSoru: Aşağıda verilen hormon çiftlerinden hangisi antagonist (zıt) çalışır?\nA) Oksitosin\ + \ - Prolaktin\nB) Kalsitonin - Parathormon\nC) Adrenalin\ + \ - Noradrenalin\nD) Östrojen - Progesteron\nE) FSH - LH\nÇözüm: Adım adım düşünelim.\ + \ Bu soruyu cevaplayabilmek için hormonların görevlerini ve birbirleri ile olan ilişkilerini bilmek gerekir.\ + \ A) Oksitosin ve Prolaktin: Oksitosin doğum sırasında uterus kasılmalarını uyarır ve süt salgılanmasını sağlar. Prolaktin ise süt üretimini uyarır. Bu iki hormon birbirini destekleyici görev yapar, zıt değildir.\ + \ B) Kalsitonin ve Parathormon: Kalsitonin kanda kalsiyum seviyesini düşürür, parathormon ise kalsiyum seviyesini yükseltir. Bu iki hormon birbirine zıt etki gösterir, antagonisttir.\ + \ C) Adrenalin ve Noradrenalin: Her ikisi de stres hormonudur ve benzer görevleri vardır. Zıt etki göstermezler.\ + \ D) Östrojen ve Progesteron: Östrojen kadınlık hormonudur, progesteron ise gebelik sırasında üretilir. Birlikte çalışırlar, zıt etki göstermezler.\ + \ E) FSH ve LH: FSH folikül gelişimini, LH ovulasyonu uyarır. Birlikte çalışırlar, zıt etki göstermezler.\ + \ Dolayısıyla verilen seçenekler arasında antagonist (zıt) çalışan hormon çifti Kalsitonin ve Parathormon'dur.\ + \ Doğru cevap B şıkkıdır.\n\nSoru: I. Besin azalması II. Avcıların artması III. Zehirli madde birikimin artması\ + \ Yukarıdaki faktörlerden hangileri çevre direncini artırır?\nA) Yalnız I\nB) Yalnız II\nC)\ + \ Yalnız III\nD) II ve III\nE) I, II ve III\nÇözüm: Adım adım düşünelim.\ + \ Çevre direnci, bir ekosistemin dışarıdan gelen olumsuz etkilere karşı direncini ifade eder. Yüksek çevre direnci, ekosistemin bu olumsuz etkilere daha iyi direnebileceği anlamına gelir.\ + \ I. Besin azalması, popülasyonların büyümesini ve sağlığını olumsuz etkiler, dolayısıyla çevre direncini artırır.\ + \ II. Avcıların artması, popülasyonların dengesini bozar ve türlerin sayısını azaltır, bu da çevre direncini artırır.\ + \ III. Zehirli madde birikiminin artması, canlıların sağlığını ve üremesini olumsuz etkiler, ekosistemin dengesini bozar, bu şekilde çevre direncini artırır.\ + \ Sorudaki faktörlerin hepsi olumsuz faktörlerdir ve ekosistemin direncini zorlar. Doğru cevap E şıkkıdır.\n\nSoru:\ + \ Gen klonlama çalışmaları sırasında; a. Vektör DNA ve istenen geni taşıyan DNA'nın kesilmesi, b. İstenen geni taşıyan DNA'nın,\ + \ vektör DNA ile birleştirilmesi, c. Bakterinin çoğalmasıyla birlikte istenen genin kopyalanması, uygulamaları yapılmaktadır.\ + \ Bu uygulamalarda; I. DNA polimeraz II. DNA ligaz III. Restriksiyon enzimi yapılarının kullanıldığı çalışma basamakları\ + \ hangi seçenekte doğru olarak verilmiştir?\ + \ I II III \nA) a, b b\ + \ a, c\nB) b a, b c\nC)\ + \ a c a, c\nD) c b, c a\nE)\ + \ b, c a a, b\nÇözüm: Adım Adım düşünelim.\ + \ I. DNA polimeraz: c (Bakterinin çoğalması ile birlikte istenen genin kopyalanması)\ + \ II. DNA ligaz: b, c (İstenen geni taşıyan DNA'nın, vektör DNA ile birleştirilmesi ve sonrasında bakterinin çoğalması ile birlikte kopyalanması)\ + \ III. Restriksiyon enzimi: a (Vektör DNA ve istenen geni taşıyan DNA'nın kesilmesi)\ + \ Doğru cevap D şıkkıdır.\n\nSoru: İnsanlardaki lizozomlar, fagositoz yapmayan hücrelerde de aktif olabilir. Hücreler metabolik faaliyetlerinin sorunsuz geçebilmesi için bazı hücresel yapılarını yıkıp yeniden yapar.\ + \ Hücresel yapıların kontrollü ve programlı şekilde yıkılması lizozomlar tarafından yapılır ve otofaji olarak bilinir.\ + \ Otofaji ile ilgili ifadelerden; I. Otofaji sonucu hücresel yapılar yıkılamadığında lizozomal depolama hastalıkları ortaya çıkar\ + \ II. Otofaji sırasında hidrolitik enzimler hücre dışında etkinlik gösterir\ + \ III. Otofaji olayında hidrolitik enzimler lizozomlarda üretilip sitoplazmaya gönderilir hangileri doğrudur?\nA) Yalnız\ + \ I\nB) I ve II\nC) I ve III\nD) II ve III\nE) I, II ve III\nÇözüm: Adım adım düşünelim.\ + \ I. Otofaji sonucu hücresel yapılar yıkılamadığında lizozomal depolama hastalıkları ortaya çıkar: Doğru\ + \ II. Otofaji sırasında hidrolitik enzimler hücre dışında etkinlik gösterir: Yanlış, hidrolitik enzimler lizozomlarda etkinlik gösterir.\ + \ III. Otofaji olayında hidrolitik enzimler lizozomlarda üretilip sitoplazmaya gönderilir: Yanlış, hidrolitik enzimler lizozomlarda üretilir ve lizozom içinde etkinlik gösterir.\ + \ Doğru cevap A şıkkıdır." +num_fewshot: 0 +dataset_name: Biology diff --git a/lm_eval/tasks/turkishmmlu/config_cot/Chemistry.yaml b/lm_eval/tasks/turkishmmlu/config_cot/Chemistry.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b002bb91f8669cc9d99ac910ebc64049b2e6eb31 --- /dev/null +++ b/lm_eval/tasks/turkishmmlu/config_cot/Chemistry.yaml @@ -0,0 +1,86 @@ +include: _turkishmmlu_cot_default_yaml +task: turkishmmlu_cot_chemistry +description: + "Soru: 16 litre CO ve CO $_{2}$ gaz karışımını yakmak için aynı koşullarda 30 litre hava kullanılmaktadır.\ + \ Buna göre başlangıç karışımındaki VCO\/VCO$_{2}$ oranı aşağıdakilerden hangisidir?(Havanın 1\/5’i oksijen gazıdır.)\nA)\ + \ 3\nB) 3\/8\nC) 2\/3\nD) 1\/5\nE) 1\/3\nÇözüm: Adım adım düşünelim.\ + \ Yakma için kullanılan hava miktarının 1\/5'i oksijen olduğuna göre; 30 litre hava içinde 30*(1\/5) = 6 litre oksijen mevcuttur.\ + \ CO ve CO$_2$ yakmak için gerekli oksijen miktarlarını inceleyelim:CO + 1\/2 O$_2$ → CO$_2$ (Her 1 mol CO için 1\/2 mol O$_2$ gerekir.)\ + \ CO$_2$ yanmaz. Bu durumda, CO'nun yandığını ve tüm 6 litre oksijeni kullandığını varsayalım. 2 mol CO (her biri 22.4 litre/mol standart koşullarda)\ + \ için 1 mol O$_2$ (22.4 litre/mol) gerekiyor. Bu nedenle, 6 litre O$_2$ ile 12 litre CO yakılabilir.\ + \ Bu bilgiyle, başlangıçtaki CO miktarını hesaplayabiliriz. Ancak burada bir yanlış anlama var; zira yakma işlemi\ + \ sonunda sadece CO kullanıldığını farzetmek, CO$_2$'nin başlangıçta orada olmadığını varsayar ki bu doğru değil.\ + \ Soruda CO ve CO$_2$ karışımından bahsedilmektedir, bu nedenle direkt CO kullanımı üzerinden gitmemiz gerek.\ + \ Yakma işlemi CO için geçerli olduğundan ve tüm oksijen CO'nun yanması için kullanıldığından, CO miktarı ve oranı\ + \ direkt olarak hesaplanmalıdır. 6 litre O$_2$, 12 litre CO yakabilir düşüncesi yanlış olmuştur, çünkü her 1 litre CO için 1\/2 litre O$_2$ gerekmektedir.\ + \ Doğru açıklamayla, \\(V_{CO}\\) miktarını \\(x\\) litre ve \\(V_{CO_2}\\) miktarını \\(16 - x\\) litre olarak alalım.\ + \ CO yanmak için 1\/2 oranında O2 kullanır:\ + \ $\\[x \\text{ litre CO} \\times \\frac{1}{2} = \\frac{x}{2} \\text{ litre O}_2\\]$\ + \ Tüm işlemde 6 litre O$_2$ kullanıldığından ve bu O$_2$ yalnızca CO'nun yanması için kullanıldığından,\ + \ $\\(\\frac{x}{2} = 6 \\rightarrow x = 12\\)$. Yani 12 litre CO ve dolayısıyla $\\(16 - 12 = 4\\)$ litre CO$_2$ vardır.\ + \ Bu durumda VCO\/VCO$_2$ oranı $\\(12 / 4 = 3\\)$'tür. Doğru cevap A şıkkıdır.\n\nSoru:\ + \ Aşağıda verilen çözeltilerden hangisinin bileşenlerinin fiziksel halleri doğru verilmiştir?\nÖrnek Çözünen\ + \ Çözücü\nA) Alkollü su Katı Sıvı\nB)\ + \ Tuzlu su Sıvı Katı\nC)\ + \ Gazoz Gaz Sıvı\nD)\ + \ Lehim Gaz Katı\nE)\ + \ Palladyumda hidrojen Katı Gaz\nÇözüm: Adım adım düşünelim.\ + \ Doğru seçenek C şıkkıdır: Gazoz. Gazoz, çözüneni bir gaz (genellikle karbondioksit) ve çözücüsü bir sıvı (su) olan bir çözeltidir. Diğer seçeneklerde çözünen ve çözücü arasındaki fiziksel haller doğru verilmemiştir.\ + \ A şıkkı: Alkollü su; çözünen etanol (alkol) katı değil, sıvı halde ve çözücü su, sıvı halde olmalıdır.\ + \ B şıkkı: Tuzlu su; çözünen tuz (genellikle sodyum klorür) katı, çözücü su ise sıvı halde olmalıdır.\ + \ D şıkkı: Lehim; çözünen katı bir madde (genellikle kalay ve kurşun alaşımı) ve çözücü katı halde olduğunda lehim olarak adlandırılmaz; ısındığında sıvı hale geçer.\ + \ E şıkkı: Palladyumda hidrojen; çözünen hidrojen gazı ve çözücüsü katı bir metal olan palladyum doğru bir açıklamadır fakat soruda istenilen \"çözücü ve çözünenin fiziksel halleri\" açısından en uygun seçenek\ + \ Gazozdur. Doğru cevap C şıkkıdır.\n\nSoru: $_{29}$X$^{2+}$ iyonu ve $_{27}$Y atomu ile ilgili,\ + \ l. s orbitallerindeki elektron sayısı farklıdır. ll. İzoelektronik taneciklerdir. lll. Yarı dolu orbital sayıları eşittir.\ + \ yargılarından hangileri yanlıştır?\nA) l, ll ve lll\nB) ll ve lll\nC) l ve ll\nD) l ve lll\nE) Yalnız lll\nÇözüm: Adım adım düşünelim.\ + \ Bu soruyu çözmek için her iki tanecik $(_{29} \\text{X}^{2+} \\text{ ve } _{27}\\text{Y})$ hakkında bazı bilgileri bilmek gerekmektedir.\ + \ İlk olarak, periyodik cetveldeki yerlerine ve elektron konfigürasyonlarına bakalım.\ + \ - $_{29}$X sembolü ile temsil edilen element Bakır (Cu)’dır ve onun +2 yüklü iyonu $_{29}\\text{Cu}^{2+}$, 29 protona\\ + \ ve 27 elektrona sahip olacaktır. Bakır'ın temel hâli için elektron konfigürasyonu $[Ar] 3d^10 4s^1$ dir,\ + \ ancak 2 elektron kaybettiğinde, 4s orbitalinden ve bir sonraki 3d orbitalinden elektronlar kaybolur ve son konfigürasyon $[Ar] 3d^9$ olur.\ + \ - $_{27}$Y sembolü ile temsil edilen element Kobalt (Co)’dır ve Kobalt'ın nötr atomu, 27 elektrona sahiptir.\ + \ Kobalt'ın elektron konfigürasyonu $[Ar] 3d^7 4s^2$ dir. Şimdi yargılara bakalım:\ + \ l. s orbitallerindeki elektron sayısı farklıdır. Bu doğru, çünkü Cu^{2+} iyonunda 4s orbitali tamamen boşken,\ + \ Co atomunda 4s orbitali 2 elektron içerir.\ + \ ll. İzoelektronik taneciklerdir. Bu yanlış, çünkü izoelektronik tanecikler aynı sayıda elektrona ve\ + \ elektron dizilimine sahiptir. Bu durumda, Cu^{2+} iyonu ve Co atomu izoelektronik değildir.\ + \ lll. Yarı dolu orbital sayıları eşittir. Bu yanlış, çünkü Cu^{2+} yalnızca bir yarı dolu 3d orbitaline (3d^9) sahipken,\ + \ Co atomunda birden fazla yarı dolu 3d orbitali bulunabilir.\ + \ Dolayısıyla, ll ve lll yargıları yanlıştır. Doğru cevap B şıkkıdır.\n\nSoru:\ + \ Elektrolitik metal kaplama, elektrokimyasal metotlar kullanılarak yüzeyde metalik film oluşturulmasıdır.\ + \ Bu işlem malzemenin korozyondan korunması, güzel görünmesi ve yüksek sertlik kazanması gibi amaçlarla uygulanır.\ + \ Aşağıda verilenlerden hangisinde bu yöntem kullanılmamıştır?\nA) Bakır kapların\ + \ kalaylanması.\nB) Araba jantlarının nikel ile kaplanması.\nC) Çeliğin ısıya ve korozyona karşı dayanıklılığını artırmak\ + \ için alüminyum ile kaplanması.\nD) Demir tavanın yüzeyinin teflon ile kaplanması.\nE) Kol saatinin altın\ + \ ile kaplanması.Çözüm: Adım adım düşünelim. Elektrolitik metal kaplama yöntemi, bir metalin başka bir metal ile kaplanmasını içerir.\ + \ Bu yöntem, genellikle elektrokimyasal süreçler kullanılarak yapılır ve metal yüzeyinin korunmasını,\ + \ estetik görünüm kazanmasını veya belirli özellikler elde etmesini amaçlar. Seçenekler arasında bahsedilen\ + \ uygulamalardan Teflon ile kaplama işlemi, elektrolitik bir süreç değildir. Teflon (PTFE),\ + \ metal bir malzeme değil, sıcaklığa ve kimyasallara karşı dirençli bir polimer türüdür. Demir tavanın\ + \ yüzeyinin Teflon ile kaplanması, elektrokimyasal bir yöntem değil, daha ziyade fiziksel veya kimyasal\ + \ buhar biriktirme gibi farklı bir süreç gerektirir.\ + \ Doğru cevap D şıkkıdır.\n\nSoru: Laboratuvarda deney yapan beş arkadaş ile ilgili aşağıdaki bilgiler veriliyor.\ + \ Hasan: Asitler ile çalışılan bir deneyde asitin üzerine su ekliyor.\ + \ İnci: Deney sonrası malzemeleri tezgâhın üzerinde bırakıp deney raporunu yazmaya koyuluyor.\ + \ Nihal: Çözeltinin çökelmesini beklemek için yan masadaki arkadaşlarıyla sohbet ediyor.\ + \ Gülsüm: Kimyasal malzemeler ile çalışırken eldiven giymiyor ve aynı bagetle iki farklı çözeltiyi karıştırıyor.\ + \ Cenk: Sodyum metalinin su ile tepkimesini yaparken gözlerine koruyucu gözlük takıyor.\ + \ Yukarıda verilen bilgilere göre hangi öğrencilerin laboratuvardaki davranışları doğrudur?\nA) Hasan ve Gülsüm\nB) Nihal,\ + \ Gülsüm ve İnci\nC) Hasan ve İnci\nD) Nihal\nE) Cenk\nÇözüm: Adım adım düşünelim.\ + \ Laboratuvarda güvenli çalışmanın bazı kuralları vardır, bu kurallar tehlikeleri azaltmak\ + \ ve güvenli bir çalışma ortamı sağlamak için önemlidir. Öğrencilerin davranışlarını bu kurallar ışığında değerlendirelim:\ + \ - Hasan, asitlerle çalışırken asidin üzerine su ekliyor. Bu işlem çok tehlikelidir çünkü asidin üzerine su eklemek\ + \ yanma ya da patlamaya sebep olabilir. Doğru yöntem, suyun üzerine yavaşça asit eklemektir.\ + \ - İnci, deney sonrası malzemeleri tezgâhın üzerinde bırakıyor ve deney raporuna geçiyor. Laboratuvar güvenliği\ + \ açısından malzemelerin doğru şekilde temizlenmesi ve düzenlenmesi gerekmektedir. Bu yüzden İnci'nin davranışı yanlıştır.\ + \ - Nihal, çözeltinin çökelmesini beklerken yan masadaki arkadaşlarıyla sohbet ediyor. Deney süresince dikkatin\ + \ dağılmaması ve her zaman deney üzerinde kontrolün sağlanması önemli olduğundan, bu davranış doğru olmasa da,\ + \ direkt olarak güvenlikle ilişkili bir hata içermiyor.\ + \ - Gülsüm, kimyasal malzemeler ile çalışırken eldiven giymiyor ve aynı bağetle iki farklı çözeltiyi karıştırıyor.\ + \ Kimyasallarla çalışırken kişisel koruyucu ekipman kullanmak çok önemlidir, bu nedenle eldiven giymemek ve çapraz\ + \ kontaminasyona yol açacak şekilde aynı aracı kullanmak yanlıştır.\ + \ - Cenk, sodyum metalinin su ile tepkimesini yaparken gözlerine koruyucu gözlük takıyor. Bu, doğru bir davranıştır\ + \ çünkü bu tür bir deney sırasında gözleri korumak son derece önemlidir.\ + \ Buna göre, laboratuvardaki doğru davranışları sergileyen öğrenci Cenk'tir. Doğru cevap E şıkkıdır." +num_fewshot: 0 +dataset_name: Chemistry diff --git a/lm_eval/tasks/turkishmmlu/config_cot/Geography.yaml b/lm_eval/tasks/turkishmmlu/config_cot/Geography.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e940a306ac8ad26d231fd96e37b5d223d0de4f0f --- /dev/null +++ b/lm_eval/tasks/turkishmmlu/config_cot/Geography.yaml @@ -0,0 +1,52 @@ +include: _turkishmmlu_cot_default_yaml +task: turkishmmlu_cot_geography +description: + "Soru: Yerkabuğunu oluşturan levha sınırları ile aşağıdakilerden hangisinin dağılışı paralellik göstermez?\nA) Tektonik depremlerin\nB) Volkanik faaliyetlerin\nC) Kaplıcaların\nD)\ + \ Orman alanlarının\nE) Fay hatlarının\nÇözüm: Adım adım düşünelim.\ + \ Bu soruyu cevaplayabilmek için, levha sınırlarının özelliklerini ve levha hareketlerinin yeryüzündeki etkilerini bilmek gerekir.\ + \ A) Tektonik depremlerin dağılışı, levha sınırlarıyla paralellik gösterir. Levhaların birbirinden uzaklaştığı veya birbirine çarptığı yerlerde depremler meydana gelir.\ + \ B) Volkanik faaliyetlerin dağılışı, levha sınırlarıyla paralellik gösterir. Özellikle açılma (divergent) sınırlarında ve çarpışma (konverjan) sınırlarında volkanik faaliyetler görülür.\ + \ C) Kaplıcaların dağılışı, levha sınırlarıyla paralellik gösterir. Yerkabuğu inceldikçe ve magma yeryüzüne yaklaştıkça kaplıcalar oluşur. Bu da genellikle levha sınırlarında gerçekleşir.\ + \ D) Orman alanlarının dağılışı, levha sınırlarıyla paralellik göstermez. Orman alanlarının dağılışı iklim, toprak yapısı, yükselti gibi faktörlere bağlıdır, doğrudan levha sınırlarıyla ilişkili değildir.\ + \ E) Fay hatlarının dağılışı, levha sınırlarıyla paralellik gösterir. Faylar, levha hareketleri sonucu oluşur ve levha sınırlarına paralel uzanırlar.\ + \ Dolayısıyla, verilen seçeneklerden sadece D şıkkı \"Orman alanlarının\" dağılışı, levha sınırlarıyla paralellik göstermez.\ + \ Doğru cevap D şıkkıdır.\n\nSoru: Geçmişten günümüze değişen iklim koşullarına rağmen, kendi yaşam bölgelerinde varlıklarını\ + \ devam ettiren kalıntı bitkilere..........denir. \ + \ Yukarıdaki boşluğa aşağıdakilerden hangisi getirilirse cümle tamamlanmış olur?\nA)\ + \ Endemik\nB) Relikt\nC) Psödomaki\nD) Antropojen Bozkır\nE) Habitat\nÇözüm: Adım adım düşünelim.\ + \ Bu soruyu cevaplayabilmek için \"relikt\" kavramının ne anlama geldiğini bilmemiz gerekir.\ + \ Relikt bitkiler, geçmiş dönemlerdeki iklim koşullarına uyum sağlamış ve günümüze kadar varlıklarını sürdürebilmiş bitki türleridir.\ + \ Bu bitkiler, çevresel koşullar değişmiş olmasına rağmen özel alanlarda hayatta kalabilmişlerdir.\ + \ Seçeneklere baktığımızda, relikt bitkileri en iyi tanımlayan seçenek B şıkkındaki \"Relikt\" seçeneğidir.\ + \ Doğru cevap B şıkkıdır.\n\nSoru: I. Üretim sürecinde doğanın dengesini bozacak ve\ + \ insan sağlığına zararlı olabilecek kimyasal madde kullanmadan yapılan bir tarım şeklidir. II. Yağışın az, sulamanın yetersiz olduğu alanlarda tarlaların bir yıl ekilip, bir yıl dinlendirilmesi şeklinde uygulanan bir tarım yöntemidir. III. Kış sıcaklık ortalamasının 0 °C’nin altına inmediği yerlerde uygulanan tarım yöntemidir.\ + \ Bu yöntem tarım ürünlerinin mevsimi dışında yetiştirilmesine olanak sağlar. Yukarıda anlatılan tarımsal faaliyetler hangisinde doğru verilmiştir?\ + \ I II III \nA) Organik Nadas Seracılık\nB) Entansif Organik Ekstansif\nC) Seracılık Nadas Organik\nD) Ekstansif Nadas Seracılık\nE)\ + \ Organik Ekstansif Seracılık\nÇözüm: Adım adım düşünelim. Bu soruyu cevaplayabilmek için, verilen tarım yöntemlerinin özelliklerini bilmemiz gerekir.\ + \ Organik tarım, üretim sürecinde doğanın dengesini bozacak ve insan sağlığına zararlı olabilecek kimyasal madde kullanmadan yapılan bir tarım şeklidir.\ + \ Nadas, yağışın az, sulamanın yetersiz olduğu alanlarda tarlaların bir yıl ekilip, bir yıl dinlendirilmesi şeklinde uygulanan bir tarım yöntemidir.\ + \ Seracılık, kış sıcaklık ortalamasının 0 °C’nin altına inmediği yerlerde uygulanan tarım yöntemidir. Bu yöntem tarım ürünlerinin mevsimi dışında yetiştirilmesine olanak sağlar.\ + \ Dolayısıyla, verilen tarım yöntemlerinin doğru karşılıkları şu şekildedir: Organik - Nadas - Seracılık\ + \ Doğru cevap A şıkkıdır.\n\nSoru:\ + \ Yeşilırmak Havzası gelişim projesi ile aşağıdakilerden hangisinin gerçekleşmesi beklenmez?\nA)\ + \ Yeşilırmak'ın akım düzensizliğinden kaynaklanan sel, taşkın, erozyon, su ve çevre kirliliği sorunlarının çözülmesi\nB)\ + \ Otlak alanlarının iyileştirilmesi\nC) Düzenli kentleşmenin sağlanması\nD) Sanayileşmenin\ + \ gerçekleşmesi\nE) Bölgeden çıkarılan ham madde kaynaklarının pazarlanması\nÇözüm: Adım adım düşünelim.\ + \ Yeşilırmak Havzası Gelişim Projesi genellikle bölgedeki çevresel sorunların çözümü, tarım alanlarının geliştirilmesi, su kaynaklarının yönetimi gibi hedeflerle yapılan bir projedir.\ + \ A) Doğru: Projede sel, taşkın, erozyon gibi akım düzensizliğinden kaynaklanan sorunların çözülmesi hedeflenir.\ + \ B) Doğru: Projede otlak alanlarının iyileştirilmesi ve tarım alanlarının geliştirilmesi gibi doğal alanların rehabilitasyonu da önemli bir hedeftir.\ + \ C) Doğru: Projede düzenli kentleşme, altyapı geliştirme gibi şehir planlaması da dikkate alınır.\ + \ D) Doğru: Projenin amaçlarından biri sanayileşmede plansızlığın giderilmesi ve sanayinin gelişmesidir.\ + \ E) Yanlış: Projedenin amaçları arasında ham madde kaynaklarının pazarlanması yer almamaktadır.\ + \ Sonuç olarak, E) Bölgeden çıkarılan ham madde kaynaklarının pazarlanması beklenmez, çünkü projenin amaçları arasında yer almamaktadır.\ + \ Doğru cevap E şıkkıdır.\n\nSoru:\ + \ 1\/2.000.000 ölçekli bir haritada 4 cm olarak gösterilen A-B arasındaki uzaklık başka bir haritada 16 cm olarak gösterilmiştir.\ + \ Buna göre ikinci haritanın ölçeği aşağıdakilerden hangisidir?\nA) 1\/100.000\nB) 1\/200.000\nC) 1\/500.000\nD) 1\/1.500.000\nE)\ + \ 1\/8.000.000\nÇözüm: Adım adım düşünelim.\ + \ İlk haritada 4 cm olarak gösterilen A-B arasındaki uzaklık, ikinci haritada 16 cm olarak gösterilmiştir.\ + \ İlk haritada 4 cm olan uzaklık, ikinci haritada 16 cm olarak gösterildiğine göre, ikinci haritanın ölçeği 4 kat büyüktür.\ + \ İlk haritanın ölçeği 1/2.000.000 olduğuna göre, ikinci haritanın ölçeği 4 kat büyük olduğundan 1/500.000 olacaktır.\ + \ Dolayısıyla, ikinci haritanın ölçeği 1/500.000'dir.\ + \ Doğru cevap C şıkkıdır." +num_fewshot: 0 +dataset_name: Geography diff --git a/lm_eval/tasks/turkishmmlu/config_cot/History.yaml b/lm_eval/tasks/turkishmmlu/config_cot/History.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cbb53a5979eeed4e685c124792f4c8f620052d3f --- /dev/null +++ b/lm_eval/tasks/turkishmmlu/config_cot/History.yaml @@ -0,0 +1,72 @@ +include: _turkishmmlu_cot_default_yaml +task: turkishmmlu_cot_history +description: + "Soru: Denizcilikte öne çıkan ............. Beyliği Balıkesir bölgesinde kuruldu.\ + \ Osmanlı Devleti’ne .............. döneminde katıldı. Bu beyliğin Osmanlı Devletine katılmasıyla Osmanlı\ + \ Devleti’nde donanma gücü oluşturuldu. Yukarıdaki metinde boş bırakılan yerlere aşağıdakilerden hangisi getirilmelidir?\nA)\ + \ Karesioğulları - Orhan Bey\nB) Hamitoğulları - I. Murat (Hüdavendigar)\nC) Ramazanoğluları\ + \ - I. Selim (Yavuz)\nD) Karamanoğulları - II. Mehmet (Fatih)\nE) Candaroğulları - II. Mehmet (Fatih)\nÇözüm: Adım adım düşünelim.\ + \ Denizcilikte öne çıkan Karesi Beyliği Balıkesir bölgesinde kurulmuştur.\ + \ Osmanlı Devleti'ne katılması ise Orhan Bey döneminde gerçekleşmiştir.\ + \ Bu dönemde Karesi Beyliği'nin Osmanlı'ya katılmasıyla Osmanlı Devleti'nde donanma gücü oluşturulmuştur.\ + \ Doğru cevap A şıkkıdır.\n\nSoru: Osmanlı Devleti’nin kuruluşu ile ilgili birçok tez ortaya atılmıştır.\ + \ Bu tezler arasında birbiriyle benzer fikirler\ + \ olmakla beraber birbirinden oldukça farklı bakış açılarını temsil eden görüşler de ortaya atılmıştır.\ + \ Bu doğrultuda Tarihçi Colin İmber’in görüşleri “Osmanlıların kökenleri hakkındaki eski ya da çağdaş kuramların\ + \ hiçbiri kesinlikle kanıtlanamaz. Osman Gazi hakkındaki geleneksel hikâyelerin neredeyse tümü hayal ürünüdür.\ + \ Çağdaş bir tarihçinin yapabileceği en iyi şey, Osmanlı tarihinin başlangıcının bir kara delikten ibaret\ + \ olduğunu kabul etmek olacaktır.” şeklindedir. Ancak 15. yüzyılda Aşıkpaşazade Osmanlı Tarihi eserini nasıl yazdığını\ + \ şu cümlelerle ifade etmiştir: \"Kendimden önce yazılmış eserleri inceledim. Bu inceleme sonucunda\ + \ önceki dönemleri kaleme aldım.\" Buna göre, Tarihçi Colin İmber ve Aşıkpaşazade’nin görüşleri dikkate\ + \ alındığında aşağıdakilerden hangisine ulaşılamaz?\nA)\ + \ Tarihî meselelerde birbirine zıt görüşler ortaya atılmıştır.\nB) Colin İmber, Osmanlı kuruluş kaynaklarını yetersiz\ + \ görmüştür.\nC) Aşıkpaşazade eserini kaynaklara dayandırmıştır.\nD) Colin İmber, Osman Bey hakkındaki görüşleri\ + \ reddetmiştir.\nE) Osmanlı kuruluşu efsanelere dayanılarak yazılmıştır.\nÇözüm: Adım adım düşünelim.\ + \ Seçenekleri gözden geçirelim:\ + \ A) Tarihi meselelerde birbirine zıt görüşler ortaya atılmıştır.\ + \ - Bu doğru, çünkü Colin İmber ve Asıkpaşazade'nin görüşleri birbirine zıt gibi görünmektedir.\ + \ B) Colin İmber, Osmanlı kuruluş kaynaklarını yetersiz görmüştür. \ + \ - Doğru, çünkü İmber'e göre Osmanlı tarihindeki kaynaklar yetersiz ve hikayeler hayal ürünüdür.\ + \ C) Asıkpaşazade eserini kaynaklara dayandırmıştır. - Doğru, çünkü Asıkpaşazade, eserini kaynaklara dayandırdığını belirtmiştir.\ + \ D) Colin İmber, Osman Bey hakkındaki görüşleri reddetmiştir.\ + \ - Doğru, çünkü İmber, Osman Gazi hakkındaki geleneksel hikayelerin çoğunun hayal ürünü olduğunu belirtmiştir.\ + \ E) Osmanlı kuruluşu efsanelere dayanılarak yazılmıştır.\ + \ - Bu yanlıştır, çünkü Asıkpaşazade'nin eserini kaynaklara dayandırdığı\ + \ ve İmber'in görüşüne göre Osmanlı tarihinin başlangıcının kesin olarak bilinmediği belirtilmiştir.\ + \ Doğru cevap E şıkkıdır.\n\nSoru: Aşağıdakilerden hangisi İstanbul İsyanları’nın sonuçlarından biri değildir?\nA) İstanbul’da\ + \ asayiş ve kamu düzeni bozulmuştur.\nB) Yeniçerilerin devlet içindeki otoritesi artmıştır.\nC) Padişah ve devlet adamlarının\ + \ otoriteleri sarsılmıştır.\nD) Tarımsal üretim azalmış, topraklar boş kalmıştır.\nE) Halkın orduya olan güveni\ + \ azalmıştır.\nÇözüm: Adım adım düşünelim.\ + \ İstanbul İsyanları'nın sonuçlarından biri olarak verilen seçeneklerin hepsi olumlu ya da olumsuz sonuçlar içermektedir.\ + \ Ancak, D seçeneği olan \"Tarımsal üretim azalmış, topraklar boş kalmıştır\" \ + \ İstanbul İsyanları'nın sonuçlarından biri olarak kabul edilmemektedir. İstanbul İsyanları genellikle\ + \ askeri, siyasi ve sosyal sonuçlarla ilişkilendirilir, tarımsal üretimdeki azalma ve toprakların boş\ + \ kalması bu isyanlarla doğrudan bağlantılı değildir.\ + \ Doğru cevap D şıkkıdır.\n\nSoru: SSCB'nin 1991'de dağılmasından sonra Rusya'nın girişimiyle kurulan teşkilat\ + \ aşağıdakilerden hangisidir?\nA) Gümrük Birliği\nB) Avrupa Birliği\nC) Bağımsız Devletler Topluluğu\nD) Şhangay\ + \ Teşkilatı\nE) AGİT\nÇözüm: Adım adım düşünelim.\ + \ Seçenekleri gözden geçirelim: A) Gümrük Birliği - SSCB'nin dağılmasından sonra Rusya'nın girişimiyle kurulan bir teşkilat değildir.\ + \ B) Avrupa Birliği - SSCB'nin dağılmasıyla ilgili değildir ve Rusya'nın girişimiyle kurulmamıştır.\ + \ C) Bağımsız Devletler Topluluğu - SSCB'nin dağılmasından sonra Rusya'nın girişimiyle kurulan bir teşkilattır.\ + \ D) Şangay Teşkilatı - SSCB'nin dağılmasıyla ilgili değildir ve Rusya'nın girişimiyle kurulmamıştır.\ + \ E) AGİT (Avrupa Güvenlik ve İşbirliği Teşkilatı) - SSCB'nin dağılmasıyla ilgili değildir ve Rusya'nın girişimiyle kurulmamıştır.\ + \ Doğru cevap C şıkkıdır.\n\nSoru: Cüveynî “Tarih-i Cihangüşa” adlı eserinde Uygur bölgesini gezerken rastladığı garip\ + \ işaretlerle yazılmış taşlardan söz etmektedir.\ + \ Daha sonra Danimarkalı Messerschmidt’in (Mesırşmit) 1721’de bu eseri görerek Avrupa bilim camiasına tanıtması üzerine birçok bilim adamı\ + \ bu yazıların kime ait olduğu hakkında tahminlerde bulunmuştur. Bu bilim insanlarının bazıları bu yazıların; Prusyalılara, İskitlere,\ + \ Yunanlara ve Latinlere ait olduğunu ileri sürmüştür. Bölgeye giden birçok araştırmacı burada kazılar yaparak yeni kalıntılar tespit etmiştir.\ + \ Kitabelerin okunuşunu ilk çözen Danimarkalı W. Thomsen (V. Tomsen) olmuş, Türklerin yaşantılarına, töresine,\ + \ kültürüne ve devlet yönetimine dair bilgilerle birlikte bu abidelerin Türklere ait olduğunu tüm dünyaya duyurmuştur.\ + \ Metne göre, tarih biliminin aşağıdaki hangi bilim dalından yararlandığı söylenemez?\nA) Arkeoloji\nB)\ + \ Nümizmatik\nC) Epigrafi\nD) Paleografya\nE) Etnografya\nÇözüm: Adım adım düşünelim.\ + \ Metinde, Cuveyni'nin \"Tarih-i Cihanguşa\" adlı eserinde Uygur bölgesinde rastladığı garip işaretlerle yazılmış taşlardan bahsedilmektedir.\ + \ Kitabelerin okunuşunu ilk çözen Danimarkalı W. Thomsen, bu abidelerin Türklere ait olduğunu tüm dünyaya duyurmuştur.\ + \ Bu ifadelerde, epigrafi ve paleografi bilim dallarından bahsedilmektedir.\ + \ Yeni kalıntıların tespitiyle ilgili olarak arkeoloji bilim dalından da bahsedilmektedir.\ + \ Türklerin yaşantılarına, töresine, kültürüne ve devlet yönetimine dair bilgilerin elde edilmesi etnografya bilim dalıyla ilgilidir.\ + \ Nümizmatik, madeni paraları ve metal para basımı tarihini inceleyen bir bilim dalıdır.\ + \ Metinde bahsedilen taşlarla ilgili olarak nümizmatik biliminin bir katkısından bahsedilmemektedir.\ + \ Sonuç olarak, verilen metne göre tarih bilimi, nümizmatik bilim dalından yararlanmamıştır.\ + \ Doğru cevap B şıkkıdır." +num_fewshot: 0 +dataset_name: History diff --git a/lm_eval/tasks/turkishmmlu/config_cot/Mathematics.yaml b/lm_eval/tasks/turkishmmlu/config_cot/Mathematics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ff08a1273649a13fca71a3586832fa3f600b4833 --- /dev/null +++ b/lm_eval/tasks/turkishmmlu/config_cot/Mathematics.yaml @@ -0,0 +1,74 @@ +include: _turkishmmlu_cot_default_yaml +task: turkishmmlu_cot_mathematics +description: + "Soru: 3 farklı fizik ve 4 farklı kimya kitabı sıra halinde bir rafa dizilecektir. Fizik veya kimya kitaplarının\ + \ birbirinden ayrılmaması koşuluyla bu kitaplar kaç farklı şekilde dizilebilir\nA) 7.3!.4! \nB) 3.3!.4!\nC) 5.3!.4!\nD)\ + \ 6.3!.4!\nE) 2!.3!.4!\nÇözüm: Adım adım düşünelim. + \ Bu tür soruları çözerken, ilgili koşulları adım adım inceleyerek ilerlemek önemlidir. Bu\ + \ problemde iki önemli koşul var: fizik kitaplarının birbirinden ayrılmaması ve kimya\ + \ kitaplarının birbirinden ayrılmaması gerekmektedir. 1. Adım: Fizik ve kimya kitapları kendi arasında ayrılmaması gerektiği için, tüm fizik\ + \ kitaplarını tek bir grup olarak ve tüm kimya kitaplarını da tek bir grup olarak\ + \ düşünebiliriz. Bu durumda, sadece iki grup olduğu için bu iki grubu sıra halinde\ + \ dizmenin 2! farklı yolu vardır. 2. Adım: Fizik kitapları kendi içinde sıralanabilir. 3 farklı fizik kitabı olduğu için, bu\ + \ kitapları sıralamanın 3! farklı yolu vardır. 3. Adım: Kimya kitapları kendi içinde sıralanabilir. 4 farklı kimya kitabı olduğu için, bu\ + \ kitapları sıralamanın 4! farklı yolu vardır. 4. Adım: Bu durumda, tüm durumların toplam sayısını bulmak için bu\ + \ kombinasyonların çarpımını alırız: 2! . 3! . 4!.\ + \ Doğru cevap E şıkkıdır.\n\nSoru: $(p \\land q')' \\Rightarrow p$ koşullu önermesinin\ + \ karşıtı aşağıdakilerden hangisidir?\nA) $p \\Rightarrow (p \\Rightarrow q)$\nB) $p \\Rightarrow (p \\land q')$\nC)\ + \ $p' \\Rightarrow q$\nD) $q' \\lor p$\nE) $p' \\lor (q \\Rightarrow p)$\nÇözüm: Adım adım düşünelim.\ + \ Verilen koşullu önermenin karşıtını bulmak için, öncelikle verilen koşullu önermeyi ve\ + \ karşıtını gözden geçirelim.\ + \ Verilen koşullu önerme: $(p \\land q')' \\Rightarrow p$\ + \ Bir koşullu önermenin karşıtını bulmak için, koşullu önermenin solundaki ifade (hipotez) ile\ + \ sağdaki ifade (sonuç) yer değiştirilmelidir. Buna göre, verilen önermenin karşıtı:\ + \ $p \\Rightarrow (p \\land q')'$\ + \ $(p \\land q')'$ ifadesini açalım:\ + \ $(p \\land q')' = p' \\lor q$\ + \ $(p \\land q')' = $p \\Rightarrow q$\ + \ Bu durumda, verilen koşullu önermenin karşıtı: $p \\Rightarrow (p \\land q')'$\ + \ $p \\Rightarrow (p \\Rightarrow q)$ şeklinde de ifade edilebilir. \ + \ Doğru cevap A şıkkıdır.\n\nSoru: A oto kiralama firması kiralanacak bir araç için sabit 200 TL\ + \ ve her bir kilometre için 20 kuruş almaktadır. B oto kiralama firması ise kiralanacak bir araç için sabit 140 TL ve her bir kilometre için 25 kuruş almaktadır.\ + \ Buna göre kiralanan birer araçla kaç kilometre yol gidilirse iki firmaya da ödenecek ücret birbirine eşit olur\nA) 1150\nB)\ + \ 1200\nC) 1250\nD) 1300\nE) 1350\nÇözüm: Adım adım düşünelim. A ve B firmalarının uyguladığı ücret planını bir denklemle ifade edebiliriz.\ + \ A firması için toplam maliyet: 200 + 0.20k TL ('k' kilometre için)\ + \ B firması için toplam maliyet: 140 + 0.25k TL\ + \ Bu iki firmaya ödenecek ücretin birbirine eşit olduğu durumu bulmak için bu iki ifadeyi birbirine eşitleriz:\ + \ 200 + 0.20k = 140 + 0.25k\ + \ Bu denklemi çözmek için öncelikle her iki tarafından da '140' çıkarırız ve '0.20k' sapmasını bir tarafa alırız:\ + \ 60 = 0.05k\ + \ Buradan 'k' değerini bulmak için:\ + \ k = 60/0.05 = 1200\ + \ Yani, araçlarla 1200 kilometre yol gidildiğinde iki firmaya da ödenecek ücret birbirine eşit olur.\ + \ Bu nedenle doğru seçenek B: 1200 olacaktır. Doğru cevap B şıkkıdır.\n\nSoru:\ + \ Merkezinin koordinatları M(-1,2) ve yarıçap uzunluğu r=3 birim olan çemberin standart denklemi\ + \ aşağıdakilerden hangisidir?\nA) $\\(\\mathrm{\\left( x-1\\right) ^{2}+\\left( y-2\\right) ^{2}=9}\\)$\nB)\ + \ $\\(\\mathrm{\\left( x-1\\right) ^{2}-\\left( y-2\\right) ^{2}=9}\\)$\nC)\ + \ $\\(\\mathrm{\\left( x+1\\right) ^{2}+\\left( y-2\\right) ^{2}=9}\\)$\nD)\ + \ $\\(\\mathrm{\\left( x+1\\right) ^{2}+\\left( y+2\\right) ^{2}=9}\\)$\nE)\ + \ $\\(\\mathrm{\\left( x+1\\right) ^{2}-\\left( y+2\\right) ^{2}=9}\\)$\nÇözüm: Adım adım düşünelim.\ + \ Çemberin standart denklemi $(x - h)^2 + (y - k)^2 = r^2$ şeklinde ifade edilir, burada\ + \ $(h, k)$ çemberin merkezinin koordinatlarıdır ve $r$ çemberin yarıçapıdır.\ + \ Verilen soruda, çemberin merkezi $M(-1,2)$ ve yarıçapının uzunluğu $r=3$ birim\ + \ olarak belirtilmiştir. Bu değerleri standart denklemde yerine koyalım:\ + \ Merkez $(-1, 2)$ olduğu için, $h = -1$ ve $k = 2$ olur. Yarıçap $r = 3$ birimi olduğu\ + \ için denklem $r^2 = 9$ şekline dönüşür.\ + \ Bunu denkleme yerleştirirsek:\ + \ $(x - (-1))^2 + (y - 2)^2 = 3^2$.\ + \ Bu da şu şekilde basitleştirilir:\ + \ $(x + 1)^2 + (y - 2)^2 = 9$.\ + \ Doğru cevap C şıkkıdır.\n\nSoru:\ + \ $\\(\\large| -8-1\\large| +\\large|-1+6\\large| -\\large|4\\large|\\)$ işleminin sonucu kaçtır?\nA) 2\nB)\ + \ 4\nC) 9\nD) 10\nE) 18\nÇözüm: Adım adım düşünelim.\ + \ Mutlak değer ifadesindeki sayı negatif olsa dahi sonuç pozitif olacaktır. Soruda\ + \ verilen ifadeyi adım adım çözeceğiz:\ + \ $\\large| -8 - 1\\large| + \\large| -1 + 6\\large| - \\large| 4\\large|$\ + \ İlk adımda her bir mutlak değer içindeki işlemi yapalım:\ + \ - $\\large| -8 - 1\\large| = \\large| -9\\large| = 9$ (Mutlak değer içindeki sonuç her\ + \ zaman pozitiftir.)\ + \ - $\\large| -1 + 6\\large| = \\large| 5\\large| = 5$.\ + \ - $\\large| 4\\large| = 4$.\ + \ Böylece ifade şu şekle dönüşür: $9 + 5 - 4$. Sonuç: $9 + 5 - 4 = 10$\ + \ Bu nedenle işlemin sonucu 10'dur. Doğru cevap D şıkkıdır.\n\n" +num_fewshot: 0 +dataset_name: Mathematics diff --git a/lm_eval/tasks/turkishmmlu/config_cot/Philosophy.yaml b/lm_eval/tasks/turkishmmlu/config_cot/Philosophy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3ef5fd615fb7524534c9bf3bfe896efe72363ceb --- /dev/null +++ b/lm_eval/tasks/turkishmmlu/config_cot/Philosophy.yaml @@ -0,0 +1,45 @@ +include: _turkishmmlu_cot_default_yaml +task: turkishmmlu_cot_philosophy +description: + "Soru: Aşağıdaki ifadelerden hangisi Kant'ın epistemolojik görüşünü yansıtan bir yargıdır?\nA)\ + \ Bilginin kaynağı hem akıl hem deneyimdir.\nB) Doğru bilgi mümkün değildir.\nC) Sezgiye\ + \ dayalı bilgi gerçektir.\nD) Akıl tek başına bilgi elde etmek için yeterlidir.\nE)\ + \ Fenomenlerin bilgisi özseldir.\nÇözüm: Adım adım düşünelim.\ + \ Kant'ın epistemolojik görüşü, bilginin kaynağı olarak hem akıl (yani a priori bilgi) hem de deneyim (a posteriori bilgi) olduğunu öne sürer.\ + \ Doğru cevap A şıkkıdır.\n\nSoru: Bir argümanın ya da argümanı oluşturan önermelerin kendi içinde\ + \ veya birbiriyle uygun olması o argümanın ya daönermenin .................. olduğunu gösterir. Parçadaki boşluğa aşağıdaki\ + \ kavramlardan hangisi getirilmelidir?\nA) Çelişik\nB) Gerçek\nC) Doğru\nD)\ + \ Tutarlı\nE) Geçerli\nÇözüm: Adım adım düşünelim.\ + \ Boşluğa getirilmesi gereken kavram, bir argümanın veya argümanı oluşturan önermelerin kendi içinde veya\ + \ birbiriyle uygun olmasını ifade eden bir kavram olmalıdır. Bu durumda, argümanın veya önermenin tutarlı olduğunu gösterir.\ + \ Doğru cevap D şıkkıdır.\n\nSoru: Bir tek teoloji yoktur. Hemen her dinin kendi teolojisi vardır;\ + \ Hristiyan ve İslam teolojisi gibi. Bunlardan hiçbiri açıklamalarında ait oldukları dinin inanç esaslarının dışına çıkmaz.\ + \ Aşağıdakilerden hangisi bu durumun sebebi olamaz?\nA) Teolojilerin ait oldukları dini baştan doğru kabul\ + \ etmeleri\nB) Teolojilerin din felsefesi gibi dine eleştirel olarak yaklaşmaları\nC) Teolojilerin ait oldukları\ + \ dinin doğruluğunu ispat etme çabaları\nD) Teolojilerin belli bir dini görüşü savunmaya çalışmaları\nE)\ + \ Teolojilerin din felsefesi gibi dinlere tarafsız yaklaşmamaları\nÇözüm: Adım adım düşünelim.\ + \ Bu soruda, verilen ifadelerden hangisinin çeşitli dinlerin kendi teolojilerinin ait oldukları dinin inanç esaslarının dışına\ + \ çıkmadığı durumu açıklamada bir sebep olamayacağı sorulmaktadır. Teolojinin temel amacı, bir dinin inanç esaslarını açıklamak ve yorumlamaktır.\ + \ Dolayısıyla, teolojilerin ait oldukları dini baştan doğru kabul etmeleri A), belli bir dinin görüşünü savunmaya çalışmaları, D)\ + \ ait oldukları dinin doğruluğunu ispat etme çabaları C) ve din felsefesi gibi dinlere tarafsız yaklaşmamaları E) bu durumun sebepleri olabilir.\ + \ Ancak, din felsefesi gibi dine eleştirel olarak yaklaşmaları B) ise teolojik çalışmaların temel amaçlarından biri değildir.\ + \ Doğru cevap B şıkkıdır. \n\nSoru: Konfüçyüs’a göre “Zenginlik ve şeref herkesin istediği şeylerdir. Eğer bunlar doğru bir yolda\ + \ kazanılmazsa pek çabuk kaybedilir. Fakirlik ve düşkünlük insanların nefret ettiği şeylerdir. Eğer insanlar dürüst davranmazlarsa\ + \ kendilerini bunlardan sıyırmalarının imkânı yoktur.” Bu açıklama ile Konfüçyüs, felsefenin hangi konusuna vurgu\ + \ yapmıştır?\nA) Bilgi\nB) Spor\nC) Varlık\nD) Siyaset\nE) Ahlak\nÇözüm: Adım adım düşünelim.\ + \ Konfüçyüs'ün bu açıklamasında vurgu yaptığı temel konu, insan davranışlarının doğruluğuyla ilgilidir.\ + \ Konfüçyüs, zenginlik, şeref, fakirlik ve düşkünlük gibi dışsal değerlerin kazanılmasının doğru bir şekilde sağlanmaması durumunda\ + \ hızlıca kaybedilebileceğini, ancak insanların dürüst davranışlar sergilemediği sürece bu tür olumsuzluklardan kaçmanın mümkün olmadığını vurgular.\ + \ Dolayısıyla, Konfüçyüs'ün bu açıklamasıyla vurguladığı konu ahlaktır.\ Doğru cevap E şıkkıdır.\n\nSoru:\ + \ İnancın temele alındığı MS 2- MS 15. yüzyıl felsefesi, kendinden önceki dönem felsefelerinden etkilenmiştir.\ + \ Örneğin Hristiyan filozofları, Hristiyanlığı yaymak için diriliş inancını destekleyen ruhun ölmemesi hatta tekrar dirilmesi\ + \ fikrini savunan Antik Yunan filozofunun görüşlerinden; İslam filozofları ise Tanrı’nın varlığı kanıtlamalarında\ + \ başka bir Antik Yunan filozofunun geliştirdiği mantıktan yararlanmışlardır. Bu parçaya göre, MS 2- 15. yüzyıl\ + \ felsefesini etkileyen filozoflar sırasıyla aşağıdaki seçeneklerden hangisinde verilmiştir?\nA)\ + \ Epiküros, Aristoteles\nB) Epiküros, Platon\nC) Platon, Aristoteles\nD) Platon, Epiküros\nE)\ + \ Aristoteles, Platon\nÇözüm: Adım adım düşünelim.\ + \ Parçada belirtilenler doğrultusunda, Hristiyan filozoflarının diriliş inancını desteklemek için Antik Yunan filozoflarının ruhun ölmemesi\ + \ ve hatta tekrar dirilmesi fikrini savunan görüşlerinden etkilendiği ifade edilmiştir. Bu görüşleri savunan Antik Yunan filozofunun Platon olduğu bilinmektedir.\ + \ İslam filozoflarının ise Tanrı'nın varlığını kanıtlamada başka bir Antik Yunan filozofunun mantığından yararlandığı ifade edilmektedir\ + \ ve bu filozofun da Aristoteles olduğu bilinmektedir. Doğru cevap C şıkkıdır." +dataset_name: Philosophy diff --git a/lm_eval/tasks/turkishmmlu/config_cot/Physics.yaml b/lm_eval/tasks/turkishmmlu/config_cot/Physics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..25d8eff98ad1e9583e556972a574f6ecfca88c65 --- /dev/null +++ b/lm_eval/tasks/turkishmmlu/config_cot/Physics.yaml @@ -0,0 +1,106 @@ +include: _turkishmmlu_cot_default_yaml +task: turkishmmlu_cot_physics +description: + "Soru: 20°C sıcaklığında R direncine sahip silindirik bakır tel eritilerek boyu ilk boyunun iki katı kadar ve yine\ + \ silindirik olacak biçimde yeni bir kalıba dökülüyor. Bu durumda yeni telin direnci kaç R olur?\nA)\ + \ 4\nB) 2\nC) 1\nD) 1/2\nE) 1/4\nÇözüm: Adım adım düşünelim.\ + \ Yeni direncin değerini bulmak için öncelikle direncin nasıl değiştiğini anlamamız gerekiyor.\ + \ Direnç \\(R\\), iletkenin direncinin uzunluğu \\(l\\), kesit alanı \\(A\\) ve\ + \ malzemenin özdirenci (veya direnç katsayısı) \\(\\rho\\) ile doğrudan ilişkilidir ve\ + \ aşağıdaki formülle ifade edilir:\ + \ \\[R = \\rho \\frac{l}{A}\\]\ + \ Bu denklemde görüldüğü üzere, direnç iletkenin uzunluğu ile doğru orantılıyken,\ + \ kesit alanı ile ters orantılıdır. Soruda verilenler doğrultusunda, bakır telin uzunluğu iki\ + \ katına çıkarılıyor, bu durumda yeni uzunluğumuz \\(2l\\) olacaktır. Bakır telin hacmi\ + \ değişmediğinden ve şekli de yine silindirik olduğundan, bu durumda kesit alan da\ + \ değişmek zorundadır; çünkü uzunluk iki katına çıkartıldığında, hacmi korumak için\ + \ kesit alanın yarıya düşürülmesi gerekiyor. Yeni kesit alanımız \\(A/2\\) olacaktır.\ + \ Yeni direnç \\(R'\\) şu şekilde hesaplanır:\ + \ \\[R' = \\rho \\frac{2l}{A/2} = 4 \\times \\rho \\frac{l}{A}\\]\ + \ Bu durumda, yeni direnç \\(R'\\) orijinal direnç \\(R\\) ile karşılaştırıldığında 4 katına\ + \ çıkmış olur. Doğru cevap A şıkkıdır.\n\nSoru: Karayip resif mürekkep balığı suyun üstünde 2 m yüksekliğe çıkmak\ + \ ve vücut uzunluğunun 50 katı kadar bir mesafeye yani toplamda 10 m uçmak için tepkili itme kullanır.\ + \ Bunun için vücuduna düşük hızda aldığı akışkan ile kütlesini arttırıp, sonra bu akışkanı yüksek hızda dışarı atarak kütlesini azaltır. Mürekkep balığının kullandığı\ + \ bu yöntem roketleri fırlatmak için ihtiyaç duyulan itiş gücünün temel mantığını oluşturur. Örneğin yükünü\ + \ kaldırabilecek yeterli itiş kuvvetinin sağlanabilmesi için Atlas V roketinde, saniyede 1000 kg’dan fazla yakıt\ + \ tüketilmesi ve egzoz gazının saniyede 4000 m süratle dışarı atılması gerekmiştir. Yer çekimine karşı yapılan bu\ + \ fırlatma işlemi rokete uçuş süresi boyunca gerekli yakıtın sağlanması ve yer çekiminden kurtulmak için saatte 40000 km gibi bir sürate ulaşması\ + \ ile mümkün olabilmektedir. Aşağıdakilerden hangisinde ya da hangilerinde\ + \ yukarıdaki metinde geçen duruma örnek verilebilir?\nI. Şişirilen balonun serbest bırakılması. II. Ateşlenen top aracının geriye doğru hareket etmesi.\ + \ III. Havada patlayan havai fişeği parçalarının farklı yönlerde ilerlemesi. IV.\ + \ Hareketli bir bilardo topu ile duran bilardo topunun hareket ettirilmesi\nA) Yalnız I\nB)\ + \ Yalnız II\nC) I ve II\nD) II ve IV\nE) I, II ve IV\nÇözüm: Adım adım düşünelim.\ + \ Yukarıdaki metinde anlatılan durum, bir cismin içerisinden hızla dışarı atılan akışkan veya gaz sayesinde\ + \ ters yönde bir itiş gücü üretilmesini ve bu sayede ileri hareket edilmesini içerir. Bu durum, roketlerin fırlatılma\ + \ prensibi ve Karayip resif mürekkep balığının hareket mekanizması ile örneklendirilmiştir. Verilen seçenekler göz önünde\ + \ bulundurulduğunda: \ + \ I. Şişirilen balonun serbest bırakılması - Bu durum, balonun içindeki hava dışarıya hızla atıldığında balonun ters yönde\ + \ hareket etmesi prensibine dayanır, bu nedenle ilgili durumla benzerdir. \ + \ II. Ateşlenen top aracının geriye doğru hareket etmesi - Bu, Newton'un üçüncü yasası (her eyleme karşılık eşit ve zıt\ + \ yönde bir tepki olur) örneğidir ve roketin veya mürekkep balığının hareketine benzer bir durumdur. Ateşlenen top, bir yöne doğru\ + \ kuvvet uyguladığında, top aracı da zıt yönde bir kuvvetle hareket eder.\ + \ III. Havada patlayan havai fişeğin parçalarının farklı yönlere ilerlemesi - Bu, içeriden\ + \ dışarıya bir patlama etkisi ile nesnelerin her yöne dağılmasını içerir ve itme\ + \ kuvvetinin her yöne eşit dağıldığı bir durumu ifade eder. Bu örnek, direkt olarak\ + \ içeriden dışarıya hızlı bir akışkan veya gazın atılması ile ileri hareket etme prensibini\ + \ taşımaz.\ + \ IV. Hareketli bir bilardo topu ile duran bilardo topunun hareket ettirilmesi - Bu, iki katı\ + \ cismin çarpışması durumunu ve momentumun korunumu ilkesini içerir, direkt olarak\ + \ itiş kuvveti prensibi ile ilgili değildir.\ + \ Böylece, açıklanan prensiplere en çok uyan seçenek I. ve II. durumları içeren seçenek doğrudur.\ + \ Doğru cevap C şıkkıdır.\n\nSoru: Yay sabiti k olan yay sarkacı ve ip uzunluğu $\\(\\ell\\)$ olan basit sarkaçtan oluşturulan\ + \ sistemler yerçekimi ivmesinin daha büyük olduğu bir gezegene götürülürse periyotları nasıl değişir?\nYay Sarkacı Basit Sarkaç\nA) Değişmez Artar\nB) Değişmez Değişmez\nC) Azalır\ + \ Değişmez\nD) Artar Azalır\nE) Değişmez Azalır\nÇözüm: Adım adım düşünelim.\ + \ Yay sarkacının periyodu, yay sabiti \\(k\\) ve sarkacın kütlesi \\(m\\) ile ilişkili olup\ + \ \\(T = 2\\pi\\sqrt{\\frac{m}{k}}\\) formülüyle hesaplanır. Yerçekimi ivmesi (\\(g\\)) bu formülde yer almadığından, yay sarkacının\ + \ periyodu yerçekimi ivmesinin değerinden bağımsızdır ve değişmez. \ + \ Basit sarkacın periyodu ise \\(T = 2\\pi\\sqrt{\\frac{\\ell}{g}}\\) formülüyle hesaplanır. Bu formülde \\(\\ell\\) ipin uzunluğunu\ + \ ve \\(g\\) yerçekimi ivmesini temsil eder. Eğer yerçekimi ivmesi \\(g\\) büyükse, denklemin paydası büyüyeceğinden, periyodun\ + \ değeri azalır. Bu durumda, yay sarkacının periyodu değişmezken, basit sarkacın periyodu azalır.\ + \ Doğru cevap E şıkkıdır.\n\nSoru: 100 g kütleli bir kap özkütlesi 1 g/cm $^{3}$ olan su ile dolu iken kütlesi 350 g oluyor.\ + \ Buna göre, kap 0,5 g/cm$^{3 }$özkütleli sıvı ile doldurulursa kütlesi kaç g olur?\nA) 75\nB)\ + \ 125\nC) 175\nD) 225\nE) 250\nÇözüm: Adım adım düşünelim.\ + \ Kap ve suyun birleşik kütlesi 350 g olduğuna göre, sadece suyun kütlesini bulabilmek için kapın kütlesini bu değerden\ + \ çıkarmamız gerekiyor. Kap zaten boşken 100 g olduğuna göre: \ + \ Su + kap = 350 g\ + \ Su = 350 g - kap\ + \ Su = 350 g - 100 g\ + \ Su = 250 g\ + \ Bu, kapın içerisine koyduğumuz suyun kütlesinin 250 g olduğunu gösterir. Su için\ + \ verilen özkütlesi 1 g/cm$^{3}$ olduğuna göre ve kütlesi 250 g ise hacmi de 250\ + \ cm$^{3}$ olmalıdır (çünkü kütlesi özkütlesi ile çarpıldığında hacmini verir).\ + \ Şimdi, 0,5 g/cm$^{3}$ öz kütleli bir sıvı ile aynı kapı doldurursak, aynı hacmi (250\ + \ cm$^{3}$) dolduracağını varsayabiliriz. Bu sıvının kütlesini bulmak için hacmi öz\ + \ kütlesi ile çarpalım:\ + \ Kütle = özkütle * hacim\ + \ Kütle = 0,5 g/cm$^{3}$ * 250 cm$^{3}$\ + \ Kütle = 125 g\ + \ Bu yeni sıvının kütlesi 125 g olacaktır. Ancak bu sadece sıvının kütlesi. Toplam\ + \ kütleyi bulmak için kapın kütlesini (100 g) de eklememiz gerekiyor:\ + \ Toplam kütle = sıvı + kap\ + \ Toplam kütle = 125 g + 100 g\ + \ Toplam kütle = 225 g\ + \ Buna göre, kap 0,5 g/cm$^{3}$ özkütleli sıvı ile doldurulursa toplam kütlesi 225 g olur.\ + \ Doğru cevap D şıkkıdır.\n\nSoru: Sıcaklık için aşağıda verilenlerden hangisi doğrudur?\nA)\ + \ Bir enerji çeşitidir.\nB) Termometre ile ölçülür.\nC) Birimi\ + \ kaloridir.\nD) Madde miktarına bağlıdır.\nE) Birimi joule' dür.\nÇözüm: Adım adım düşünelim.\ + \ Sıcaklığın ne olduğunu ve nasıl ölçüldüğünü ele alarak, verilen seçenekleri inceleyelim:\ + \ A) \"Bir enerji çeşididir.\" - Bu ifade doğru değildir. Sıcaklık, enerjinin bir çeşidi değil,\ + \ malzemenin veya cismin ortalama kinetik enerjisinin bir ölçüsüdür. Dolayısıyla,\ + \ enerjinin bir türü olarak değil, enerjinin belli bir durumunu ifade eder.\ + \ B) \"Termometre ile ölçülür.\" - Bu ifade doğrudur. Sıcaklık termometre gibi özel\ + \ aygıtlar kullanılarak ölçülür. Bu aygıtlar, farklı sıcaklık değerlerinde farklı tepkiler verir\ + \ ve bu sayede sıcaklığın ne olduğunu belirleyebiliriz.\ + \ C) \"Birimi kaloridir.\" - Bu ifade yanlıştır. Kalori, enerji birimidir ve özellikle ısı\ + \ enerjisinin miktarını ölçmek için kullanılır. Sıcaklığın birimi ise derece Celsius (°C), derece\ + \ Fahrenheit (°F), veya Kelvin (K) gibi ölçü birimleridir.\ + \ D) \"Madde miktarına bağlıdır.\" - Bu ifade yanlıştır. Sıcaklık, bir maddenin miktarından\ + \ bağımsız, maddenin ortalama kinetik enerjisinin bir ölçüsüdür. Yani, madde miktarı\ + \ sıcaklığı doğrudan etkilemez; aynı sıcaklıktaki iki farklı miktar madde, biri fazla biri az\ + \ olsa dahi, aynı sıcaklığa sahip olabilir.\ + \ E) \"Birimi joule'dür.\" - Bu ifade de yanlıştır. Joule, enerjinin SI birimidir ve sıcaklığın\ + \ birimi değildir. Daha önce belirtildiği gibi, sıcaklığın birimi derece Celsius, derece\ + \ Fahrenheit veya Kelvin'dir.\ + \ Doğru cevap B şıkkıdır." +num_fewshot: 0 +dataset_name: Physics diff --git a/lm_eval/tasks/turkishmmlu/config_cot/Religion_and_Ethics.yaml b/lm_eval/tasks/turkishmmlu/config_cot/Religion_and_Ethics.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a81b9b803e81626fd6ff15bd56a40794247c443e --- /dev/null +++ b/lm_eval/tasks/turkishmmlu/config_cot/Religion_and_Ethics.yaml @@ -0,0 +1,51 @@ +include: _turkishmmlu_cot_default_yaml +task: turkishmmlu_cot_religion_and_ethics +description: + "Soru: (I) Allah’a (c.c.) inanmak insanın hayatında olumlu değişiklikler yapar. (II) Bunun nedeni imanın; duygu, düşünce ve davranışlar üzerinde etkili olmasıdır.\ + \ (III) İnsanın Allah (c.c.) ile irtibatı dua, ibadet, tövbe ve Kur’an okuma gibi yollarla gerçekleşir. (IV) İnsan büyük oranda inancına göre görür, düşünür ve davranışlar sergiler.\ + \ (V) Bu durumu açıklayan birçok ayet ve hadis bulunmaktadır. Bu parçadaki numaralanmış cümlelerden hangisi düşüncenin akışını bozmaktadır?\nA)\ + \ I\nB) II\nC) III\nD) IV\nE) V\nÇözüm: Adım adım düşünelim.\ + \ Metinde düşüncenin akışını bozan cümle C) III'dür. Çünkü diğer cümleler imanın insan üzerindeki etkilerine odaklanırken,\ + \ III numaralı cümle insanın Allah ile irtibatını nasıl gerçekleştirdiğine dair bir açıklama sunar, bu da ana fikrin akışını biraz keser. \ + \ Doğru cevap C şıkkıdır.\n\nSoru: İslam ahlakının kaynağı aşağıdakilerden hangisidir?\nA)\ + \ Kur’an ve sünnet\nB) Birey ve toplum\nC) Terbiye ve deneyim\nD)\ + \ Akıl ve düşünce\nE) Vicdan ve bilgi\nÇözüm: Adım adım düşünelim.\ + \ İslam ahlakının kaynağı \"Kur'an ve Sünnet\"tir. \ + \ Kur'an, İslam'ın kutsal kitabıdır ve peygamberin sözleri, eylemleri ve onayladıkları olan sünnet, İslam ahlakının temelini oluşturur.\ + \ Doğru cevap A şıkkıdır.\n\nSoru: Dinimizin ibadetler konusunda önem verdiği temel ilkelerden biri ihlastır.\ + \ Dolayısıyla İslami açıdan ibadetlerin, sadece Yüce Allah’ın (c.c.) rızasını kazanmak amacıyla yapılması esastır.\ + \ Mümin, her zaman bu bilinçte olmalıdır.\nİbadetlerini bu bilinçle, severek, isteyerek ve samimiyetle yapmalıdır.\ + \ Nitekim bizler her gün namazlarda okuduğumuz Fâtiha suresinde, ---- diyerek bunu ifade ediyoruz.\ + \ Verilen parçada boş bırakılan yere aşağıdaki ayetlerden hangisi getirilmelidir?\nA)\ + \ “Rahmân ve rahîm olan Allah’ın adıyla.” (Fâtiha suresi, 1. ayet.) \nB)\ + \ “Hamd (övme ve övülme), âlemlerin Rabbi Allah’a mahsustur. ” (Fâtiha suresi, 2. ayet.) \nC)\ + \ “O, rahmândır ve rahîmdir. Ceza gününün malikidir. ” (Fâtiha suresi, 3-4. ayetler.) \nD)\ + \ “Yalnız sana ibadet eder ve yalnız senden yardım dileriz.” (Fâtiha suresi, 5. ayet.) \nE)\ + \ “Bize doğru yolu göster. ” (Fâtiha suresi, 6. ayet.)\nÇözüm: Adım adım düşünelim. \ + \ Verilen parçada ifade edildiği gibi, İslami ibadetlerin temel amacı Allah'ın rızasını kazanmaktır ve bu amaçla ibadetlerin ihlas ile yapılması önemlidir. Namazlarda okunan Fatiha suresinde bu bilinci ifade eden bir ayet bulunmaktadır.\ + \ Fatiha suresinin ayetleri şunlardır:\ + \ Rahmân ve Rahîm olan Allah'ın adıyla.\ + \ Hamd (övme ve övülme) âlemlerin Rabbi Allah'a mahsustur.\ + \ O, Rahmân'dır ve Rahîm'dir.\ + \ Ceza gününün malikidir. \ + \ Yalnız sana ibadet eder ve yalnız senden yardım dileriz.\ + \ Bize doğru yolu göster.\ + \ İbadetlerin ihlas ile yapılması ve sadece Allah'ın rızasını gözeterek gerçekleştirilmesi kavramını ifade eden ayet ise D) Yalnız sana ibadet eder ve yalnız senden yardım dileriz. şeklindeki 5. ayettir.\ + \ Doğru cevap D şıkkıdır.\n\nSoru: Müminler ancak kardeştirler. Öyleyse kardeşlerinizin arasını düzeltin.\ + \ Allah’a karşı gelmekten sakının ki size merhamet edilsin.” (Hucurât suresi, 10. ayet.) Ayetin asıl konusu nedir?\nA)\ + \ Adalet\nB) Kardeşlik\nC) İtaat\nD) Bağışlama\nE) Sorumluluk\nÇözüm: Adım adım düşünelim.\ + \ Bu ayetin asıl konusu \"Kardeşlik\"tir. Çünkü ayette müminlerin birbirlerine kardeş oldukları\ + \ ve aralarındaki ilişkilerin düzeltilmesi gerektiği vurgulanır. Doğru cevap B şıkkıdır.\n\nSoru:\ + \ İslam medeniyeti, Emeviler döneminde oluşum sürecine girmiş, Abbasiler döneminde gelişmiş, Selçuklular döneminde bu gelişim devam etmiş, \ + \ Osmanlı döneminde ise kurum ve kuruluşlarıyla en parlak devrini yaşamıştır. İslam medeniyetinin mimari, edebiyat, sanat, matematik, astronomi, tıp vb. \ + \ alanlarda ortaya koyduğu örnekler oldukça fazladır. Farabi, İbn Sina, Biruni, Harezmî, İbn Rüşd, İbn Haldun gibi ilim adamları kendi dönemlerinde, felsefe, mantık, matematik,\ + \ tıp, astronomi gibi alanlarda tüm dünyaya zengin ilmi ürünler sunmuşlardır. Bu parçanın ana konusu aşağıdakilerden hangisidir?\nA)\ + \ Medeniyetin en önemli unsurları\nB) Medeniyete katkı sağlayan İslam âlimleri\nC)\ + \ İslam medeniyetiyle ortaya çıkan kurumlar\nD) İslam medeniyetinin en parlak dönemleri\nE)\ + \ İslam medeniyetinin oluşma süreci\nÇözüm: Adım adım düşünelim.\ + \ Bu parçanın ana konusu, İslam medeniyetinin gelişimini ve etkisini tanımlayan bir açıklamadır.\ + \ Parça, İslam medeniyetinin Emeviler, Abbasiler, Selçuklular ve Osmanlılar dönemlerindeki oluşumunu ve gelişimini kapsamlı bir şekilde ele alırken,\ + \ İslam medeniyetinin farklı alanlardaki önemli katkılarını ve ilim adamlarının rollerini vurgular.\ + \ Doğru cevap E şıkkıdır." +dataset_name: Religion_and_Ethics +num_fewshot: 0 diff --git a/lm_eval/tasks/turkishmmlu/config_cot/Turkish_Language_and_Literature.yaml b/lm_eval/tasks/turkishmmlu/config_cot/Turkish_Language_and_Literature.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2e1ee51ae39475693de41fff1a0da13cb10ac466 --- /dev/null +++ b/lm_eval/tasks/turkishmmlu/config_cot/Turkish_Language_and_Literature.yaml @@ -0,0 +1,59 @@ +include: _turkishmmlu_cot_default_yaml +task: turkishmmlu_cot_turkish_language_and_literature +description: + "Soru: Aşağıdakilerden hangisi edebiyat tarihinin konuları arasında yer almaz?\nA) Edebî dönemler\nB)\ + \ Şair ve yazarların edebî kişiliği\nC) Şair ve yazarların hayatları\nD) Edebî eserler\nE)\ + \ Arkeolojik eserler\nÇözüm: Adım adım düşünelim.\ + \ Edebiyat tarihinin konularını gözden geçirelim:\ + \ A) Edebî dönemler - Edebiyat tarihi, farklı edebi dönemleri ve bu dönemlerdeki edebi akımları inceler.\ + \ B) Şair ve yazarların edebî kişiliği - Edebiyat tarihi, şair ve yazarların eserlerindeki temaları, tarzları ve edebi kişiliklerini de ele alır.\ + \ C) Şair ve yazarların hayatları - Edebiyat tarihi, şair ve yazarların biyografilerini ve hayat hikayelerini de inceler.\ + \ D) Edebî eserler - Edebiyat tarihi, farklı edebi eserleri, bu eserlerin türlerini, temalarını ve döneminin özelliklerini inceler.\ + \ E) Arkeolojik eserler - Edebiyat tarihi, genellikle arkeolojik eserlerle ilgilenmez. Arkeolojik eserler,\ + \ arkeoloji biliminin konusudur ve genellikle tarih ve kültürel antropoloji alanlarında incelenir.\ + \ Sonuç olarak, edebiyat tarihinin konuları arasında yer almayan (farklı bir alana ait olan) seçenek E) Arkeolojik eserlerdir.\ + \ Doğru cevap E şıkkıdır.\n\nSoru: Osmanlı Devleti'nin son döneminde ortaya çıkan düşünce akımlarının\ + \ temsilcileri hangi seçenekte doğru verilmiştir?\nA) İslamcılık: Sait Halim Paşa, Mehmet Emin Yurdakul\nB)\ + \ Türkçülük: Ziya Gökalp, Ömer Seyfettin\nC) Batıcılık: Tevfik Fikret, Eşref Edip\nD) Osmanlıcılık:\ + \ Mehmet Akif Ersoy, Sait Halim Paşa\nE) Medeniyetçilik: Münif Paşa, Ali Canip Yöntem\nÇözüm: Adım adım düşünelim.\ + \ Osmanlı Devleti'nin son döneminde ortaya çıkan düşünce akımlarının temsilcilerini inceleyelim:\ + \ A) İslamcılık: Sait Halim Paşa, Mehmet Emin Yurdakul - İslamcılık akımının temsilcileri arasında Sait Halim Paşa ve Mehmet Emin Yurdakul yer almamaktadır.\ + \ B) Türkçülük: Ziya Gökalp, Ömer Seyfettin - Türkçülük akımının önde gelen temsilcileri arasında Ziya Gökalp ve Ömer Seyfettin bulunmaktadır. Bu seçenek doğru görünmektedir.\ + \ C) Batıcılık: Tevfik Fikret, Eşref Edip - Batıcılık akımının temsilcileri arasında Tevfik Fikret ve Eşref Edip bulunmamaktadır.\ + \ D) Osmanlıcılık: Mehmet Akif Ersoy, Sait Halim Paşa - Osmanlıcılık akımının temsilcileri arasında Mehmet Akif Ersoy ve Sait Halim Paşa bulunmamaktadır.\ + \ E) Medeniyetçilik: Münif Paşa, Ali Canip Yöntem - Medeniyetçilik akımının temsilcileri arasında Münif Paşa ve Ali Canip Yöntem bulunmamaktadır.\ + \ Doğru cevap B şıkkıdır.\n\nSoru: Birçok edebiyatçı ve eleştirmene göre şairlik yeteneği Tevfik Fikret'ten bile daha ileridedir.\ + \ Fakat siyasi görüşleri daima sanatının önünde yer almıştır.\ + \ Millî Mücadele Dönemi'nde Anadolu hareketini eleştirmiş, saltanat yanlısı görüşler ortaya koymuştur. Cumhuriyet Dönemi'nde de siyasi fikirleri yüzünden çokça eleştirilmiştir.\ + \ Parçada bahsedilen Servetifünun şairi aşağıdakilerden hangisidir?\nA) Süleyman Nazif\nB) Hüseyin Cahit Yalçın\nC) Mehmet Rauf\nD) Cenab\ + \ Şahabettin\nE) Halit Ziya Uşaklıgil\nÇözüm: Adım adım düşünelim.\ + \ Parçada bahsedilen şairin siyasi görüşlerinin sanatından daha önde olduğu ve Milli Mücadele Dönemi'nde Anadolu hareketini eleştirdiği,\ + \ saltanat yanlısı görüşler ortaya koyduğu ve Cumhuriyet Dönemi'nde de siyasi fikirleri nedeniyle eleştirildiği belirtilmektedir.\ + \ Bu özellikler ışığında, Servet-i Fünun döneminin önemli şairlerinden olan Cenab Şahabettin'i işaretlemek gerekir.\ + \ Doğru cevap D şıkkıdır.\n\nSoru: Selime’nin odasında biraz kaldıktan sonra otelden çıktım ( ) kırlara doğru yürüdüm ( )\ + \ Tozlu bir yol ( ) İki yanında fırınlar ( ) kahveler ( ) büyük ambarlar var ( )\ + \ Bu parçada yay ayraçlarla gösterilen yerlere aşağıdaki noktalama işaretlerinden hangisinde\ + \ verilenler getirilmelidir?\nA) (,) (.) (...) (,) (,) (.)\nB) (,) (!) (.) (,) (;) (...)\nC)\ + \ (...) (.) (...) (,) (,) (.)\nD) (,) (.) (.) (,) (,) (.)\nE) (,) (.) (;) (,) (,) (.)\nÇözüm: Adım adım düşünelim.\ + \ Parçada yay ayraçlarla gösterilen yerlere uygun noktalama işaretlerini belirlememiz gerekiyor.\ + \ Parçada yer alan noktalama işaretlerini dikkate alarak, uygun noktalama işaretlerini belirleyelim:\ + \ Selime’nin odasında biraz kaldıktan sonra otelden çıktım (,) kırlara doğru yürüdüm (.)\ + \ Tozlu bir yol (...) İki yanında fırınlar (,) kahveler (,) büyük ambarlar var (.)\ + \ Parçada yer alan noktalama işaretlerine göre, doğru sıralama şu şekildedir: (,) (.) (...) (,) (,) (.)\ + \ Doğru cevap A şıkkıdır.\n\nSoru: HOROZLA İNCİ Bir horoz inci bulur, kuyumcuya gider: “Al, şuna bak, der, pırıl pırıl,\ + \ ne özrü ne kusuru var. Fakat sen bana bir avuç mısır ver, benim işime o yarar.”\ + \ Bir cahile bir kitap miras kalır. Adam kitabı alır, komşusu kitapçıya gider:\ + \ “Bak, ne güzel kitap, der, fakat sen bana beş on kuruş ver, benim işime o yarar.”\ + \ Nâzım Hikmet , La Fontaine’den Masallar Bu fablda; I. Teşhis ve intak sanatlarından yararlanıldığı,\ + \ II. Fablın kahramanlarından birinin hayvan olduğu, III. Fablda zamanın belli olduğu bilgilerinden hangilerine\ + \ ulaşılabilir?\nA) Yalnız I\nB) Yalnız II\nC) I ve II\nD) I ve III\nE) II ve III\nÇözüm: Adım adım düşünelim. + \ Verilen metinde, bir horoz ve bir cahil (insan) arasında bir benzetme yapılmıştır. Bu fabldan hangi bilgilere ulaşılabilir gözden geçirelim:\ + \ I. Teşhis ve intak sanatlarından yararlanıldığı - Metinde, horozun inciyi bulduktan sonra kuyumcuya gidip onun değerini belirlemesi anlatılmaktadır.\ + \ Bu, bir tür değerlendirme ve tanımlama sürecidir. Dolayısıyla, I doğrudur.\ + \ II. Fablın kahramanlarından birinin hayvan olduğu - Metinde, horozun inci bulduğu ve cahilin bir kitap bulduğu anlatılmaktadır.\ + \ Horoz bir hayvandır, dolayısıyla fablın bir hayvan karakterine sahip olduğunu söyleyebiliriz. II doğrudur.\ + \ III. Fablda zamanın belli olduğu - Metinde, zaman belirtilmemiştir. Dolayısıyla, III hakkında bir bilgiye ulaşılamaz.\ + \ Sonuç olarak, I ve II doğru yanıtlardır, ancak III hakkında bir bilgiye ulaşılamaz.\ + \ Doğru cevap D şıkkıdır." +num_fewshot: 0 +dataset_name: Turkish_Language_and_Literature diff --git a/lm_eval/tasks/turkishmmlu/config_cot/_turkishmmlu_cot_default_yaml b/lm_eval/tasks/turkishmmlu/config_cot/_turkishmmlu_cot_default_yaml new file mode 100644 index 0000000000000000000000000000000000000000..03986c163be12ab79c7497f94d6f3625499a7af7 --- /dev/null +++ b/lm_eval/tasks/turkishmmlu/config_cot/_turkishmmlu_cot_default_yaml @@ -0,0 +1,34 @@ +tag: turkishmmlu_cot +task: null +dataset_path: AYueksel/TurkishMMLU +dataset_name: TurkishMMLU +test_split: test +fewshot_split: dev +output_type: generate_until +doc_to_text: "\n\nSoru: {{ question.strip() }}\nA) {{ choices[0] }}\nB) {{ choices[1] }}\nC) {{ choices[2] }}\nD) {{ choices[3] }}\nE) {{ choices[4] }}\nÇözüm: Adım adım düşünelim." +doc_to_target: "{{ answer }}" +filter_list: + - name: "get-answer" + filter: + - function: "regex" + regex_pattern: "([A-E])(?= şıkkıdır| şıkkı)" + - function: "take_first" + - name: "get-choice" + filter: + - function: "regex" + regex_pattern: "([A-E])" + - function: "take_first" +generation_kwargs: + until: + - "" + do_sample: false + temperature: 0.0 +num_fewshot: 0 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + ignore_case: true + ignore_punctuation: true +metadata: + version: 0.0 diff --git a/lm_eval/tasks/unitxt/20_newsgroups.yaml b/lm_eval/tasks/unitxt/20_newsgroups.yaml index 1a5af68a3108c963cf4d40aea627f7e8206f6c8c..f2444bd24f9133737df0e9dfaa31b8755ffbd94f 100644 --- a/lm_eval/tasks/unitxt/20_newsgroups.yaml +++ b/lm_eval/tasks/unitxt/20_newsgroups.yaml @@ -1,3 +1,3 @@ -include: unitxt_tasks.classification.multi_class task: 20_newsgroups -dataset_name: card=cards.20_newsgroups,template=templates.classification.multi_class.title +include: unitxt +recipe: card=cards.20_newsgroups,template=templates.classification.multi_class.title diff --git a/lm_eval/tasks/unitxt/README.md b/lm_eval/tasks/unitxt/README.md index 1cfc850834c2255ea9bc1b8f08113b3e26fa42d2..06bee13380bb01107e1b98e6253ea6655f1b6499 100644 --- a/lm_eval/tasks/unitxt/README.md +++ b/lm_eval/tasks/unitxt/README.md @@ -1,15 +1,17 @@ # Unitxt +Unitxt is a library for customizable textual data preparation and evaluation tailored to generative language models. Unitxt natively integrates with common libraries like HuggingFace and LM-eval-harness and deconstructs processing flows into modular components, enabling easy customization and sharing between practitioners. These components encompass model-specific formats, task prompts, and many other comprehensive dataset processing definitions. These components are centralized in the Unitxt-Catalog, thus fostering collaboration and exploration in modern textual data workflows. + +The full Unitxt catalog can be viewed in an [online explorer](https://unitxt.readthedocs.io/en/latest/docs/demo.html). + +Read more about Unitxt at [www.unitxt.ai](https://www.unitxt.ai/). + ### Paper Title: `Unitxt: Flexible, Shareable and Reusable Data Preparation and Evaluation for Generative AI` -Abstract: `https://arxiv.org/abs/2401.14019` - -Unitxt is a library for customizable textual data preparation and evaluation tailored to generative language models. Unitxt natively integrates with common libraries like HuggingFace and LM-eval-harness and deconstructs processing flows into modular components, enabling easy customization and sharing between practitioners. These components encompass model-specific formats, task prompts, and many other comprehensive dataset processing definitions. These components are centralized in the Unitxt-Catalog, thus fostering collaboration and exploration in modern textual data workflows. +Abstract: [link](https://arxiv.org/abs/2401.14019) -The full Unitxt catalog can be viewed in an online explorer. `https://unitxt.readthedocs.io/en/latest/docs/demo.html` -Homepage: https://unitxt.readthedocs.io/en/latest/index.html ### Citation @@ -36,42 +38,4 @@ The full list of Unitxt tasks currently supported can be seen under `tasks/unitx ### Adding tasks -You can add additional tasks from the Unitxt catalog by generating new LM-Eval yaml files for these datasets. - -The Unitxt task yaml files are generated via the `generate_yamls.py` script in the `tasks/unitxt` directory. - -To add a yaml file for an existing dataset Unitxt which is not yet in LM-Eval: -1. Add the card name to the `unitxt_datasets` file in the `tasks/unitxt` directory. -2. The generate_yaml.py contains the default Unitxt [template](https://unitxt.readthedocs.io/en/latest/docs/adding_template.html) used for each kind of NLP task in the `default_template_per_task` dictionary. If the dataset is of a Unitxt task type, previously not used in LM-Eval, you will need to add a default template for it in the dictionary. - -``` -default_template_per_task = { - "tasks.classification.multi_label" : "templates.classification.multi_label.title" , - "tasks.classification.multi_class" : "templates.classification.multi_class.title" , - "tasks.summarization.abstractive" : "templates.summarization.abstractive.full", - "tasks.regression.two_texts" : "templates.regression.two_texts.simple", - "tasks.qa.with_context.extractive" : "templates.qa.with_context.simple", - "tasks.grammatical_error_correction" : "templates.grammatical_error_correction.simple", - "tasks.span_labeling.extraction" : "templates.span_labeling.extraction.title" -} -``` -3. Run `python generate_yaml.py` (this will generate all the datasets listed in the `unitxt_datasets`) - -If you want to add a new dataset to the Unitxt catalog, see the Unitxt documentation: - -https://unitxt.readthedocs.io/en/latest/docs/adding_dataset.html - - - -### Checklist - -For adding novel benchmarks/datasets to the library: -* [ ] Is the task an existing benchmark in the literature? - * [ ] Have you referenced the original paper that introduced the task? - * [ ] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? - - -If other tasks on this dataset are already supported: -* [ ] Is the "Main" variant of this task clearly denoted? -* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates? -* [ ] Have you noted which, if any, published evaluation setups are matched by this variant? +See the [adding tasks guide](https://www.unitxt.ai/en/latest/docs/lm_eval.html#). diff --git a/lm_eval/tasks/unitxt/ag_news.yaml b/lm_eval/tasks/unitxt/ag_news.yaml index 32914bda93f5c4a4bd31153e0d547ef7cdecec29..792ce0b4b48ee8f986ac5207b2b5821cc0e34800 100644 --- a/lm_eval/tasks/unitxt/ag_news.yaml +++ b/lm_eval/tasks/unitxt/ag_news.yaml @@ -1,3 +1,3 @@ -include: unitxt_tasks.classification.multi_class task: ag_news -dataset_name: card=cards.ag_news,template=templates.classification.multi_class.title +include: unitxt +recipe: card=cards.ag_news,template=templates.classification.multi_class.title diff --git a/lm_eval/tasks/unitxt/argument_topic.yaml b/lm_eval/tasks/unitxt/argument_topic.yaml index c333e194d636d2f558a8f2d7d7e0508edf33fd96..d04810cd49f1a7bf2f344a2d30e1a1f4faa2deba 100644 --- a/lm_eval/tasks/unitxt/argument_topic.yaml +++ b/lm_eval/tasks/unitxt/argument_topic.yaml @@ -1,3 +1,3 @@ -include: unitxt_tasks.classification.multi_class task: argument_topic -dataset_name: card=cards.argument_topic,template=templates.classification.multi_class.title +include: unitxt +recipe: card=cards.argument_topic,template=templates.classification.multi_class.title diff --git a/lm_eval/tasks/unitxt/atis.yaml b/lm_eval/tasks/unitxt/atis.yaml index 1e7979be2f47e66e28f5841a3a045035e31adf8d..e9a26697accf1c623ac1cfbea228dda00167dc02 100644 --- a/lm_eval/tasks/unitxt/atis.yaml +++ b/lm_eval/tasks/unitxt/atis.yaml @@ -1,3 +1,3 @@ -include: unitxt_tasks.span_labeling.extraction task: atis -dataset_name: card=cards.atis,template=templates.span_labeling.extraction.title +include: unitxt +recipe: card=cards.atis,template=templates.span_labeling.extraction.title diff --git a/lm_eval/tasks/unitxt/banking77.yaml b/lm_eval/tasks/unitxt/banking77.yaml index a888e6a493f945d83c5e7c1f8f7f56b3301bc629..6475575dd82439d4180ffa7a7b93d54cf9d8006c 100644 --- a/lm_eval/tasks/unitxt/banking77.yaml +++ b/lm_eval/tasks/unitxt/banking77.yaml @@ -1,3 +1,3 @@ -include: unitxt_tasks.classification.multi_class task: banking77 -dataset_name: card=cards.banking77,template=templates.classification.multi_class.title +include: unitxt +recipe: card=cards.banking77,template=templates.classification.multi_class.title diff --git a/lm_eval/tasks/unitxt/claim_stance_topic.yaml b/lm_eval/tasks/unitxt/claim_stance_topic.yaml index 5f8a6133f65b4687e40b1241e3f4af3f688dcf00..2a2469d5ff78a6b5b4bc72ff6e867d94cf1ecee3 100644 --- a/lm_eval/tasks/unitxt/claim_stance_topic.yaml +++ b/lm_eval/tasks/unitxt/claim_stance_topic.yaml @@ -1,3 +1,3 @@ -include: unitxt_tasks.classification.multi_class task: claim_stance_topic -dataset_name: card=cards.claim_stance_topic,template=templates.classification.multi_class.title +include: unitxt +recipe: card=cards.claim_stance_topic,template=templates.classification.multi_class.title diff --git a/lm_eval/tasks/unitxt/cnn_dailymail.yaml b/lm_eval/tasks/unitxt/cnn_dailymail.yaml index f3f26cd5309e8d95ed445fbf9e2e0db17b6b76ff..aa3748c806824bbca8ae8db40f7112db3bd877f3 100644 --- a/lm_eval/tasks/unitxt/cnn_dailymail.yaml +++ b/lm_eval/tasks/unitxt/cnn_dailymail.yaml @@ -1,3 +1,3 @@ -include: unitxt_tasks.summarization.abstractive task: cnn_dailymail -dataset_name: card=cards.cnn_dailymail,template=templates.summarization.abstractive.full +include: unitxt +recipe: card=cards.cnn_dailymail,template=templates.summarization.abstractive.full diff --git a/lm_eval/tasks/unitxt/coedit_gec.yaml b/lm_eval/tasks/unitxt/coedit_gec.yaml index 619be722bfb70ff355097e38ffaad9e68b9d7523..4959064696816a22e7c084a45497b0670f796950 100644 --- a/lm_eval/tasks/unitxt/coedit_gec.yaml +++ b/lm_eval/tasks/unitxt/coedit_gec.yaml @@ -1,3 +1,3 @@ -include: unitxt_tasks.grammatical_error_correction task: coedit_gec -dataset_name: card=cards.coedit_gec,template=templates.grammatical_error_correction.simple +include: unitxt +recipe: card=cards.coedit_gec,template=templates.grammatical_error_correction.simple diff --git a/lm_eval/tasks/unitxt/dbpedia_14.yaml b/lm_eval/tasks/unitxt/dbpedia_14.yaml index 6ef2df71686c52db2f8e5a1ee942176b782eb54c..b26d65a72be22c2faae0080d4d5c223062e67d9a 100644 --- a/lm_eval/tasks/unitxt/dbpedia_14.yaml +++ b/lm_eval/tasks/unitxt/dbpedia_14.yaml @@ -1,3 +1,3 @@ -include: unitxt_tasks.classification.multi_class task: dbpedia_14 -dataset_name: card=cards.dbpedia_14,template=templates.classification.multi_class.title +include: unitxt +recipe: card=cards.dbpedia_14,template=templates.classification.multi_class.title diff --git a/lm_eval/tasks/unitxt/doc_vqa.yaml b/lm_eval/tasks/unitxt/doc_vqa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..cbf274d4bb2fbe01ffa59ea0a2ad50da02e6b0ac --- /dev/null +++ b/lm_eval/tasks/unitxt/doc_vqa.yaml @@ -0,0 +1,3 @@ +task: doc_vqa +include: unitxt_multimodal +recipe: card=cards.doc_vqa.en,template=templates.qa.with_context.title diff --git a/lm_eval/tasks/unitxt/ethos_binary.yaml b/lm_eval/tasks/unitxt/ethos_binary.yaml index 8a3b16701608fcab26d3322d10ea05b540464dbf..3976de43ace0048784a0c802777fd815976571ba 100644 --- a/lm_eval/tasks/unitxt/ethos_binary.yaml +++ b/lm_eval/tasks/unitxt/ethos_binary.yaml @@ -1,3 +1,3 @@ -include: unitxt_tasks.classification.multi_class task: ethos_binary -dataset_name: card=cards.ethos_binary,template=templates.classification.multi_class.title +include: unitxt +recipe: card=cards.ethos_binary,template=templates.classification.multi_class.title diff --git a/lm_eval/tasks/unitxt/financial_tweets.yaml b/lm_eval/tasks/unitxt/financial_tweets.yaml index 8d804e7adbd46ca0cdb4cc0ffd02e07b03c9c2d2..7b4bb9e538238b2bd4fe7d11c31389a11fadbe7a 100644 --- a/lm_eval/tasks/unitxt/financial_tweets.yaml +++ b/lm_eval/tasks/unitxt/financial_tweets.yaml @@ -1,3 +1,3 @@ -include: unitxt_tasks.classification.multi_class task: financial_tweets -dataset_name: card=cards.financial_tweets,template=templates.classification.multi_class.title +include: unitxt +recipe: card=cards.financial_tweets,template=templates.classification.multi_class.title diff --git a/lm_eval/tasks/unitxt/generate_yamls.py b/lm_eval/tasks/unitxt/generate_yamls.py deleted file mode 100644 index b1b9c004bb6009d4a6474f991860d9268a096f21..0000000000000000000000000000000000000000 --- a/lm_eval/tasks/unitxt/generate_yamls.py +++ /dev/null @@ -1,135 +0,0 @@ -# -# This file generates a set of LM eval harness yaml file -# that load unitxt datasets (https://github.com/IBM/unitxt) -# - -import unitxt_wrapper -import yaml -from unitxt.artifact import fetch_artifact -from unitxt.standard import StandardRecipe - - -# This code is required to properly dump LM harness YAML that contains references to functions -def function_representer(dumper: yaml.SafeDumper, func) -> yaml.nodes.MappingNode: - return dumper.represent_scalar( - "!function", f"{func.__module__}.{func.__name__}", style=None - ) - - -def write_task_yaml(filename, data): - yaml.add_representer(type(data["process_results"]), function_representer) - with open(filename, "w") as stream: - yaml.dump(data, stream, sort_keys=False) - - -def write_card_yaml(filename, data): - with open(filename, "w") as stream: - yaml.dump(data, stream, sort_keys=False) - - -default_template_per_task = { - "tasks.classification.multi_label": "templates.classification.multi_label.title", - "tasks.classification.multi_class": "templates.classification.multi_class.title", - "tasks.summarization.abstractive": "templates.summarization.abstractive.full", - "tasks.regression.two_texts": "templates.regression.two_texts.simple", - "tasks.qa.with_context.extractive": "templates.qa.with_context.simple", - "tasks.grammatical_error_correction": "templates.grammatical_error_correction.simple", - "tasks.span_labeling.extraction": "templates.span_labeling.extraction.title", -} - - -def generate_task_yaml(task: str): - """ - Generate an LM Eval Harness YAML file based on a Unitxt task defintion. - The output YAML is based on 'template.yaml.file' found in current directoy. - - The common template is filled the the specific metrics for the task. - It still leaves the 'dataset_name' and 'task name' unspecified. - """ - print("*" * 80) - print("*") - print(f"* Generating YAML base file for task {task}") - print("*") - task_definition, _ = fetch_artifact(task) - data = { - "group": ["unitxt"], - "dataset_path": "unitxt/data", - "output_type": "generate_until", - "training_split": "train", - "validation_split": "test", - "doc_to_text": "{{source}}", - "doc_to_target": "target", - "process_results": unitxt_wrapper.process_results, - "generation_kwargs": {"until": [""]}, - "metric_list": [], - "metadata": {"verison": 1.0}, - } - - for metric_name in task_definition.metrics: - new_metric = {"metric": "", "aggregation": "unitxt", "higher_is_better": True} - new_metric["metric"] = metric_name.replace("metrics.", "unitxt_") - data["metric_list"].append(new_metric) - - write_task_yaml(f"unitxt_{task}", data) - - -def generate_card_yaml(card: str): - """ - Generate an LM Eval Harness YAML file based on the Unitxt dataset card. - It includes the task YAML for the dataset, and overrides the 'dataset_name' and 'task' with the card. - """ - - print("*" * 80) - print("*") - print(f"* Generating YAML file for unitxt dataset {card}") - print("*") - - card_definition, _ = fetch_artifact(f"cards.{card}") - task = card_definition.task.__id__ - if task in default_template_per_task: - template = default_template_per_task[task] - else: - raise ValueError( - f"Default template was not defined for task {task} in 'default_template_per_task' dict in generate_yamls.py" - ) - data = {} - data["include"] = f"unitxt_{task}" - data["task"] = card - data["dataset_name"] = f"card=cards.{card},template={template}" - # This is faster that the load_dataset approach - # dataset = load_dataset('unitxt/data', data["dataset_name"]+",loader_limit=100",trust_remote_code=True) - recipe = StandardRecipe(card=f"cards.{card}", template=template, loader_limit=100) - stream = recipe() - dataset = stream.to_dataset() - print(dataset) - print("Sample input:") - print(dataset["test"][0]["source"]) - print("Sample output:") - print(dataset["test"][0]["target"]) - write_card_yaml(f"{card}.yaml", data) - - -def main(): - for task in default_template_per_task.keys(): - try: - generate_task_yaml(task) - except Exception as e: - print(f"Unable to generate YAML for {task} due to:") - print(e) - raise (e) - with open("unitxt_datasets") as f: - for unitxt_dataset in f: - unitxt_dataset = unitxt_dataset.strip() - if unitxt_dataset.startswith("### END ###"): - exit(0) - if not unitxt_dataset.startswith("#"): - try: - generate_card_yaml(unitxt_dataset) - except Exception as e: - print(f"Unable to generate YAML for {unitxt_dataset} due to:") - print(e) - raise e - - -if __name__ == "__main__": - main() diff --git a/lm_eval/tasks/unitxt/law_stack_exchange.yaml b/lm_eval/tasks/unitxt/law_stack_exchange.yaml index 780af103c8bd21d47187ba3e4ba5d6646abdb2c3..d0c589a3d69da65256799d9c6f15cd4a48a7fadd 100644 --- a/lm_eval/tasks/unitxt/law_stack_exchange.yaml +++ b/lm_eval/tasks/unitxt/law_stack_exchange.yaml @@ -1,3 +1,3 @@ -include: unitxt_tasks.classification.multi_class task: law_stack_exchange -dataset_name: card=cards.law_stack_exchange,template=templates.classification.multi_class.title +include: unitxt +recipe: card=cards.law_stack_exchange,template=templates.classification.multi_class.title diff --git a/lm_eval/tasks/unitxt/ledgar.yaml b/lm_eval/tasks/unitxt/ledgar.yaml index 6d93ef9d289f0720ec4a897e99d44d2d7f2635a9..1c31589764197998f0cc4bd89b256a9e7e83cd22 100644 --- a/lm_eval/tasks/unitxt/ledgar.yaml +++ b/lm_eval/tasks/unitxt/ledgar.yaml @@ -1,3 +1,3 @@ -include: unitxt_tasks.classification.multi_class task: ledgar -dataset_name: card=cards.ledgar,template=templates.classification.multi_class.title +include: unitxt +recipe: card=cards.ledgar,template=templates.classification.multi_class.title diff --git a/lm_eval/tasks/unitxt/medical_abstracts.yaml b/lm_eval/tasks/unitxt/medical_abstracts.yaml index dd4e87b688b359b75aac6b984916adc44022ad1e..74cfef0b685d5c4f583e379df107ed404ae81aed 100644 --- a/lm_eval/tasks/unitxt/medical_abstracts.yaml +++ b/lm_eval/tasks/unitxt/medical_abstracts.yaml @@ -1,3 +1,3 @@ -include: unitxt_tasks.classification.multi_class task: medical_abstracts -dataset_name: card=cards.medical_abstracts,template=templates.classification.multi_class.title +include: unitxt +recipe: card=cards.medical_abstracts,template=templates.classification.multi_class.title diff --git a/lm_eval/tasks/unitxt/stsb.yaml b/lm_eval/tasks/unitxt/stsb.yaml index 44e78c58484c9ec0306128b22d4746875f23569f..8d91b0e13c6a7327efd3c9efd36183ae87ef242c 100644 --- a/lm_eval/tasks/unitxt/stsb.yaml +++ b/lm_eval/tasks/unitxt/stsb.yaml @@ -1,3 +1,3 @@ -include: unitxt_tasks.regression.two_texts task: stsb -dataset_name: card=cards.stsb,template=templates.regression.two_texts.simple +include: unitxt +recipe: card=cards.stsb,template=templates.regression.two_texts.simple diff --git a/lm_eval/tasks/unitxt/task.py b/lm_eval/tasks/unitxt/task.py new file mode 100644 index 0000000000000000000000000000000000000000..12b2760864b16040dc69d606a1ac5d547dcea2f6 --- /dev/null +++ b/lm_eval/tasks/unitxt/task.py @@ -0,0 +1,195 @@ +""" +In the dynamic landscape of generative NLP, traditional text processing pipelines limit research flexibility and reproducibility, as they are tailored to specific dataset, task, and model combinations. The escalating complexity, involving system prompts, model-specific formats, instructions, and more, calls for a shift to a structured, modular, and customizable solution. + +Addressing this need, we present Unitxt, an innovative library for customizable textual data preparation and evaluation tailored to generative language models. Unitxt natively integrates with common libraries like HuggingFace and LM-eval-harness and deconstructs processing flows into modular components, enabling easy customization and sharing between practitioners. These components encompass model-specific formats, task prompts, and many other comprehensive dataset processing definitions. The Unitxt-Catalog centralizes these components, fostering collaboration and exploration in modern textual data workflows. Beyond being a tool, Unitxt is a community-driven platform, empowering users to build, share, and advance their pipelines collaboratively. +""" + +import importlib.util +import re +from functools import partial +from typing import Any, Dict, Optional + +import datasets +import evaluate + +from lm_eval.api.instance import Instance +from lm_eval.api.task import ConfigurableTask + + +_CITATION = """ +@misc{bandel2024unitxt, + title={Unitxt: Flexible, Shareable and Reusable Data Preparation and Evaluation for Generative AI}, + author={Elron Bandel and Yotam Perlitz and Elad Venezian and Roni Friedman-Melamed and Ofir Arviv and Matan Orbach and Shachar Don-Yehyia and Dafna Sheinwald and Ariel Gera and Leshem Choshen and Michal Shmueli-Scheuer and Yoav Katz}, + year={2024}, + eprint={2401.14019}, + archivePrefix={arXiv}, + primaryClass={cs.CL} +} +""" + + +def is_unitxt_installed() -> bool: + return importlib.util.find_spec("unitxt") is not None + + +def score(items, metric): + predictions, references = zip(*items) + evaluator = evaluate.load("unitxt/metric") + for reference in references: + reference["metrics"] = [metric] + results = evaluator.compute(predictions=predictions, references=references) + return results[0]["score"]["global"]["score"] + + +class Unitxt(ConfigurableTask): + VERSION = 0 + + def __init__( + self, + config: Optional[dict] = None, + ) -> None: + if config is None: + config = {} + assert "recipe" in config, "Unitxt task must have a 'recipe' string." + super().__init__( + config={ + "metadata": {"version": self.VERSION}, + "dataset_name": config["recipe"], + } + ) + self.image_decoder = datasets.Image() + self.metrics = self.dataset["test"][0]["metrics"] + + def download(self, dataset_kwargs: Optional[Dict[str, Any]] = None) -> None: + if is_unitxt_installed(): + from unitxt import load_dataset + + self.dataset = load_dataset(self.DATASET_NAME) + else: + self.dataset = datasets.load_dataset( + name=self.DATASET_NAME, + path="unitxt/data", + trust_remote_code=True, + ) + + def has_training_docs(self): + return "train" in self.dataset + + def has_validation_docs(self): + return "validation" in self.dataset + + def has_test_docs(self): + return "test" in self.dataset + + def training_docs(self): + return self.dataset["train"] + + def validation_docs(self): + return self.dataset["validation"] + + def test_docs(self): + return self.dataset["test"] + + def doc_to_text(self, doc): + return doc["source"] + + def should_decontaminate(self): + return False + + def doc_to_target(self, doc): + doc["target"] + + def get_arguments(self, doc, ctx): + return (ctx, {"until": ["\n"]}) + + def construct_requests(self, doc, ctx, **kwargs): + """Uses RequestFactory to construct Requests and returns an iterable of + Requests which will be sent to the LM. + + :param doc: + The document as returned from training_docs, validation_docs, or test_docs. + :param ctx: str + The context string, generated by fewshot_context. This includes the natural + language description, as well as the few shot examples, and the question + part of the document for `doc`. + """ + return [ + Instance( + request_type="generate_until", + doc=doc, + arguments=self.get_arguments(doc, ctx), + idx=0, + **kwargs, + ) + ] + + def process_results(self, doc, results): + """Take a single document and the LM results and evaluates, returning a + dict where keys are the names of submetrics and values are the values of + the metric for that one document + + :param doc: + The document as returned from training_docs, validation_docs, or test_docs. + :param results: + The results of the requests created in construct_requests. + """ + + continuation = results[0] + + predictions = continuation + + references = doc + return { + metric.replace("metrics.", ""): (predictions, references) + for metric in self.metrics + } + + def aggregation(self): + """ + :returns: {str: [float] -> float} + A dictionary where keys are the names of submetrics and values are + functions that aggregate a list of metrics + """ + return { + metric.replace("metrics.", ""): partial(score, metric=metric) + for metric in self.metrics + } + + def higher_is_better(self): + """ + :returns: {str: bool} + A dictionary where keys are the names of submetrics and values are + whether a higher value of the submetric is better + """ + return {metric.replace("metrics.", ""): True for metric in self.metrics} + + +images_regex = r'' +image_source_regex = r'", doc["source"]) + + def doc_to_image(self, doc): + images = extract_images(doc["source"], doc) + return [self.image_decoder.decode_example(image) for image in images] + + def get_arguments(self, doc, ctx): + return (ctx, {"until": ["\n"]}, {"visual": self.doc_to_image(doc)}) diff --git a/lm_eval/tasks/unitxt/unfair_tos.yaml b/lm_eval/tasks/unitxt/unfair_tos.yaml index 2c52c31655eef9a799048309f717967bc8dcb084..b401dfeff4d06d3b3c96b18f00ad211b4607b46e 100644 --- a/lm_eval/tasks/unitxt/unfair_tos.yaml +++ b/lm_eval/tasks/unitxt/unfair_tos.yaml @@ -1,3 +1,3 @@ -include: unitxt_tasks.classification.multi_label task: unfair_tos -dataset_name: card=cards.unfair_tos,template=templates.classification.multi_label.title +include: unitxt +recipe: card=cards.unfair_tos,template=templates.classification.multi_label.title diff --git a/lm_eval/tasks/unitxt/unitxt b/lm_eval/tasks/unitxt/unitxt new file mode 100644 index 0000000000000000000000000000000000000000..e6902c46d4a0342e10360715be125178ecd58aad --- /dev/null +++ b/lm_eval/tasks/unitxt/unitxt @@ -0,0 +1 @@ +class: !function task.Unitxt diff --git a/lm_eval/tasks/unitxt/unitxt_datasets b/lm_eval/tasks/unitxt/unitxt_datasets deleted file mode 100644 index cdafa0cf256979a37b726a8f10e1931960faa17d..0000000000000000000000000000000000000000 --- a/lm_eval/tasks/unitxt/unitxt_datasets +++ /dev/null @@ -1,18 +0,0 @@ -coedit_gec -atis -20_newsgroups -ag_news -argument_topic -banking77 -claim_stance_topic -cnn_dailymail -dbpedia_14 -ethos_binary -financial_tweets -law_stack_exchange -ledgar -medical_abstracts -stsb -unfair_tos -xsum -yahoo_answers_topics diff --git a/lm_eval/tasks/unitxt/unitxt_multimodal b/lm_eval/tasks/unitxt/unitxt_multimodal new file mode 100644 index 0000000000000000000000000000000000000000..fbcff691fb39bb3f89a58e7965d7193a7b8d442a --- /dev/null +++ b/lm_eval/tasks/unitxt/unitxt_multimodal @@ -0,0 +1 @@ +class: !function task.UnitxtMultiModal diff --git a/lm_eval/tasks/unitxt/unitxt_tasks.classification.multi_class b/lm_eval/tasks/unitxt/unitxt_tasks.classification.multi_class deleted file mode 100644 index 7e248b834e7fc41bed423119f325955cbc179d07..0000000000000000000000000000000000000000 --- a/lm_eval/tasks/unitxt/unitxt_tasks.classification.multi_class +++ /dev/null @@ -1,24 +0,0 @@ -group: -- unitxt -dataset_path: unitxt/data -output_type: generate_until -training_split: train -validation_split: test -doc_to_text: '{{source}}' -doc_to_target: target -process_results: !function 'unitxt_wrapper.process_results' -generation_kwargs: - until: - - -metric_list: -- metric: unitxt_f1_micro - aggregation: unitxt - higher_is_better: true -- metric: unitxt_accuracy - aggregation: unitxt - higher_is_better: true -- metric: unitxt_f1_macro - aggregation: unitxt - higher_is_better: true -metadata: - verison: 1.0 diff --git a/lm_eval/tasks/unitxt/unitxt_tasks.classification.multi_label b/lm_eval/tasks/unitxt/unitxt_tasks.classification.multi_label deleted file mode 100644 index 871ebda730349587a65143dd2e7a912753c40876..0000000000000000000000000000000000000000 --- a/lm_eval/tasks/unitxt/unitxt_tasks.classification.multi_label +++ /dev/null @@ -1,24 +0,0 @@ -group: -- unitxt -dataset_path: unitxt/data -output_type: generate_until -training_split: train -validation_split: test -doc_to_text: '{{source}}' -doc_to_target: target -process_results: !function 'unitxt_wrapper.process_results' -generation_kwargs: - until: - - -metric_list: -- metric: unitxt_f1_micro_multi_label - aggregation: unitxt - higher_is_better: true -- metric: unitxt_accuracy - aggregation: unitxt - higher_is_better: true -- metric: unitxt_f1_macro_multi_label - aggregation: unitxt - higher_is_better: true -metadata: - verison: 1.0 diff --git a/lm_eval/tasks/unitxt/unitxt_tasks.grammatical_error_correction b/lm_eval/tasks/unitxt/unitxt_tasks.grammatical_error_correction deleted file mode 100644 index 71dbfdda552452edb3fcd12b16563e156dc91d6b..0000000000000000000000000000000000000000 --- a/lm_eval/tasks/unitxt/unitxt_tasks.grammatical_error_correction +++ /dev/null @@ -1,24 +0,0 @@ -group: -- unitxt -dataset_path: unitxt/data -output_type: generate_until -training_split: train -validation_split: test -doc_to_text: '{{source}}' -doc_to_target: target -process_results: !function 'unitxt_wrapper.process_results' -generation_kwargs: - until: - - -metric_list: -- metric: unitxt_char_edit_dist_accuracy - aggregation: unitxt - higher_is_better: true -- metric: unitxt_rouge - aggregation: unitxt - higher_is_better: true -- metric: unitxt_char_edit_distance[reference_field=original_text] - aggregation: unitxt - higher_is_better: true -metadata: - verison: 1.0 diff --git a/lm_eval/tasks/unitxt/unitxt_tasks.qa.with_context.extractive b/lm_eval/tasks/unitxt/unitxt_tasks.qa.with_context.extractive deleted file mode 100644 index 72ecb3134ac2026e30020162763ef4f3cee6a245..0000000000000000000000000000000000000000 --- a/lm_eval/tasks/unitxt/unitxt_tasks.qa.with_context.extractive +++ /dev/null @@ -1,18 +0,0 @@ -group: -- unitxt -dataset_path: unitxt/data -output_type: generate_until -training_split: train -validation_split: test -doc_to_text: '{{source}}' -doc_to_target: target -process_results: !function 'unitxt_wrapper.process_results' -generation_kwargs: - until: - - -metric_list: -- metric: unitxt_squad - aggregation: unitxt - higher_is_better: true -metadata: - verison: 1.0 diff --git a/lm_eval/tasks/unitxt/unitxt_tasks.regression.two_texts b/lm_eval/tasks/unitxt/unitxt_tasks.regression.two_texts deleted file mode 100644 index 6db5d5876475c12674685e72305fdb38f40d254d..0000000000000000000000000000000000000000 --- a/lm_eval/tasks/unitxt/unitxt_tasks.regression.two_texts +++ /dev/null @@ -1,18 +0,0 @@ -group: -- unitxt -dataset_path: unitxt/data -output_type: generate_until -training_split: train -validation_split: test -doc_to_text: '{{source}}' -doc_to_target: target -process_results: !function 'unitxt_wrapper.process_results' -generation_kwargs: - until: - - -metric_list: -- metric: unitxt_spearman - aggregation: unitxt - higher_is_better: true -metadata: - verison: 1.0 diff --git a/lm_eval/tasks/unitxt/unitxt_tasks.span_labeling.extraction b/lm_eval/tasks/unitxt/unitxt_tasks.span_labeling.extraction deleted file mode 100644 index 60a5e74a403e834f5c99e777ad4f2550fb01f315..0000000000000000000000000000000000000000 --- a/lm_eval/tasks/unitxt/unitxt_tasks.span_labeling.extraction +++ /dev/null @@ -1,18 +0,0 @@ -group: -- unitxt -dataset_path: unitxt/data -output_type: generate_until -training_split: train -validation_split: test -doc_to_text: '{{source}}' -doc_to_target: target -process_results: !function 'unitxt_wrapper.process_results' -generation_kwargs: - until: - - -metric_list: -- metric: unitxt_ner - aggregation: unitxt - higher_is_better: true -metadata: - verison: 1.0 diff --git a/lm_eval/tasks/unitxt/unitxt_tasks.summarization.abstractive b/lm_eval/tasks/unitxt/unitxt_tasks.summarization.abstractive deleted file mode 100644 index 4b5d97f942f7bef1ba506e6dda1e1e9f4e9d343f..0000000000000000000000000000000000000000 --- a/lm_eval/tasks/unitxt/unitxt_tasks.summarization.abstractive +++ /dev/null @@ -1,18 +0,0 @@ -group: -- unitxt -dataset_path: unitxt/data -output_type: generate_until -training_split: train -validation_split: test -doc_to_text: '{{source}}' -doc_to_target: target -process_results: !function 'unitxt_wrapper.process_results' -generation_kwargs: - until: - - -metric_list: -- metric: unitxt_rouge - aggregation: unitxt - higher_is_better: true -metadata: - verison: 1.0 diff --git a/lm_eval/tasks/unitxt/unitxt_wrapper.py b/lm_eval/tasks/unitxt/unitxt_wrapper.py deleted file mode 100644 index cfb2ec3830aa95b9bf48663623b9138ce8d20869..0000000000000000000000000000000000000000 --- a/lm_eval/tasks/unitxt/unitxt_wrapper.py +++ /dev/null @@ -1,46 +0,0 @@ -try: - from unitxt import evaluate -except ImportError: - raise ImportError( - "Package 'unitxt' is not installed. To install it, use `pip install 'lm_eval[unitxt]'`" - ) - -from lm_eval.api.registry import AGGREGATION_REGISTRY, METRIC_REGISTRY, register_metric - - -def unitxt_agg_metric(items): - preds = [pred[0] for pred, _, _ in items] - refs = [ref for _, ref, _ in items] - metric_name = items[0][2].replace("unitxt_", "metrics.") - for ref in refs: - ref["metrics"] = [metric_name] - - result_metrics = evaluate(preds, refs) - return result_metrics[0]["score"]["global"]["score"] - - -AGGREGATION_REGISTRY["unitxt"] = unitxt_agg_metric - - -def unitxt_metric(items): # This is a passthrough function - return items - - -def process_results(doc, results): - metrics = doc["metrics"] - scores = {} - for metric in metrics: - metric = metric.replace("metrics.", "unitxt_") - scores[metric] = (results, doc, metric) - - if metric not in METRIC_REGISTRY: - register_metric( - metric=metric, - higher_is_better=True, - output_type="generate_until", - aggregation="unitxt", - )(unitxt_metric) - return scores - - -# diff --git a/lm_eval/tasks/unitxt/xsum.yaml b/lm_eval/tasks/unitxt/xsum.yaml index 309e3039deef20eb52e4ed859b5cf89b8c0800b9..6fe2999dca43cb86ca91078c869f6622d7e01733 100644 --- a/lm_eval/tasks/unitxt/xsum.yaml +++ b/lm_eval/tasks/unitxt/xsum.yaml @@ -1,3 +1,3 @@ -include: unitxt_tasks.summarization.abstractive task: xsum -dataset_name: card=cards.xsum,template=templates.summarization.abstractive.full +include: unitxt +recipe: card=cards.xsum,template=templates.summarization.abstractive.full diff --git a/lm_eval/tasks/unitxt/yahoo_answers_topics.yaml b/lm_eval/tasks/unitxt/yahoo_answers_topics.yaml index 3cb8ad2006f0de5157aecdec41fd3fe72740cc77..6bf12faedbd58109333426b99ada5d14aa3e9f06 100644 --- a/lm_eval/tasks/unitxt/yahoo_answers_topics.yaml +++ b/lm_eval/tasks/unitxt/yahoo_answers_topics.yaml @@ -1,3 +1,3 @@ -include: unitxt_tasks.classification.multi_class task: yahoo_answers_topics -dataset_name: card=cards.yahoo_answers_topics,template=templates.classification.multi_class.title +include: unitxt +recipe: card=cards.yahoo_answers_topics,template=templates.classification.multi_class.title diff --git a/lm_eval/tasks/unscramble/anagrams1.yaml b/lm_eval/tasks/unscramble/anagrams1.yaml index 392fc78f94a15d7b4d0643e1dc414c9748280b0c..ee044474f5f44e1b63f065a919e4604dfc242756 100644 --- a/lm_eval/tasks/unscramble/anagrams1.yaml +++ b/lm_eval/tasks/unscramble/anagrams1.yaml @@ -1,4 +1,4 @@ -group: +tag: - unscramble task: anagrams1 dataset_path: EleutherAI/unscramble diff --git a/lm_eval/tasks/unscramble/anagrams2.yaml b/lm_eval/tasks/unscramble/anagrams2.yaml index 9f1dbe6fb5fb25d45475babe4e6c3771d9c89fe9..cb5e91dec2c0bb000441f83c52c7871cdc93b382 100644 --- a/lm_eval/tasks/unscramble/anagrams2.yaml +++ b/lm_eval/tasks/unscramble/anagrams2.yaml @@ -1,4 +1,4 @@ -group: +tag: - unscramble task: anagrams2 dataset_path: EleutherAI/unscramble diff --git a/lm_eval/tasks/unscramble/cycle_letters.yaml b/lm_eval/tasks/unscramble/cycle_letters.yaml index dc7ccf977cb0b5f8e372739a2c53fa4bbf5c92d3..b433b16737958065b3d6a6c224610eecea6634c3 100644 --- a/lm_eval/tasks/unscramble/cycle_letters.yaml +++ b/lm_eval/tasks/unscramble/cycle_letters.yaml @@ -1,4 +1,4 @@ -group: +tag: - unscramble task: cycle_letters dataset_path: EleutherAI/unscramble diff --git a/lm_eval/tasks/unscramble/random_insertion.yaml b/lm_eval/tasks/unscramble/random_insertion.yaml index 189c2415f12a49334dd18e6a6c63c36cfc9300f7..f537620d8b8c234ef43658779d7e43b207e1e1a3 100644 --- a/lm_eval/tasks/unscramble/random_insertion.yaml +++ b/lm_eval/tasks/unscramble/random_insertion.yaml @@ -1,4 +1,4 @@ -group: +tag: - unscramble task: random_insertion dataset_path: EleutherAI/unscramble diff --git a/lm_eval/tasks/unscramble/reversed_words.yaml b/lm_eval/tasks/unscramble/reversed_words.yaml index d1bc37dc57b1ae4a3e4608c643e63a6b97dd49c7..b074de6eaaac0433c739451a2f0eec8fbc393a0f 100644 --- a/lm_eval/tasks/unscramble/reversed_words.yaml +++ b/lm_eval/tasks/unscramble/reversed_words.yaml @@ -1,4 +1,4 @@ -group: +tag: - unscramble task: reversed_words dataset_path: EleutherAI/unscramble diff --git a/lm_eval/tasks/webqs/webqs.yaml b/lm_eval/tasks/webqs/webqs.yaml index 71aaac1eba31ccc02ea06ceefa28aedf127732d2..d725d1ddccd554c14a1a4cb4976f22b84749dc4a 100644 --- a/lm_eval/tasks/webqs/webqs.yaml +++ b/lm_eval/tasks/webqs/webqs.yaml @@ -1,4 +1,4 @@ -group: +tag: - freebase task: webqs dataset_path: web_questions diff --git a/lm_eval/tasks/winogrande/default.yaml b/lm_eval/tasks/winogrande/default.yaml index 213f0727fea6ef8d5b6f87a78f093de89b6f80f6..12e0077a70f79a333c273b4be2feddc498f8fa31 100644 --- a/lm_eval/tasks/winogrande/default.yaml +++ b/lm_eval/tasks/winogrande/default.yaml @@ -15,3 +15,5 @@ metric_list: higher_is_better: true metadata: version: 1.0 +dataset_kwargs: + trust_remote_code: true diff --git a/lm_eval/tasks/wmdp/README.md b/lm_eval/tasks/wmdp/README.md index f6074d47102d60dbf6acc4408eb64ee4d379559f..aadde0c1dce80cf0f6fe17fbb8a2a1563f34051b 100644 --- a/lm_eval/tasks/wmdp/README.md +++ b/lm_eval/tasks/wmdp/README.md @@ -24,7 +24,7 @@ Homepage: https://wmdp.ai } ``` -### Groups and Tasks +### Groups, Tags, and Tasks #### Groups diff --git a/lm_eval/tasks/wmdp/_default_template_yaml b/lm_eval/tasks/wmdp/_default_template_yaml index 4e9d1c804bc2e248feaa1d132de6f0279f032d0c..7c2f25c1d7ae111422411d8d27e6210300adff4f 100644 --- a/lm_eval/tasks/wmdp/_default_template_yaml +++ b/lm_eval/tasks/wmdp/_default_template_yaml @@ -1,5 +1,4 @@ dataset_path: cais/wmdp -group: wmdp test_split: test training_split: null validation_split: null @@ -13,4 +12,4 @@ metric_list: aggregation: mean higher_is_better: true metadata: - version: 0 + version: 1 diff --git a/lm_eval/tasks/wmdp/_wmdp.yaml b/lm_eval/tasks/wmdp/_wmdp.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ec1c795264d990b018d1112bf490591c43a815fa --- /dev/null +++ b/lm_eval/tasks/wmdp/_wmdp.yaml @@ -0,0 +1,11 @@ +group: wmdp +task: + - wmdp_bio + - wmdp_chem + - wmdp_cyber +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: True +metadata: + version: 1 diff --git a/lm_eval/tasks/wmt2016/README.md b/lm_eval/tasks/wmt2016/README.md index 14e9edcabda36b35c83c7f8a771a633c7287e2a3..56b2e4ab12215261fe6d7fcf00a7e69006fc48dd 100644 --- a/lm_eval/tasks/wmt2016/README.md +++ b/lm_eval/tasks/wmt2016/README.md @@ -27,11 +27,7 @@ Homepage: https://huggingface.co/datasets/wmt16 } ``` -### Groups and Tasks - -#### Groups - -* `wmt-t5-prompt`: Group for all wmt tasks with prompt templates used for T5 (`Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer`) +### Groups, Tags, and Tasks #### Tasks diff --git a/lm_eval/tasks/wmt2016/ro_en-t5_prompt.yaml b/lm_eval/tasks/wmt2016/ro_en-t5_prompt.yaml index fae5db7ed2cfa51ea551712ab498d1da7f01ef6f..5009d8767da9da38c1ea58a594c11c22a0db6d52 100644 --- a/lm_eval/tasks/wmt2016/ro_en-t5_prompt.yaml +++ b/lm_eval/tasks/wmt2016/ro_en-t5_prompt.yaml @@ -1,5 +1,3 @@ -group: - - wmt-t5-prompt task: wmt-ro-en-t5-prompt dataset_path: wmt16 dataset_name: ro-en diff --git a/lm_eval/tasks/xcopa/_xcopa.yaml b/lm_eval/tasks/xcopa/_xcopa.yaml new file mode 100644 index 0000000000000000000000000000000000000000..c73141e6e810155ec7fcb7dcb864c8991176a195 --- /dev/null +++ b/lm_eval/tasks/xcopa/_xcopa.yaml @@ -0,0 +1,19 @@ +group: xcopa +task: + - xcopa_et + - xcopa_ht + - xcopa_id + - xcopa_it + - xcopa_qu + - xcopa_sw + - xcopa_ta + - xcopa_th + - xcopa_tr + - xcopa_vi + - xcopa_zh +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: True +metadata: + version: 1.0 diff --git a/lm_eval/tasks/xcopa/default_et.yaml b/lm_eval/tasks/xcopa/default_et.yaml index 9f2b0b73b585ab1a6e4c946237e52dba283a830a..76e750ffe322f0f91b4ccdf146600e3f5bd28bdf 100644 --- a/lm_eval/tasks/xcopa/default_et.yaml +++ b/lm_eval/tasks/xcopa/default_et.yaml @@ -1,4 +1,3 @@ -group: xcopa task: xcopa_et dataset_path: xcopa dataset_name: et diff --git a/lm_eval/tasks/xnli/_xnli.yaml b/lm_eval/tasks/xnli/_xnli.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f6e350c85de1690e04fff7ccc13bf3ea98f85f35 --- /dev/null +++ b/lm_eval/tasks/xnli/_xnli.yaml @@ -0,0 +1,23 @@ +group: xnli +task: + - xnli_ar + - xnli_bg + - xnli_de + - xnli_el + - xnli_en + - xnli_es + - xnli_fr + - xnli_hi + - xnli_ru + - xnli_sw + - xnli_th + - xnli_tr + - xnli_ur + - xnli_vi + - xnli_zh +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/xnli/xnli_common_yaml b/lm_eval/tasks/xnli/xnli_common_yaml index 0201459d35817d6b08b44edaf3358d706de5c20f..4dc4ab4fae8c39f88196fe31d98e8235e17e4d36 100644 --- a/lm_eval/tasks/xnli/xnli_common_yaml +++ b/lm_eval/tasks/xnli/xnli_common_yaml @@ -1,7 +1,6 @@ # This file will be included in the generated language-specific task configs. # It doesn't have a yaml file extension as it is not meant to be imported directly # by the harness. -group: xnli task: null dataset_path: xnli dataset_name: null diff --git a/lm_eval/tasks/xnli_eu/README.md b/lm_eval/tasks/xnli_eu/README.md index 4efffa3ca786370e96fa09d81393cb97722bc502..ce646d4d9cb4e4e93a8a55d16b11b0cbf290225e 100644 --- a/lm_eval/tasks/xnli_eu/README.md +++ b/lm_eval/tasks/xnli_eu/README.md @@ -24,9 +24,9 @@ Homepage: https://github.com/hitz-zentroa/xnli-eu } ``` -### Groups and Tasks +### Groups, Tags, and Tasks -#### Groups +#### Tags * `xnli_eu_mt_native`: Includes MT and Native variants of the XNLIeu dataset. diff --git a/lm_eval/tasks/xnli_eu/xnli_common_yaml b/lm_eval/tasks/xnli_eu/xnli_common_yaml index fe2a43afe381984289584aa7207c4405d762d0a2..4950a8996806739858b4261f9d0b005cd508fafe 100644 --- a/lm_eval/tasks/xnli_eu/xnli_common_yaml +++ b/lm_eval/tasks/xnli_eu/xnli_common_yaml @@ -1,4 +1,3 @@ -group: xnli task: null dataset_path: xnli dataset_name: null diff --git a/lm_eval/tasks/xnli_eu/xnli_eu_mt.yaml b/lm_eval/tasks/xnli_eu/xnli_eu_mt.yaml index 4674157ccc3f94962c3f3353f0705b383fd11366..c0fbf5416b4c10bc640a25f8a3a63dd5fb903128 100644 --- a/lm_eval/tasks/xnli_eu/xnli_eu_mt.yaml +++ b/lm_eval/tasks/xnli_eu/xnli_eu_mt.yaml @@ -1,4 +1,4 @@ include: xnli_eu.yaml -group: xnli_eu_mt_native +tag: xnli_eu_mt_native task: xnli_eu_mt dataset_name: eu_mt diff --git a/lm_eval/tasks/xnli_eu/xnli_eu_native.yaml b/lm_eval/tasks/xnli_eu/xnli_eu_native.yaml index 86e5eb06c5dd5878ad097144e54a4a9eb39226e3..e841f37e7ff36b238b85f05f9de7fd7fc488cbb2 100644 --- a/lm_eval/tasks/xnli_eu/xnli_eu_native.yaml +++ b/lm_eval/tasks/xnli_eu/xnli_eu_native.yaml @@ -1,5 +1,5 @@ include: xnli_eu.yaml -group: xnli_eu_mt_native +tag: xnli_eu_mt_native task: xnli_eu_native training_split: null validation_split: null diff --git a/lm_eval/tasks/xquad/README.md b/lm_eval/tasks/xquad/README.md new file mode 100755 index 0000000000000000000000000000000000000000..b6ee68294f713a8b873a4b145bb805d508cbfc7d --- /dev/null +++ b/lm_eval/tasks/xquad/README.md @@ -0,0 +1,62 @@ +# XQuAD + +### Paper + +Title: `On the Cross-lingual Transferability of Monolingual Representations` + +Abstract: https://aclanthology.org/2020.acl-main.421.pdf + +XQuAD (Cross-lingual Question Answering Dataset) is a benchmark dataset for evaluating cross-lingual question answering performance. The dataset consists of a subset of 240 paragraphs and 1190 question-answer pairs from the development set of SQuAD v1.1 (Rajpurkar et al., 2016) together with their professional translations into ten languages: Spanish, German, Greek, Russian, Turkish, Arabic, Vietnamese, Thai, Chinese, and Hindi. Consequently, the dataset is entirely parallel across 11 languages. + +Homepage: https://github.com/deepmind/xquad + + +### Citation + +``` +@article{Artetxe:etal:2019, + author = {Mikel Artetxe and Sebastian Ruder and Dani Yogatama}, + title = {On the cross-lingual transferability of monolingual representations}, + journal = {CoRR}, + volume = {abs/1910.11856}, + year = {2019}, + archivePrefix = {arXiv}, + eprint = {1910.11856} +} +``` + +### Groups and Tasks + +#### Groups + +* `xquad`: All available languages. + +#### Tasks +Perform extractive question answering for each language's subset of XQuAD. +* `xquad_ar`: Arabic +* `xquad_de`: German +* `xquad_el`: Greek +* `xquad_en`: English +* `xquad_es`: Spanish +* `xquad_hi`: Hindi +* `xquad_ro`: Romanian +* `xquad_ru`: Russian +* `xquad_th`: Thai +* `xquad_tr`: Turkish +* `xquad_vi`: Vietnamese +* `xquad_zh`: Chinese + + + +### Checklist + +For adding novel benchmarks/datasets to the library: +* [x] Is the task an existing benchmark in the literature? + * [x] Have you referenced the original paper that introduced the task? + * [x] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test? + + +If other tasks on this dataset are already supported: +* [ ] Is the "Main" variant of this task clearly denoted? +* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates? +* [ ] Have you noted which, if any, published evaluation setups are matched by this variant? diff --git a/lm_eval/tasks/xquad/utils.py b/lm_eval/tasks/xquad/utils.py new file mode 100755 index 0000000000000000000000000000000000000000..83e2e9ecef4a3ae990b4e54a3869978c24dfc127 --- /dev/null +++ b/lm_eval/tasks/xquad/utils.py @@ -0,0 +1,15 @@ +import re +from itertools import product + +import evaluate +import transformers.data.metrics.squad_metrics as squad_metrics + +from lm_eval.utils import general_detokenize + + +def process_results_qa(doc, results): + preds = results[0] + reference = doc["answers"]["text"][0] + f1_sum = squad_metrics.compute_f1(reference, preds) + exact_match = squad_metrics.compute_exact(reference, preds) + return {"f1": f1_sum, "exact_match": exact_match} diff --git a/lm_eval/tasks/xquad/xquad_ar.yaml b/lm_eval/tasks/xquad/xquad_ar.yaml new file mode 100755 index 0000000000000000000000000000000000000000..4a843009c580e85e9cf59ddf812ebe56fc0e1968 --- /dev/null +++ b/lm_eval/tasks/xquad/xquad_ar.yaml @@ -0,0 +1,4 @@ +include: xquad_common_yaml +task: xquad_ar +dataset_name: xquad.ar +doc_to_text: "سيا: {{context}}\n\nسؤال: {{question}}\n\nإجابة:" diff --git a/lm_eval/tasks/xquad/xquad_common_yaml b/lm_eval/tasks/xquad/xquad_common_yaml new file mode 100755 index 0000000000000000000000000000000000000000..134a1065844823a42a6f1cc97841a683d767547f --- /dev/null +++ b/lm_eval/tasks/xquad/xquad_common_yaml @@ -0,0 +1,27 @@ +# This file will be included in the generated language-specific task configs. +# It doesn't have a yaml file extension as it is not meant to be imported directly +# by the harness. +tag: xquad +task: null +dataset_path: xquad +dataset_name: null +output_type: generate_until +validation_split: validation +doc_to_text: null +doc_to_target: '{{answers["text"][0]}}' +process_results: !function utils.process_results_qa +target_delimiter: ' ' +generation_kwargs: + until: + - "\n" + do_sample: false + temperature: 0.0 +metric_list: + - metric: exact_match + aggregation: mean + higher_is_better: true + - metric: f1 + aggregation: mean + higher_is_better: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/xquad/xquad_de.yaml b/lm_eval/tasks/xquad/xquad_de.yaml new file mode 100755 index 0000000000000000000000000000000000000000..f6d44f6b89575577fd3e69993ddd62d0ca0a543a --- /dev/null +++ b/lm_eval/tasks/xquad/xquad_de.yaml @@ -0,0 +1,4 @@ +include: xquad_common_yaml +task: xquad_de +dataset_name: xquad.de +doc_to_text: "Kontext: {{context}}\n\nFrage: {{question}}\n\nAntwort:" diff --git a/lm_eval/tasks/xquad/xquad_el.yaml b/lm_eval/tasks/xquad/xquad_el.yaml new file mode 100755 index 0000000000000000000000000000000000000000..1b54cdb03b637d708a4585f359b49b2eb0bcbf48 --- /dev/null +++ b/lm_eval/tasks/xquad/xquad_el.yaml @@ -0,0 +1,4 @@ +include: xquad_common_yaml +task: xquad_el +dataset_name: xquad.el +doc_to_text: "Συμφραζόμενα: {{context}}\n\nΕρώτηση: {{question}}\n\nΑπάντηση:" diff --git a/lm_eval/tasks/xquad/xquad_en.yaml b/lm_eval/tasks/xquad/xquad_en.yaml new file mode 100755 index 0000000000000000000000000000000000000000..54ac757596fdb2ababca60a78809cb71dabc3d75 --- /dev/null +++ b/lm_eval/tasks/xquad/xquad_en.yaml @@ -0,0 +1,4 @@ +include: xquad_common_yaml +task: xquad_en +dataset_name: xquad.en +doc_to_text: "Context: {{context}}\n\nQuestion: {{question}}\n\nAnswer:" diff --git a/lm_eval/tasks/xquad/xquad_es.yaml b/lm_eval/tasks/xquad/xquad_es.yaml new file mode 100755 index 0000000000000000000000000000000000000000..1322c90c3da80a74b1fab62ba36c2366031a850d --- /dev/null +++ b/lm_eval/tasks/xquad/xquad_es.yaml @@ -0,0 +1,4 @@ +include: xquad_common_yaml +task: xquad_es +dataset_name: xquad.es +doc_to_text: "Contexto: {{context}}\n\nPregunta: {{question}}\n\nRespuesta:" diff --git a/lm_eval/tasks/xquad/xquad_hi.yaml b/lm_eval/tasks/xquad/xquad_hi.yaml new file mode 100755 index 0000000000000000000000000000000000000000..8daa5dd81706b0870ddf82555e47f780fc5b45b9 --- /dev/null +++ b/lm_eval/tasks/xquad/xquad_hi.yaml @@ -0,0 +1,4 @@ +include: xquad_common_yaml +task: xquad_hi +dataset_name: xquad.hi +doc_to_text: "प्रसंग: {{context}}\n\nसवाल: {{question}}\n\nउत्तर:" diff --git a/lm_eval/tasks/xquad/xquad_ro.yaml b/lm_eval/tasks/xquad/xquad_ro.yaml new file mode 100755 index 0000000000000000000000000000000000000000..4f8023da8d5a7626dadf3516a7de084930e22545 --- /dev/null +++ b/lm_eval/tasks/xquad/xquad_ro.yaml @@ -0,0 +1,4 @@ +include: xquad_common_yaml +task: xquad_ro +dataset_name: xquad.ro +doc_to_text: "Context: {{context}}\n\nÎntrebare: {{question}}\n\nRăspuns:" diff --git a/lm_eval/tasks/xquad/xquad_ru.yaml b/lm_eval/tasks/xquad/xquad_ru.yaml new file mode 100755 index 0000000000000000000000000000000000000000..72b1d1050e7a5d85c093f9b3a070f7877d970c30 --- /dev/null +++ b/lm_eval/tasks/xquad/xquad_ru.yaml @@ -0,0 +1,4 @@ +include: xquad_common_yaml +task: xquad_ru +dataset_name: xquad.ru +doc_to_text: "Контекст: {{context}}\n\nВопрос: {{question}}\n\nОтвет:" diff --git a/lm_eval/tasks/xquad/xquad_th.yaml b/lm_eval/tasks/xquad/xquad_th.yaml new file mode 100755 index 0000000000000000000000000000000000000000..15569ce189fa48becda9e4fa45fb26974ab9e1c8 --- /dev/null +++ b/lm_eval/tasks/xquad/xquad_th.yaml @@ -0,0 +1,4 @@ +include: xquad_common_yaml +task: xquad_th +dataset_name: xquad.th +doc_to_text: "บริบท: {{context}}\n\nคำถาม: {{question}}\n\nคำตอบ:" diff --git a/lm_eval/tasks/xquad/xquad_tr.yaml b/lm_eval/tasks/xquad/xquad_tr.yaml new file mode 100755 index 0000000000000000000000000000000000000000..4f6c43db0093b2a4d4bfec83056561befd9340c0 --- /dev/null +++ b/lm_eval/tasks/xquad/xquad_tr.yaml @@ -0,0 +1,4 @@ +include: xquad_common_yaml +task: xquad_tr +dataset_name: xquad.tr +doc_to_text: "Bağlam: {{context}}\n\nSoru: {{question}}\n\nCevap:" diff --git a/lm_eval/tasks/xquad/xquad_vi.yaml b/lm_eval/tasks/xquad/xquad_vi.yaml new file mode 100755 index 0000000000000000000000000000000000000000..8373902a11b36cad9595fe5f39c62fe59971b35f --- /dev/null +++ b/lm_eval/tasks/xquad/xquad_vi.yaml @@ -0,0 +1,4 @@ +include: xquad_common_yaml +task: xquad_vi +dataset_name: xquad.vi +doc_to_text: "Bối cảnh: {{context}}\n\nCâu hỏi: {{question}}\n\nTrả lời:" diff --git a/lm_eval/tasks/xquad/xquad_zh.yaml b/lm_eval/tasks/xquad/xquad_zh.yaml new file mode 100755 index 0000000000000000000000000000000000000000..f0c7632aa1d51cdc70c2465388cda848b1c38f67 --- /dev/null +++ b/lm_eval/tasks/xquad/xquad_zh.yaml @@ -0,0 +1,4 @@ +include: xquad_common_yaml +task: xquad_zh +dataset_name: xquad.zh +doc_to_text: "语境: {{context}}\n\n问题: {{question}}\n\n回答:" diff --git a/lm_eval/tasks/xstorycloze/_xstorycloze.yaml b/lm_eval/tasks/xstorycloze/_xstorycloze.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0b9b41be5db54451d69bde525e53b495271a49f6 --- /dev/null +++ b/lm_eval/tasks/xstorycloze/_xstorycloze.yaml @@ -0,0 +1,19 @@ +group: xstorycloze +task: + - xstorycloze_ar + - xstorycloze_en + - xstorycloze_es + - xstorycloze_eu + - xstorycloze_hi + - xstorycloze_id + - xstorycloze_my + - xstorycloze_ru + - xstorycloze_sw + - xstorycloze_te + - xstorycloze_zh +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/xstorycloze/default_ar.yaml b/lm_eval/tasks/xstorycloze/default_ar.yaml index 2a52966d5a76138be4821d38c5bd639701586061..836a4cb05484e21179331e98daf9893abe5d6b3a 100644 --- a/lm_eval/tasks/xstorycloze/default_ar.yaml +++ b/lm_eval/tasks/xstorycloze/default_ar.yaml @@ -1,4 +1,3 @@ -group: xstorycloze task: xstorycloze_ar dataset_path: juletxara/xstory_cloze dataset_name: ar diff --git a/lm_eval/tasks/xwinograd/_xwinograd.yaml b/lm_eval/tasks/xwinograd/_xwinograd.yaml new file mode 100644 index 0000000000000000000000000000000000000000..24fbcdefacca2c9a6798504030c86ed2ad50b9ac --- /dev/null +++ b/lm_eval/tasks/xwinograd/_xwinograd.yaml @@ -0,0 +1,14 @@ +group: xwinograd +task: + - xwinograd_en + - xwinograd_fr + - xwinograd_jp + - xwinograd_pt + - xwinograd_ru + - xwinograd_zh +aggregate_metric_list: + - metric: acc + aggregation: mean + weight_by_size: true +metadata: + version: 1.0 diff --git a/lm_eval/tasks/xwinograd/xwinograd_common_yaml b/lm_eval/tasks/xwinograd/xwinograd_common_yaml index 86554820e9aa6087b00bb6f552c70638cd02eaeb..b3e683262e676948907a2641ef9782fa29655ac0 100644 --- a/lm_eval/tasks/xwinograd/xwinograd_common_yaml +++ b/lm_eval/tasks/xwinograd/xwinograd_common_yaml @@ -1,8 +1,6 @@ # This file will be included in the generated language-specific task configs. # It doesn't have a yaml file extension as it is not meant to be imported directly # by the harness. -group: - - xwinograd dataset_path: Muennighoff/xwinograd dataset_name: null # Overridden by language-specific config. output_type: multiple_choice diff --git a/lm_eval/utils.py b/lm_eval/utils.py index 486f960fd851745805597fb6deea2ed032140742..7166e24d0723e397f00347d6f14eb14e5902a452 100644 --- a/lm_eval/utils.py +++ b/lm_eval/utils.py @@ -163,7 +163,7 @@ def get_file_datetime(filename: str) -> str: """ Given the results and sample results filenames, extracts and returns the datetime. """ - return filename[filename.rfind("_") + 1 :].replace(".json", "") + return filename[filename.rfind("_") + 1 :].replace(".jsonl", "") def sanitize_model_name(model_name: str) -> str: @@ -308,7 +308,7 @@ class Reorderer: return res -def make_table(result_dict, column: str = "results", sort_results: bool = True): +def make_table(result_dict, column: str = "results", sort_results: bool = False): """Generate table of results.""" from pytablewriter import LatexTableWriter, MarkdownTableWriter @@ -338,20 +338,21 @@ def make_table(result_dict, column: str = "results", sort_results: bool = True): keys = result_dict[column].keys() if sort_results: - # sort entries alphabetically + # sort entries alphabetically by task or group name. + # NOTE: we default here to false, because order matters for multi-level table printing a la mmlu. + # sorting here would mess that up keys = sorted(keys) for k in keys: dic = result_dict[column][k] - version = result_dict["versions"].get(k, "N/A") - n = str(result_dict["n-shot"][k]) + version = result_dict["versions"].get(k, " N/A") + n = str(result_dict.get("n-shot", " ").get(k, " ")) higher_is_better = result_dict.get("higher_is_better", {}).get(k, {}) if "alias" in dic: k = dic.pop("alias") metric_items = dic.items() - if sort_results: - metric_items = sorted(metric_items) + metric_items = sorted(metric_items) for (mf), v in metric_items: m, _, f = mf.partition(",") @@ -360,13 +361,14 @@ def make_table(result_dict, column: str = "results", sort_results: bool = True): hib = HIGHER_IS_BETTER_SYMBOLS.get(higher_is_better.get(m), "") + v = "%.4f" % v if isinstance(v, float) else v + if m + "_stderr" + "," + f in dic: se = dic[m + "_stderr" + "," + f] - if se != "N/A": - se = "%.4f" % se - values.append([k, version, f, n, m, hib, "%.4f" % v, "±", se]) + se = " N/A" if se == "N/A" else "%.4f" % se + values.append([k, version, f, n, m, hib, v, "±", se]) else: - values.append([k, version, f, n, m, hib, "%.4f" % v, "", ""]) + values.append([k, version, f, n, m, hib, v, "", ""]) k = "" version = "" md_writer.value_matrix = values @@ -469,7 +471,9 @@ def regex_replace(string, pattern, repl, count: int = 0): return re.sub(pattern, repl, string, count=count) -env = Environment(loader=BaseLoader, undefined=StrictUndefined) +env = Environment( + loader=BaseLoader, undefined=StrictUndefined, keep_trailing_newline=True +) env.filters["regex_replace"] = regex_replace @@ -485,3 +489,13 @@ def create_iterator(raw_iterator, *, rank=0, world_size=1, limit=None): among ranks in multigpu setting or only pulling a sample of documents """ return islice(raw_iterator, rank, limit, world_size) + + +def weighted_f1_score(items): + from sklearn.metrics import f1_score + + unzipped_list = list(zip(*items)) + golds = unzipped_list[0] + preds = unzipped_list[1] + fscore = f1_score(golds, preds, average="weighted") + return fscore diff --git a/pyproject.toml b/pyproject.toml index 3818a1a80a504b6e5ff52f72889a5f85a4b86838..79d3dc3d411934ae9e81d6a5e818d799fc8c273e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "lm_eval" -version = "0.4.2" +version = "0.4.5" authors = [ {name="EleutherAI", email="contact@eleuther.ai"} ] @@ -57,17 +57,17 @@ Homepage = "https://github.com/EleutherAI/lm-evaluation-harness" Repository = "https://github.com/EleutherAI/lm-evaluation-harness" [project.optional-dependencies] -anthropic = ["anthropic"] +api = ["requests", "aiohttp", "tenacity", "tqdm", "tiktoken"] dev = ["pytest", "pytest-cov", "pytest-xdist", "pre-commit", "mypy"] deepsparse = ["deepsparse-nightly[llm]>=1.8.0.20240404"] gptq = ["auto-gptq[triton]>=0.6.0"] hf_transfer = ["hf_transfer"] -ifeval = ["langdetect", "immutabledict"] +ibm_watsonx_ai = ["ibm_watsonx_ai"] +ifeval = ["langdetect", "immutabledict", "nltk>=3.9.1"] neuronx = ["optimum[neuronx]"] mamba = ["mamba_ssm", "causal-conv1d==1.0.2"] math = ["sympy>=1.12", "antlr4-python3-runtime==4.11"] multilingual = ["nagisa>=0.2.7", "jieba>=0.42.1", "pycountry"] -openai = ["openai==1.3.9", "tiktoken"] optimum = ["optimum[openvino]"] promptsource = ["promptsource>=0.2.3"] sentencepiece = ["sentencepiece>=0.1.98"] @@ -76,13 +76,15 @@ testing = ["pytest", "pytest-cov", "pytest-xdist"] vllm = ["vllm>=0.4.2"] zeno = ["pandas", "zeno-client"] wandb = ["wandb>=0.16.3", "pandas", "numpy"] -unitxt = ["unitxt"] +gptqmodel = ["gptqmodel>=1.0.9"] +japanese_leaderboard = ["emoji==2.14.0", "neologdn==0.5.3", "fugashi[unidic-lite]", "rouge_score>=0.1.2"] all = [ "lm_eval[anthropic]", "lm_eval[dev]", "lm_eval[deepsparse]", "lm_eval[gptq]", "lm_eval[hf_transfer]", + "lm_eval[ibm_watsonx_ai]", "lm_eval[ifeval]", "lm_eval[mamba]", "lm_eval[math]", @@ -95,7 +97,7 @@ all = [ "lm_eval[vllm]", "lm_eval[zeno]", "lm_eval[wandb]", - "lm_eval[unitxt]" + "lm_eval[japanese_leaderboard]", ] [tool.ruff.lint] @@ -107,3 +109,4 @@ known-first-party = ["lm_eval"] [tool.ruff.lint.extend-per-file-ignores] "__init__.py" = ["F401","F402","F403"] +"utils.py" = ["F401"] diff --git a/scripts/clean_training_data/generate_13_grams.py b/scripts/clean_training_data/generate_13_grams.py index e508f266e9bfbe1cdf6f93de478c2d60d490d557..530c35ecd7785dbb9f97c5fcecc7e1ce776cc5b0 100644 --- a/scripts/clean_training_data/generate_13_grams.py +++ b/scripts/clean_training_data/generate_13_grams.py @@ -55,7 +55,7 @@ def yield_pile(start_offsets=None, checkpoint_offset=None): print( "We expect the pile archives to be in the 'pile' directory, but this was not found." ) - raise Exception("Pile directory not found.") + raise FileNotFoundError("Pile directory not found.") files = list(sorted(glob.glob(os.path.join(directory, "*.jsonl.zst*")))) diff --git a/scripts/model_comparator.py b/scripts/model_comparator.py index 26fe53961ab4972980fd55e30f89027a3098f8f2..55f4f3b15468b2f46e590cbfd82d7902f1d9a16f 100644 --- a/scripts/model_comparator.py +++ b/scripts/model_comparator.py @@ -4,7 +4,6 @@ from typing import Dict, List, Tuple import numpy as np import pandas as pd -import scipy.stats import torch import lm_eval.evaluator @@ -23,11 +22,13 @@ def memory_stats(): def calculate_z_value(res1: Dict, res2: Dict) -> Tuple[float, float]: + from scipy.stats.norm import sf + acc1, acc2 = res1["acc,none"], res2["acc,none"] st_err1, st_err2 = res1["acc_stderr,none"], res2["acc_stderr,none"] Z = (acc1 - acc2) / np.sqrt((st_err1**2) + (st_err2**2)) # Determining the p-value - p_value = 2 * scipy.stats.norm.sf(abs(Z)) # two-tailed test + p_value = 2 * sf(abs(Z)) # two-tailed test return Z, p_value diff --git a/scripts/write_out.py b/scripts/write_out.py index 6ff5a4304ed7798f8e375abeb8a5f30cb2aedcea..a5122df996761cf9f1345fe0fd53c6a9fe4ecb71 100644 --- a/scripts/write_out.py +++ b/scripts/write_out.py @@ -5,6 +5,7 @@ import random import numpy as np from lm_eval import tasks +from lm_eval.evaluator_utils import get_task_list from lm_eval.tasks import TaskManager from lm_eval.utils import eval_logger, join_iters @@ -51,9 +52,8 @@ def main(): task_dict = tasks.get_task_dict(task_names, task_manager) os.makedirs(args.output_base_path, exist_ok=True) - for task_name, task in task_dict.items(): - if isinstance(task, tuple): - _, task = task + for task in [x.task for x in get_task_list(task_dict)]: + task_name = task.config.task rnd = random.Random() rnd.seed(args.seed) diff --git a/scripts/zeno_visualize.py b/scripts/zeno_visualize.py index f2772a235579b64cb05353a950a716e104a44cb2..4bc7e03bf8417646ee5ffad9bd7f25e332dac597 100644 --- a/scripts/zeno_visualize.py +++ b/scripts/zeno_visualize.py @@ -173,19 +173,19 @@ def generate_dataset( instance = [""] * len(ids) if config["output_type"] == "loglikelihood": - instance = [x["arguments"][0][0] for x in data] - labels = [x["arguments"][0][1] for x in data] + instance = [x["arguments"]["gen_args_0"]["arg_0"] for x in data] + labels = [x["arguments"]["gen_args_0"]["arg_1"] for x in data] elif config["output_type"] == "multiple_choice": instance = [ - x["arguments"][0][0] + x["arguments"]["gen_args_0"]["arg_0"] + "\n\n" + "\n".join([f"- {y[1]}" for y in x["arguments"]]) for x in data ] elif config["output_type"] == "loglikelihood_rolling": - instance = [x["arguments"][0][0] for x in data] + instance = [x["arguments"]["gen_args_0"]["arg_0"] for x in data] elif config["output_type"] == "generate_until": - instance = [x["arguments"][0][0] for x in data] + instance = [x["arguments"]["gen_args_0"]["arg_0"] for x in data] return pd.DataFrame( { diff --git a/templates/new_yaml_task/README.md b/templates/new_yaml_task/README.md index 3028712286ef77e503c97ffbe5ce39c7ec7b14e2..3b7259c79cfe8353d3318674e212f8cccd57409f 100644 --- a/templates/new_yaml_task/README.md +++ b/templates/new_yaml_task/README.md @@ -17,12 +17,16 @@ Homepage: `homepage to the benchmark's website goes here, if applicable` BibTeX-formatted citation goes here ``` -### Groups and Tasks +### Groups, Tags, and Tasks #### Groups * `group_name`: `Short description` +#### Tags + +* `tag_name`: `Short description` + #### Tasks * `task_name`: `1-sentence description of what this particular task does` diff --git a/tests/models/test_api.py b/tests/models/test_api.py new file mode 100644 index 0000000000000000000000000000000000000000..1bca2f7bdbc479d6c8c45171347f11dd8c8892d9 --- /dev/null +++ b/tests/models/test_api.py @@ -0,0 +1,149 @@ +from unittest.mock import MagicMock, patch + +import pytest + +from lm_eval.models.openai_completions import LocalCompletionsAPI + + +@pytest.fixture +def api(): + return LocalCompletionsAPI( + base_url="http://test-url.com", tokenizer_backend=None, model="gpt-3.5-turbo" + ) + + +@pytest.fixture +def api_tokenized(): + return LocalCompletionsAPI( + base_url="http://test-url.com", + model="EleutherAI/pythia-1b", + tokenizer_backend="huggingface", + ) + + +def test_create_payload_generate(api): + messages = ["Generate a story"] + gen_kwargs = { + "max_tokens": 100, + "temperature": 0.7, + "until": ["The End"], + "do_sample": True, + "seed": 1234, + } + payload = api._create_payload(messages, generate=True, gen_kwargs=gen_kwargs) + + assert payload == { + "prompt": ["Generate a story"], + "model": "gpt-3.5-turbo", + "max_tokens": 100, + "temperature": 0.7, + "stop": ["The End"], + "seed": 1234, + } + + +def test_create_payload_loglikelihood(api): + messages = ["The capital of France is"] + payload = api._create_payload(messages, generate=False, gen_kwargs=None) + + assert payload == { + "model": "gpt-3.5-turbo", + "prompt": ["The capital of France is"], + "max_tokens": 1, + "logprobs": 1, + "echo": True, + "temperature": 0, + "seed": 1234, + } + + +@pytest.mark.parametrize( + "input_messages, generate, gen_kwargs, expected_payload", + [ + ( + ["Hello, how are"], + True, + {"max_gen_toks": 100, "temperature": 0.7}, + { + "prompt": "Hello, how are", + "model": "gpt-3.5-turbo", + "max_tokens": 100, + "temperature": 0.7, + "stop": ["<|endoftext|>"], + "seed": 1234, + }, + ), + ( + ["Hello, how are", "you"], + True, + {}, + { + "prompt": "Hello, how are", + "model": "gpt-3.5-turbo", + "max_tokens": 256, + "temperature": 0, + "stop": ["<|endoftext|>"], + "seed": 1234, + }, + ), + ], +) +def test_model_generate_call_usage( + api, input_messages, generate, gen_kwargs, expected_payload +): + with patch("requests.post") as mock_post: + mock_response = MagicMock() + mock_response.json.return_value = {"result": "success"} + mock_post.return_value = mock_response + + # Act + result = api.model_call( + input_messages, generate=generate, gen_kwargs=gen_kwargs + ) + + # Assert + mock_post.assert_called_once() + _, kwargs = mock_post.call_args + assert "json" in kwargs + assert kwargs["json"] == expected_payload + assert result == {"result": "success"} + + +@pytest.mark.parametrize( + "input_messages, generate, gen_kwargs, expected_payload", + [ + ( + [[1, 2, 3, 4, 5]], + False, + None, + { + "model": "EleutherAI/pythia-1b", + "prompt": [[1, 2, 3, 4, 5]], + "max_tokens": 1, + "logprobs": 1, + "echo": True, + "seed": 1234, + "temperature": 0, + }, + ), + ], +) +def test_model_tokenized_call_usage( + api_tokenized, input_messages, generate, gen_kwargs, expected_payload +): + with patch("requests.post") as mock_post: + mock_response = MagicMock() + mock_response.json.return_value = {"result": "success"} + mock_post.return_value = mock_response + + # Act + result = api_tokenized.model_call( + input_messages, generate=generate, gen_kwargs=gen_kwargs + ) + + # Assert + mock_post.assert_called_once() + _, kwargs = mock_post.call_args + assert "json" in kwargs + assert kwargs["json"] == expected_payload + assert result == {"result": "success"} diff --git a/tests/models/test_gptqmodel.py b/tests/models/test_gptqmodel.py new file mode 100644 index 0000000000000000000000000000000000000000..9e3831c8fbacd96cae75f462fd14c57b048acd77 --- /dev/null +++ b/tests/models/test_gptqmodel.py @@ -0,0 +1,54 @@ +from typing import List + +import pytest + +import lm_eval + + +def assert_less_than(value, threshold, desc): + if value is not None: + assert float(value) < threshold, f"{desc} should be less than {threshold}" + + +@pytest.mark.skip(reason="requires CUDA") +class Test_GPTQModel: + gptqmodel = pytest.importorskip("gptqmodel", minversion="1.0.9") + MODEL_ID = "ModelCloud/Opt-125-GPTQ-4bit-10-25-2024" + + def test_gptqmodel(self) -> None: + acc = "acc" + acc_norm = "acc_norm" + acc_value = None + acc_norm_value = None + task = "arc_easy" + + model_args = f"pretrained={self.MODEL_ID},gptqmodel=True" + + tasks: List[str] = [task] + + results = lm_eval.simple_evaluate( + model="hf", + model_args=model_args, + tasks=tasks, + device="cuda", + ) + + column = "results" + dic = results.get(column, {}).get(self.task) + if dic is not None: + if "alias" in dic: + _ = dic.pop("alias") + items = sorted(dic.items()) + for k, v in items: + m, _, f = k.partition(",") + if m.endswith("_stderr"): + continue + + if m == acc: + acc_value = "%.4f" % v if isinstance(v, float) else v + + if m == acc_norm: + acc_norm_value = "%.4f" % v if isinstance(v, float) else v + + assert_less_than(acc_value, 0.43, "acc") + assert_less_than(acc_norm_value, 0.39, "acc_norm") diff --git a/tests/models/test_huggingface.py b/tests/models/test_huggingface.py index 9f495402397a7b336384bb8d2ffe8fd24ddce706..442696928c065551bf2ff781420b9459359d5ddf 100644 --- a/tests/models/test_huggingface.py +++ b/tests/models/test_huggingface.py @@ -5,7 +5,9 @@ import sys from pathlib import Path import numpy as np +import tokenizers import torch +from packaging.version import parse as parse_version from lm_eval import tasks from lm_eval.api.instance import Instance @@ -145,4 +147,7 @@ class Test_HFLM: context = self.LM.tok_batch_encode([TEST_STRING])[0] res = self.LM._model_generate(context, max_length=10, stop=["\n\n"]) res = self.LM.tok_decode(res[0]) - assert res == "foo bar\n!info bar" + if parse_version(tokenizers.__version__) >= parse_version("0.20.0"): + assert res == "foo bar\n !info bar" + else: + assert res == "foo bar\n!info bar" diff --git a/tests/models/test_neuron_optimum.py b/tests/models/test_neuron_optimum.py deleted file mode 100644 index 564d52303968e210439a7931f012487a959a367f..0000000000000000000000000000000000000000 --- a/tests/models/test_neuron_optimum.py +++ /dev/null @@ -1,26 +0,0 @@ -import pytest -import torch - -from lm_eval.models.neuron_optimum import wrap_constant_batch_size - - -def test_wrap_constant_batch_size(): - class Tester: - def __init__(self, batch_size): - self.batch_size = batch_size - - @wrap_constant_batch_size - def test_constant_batch_size(self, inputs): - assert len(inputs) == self.batch_size - return inputs - - batch_size_test = 8 - for i in range(1, batch_size_test + 1): - tensor = torch.ones([i, 2, 2]) - out = Tester(batch_size=batch_size_test).test_constant_batch_size(tensor) - torch.testing.assert_allclose(out, tensor) - - with pytest.raises(ValueError): - Tester(batch_size=batch_size_test).test_constant_batch_size( - torch.ones([batch_size_test + 1, 2, 2]) - ) diff --git a/tests/test_evaluator.py b/tests/test_evaluator.py index d5edf9aec2424811bcb527add6f80b4d25469f40..cc5f70fa73edf6a9dbd4d8d325dc3cc868eace6c 100644 --- a/tests/test_evaluator.py +++ b/tests/test_evaluator.py @@ -33,6 +33,7 @@ os.environ["TOKENIZERS_PARALLELISM"] = "false" 10000, ), ], + ids=lambda d: f"{d}", ) def test_evaluator( task_name: List[str], limit: int, model: str, model_args: str, bootstrap_iters: int @@ -89,7 +90,7 @@ def test_evaluator( "pretrained=EleutherAI/pythia-14m,dtype=float32,device=cpu", ), ( - ["mmlu_abstract_algebra", "mmlu_global_facts", "mmlu_public_relations"], + ["mmlu_stem"], 10, "hf", "pretrained=EleutherAI/pythia-14m,dtype=float32,device=cpu", @@ -107,6 +108,7 @@ def test_evaluator( "pretrained=EleutherAI/pythia-14m,dtype=float32,device=cpu", ), ], + ids=lambda d: f"{d}", ) def test_printed_results(task_name: List[str], limit: int, model: str, model_args: str): results = evaluator.simple_evaluate( @@ -144,6 +146,6 @@ def test_printed_results(task_name: List[str], limit: int, model: str, model_arg try: t1_item = float(t1_item) t2_item = float(t2_item) - assert abs(t1_item - t2_item) < 0.1 + assert abs(t1_item - t2_item) < 0.3 except ValueError: assert t1_item == t2_item diff --git a/tests/test_prompt.py b/tests/test_prompt.py new file mode 100644 index 0000000000000000000000000000000000000000..d1e5deb06507aaf85c1d17d88d6237ff0f042875 --- /dev/null +++ b/tests/test_prompt.py @@ -0,0 +1,119 @@ +import random +from typing import List + +import numpy as np +import pytest + +from lm_eval import tasks +from lm_eval.tasks import TaskManager +from lm_eval.utils import join_iters + + +MMLU_ANATOMY_ZERO_SHOT = """The following are multiple choice questions (with answers) about anatomy. + +A lesion causing compression of the facial nerve at the stylomastoid foramen will cause ipsilateral +A. paralysis of the facial muscles. +B. paralysis of the facial muscles and loss of taste. +C. paralysis of the facial muscles, loss of taste and lacrimation. +D. paralysis of the facial muscles, loss of taste, lacrimation and decreased salivation. +Answer:""" + +MMLU_ANATOMY_FIVE_SHOT = """The following are multiple choice questions (with answers) about anatomy. + +What is the embryological origin of the hyoid bone? +A. The first pharyngeal arch +B. The first and second pharyngeal arches +C. The second pharyngeal arch +D. The second and third pharyngeal arches +Answer: D + +Which of these branches of the trigeminal nerve contain somatic motor processes? +A. The supraorbital nerve +B. The infraorbital nerve +C. The mental nerve +D. None of the above +Answer: D + +The pleura +A. have no sensory innervation. +B. are separated by a 2 mm space. +C. extend into the neck. +D. are composed of respiratory epithelium. +Answer: C + +In Angle's Class II Div 2 occlusion there is +A. excess overbite of the upper lateral incisors. +B. negative overjet of the upper central incisors. +C. excess overjet of the upper lateral incisors. +D. excess overjet of the upper central incisors. +Answer: C + +Which of the following is the body cavity that contains the pituitary gland? +A. Abdominal +B. Cranial +C. Pleural +D. Spinal +Answer: B + +A lesion causing compression of the facial nerve at the stylomastoid foramen will cause ipsilateral +A. paralysis of the facial muscles. +B. paralysis of the facial muscles and loss of taste. +C. paralysis of the facial muscles, loss of taste and lacrimation. +D. paralysis of the facial muscles, loss of taste, lacrimation and decreased salivation. +Answer:""" + + +@pytest.mark.parametrize( + "task_names,sets,num_fewshot,seed,num_examples,expected_prompt", + [ + (["mmlu_anatomy"], "test", 0, 42, 1, MMLU_ANATOMY_ZERO_SHOT), + (["mmlu_anatomy"], "test", 5, 42, 1, MMLU_ANATOMY_FIVE_SHOT), + ], +) +def test_mmlu_prompt_rendering( + task_names: List[str], + sets: str, + num_fewshot: int, + seed: int, + num_examples: int, + expected_prompt: str, +): + np.random.seed(seed) + + task_manager = TaskManager() + task_dict = tasks.get_task_dict(task_names, task_manager) + + for task_name, task in task_dict.items(): + if isinstance(task, tuple): + _, task = task + + rnd = random.Random() + rnd.seed(seed) + + iters = [] + + for set in sets.split(","): + docs = None + if set == "train" and task.has_training_docs(): + docs = task.training_docs() + if set == "val" and task.has_validation_docs(): + docs = task.validation_docs() + if set == "test" and task.has_test_docs(): + docs = task.test_docs() + if docs is not None: + iters.append(docs) + + if len(iters) == 0: + raise ValueError + + docs = join_iters(iters) + + for i, doc in ( + zip(range(num_examples), docs) if num_examples > 0 else enumerate(docs) + ): + ctx = task.fewshot_context( + doc=doc, + num_fewshot=num_fewshot, + ) + + assert ctx == expected_prompt diff --git a/tests/test_task_manager.py b/tests/test_task_manager.py new file mode 100644 index 0000000000000000000000000000000000000000..1b10d3291a02d4a4b12fafde67641c0e38288233 --- /dev/null +++ b/tests/test_task_manager.py @@ -0,0 +1,73 @@ +import tempfile +from pathlib import Path + +import pytest + +from lm_eval.tasks import TaskManager + + +@pytest.fixture(scope="module") +def custom_task_name(): + return "zzz_my_python_task" + + +@pytest.fixture(scope="module") +def custom_task_tag(): + return "zzz-tag" + + +@pytest.fixture(scope="module") +def task_yaml(pytestconfig, custom_task_name, custom_task_tag): + yield f"""include: {pytestconfig.rootpath}/lm_eval/tasks/hellaswag/hellaswag.yaml +task: {custom_task_name} +class: !function {custom_task_name}.MockPythonTask +tag: + - {custom_task_tag} +""" + + +@pytest.fixture(scope="module") +def task_code(): + return """ +from lm_eval.tasks import ConfigurableTask + +class MockPythonTask(ConfigurableTask): + + def __init__( + self, + data_dir=None, + cache_dir=None, + download_mode=None, + config=None, + ) -> None: + config.pop("class") + super().__init__(data_dir, cache_dir, download_mode, config) +""" + + +@pytest.fixture(scope="module") +def custom_task_files_dir(task_yaml, task_code, custom_task_name): + with tempfile.TemporaryDirectory() as temp_dir: + yaml_path = Path(temp_dir) / f"{custom_task_name}.yaml" + with open(yaml_path, "w") as f: + f.write(task_yaml) + pysource_path = Path(temp_dir) / f"{custom_task_name}.py" + with open(pysource_path, "w") as f: + f.write(task_code) + yield temp_dir + + +def test_python_task_inclusion( + custom_task_files_dir: Path, custom_task_name: str, custom_task_tag: str +): + task_manager = TaskManager( + verbosity="INFO", include_path=str(custom_task_files_dir) + ) + # check if python tasks enters the global task_index + assert custom_task_name in task_manager.task_index + # check if subtask is present + assert custom_task_name in task_manager.all_subtasks + # check if tag is present + assert custom_task_tag in task_manager.all_tags + # check if it can be loaded by tag (custom_task_tag) + assert custom_task_name in task_manager.load_task_or_group(custom_task_tag) diff --git a/tests/test_tasks.py b/tests/test_tasks.py index c9453d54973ea9f094d9bfcaec1b30f8eeb4047d..e24109966effe396bb1ecd5768325db85aa7aa53 100644 --- a/tests/test_tasks.py +++ b/tests/test_tasks.py @@ -1,14 +1,17 @@ import os from itertools import islice +import datasets import pytest import lm_eval.tasks as tasks from lm_eval.api.task import ConfigurableTask +from lm_eval.evaluator_utils import get_task_list from .utils import new_tasks +datasets.config.HF_DATASETS_TRUST_REMOTE_CODE = True os.environ["TOKENIZERS_PARALLELISM"] = "false" task_manager = tasks.TaskManager() # Default Task @@ -20,10 +23,11 @@ def task_class(): # CI: new_tasks checks if any modifications have been made task_classes = new_tasks() # Check if task_classes is empty - if task_classes: - return list(task_manager.load_task_or_group(task_classes).values()) - else: - return list(task_manager.load_task_or_group(TASKS).values()) + task_classes = task_classes if task_classes else TASKS + res = tasks.get_task_dict(task_classes, task_manager) + res = [x.task for x in get_task_list(res)] + + return res @pytest.fixture() @@ -101,7 +105,11 @@ class TestNewTasks: ) _array_target = [task.doc_to_target(doc) for doc in arr] if task._config.output_type == "multiple_choice": - assert all(isinstance(label, int) for label in _array_target) + # TODO: label can be string or int; add better test conditions + assert all( + (isinstance(label, int) or isinstance(label, str)) + for label in _array_target + ) def test_build_all_requests(self, task_class, limit): task_class.build_all_requests(rank=1, limit=limit, world_size=1) diff --git a/tests/testdata/ai2_arc_10_hf_pretrained-EleutherAI-pythia-14m-dtype-float32-device-cpu.txt b/tests/testdata/ai2_arc_10_hf_pretrained-EleutherAI-pythia-14m-dtype-float32-device-cpu.txt index 79a693321ad1bf6fb0d65cab287569e4bfc0d459..6ba01f21d5711e90b22bafd9979f7a517b98b0ef 100644 --- a/tests/testdata/ai2_arc_10_hf_pretrained-EleutherAI-pythia-14m-dtype-float32-device-cpu.txt +++ b/tests/testdata/ai2_arc_10_hf_pretrained-EleutherAI-pythia-14m-dtype-float32-device-cpu.txt @@ -1,8 +1,6 @@ -| Tasks |Version|Filter|n-shot| Metric | |Value| |Stderr| -|----------------|-------|------|-----:|--------|---|----:|---|------| -|ai2_arc |N/A |none | 0|acc |↑ | 0.15|± |N/A | -| | |none | 0|acc_norm|↑ | 0.05|± |N/A | -| - arc_challenge| 1|none | 0|acc |↑ | 0.00|± |N/A | -| | |none | 0|acc_norm|↑ | 0.00|± |N/A | -| - arc_easy | 1|none | 0|acc |↑ | 0.30|± |N/A | -| | |none | 0|acc_norm|↑ | 0.10|± |N/A | \ No newline at end of file +| Tasks |Version|Filter|n-shot| Metric | |Value| |Stderr| +|-------------|------:|------|-----:|--------|---|----:|---|------| +|arc_challenge| 1|none | 0|acc |↑ | 0.0|± | N/A| +| | |none | 0|acc_norm|↑ | 0.0|± | N/A| +|arc_easy | 1|none | 0|acc |↑ | 0.3|± | N/A| +| | |none | 0|acc_norm|↑ | 0.1|± | N/A| \ No newline at end of file diff --git a/tests/testdata/lambada_openai_10_hf_pretrained-EleutherAI-pythia-14m-dtype-float32-device-cpu.txt b/tests/testdata/lambada_openai_10_hf_pretrained-EleutherAI-pythia-14m-dtype-float32-device-cpu.txt index 94bb6255fd045eae45e53e8c985e12100133e8bf..a3ea263b794b561212250ea3d962d1ba8eb89bde 100644 --- a/tests/testdata/lambada_openai_10_hf_pretrained-EleutherAI-pythia-14m-dtype-float32-device-cpu.txt +++ b/tests/testdata/lambada_openai_10_hf_pretrained-EleutherAI-pythia-14m-dtype-float32-device-cpu.txt @@ -1,4 +1,4 @@ | Tasks |Version|Filter|n-shot| Metric | | Value | |Stderr| |--------------|------:|------|-----:|----------|---|-------:|---|------| -|lambada_openai| 1|none | 0|acc |↑ | 0.1000|± |N/A | -| | |none | 0|perplexity|↓ |605.4879|± |N/A | \ No newline at end of file +|lambada_openai| 1|none | 0|acc |↑ | 0.1000|± | N/A| +| | |none | 0|perplexity|↓ |605.3866|± | N/A| \ No newline at end of file diff --git a/tests/testdata/mmlu_abstract_algebra-mmlu_global_facts-mmlu_public_relations_10_hf_pretrained-EleutherAI-pythia-14m-dtype-float32-device-cpu.txt b/tests/testdata/mmlu_abstract_algebra-mmlu_global_facts-mmlu_public_relations_10_hf_pretrained-EleutherAI-pythia-14m-dtype-float32-device-cpu.txt deleted file mode 100644 index 085534aef531036c907b7e6e8816ca6fd8820700..0000000000000000000000000000000000000000 --- a/tests/testdata/mmlu_abstract_algebra-mmlu_global_facts-mmlu_public_relations_10_hf_pretrained-EleutherAI-pythia-14m-dtype-float32-device-cpu.txt +++ /dev/null @@ -1,5 +0,0 @@ -| Tasks |Version|Filter|n-shot|Metric| |Value| |Stderr| -|----------------|------:|------|-----:|------|---|----:|---|------| -|abstract_algebra| 0|none | 0|acc |↑ | 0.2|± |N/A | -|global_facts | 0|none | 0|acc |↑ | 0.2|± |N/A | -|public_relations| 0|none | 0|acc |↑ | 0.2|± |N/A | \ No newline at end of file diff --git a/tests/testdata/mmlu_stem_10_hf_pretrained-EleutherAI-pythia-14m-dtype-float32-device-cpu.txt b/tests/testdata/mmlu_stem_10_hf_pretrained-EleutherAI-pythia-14m-dtype-float32-device-cpu.txt new file mode 100644 index 0000000000000000000000000000000000000000..8c6c56c1261f69f89dff5ac169fd75d16a5a9f43 --- /dev/null +++ b/tests/testdata/mmlu_stem_10_hf_pretrained-EleutherAI-pythia-14m-dtype-float32-device-cpu.txt @@ -0,0 +1,22 @@ +| Tasks |Version|Filter|n-shot|Metric| |Value | |Stderr| +|-------------------------------|------:|------|-----:|------|---|-----:|---|------| +|stem | 2|none | |acc |↑ |0.2474|± | N/A| +| - abstract_algebra | 1|none | 0|acc |↑ |0.2000|± | N/A| +| - anatomy | 1|none | 0|acc |↑ |0.3000|± | N/A| +| - astronomy | 1|none | 0|acc |↑ |0.1000|± | N/A| +| - college_biology | 1|none | 0|acc |↑ |0.3000|± | N/A| +| - college_chemistry | 1|none | 0|acc |↑ |0.1000|± | N/A| +| - college_computer_science | 1|none | 0|acc |↑ |0.2000|± | N/A| +| - college_mathematics | 1|none | 0|acc |↑ |0.2000|± | N/A| +| - college_physics | 1|none | 0|acc |↑ |0.3000|± | N/A| +| - computer_security | 1|none | 0|acc |↑ |0.5000|± | N/A| +| - conceptual_physics | 1|none | 0|acc |↑ |0.3000|± | N/A| +| - electrical_engineering | 1|none | 0|acc |↑ |0.4000|± | N/A| +| - elementary_mathematics | 1|none | 0|acc |↑ |0.0000|± | N/A| +| - high_school_biology | 1|none | 0|acc |↑ |0.3000|± | N/A| +| - high_school_chemistry | 1|none | 0|acc |↑ |0.4000|± | N/A| +| - high_school_computer_science| 1|none | 0|acc |↑ |0.3000|± | N/A| +| - high_school_mathematics | 1|none | 0|acc |↑ |0.2000|± | N/A| +| - high_school_physics | 1|none | 0|acc |↑ |0.3000|± | N/A| +| - high_school_statistics | 1|none | 0|acc |↑ |0.0000|± | N/A| +| - machine_learning | 1|none | 0|acc |↑ |0.3000|± | N/A| \ No newline at end of file diff --git a/tests/testdata/wikitext_10_hf_pretrained-EleutherAI-pythia-14m-dtype-float32-device-cpu.txt b/tests/testdata/wikitext_10_hf_pretrained-EleutherAI-pythia-14m-dtype-float32-device-cpu.txt index 5f78f2c189b110d61b25108da65910dead8deb5b..654e63ee7cb083d729d9be56e0d9d71ef1805928 100644 --- a/tests/testdata/wikitext_10_hf_pretrained-EleutherAI-pythia-14m-dtype-float32-device-cpu.txt +++ b/tests/testdata/wikitext_10_hf_pretrained-EleutherAI-pythia-14m-dtype-float32-device-cpu.txt @@ -1,5 +1,5 @@ | Tasks |Version|Filter|n-shot| Metric | | Value | |Stderr| |--------|------:|------|-----:|---------------|---|-------:|---|------| -|wikitext| 2|none | 0|bits_per_byte |↓ | 1.3394|± |N/A | -| | |none | 0|byte_perplexity|↓ | 2.5304|± |N/A | -| | |none | 0|word_perplexity|↓ |130.4812|± |N/A | \ No newline at end of file +|wikitext| 2|none | 0|bits_per_byte |↓ | 1.3394|± | N/A| +| | |none | 0|byte_perplexity|↓ | 2.5304|± | N/A| +| | |none | 0|word_perplexity|↓ |130.4801|± | N/A| \ No newline at end of file diff --git a/tests/testyamls/test-01.yaml b/tests/testyamls/test-01.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4e4367eeb3651bd49540009949fc080b9dba0a59 --- /dev/null +++ b/tests/testyamls/test-01.yaml @@ -0,0 +1,45 @@ +group: test-1 +group_alias: test 1 +task: + - piqa # string task + - ai2_arc # string tag + - task: super-glue-lm-eval-v1 # Should this be spread out? + num_fewshot: 3 + - task: swag # dict registered task + num_fewshot: 2 + - task: mmlu + num_fewshot: 5 + - group: nli-tasks # dict group + task: + - anli + - boolq + - sglue_rte + num_fewshot: 4 + metric_list: + - metric: brier_score + - task: sciq # dict registered task duplicate + task_alias: sciq 2-shot + num_fewshot: 2 + - task: sciq # dict registered task duplicate + task_alias: sciq 4-shot + num_fewshot: 4 + - task: sciq # dict registered task duplicate + task_alias: sciq 6-shot + num_fewshot: 6 + - task: siqa_custom # dict task + dataset_path: social_i_qa + dataset_name: null + output_type: multiple_choice + training_split: train + validation_split: validation + doc_to_text: "Question: {{context}} {{question}}\nAnswer:" + target_delimiter: " " + doc_to_choice: + - "{{answerA}}" + - "{{answerB}}" + - "{{answerC}}" + doc_to_target: "{{ (label|int) - 1 }}" + metric_list: + - metric: acc + aggregation: mean + higher_is_better: true