Add the original `LAMBADA` dataset

lm_eval/datasets/lambada/dataset_infos.json

-{"original": {"description": "LAMBADA is a dataset to evaluate the capabilities of computational models for text\nunderstanding by means of a word prediction task. LAMBADA is a collection of narrative\ntexts sharing the characteristic that human subjects are able to guess their last\nword if they are exposed to the whole text, but not if they only see the last\nsentence preceding the target word. To succeed on LAMBADA, computational models\ncannot simply rely on local context, but must be able to keep track of information\nin the broader discourse.\n\nThe LAMBADA dataset", "citation": "@misc{\n    author={Paperno, Denis and Kruszewski, Germ\u00e1n and Lazaridou, Angeliki and Pham, Quan Ngoc and Bernardi, Raffaella and Pezzelle, Sandro and Baroni, Marco and Boleda, Gemma and Fern\u00e1ndez, Raquel}, \n    title={The LAMBADA dataset},\n    DOI={10.5281/zenodo.2630551},\n    publisher={Zenodo},\n    year={2016},\n    month={Aug}\n}\n", "homepage": "https://zenodo.org/record/2630551#.X4Xzn5NKjUI", "license": "", "features": {"text": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "lambada", "config_name": "original", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"validation": {"name": "validation", "num_bytes": 1709449, "num_examples": 5153, "dataset_name": "lambada"}}, "download_checksums": {"http://eaidata.bmk.sh/data/lambada_test.jsonl": {"num_bytes": 1819752, "checksum": "4aa8d02cd17c719165fc8a7887fddd641f43fcafa4b1c806ca8abc31fabdb226"}}, "download_size": 1819752, "post_processing_size": null, "dataset_size": 1709449, "size_in_bytes": 3529201}, "en": {"description": "LAMBADA is a dataset to evaluate the capabilities of computational models for text\nunderstanding by means of a word prediction task. LAMBADA is a collection of narrative\ntexts sharing the characteristic that human subjects are able to guess their last\nword if they are exposed to the whole text, but not if they only see the last\nsentence preceding the target word. To succeed on LAMBADA, computational models\ncannot simply rely on local context, but must be able to keep track of information\nin the broader discourse.\n\nThe English translated LAMBADA dataset", "citation": "@misc{\n    author={Paperno, Denis and Kruszewski, Germ\u00e1n and Lazaridou, Angeliki and Pham, Quan Ngoc and Bernardi, Raffaella and Pezzelle, Sandro and Baroni, Marco and Boleda, Gemma and Fern\u00e1ndez, Raquel}, \n    title={The LAMBADA dataset},\n    DOI={10.5281/zenodo.2630551},\n    publisher={Zenodo},\n    year={2016},\n    month={Aug}\n}\n", "homepage": "https://zenodo.org/record/2630551#.X4Xzn5NKjUI", "license": "", "features": {"text": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "lambada", "config_name": "en", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"validation": {"name": "validation", "num_bytes": 1709449, "num_examples": 5153, "dataset_name": "lambada"}}, "download_checksums": {"http://eaidata.bmk.sh/data/lambada_test_en.jsonl": {"num_bytes": 1819752, "checksum": "4aa8d02cd17c719165fc8a7887fddd641f43fcafa4b1c806ca8abc31fabdb226"}}, "download_size": 1819752, "post_processing_size": null, "dataset_size": 1709449, "size_in_bytes": 3529201}, "fr": {"description": "LAMBADA is a dataset to evaluate the capabilities of computational models for text\nunderstanding by means of a word prediction task. LAMBADA is a collection of narrative\ntexts sharing the characteristic that human subjects are able to guess their last\nword if they are exposed to the whole text, but not if they only see the last\nsentence preceding the target word. To succeed on LAMBADA, computational models\ncannot simply rely on local context, but must be able to keep track of information\nin the broader discourse.\n\nThe French translated LAMBADA dataset", "citation": "@misc{\n    author={Paperno, Denis and Kruszewski, Germ\u00e1n and Lazaridou, Angeliki and Pham, Quan Ngoc and Bernardi, Raffaella and Pezzelle, Sandro and Baroni, Marco and Boleda, Gemma and Fern\u00e1ndez, Raquel}, \n    title={The LAMBADA dataset},\n    DOI={10.5281/zenodo.2630551},\n    publisher={Zenodo},\n    year={2016},\n    month={Aug}\n}\n", "homepage": "https://zenodo.org/record/2630551#.X4Xzn5NKjUI", "license": "", "features": {"text": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "lambada", "config_name": "fr", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"validation": {"name": "validation", "num_bytes": 1948795, "num_examples": 5153, "dataset_name": "lambada"}}, "download_checksums": {"http://eaidata.bmk.sh/data/lambada_test_fr.jsonl": {"num_bytes": 2028703, "checksum": "941ec6a73dba7dc91c860bf493eb66a527cd430148827a4753a4535a046bf362"}}, "download_size": 2028703, "post_processing_size": null, "dataset_size": 1948795, "size_in_bytes": 3977498}, "de": {"description": "LAMBADA is a dataset to evaluate the capabilities of computational models for text\nunderstanding by means of a word prediction task. LAMBADA is a collection of narrative\ntexts sharing the characteristic that human subjects are able to guess their last\nword if they are exposed to the whole text, but not if they only see the last\nsentence preceding the target word. To succeed on LAMBADA, computational models\ncannot simply rely on local context, but must be able to keep track of information\nin the broader discourse.\n\nThe German translated LAMBADA dataset", "citation": "@misc{\n    author={Paperno, Denis and Kruszewski, Germ\u00e1n and Lazaridou, Angeliki and Pham, Quan Ngoc and Bernardi, Raffaella and Pezzelle, Sandro and Baroni, Marco and Boleda, Gemma and Fern\u00e1ndez, Raquel}, \n    title={The LAMBADA dataset},\n    DOI={10.5281/zenodo.2630551},\n    publisher={Zenodo},\n    year={2016},\n    month={Aug}\n}\n", "homepage": "https://zenodo.org/record/2630551#.X4Xzn5NKjUI", "license": "", "features": {"text": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "lambada", "config_name": "de", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"validation": {"name": "validation", "num_bytes": 1904576, "num_examples": 5153, "dataset_name": "lambada"}}, "download_checksums": {"http://eaidata.bmk.sh/data/lambada_test_de.jsonl": {"num_bytes": 1985231, "checksum": "51c6c1795894c46e88e4c104b5667f488efe79081fb34d746b82b8caa663865e"}}, "download_size": 1985231, "post_processing_size": null, "dataset_size": 1904576, "size_in_bytes": 3889807}, "it": {"description": "LAMBADA is a dataset to evaluate the capabilities of computational models for text\nunderstanding by means of a word prediction task. LAMBADA is a collection of narrative\ntexts sharing the characteristic that human subjects are able to guess their last\nword if they are exposed to the whole text, but not if they only see the last\nsentence preceding the target word. To succeed on LAMBADA, computational models\ncannot simply rely on local context, but must be able to keep track of information\nin the broader discourse.\n\nThe Italian translated LAMBADA dataset", "citation": "@misc{\n    author={Paperno, Denis and Kruszewski, Germ\u00e1n and Lazaridou, Angeliki and Pham, Quan Ngoc and Bernardi, Raffaella and Pezzelle, Sandro and Baroni, Marco and Boleda, Gemma and Fern\u00e1ndez, Raquel}, \n    title={The LAMBADA dataset},\n    DOI={10.5281/zenodo.2630551},\n    publisher={Zenodo},\n    year={2016},\n    month={Aug}\n}\n", "homepage": "https://zenodo.org/record/2630551#.X4Xzn5NKjUI", "license": "", "features": {"text": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "lambada", "config_name": "it", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"validation": {"name": "validation", "num_bytes": 1813420, "num_examples": 5153, "dataset_name": "lambada"}}, "download_checksums": {"http://eaidata.bmk.sh/data/lambada_test_it.jsonl": {"num_bytes": 1894613, "checksum": "86654237716702ab74f42855ae5a78455c1b0e50054a4593fb9c6fcf7fad0850"}}, "download_size": 1894613, "post_processing_size": null, "dataset_size": 1813420, "size_in_bytes": 3708033}, "es": {"description": "LAMBADA is a dataset to evaluate the capabilities of computational models for text\nunderstanding by means of a word prediction task. LAMBADA is a collection of narrative\ntexts sharing the characteristic that human subjects are able to guess their last\nword if they are exposed to the whole text, but not if they only see the last\nsentence preceding the target word. To succeed on LAMBADA, computational models\ncannot simply rely on local context, but must be able to keep track of information\nin the broader discourse.\n\nThe Spanish translated LAMBADA dataset", "citation": "@misc{\n    author={Paperno, Denis and Kruszewski, Germ\u00e1n and Lazaridou, Angeliki and Pham, Quan Ngoc and Bernardi, Raffaella and Pezzelle, Sandro and Baroni, Marco and Boleda, Gemma and Fern\u00e1ndez, Raquel}, \n    title={The LAMBADA dataset},\n    DOI={10.5281/zenodo.2630551},\n    publisher={Zenodo},\n    year={2016},\n    month={Aug}\n}\n", "homepage": "https://zenodo.org/record/2630551#.X4Xzn5NKjUI", "license": "", "features": {"text": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "lambada", "config_name": "es", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"validation": {"name": "validation", "num_bytes": 1821735, "num_examples": 5153, "dataset_name": "lambada"}}, "download_checksums": {"http://eaidata.bmk.sh/data/lambada_test_es.jsonl": {"num_bytes": 1902349, "checksum": "ffd760026c647fb43c67ce1bc56fd527937304b348712dce33190ea6caba6f9c"}}, "download_size": 1902349, "post_processing_size": null, "dataset_size": 1821735, "size_in_bytes": 3724084}}

lm_eval/datasets/lambada_openai/dataset_infos.json

+{"original": {"description": "LAMBADA is a dataset to evaluate the capabilities of computational models for text\nunderstanding by means of a word prediction task. LAMBADA is a collection of narrative\ntexts sharing the characteristic that human subjects are able to guess their last\nword if they are exposed to the whole text, but not if they only see the last\nsentence preceding the target word. To succeed on LAMBADA, computational models\ncannot simply rely on local context, but must be able to keep track of information\nin the broader discourse.\n\nThe LAMBADA dataset", "citation": "@misc{\n    author={Paperno, Denis and Kruszewski, Germ\u00e1n and Lazaridou, Angeliki and Pham, Quan Ngoc and Bernardi, Raffaella and Pezzelle, Sandro and Baroni, Marco and Boleda, Gemma and Fern\u00e1ndez, Raquel}, \n    title={The LAMBADA dataset},\n    DOI={10.5281/zenodo.2630551},\n    publisher={Zenodo},\n    year={2016},\n    month={Aug}\n}\n", "homepage": "https://zenodo.org/record/2630551#.X4Xzn5NKjUI", "license": "", "features": {"text": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "lambada", "config_name": "original", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"validation": {"name": "validation", "num_bytes": 1709449, "num_examples": 5153, "dataset_name": "lambada"}}, "download_checksums": {"http://eaidata.bmk.sh/data/lambada_test.jsonl": {"num_bytes": 1819752, "checksum": "4aa8d02cd17c719165fc8a7887fddd641f43fcafa4b1c806ca8abc31fabdb226"}}, "download_size": 1819752, "post_processing_size": null, "dataset_size": 1709449, "size_in_bytes": 3529201}, "en": {"description": "The LAMBADA dataset as processed by OpenAI. It is used to evaluate the capabilities\nof computational models for text understanding by means of a word prediction task.\nLAMBADA is a collection of narrative texts sharing the characteristic that human subjects\nare able to guess their last word if they are exposed to the whole text, but not\nif they only see the last sentence preceding the target word. To succeed on LAMBADA,\ncomputational models cannot simply rely on local context, but must be able to keep track\nof information in the broader discourse.\n\nReference: https://github.com/openai/gpt-2/issues/131#issuecomment-497136199\n\nThe English translated LAMBADA OpenAI dataset", "citation": "@misc{\n    author={Paperno, Denis and Kruszewski, Germ\u00e1n and Lazaridou, Angeliki and Pham, Quan Ngoc and Bernardi, Raffaella and Pezzelle, Sandro and Baroni, Marco and Boleda, Gemma and Fern\u00e1ndez, Raquel},\n    title={The LAMBADA dataset},\n    DOI={10.5281/zenodo.2630551},\n    publisher={Zenodo},\n    year={2016},\n    month={Aug}\n}\n", "homepage": "https://zenodo.org/record/2630551#.X4Xzn5NKjUI", "license": "", "features": {"text": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "lambada_openai", "config_name": "en", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"validation": {"name": "validation", "num_bytes": 1709449, "num_examples": 5153, "dataset_name": "lambada_openai"}}, "download_checksums": {"http://eaidata.bmk.sh/data/lambada_test_en.jsonl": {"num_bytes": 1819752, "checksum": "4aa8d02cd17c719165fc8a7887fddd641f43fcafa4b1c806ca8abc31fabdb226"}}, "download_size": 1819752, "post_processing_size": null, "dataset_size": 1709449, "size_in_bytes": 3529201}, "fr": {"description": "The LAMBADA dataset as processed by OpenAI. It is used to evaluate the capabilities\nof computational models for text understanding by means of a word prediction task.\nLAMBADA is a collection of narrative texts sharing the characteristic that human subjects\nare able to guess their last word if they are exposed to the whole text, but not\nif they only see the last sentence preceding the target word. To succeed on LAMBADA,\ncomputational models cannot simply rely on local context, but must be able to keep track\nof information in the broader discourse.\n\nReference: https://github.com/openai/gpt-2/issues/131#issuecomment-497136199\n\nThe French translated LAMBADA OpenAI dataset", "citation": "@misc{\n    author={Paperno, Denis and Kruszewski, Germ\u00e1n and Lazaridou, Angeliki and Pham, Quan Ngoc and Bernardi, Raffaella and Pezzelle, Sandro and Baroni, Marco and Boleda, Gemma and Fern\u00e1ndez, Raquel},\n    title={The LAMBADA dataset},\n    DOI={10.5281/zenodo.2630551},\n    publisher={Zenodo},\n    year={2016},\n    month={Aug}\n}\n", "homepage": "https://zenodo.org/record/2630551#.X4Xzn5NKjUI", "license": "", "features": {"text": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "lambada_openai", "config_name": "fr", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"validation": {"name": "validation", "num_bytes": 1948795, "num_examples": 5153, "dataset_name": "lambada_openai"}}, "download_checksums": {"http://eaidata.bmk.sh/data/lambada_test_fr.jsonl": {"num_bytes": 2028703, "checksum": "941ec6a73dba7dc91c860bf493eb66a527cd430148827a4753a4535a046bf362"}}, "download_size": 2028703, "post_processing_size": null, "dataset_size": 1948795, "size_in_bytes": 3977498}, "de": {"description": "The LAMBADA dataset as processed by OpenAI. It is used to evaluate the capabilities\nof computational models for text understanding by means of a word prediction task.\nLAMBADA is a collection of narrative texts sharing the characteristic that human subjects\nare able to guess their last word if they are exposed to the whole text, but not\nif they only see the last sentence preceding the target word. To succeed on LAMBADA,\ncomputational models cannot simply rely on local context, but must be able to keep track\nof information in the broader discourse.\n\nReference: https://github.com/openai/gpt-2/issues/131#issuecomment-497136199\n\nThe German translated LAMBADA OpenAI dataset", "citation": "@misc{\n    author={Paperno, Denis and Kruszewski, Germ\u00e1n and Lazaridou, Angeliki and Pham, Quan Ngoc and Bernardi, Raffaella and Pezzelle, Sandro and Baroni, Marco and Boleda, Gemma and Fern\u00e1ndez, Raquel},\n    title={The LAMBADA dataset},\n    DOI={10.5281/zenodo.2630551},\n    publisher={Zenodo},\n    year={2016},\n    month={Aug}\n}\n", "homepage": "https://zenodo.org/record/2630551#.X4Xzn5NKjUI", "license": "", "features": {"text": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "lambada_openai", "config_name": "de", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"validation": {"name": "validation", "num_bytes": 1904576, "num_examples": 5153, "dataset_name": "lambada_openai"}}, "download_checksums": {"http://eaidata.bmk.sh/data/lambada_test_de.jsonl": {"num_bytes": 1985231, "checksum": "51c6c1795894c46e88e4c104b5667f488efe79081fb34d746b82b8caa663865e"}}, "download_size": 1985231, "post_processing_size": null, "dataset_size": 1904576, "size_in_bytes": 3889807}, "it": {"description": "The LAMBADA dataset as processed by OpenAI. It is used to evaluate the capabilities\nof computational models for text understanding by means of a word prediction task.\nLAMBADA is a collection of narrative texts sharing the characteristic that human subjects\nare able to guess their last word if they are exposed to the whole text, but not\nif they only see the last sentence preceding the target word. To succeed on LAMBADA,\ncomputational models cannot simply rely on local context, but must be able to keep track\nof information in the broader discourse.\n\nReference: https://github.com/openai/gpt-2/issues/131#issuecomment-497136199\n\nThe Italian translated LAMBADA OpenAI dataset", "citation": "@misc{\n    author={Paperno, Denis and Kruszewski, Germ\u00e1n and Lazaridou, Angeliki and Pham, Quan Ngoc and Bernardi, Raffaella and Pezzelle, Sandro and Baroni, Marco and Boleda, Gemma and Fern\u00e1ndez, Raquel},\n    title={The LAMBADA dataset},\n    DOI={10.5281/zenodo.2630551},\n    publisher={Zenodo},\n    year={2016},\n    month={Aug}\n}\n", "homepage": "https://zenodo.org/record/2630551#.X4Xzn5NKjUI", "license": "", "features": {"text": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "lambada_openai", "config_name": "it", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"validation": {"name": "validation", "num_bytes": 1813420, "num_examples": 5153, "dataset_name": "lambada_openai"}}, "download_checksums": {"http://eaidata.bmk.sh/data/lambada_test_it.jsonl": {"num_bytes": 1894613, "checksum": "86654237716702ab74f42855ae5a78455c1b0e50054a4593fb9c6fcf7fad0850"}}, "download_size": 1894613, "post_processing_size": null, "dataset_size": 1813420, "size_in_bytes": 3708033}, "es": {"description": "The LAMBADA dataset as processed by OpenAI. It is used to evaluate the capabilities\nof computational models for text understanding by means of a word prediction task.\nLAMBADA is a collection of narrative texts sharing the characteristic that human subjects\nare able to guess their last word if they are exposed to the whole text, but not\nif they only see the last sentence preceding the target word. To succeed on LAMBADA,\ncomputational models cannot simply rely on local context, but must be able to keep track\nof information in the broader discourse.\n\nReference: https://github.com/openai/gpt-2/issues/131#issuecomment-497136199\n\nThe Spanish translated LAMBADA OpenAI dataset", "citation": "@misc{\n    author={Paperno, Denis and Kruszewski, Germ\u00e1n and Lazaridou, Angeliki and Pham, Quan Ngoc and Bernardi, Raffaella and Pezzelle, Sandro and Baroni, Marco and Boleda, Gemma and Fern\u00e1ndez, Raquel},\n    title={The LAMBADA dataset},\n    DOI={10.5281/zenodo.2630551},\n    publisher={Zenodo},\n    year={2016},\n    month={Aug}\n}\n", "homepage": "https://zenodo.org/record/2630551#.X4Xzn5NKjUI", "license": "", "features": {"text": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "lambada_openai", "config_name": "es", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"validation": {"name": "validation", "num_bytes": 1821735, "num_examples": 5153, "dataset_name": "lambada_openai"}}, "download_checksums": {"http://eaidata.bmk.sh/data/lambada_test_es.jsonl": {"num_bytes": 1902349, "checksum": "ffd760026c647fb43c67ce1bc56fd527937304b348712dce33190ea6caba6f9c"}}, "download_size": 1902349, "post_processing_size": null, "dataset_size": 1821735, "size_in_bytes": 3724084}, "default": {"description": "The LAMBADA dataset as processed by OpenAI. It is used to evaluate the capabilities\nof computational models for text understanding by means of a word prediction task.\nLAMBADA is a collection of narrative texts sharing the characteristic that human subjects\nare able to guess their last word if they are exposed to the whole text, but not\nif they only see the last sentence preceding the target word. To succeed on LAMBADA,\ncomputational models cannot simply rely on local context, but must be able to keep track\nof information in the broader discourse.\n\nReference: https://github.com/openai/gpt-2/issues/131#issuecomment-497136199\n\nPre-processed English LAMBADA dataset from OpenAI", "citation": "@misc{\n    author={Paperno, Denis and Kruszewski, Germ\u00e1n and Lazaridou, Angeliki and Pham, Quan Ngoc and Bernardi, Raffaella and Pezzelle, Sandro and Baroni, Marco and Boleda, Gemma and Fern\u00e1ndez, Raquel},\n    title={The LAMBADA dataset},\n    DOI={10.5281/zenodo.2630551},\n    publisher={Zenodo},\n    year={2016},\n    month={Aug}\n}\n", "homepage": "https://zenodo.org/record/2630551#.X4Xzn5NKjUI", "license": "", "features": {"text": {"dtype": "string", "id": null, "_type": "Value"}}, "post_processed": null, "supervised_keys": null, "task_templates": null, "builder_name": "lambada_openai", "config_name": "default", "version": {"version_str": "0.0.1", "description": null, "major": 0, "minor": 0, "patch": 1}, "splits": {"validation": {"name": "validation", "num_bytes": 1709449, "num_examples": 5153, "dataset_name": "lambada_openai"}}, "download_checksums": {"https://openaipublic.blob.core.windows.net/gpt-2/data/lambada_test.jsonl": {"num_bytes": 1819752, "checksum": "4aa8d02cd17c719165fc8a7887fddd641f43fcafa4b1c806ca8abc31fabdb226"}}, "download_size": 1819752, "post_processing_size": null, "dataset_size": 1709449, "size_in_bytes": 3529201}}
\ No newline at end of file

lm_eval/datasets/lambada/lambada.py → lm_eval/datasets/lambada_openai/lambada_openai.py

-# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
+# Copyright 2022 The HuggingFace Datasets Authors and the current dataset script contributor.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # TODO: Address all TODOs and remove all explanatory comments
-"""LAMBADA dataset."""
+"""LAMBADA (OpenAI) dataset."""
 
 
 import json
@@ -32,13 +32,15 @@ _CITATION = """\
 """
 
 _DESCRIPTION = """\
-LAMBADA is a dataset to evaluate the capabilities of computational models for text
-understanding by means of a word prediction task. LAMBADA is a collection of narrative
-texts sharing the characteristic that human subjects are able to guess their last
-word if they are exposed to the whole text, but not if they only see the last
-sentence preceding the target word. To succeed on LAMBADA, computational models
-cannot simply rely on local context, but must be able to keep track of information
-in the broader discourse.
+The LAMBADA dataset as processed by OpenAI. It is used to evaluate the capabilities
+of computational models for text understanding by means of a word prediction task.
+LAMBADA is a collection of narrative texts sharing the characteristic that human subjects
+are able to guess their last word if they are exposed to the whole text, but not
+if they only see the last sentence preceding the target word. To succeed on LAMBADA,
+computational models cannot simply rely on local context, but must be able to keep track
+of information in the broader discourse.
+
+Reference: https://github.com/openai/gpt-2/issues/131#issuecomment-497136199
 """
 
 _HOMEPAGE = "https://zenodo.org/record/2630551#.X4Xzn5NKjUI"
@@ -47,7 +49,7 @@ _HOMEPAGE = "https://zenodo.org/record/2630551#.X4Xzn5NKjUI"
 _LICENSE = ""
 
 _URLS = {
-    "original": "http://eaidata.bmk.sh/data/lambada_test.jsonl",
+    "default": "https://openaipublic.blob.core.windows.net/gpt-2/data/lambada_test.jsonl",
     "en": "http://eaidata.bmk.sh/data/lambada_test_en.jsonl",
     "fr": "http://eaidata.bmk.sh/data/lambada_test_fr.jsonl",
     "de": "http://eaidata.bmk.sh/data/lambada_test_de.jsonl",
@@ -56,43 +58,45 @@ _URLS = {
 }
 
 
-class Lambada(datasets.GeneratorBasedBuilder):
+class LambadaOpenAI(datasets.GeneratorBasedBuilder):
     """LAMBADA is a dataset to evaluate the capabilities of computational models for text understanding by means of a word prediction task."""
 
     VERSION = datasets.Version("0.0.1")
 
     BUILDER_CONFIGS = [
         datasets.BuilderConfig(
-            name="original", version=VERSION, description="The LAMBADA dataset"
+            name="default",
+            version=VERSION,
+            description="Pre-processed English LAMBADA dataset from OpenAI",
         ),
         datasets.BuilderConfig(
             name="en",
             version=VERSION,
-            description="The English translated LAMBADA dataset",
+            description="The English translated LAMBADA OpenAI dataset",
         ),
         datasets.BuilderConfig(
             name="fr",
             version=VERSION,
-            description="The French translated LAMBADA dataset",
+            description="The French translated LAMBADA OpenAI dataset",
         ),
         datasets.BuilderConfig(
             name="de",
             version=VERSION,
-            description="The German translated LAMBADA dataset",
+            description="The German translated LAMBADA OpenAI dataset",
         ),
         datasets.BuilderConfig(
             name="it",
             version=VERSION,
-            description="The Italian translated LAMBADA dataset",
+            description="The Italian translated LAMBADA OpenAI dataset",
         ),
         datasets.BuilderConfig(
             name="es",
             version=VERSION,
-            description="The Spanish translated LAMBADA dataset",
+            description="The Spanish translated LAMBADA OpenAI dataset",
         ),
     ]
 
-    DEFAULT_CONFIG_NAME = "original"
+    DEFAULT_CONFIG_NAME = "default"
 
     def _info(self):
         features = datasets.Features(

lm_eval/tasks/__init__.py

@@ -104,8 +104,10 @@ TASK_REGISTRY = {
     # Order by benchmark/genre?
     "coqa": coqa.CoQA,
     "drop": drop.DROP,
-    "lambada": lambada.LAMBADA,
-    "lambada_cloze": lambada_cloze.LAMBADA_cloze,
+    "lambada_openai": lambada.LambadaOpenAI,
+    "lambada_standard": lambada.LambadaStandard,
+    "lambada_openai_cloze": lambada_cloze.LambadaOpenAICloze,
+    "lambada_standard_cloze": lambada_cloze.LambadaStandardCloze,
     # multilingual lambada
     **lambada_multilingual.construct_tasks(),
     "wikitext": wikitext.WikiText,

lm_eval/tasks/lambada.py

@@ -13,7 +13,7 @@ in the broader discourse.
 Homepage: https://zenodo.org/record/2630551#.X4Xzn5NKjUI
 """
 import inspect
-import lm_eval.datasets.lambada.lambada
+import lm_eval.datasets.lambada_openai.lambada_openai
 from lm_eval.base import Task, rf
 from lm_eval.metrics import mean, perplexity
 
@@ -30,27 +30,20 @@ _CITATION = """
 """
 
 
-class LAMBADA(Task):
-    VERSION = 0
-    DATASET_PATH = inspect.getfile(lm_eval.datasets.lambada.lambada)
-
-    def has_training_docs(self):
-        return False
-
-    def has_validation_docs(self):
-        return True
-
-    def has_test_docs(self):
-        return False
+class LambadaBase(Task):
+    VERSION = None
 
     def training_docs(self):
-        pass
+        if self.has_training_docs():
+            return self.dataset["train"]
 
     def validation_docs(self):
-        return self.dataset["validation"]
+        if self.has_validation_docs():
+            return self.dataset["validation"]
 
     def test_docs(self):
-        pass
+        if self.has_test_docs():
+            return self.dataset["test"]
 
     def doc_to_text(self, doc):
         return doc["text"].rsplit(" ", 1)[0]
@@ -79,3 +72,39 @@ class LAMBADA(Task):
 
     def higher_is_better(self):
         return {"ppl": False, "acc": True}
+
+
+class LambadaStandard(LambadaBase):
+    """The LAMBADA task using the standard original LAMBADA dataset."""
+
+    VERSION = 0
+    DATASET_PATH = "lambada"
+
+    def has_training_docs(self):
+        return False
+
+    def has_validation_docs(self):
+        return True
+
+    def has_test_docs(self):
+        return True
+
+
+class LambadaOpenAI(LambadaBase):
+    """The LAMBADA task using the LAMBADA OpenAI dataset, a modified version of the
+    original LAMBADA dataset created by OpenAI for evaluating their GPT-2 model.
+
+    Reference: https://github.com/openai/gpt-2/issues/131#issuecomment-497136199
+    """
+
+    VERSION = 0
+    DATASET_PATH = inspect.getfile(lm_eval.datasets.lambada_openai.lambada_openai)
+
+    def has_training_docs(self):
+        return False
+
+    def has_validation_docs(self):
+        return True
+
+    def has_test_docs(self):
+        return False

lm_eval/tasks/lambada_cloze.py

@@ -13,7 +13,7 @@ in the broader discourse.
 
 Homepage: https://zenodo.org/record/2630551#.X4Xzn5NKjUI
 """
-from lm_eval.tasks.lambada import LAMBADA
+from lm_eval.tasks.lambada import LambadaOpenAI, LambadaStandard
 
 
 _CITATION = """
@@ -28,7 +28,27 @@ _CITATION = """
 """
 
 
-class LAMBADA_cloze(LAMBADA):
+class LambadaStandardCloze(LambadaStandard):
+    """Cloze-style LambadaStandard."""
+
+    VERSION = 0
+
+    def doc_to_text(self, doc):
+        return doc["text"].rsplit(" ", 1)[0] + " ____. ->"
+
+    def should_decontaminate(self):
+        return True
+
+    def doc_to_decontamination_query(self, doc):
+        return doc["text"]
+
+    def doc_to_target(self, doc):
+        return " " + doc["text"].rsplit(" ", 1)[1]
+
+
+class LambadaOpenAICloze(LambadaOpenAI):
+    """Cloze-style LambadaOpenAI."""
+
     VERSION = 0
 
     def doc_to_text(self, doc):

lm_eval/tasks/lambada_multilingual.py

 """
-The LAMBADA dataset: Word prediction requiring a broad discourse context∗
+The LAMBADA (OpenAI) dataset: Word prediction requiring a broad discourse context∗
 https://arxiv.org/pdf/1606.06031.pdf
 
-The LAMBADA dataset machine-translated to other languages.
+The LAMBADA OpenAI dataset machine-translated to other languages.
 LAMBADA is a dataset to evaluate the capabilities of computational models for text
 understanding by means of a word prediction task. LAMBADA is a collection of narrative
 passages sharing the characteristic that human subjects are able to guess their last
@@ -12,8 +12,10 @@ cannot simply rely on local context, but must be able to keep track of informati
 in the broader discourse.
 
 Homepage: https://zenodo.org/record/2630551#.X4Xzn5NKjUI
+
+Reference (OpenAI): https://github.com/openai/gpt-2/issues/131#issuecomment-497136199
 """
-from . import lambada
+from .lambada import LambadaOpenAI
 
 
 _CITATION = """
@@ -28,41 +30,42 @@ _CITATION = """
 """
 
 
-class MultilingualLAMBADA(lambada.LAMBADA):
+class LambadaOpenAIMultilingualEnglish(LambadaOpenAI):
     VERSION = 0
-
-
-class MultilingualLAMBADAEN(MultilingualLAMBADA):
     DATASET_NAME = "en"
 
 
-class MultilingualLAMBADAFR(MultilingualLAMBADA):
+class LambadaOpenAIMultilingualFrench(LambadaOpenAI):
+    VERSION = 0
     DATASET_NAME = "fr"
 
 
-class MultilingualLAMBADADE(MultilingualLAMBADA):
+class LambadaOpenAIMultilingualGerman(LambadaOpenAI):
+    VERSION = 0
     DATASET_NAME = "de"
 
 
-class MultilingualLAMBADAIT(MultilingualLAMBADA):
+class LambadaOpenAIMultilingualItalian(LambadaOpenAI):
+    VERSION = 0
     DATASET_NAME = "it"
 
 
-class MultilingualLAMBADAES(MultilingualLAMBADA):
+class LambadaOpenAIMultilingualSpanish(LambadaOpenAI):
+    VERSION = 0
     DATASET_NAME = "es"
 
 
 LANG_CLASSES = [
-    MultilingualLAMBADAEN,
-    MultilingualLAMBADAFR,
-    MultilingualLAMBADADE,
-    MultilingualLAMBADAIT,
-    MultilingualLAMBADAES,
+    LambadaOpenAIMultilingualEnglish,
+    LambadaOpenAIMultilingualFrench,
+    LambadaOpenAIMultilingualGerman,
+    LambadaOpenAIMultilingualItalian,
+    LambadaOpenAIMultilingualSpanish,
 ]
 
 
 def construct_tasks():
     tasks = {}
     for lang_class in LANG_CLASSES:
-        tasks[f"lambada_mt_{lang_class.DATASET_NAME}"] = lang_class
+        tasks[f"lambada_openai_mt_{lang_class.DATASET_NAME}"] = lang_class
     return tasks

tests/testdata/lambada_openai-v0-loglikelihood

+6829e6a8aa5922e6c92dd31403cc060f242dc0ede4a775e085a70da095ab2e20
\ No newline at end of file

tests/testdata/lambada_openai-v0-res.json

+{"results": {"lambada_openai": {"acc": 0.0, "acc_stderr": 0.0, "ppl": 1.6479047769869253, "ppl_stderr": 0.006497321146240192}}, "versions": {"lambada_openai": 0}}
\ No newline at end of file

tests/testdata/lambada_openai_cloze-v0-loglikelihood

+7655e748b63ae7e9911411d2d2a2577221d6c861ca4448509992541294d689f3
\ No newline at end of file

tests/testdata/lambada_openai_cloze-v0-res.json

+{"results": {"lambada_openai_cloze": {"acc": 0.0, "acc_stderr": 0.0, "ppl": 1.6479047769869253, "ppl_stderr": 0.006497321146240192}}, "versions": {"lambada_openai_cloze": 0}}
\ No newline at end of file

tests/testdata/lambada_openai_mt_de-v0-loglikelihood

+5ad125e1708499832b2cee8c3388f89f9c0277010fd96fbd3359039ce8105984
\ No newline at end of file

tests/testdata/lambada_openai_mt_de-v0-res.json

+{"results": {"lambada_openai_mt_de": {"acc": 0.0, "acc_stderr": 0.0, "ppl": 1.6479047769869253, "ppl_stderr": 0.006497321146240192}}, "versions": {"lambada_openai_mt_de": 0}}
\ No newline at end of file

tests/testdata/lambada_openai_mt_en-v0-loglikelihood

+6829e6a8aa5922e6c92dd31403cc060f242dc0ede4a775e085a70da095ab2e20
\ No newline at end of file

tests/testdata/lambada_openai_mt_en-v0-res.json

+{"results": {"lambada_openai_mt_en": {"acc": 0.0, "acc_stderr": 0.0, "ppl": 1.6479047769869253, "ppl_stderr": 0.006497321146240192}}, "versions": {"lambada_openai_mt_en": 0}}
\ No newline at end of file

tests/testdata/lambada_openai_mt_es-v0-loglikelihood

+4a88f4b316c72fe0396c382d6cbb33568ac4d0ad225150d3536635c085359fc9
\ No newline at end of file

tests/testdata/lambada_openai_mt_es-v0-res.json

+{"results": {"lambada_openai_mt_es": {"acc": 0.0, "acc_stderr": 0.0, "ppl": 1.6479047769869253, "ppl_stderr": 0.006497321146240192}}, "versions": {"lambada_openai_mt_es": 0}}
\ No newline at end of file

tests/testdata/lambada_openai_mt_fr-v0-loglikelihood

+5d16f4a0c51dc6d7b6df2ebeba2bbfa51e700b843779b559b3d90183d7b02a11
\ No newline at end of file

tests/testdata/lambada_openai_mt_fr-v0-res.json

+{"results": {"lambada_openai_mt_fr": {"acc": 0.0, "acc_stderr": 0.0, "ppl": 1.6479047769869253, "ppl_stderr": 0.006497321146240192}}, "versions": {"lambada_openai_mt_fr": 0}}
\ No newline at end of file