{"coqa":{"description":"CoQA is a large-scale dataset for building Conversational Question Answering\nsystems. The goal of the CoQA challenge is to measure the ability of machines to\nunderstand a text passage and answer a series of interconnected questions that\nappear in a conversation.\n","citation":"@misc{reddy2018coqa,\n title={CoQA: A Conversational Question Answering Challenge},\n author={Siva Reddy and Danqi Chen and Christopher D. Manning},\n year={2018},\n eprint={1808.07042},\n archivePrefix={arXiv},\n primaryClass={cs.CL}\n}\n","homepage":"https://stanfordnlp.github.io/coqa/","license":"","features":{"id":{"dtype":"string","id":null,"_type":"Value"},"source":{"dtype":"string","id":null,"_type":"Value"},"story":{"dtype":"string","id":null,"_type":"Value"},"questions":{"feature":{"input_text":{"dtype":"string","id":null,"_type":"Value"},"turn_id":{"dtype":"int32","id":null,"_type":"Value"}},"length":-1,"id":null,"_type":"Sequence"},"answers":{"feature":{"span_start":{"dtype":"int32","id":null,"_type":"Value"},"span_end":{"dtype":"int32","id":null,"_type":"Value"},"span_text":{"dtype":"string","id":null,"_type":"Value"},"input_text":{"dtype":"string","id":null,"_type":"Value"},"turn_id":{"dtype":"int32","id":null,"_type":"Value"}},"length":-1,"id":null,"_type":"Sequence"},"additional_answers":{"0":{"feature":{"span_start":{"dtype":"int32","id":null,"_type":"Value"},"span_end":{"dtype":"int32","id":null,"_type":"Value"},"span_text":{"dtype":"string","id":null,"_type":"Value"},"input_text":{"dtype":"string","id":null,"_type":"Value"},"turn_id":{"dtype":"int32","id":null,"_type":"Value"}},"length":-1,"id":null,"_type":"Sequence"},"1":{"feature":{"span_start":{"dtype":"int32","id":null,"_type":"Value"},"span_end":{"dtype":"int32","id":null,"_type":"Value"},"span_text":{"dtype":"string","id":null,"_type":"Value"},"input_text":{"dtype":"string","id":null,"_type":"Value"},"turn_id":{"dtype":"int32","id":null,"_type":"Value"}},"length":-1,"id":null,"_type":"Sequence"},"2":{"feature":{"span_start":{"dtype":"int32","id":null,"_type":"Value"},"span_end":{"dtype":"int32","id":null,"_type":"Value"},"span_text":{"dtype":"string","id":null,"_type":"Value"},"input_text":{"dtype":"string","id":null,"_type":"Value"},"turn_id":{"dtype":"int32","id":null,"_type":"Value"}},"length":-1,"id":null,"_type":"Sequence"}}},"post_processed":null,"supervised_keys":null,"task_templates":null,"builder_name":"coqa","config_name":"coqa","version":{"version_str":"0.0.1","description":null,"major":0,"minor":0,"patch":1},"splits":{"train":{"name":"train","num_bytes":26250528,"num_examples":7199,"dataset_name":"coqa"},"validation":{"name":"validation","num_bytes":3765933,"num_examples":500,"dataset_name":"coqa"}},"download_checksums":{"https://nlp.stanford.edu/data/coqa/coqa-train-v1.0.json":{"num_bytes":49001836,"checksum":"b0fdb2bc1bd38dd3ca2ce5fa2ac3e02c6288ac914f241ac409a655ffb6619fa6"},"https://nlp.stanford.edu/data/coqa/coqa-dev-v1.0.json":{"num_bytes":9090845,"checksum":"dfa367a9733ce53222918d0231d9b3bedc2b8ee831a2845f62dfc70701f2540a"}},"download_size":58092681,"post_processing_size":null,"dataset_size":30016461,"size_in_bytes":88109142}}
\ No newline at end of file
{"coqa":{"description":"CoQA is a large-scale dataset for building Conversational Question Answering\nsystems. The goal of the CoQA challenge is to measure the ability of machines to\nunderstand a text passage and answer a series of interconnected questions that\nappear in a conversation.\n","citation":"@misc{reddy2018coqa,\n title={CoQA: A Conversational Question Answering Challenge},\n author={Siva Reddy and Danqi Chen and Christopher D. Manning},\n year={2018},\n eprint={1808.07042},\n archivePrefix={arXiv},\n primaryClass={cs.CL}\n}\n","homepage":"https://stanfordnlp.github.io/coqa/","license":"","features":{"id":{"dtype":"string","id":null,"_type":"Value"},"source":{"dtype":"string","id":null,"_type":"Value"},"story":{"dtype":"string","id":null,"_type":"Value"},"questions":{"feature":{"input_text":{"dtype":"string","id":null,"_type":"Value"},"turn_id":{"dtype":"int32","id":null,"_type":"Value"}},"length":-1,"id":null,"_type":"Sequence"},"answers":{"feature":{"span_start":{"dtype":"int32","id":null,"_type":"Value"},"span_end":{"dtype":"int32","id":null,"_type":"Value"},"span_text":{"dtype":"string","id":null,"_type":"Value"},"input_text":{"dtype":"string","id":null,"_type":"Value"},"turn_id":{"dtype":"int32","id":null,"_type":"Value"}},"length":-1,"id":null,"_type":"Sequence"},"additional_answers":{"0":{"feature":{"span_start":{"dtype":"int32","id":null,"_type":"Value"},"span_end":{"dtype":"int32","id":null,"_type":"Value"},"span_text":{"dtype":"string","id":null,"_type":"Value"},"input_text":{"dtype":"string","id":null,"_type":"Value"},"turn_id":{"dtype":"int32","id":null,"_type":"Value"}},"length":-1,"id":null,"_type":"Sequence"},"1":{"feature":{"span_start":{"dtype":"int32","id":null,"_type":"Value"},"span_end":{"dtype":"int32","id":null,"_type":"Value"},"span_text":{"dtype":"string","id":null,"_type":"Value"},"input_text":{"dtype":"string","id":null,"_type":"Value"},"turn_id":{"dtype":"int32","id":null,"_type":"Value"}},"length":-1,"id":null,"_type":"Sequence"},"2":{"feature":{"span_start":{"dtype":"int32","id":null,"_type":"Value"},"span_end":{"dtype":"int32","id":null,"_type":"Value"},"span_text":{"dtype":"string","id":null,"_type":"Value"},"input_text":{"dtype":"string","id":null,"_type":"Value"},"turn_id":{"dtype":"int32","id":null,"_type":"Value"}},"length":-1,"id":null,"_type":"Sequence"}}},"post_processed":null,"supervised_keys":null,"task_templates":null,"builder_name":"coqa","config_name":"coqa","version":{"version_str":"0.0.1","description":null,"major":0,"minor":0,"patch":1},"splits":{"train":{"name":"train","num_bytes":26250528,"num_examples":7199,"dataset_name":"coqa"},"validation":{"name":"validation","num_bytes":3765933,"num_examples":500,"dataset_name":"coqa"}},"download_checksums":{"https://nlp.stanford.edu/data/coqa/coqa-train-v1.0.json":{"num_bytes":49001836,"checksum":"b0fdb2bc1bd38dd3ca2ce5fa2ac3e02c6288ac914f241ac409a655ffb6619fa6"},"https://nlp.stanford.edu/data/coqa/coqa-dev-v1.0.json":{"num_bytes":9090845,"checksum":"dfa367a9733ce53222918d0231d9b3bedc2b8ee831a2845f62dfc70701f2540a"}},"download_size":58092681,"post_processing_size":null,"dataset_size":30016461,"size_in_bytes":88109142}}
{"drop":{"description":"DROP is a QA dataset which tests comprehensive understanding of paragraphs. In \nthis crowdsourced, adversarially-created, 96k question-answering benchmark, a \nsystem must resolve multiple references in a question, map them onto a paragraph,\nand perform discrete operations over them (such as addition, counting, or sorting).\n","citation":"@misc{dua2019drop,\n title={DROP: A Reading Comprehension Benchmark Requiring Discrete Reasoning Over Paragraphs}, \n author={Dheeru Dua and Yizhong Wang and Pradeep Dasigi and Gabriel Stanovsky and Sameer Singh and Matt Gardner},\n year={2019},\n eprint={1903.00161},\n archivePrefix={arXiv},\n primaryClass={cs.CL}\n}\n","homepage":"https://allenai.org/data/drop","license":"","features":{"section_id":{"dtype":"string","id":null,"_type":"Value"},"passage":{"dtype":"string","id":null,"_type":"Value"},"question":{"dtype":"string","id":null,"_type":"Value"},"query_id":{"dtype":"string","id":null,"_type":"Value"},"answer":{"number":{"dtype":"string","id":null,"_type":"Value"},"date":{"day":{"dtype":"string","id":null,"_type":"Value"},"month":{"dtype":"string","id":null,"_type":"Value"},"year":{"dtype":"string","id":null,"_type":"Value"}},"spans":{"feature":{"dtype":"string","id":null,"_type":"Value"},"length":-1,"id":null,"_type":"Sequence"},"worker_id":{"dtype":"string","id":null,"_type":"Value"},"hit_id":{"dtype":"string","id":null,"_type":"Value"}},"validated_answers":{"feature":{"number":{"dtype":"string","id":null,"_type":"Value"},"date":{"day":{"dtype":"string","id":null,"_type":"Value"},"month":{"dtype":"string","id":null,"_type":"Value"},"year":{"dtype":"string","id":null,"_type":"Value"}},"spans":{"feature":{"dtype":"string","id":null,"_type":"Value"},"length":-1,"id":null,"_type":"Sequence"},"worker_id":{"dtype":"string","id":null,"_type":"Value"},"hit_id":{"dtype":"string","id":null,"_type":"Value"}},"length":-1,"id":null,"_type":"Sequence"}},"post_processed":null,"supervised_keys":null,"task_templates":null,"builder_name":"drop","config_name":"drop","version":{"version_str":"0.0.1","description":null,"major":0,"minor":0,"patch":1},"splits":{"train":{"name":"train","num_bytes":108858121,"num_examples":77409,"dataset_name":"drop"},"validation":{"name":"validation","num_bytes":12560739,"num_examples":9536,"dataset_name":"drop"}},"download_checksums":{"https://s3-us-west-2.amazonaws.com/allennlp/datasets/drop/drop_dataset.zip":{"num_bytes":8308692,"checksum":"39d2278a29fd729de301b111a45f434c24834f40df8f4ff116d864589e3249d6"}},"download_size":8308692,"post_processing_size":null,"dataset_size":121418860,"size_in_bytes":129727552}}
\ No newline at end of file
{"drop":{"description":"DROP is a QA dataset which tests comprehensive understanding of paragraphs. In \nthis crowdsourced, adversarially-created, 96k question-answering benchmark, a \nsystem must resolve multiple references in a question, map them onto a paragraph,\nand perform discrete operations over them (such as addition, counting, or sorting).\n","citation":"@misc{dua2019drop,\n title={DROP: A Reading Comprehension Benchmark Requiring Discrete Reasoning Over Paragraphs}, \n author={Dheeru Dua and Yizhong Wang and Pradeep Dasigi and Gabriel Stanovsky and Sameer Singh and Matt Gardner},\n year={2019},\n eprint={1903.00161},\n archivePrefix={arXiv},\n primaryClass={cs.CL}\n}\n","homepage":"https://allenai.org/data/drop","license":"","features":{"section_id":{"dtype":"string","id":null,"_type":"Value"},"passage":{"dtype":"string","id":null,"_type":"Value"},"question":{"dtype":"string","id":null,"_type":"Value"},"query_id":{"dtype":"string","id":null,"_type":"Value"},"answer":{"number":{"dtype":"string","id":null,"_type":"Value"},"date":{"day":{"dtype":"string","id":null,"_type":"Value"},"month":{"dtype":"string","id":null,"_type":"Value"},"year":{"dtype":"string","id":null,"_type":"Value"}},"spans":{"feature":{"dtype":"string","id":null,"_type":"Value"},"length":-1,"id":null,"_type":"Sequence"},"worker_id":{"dtype":"string","id":null,"_type":"Value"},"hit_id":{"dtype":"string","id":null,"_type":"Value"}},"validated_answers":{"feature":{"number":{"dtype":"string","id":null,"_type":"Value"},"date":{"day":{"dtype":"string","id":null,"_type":"Value"},"month":{"dtype":"string","id":null,"_type":"Value"},"year":{"dtype":"string","id":null,"_type":"Value"}},"spans":{"feature":{"dtype":"string","id":null,"_type":"Value"},"length":-1,"id":null,"_type":"Sequence"},"worker_id":{"dtype":"string","id":null,"_type":"Value"},"hit_id":{"dtype":"string","id":null,"_type":"Value"}},"length":-1,"id":null,"_type":"Sequence"}},"post_processed":null,"supervised_keys":null,"task_templates":null,"builder_name":"drop","config_name":"drop","version":{"version_str":"0.0.1","description":null,"major":0,"minor":0,"patch":1},"splits":{"train":{"name":"train","num_bytes":108858121,"num_examples":77409,"dataset_name":"drop"},"validation":{"name":"validation","num_bytes":12560739,"num_examples":9536,"dataset_name":"drop"}},"download_checksums":{"https://s3-us-west-2.amazonaws.com/allennlp/datasets/drop/drop_dataset.zip":{"num_bytes":8308692,"checksum":"39d2278a29fd729de301b111a45f434c24834f40df8f4ff116d864589e3249d6"}},"download_size":8308692,"post_processing_size":null,"dataset_size":121418860,"size_in_bytes":129727552}}
{"gsm8k":{"description":"State-of-the-art language models can match human performance on many tasks, but \nthey still struggle to robustly perform multi-step mathematical reasoning. To \ndiagnose the failures of current models and support research, we introduce GSM8K,\na dataset of 8.5K high quality linguistically diverse grade school math word problems.\nWe find that even the largest transformer models fail to achieve high test performance, \ndespite the conceptual simplicity of this problem distribution.\n","citation":"@misc{cobbe2021training,\n title={Training Verifiers to Solve Math Word Problems},\n author={Karl Cobbe and Vineet Kosaraju and Mohammad Bavarian and Jacob Hilton and Reiichiro Nakano and Christopher Hesse and John Schulman},\n year={2021},\n eprint={2110.14168},\n archivePrefix={arXiv},\n primaryClass={cs.LG}\n}\n","homepage":"https://github.com/openai/grade-school-math","license":"","features":{"question":{"dtype":"string","id":null,"_type":"Value"},"answer":{"dtype":"string","id":null,"_type":"Value"}},"post_processed":null,"supervised_keys":null,"task_templates":null,"builder_name":"gsm8_k","config_name":"gsm8k","version":{"version_str":"0.0.1","description":null,"major":0,"minor":0,"patch":1},"splits":{"train":{"name":"train","num_bytes":3963202,"num_examples":7473,"dataset_name":"gsm8_k"},"test":{"name":"test","num_bytes":713732,"num_examples":1319,"dataset_name":"gsm8_k"}},"download_checksums":{"https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/train.jsonl":{"num_bytes":4166206,"checksum":"17f347dc51477c50d4efb83959dbb7c56297aba886e5544ee2aaed3024813465"},"https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl":{"num_bytes":749738,"checksum":"3730d312f6e3440559ace48831e51066acaca737f6eabec99bccb9e4b3c39d14"}},"download_size":4915944,"post_processing_size":null,"dataset_size":4676934,"size_in_bytes":9592878}}
{"es":{"description":"HEAD-QA is a multi-choice HEAlthcare Dataset. The questions come from exams to access a specialized position in the\nSpanish healthcare system, and are challenging even for highly specialized humans. They are designed by the Ministerio\nde Sanidad, Consumo y Bienestar Social.\nThe dataset contains questions about the following topics: medicine, nursing, psychology, chemistry, pharmacology and biology.\n","citation":"@inproceedings{vilares-gomez-rodriguez-2019-head,\n title = \"{HEAD}-{QA}: A Healthcare Dataset for Complex Reasoning\",\n author = \"Vilares, David and\n G{'o}mez-Rodr{'i}guez, Carlos\",\n booktitle = \"Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics\",\n month = jul,\n year = \"2019\",\n address = \"Florence, Italy\",\n publisher = \"Association for Computational Linguistics\",\n url = \"https://www.aclweb.org/anthology/P19-1092\",\n doi = \"10.18653/v1/P19-1092\",\n pages = \"960--966\",\n abstract = \"We present HEAD-QA, a multi-choice question answering testbed to encourage research on complex reasoning. The questions come from exams to access a specialized position in the Spanish healthcare system, and are challenging even for highly specialized humans. We then consider monolingual (Spanish) and cross-lingual (to English) experiments with information retrieval and neural techniques. We show that: (i) HEAD-QA challenges current methods, and (ii) the results lag well behind human performance, demonstrating its usefulness as a benchmark for future work.\",\n}\n","homepage":"https://aghie.github.io/head-qa/","license":"MIT License","features":{"name":{"dtype":"string","id":null,"_type":"Value"},"year":{"dtype":"string","id":null,"_type":"Value"},"category":{"dtype":"string","id":null,"_type":"Value"},"qid":{"dtype":"int32","id":null,"_type":"Value"},"qtext":{"dtype":"string","id":null,"_type":"Value"},"ra":{"dtype":"int32","id":null,"_type":"Value"},"answers":[{"aid":{"dtype":"int32","id":null,"_type":"Value"},"atext":{"dtype":"string","id":null,"_type":"Value"}}]},"post_processed":null,"supervised_keys":null,"task_templates":null,"builder_name":"head_qa","config_name":"es","version":{"version_str":"1.1.0","description":null,"major":1,"minor":1,"patch":0},"splits":{"train":{"name":"train","num_bytes":1196021,"num_examples":2657,"dataset_name":"head_qa"},"test":{"name":"test","num_bytes":1169819,"num_examples":2742,"dataset_name":"head_qa"},"validation":{"name":"validation","num_bytes":556924,"num_examples":1366,"dataset_name":"head_qa"}},"download_checksums":{"https://drive.google.com/uc?export=download&confirm=t&id=1a_95N5zQQoUCq8IBNVZgziHbeM-QxG2t":{"num_bytes":79365502,"checksum":"6ec29a3f55153d167f0bdf05395558919ba0b1df9c63e79ffceda2a09884ad8b"}},"download_size":79365502,"post_processing_size":null,"dataset_size":2922764,"size_in_bytes":82288266},"en":{"description":"HEAD-QA is a multi-choice HEAlthcare Dataset. The questions come from exams to access a specialized position in the\nSpanish healthcare system, and are challenging even for highly specialized humans. They are designed by the Ministerio\nde Sanidad, Consumo y Bienestar Social.\nThe dataset contains questions about the following topics: medicine, nursing, psychology, chemistry, pharmacology and biology.\n","citation":"@inproceedings{vilares-gomez-rodriguez-2019-head,\n title = \"{HEAD}-{QA}: A Healthcare Dataset for Complex Reasoning\",\n author = \"Vilares, David and\n G{'o}mez-Rodr{'i}guez, Carlos\",\n booktitle = \"Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics\",\n month = jul,\n year = \"2019\",\n address = \"Florence, Italy\",\n publisher = \"Association for Computational Linguistics\",\n url = \"https://www.aclweb.org/anthology/P19-1092\",\n doi = \"10.18653/v1/P19-1092\",\n pages = \"960--966\",\n abstract = \"We present HEAD-QA, a multi-choice question answering testbed to encourage research on complex reasoning. The questions come from exams to access a specialized position in the Spanish healthcare system, and are challenging even for highly specialized humans. We then consider monolingual (Spanish) and cross-lingual (to English) experiments with information retrieval and neural techniques. We show that: (i) HEAD-QA challenges current methods, and (ii) the results lag well behind human performance, demonstrating its usefulness as a benchmark for future work.\",\n}\n","homepage":"https://aghie.github.io/head-qa/","license":"MIT License","features":{"name":{"dtype":"string","id":null,"_type":"Value"},"year":{"dtype":"string","id":null,"_type":"Value"},"category":{"dtype":"string","id":null,"_type":"Value"},"qid":{"dtype":"int32","id":null,"_type":"Value"},"qtext":{"dtype":"string","id":null,"_type":"Value"},"ra":{"dtype":"int32","id":null,"_type":"Value"},"answers":[{"aid":{"dtype":"int32","id":null,"_type":"Value"},"atext":{"dtype":"string","id":null,"_type":"Value"}}]},"post_processed":null,"supervised_keys":null,"task_templates":null,"builder_name":"head_qa","config_name":"en","version":{"version_str":"1.1.0","description":null,"major":1,"minor":1,"patch":0},"splits":{"train":{"name":"train","num_bytes":1123151,"num_examples":2657,"dataset_name":"head_qa"},"test":{"name":"test","num_bytes":1097349,"num_examples":2742,"dataset_name":"head_qa"},"validation":{"name":"validation","num_bytes":523462,"num_examples":1366,"dataset_name":"head_qa"}},"download_checksums":{"https://drive.google.com/uc?export=download&confirm=t&id=1a_95N5zQQoUCq8IBNVZgziHbeM-QxG2t":{"num_bytes":79365502,"checksum":"6ec29a3f55153d167f0bdf05395558919ba0b1df9c63e79ffceda2a09884ad8b"}},"download_size":79365502,"post_processing_size":null,"dataset_size":2743962,"size_in_bytes":82109464}}
\ No newline at end of file
{"es":{"description":"HEAD-QA is a multi-choice HEAlthcare Dataset. The questions come from exams to access a specialized position in the\nSpanish healthcare system, and are challenging even for highly specialized humans. They are designed by the Ministerio\nde Sanidad, Consumo y Bienestar Social.\nThe dataset contains questions about the following topics: medicine, nursing, psychology, chemistry, pharmacology and biology.\n","citation":"@inproceedings{vilares-gomez-rodriguez-2019-head,\n title = \"{HEAD}-{QA}: A Healthcare Dataset for Complex Reasoning\",\n author = \"Vilares, David and\n G{'o}mez-Rodr{'i}guez, Carlos\",\n booktitle = \"Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics\",\n month = jul,\n year = \"2019\",\n address = \"Florence, Italy\",\n publisher = \"Association for Computational Linguistics\",\n url = \"https://www.aclweb.org/anthology/P19-1092\",\n doi = \"10.18653/v1/P19-1092\",\n pages = \"960--966\",\n abstract = \"We present HEAD-QA, a multi-choice question answering testbed to encourage research on complex reasoning. The questions come from exams to access a specialized position in the Spanish healthcare system, and are challenging even for highly specialized humans. We then consider monolingual (Spanish) and cross-lingual (to English) experiments with information retrieval and neural techniques. We show that: (i) HEAD-QA challenges current methods, and (ii) the results lag well behind human performance, demonstrating its usefulness as a benchmark for future work.\",\n}\n","homepage":"https://aghie.github.io/head-qa/","license":"MIT License","features":{"name":{"dtype":"string","id":null,"_type":"Value"},"year":{"dtype":"string","id":null,"_type":"Value"},"category":{"dtype":"string","id":null,"_type":"Value"},"qid":{"dtype":"int32","id":null,"_type":"Value"},"qtext":{"dtype":"string","id":null,"_type":"Value"},"ra":{"dtype":"int32","id":null,"_type":"Value"},"answers":[{"aid":{"dtype":"int32","id":null,"_type":"Value"},"atext":{"dtype":"string","id":null,"_type":"Value"}}]},"post_processed":null,"supervised_keys":null,"task_templates":null,"builder_name":"head_qa","config_name":"es","version":{"version_str":"1.1.0","description":null,"major":1,"minor":1,"patch":0},"splits":{"train":{"name":"train","num_bytes":1196021,"num_examples":2657,"dataset_name":"head_qa"},"test":{"name":"test","num_bytes":1169819,"num_examples":2742,"dataset_name":"head_qa"},"validation":{"name":"validation","num_bytes":556924,"num_examples":1366,"dataset_name":"head_qa"}},"download_checksums":{"https://drive.google.com/uc?export=download&confirm=t&id=1a_95N5zQQoUCq8IBNVZgziHbeM-QxG2t":{"num_bytes":79365502,"checksum":"6ec29a3f55153d167f0bdf05395558919ba0b1df9c63e79ffceda2a09884ad8b"}},"download_size":79365502,"post_processing_size":null,"dataset_size":2922764,"size_in_bytes":82288266},"en":{"description":"HEAD-QA is a multi-choice HEAlthcare Dataset. The questions come from exams to access a specialized position in the\nSpanish healthcare system, and are challenging even for highly specialized humans. They are designed by the Ministerio\nde Sanidad, Consumo y Bienestar Social.\nThe dataset contains questions about the following topics: medicine, nursing, psychology, chemistry, pharmacology and biology.\n","citation":"@inproceedings{vilares-gomez-rodriguez-2019-head,\n title = \"{HEAD}-{QA}: A Healthcare Dataset for Complex Reasoning\",\n author = \"Vilares, David and\n G{'o}mez-Rodr{'i}guez, Carlos\",\n booktitle = \"Proceedings of the 57th Annual Meeting of the Association for Computational Linguistics\",\n month = jul,\n year = \"2019\",\n address = \"Florence, Italy\",\n publisher = \"Association for Computational Linguistics\",\n url = \"https://www.aclweb.org/anthology/P19-1092\",\n doi = \"10.18653/v1/P19-1092\",\n pages = \"960--966\",\n abstract = \"We present HEAD-QA, a multi-choice question answering testbed to encourage research on complex reasoning. The questions come from exams to access a specialized position in the Spanish healthcare system, and are challenging even for highly specialized humans. We then consider monolingual (Spanish) and cross-lingual (to English) experiments with information retrieval and neural techniques. We show that: (i) HEAD-QA challenges current methods, and (ii) the results lag well behind human performance, demonstrating its usefulness as a benchmark for future work.\",\n}\n","homepage":"https://aghie.github.io/head-qa/","license":"MIT License","features":{"name":{"dtype":"string","id":null,"_type":"Value"},"year":{"dtype":"string","id":null,"_type":"Value"},"category":{"dtype":"string","id":null,"_type":"Value"},"qid":{"dtype":"int32","id":null,"_type":"Value"},"qtext":{"dtype":"string","id":null,"_type":"Value"},"ra":{"dtype":"int32","id":null,"_type":"Value"},"answers":[{"aid":{"dtype":"int32","id":null,"_type":"Value"},"atext":{"dtype":"string","id":null,"_type":"Value"}}]},"post_processed":null,"supervised_keys":null,"task_templates":null,"builder_name":"head_qa","config_name":"en","version":{"version_str":"1.1.0","description":null,"major":1,"minor":1,"patch":0},"splits":{"train":{"name":"train","num_bytes":1123151,"num_examples":2657,"dataset_name":"head_qa"},"test":{"name":"test","num_bytes":1097349,"num_examples":2742,"dataset_name":"head_qa"},"validation":{"name":"validation","num_bytes":523462,"num_examples":1366,"dataset_name":"head_qa"}},"download_checksums":{"https://drive.google.com/uc?export=download&confirm=t&id=1a_95N5zQQoUCq8IBNVZgziHbeM-QxG2t":{"num_bytes":79365502,"checksum":"6ec29a3f55153d167f0bdf05395558919ba0b1df9c63e79ffceda2a09884ad8b"}},"download_size":79365502,"post_processing_size":null,"dataset_size":2743962,"size_in_bytes":82109464}}
{"commonsense":{"description":"The ETHICS dataset is a benchmark that spans concepts in justice, well-being,\nduties, virtues, and commonsense morality. Models predict widespread moral\njudgments about diverse text scenarios. This requires connecting physical and\nsocial world knowledge to value judgements, a capability that may enable us\nto steer chatbot outputs or eventually regularize open-ended reinforcement\nlearning agents.\n\nThe Commonsense subset contains examples focusing on moral standards and principles that most people intuitively accept.","citation":"@article{hendrycks2021ethics\n title={Aligning AI With Shared Human Values},\n author={Dan Hendrycks and Collin Burns and Steven Basart and Andrew Critch and Jerry Li and Dawn Song and Jacob Steinhardt},\n journal={Proceedings of the International Conference on Learning Representations (ICLR)},\n year={2021}\n}\n","homepage":"https://github.com/hendrycks/ethics","license":"","features":{"label":{"dtype":"int32","id":null,"_type":"Value"},"input":{"dtype":"string","id":null,"_type":"Value"},"is_short":{"dtype":"bool","id":null,"_type":"Value"},"edited":{"dtype":"bool","id":null,"_type":"Value"}},"post_processed":null,"supervised_keys":null,"task_templates":null,"builder_name":"hendrycks_ethics","config_name":"commonsense","version":{"version_str":"0.0.1","description":null,"major":0,"minor":0,"patch":1},"splits":{"train":{"name":"train","num_bytes":14435215,"num_examples":13910,"dataset_name":"hendrycks_ethics"},"test":{"name":"test","num_bytes":3150094,"num_examples":3885,"dataset_name":"hendrycks_ethics"}},"download_checksums":{"https://people.eecs.berkeley.edu/~hendrycks/ethics.tar":{"num_bytes":35585024,"checksum":"40acbf1ac0da79a2aabef394d58889136b8d38b05be09482006de2453fb06333"}},"download_size":35585024,"post_processing_size":null,"dataset_size":17585309,"size_in_bytes":53170333},"deontology":{"description":"The ETHICS dataset is a benchmark that spans concepts in justice, well-being,\nduties, virtues, and commonsense morality. Models predict widespread moral\njudgments about diverse text scenarios. This requires connecting physical and\nsocial world knowledge to value judgements, a capability that may enable us\nto steer chatbot outputs or eventually regularize open-ended reinforcement\nlearning agents.\n\nThe Deontology subset contains examples focusing on whether an act is required, permitted, or forbidden according to a set of rules or constraints","citation":"@article{hendrycks2021ethics\n title={Aligning AI With Shared Human Values},\n author={Dan Hendrycks and Collin Burns and Steven Basart and Andrew Critch and Jerry Li and Dawn Song and Jacob Steinhardt},\n journal={Proceedings of the International Conference on Learning Representations (ICLR)},\n year={2021}\n}\n","homepage":"https://github.com/hendrycks/ethics","license":"","features":{"group_id":{"dtype":"int32","id":null,"_type":"Value"},"label":{"dtype":"int32","id":null,"_type":"Value"},"scenario":{"dtype":"string","id":null,"_type":"Value"},"excuse":{"dtype":"string","id":null,"_type":"Value"}},"post_processed":null,"supervised_keys":null,"task_templates":null,"builder_name":"hendrycks_ethics","config_name":"deontology","version":{"version_str":"0.0.1","description":null,"major":0,"minor":0,"patch":1},"splits":{"train":{"name":"train","num_bytes":1931475,"num_examples":18164,"dataset_name":"hendrycks_ethics"},"test":{"name":"test","num_bytes":384602,"num_examples":3596,"dataset_name":"hendrycks_ethics"}},"download_checksums":{"https://people.eecs.berkeley.edu/~hendrycks/ethics.tar":{"num_bytes":35585024,"checksum":"40acbf1ac0da79a2aabef394d58889136b8d38b05be09482006de2453fb06333"}},"download_size":35585024,"post_processing_size":null,"dataset_size":2316077,"size_in_bytes":37901101},"justice":{"description":"The ETHICS dataset is a benchmark that spans concepts in justice, well-being,\nduties, virtues, and commonsense morality. Models predict widespread moral\njudgments about diverse text scenarios. This requires connecting physical and\nsocial world knowledge to value judgements, a capability that may enable us\nto steer chatbot outputs or eventually regularize open-ended reinforcement\nlearning agents.\n\nThe Justice subset contains examples focusing on how a character treats another person","citation":"@article{hendrycks2021ethics\n title={Aligning AI With Shared Human Values},\n author={Dan Hendrycks and Collin Burns and Steven Basart and Andrew Critch and Jerry Li and Dawn Song and Jacob Steinhardt},\n journal={Proceedings of the International Conference on Learning Representations (ICLR)},\n year={2021}\n}\n","homepage":"https://github.com/hendrycks/ethics","license":"","features":{"group_id":{"dtype":"int32","id":null,"_type":"Value"},"label":{"dtype":"int32","id":null,"_type":"Value"},"scenario":{"dtype":"string","id":null,"_type":"Value"}},"post_processed":null,"supervised_keys":null,"task_templates":null,"builder_name":"hendrycks_ethics","config_name":"justice","version":{"version_str":"0.0.1","description":null,"major":0,"minor":0,"patch":1},"splits":{"train":{"name":"train","num_bytes":2516501,"num_examples":21791,"dataset_name":"hendrycks_ethics"},"test":{"name":"test","num_bytes":309427,"num_examples":2704,"dataset_name":"hendrycks_ethics"}},"download_checksums":{"https://people.eecs.berkeley.edu/~hendrycks/ethics.tar":{"num_bytes":35585024,"checksum":"40acbf1ac0da79a2aabef394d58889136b8d38b05be09482006de2453fb06333"}},"download_size":35585024,"post_processing_size":null,"dataset_size":2825928,"size_in_bytes":38410952},"utilitarianism":{"description":"The ETHICS dataset is a benchmark that spans concepts in justice, well-being,\nduties, virtues, and commonsense morality. Models predict widespread moral\njudgments about diverse text scenarios. This requires connecting physical and\nsocial world knowledge to value judgements, a capability that may enable us\nto steer chatbot outputs or eventually regularize open-ended reinforcement\nlearning agents.\n\nThe Utilitarianism subset contains scenarios that should be ranked from most pleasant to least pleasant for the person in the scenario","citation":"@article{hendrycks2021ethics\n title={Aligning AI With Shared Human Values},\n author={Dan Hendrycks and Collin Burns and Steven Basart and Andrew Critch and Jerry Li and Dawn Song and Jacob Steinhardt},\n journal={Proceedings of the International Conference on Learning Representations (ICLR)},\n year={2021}\n}\n","homepage":"https://github.com/hendrycks/ethics","license":"","features":{"activity":{"dtype":"string","id":null,"_type":"Value"},"baseline":{"dtype":"string","id":null,"_type":"Value"},"rating":{"dtype":"string","id":null,"_type":"Value"}},"post_processed":null,"supervised_keys":null,"task_templates":null,"builder_name":"hendrycks_ethics","config_name":"utilitarianism","version":{"version_str":"0.0.1","description":null,"major":0,"minor":0,"patch":1},"splits":{"train":{"name":"train","num_bytes":2241770,"num_examples":13738,"dataset_name":"hendrycks_ethics"},"test":{"name":"test","num_bytes":749768,"num_examples":4808,"dataset_name":"hendrycks_ethics"}},"download_checksums":{"https://people.eecs.berkeley.edu/~hendrycks/ethics.tar":{"num_bytes":35585024,"checksum":"40acbf1ac0da79a2aabef394d58889136b8d38b05be09482006de2453fb06333"}},"download_size":35585024,"post_processing_size":null,"dataset_size":2991538,"size_in_bytes":38576562},"virtue":{"description":"The ETHICS dataset is a benchmark that spans concepts in justice, well-being,\nduties, virtues, and commonsense morality. Models predict widespread moral\njudgments about diverse text scenarios. This requires connecting physical and\nsocial world knowledge to value judgements, a capability that may enable us\nto steer chatbot outputs or eventually regularize open-ended reinforcement\nlearning agents.\n\nThe Virtue subset contains scenarios focusing on whether virtues or vices are being exemplified","citation":"@article{hendrycks2021ethics\n title={Aligning AI With Shared Human Values},\n author={Dan Hendrycks and Collin Burns and Steven Basart and Andrew Critch and Jerry Li and Dawn Song and Jacob Steinhardt},\n journal={Proceedings of the International Conference on Learning Representations (ICLR)},\n year={2021}\n}\n","homepage":"https://github.com/hendrycks/ethics","license":"","features":{"group_id":{"dtype":"int32","id":null,"_type":"Value"},"label":{"dtype":"int32","id":null,"_type":"Value"},"scenario":{"dtype":"string","id":null,"_type":"Value"},"trait":{"dtype":"string","id":null,"_type":"Value"}},"post_processed":null,"supervised_keys":null,"task_templates":null,"builder_name":"hendrycks_ethics","config_name":"virtue","version":{"version_str":"0.0.1","description":null,"major":0,"minor":0,"patch":1},"splits":{"train":{"name":"train","num_bytes":2640328,"num_examples":28245,"dataset_name":"hendrycks_ethics"},"test":{"name":"test","num_bytes":473473,"num_examples":4975,"dataset_name":"hendrycks_ethics"}},"download_checksums":{"https://people.eecs.berkeley.edu/~hendrycks/ethics.tar":{"num_bytes":35585024,"checksum":"40acbf1ac0da79a2aabef394d58889136b8d38b05be09482006de2453fb06333"}},"download_size":35585024,"post_processing_size":null,"dataset_size":3113801,"size_in_bytes":38698825}}
\ No newline at end of file
{"commonsense":{"description":"The ETHICS dataset is a benchmark that spans concepts in justice, well-being,\nduties, virtues, and commonsense morality. Models predict widespread moral\njudgments about diverse text scenarios. This requires connecting physical and\nsocial world knowledge to value judgements, a capability that may enable us\nto steer chatbot outputs or eventually regularize open-ended reinforcement\nlearning agents.\n\nThe Commonsense subset contains examples focusing on moral standards and principles that most people intuitively accept.","citation":"@article{hendrycks2021ethics\n title={Aligning AI With Shared Human Values},\n author={Dan Hendrycks and Collin Burns and Steven Basart and Andrew Critch and Jerry Li and Dawn Song and Jacob Steinhardt},\n journal={Proceedings of the International Conference on Learning Representations (ICLR)},\n year={2021}\n}\n","homepage":"https://github.com/hendrycks/ethics","license":"","features":{"label":{"dtype":"int32","id":null,"_type":"Value"},"input":{"dtype":"string","id":null,"_type":"Value"},"is_short":{"dtype":"bool","id":null,"_type":"Value"},"edited":{"dtype":"bool","id":null,"_type":"Value"}},"post_processed":null,"supervised_keys":null,"task_templates":null,"builder_name":"hendrycks_ethics","config_name":"commonsense","version":{"version_str":"0.0.1","description":null,"major":0,"minor":0,"patch":1},"splits":{"train":{"name":"train","num_bytes":14435215,"num_examples":13910,"dataset_name":"hendrycks_ethics"},"test":{"name":"test","num_bytes":3150094,"num_examples":3885,"dataset_name":"hendrycks_ethics"}},"download_checksums":{"https://people.eecs.berkeley.edu/~hendrycks/ethics.tar":{"num_bytes":35585024,"checksum":"40acbf1ac0da79a2aabef394d58889136b8d38b05be09482006de2453fb06333"}},"download_size":35585024,"post_processing_size":null,"dataset_size":17585309,"size_in_bytes":53170333},"deontology":{"description":"The ETHICS dataset is a benchmark that spans concepts in justice, well-being,\nduties, virtues, and commonsense morality. Models predict widespread moral\njudgments about diverse text scenarios. This requires connecting physical and\nsocial world knowledge to value judgements, a capability that may enable us\nto steer chatbot outputs or eventually regularize open-ended reinforcement\nlearning agents.\n\nThe Deontology subset contains examples focusing on whether an act is required, permitted, or forbidden according to a set of rules or constraints","citation":"@article{hendrycks2021ethics\n title={Aligning AI With Shared Human Values},\n author={Dan Hendrycks and Collin Burns and Steven Basart and Andrew Critch and Jerry Li and Dawn Song and Jacob Steinhardt},\n journal={Proceedings of the International Conference on Learning Representations (ICLR)},\n year={2021}\n}\n","homepage":"https://github.com/hendrycks/ethics","license":"","features":{"group_id":{"dtype":"int32","id":null,"_type":"Value"},"label":{"dtype":"int32","id":null,"_type":"Value"},"scenario":{"dtype":"string","id":null,"_type":"Value"},"excuse":{"dtype":"string","id":null,"_type":"Value"}},"post_processed":null,"supervised_keys":null,"task_templates":null,"builder_name":"hendrycks_ethics","config_name":"deontology","version":{"version_str":"0.0.1","description":null,"major":0,"minor":0,"patch":1},"splits":{"train":{"name":"train","num_bytes":1931475,"num_examples":18164,"dataset_name":"hendrycks_ethics"},"test":{"name":"test","num_bytes":384602,"num_examples":3596,"dataset_name":"hendrycks_ethics"}},"download_checksums":{"https://people.eecs.berkeley.edu/~hendrycks/ethics.tar":{"num_bytes":35585024,"checksum":"40acbf1ac0da79a2aabef394d58889136b8d38b05be09482006de2453fb06333"}},"download_size":35585024,"post_processing_size":null,"dataset_size":2316077,"size_in_bytes":37901101},"justice":{"description":"The ETHICS dataset is a benchmark that spans concepts in justice, well-being,\nduties, virtues, and commonsense morality. Models predict widespread moral\njudgments about diverse text scenarios. This requires connecting physical and\nsocial world knowledge to value judgements, a capability that may enable us\nto steer chatbot outputs or eventually regularize open-ended reinforcement\nlearning agents.\n\nThe Justice subset contains examples focusing on how a character treats another person","citation":"@article{hendrycks2021ethics\n title={Aligning AI With Shared Human Values},\n author={Dan Hendrycks and Collin Burns and Steven Basart and Andrew Critch and Jerry Li and Dawn Song and Jacob Steinhardt},\n journal={Proceedings of the International Conference on Learning Representations (ICLR)},\n year={2021}\n}\n","homepage":"https://github.com/hendrycks/ethics","license":"","features":{"group_id":{"dtype":"int32","id":null,"_type":"Value"},"label":{"dtype":"int32","id":null,"_type":"Value"},"scenario":{"dtype":"string","id":null,"_type":"Value"}},"post_processed":null,"supervised_keys":null,"task_templates":null,"builder_name":"hendrycks_ethics","config_name":"justice","version":{"version_str":"0.0.1","description":null,"major":0,"minor":0,"patch":1},"splits":{"train":{"name":"train","num_bytes":2516501,"num_examples":21791,"dataset_name":"hendrycks_ethics"},"test":{"name":"test","num_bytes":309427,"num_examples":2704,"dataset_name":"hendrycks_ethics"}},"download_checksums":{"https://people.eecs.berkeley.edu/~hendrycks/ethics.tar":{"num_bytes":35585024,"checksum":"40acbf1ac0da79a2aabef394d58889136b8d38b05be09482006de2453fb06333"}},"download_size":35585024,"post_processing_size":null,"dataset_size":2825928,"size_in_bytes":38410952},"utilitarianism":{"description":"The ETHICS dataset is a benchmark that spans concepts in justice, well-being,\nduties, virtues, and commonsense morality. Models predict widespread moral\njudgments about diverse text scenarios. This requires connecting physical and\nsocial world knowledge to value judgements, a capability that may enable us\nto steer chatbot outputs or eventually regularize open-ended reinforcement\nlearning agents.\n\nThe Utilitarianism subset contains scenarios that should be ranked from most pleasant to least pleasant for the person in the scenario","citation":"@article{hendrycks2021ethics\n title={Aligning AI With Shared Human Values},\n author={Dan Hendrycks and Collin Burns and Steven Basart and Andrew Critch and Jerry Li and Dawn Song and Jacob Steinhardt},\n journal={Proceedings of the International Conference on Learning Representations (ICLR)},\n year={2021}\n}\n","homepage":"https://github.com/hendrycks/ethics","license":"","features":{"activity":{"dtype":"string","id":null,"_type":"Value"},"baseline":{"dtype":"string","id":null,"_type":"Value"},"rating":{"dtype":"string","id":null,"_type":"Value"}},"post_processed":null,"supervised_keys":null,"task_templates":null,"builder_name":"hendrycks_ethics","config_name":"utilitarianism","version":{"version_str":"0.0.1","description":null,"major":0,"minor":0,"patch":1},"splits":{"train":{"name":"train","num_bytes":2241770,"num_examples":13738,"dataset_name":"hendrycks_ethics"},"test":{"name":"test","num_bytes":749768,"num_examples":4808,"dataset_name":"hendrycks_ethics"}},"download_checksums":{"https://people.eecs.berkeley.edu/~hendrycks/ethics.tar":{"num_bytes":35585024,"checksum":"40acbf1ac0da79a2aabef394d58889136b8d38b05be09482006de2453fb06333"}},"download_size":35585024,"post_processing_size":null,"dataset_size":2991538,"size_in_bytes":38576562},"virtue":{"description":"The ETHICS dataset is a benchmark that spans concepts in justice, well-being,\nduties, virtues, and commonsense morality. Models predict widespread moral\njudgments about diverse text scenarios. This requires connecting physical and\nsocial world knowledge to value judgements, a capability that may enable us\nto steer chatbot outputs or eventually regularize open-ended reinforcement\nlearning agents.\n\nThe Virtue subset contains scenarios focusing on whether virtues or vices are being exemplified","citation":"@article{hendrycks2021ethics\n title={Aligning AI With Shared Human Values},\n author={Dan Hendrycks and Collin Burns and Steven Basart and Andrew Critch and Jerry Li and Dawn Song and Jacob Steinhardt},\n journal={Proceedings of the International Conference on Learning Representations (ICLR)},\n year={2021}\n}\n","homepage":"https://github.com/hendrycks/ethics","license":"","features":{"group_id":{"dtype":"int32","id":null,"_type":"Value"},"label":{"dtype":"int32","id":null,"_type":"Value"},"scenario":{"dtype":"string","id":null,"_type":"Value"},"trait":{"dtype":"string","id":null,"_type":"Value"}},"post_processed":null,"supervised_keys":null,"task_templates":null,"builder_name":"hendrycks_ethics","config_name":"virtue","version":{"version_str":"0.0.1","description":null,"major":0,"minor":0,"patch":1},"splits":{"train":{"name":"train","num_bytes":2640328,"num_examples":28245,"dataset_name":"hendrycks_ethics"},"test":{"name":"test","num_bytes":473473,"num_examples":4975,"dataset_name":"hendrycks_ethics"}},"download_checksums":{"https://people.eecs.berkeley.edu/~hendrycks/ethics.tar":{"num_bytes":35585024,"checksum":"40acbf1ac0da79a2aabef394d58889136b8d38b05be09482006de2453fb06333"}},"download_size":35585024,"post_processing_size":null,"dataset_size":3113801,"size_in_bytes":38698825}}
@@ -71,54 +71,64 @@ class HendrycksEthics(datasets.GeneratorBasedBuilder):
EthicsConfig(
name="commonsense",
prefix="cm",
features=datasets.Features({
"label":datasets.Value("int32"),
"input":datasets.Value("string"),
"is_short":datasets.Value("bool"),
"edited":datasets.Value("bool"),
}),
description="The Commonsense subset contains examples focusing on moral standards and principles that most people intuitively accept."
features=datasets.Features(
{
"label":datasets.Value("int32"),
"input":datasets.Value("string"),
"is_short":datasets.Value("bool"),
"edited":datasets.Value("bool"),
}
),
description="The Commonsense subset contains examples focusing on moral standards and principles that most people intuitively accept.",
),
EthicsConfig(
name="deontology",
prefix="deontology",
features=datasets.Features({
"group_id":datasets.Value("int32"),
"label":datasets.Value("int32"),
"scenario":datasets.Value("string"),
"excuse":datasets.Value("string"),
}),
features=datasets.Features(
{
"group_id":datasets.Value("int32"),
"label":datasets.Value("int32"),
"scenario":datasets.Value("string"),
"excuse":datasets.Value("string"),
}
),
description="The Deontology subset contains examples focusing on whether an act is required, permitted, or forbidden according to a set of rules or constraints",
),
EthicsConfig(
name="justice",
prefix="justice",
features=datasets.Features({
"group_id":datasets.Value("int32"),
"label":datasets.Value("int32"),
"scenario":datasets.Value("string"),
}),
features=datasets.Features(
{
"group_id":datasets.Value("int32"),
"label":datasets.Value("int32"),
"scenario":datasets.Value("string"),
}
),
description="The Justice subset contains examples focusing on how a character treats another person",
),
EthicsConfig(
name="utilitarianism",
prefix="util",
features=datasets.Features({
"activity":datasets.Value("string"),
"baseline":datasets.Value("string"),
"rating":datasets.Value("string"),# Empty rating.
}),
features=datasets.Features(
{
"activity":datasets.Value("string"),
"baseline":datasets.Value("string"),
"rating":datasets.Value("string"),# Empty rating.
}
),
description="The Utilitarianism subset contains scenarios that should be ranked from most pleasant to least pleasant for the person in the scenario",
),
EthicsConfig(
name="virtue",
prefix="virtue",
features=datasets.Features({
"group_id":datasets.Value("int32"),
"label":datasets.Value("int32"),
"scenario":datasets.Value("string"),
"trait":datasets.Value("string"),
}),
features=datasets.Features(
{
"group_id":datasets.Value("int32"),
"label":datasets.Value("int32"),
"scenario":datasets.Value("string"),
"trait":datasets.Value("string"),
}
),
description="The Virtue subset contains scenarios focusing on whether virtues or vices are being exemplified",
),
]
...
...
@@ -140,7 +150,12 @@ class HendrycksEthics(datasets.GeneratorBasedBuilder):
name=datasets.Split.TRAIN,
# These kwargs will be passed to _generate_examples
{"algebra":{"description":"MATH is a dataset of 12,500 challenging competition mathematics problems. Each\nproblem in Math has a full step-by-step solution which can be used to teach\nmodels to generate answer derivations and explanations.\n","citation":"@article{hendrycksmath2021,\n title={Measuring Mathematical Problem Solving With the Math Dataset},\n author={Dan Hendrycks and Collin Burns and Saurav Kadavath and Akul Arora and Steven Basart and Eric Tang and Dawn Song and Jacob Steinhardt},\n journal={NeurIPS},\n year={2021}\n}\n","homepage":"https://github.com/hendrycks/math","license":"","features":{"problem":{"dtype":"string","id":null,"_type":"Value"},"level":{"dtype":"string","id":null,"_type":"Value"},"type":{"dtype":"string","id":null,"_type":"Value"},"solution":{"dtype":"string","id":null,"_type":"Value"}},"post_processed":null,"supervised_keys":null,"task_templates":null,"builder_name":"hendrycks_math","config_name":"algebra","version":{"version_str":"0.0.1","description":null,"major":0,"minor":0,"patch":1},"splits":{"train":{"name":"train","num_bytes":955021,"num_examples":1744,"dataset_name":"hendrycks_math"},"test":{"name":"test","num_bytes":648291,"num_examples":1187,"dataset_name":"hendrycks_math"}},"download_checksums":{"https://people.eecs.berkeley.edu/~hendrycks/MATH.tar":{"num_bytes":20327936,"checksum":"0fbe4fad0df66942db6c221cdcc95b298cc7f4595a2f0f518360cce84e90d9ac"}},"download_size":20327936,"post_processing_size":null,"dataset_size":1603312,"size_in_bytes":21931248},"counting_and_probability":{"description":"MATH is a dataset of 12,500 challenging competition mathematics problems. Each\nproblem in Math has a full step-by-step solution which can be used to teach\nmodels to generate answer derivations and explanations.\n","citation":"@article{hendrycksmath2021,\n title={Measuring Mathematical Problem Solving With the Math Dataset},\n author={Dan Hendrycks and Collin Burns and Saurav Kadavath and Akul Arora and Steven Basart and Eric Tang and Dawn Song and Jacob Steinhardt},\n journal={NeurIPS},\n year={2021}\n}\n","homepage":"https://github.com/hendrycks/math","license":"","features":{"problem":{"dtype":"string","id":null,"_type":"Value"},"level":{"dtype":"string","id":null,"_type":"Value"},"type":{"dtype":"string","id":null,"_type":"Value"},"solution":{"dtype":"string","id":null,"_type":"Value"}},"post_processed":null,"supervised_keys":null,"task_templates":null,"builder_name":"hendrycks_math","config_name":"counting_and_probability","version":{"version_str":"0.0.1","description":null,"major":0,"minor":0,"patch":1},"splits":{"train":{"name":"train","num_bytes":667385,"num_examples":771,"dataset_name":"hendrycks_math"},"test":{"name":"test","num_bytes":353803,"num_examples":474,"dataset_name":"hendrycks_math"}},"download_checksums":{"https://people.eecs.berkeley.edu/~hendrycks/MATH.tar":{"num_bytes":20327936,"checksum":"0fbe4fad0df66942db6c221cdcc95b298cc7f4595a2f0f518360cce84e90d9ac"}},"download_size":20327936,"post_processing_size":null,"dataset_size":1021188,"size_in_bytes":21349124},"geometry":{"description":"MATH is a dataset of 12,500 challenging competition mathematics problems. Each\nproblem in Math has a full step-by-step solution which can be used to teach\nmodels to generate answer derivations and explanations.\n","citation":"@article{hendrycksmath2021,\n title={Measuring Mathematical Problem Solving With the Math Dataset},\n author={Dan Hendrycks and Collin Burns and Saurav Kadavath and Akul Arora and Steven Basart and Eric Tang and Dawn Song and Jacob Steinhardt},\n journal={NeurIPS},\n year={2021}\n}\n","homepage":"https://github.com/hendrycks/math","license":"","features":{"problem":{"dtype":"string","id":null,"_type":"Value"},"level":{"dtype":"string","id":null,"_type":"Value"},"type":{"dtype":"string","id":null,"_type":"Value"},"solution":{"dtype":"string","id":null,"_type":"Value"}},"post_processed":null,"supervised_keys":null,"task_templates":null,"builder_name":"hendrycks_math","config_name":"geometry","version":{"version_str":"0.0.1","description":null,"major":0,"minor":0,"patch":1},"splits":{"train":{"name":"train","num_bytes":1077241,"num_examples":870,"dataset_name":"hendrycks_math"},"test":{"name":"test","num_bytes":523126,"num_examples":479,"dataset_name":"hendrycks_math"}},"download_checksums":{"https://people.eecs.berkeley.edu/~hendrycks/MATH.tar":{"num_bytes":20327936,"checksum":"0fbe4fad0df66942db6c221cdcc95b298cc7f4595a2f0f518360cce84e90d9ac"}},"download_size":20327936,"post_processing_size":null,"dataset_size":1600367,"size_in_bytes":21928303},"intermediate_algebra":{"description":"MATH is a dataset of 12,500 challenging competition mathematics problems. Each\nproblem in Math has a full step-by-step solution which can be used to teach\nmodels to generate answer derivations and explanations.\n","citation":"@article{hendrycksmath2021,\n title={Measuring Mathematical Problem Solving With the Math Dataset},\n author={Dan Hendrycks and Collin Burns and Saurav Kadavath and Akul Arora and Steven Basart and Eric Tang and Dawn Song and Jacob Steinhardt},\n journal={NeurIPS},\n year={2021}\n}\n","homepage":"https://github.com/hendrycks/math","license":"","features":{"problem":{"dtype":"string","id":null,"_type":"Value"},"level":{"dtype":"string","id":null,"_type":"Value"},"type":{"dtype":"string","id":null,"_type":"Value"},"solution":{"dtype":"string","id":null,"_type":"Value"}},"post_processed":null,"supervised_keys":null,"task_templates":null,"builder_name":"hendrycks_math","config_name":"intermediate_algebra","version":{"version_str":"0.0.1","description":null,"major":0,"minor":0,"patch":1},"splits":{"train":{"name":"train","num_bytes":1157476,"num_examples":1295,"dataset_name":"hendrycks_math"},"test":{"name":"test","num_bytes":795070,"num_examples":903,"dataset_name":"hendrycks_math"}},"download_checksums":{"https://people.eecs.berkeley.edu/~hendrycks/MATH.tar":{"num_bytes":20327936,"checksum":"0fbe4fad0df66942db6c221cdcc95b298cc7f4595a2f0f518360cce84e90d9ac"}},"download_size":20327936,"post_processing_size":null,"dataset_size":1952546,"size_in_bytes":22280482},"number_theory":{"description":"MATH is a dataset of 12,500 challenging competition mathematics problems. Each\nproblem in Math has a full step-by-step solution which can be used to teach\nmodels to generate answer derivations and explanations.\n","citation":"@article{hendrycksmath2021,\n title={Measuring Mathematical Problem Solving With the Math Dataset},\n author={Dan Hendrycks and Collin Burns and Saurav Kadavath and Akul Arora and Steven Basart and Eric Tang and Dawn Song and Jacob Steinhardt},\n journal={NeurIPS},\n year={2021}\n}\n","homepage":"https://github.com/hendrycks/math","license":"","features":{"problem":{"dtype":"string","id":null,"_type":"Value"},"level":{"dtype":"string","id":null,"_type":"Value"},"type":{"dtype":"string","id":null,"_type":"Value"},"solution":{"dtype":"string","id":null,"_type":"Value"}},"post_processed":null,"supervised_keys":null,"task_templates":null,"builder_name":"hendrycks_math","config_name":"number_theory","version":{"version_str":"0.0.1","description":null,"major":0,"minor":0,"patch":1},"splits":{"train":{"name":"train","num_bytes":595793,"num_examples":869,"dataset_name":"hendrycks_math"},"test":{"name":"test","num_bytes":349455,"num_examples":540,"dataset_name":"hendrycks_math"}},"download_checksums":{"https://people.eecs.berkeley.edu/~hendrycks/MATH.tar":{"num_bytes":20327936,"checksum":"0fbe4fad0df66942db6c221cdcc95b298cc7f4595a2f0f518360cce84e90d9ac"}},"download_size":20327936,"post_processing_size":null,"dataset_size":945248,"size_in_bytes":21273184},"prealgebra":{"description":"MATH is a dataset of 12,500 challenging competition mathematics problems. Each\nproblem in Math has a full step-by-step solution which can be used to teach\nmodels to generate answer derivations and explanations.\n","citation":"@article{hendrycksmath2021,\n title={Measuring Mathematical Problem Solving With the Math Dataset},\n author={Dan Hendrycks and Collin Burns and Saurav Kadavath and Akul Arora and Steven Basart and Eric Tang and Dawn Song and Jacob Steinhardt},\n journal={NeurIPS},\n year={2021}\n}\n","homepage":"https://github.com/hendrycks/math","license":"","features":{"problem":{"dtype":"string","id":null,"_type":"Value"},"level":{"dtype":"string","id":null,"_type":"Value"},"type":{"dtype":"string","id":null,"_type":"Value"},"solution":{"dtype":"string","id":null,"_type":"Value"}},"post_processed":null,"supervised_keys":null,"task_templates":null,"builder_name":"hendrycks_math","config_name":"prealgebra","version":{"version_str":"0.0.1","description":null,"major":0,"minor":0,"patch":1},"splits":{"train":{"name":"train","num_bytes":715611,"num_examples":1205,"dataset_name":"hendrycks_math"},"test":{"name":"test","num_bytes":510195,"num_examples":871,"dataset_name":"hendrycks_math"}},"download_checksums":{"https://people.eecs.berkeley.edu/~hendrycks/MATH.tar":{"num_bytes":20327936,"checksum":"0fbe4fad0df66942db6c221cdcc95b298cc7f4595a2f0f518360cce84e90d9ac"}},"download_size":20327936,"post_processing_size":null,"dataset_size":1225806,"size_in_bytes":21553742},"precalculus":{"description":"MATH is a dataset of 12,500 challenging competition mathematics problems. Each\nproblem in Math has a full step-by-step solution which can be used to teach\nmodels to generate answer derivations and explanations.\n","citation":"@article{hendrycksmath2021,\n title={Measuring Mathematical Problem Solving With the Math Dataset},\n author={Dan Hendrycks and Collin Burns and Saurav Kadavath and Akul Arora and Steven Basart and Eric Tang and Dawn Song and Jacob Steinhardt},\n journal={NeurIPS},\n year={2021}\n}\n","homepage":"https://github.com/hendrycks/math","license":"","features":{"problem":{"dtype":"string","id":null,"_type":"Value"},"level":{"dtype":"string","id":null,"_type":"Value"},"type":{"dtype":"string","id":null,"_type":"Value"},"solution":{"dtype":"string","id":null,"_type":"Value"}},"post_processed":null,"supervised_keys":null,"task_templates":null,"builder_name":"hendrycks_math","config_name":"precalculus","version":{"version_str":"0.0.1","description":null,"major":0,"minor":0,"patch":1},"splits":{"train":{"name":"train","num_bytes":816245,"num_examples":746,"dataset_name":"hendrycks_math"},"test":{"name":"test","num_bytes":552893,"num_examples":546,"dataset_name":"hendrycks_math"}},"download_checksums":{"https://people.eecs.berkeley.edu/~hendrycks/MATH.tar":{"num_bytes":20327936,"checksum":"0fbe4fad0df66942db6c221cdcc95b298cc7f4595a2f0f518360cce84e90d9ac"}},"download_size":20327936,"post_processing_size":null,"dataset_size":1369138,"size_in_bytes":21697074}}
\ No newline at end of file
{"algebra":{"description":"MATH is a dataset of 12,500 challenging competition mathematics problems. Each\nproblem in Math has a full step-by-step solution which can be used to teach\nmodels to generate answer derivations and explanations.\n","citation":"@article{hendrycksmath2021,\n title={Measuring Mathematical Problem Solving With the Math Dataset},\n author={Dan Hendrycks and Collin Burns and Saurav Kadavath and Akul Arora and Steven Basart and Eric Tang and Dawn Song and Jacob Steinhardt},\n journal={NeurIPS},\n year={2021}\n}\n","homepage":"https://github.com/hendrycks/math","license":"","features":{"problem":{"dtype":"string","id":null,"_type":"Value"},"level":{"dtype":"string","id":null,"_type":"Value"},"type":{"dtype":"string","id":null,"_type":"Value"},"solution":{"dtype":"string","id":null,"_type":"Value"}},"post_processed":null,"supervised_keys":null,"task_templates":null,"builder_name":"hendrycks_math","config_name":"algebra","version":{"version_str":"0.0.1","description":null,"major":0,"minor":0,"patch":1},"splits":{"train":{"name":"train","num_bytes":955021,"num_examples":1744,"dataset_name":"hendrycks_math"},"test":{"name":"test","num_bytes":648291,"num_examples":1187,"dataset_name":"hendrycks_math"}},"download_checksums":{"https://people.eecs.berkeley.edu/~hendrycks/MATH.tar":{"num_bytes":20327936,"checksum":"0fbe4fad0df66942db6c221cdcc95b298cc7f4595a2f0f518360cce84e90d9ac"}},"download_size":20327936,"post_processing_size":null,"dataset_size":1603312,"size_in_bytes":21931248},"counting_and_probability":{"description":"MATH is a dataset of 12,500 challenging competition mathematics problems. Each\nproblem in Math has a full step-by-step solution which can be used to teach\nmodels to generate answer derivations and explanations.\n","citation":"@article{hendrycksmath2021,\n title={Measuring Mathematical Problem Solving With the Math Dataset},\n author={Dan Hendrycks and Collin Burns and Saurav Kadavath and Akul Arora and Steven Basart and Eric Tang and Dawn Song and Jacob Steinhardt},\n journal={NeurIPS},\n year={2021}\n}\n","homepage":"https://github.com/hendrycks/math","license":"","features":{"problem":{"dtype":"string","id":null,"_type":"Value"},"level":{"dtype":"string","id":null,"_type":"Value"},"type":{"dtype":"string","id":null,"_type":"Value"},"solution":{"dtype":"string","id":null,"_type":"Value"}},"post_processed":null,"supervised_keys":null,"task_templates":null,"builder_name":"hendrycks_math","config_name":"counting_and_probability","version":{"version_str":"0.0.1","description":null,"major":0,"minor":0,"patch":1},"splits":{"train":{"name":"train","num_bytes":667385,"num_examples":771,"dataset_name":"hendrycks_math"},"test":{"name":"test","num_bytes":353803,"num_examples":474,"dataset_name":"hendrycks_math"}},"download_checksums":{"https://people.eecs.berkeley.edu/~hendrycks/MATH.tar":{"num_bytes":20327936,"checksum":"0fbe4fad0df66942db6c221cdcc95b298cc7f4595a2f0f518360cce84e90d9ac"}},"download_size":20327936,"post_processing_size":null,"dataset_size":1021188,"size_in_bytes":21349124},"geometry":{"description":"MATH is a dataset of 12,500 challenging competition mathematics problems. Each\nproblem in Math has a full step-by-step solution which can be used to teach\nmodels to generate answer derivations and explanations.\n","citation":"@article{hendrycksmath2021,\n title={Measuring Mathematical Problem Solving With the Math Dataset},\n author={Dan Hendrycks and Collin Burns and Saurav Kadavath and Akul Arora and Steven Basart and Eric Tang and Dawn Song and Jacob Steinhardt},\n journal={NeurIPS},\n year={2021}\n}\n","homepage":"https://github.com/hendrycks/math","license":"","features":{"problem":{"dtype":"string","id":null,"_type":"Value"},"level":{"dtype":"string","id":null,"_type":"Value"},"type":{"dtype":"string","id":null,"_type":"Value"},"solution":{"dtype":"string","id":null,"_type":"Value"}},"post_processed":null,"supervised_keys":null,"task_templates":null,"builder_name":"hendrycks_math","config_name":"geometry","version":{"version_str":"0.0.1","description":null,"major":0,"minor":0,"patch":1},"splits":{"train":{"name":"train","num_bytes":1077241,"num_examples":870,"dataset_name":"hendrycks_math"},"test":{"name":"test","num_bytes":523126,"num_examples":479,"dataset_name":"hendrycks_math"}},"download_checksums":{"https://people.eecs.berkeley.edu/~hendrycks/MATH.tar":{"num_bytes":20327936,"checksum":"0fbe4fad0df66942db6c221cdcc95b298cc7f4595a2f0f518360cce84e90d9ac"}},"download_size":20327936,"post_processing_size":null,"dataset_size":1600367,"size_in_bytes":21928303},"intermediate_algebra":{"description":"MATH is a dataset of 12,500 challenging competition mathematics problems. Each\nproblem in Math has a full step-by-step solution which can be used to teach\nmodels to generate answer derivations and explanations.\n","citation":"@article{hendrycksmath2021,\n title={Measuring Mathematical Problem Solving With the Math Dataset},\n author={Dan Hendrycks and Collin Burns and Saurav Kadavath and Akul Arora and Steven Basart and Eric Tang and Dawn Song and Jacob Steinhardt},\n journal={NeurIPS},\n year={2021}\n}\n","homepage":"https://github.com/hendrycks/math","license":"","features":{"problem":{"dtype":"string","id":null,"_type":"Value"},"level":{"dtype":"string","id":null,"_type":"Value"},"type":{"dtype":"string","id":null,"_type":"Value"},"solution":{"dtype":"string","id":null,"_type":"Value"}},"post_processed":null,"supervised_keys":null,"task_templates":null,"builder_name":"hendrycks_math","config_name":"intermediate_algebra","version":{"version_str":"0.0.1","description":null,"major":0,"minor":0,"patch":1},"splits":{"train":{"name":"train","num_bytes":1157476,"num_examples":1295,"dataset_name":"hendrycks_math"},"test":{"name":"test","num_bytes":795070,"num_examples":903,"dataset_name":"hendrycks_math"}},"download_checksums":{"https://people.eecs.berkeley.edu/~hendrycks/MATH.tar":{"num_bytes":20327936,"checksum":"0fbe4fad0df66942db6c221cdcc95b298cc7f4595a2f0f518360cce84e90d9ac"}},"download_size":20327936,"post_processing_size":null,"dataset_size":1952546,"size_in_bytes":22280482},"number_theory":{"description":"MATH is a dataset of 12,500 challenging competition mathematics problems. Each\nproblem in Math has a full step-by-step solution which can be used to teach\nmodels to generate answer derivations and explanations.\n","citation":"@article{hendrycksmath2021,\n title={Measuring Mathematical Problem Solving With the Math Dataset},\n author={Dan Hendrycks and Collin Burns and Saurav Kadavath and Akul Arora and Steven Basart and Eric Tang and Dawn Song and Jacob Steinhardt},\n journal={NeurIPS},\n year={2021}\n}\n","homepage":"https://github.com/hendrycks/math","license":"","features":{"problem":{"dtype":"string","id":null,"_type":"Value"},"level":{"dtype":"string","id":null,"_type":"Value"},"type":{"dtype":"string","id":null,"_type":"Value"},"solution":{"dtype":"string","id":null,"_type":"Value"}},"post_processed":null,"supervised_keys":null,"task_templates":null,"builder_name":"hendrycks_math","config_name":"number_theory","version":{"version_str":"0.0.1","description":null,"major":0,"minor":0,"patch":1},"splits":{"train":{"name":"train","num_bytes":595793,"num_examples":869,"dataset_name":"hendrycks_math"},"test":{"name":"test","num_bytes":349455,"num_examples":540,"dataset_name":"hendrycks_math"}},"download_checksums":{"https://people.eecs.berkeley.edu/~hendrycks/MATH.tar":{"num_bytes":20327936,"checksum":"0fbe4fad0df66942db6c221cdcc95b298cc7f4595a2f0f518360cce84e90d9ac"}},"download_size":20327936,"post_processing_size":null,"dataset_size":945248,"size_in_bytes":21273184},"prealgebra":{"description":"MATH is a dataset of 12,500 challenging competition mathematics problems. Each\nproblem in Math has a full step-by-step solution which can be used to teach\nmodels to generate answer derivations and explanations.\n","citation":"@article{hendrycksmath2021,\n title={Measuring Mathematical Problem Solving With the Math Dataset},\n author={Dan Hendrycks and Collin Burns and Saurav Kadavath and Akul Arora and Steven Basart and Eric Tang and Dawn Song and Jacob Steinhardt},\n journal={NeurIPS},\n year={2021}\n}\n","homepage":"https://github.com/hendrycks/math","license":"","features":{"problem":{"dtype":"string","id":null,"_type":"Value"},"level":{"dtype":"string","id":null,"_type":"Value"},"type":{"dtype":"string","id":null,"_type":"Value"},"solution":{"dtype":"string","id":null,"_type":"Value"}},"post_processed":null,"supervised_keys":null,"task_templates":null,"builder_name":"hendrycks_math","config_name":"prealgebra","version":{"version_str":"0.0.1","description":null,"major":0,"minor":0,"patch":1},"splits":{"train":{"name":"train","num_bytes":715611,"num_examples":1205,"dataset_name":"hendrycks_math"},"test":{"name":"test","num_bytes":510195,"num_examples":871,"dataset_name":"hendrycks_math"}},"download_checksums":{"https://people.eecs.berkeley.edu/~hendrycks/MATH.tar":{"num_bytes":20327936,"checksum":"0fbe4fad0df66942db6c221cdcc95b298cc7f4595a2f0f518360cce84e90d9ac"}},"download_size":20327936,"post_processing_size":null,"dataset_size":1225806,"size_in_bytes":21553742},"precalculus":{"description":"MATH is a dataset of 12,500 challenging competition mathematics problems. Each\nproblem in Math has a full step-by-step solution which can be used to teach\nmodels to generate answer derivations and explanations.\n","citation":"@article{hendrycksmath2021,\n title={Measuring Mathematical Problem Solving With the Math Dataset},\n author={Dan Hendrycks and Collin Burns and Saurav Kadavath and Akul Arora and Steven Basart and Eric Tang and Dawn Song and Jacob Steinhardt},\n journal={NeurIPS},\n year={2021}\n}\n","homepage":"https://github.com/hendrycks/math","license":"","features":{"problem":{"dtype":"string","id":null,"_type":"Value"},"level":{"dtype":"string","id":null,"_type":"Value"},"type":{"dtype":"string","id":null,"_type":"Value"},"solution":{"dtype":"string","id":null,"_type":"Value"}},"post_processed":null,"supervised_keys":null,"task_templates":null,"builder_name":"hendrycks_math","config_name":"precalculus","version":{"version_str":"0.0.1","description":null,"major":0,"minor":0,"patch":1},"splits":{"train":{"name":"train","num_bytes":816245,"num_examples":746,"dataset_name":"hendrycks_math"},"test":{"name":"test","num_bytes":552893,"num_examples":546,"dataset_name":"hendrycks_math"}},"download_checksums":{"https://people.eecs.berkeley.edu/~hendrycks/MATH.tar":{"num_bytes":20327936,"checksum":"0fbe4fad0df66942db6c221cdcc95b298cc7f4595a2f0f518360cce84e90d9ac"}},"download_size":20327936,"post_processing_size":null,"dataset_size":1369138,"size_in_bytes":21697074}}
{"original":{"description":"LAMBADA is a dataset to evaluate the capabilities of computational models for text\nunderstanding by means of a word prediction task. LAMBADA is a collection of narrative\ntexts sharing the characteristic that human subjects are able to guess their last\nword if they are exposed to the whole text, but not if they only see the last\nsentence preceding the target word. To succeed on LAMBADA, computational models\ncannot simply rely on local context, but must be able to keep track of information\nin the broader discourse.\n\nThe LAMBADA dataset","citation":"@misc{\n author={Paperno, Denis and Kruszewski, Germ\u00e1n and Lazaridou, Angeliki and Pham, Quan Ngoc and Bernardi, Raffaella and Pezzelle, Sandro and Baroni, Marco and Boleda, Gemma and Fern\u00e1ndez, Raquel}, \n title={The LAMBADA dataset},\n DOI={10.5281/zenodo.2630551},\n publisher={Zenodo},\n year={2016},\n month={Aug}\n}\n","homepage":"https://zenodo.org/record/2630551#.X4Xzn5NKjUI","license":"","features":{"text":{"dtype":"string","id":null,"_type":"Value"}},"post_processed":null,"supervised_keys":null,"task_templates":null,"builder_name":"lambada","config_name":"original","version":{"version_str":"0.0.1","description":null,"major":0,"minor":0,"patch":1},"splits":{"validation":{"name":"validation","num_bytes":1709449,"num_examples":5153,"dataset_name":"lambada"}},"download_checksums":{"http://eaidata.bmk.sh/data/lambada_test.jsonl":{"num_bytes":1819752,"checksum":"4aa8d02cd17c719165fc8a7887fddd641f43fcafa4b1c806ca8abc31fabdb226"}},"download_size":1819752,"post_processing_size":null,"dataset_size":1709449,"size_in_bytes":3529201},"en":{"description":"LAMBADA is a dataset to evaluate the capabilities of computational models for text\nunderstanding by means of a word prediction task. LAMBADA is a collection of narrative\ntexts sharing the characteristic that human subjects are able to guess their last\nword if they are exposed to the whole text, but not if they only see the last\nsentence preceding the target word. To succeed on LAMBADA, computational models\ncannot simply rely on local context, but must be able to keep track of information\nin the broader discourse.\n\nThe English translated LAMBADA dataset","citation":"@misc{\n author={Paperno, Denis and Kruszewski, Germ\u00e1n and Lazaridou, Angeliki and Pham, Quan Ngoc and Bernardi, Raffaella and Pezzelle, Sandro and Baroni, Marco and Boleda, Gemma and Fern\u00e1ndez, Raquel}, \n title={The LAMBADA dataset},\n DOI={10.5281/zenodo.2630551},\n publisher={Zenodo},\n year={2016},\n month={Aug}\n}\n","homepage":"https://zenodo.org/record/2630551#.X4Xzn5NKjUI","license":"","features":{"text":{"dtype":"string","id":null,"_type":"Value"}},"post_processed":null,"supervised_keys":null,"task_templates":null,"builder_name":"lambada","config_name":"en","version":{"version_str":"0.0.1","description":null,"major":0,"minor":0,"patch":1},"splits":{"validation":{"name":"validation","num_bytes":1709449,"num_examples":5153,"dataset_name":"lambada"}},"download_checksums":{"http://eaidata.bmk.sh/data/lambada_test_en.jsonl":{"num_bytes":1819752,"checksum":"4aa8d02cd17c719165fc8a7887fddd641f43fcafa4b1c806ca8abc31fabdb226"}},"download_size":1819752,"post_processing_size":null,"dataset_size":1709449,"size_in_bytes":3529201},"fr":{"description":"LAMBADA is a dataset to evaluate the capabilities of computational models for text\nunderstanding by means of a word prediction task. LAMBADA is a collection of narrative\ntexts sharing the characteristic that human subjects are able to guess their last\nword if they are exposed to the whole text, but not if they only see the last\nsentence preceding the target word. To succeed on LAMBADA, computational models\ncannot simply rely on local context, but must be able to keep track of information\nin the broader discourse.\n\nThe French translated LAMBADA dataset","citation":"@misc{\n author={Paperno, Denis and Kruszewski, Germ\u00e1n and Lazaridou, Angeliki and Pham, Quan Ngoc and Bernardi, Raffaella and Pezzelle, Sandro and Baroni, Marco and Boleda, Gemma and Fern\u00e1ndez, Raquel}, \n title={The LAMBADA dataset},\n DOI={10.5281/zenodo.2630551},\n publisher={Zenodo},\n year={2016},\n month={Aug}\n}\n","homepage":"https://zenodo.org/record/2630551#.X4Xzn5NKjUI","license":"","features":{"text":{"dtype":"string","id":null,"_type":"Value"}},"post_processed":null,"supervised_keys":null,"task_templates":null,"builder_name":"lambada","config_name":"fr","version":{"version_str":"0.0.1","description":null,"major":0,"minor":0,"patch":1},"splits":{"validation":{"name":"validation","num_bytes":1948795,"num_examples":5153,"dataset_name":"lambada"}},"download_checksums":{"http://eaidata.bmk.sh/data/lambada_test_fr.jsonl":{"num_bytes":2028703,"checksum":"941ec6a73dba7dc91c860bf493eb66a527cd430148827a4753a4535a046bf362"}},"download_size":2028703,"post_processing_size":null,"dataset_size":1948795,"size_in_bytes":3977498},"de":{"description":"LAMBADA is a dataset to evaluate the capabilities of computational models for text\nunderstanding by means of a word prediction task. LAMBADA is a collection of narrative\ntexts sharing the characteristic that human subjects are able to guess their last\nword if they are exposed to the whole text, but not if they only see the last\nsentence preceding the target word. To succeed on LAMBADA, computational models\ncannot simply rely on local context, but must be able to keep track of information\nin the broader discourse.\n\nThe German translated LAMBADA dataset","citation":"@misc{\n author={Paperno, Denis and Kruszewski, Germ\u00e1n and Lazaridou, Angeliki and Pham, Quan Ngoc and Bernardi, Raffaella and Pezzelle, Sandro and Baroni, Marco and Boleda, Gemma and Fern\u00e1ndez, Raquel}, \n title={The LAMBADA dataset},\n DOI={10.5281/zenodo.2630551},\n publisher={Zenodo},\n year={2016},\n month={Aug}\n}\n","homepage":"https://zenodo.org/record/2630551#.X4Xzn5NKjUI","license":"","features":{"text":{"dtype":"string","id":null,"_type":"Value"}},"post_processed":null,"supervised_keys":null,"task_templates":null,"builder_name":"lambada","config_name":"de","version":{"version_str":"0.0.1","description":null,"major":0,"minor":0,"patch":1},"splits":{"validation":{"name":"validation","num_bytes":1904576,"num_examples":5153,"dataset_name":"lambada"}},"download_checksums":{"http://eaidata.bmk.sh/data/lambada_test_de.jsonl":{"num_bytes":1985231,"checksum":"51c6c1795894c46e88e4c104b5667f488efe79081fb34d746b82b8caa663865e"}},"download_size":1985231,"post_processing_size":null,"dataset_size":1904576,"size_in_bytes":3889807},"it":{"description":"LAMBADA is a dataset to evaluate the capabilities of computational models for text\nunderstanding by means of a word prediction task. LAMBADA is a collection of narrative\ntexts sharing the characteristic that human subjects are able to guess their last\nword if they are exposed to the whole text, but not if they only see the last\nsentence preceding the target word. To succeed on LAMBADA, computational models\ncannot simply rely on local context, but must be able to keep track of information\nin the broader discourse.\n\nThe Italian translated LAMBADA dataset","citation":"@misc{\n author={Paperno, Denis and Kruszewski, Germ\u00e1n and Lazaridou, Angeliki and Pham, Quan Ngoc and Bernardi, Raffaella and Pezzelle, Sandro and Baroni, Marco and Boleda, Gemma and Fern\u00e1ndez, Raquel}, \n title={The LAMBADA dataset},\n DOI={10.5281/zenodo.2630551},\n publisher={Zenodo},\n year={2016},\n month={Aug}\n}\n","homepage":"https://zenodo.org/record/2630551#.X4Xzn5NKjUI","license":"","features":{"text":{"dtype":"string","id":null,"_type":"Value"}},"post_processed":null,"supervised_keys":null,"task_templates":null,"builder_name":"lambada","config_name":"it","version":{"version_str":"0.0.1","description":null,"major":0,"minor":0,"patch":1},"splits":{"validation":{"name":"validation","num_bytes":1813420,"num_examples":5153,"dataset_name":"lambada"}},"download_checksums":{"http://eaidata.bmk.sh/data/lambada_test_it.jsonl":{"num_bytes":1894613,"checksum":"86654237716702ab74f42855ae5a78455c1b0e50054a4593fb9c6fcf7fad0850"}},"download_size":1894613,"post_processing_size":null,"dataset_size":1813420,"size_in_bytes":3708033},"es":{"description":"LAMBADA is a dataset to evaluate the capabilities of computational models for text\nunderstanding by means of a word prediction task. LAMBADA is a collection of narrative\ntexts sharing the characteristic that human subjects are able to guess their last\nword if they are exposed to the whole text, but not if they only see the last\nsentence preceding the target word. To succeed on LAMBADA, computational models\ncannot simply rely on local context, but must be able to keep track of information\nin the broader discourse.\n\nThe Spanish translated LAMBADA dataset","citation":"@misc{\n author={Paperno, Denis and Kruszewski, Germ\u00e1n and Lazaridou, Angeliki and Pham, Quan Ngoc and Bernardi, Raffaella and Pezzelle, Sandro and Baroni, Marco and Boleda, Gemma and Fern\u00e1ndez, Raquel}, \n title={The LAMBADA dataset},\n DOI={10.5281/zenodo.2630551},\n publisher={Zenodo},\n year={2016},\n month={Aug}\n}\n","homepage":"https://zenodo.org/record/2630551#.X4Xzn5NKjUI","license":"","features":{"text":{"dtype":"string","id":null,"_type":"Value"}},"post_processed":null,"supervised_keys":null,"task_templates":null,"builder_name":"lambada","config_name":"es","version":{"version_str":"0.0.1","description":null,"major":0,"minor":0,"patch":1},"splits":{"validation":{"name":"validation","num_bytes":1821735,"num_examples":5153,"dataset_name":"lambada"}},"download_checksums":{"http://eaidata.bmk.sh/data/lambada_test_es.jsonl":{"num_bytes":1902349,"checksum":"ffd760026c647fb43c67ce1bc56fd527937304b348712dce33190ea6caba6f9c"}},"download_size":1902349,"post_processing_size":null,"dataset_size":1821735,"size_in_bytes":3724084}}
# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# TODO: Address all TODOs and remove all explanatory comments
"""LAMBADA dataset."""
importjson
importdatasets
_CITATION="""\
@misc{
author={Paperno, Denis and Kruszewski, Germán and Lazaridou, Angeliki and Pham, Quan Ngoc and Bernardi, Raffaella and Pezzelle, Sandro and Baroni, Marco and Boleda, Gemma and Fernández, Raquel},
title={The LAMBADA dataset},
DOI={10.5281/zenodo.2630551},
publisher={Zenodo},
year={2016},
month={Aug}
}
"""
_DESCRIPTION="""\
LAMBADA is a dataset to evaluate the capabilities of computational models for text
understanding by means of a word prediction task. LAMBADA is a collection of narrative
texts sharing the characteristic that human subjects are able to guess their last
word if they are exposed to the whole text, but not if they only see the last
sentence preceding the target word. To succeed on LAMBADA, computational models
cannot simply rely on local context, but must be able to keep track of information
{"logiqa":{"description":"LogiQA is a dataset for testing human logical reasoning. It consists of 8,678 QA\ninstances, covering multiple types of deductive reasoning. Results show that state-\nof-the-art neural models perform by far worse than human ceiling. The dataset can\nalso serve as a benchmark for reinvestigating logical AI under the deep learning\nNLP setting.\n","citation":"@misc{liu2020logiqa,\n title={LogiQA: A Challenge Dataset for Machine Reading Comprehension with Logical Reasoning}, \n author={Jian Liu and Leyang Cui and Hanmeng Liu and Dandan Huang and Yile Wang and Yue Zhang},\n year={2020},\n eprint={2007.08124},\n archivePrefix={arXiv},\n primaryClass={cs.CL}\n}\n","homepage":"https://github.com/lgw863/LogiQA-dataset","license":"","features":{"label":{"dtype":"string","id":null,"_type":"Value"},"context":{"dtype":"string","id":null,"_type":"Value"},"question":{"dtype":"string","id":null,"_type":"Value"},"options":{"feature":{"dtype":"string","id":null,"_type":"Value"},"length":-1,"id":null,"_type":"Sequence"}},"post_processed":null,"supervised_keys":null,"task_templates":null,"builder_name":"logiqa","config_name":"logiqa","version":{"version_str":"0.0.1","description":null,"major":0,"minor":0,"patch":1},"splits":{"train":{"name":"train","num_bytes":6419852,"num_examples":7376,"dataset_name":"logiqa"},"test":{"name":"test","num_bytes":571705,"num_examples":651,"dataset_name":"logiqa"},"validation":{"name":"validation","num_bytes":562437,"num_examples":651,"dataset_name":"logiqa"}},"download_checksums":{"https://raw.githubusercontent.com/lgw863/LogiQA-dataset/master/Train.txt":{"num_bytes":6281272,"checksum":"7d5bb1f58278e33b395744cd2ad8d7600faa0b3c4d615c659a44ec1181d759fa"},"https://raw.githubusercontent.com/lgw863/LogiQA-dataset/master/Test.txt":{"num_bytes":559060,"checksum":"359acb78c37802208f7fde9e2f6574b8526527c63d6a336f90a53f1932cb4701"},"https://raw.githubusercontent.com/lgw863/LogiQA-dataset/master/Eval.txt":{"num_bytes":550021,"checksum":"4c49e6753b7262c001506b9151135abf722247035ab075dad93acdea5789c01f"}},"download_size":7390353,"post_processing_size":null,"dataset_size":7553994,"size_in_bytes":14944347}}
\ No newline at end of file
{"logiqa":{"description":"LogiQA is a dataset for testing human logical reasoning. It consists of 8,678 QA\ninstances, covering multiple types of deductive reasoning. Results show that state-\nof-the-art neural models perform by far worse than human ceiling. The dataset can\nalso serve as a benchmark for reinvestigating logical AI under the deep learning\nNLP setting.\n","citation":"@misc{liu2020logiqa,\n title={LogiQA: A Challenge Dataset for Machine Reading Comprehension with Logical Reasoning}, \n author={Jian Liu and Leyang Cui and Hanmeng Liu and Dandan Huang and Yile Wang and Yue Zhang},\n year={2020},\n eprint={2007.08124},\n archivePrefix={arXiv},\n primaryClass={cs.CL}\n}\n","homepage":"https://github.com/lgw863/LogiQA-dataset","license":"","features":{"label":{"dtype":"string","id":null,"_type":"Value"},"context":{"dtype":"string","id":null,"_type":"Value"},"question":{"dtype":"string","id":null,"_type":"Value"},"options":{"feature":{"dtype":"string","id":null,"_type":"Value"},"length":-1,"id":null,"_type":"Sequence"}},"post_processed":null,"supervised_keys":null,"task_templates":null,"builder_name":"logiqa","config_name":"logiqa","version":{"version_str":"0.0.1","description":null,"major":0,"minor":0,"patch":1},"splits":{"train":{"name":"train","num_bytes":6419852,"num_examples":7376,"dataset_name":"logiqa"},"test":{"name":"test","num_bytes":571705,"num_examples":651,"dataset_name":"logiqa"},"validation":{"name":"validation","num_bytes":562437,"num_examples":651,"dataset_name":"logiqa"}},"download_checksums":{"https://raw.githubusercontent.com/lgw863/LogiQA-dataset/master/Train.txt":{"num_bytes":6281272,"checksum":"7d5bb1f58278e33b395744cd2ad8d7600faa0b3c4d615c659a44ec1181d759fa"},"https://raw.githubusercontent.com/lgw863/LogiQA-dataset/master/Test.txt":{"num_bytes":559060,"checksum":"359acb78c37802208f7fde9e2f6574b8526527c63d6a336f90a53f1932cb4701"},"https://raw.githubusercontent.com/lgw863/LogiQA-dataset/master/Eval.txt":{"num_bytes":550021,"checksum":"4c49e6753b7262c001506b9151135abf722247035ab075dad93acdea5789c01f"}},"download_size":7390353,"post_processing_size":null,"dataset_size":7553994,"size_in_bytes":14944347}}
