Merge branch 'EleutherAI:main' into main

lm_eval/tasks/noticia/utils.py

+import string
+
+import evaluate
+
+
+def clean_text(text: str) -> str:
+    # Remove punctuation
+    text = text.translate(str.maketrans("", "", string.punctuation))
+
+    # Remove newlines and multiple spaces
+    text = text.replace("\n", " ").strip()
+    text = " ".join(text.split()).strip()
+
+    # lowercase
+    text = text.lower()
+
+    return text
+
+
+def rouge1(items):
+    """
+    # passthrough for efficiency
+    """
+    return items
+
+
+def average_len(items):
+    """
+    # passthrough for efficiency
+    """
+    return items
+
+
+def rouge1_agg(items):
+    """
+    Higher is better
+    """
+
+    refs = list(zip(*items))[0]
+    refs = [[clean_text(ref)] for ref in refs]
+    # print("refs", refs)
+    preds = [clean_text(x) for x in list(zip(*items))[1]]
+    # print("preds", preds)
+    rouge_scorer = evaluate.load("rouge")
+    return rouge_scorer.compute(predictions=preds, references=refs)["rouge1"]
+
+
+def average_len_agg(items):
+    """
+    Higher is better
+    """
+
+    preds = [clean_text(x) for x in list(zip(*items))[1]]
+
+    return sum(len(x.split()) for x in preds) / len(preds)

lm_eval/tasks/nq_open/README.md

+### Paper
+
+Question Answering dataset based on aggregated user queries from Google Search.
+
+Homepage: https://research.google/pubs/natural-questions-a-benchmark-for-question-answering-research/
+
+Homepage: [google-research-datasets/natural-questions@master/nq_open](https://github.com/google-research-datasets/natural-questions/tree/master/nq_open)
+
+Paper: [aclanthology.org/P19-1612](https://aclanthology.org/P19-1612/)
+
+Derived from the Natural Questions dataset, introduced in https://storage.googleapis.com/gweb-research2023-media/pubtools/pdf/1f7b46b5378d757553d3e92ead36bda2e4254244.pdf .
+
+
+### Citation
+
+```
+@article{47761,
+title	= {Natural Questions: a Benchmark for Question Answering Research},
+author	= {Tom Kwiatkowski and Jennimaria Palomaki and Olivia Redfield and Michael Collins and Ankur Parikh and Chris Alberti and Danielle Epstein and Illia Polosukhin and Matthew Kelcey and Jacob Devlin and Kenton Lee and Kristina N. Toutanova and Llion Jones and Ming-Wei Chang and Andrew Dai and Jakob Uszkoreit and Quoc Le and Slav Petrov},
+year	= {2019},
+journal	= {Transactions of the Association of Computational Linguistics}}
+```
+
+### Tasks
+
+* `nq_open`

lm_eval/tasks/paloma/README.md

+# Paloma
+
+### Paper
+Title: Paloma: A Benchmark for Evaluating Language Model Fit
+
+Abstract: https://arxiv.org/abs/2312.10523v1
+
+Paloma is a comprehensive benchmark designed to evaluate open language models across a wide range of domains, ranging from niche artist communities to mental health forums on Reddit. It assesses the performance of various models across 585 distinct domains.
+
+Homepage: https://allenai.org/olmo
+
+
+### Note
+
+If you are running the entire `paloma` benchmark (or just `paloma_dolma_100_programing_languages`) with a HuggingFace model, make sure to pass `logits_cache=False` to `--model_args`, for example:
+```
+lm_eval --model hf --model_args pretrained=EleutherAI/pythia-160m,logits_cache=False --tasks paloma
+```
+
+
+### Citation
+```
+@article{paloma,
+  title={{Paloma}: A Benchmark for Evaluating Language Model Fit},
+  author={Magnusson, Ian and Bhagia, Akshita and Hofmann, Valentin and Soldaini, Luca and Harsh Jha, Ananya and Tafjord, Oyvind and Schwenk,Dustin and Walsh, Evan Pete and Elazar, Yanai and Lo, Kyle and Groenveld,Dirk and Beltagy,Iz and  Hajishirz,Hanneneh and Smith, Noah A. and Richardson,Kyle and Dodge,Jesse},
+  journal={technical report},
+  year={2023},
+  url={https://paloma.allen.ai/}
+}
+```
+
+### Groups and Tasks
+
+#### Groups
+
+* `paloma`
+
+#### Tasks
+
+* `paloma_4chan_meta_sep`
+* `paloma_c4_100_domains`
+* `paloma_c4_en`
+* `paloma_dolma_100_programing_languages`
+* `paloma_dolma_100_subreddits`
+* `paloma_dolma-v1_5`
+* `paloma_falcon-refinedweb`
+* `paloma_gab`
+* `paloma_m2d2_s2orc_unsplit`
+* `paloma_m2d2_wikipedia_unsplit`
+* `paloma_manosphere_meta_sep`
+* `paloma_mc4`
+* `paloma_ptb`
+* `paloma_redpajama`
+* `paloma_twitterAAE_HELM_fixed`
+* `paloma_wikitext_103`
+
+### Checklist
+
+For adding novel benchmarks/datasets to the library:
+* [ ] Is the task an existing benchmark in the literature?
+  * [ ] Have you referenced the original paper that introduced the task?
+  * [ ] If yes, does the original paper provide a reference implementation? If so, have you checked against the reference implementation and documented how to run such a test?
+
+
+If other tasks on this dataset are already supported:
+* [ ] Is the "Main" variant of this task clearly denoted?
+* [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates?
+* [ ] Have you noted which, if any, published evaluation setups are matched by this variant?

lm_eval/tasks/paloma/_paloma_template

+group:
+  - paloma
+dataset_path: allenai/paloma
+output_type: loglikelihood_rolling
+validation_split: val
+test_split: test
+doc_to_text: ""
+doc_to_target: !function paloma_utils.doc_to_target
+should_decontaminate: true
+doc_to_decontamination_query: !function paloma_utils.doc_to_target
+metric_list:
+  - metric: word_perplexity
+    aggregation: weighted_perplexity
+    higher_is_better: false
+  - metric: byte_perplexity
+    aggregation: weighted_perplexity
+    higher_is_better: false
+  - metric: bits_per_byte
+    aggregation: bits_per_byte
+    higher_is_better: false
+metadata:
+  version: 1

lm_eval/tasks/paloma/paloma_4chan_meta_sep.yaml

+include: _paloma_template
+task: paloma_4chan_meta_sep
+task_alias: 4chan
+dataset_name: 4chan_meta_sep

lm_eval/tasks/paloma/paloma_c4_100_domains.yaml

+include: _paloma_template
+task: paloma_c4_100_domains
+task_alias: C4 100 Domains
+dataset_name: c4_100_domains

lm_eval/tasks/paloma/paloma_c4_en.yaml

+include: _paloma_template
+task: paloma_c4_en
+task_alias: C4
+dataset_name: c4_en

lm_eval/tasks/paloma/paloma_dolma-v1_5.yaml

+include: _paloma_template
+task: paloma_dolma-v1_5
+task_alias: Dolma V1.5
+dataset_name: dolma-v1_5

lm_eval/tasks/paloma/paloma_dolma_100_programing_languages.yaml

+include: _paloma_template
+task: paloma_dolma_100_programing_languages
+task_alias: 100 PLs
+dataset_name: dolma_100_programing_languages

lm_eval/tasks/paloma/paloma_dolma_100_subreddits.yaml

+include: _paloma_template
+task: paloma_dolma_100_subreddits
+task_alias: 100 Subreddits
+dataset_name: dolma_100_subreddits

lm_eval/tasks/paloma/paloma_falcon-refinedweb.yaml

+include: _paloma_template
+task: paloma_falcon-refinedweb
+task_alias: Falcon
+dataset_name: falcon-refinedweb

lm_eval/tasks/paloma/paloma_gab.yaml

+include: _paloma_template
+task: paloma_gab
+task_alias: Gab
+dataset_name: gab

lm_eval/tasks/paloma/paloma_m2d2_s2orc_unsplit.yaml

+include: _paloma_template
+task: paloma_m2d2_s2orc_unsplit
+task_alias: M2D2 S2ORC
+dataset_name: m2d2_s2orc_unsplit

lm_eval/tasks/paloma/paloma_m2d2_wikipedia_unsplit.yaml

+include: _paloma_template
+task: paloma_m2d2_wikipedia_unsplit
+task_alias: M2D2 Wikipedia
+dataset_name: m2d2_wikipedia_unsplit

lm_eval/tasks/paloma/paloma_manosphere_meta_sep.yaml

+include: _paloma_template
+task: paloma_manosphere_meta_sep
+task_alias: Manosphere
+dataset_name: manosphere_meta_sep

lm_eval/tasks/paloma/paloma_mc4.yaml

+include: _paloma_template
+task: paloma_mc4
+task_alias: mC4
+dataset_name: mc4

lm_eval/tasks/paloma/paloma_ptb.yaml

+include: _paloma_template
+task: paloma_ptb
+task_alias: PTB
+dataset_name: ptb

lm_eval/tasks/paloma/paloma_redpajama.yaml

+include: _paloma_template
+task: paloma_redpajama
+task_alias: RedPajama
+dataset_name: redpajama

lm_eval/tasks/paloma/paloma_twitterAAE_HELM_fixed.yaml

+include: _paloma_template
+task: paloma_twitterAAE_HELM_fixed
+task_alias: Twitter AAE
+dataset_name: twitterAAE_HELM_fixed

lm_eval/tasks/paloma/paloma_utils.py

+def doc_to_target(doc):
+    return str(doc["text"])