Added NorEval, a novel Norwegian benchmark (#2919)

* added noreval

* added a checklist for noreval

* run pre-commit

* changed imports and added short noreval description

* fixed norsumm path

* refactored multi-folder tasks

* refactored multi-folder tasks

lm_eval/tasks/noreval/norcommonsenseqa/nno/norcommonsenseqa_nno_p1.yaml

+tag: norcommonsenseqa_nno
+dataset_name: nn
+task: norcommonsenseqa_nno_p1
+include: ../_norcommonsenseqa_yaml
+doc_to_text: "{{question}}\nKva av følgande moglege svar er det rette?\nA: {{choices.text[0]}}\nB: {{choices.text[1]}}\nC: {{choices.text[2]}}\nD: {{choices.text[3]}}\nE: {{choices.text[4]}}\n\nSvar:"
+doc_to_choice: "{{choices.label}}"

lm_eval/tasks/noreval/norcommonsenseqa/nno/norcommonsenseqa_nno_p2.yaml

+tag: norcommonsenseqa_nno
+dataset_name: nn
+task: norcommonsenseqa_nno_p2
+include: ../_norcommonsenseqa_yaml
+doc_to_text: "Gitt alternativa under, kva er svaret på følgande spørsmål: {{question}}\n\nAlternativ:\n- {{choices.text[0]}}\n- {{choices.text[1]}}\n- {{choices.text[2]}}\n- {{choices.text[3]}}\n- {{choices.text[4]}}\n\nSvar:"
+doc_to_choice: "{{choices.label}}"

lm_eval/tasks/noreval/norcommonsenseqa/nno/norcommonsenseqa_nno_p3.yaml

+tag: norcommonsenseqa_nno
+dataset_name: nn
+task: norcommonsenseqa_nno_p3
+include: ../_norcommonsenseqa_yaml
+doc_to_text: "{{question}}\nVel rett svar blant desse alternativa:\n– {{choices.text[0]}}\n– {{choices.text[1]}}\n– {{choices.text[2]}}\n– {{choices.text[3]}}\n– {{choices.text[4]}}\n\nSvar:"
+doc_to_choice: "{{choices.text}}"

lm_eval/tasks/noreval/norcommonsenseqa/nno/norcommonsenseqa_nno_p4.yaml

+tag: norcommonsenseqa_nno
+dataset_name: nn
+task: norcommonsenseqa_nno_p4
+include: ../_norcommonsenseqa_yaml
+doc_to_text: "{{question}}\nA: {{choices.text[0]}}\nB: {{choices.text[1]}}\nC: {{choices.text[2]}}\nD: {{choices.text[3]}}\nE: {{choices.text[4]}}\n\nEr det rette svaret A, B, C, D, eller E?\n\nSvar:"
+doc_to_choice: "{{choices.label}}"

lm_eval/tasks/noreval/norcommonsenseqa/nob/norcommonsenseqa_nob_p0.yaml

+tag: norcommonsenseqa_nob
+dataset_name: nb
+task: norcommonsenseqa_nob_p0
+include: ../_norcommonsenseqa_yaml
+doc_to_text: "Spørsmål: {{question}}\n\nSvar:"
+doc_to_choice: "{{choices.text}}"

lm_eval/tasks/noreval/norcommonsenseqa/nob/norcommonsenseqa_nob_p1.yaml

+tag: norcommonsenseqa_nob
+dataset_name: nb
+task: norcommonsenseqa_nob_p1
+include: ../_norcommonsenseqa_yaml
+doc_to_text: "{{question}}\nHvilket av følgende mulige svar er det riktige?\nA: {{choices.text[0]}}\nB: {{choices.text[1]}}\nC: {{choices.text[2]}}\nD: {{choices.text[3]}}\nE: {{choices.text[4]}}\n\nSvar:"
+doc_to_choice: "{{choices.label}}"

lm_eval/tasks/noreval/norcommonsenseqa/nob/norcommonsenseqa_nob_p2.yaml

+tag: norcommonsenseqa_nob
+dataset_name: nb
+task: norcommonsenseqa_nob_p2
+include: ../_norcommonsenseqa_yaml
+doc_to_text: "Gitt alternativene under, hva er svaret på følgende spørsmål: {{question}}\n\nAlternativer:\n- {{choices.text[0]}}\n- {{choices.text[1]}}\n- {{choices.text[2]}}\n- {{choices.text[3]}}\n- {{choices.text[4]}}\n\nSvar:"
+doc_to_choice: "{{choices.text}}"

lm_eval/tasks/noreval/norcommonsenseqa/nob/norcommonsenseqa_nob_p3.yaml

+tag: norcommonsenseqa_nob
+dataset_name: nb
+task: norcommonsenseqa_nob_p3
+include: ../_norcommonsenseqa_yaml
+doc_to_text: "{{question}}\nVelg riktig svar blant disse alternativene:\n– {{choices.text[0]}}\n– {{choices.text[1]}}\n– {{choices.text[2]}}\n– {{choices.text[3]}}\n– {{choices.text[4]}}\n\nSvar:"
+doc_to_choice: "{{choices.text}}"

lm_eval/tasks/noreval/norcommonsenseqa/nob/norcommonsenseqa_nob_p4.yaml

+tag: norcommonsenseqa_nob
+dataset_name: nb
+task: norcommonsenseqa_nob_p4
+include: ../_norcommonsenseqa_yaml
+doc_to_text: "{{question}}\nA: {{choices.text[0]}}\nB: {{choices.text[1]}}\nC: {{choices.text[2]}}\nD: {{choices.text[3]}}\nE: {{choices.text[4]}}\n\nEr det riktige svaret A, B, C, D, eller E?\n\nSvar:"
+doc_to_choice: "{{choices.label}}"

lm_eval/tasks/noreval/norec/_norec_yaml

+dataset_name: binary
+output_type: multiple_choice
+training_split: train
+test_split: test
+doc_to_target: sentiment
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+  - metric: f1
+    aggregation: !function utils.multi_f1
+    higher_is_better: true
+metadata:
+  version: 1.0

lm_eval/tasks/noreval/norec/norec_document/norec_document_p0.yaml

+tag: norec_document
+dataset_path: ltg/norec_document
+task: norec_document_p0
+include: ../_norec_yaml
+doc_to_text: "Tekst: {{review}}\nSentiment:"
+doc_to_choice: ["negativ", "positiv"]

lm_eval/tasks/noreval/norec/norec_document/norec_document_p1.yaml

+tag: norec_document
+dataset_path: ltg/norec_document
+task: norec_document_p1
+include: ../_norec_yaml
+doc_to_text: "Tekst: {{review}}\nEr anmeldelsen \"positiv\" eller \"negativ\"?"
+doc_to_choice: ["negativ", "positiv"]

lm_eval/tasks/noreval/norec/norec_document/norec_document_p2.yaml

+tag: norec_document
+dataset_path: ltg/norec_document
+task: norec_document_p2
+include: ../_norec_yaml
+doc_to_text: "Er polariteten til følgende anmeldelse positiv eller negativ?\nAnmeldelse: {{review}}\nAnmeldelsen er"
+doc_to_choice: ["negativ", "positiv"]

lm_eval/tasks/noreval/norec/norec_document/norec_document_p3.yaml

+tag: norec_document
+dataset_path: ltg/norec_document
+task: norec_document_p3
+include: ../_norec_yaml
+doc_to_text: "Anmeldelse: {{review}}\nEr anmelderen positiv eller negativ?"
+doc_to_choice: ["negativ", "positiv"]

lm_eval/tasks/noreval/norec/norec_document/norec_document_p4.yaml

+tag: norec_document
+dataset_path: ltg/norec_document
+task: norec_document_p4
+include: ../_norec_yaml
+doc_to_text: "Anmeldelse: {{review}}\nVil du oppsummere anmeldelsen som \"bra\" eller \"dårlig\"?"
+doc_to_choice: ["dårlig", "bra"]

lm_eval/tasks/noreval/norec/norec_sentence/norec_sentence_p0.yaml

+tag: norec_sentence
+dataset_path: ltg/norec_sentence
+task: norec_sentence_p0
+include: ../_norec_yaml
+doc_to_text: "Tekst: {{review}}\nSentiment:"
+doc_to_choice: ["negativ", "positiv"]

lm_eval/tasks/noreval/norec/norec_sentence/norec_sentence_p1.yaml

+tag: norec_sentence
+dataset_path: ltg/norec_sentence
+task: norec_sentence_p1
+include: ../_norec_yaml
+doc_to_text: "{{review}}\nEr denne setningen \"positiv\" eller \"negativ\"?"
+doc_to_choice: ["negativ", "positiv"]

lm_eval/tasks/noreval/norec/norec_sentence/norec_sentence_p2.yaml

+tag: norec_sentence
+dataset_path: ltg/norec_sentence
+task: norec_sentence_p2
+include: ../_norec_yaml
+doc_to_text: "{{review}}\nHva slags sentiment uttrykker anmelderen?"
+doc_to_choice: ["negativ", "positiv"]

lm_eval/tasks/noreval/norec/norec_sentence/norec_sentence_p3.yaml

+tag: norec_sentence
+dataset_path: ltg/norec_sentence
+task: norec_sentence_p3
+include: ../_norec_yaml
+doc_to_text: "{{review}}\nEr anmeldelsen \"positiv\" eller \"negativ\"?"
+doc_to_choice: ["negativ", "positiv"]

lm_eval/tasks/noreval/norec/norec_sentence/norec_sentence_p4.yaml

+tag: norec_sentence
+dataset_path: ltg/norec_sentence
+task: norec_sentence_p4
+include: ../_norec_yaml
+doc_to_text: "{{review}}\nEr denne setningen positiv eller negativ?"
+doc_to_choice: ["negativ", "positiv"]