Add LM-SynEval Benchmark (#3184)

* add lm_syneval

* edit readme

* update task readme

* formatting fixes

* run linting

* add descriptions and examples

* clean readme formatting

lm_eval/tasks/README.md

@@ -87,6 +87,7 @@
 | [leaderboard](leaderboard/README.md)                                     | Task group used by Hugging Face's [Open LLM Leaderboard v2](https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard). Those tasks are static and will not change through time                                                                                                                                          | English                                                                                                                       |
 | [lingoly](lingoly/README.md)                                             | Challenging logical reasoning benchmark in low-resource languages with controls for memorization                                                                                                                                                                                                                                       | English, Multilingual                                                                                                         |
 | [libra](libra/README.md)                                                 | Evaluates long-context understanding in Russian across four complexity levels                                                                                                                                                                                                                                                          | Russian (MT)                                                                                                               |
+| [lm_syneval](lm_syneval/README.md)                                                 | Evaluates the syntactic capabilities of language models.                                                                                                                                                                                                                                                          | English                |
 | [logiqa](logiqa/README.md)                                               | Logical reasoning tasks requiring advanced inference and deduction.                                                                                                                                                                                                                                                                    | English, Chinese                                                                                                              |
 | [logiqa2](logiqa2/README.md)                                             | Large-scale logical reasoning dataset adapted from the Chinese Civil Service Examination.                                                                                                                                                                                                                                              | English, Chinese                                                                                                              |
 | [mastermind](mastermind/README.md)                                       | Reasoning benchmark based on the board game of Mastermind.                                                                                                                                                                                                                                                                             | English                                                                                                                       |

lm_eval/tasks/lm_syneval/_template_yaml

+dataset_path: jmichaelov/lm_syneval
+output_type: multiple_choice
+test_split: test
+doc_to_text: ""
+target_delimiter: ""
+doc_to_target: 0
+doc_to_choice: "{{[sentence_good, sentence_bad]}}"
+num_fewshot: 0
+metric_list:
+  - metric: acc
+    aggregation: mean
+    higher_is_better: true
+metadata:
+  version: 1.0

lm_eval/tasks/lm_syneval/lm_syneval__agreement__long_vp_coord__plur_MS_LMV_LMV.yaml

+dataset_name: lm_syneval__agreement__long_vp_coord__plur_MS_LMV_LMV
+include: _template_yaml
+task: lm_syneval__agreement__long_vp_coord__plur_MS_LMV_LMV

lm_eval/tasks/lm_syneval/lm_syneval__agreement__long_vp_coord__sing_MS_LMV_LMV.yaml

+dataset_name: lm_syneval__agreement__long_vp_coord__sing_MS_LMV_LMV
+include: _template_yaml
+task: lm_syneval__agreement__long_vp_coord__sing_MS_LMV_LMV

lm_eval/tasks/lm_syneval/lm_syneval__agreement__obj_rel_across_anim__plur_MS_MV_plur_ES_EV.yaml

+dataset_name: lm_syneval__agreement__obj_rel_across_anim__plur_MS_MV_plur_ES_EV
+include: _template_yaml
+task: lm_syneval__agreement__obj_rel_across_anim__plur_MS_MV_plur_ES_EV

lm_eval/tasks/lm_syneval/lm_syneval__agreement__obj_rel_across_anim__plur_MS_MV_sing_ES_EV.yaml

+dataset_name: lm_syneval__agreement__obj_rel_across_anim__plur_MS_MV_sing_ES_EV
+include: _template_yaml
+task: lm_syneval__agreement__obj_rel_across_anim__plur_MS_MV_sing_ES_EV

lm_eval/tasks/lm_syneval/lm_syneval__agreement__obj_rel_across_anim__sing_MS_MV_plur_ES_EV.yaml

+dataset_name: lm_syneval__agreement__obj_rel_across_anim__sing_MS_MV_plur_ES_EV
+include: _template_yaml
+task: lm_syneval__agreement__obj_rel_across_anim__sing_MS_MV_plur_ES_EV

lm_eval/tasks/lm_syneval/lm_syneval__agreement__obj_rel_across_anim__sing_MS_MV_sing_ES_EV.yaml

+dataset_name: lm_syneval__agreement__obj_rel_across_anim__sing_MS_MV_sing_ES_EV
+include: _template_yaml
+task: lm_syneval__agreement__obj_rel_across_anim__sing_MS_MV_sing_ES_EV

lm_eval/tasks/lm_syneval/lm_syneval__agreement__obj_rel_across_inanim__plur_IS_IV_plur_ES_EV.yaml

+dataset_name: lm_syneval__agreement__obj_rel_across_inanim__plur_IS_IV_plur_ES_EV
+include: _template_yaml
+task: lm_syneval__agreement__obj_rel_across_inanim__plur_IS_IV_plur_ES_EV

lm_eval/tasks/lm_syneval/lm_syneval__agreement__obj_rel_across_inanim__plur_IS_IV_sing_ES_EV.yaml

+dataset_name: lm_syneval__agreement__obj_rel_across_inanim__plur_IS_IV_sing_ES_EV
+include: _template_yaml
+task: lm_syneval__agreement__obj_rel_across_inanim__plur_IS_IV_sing_ES_EV

lm_eval/tasks/lm_syneval/lm_syneval__agreement__obj_rel_across_inanim__sing_IS_IV_plur_ES_EV.yaml

+dataset_name: lm_syneval__agreement__obj_rel_across_inanim__sing_IS_IV_plur_ES_EV
+include: _template_yaml
+task: lm_syneval__agreement__obj_rel_across_inanim__sing_IS_IV_plur_ES_EV

lm_eval/tasks/lm_syneval/lm_syneval__agreement__obj_rel_across_inanim__sing_IS_IV_sing_ES_EV.yaml

+dataset_name: lm_syneval__agreement__obj_rel_across_inanim__sing_IS_IV_sing_ES_EV
+include: _template_yaml
+task: lm_syneval__agreement__obj_rel_across_inanim__sing_IS_IV_sing_ES_EV

lm_eval/tasks/lm_syneval/lm_syneval__agreement__obj_rel_no_comp_across_anim__plur_MS_MV_plur_ES_EV.yaml

+dataset_name: lm_syneval__agreement__obj_rel_no_comp_across_anim__plur_MS_MV_plur_ES_EV
+include: _template_yaml
+task: lm_syneval__agreement__obj_rel_no_comp_across_anim__plur_MS_MV_plur_ES_EV

lm_eval/tasks/lm_syneval/lm_syneval__agreement__obj_rel_no_comp_across_anim__plur_MS_MV_sing_ES_EV.yaml

+dataset_name: lm_syneval__agreement__obj_rel_no_comp_across_anim__plur_MS_MV_sing_ES_EV
+include: _template_yaml
+task: lm_syneval__agreement__obj_rel_no_comp_across_anim__plur_MS_MV_sing_ES_EV

lm_eval/tasks/lm_syneval/lm_syneval__agreement__obj_rel_no_comp_across_anim__sing_MS_MV_plur_ES_EV.yaml

+dataset_name: lm_syneval__agreement__obj_rel_no_comp_across_anim__sing_MS_MV_plur_ES_EV
+include: _template_yaml
+task: lm_syneval__agreement__obj_rel_no_comp_across_anim__sing_MS_MV_plur_ES_EV

lm_eval/tasks/lm_syneval/lm_syneval__agreement__obj_rel_no_comp_across_anim__sing_MS_MV_sing_ES_EV.yaml

+dataset_name: lm_syneval__agreement__obj_rel_no_comp_across_anim__sing_MS_MV_sing_ES_EV
+include: _template_yaml
+task: lm_syneval__agreement__obj_rel_no_comp_across_anim__sing_MS_MV_sing_ES_EV

lm_eval/tasks/lm_syneval/lm_syneval__agreement__obj_rel_no_comp_across_inanim__plur_IS_IV_plur_ES_EV.yaml

+dataset_name: lm_syneval__agreement__obj_rel_no_comp_across_inanim__plur_IS_IV_plur_ES_EV
+include: _template_yaml
+task: lm_syneval__agreement__obj_rel_no_comp_across_inanim__plur_IS_IV_plur_ES_EV

lm_eval/tasks/lm_syneval/lm_syneval__agreement__obj_rel_no_comp_across_inanim__plur_IS_IV_sing_ES_EV.yaml

+dataset_name: lm_syneval__agreement__obj_rel_no_comp_across_inanim__plur_IS_IV_sing_ES_EV
+include: _template_yaml
+task: lm_syneval__agreement__obj_rel_no_comp_across_inanim__plur_IS_IV_sing_ES_EV

lm_eval/tasks/lm_syneval/lm_syneval__agreement__obj_rel_no_comp_across_inanim__sing_IS_IV_plur_ES_EV.yaml

+dataset_name: lm_syneval__agreement__obj_rel_no_comp_across_inanim__sing_IS_IV_plur_ES_EV
+include: _template_yaml
+task: lm_syneval__agreement__obj_rel_no_comp_across_inanim__sing_IS_IV_plur_ES_EV