aggregate by group (total and categories) (#2643)

lm_eval/tasks/kmmlu/direct/kmmlu_direct_taxation.yaml

 dataset_name: Taxation
 include: _direct_kmmlu_yaml
 task: kmmlu_direct_taxation
+tag: kmmlu_direct_humss_tasks

lm_eval/tasks/kmmlu/direct/kmmlu_direct_telecommunications_and_wireless_technology.yaml

 dataset_name: Telecommunications-and-Wireless-Technology
 include: _direct_kmmlu_yaml
 task: kmmlu_direct_telecommunications_and_wireless_technology
+tag: kmmlu_direct_applied_science_tasks

lm_eval/tasks/kmmlu/direct_hard/_direct_hard_kmmlu_yaml

-tag:
-    - kmmlu
-    - kmmlu_hard_direct
 dataset_path: HAERAE-HUB/KMMLU-HARD
 output_type: generate_until
 test_split: test

lm_eval/tasks/kmmlu/direct_hard/_kmmlu_direct_hard.yaml

+group: kmmlu_direct_hard
+task:
+  - kmmlu_direct_hard_stem
+  - kmmlu_direct_hard_other
+  - kmmlu_direct_hard_applied_science
+  - kmmlu_direct_hard_humss
+aggregate_metric_list:
+  - metric: exact_match
+    weight_by_size: True
+metadata:
+  version: 2.0

lm_eval/tasks/kmmlu/direct_hard/_kmmlu_direct_hard_applied_science.yaml

+group: kmmlu_direct_hard_applied_science
+task:
+  - kmmlu_direct_hard_applied_science_tasks
+aggregate_metric_list:
+  - metric: exact_match
+    weight_by_size: True
+metadata:
+  version: 2.0

lm_eval/tasks/kmmlu/direct_hard/_kmmlu_direct_hard_humss.yaml

+group: kmmlu_direct_hard_humss
+task:
+  - kmmlu_direct_hard_humss_tasks
+aggregate_metric_list:
+  - metric: exact_match
+    weight_by_size: True
+metadata:
+  version: 2.0

lm_eval/tasks/kmmlu/direct_hard/_kmmlu_direct_hard_other.yaml

+group: kmmlu_direct_hard_other
+task:
+  - kmmlu_direct_hard_other_tasks
+aggregate_metric_list:
+  - metric: exact_match
+    weight_by_size: True
+metadata:
+  version: 2.0

lm_eval/tasks/kmmlu/direct_hard/_kmmlu_direct_hard_stem.yaml

+group: kmmlu_direct_hard_stem
+task:
+  - kmmlu_direct_hard_stem_tasks
+aggregate_metric_list:
+  - metric: exact_match
+    weight_by_size: True
+metadata:
+  version: 2.0

lm_eval/tasks/kmmlu/direct_hard/kmmlu_direct_hard_accounting.yaml

 dataset_name: accounting
 include: _direct_hard_kmmlu_yaml
-task: kmmlu_hard_direct_accounting
+task: kmmlu_direct_hard_accounting
+tag: kmmlu_direct_hard_humss_tasks

lm_eval/tasks/kmmlu/direct_hard/kmmlu_direct_hard_agricultural_sciences.yaml

 dataset_name: agricultural_sciences
 include: _direct_hard_kmmlu_yaml
-task: kmmlu_hard_direct_agricultural_sciences
+task: kmmlu_direct_hard_agricultural_sciences
+tag: kmmlu_direct_hard_other_tasks

lm_eval/tasks/kmmlu/direct_hard/kmmlu_direct_hard_aviation_engineering_and_maintenance.yaml

 dataset_name: aviation_engineering_and_maintenance
 include: _direct_hard_kmmlu_yaml
-task: kmmlu_hard_direct_aviation_engineering_and_maintenance
+task: kmmlu_direct_hard_aviation_engineering_and_maintenance
+tag: kmmlu_direct_hard_applied_science_tasks

lm_eval/tasks/kmmlu/direct_hard/kmmlu_direct_hard_biology.yaml

 dataset_name: biology
 include: _direct_hard_kmmlu_yaml
-task: kmmlu_hard_direct_biology
+task: kmmlu_direct_hard_biology
+tag: kmmlu_direct_hard_stem_tasks

lm_eval/tasks/kmmlu/direct_hard/kmmlu_direct_hard_chemical_engineering.yaml

 dataset_name: chemical_engineering
 include: _direct_hard_kmmlu_yaml
-task: kmmlu_hard_direct_chemical_engineering
+task: kmmlu_direct_hard_chemical_engineering
+tag: kmmlu_direct_hard_stem_tasks

lm_eval/tasks/kmmlu/direct_hard/kmmlu_direct_hard_chemistry.yaml

 dataset_name: chemistry
 include: _direct_hard_kmmlu_yaml
-task: kmmlu_hard_direct_chemistry
+task: kmmlu_direct_hard_chemistry
+tag: kmmlu_direct_hard_stem_tasks

lm_eval/tasks/kmmlu/direct_hard/kmmlu_direct_hard_civil_engineering.yaml

 dataset_name: civil_engineering
 include: _direct_hard_kmmlu_yaml
-task: kmmlu_hard_direct_civil_engineering
+task: kmmlu_direct_hard_civil_engineering
+tag: kmmlu_direct_hard_stem_tasks

lm_eval/tasks/kmmlu/direct_hard/kmmlu_direct_hard_computer_science.yaml

 dataset_name: computer_science
 include: _direct_hard_kmmlu_yaml
-task: kmmlu_hard_direct_computer_science
+task: kmmlu_direct_hard_computer_science
+tag: kmmlu_direct_hard_stem_tasks

lm_eval/tasks/kmmlu/direct_hard/kmmlu_direct_hard_construction.yaml

 dataset_name: construction
 include: _direct_hard_kmmlu_yaml
-task: kmmlu_hard_direct_construction
+task: kmmlu_direct_hard_construction
+tag: kmmlu_direct_hard_other_tasks

lm_eval/tasks/kmmlu/direct_hard/kmmlu_direct_hard_criminal_law.yaml

 dataset_name: criminal_law
 include: _direct_hard_kmmlu_yaml
-task: kmmlu_hard_direct_criminal_law
+task: kmmlu_direct_hard_criminal_law
+tag: kmmlu_direct_hard_humss_tasks

lm_eval/tasks/kmmlu/direct_hard/kmmlu_direct_hard_ecology.yaml

 dataset_name: ecology
 include: _direct_hard_kmmlu_yaml
-task: kmmlu_hard_direct_ecology
+task: kmmlu_direct_hard_ecology
+tag: kmmlu_direct_hard_stem_tasks

lm_eval/tasks/kmmlu/direct_hard/kmmlu_direct_hard_economics.yaml

 dataset_name: economics
 include: _direct_hard_kmmlu_yaml
-task: kmmlu_hard_direct_economics
+task: kmmlu_direct_hard_economics
+tag: kmmlu_direct_hard_humss_tasks