Setting trust_remote_code to True for HuggingFace datasets compatibility (#1487)

* setting trust_remote_code

* dataset list no notebooks

* respect trust remote code

* Address changes, move cli options and change datasets

* fix task for tests

* headqa

* remove kobest

* pin datasets and address comments

* clean up space

lm_eval/tasks/model_written_evals/sycophancy/sycophancy_on_nlp_survey.yaml

@@ -12,3 +12,5 @@ metric_list:
   - metric: acc
 metadata:
   version: 0.0
+dataset_kwargs:
+  trust_remote_code: true

lm_eval/tasks/model_written_evals/sycophancy/sycophancy_on_philpapers2020.yaml

@@ -12,3 +12,5 @@ metric_list:
   - metric: acc
 metadata:
   version: 0.0
+dataset_kwargs:
+  trust_remote_code: true

lm_eval/tasks/model_written_evals/sycophancy/sycophancy_on_political_typology_quiz.yaml

@@ -12,3 +12,5 @@ metric_list:
   - metric: acc
 metadata:
   version: 0.0
+dataset_kwargs:
+  trust_remote_code: true

lm_eval/tasks/mutual/mutual.yaml

@@ -23,3 +23,5 @@ metric_list:
     higher_is_better: true
 metadata:
   version: 2.0
+dataset_kwargs:
+  trust_remote_code: true

lm_eval/tasks/race/race.yaml

@@ -12,3 +12,5 @@ metric_list:
     higher_is_better: true
 metadata:
   version: 2.0
+dataset_kwargs:
+  trust_remote_code: true

lm_eval/tasks/unscramble/anagrams1.yaml

@@ -18,3 +18,5 @@ metric_list:
     ignore_punctuation: false
 metadata:
   version: 2.0
+dataset_kwargs:
+  trust_remote_code: true

lm_eval/tasks/unscramble/anagrams2.yaml

@@ -18,3 +18,5 @@ metric_list:
     ignore_punctuation: false
 metadata:
   version: 2.0
+dataset_kwargs:
+  trust_remote_code: true

lm_eval/tasks/unscramble/cycle_letters.yaml

@@ -18,3 +18,5 @@ metric_list:
     ignore_punctuation: false
 metadata:
   version: 2.0
+dataset_kwargs:
+  trust_remote_code: true

lm_eval/tasks/unscramble/random_insertion.yaml

@@ -18,3 +18,5 @@ metric_list:
     ignore_punctuation: false
 metadata:
   version: 2.0
+dataset_kwargs:
+  trust_remote_code: true

lm_eval/tasks/wikitext/wikitext.yaml

@@ -16,3 +16,5 @@ metric_list:
   - metric: bits_per_byte
 metadata:
   version: 2.0
+dataset_kwargs:
+  trust_remote_code: true

pyproject.toml

@@ -21,7 +21,7 @@ license = { "text" = "MIT" }
 dependencies = [
     "accelerate>=0.21.0",
     "evaluate",
-    "datasets>=2.14.0",
+    "datasets>=2.16.0",
     "evaluate>=0.4.0",
     "jsonlines",
     "numexpr",