Merge pull request #4 from EleutherAI/master

Update cfoster0 fork

requirements.txt

+black==20.8b1
+best_download>=0.0.5
+datasets>=1.2.1
+click>=7.1
+scikit-learn>=0.24.1
+torch>=1.7
+transformers>=4.1
+sqlitedict==1.6.0
+pytablewriter
\ No newline at end of file

scripts/make_table.py

+from lm_eval import tasks
+from pytablewriter import MarkdownTableWriter
+
+writer = MarkdownTableWriter()
+writer.headers = ["Task Name", "Train", "Val", "Test", "Metrics"]
+
+values = []
+
+def chk(tf):
+    if tf:
+        return '✓'
+    else:
+        return ' '
+
+for tname, Task in tasks.TASK_REGISTRY.items():
+    task = Task()
+
+    values.append([tname,chk(task.has_training_docs()),chk(task.has_validation_docs()),chk(task.has_test_docs()),', '.join(task.aggregation().keys())])
+
+writer.value_matrix = values
+
+print(writer.dumps())
\ No newline at end of file

tests/test_models.py

+import lm_eval.models as models
+import lm_eval.base as base
+
+def test_gpt2():
+    gpt2 = models.get_model('gpt2').create_from_arg_string("device=cpu")
+    (ll_dog, ig_dog), (ll_cat, ig_cat) = gpt2.loglikelihood([
+        ('The quick brown fox jumps over the lazy', ' dog'),
+        ('The quick brown fox jumps over the lazy', ' cat'),
+    ])
+
+    assert ll_dog > ll_cat
+    assert not ig_cat
+
+    # test empty context
+    gpt2.loglikelihood([('', 'test')])
\ No newline at end of file