Merge remote-tracking branch 'origin/master' into fazz/refactor-task-coqa

.github/workflows/python-app.yml

+# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: Python application
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Cache
+      uses: actions/cache@v2.1.3
+      with:
+        # A list of files, directories, and wildcard patterns to cache and restore
+        path: |
+          data
+          ~/.cache
+        # An explicit key for restoring and saving the cache
+        key: evaldata-cache
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.9
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install flake8 pytest pytest-cov
+        pip install -e .
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+    - name: Lint with flake8
+      run: |
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+    - name: Test with pytest
+      run: |
+        pytest --cov=lm_eval/ tests/
+    - name: Upload to codecov
+      run: |
+        bash <(curl -s https://codecov.io/bash)
\ No newline at end of file

README.md

 # Evaluation Harness for Large Language Models
 
+![](https://github.com/EleutherAI/lm-evaluation-harness/workflows/Python%20application/badge.svg)
+[![codecov](https://codecov.io/gh/EleutherAI/lm-evaluation-harness/branch/master/graph/badge.svg?token=JSG3O2427J)](https://codecov.io/gh/EleutherAI/lm-evaluation-harness)
+
 ## Overview 
 
 The goal of this project is to build a set of tools for evaluating LMs on typical NLU tasks, based on evaluation of GPT-3 as described in https://arxiv.org/pdf/2005.14165.pdf. Following the initial description, this repo should support 3 functions:

lm_eval/tasks/__init__.py

@@ -46,7 +46,7 @@ TASK_REGISTRY = {
     "lambada": lambada.LAMBADA,
     "piqa": piqa.PiQA,
 
-    "triviaqa": triviaqa.TriviaQA,
+    #"triviaqa": triviaqa.TriviaQA,
     # "arc_easy": arc.ARCEasy, # not implemented yet
     # "arc_challenge": arc.ARCChallenge, # not implemented yet
     # "quac": quac.QuAC, # not implemented yet

lm_eval/tasks/naturalqs.py

 from . common import HFTask
 from itertools import islice
+import random
 
 class NaturalQs(HFTask):
+    # TODO: naturalqs has a *really* large train set that huggingface just
+    # automatically downloads even if you dont use it. we should try and only 
+    # download the val set and not even bother with the train set. 
+
     DATASET_PATH = "natural_questions"
     DATASET_NAME = None
 

lm_eval/tasks/race.py

@@ -23,7 +23,8 @@ class RACE(HFTask):
         return True
 
     def _collate_data(self, set):
-        if set in self.cache: return self.cache[set]
+        if set in self.cache:
+            return self.cache[set]
         # One big issue with HF's implementation of this dataset: it makes a
         # separate document for each question; meanwhile, in the GPT3 paper it
         # is shown that one document is made per passage.

lm_eval/tasks/triviaqa.py

@@ -21,7 +21,7 @@ class TriviaQA(Dataset):
         return True
 
     def has_test_docs(self):
-        return True
+        return False
 
     def training_docs(self):
         return json.load(open('data/triviaqa/triviaqa-unfiltered/unfiltered-web-train.json'))['Data']

lm_eval/tasks/wikitext.py

@@ -14,9 +14,11 @@ class WikiText103(NLP_TASK):
 
     def doc_to_text(self, doc):
         # TODO: implement
+        pass
 
     def doc_to_target(self, doc):
         # TODO: implement
+        pass
 
     def construct_requests(self, doc, ctx):
         """ Uses RequestFactory to construct Requests and returns an iterable of 
@@ -74,9 +76,11 @@ class WikiText2(NLP_TASK):
 
     def doc_to_text(self, doc):
         # TODO: implement
+        pass
 
     def doc_to_target(self, doc):
         # TODO: implement
+        pass
 
     def construct_requests(self, doc, ctx):
         """ Uses RequestFactory to construct Requests and returns an iterable of