Logging Updates (Alphabetize table printouts, fix eval tracker bug) (#1774) (#1791)

* fix auto-batch size bug for seq2seq models

* alphabetize task + group tables ; fix eval tracker bug

* fix eval tracker bug

lm_eval/logging/evaluation_tracker.py

@@ -81,7 +81,7 @@ class EvaluationTracker:
 
     def __init__(
         self,
-        output_path: str = "",
+        output_path: str = None,
         hub_results_org: str = "",
         hub_repo_name: str = "",
         push_results_to_hub: bool = False,

lm_eval/models/huggingface.py

@@ -667,6 +667,8 @@ class HFLM(TemplateLM):
             max_cont_enc = len(continuation_enc[-(self.max_length + 1) :])
         else:
             max_length = self.max_length
+            max_context_enc = max_length
+            max_cont_enc = max_length
 
         # if OOM, then halves batch_size and tries again
         @find_executable_batch_size(starting_batch_size=self.max_batch_size)

lm_eval/utils.py

@@ -242,7 +242,7 @@ class Reorderer:
         return res
 
 
-def make_table(result_dict, column: str = "results"):
+def make_table(result_dict, column: str = "results", sort_results: bool = True):
     """Generate table of results."""
     from pytablewriter import LatexTableWriter, MarkdownTableWriter
 
@@ -269,7 +269,12 @@ def make_table(result_dict, column: str = "results"):
 
     values = []
 
-    for k, dic in result_dict[column].items():
+    keys = result_dict[column].keys()
+    if sort_results:
+        # sort entries alphabetically
+        keys = sorted(keys)
+    for k in keys:
+        dic = result_dict[column][k]
         version = result_dict["versions"].get(k, "N/A")
         n = str(result_dict["n-shot"][k])