Skip to content

GitLab

Commit 9d0df41b authored by haileyschoelkopf's avatar haileyschoelkopf
Browse files

add fewshot_config field

parent 624fb207
Hide whitespace changes
Inline Side-by-side
Showing with 4 additions and 3 deletions
lm_eval/api/task.py
View file @ 9d0df41b
...@@ -75,6 +75,7 @@ class TaskConfig(dict): ...@@ -75,6 +75,7 @@ class TaskConfig(dict):
description: str = "" description: str = ""
target_delimiter: str = " " target_delimiter: str = " "
fewshot_delimiter: str = "\n\n" fewshot_delimiter: str = "\n\n"
fewshot_config: dict = None
# runtime configuration options # runtime configuration options
num_fewshot: int = 0 num_fewshot: int = 0
# scoring options # scoring options
...@@ -629,9 +630,9 @@ class ConfigurableTask(Task): ...@@ -629,9 +630,9 @@ class ConfigurableTask(Task):
self.prompt = None self.prompt = None
if self.fewshot_docs() is not None: if self.fewshot_docs() is not None:
self.sampler = samplers.Sampler( self.sampler = samplers.get_sampler(
list(self.fewshot_docs()), self, rnd=random.Random(1234) self.config.fewshot_config.get("sampler", "default")
) )(list(self.fewshot_docs()), self, rnd=random.Random(1234))
if self.has_test_docs(): if self.has_test_docs():
self.task_docs = self.test_docs() self.task_docs = self.test_docs()
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment