`arithmetic`: set target delimiter to empty string (#2701)

* set target delimiter to empty string

* nit

* add warning

lm_eval/api/samplers.py

+import logging
+import warnings
 from functools import partial
 from typing import TYPE_CHECKING, Iterable, Optional, Union
 
@@ -9,6 +11,8 @@ if TYPE_CHECKING:
 
     from lm_eval.api.task import ConfigurableTask, Task
 
+eval_logger = logging.getLogger("lm-eval")
+
 
 class ContextSampler:
     def __init__(
@@ -97,6 +101,13 @@ class ContextSampler:
                 labeled_examples += self.doc_to_choice(doc)[doc_content]
 
             if doc_target != "":
+                if self.target_delimiter.isspace() and str(doc_target)[0].isspace():
+                    # TODO: add logger warn once here.
+                    warnings.warn(
+                        "Both target_delimiter and target start with a space. This may cause issues.",
+                        Warning,
+                        stacklevel=2,
+                    )
                 labeled_examples += self.target_delimiter
                 labeled_examples += prefix
                 labeled_examples += (

lm_eval/tasks/arithmetic/README.md

@@ -58,3 +58,6 @@ If other tasks on this dataset are already supported:
 * [ ] Is the "Main" variant of this task clearly denoted?
 * [ ] Have you provided a short sentence in a README on what each new variant adds / evaluates?
 * [ ] Have you noted which, if any, published evaluation setups are matched by this variant?
+
+### Changelog
+version 2.0: (2025-Feb-14) set target delimiter to "" as the targets already start with a space.

lm_eval/tasks/arithmetic/arithmetic_1dc.yaml

@@ -8,11 +8,12 @@ validation_split: validation
 test_split: null
 doc_to_text: "{{context}}"
 doc_to_target: "{{completion}}"
+target_delimiter: ""
 metric_list:
   - metric: acc
     aggregation: mean
     higher_is_better: true
 metadata:
-  version: 1.0
+  version: 2.0
 dataset_kwargs:
   trust_remote_code: true