Unverified Commit 538b4393 authored by Haodong Duan's avatar Haodong Duan Committed by GitHub
Browse files

[Fix] Fix seed in HFEvaluator (#122)

parent 46c96457
import random
from typing import List from typing import List
import evaluate import evaluate
import numpy as np
from opencompass.registry import ICL_EVALUATORS from opencompass.registry import ICL_EVALUATORS
...@@ -12,10 +14,15 @@ class HuggingfaceEvaluator(BaseEvaluator): ...@@ -12,10 +14,15 @@ class HuggingfaceEvaluator(BaseEvaluator):
Args: Args:
metric (str): Metric name in evaluate module. metric (str): Metric name in evaluate module.
seed (int): There exists some randomness during the calculation of some
metrics, thus we set a fixed random seed for reproducing. Defaults
to 0.
""" """
def __init__(self, metric: str) -> None: def __init__(self, metric: str, seed: int = 0) -> None:
self.metric = metric self.metric = metric
random.seed(seed)
np.random.seed(seed)
super().__init__() super().__init__()
def _preprocess(self, predictions: List, references: List) -> dict: def _preprocess(self, predictions: List, references: List) -> dict:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment