"git@developer.sourcefind.cn:orangecat/ollama.git" did not exist on "152ab524c2e909f4f10d2f1b48e5ac06298bc758"
Commit f650197a authored by Baber's avatar Baber
Browse files

refactor build_filter_ensemble to simplify filter creation

parent 70f5e2f0
from functools import partial from functools import partial
from typing import Iterable, List, Optional, Union from typing import Optional, Union
from lm_eval.api.filter import FilterEnsemble from lm_eval.api.filter import FilterEnsemble
from lm_eval.api.registry import get_filter from lm_eval.api.registry import get_filter
...@@ -8,18 +8,16 @@ from . import custom, extraction, selection, transformation ...@@ -8,18 +8,16 @@ from . import custom, extraction, selection, transformation
def build_filter_ensemble( def build_filter_ensemble(
filter_name: str, components: list[tuple[str, Optional[dict]]] filter_name: str,
components: list[tuple[str, Optional[dict[str, Union[str, int, float]]]]],
) -> FilterEnsemble: ) -> FilterEnsemble:
""" """
Create a filtering pipeline. Create a filtering pipeline.
""" """
filters = [] # create filters given its name in the registry, and add each as a pipeline step
for function, kwargs in components: return FilterEnsemble(
if kwargs is None: name=filter_name,
kwargs = {} filters=[
# create a filter given its name in the registry partial(get_filter(func), **(kwargs or {})) for func, kwargs in components
f = partial(get_filter(function), **kwargs) ],
# add the filter as a pipeline step )
filters.append(f)
return FilterEnsemble(name=filter_name, filters=filters)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment