Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
gaoqiong
lm-evaluation-harness
Commits
d13b1f56
Commit
d13b1f56
authored
Jul 02, 2024
by
haileyschoelkopf
Browse files
fix merge conflicts?
parent
de4400c5
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
19 additions
and
21 deletions
+19
-21
lm_eval/api/task.py
lm_eval/api/task.py
+15
-17
lm_eval/evaluator_utils.py
lm_eval/evaluator_utils.py
+4
-4
No files found.
lm_eval/api/task.py
View file @
d13b1f56
...
@@ -22,7 +22,6 @@ from typing import (
...
@@ -22,7 +22,6 @@ from typing import (
import
datasets
import
datasets
import
numpy
as
np
import
numpy
as
np
import
shortuuid
from
tqdm
import
tqdm
from
tqdm
import
tqdm
from
lm_eval
import
utils
from
lm_eval
import
utils
...
@@ -137,9 +136,8 @@ class ConfigurableGroup(abc.ABC):
...
@@ -137,9 +136,8 @@ class ConfigurableGroup(abc.ABC):
self
,
self
,
config
:
Optional
[
dict
]
=
None
,
config
:
Optional
[
dict
]
=
None
,
)
->
None
:
)
->
None
:
# Create a unique identifier ID
self
.
_config
=
GroupConfig
(
**
config
)
self
.
_config
=
GroupConfig
(
**
config
)
self
.
_task_id
=
self
.
_config
.
group
#
self._task_id = self._config.group
@
property
@
property
def
group
(
self
):
def
group
(
self
):
...
@@ -157,13 +155,13 @@ class ConfigurableGroup(abc.ABC):
...
@@ -157,13 +155,13 @@ class ConfigurableGroup(abc.ABC):
def
config
(
self
):
def
config
(
self
):
return
self
.
_config
.
to_dict
()
return
self
.
_config
.
to_dict
()
@
property
#
@property
def
task_id
(
self
)
->
Any
:
#
def task_id(self) -> Any:
return
self
.
_task_id
#
return self._task_id
@
task_id
.
setter
#
@task_id.setter
def
task_id
(
self
,
value
):
#
def task_id(self, value):
self
.
_task_id
=
value
#
self._task_id = value
@
property
@
property
def
group_name
(
self
)
->
Any
:
def
group_name
(
self
)
->
Any
:
...
@@ -363,7 +361,7 @@ class Task(abc.ABC):
...
@@ -363,7 +361,7 @@ class Task(abc.ABC):
self
.
_instances
:
Optional
[
List
[
Instance
]]
=
None
self
.
_instances
:
Optional
[
List
[
Instance
]]
=
None
# Create a unique identifier ID
# Create a unique identifier ID
self
.
_task_id
=
shortuuid
.
uuid
()[:
8
]
#
self._task_id = shortuuid.uuid()[:8]
self
.
_config
:
TaskConfig
=
TaskConfig
({
**
config
})
if
config
else
TaskConfig
()
self
.
_config
:
TaskConfig
=
TaskConfig
({
**
config
})
if
config
else
TaskConfig
()
self
.
_filters
=
[
build_filter_ensemble
(
"none"
,
[[
"take_first"
,
None
]])]
self
.
_filters
=
[
build_filter_ensemble
(
"none"
,
[[
"take_first"
,
None
]])]
...
@@ -820,13 +818,13 @@ class Task(abc.ABC):
...
@@ -820,13 +818,13 @@ class Task(abc.ABC):
)
)
return
doc_iterator
return
doc_iterator
@
property
#
@property
def
task_id
(
self
)
->
Any
:
#
def task_id(self) -> Any:
return
self
.
_task_id
#
return self._task_id
@
task_id
.
setter
#
@task_id.setter
def
task_id
(
self
,
value
):
#
def task_id(self, value):
self
.
_task_id
=
value
#
self._task_id = value
class
ConfigurableTask
(
Task
):
class
ConfigurableTask
(
Task
):
...
@@ -842,7 +840,7 @@ class ConfigurableTask(Task):
...
@@ -842,7 +840,7 @@ class ConfigurableTask(Task):
config
:
Optional
[
dict
]
=
None
,
config
:
Optional
[
dict
]
=
None
,
)
->
None
:
# TODO no super() call here
)
->
None
:
# TODO no super() call here
# Create a unique identifier ID
# Create a unique identifier ID
self
.
_task_id
=
shortuuid
.
uuid
()[:
8
]
#
self._task_id = shortuuid.uuid()[:8]
# Get pre-configured attributes
# Get pre-configured attributes
self
.
_config
=
self
.
CONFIG
self
.
_config
=
self
.
CONFIG
...
...
lm_eval/evaluator_utils.py
View file @
d13b1f56
...
@@ -40,7 +40,7 @@ class TaskOutput:
...
@@ -40,7 +40,7 @@ class TaskOutput:
self
,
self
,
task
=
None
,
task
=
None
,
task_name
=
None
,
task_name
=
None
,
task_id
=
None
,
#
task_id=None,
task_config
=
None
,
task_config
=
None
,
version
=
None
,
version
=
None
,
group_name
=
None
,
group_name
=
None
,
...
@@ -52,7 +52,7 @@ class TaskOutput:
...
@@ -52,7 +52,7 @@ class TaskOutput:
self
.
task
=
task
self
.
task
=
task
self
.
task_config
=
task_config
self
.
task_config
=
task_config
self
.
task_name
=
task_name
self
.
task_name
=
task_name
self
.
task_id
=
task_id
#
self.task_id = task_id
self
.
group_name
=
group_name
self
.
group_name
=
group_name
self
.
version
=
version
self
.
version
=
version
self
.
n_shot
=
n_shot
self
.
n_shot
=
n_shot
...
@@ -78,7 +78,7 @@ class TaskOutput:
...
@@ -78,7 +78,7 @@ class TaskOutput:
task
=
task
,
task_name
=
task_name
,
is_group
=
is_group
,
group_name
=
group_name
task
=
task
,
task_name
=
task_name
,
is_group
=
is_group
,
group_name
=
group_name
)
)
version
=
task
.
VERSION
version
=
task
.
VERSION
task_id
=
task
.
task_id
#
task_id = task.task_id
task_config
=
dict
(
task
.
dump_config
())
task_config
=
dict
(
task
.
dump_config
())
if
(
n_shot
:
=
task_config
.
get
(
"num_fewshot"
))
==
0
:
if
(
n_shot
:
=
task_config
.
get
(
"num_fewshot"
))
==
0
:
n_shot
=
task_config
.
get
(
"metadata"
,
{}).
get
(
"num_fewshot"
,
0
)
n_shot
=
task_config
.
get
(
"metadata"
,
{}).
get
(
"num_fewshot"
,
0
)
...
@@ -87,7 +87,7 @@ class TaskOutput:
...
@@ -87,7 +87,7 @@ class TaskOutput:
return
cls
(
return
cls
(
task
=
task
,
task
=
task
,
task_name
=
task_name
,
task_name
=
task_name
,
task_id
=
task_id
,
#
task_id=task_id,
task_config
=
task_config
,
task_config
=
task_config
,
group_name
=
group_name
,
group_name
=
group_name
,
version
=
version
,
version
=
version
,
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment