Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
gaoqiong
lm-evaluation-harness
Commits
c74e2761
Commit
c74e2761
authored
Dec 06, 2023
by
lintangsutawika
Browse files
reformat
parent
cc572624
Changes
155
Show whitespace changes
Inline
Side-by-side
Showing
20 changed files
with
38 additions
and
35 deletions
+38
-35
lm_eval/tasks/arc/alternative_worlds/style_06/c.yaml
lm_eval/tasks/arc/alternative_worlds/style_06/c.yaml
+2
-2
lm_eval/tasks/arc/alternative_worlds/style_07/a.yaml
lm_eval/tasks/arc/alternative_worlds/style_07/a.yaml
+2
-2
lm_eval/tasks/arc/alternative_worlds/style_07/b.yaml
lm_eval/tasks/arc/alternative_worlds/style_07/b.yaml
+2
-2
lm_eval/tasks/arc/alternative_worlds/style_07/c.yaml
lm_eval/tasks/arc/alternative_worlds/style_07/c.yaml
+2
-2
lm_eval/tasks/arc/alternative_worlds/style_08/a.yaml
lm_eval/tasks/arc/alternative_worlds/style_08/a.yaml
+2
-2
lm_eval/tasks/arc/alternative_worlds/style_08/b.yaml
lm_eval/tasks/arc/alternative_worlds/style_08/b.yaml
+2
-2
lm_eval/tasks/arc/alternative_worlds/style_08/c.yaml
lm_eval/tasks/arc/alternative_worlds/style_08/c.yaml
+2
-2
lm_eval/tasks/arc/alternative_worlds/styles.py
lm_eval/tasks/arc/alternative_worlds/styles.py
+12
-9
lm_eval/tasks/arithmetic/alternative_worlds/_template_yaml
lm_eval/tasks/arithmetic/alternative_worlds/_template_yaml
+1
-1
lm_eval/tasks/arithmetic/alternative_worlds/arithmetic_alt.yaml
...l/tasks/arithmetic/alternative_worlds/arithmetic_alt.yaml
+1
-1
lm_eval/tasks/arithmetic/alternative_worlds/style_00/_template_00_yaml
.../arithmetic/alternative_worlds/style_00/_template_00_yaml
+1
-1
lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_1dc.yaml
...rithmetic/alternative_worlds/style_00/arithmetic_1dc.yaml
+1
-1
lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_2da.yaml
...rithmetic/alternative_worlds/style_00/arithmetic_2da.yaml
+1
-1
lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_2dm.yaml
...rithmetic/alternative_worlds/style_00/arithmetic_2dm.yaml
+1
-1
lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_2ds.yaml
...rithmetic/alternative_worlds/style_00/arithmetic_2ds.yaml
+1
-1
lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_3da.yaml
...rithmetic/alternative_worlds/style_00/arithmetic_3da.yaml
+1
-1
lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_3ds.yaml
...rithmetic/alternative_worlds/style_00/arithmetic_3ds.yaml
+1
-1
lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_4da.yaml
...rithmetic/alternative_worlds/style_00/arithmetic_4da.yaml
+1
-1
lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_4ds.yaml
...rithmetic/alternative_worlds/style_00/arithmetic_4ds.yaml
+1
-1
lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_5da.yaml
...rithmetic/alternative_worlds/style_00/arithmetic_5da.yaml
+1
-1
No files found.
lm_eval/tasks/arc/alternative_worlds/style_06/c.yaml
View file @
c74e2761
lm_eval/tasks/arc/alternative_worlds/style_07/a.yaml
View file @
c74e2761
lm_eval/tasks/arc/alternative_worlds/style_07/b.yaml
View file @
c74e2761
lm_eval/tasks/arc/alternative_worlds/style_07/c.yaml
View file @
c74e2761
lm_eval/tasks/arc/alternative_worlds/style_08/a.yaml
View file @
c74e2761
lm_eval/tasks/arc/alternative_worlds/style_08/b.yaml
View file @
c74e2761
lm_eval/tasks/arc/alternative_worlds/style_08/c.yaml
View file @
c74e2761
lm_eval/tasks/arc/alternative_worlds/styles.py
View file @
c74e2761
import
string
import
string
from
functools
import
partial
from
functools
import
partial
def
doc_to_text_base
(
alphabet
,
style
,
doc
):
def
doc_to_text_base
(
alphabet
,
style
,
doc
):
choices
=
doc
[
"choices"
][
"text"
]
choices
=
doc
[
"choices"
][
"text"
]
...
@@ -13,19 +14,21 @@ def doc_to_text_base(alphabet, style, doc):
...
@@ -13,19 +14,21 @@ def doc_to_text_base(alphabet, style, doc):
else
:
else
:
choice_string
=
"{} {}"
choice_string
=
"{} {}"
doc_to_text
=
"
\n\n
"
.
join
([
doc_to_text
=
"
\n\n
"
.
join
(
"Question: "
+
doc
[
"question"
]
+
"
\n
Answer:"
,
[
]
+
[
"Question: "
+
doc
[
"question"
]
+
"
\n
Answer:"
,
choice_string
.
format
(
i
,
j
)
for
i
,
j
in
zip
(
letter_list
,
choices
)
]
]
+
[
choice_string
.
format
(
i
,
j
)
for
i
,
j
in
zip
(
letter_list
,
choices
)]
)
)
return
doc_to_text
return
doc_to_text
# Full continuation
# Full continuation
def
choice_A
(
doc
):
def
choice_A
(
doc
):
return
doc
[
"choices"
][
"text"
]
return
doc
[
"choices"
][
"text"
]
# Letters only
# Letters only
def
choice_B
(
alphabet
,
style
,
doc
):
def
choice_B
(
alphabet
,
style
,
doc
):
...
@@ -34,10 +37,11 @@ def choice_B(alphabet, style, doc):
...
@@ -34,10 +37,11 @@ def choice_B(alphabet, style, doc):
letter_list
=
[
style
.
format
(
letter
)
for
letter
in
alphabet
[
0
:
num
]]
letter_list
=
[
style
.
format
(
letter
)
for
letter
in
alphabet
[
0
:
num
]]
if
"
\t
"
in
style
:
if
"
\t
"
in
style
:
letter_list
=
[
letter
.
replace
(
"
\t
"
,
""
)
for
letter
in
letter_list
]
letter_list
=
[
letter
.
replace
(
"
\t
"
,
""
)
for
letter
in
letter_list
]
return
letter_list
return
letter_list
# Letters + Full continuation
# Letters + Full continuation
def
choice_C
(
alphabet
,
style
,
doc
):
def
choice_C
(
alphabet
,
style
,
doc
):
...
@@ -46,9 +50,10 @@ def choice_C(alphabet, style, doc):
...
@@ -46,9 +50,10 @@ def choice_C(alphabet, style, doc):
letter_list
=
[
style
.
format
(
letter
)
for
letter
in
alphabet
[
0
:
num
]]
letter_list
=
[
style
.
format
(
letter
)
for
letter
in
alphabet
[
0
:
num
]]
if
"
\t
"
not
in
style
:
if
"
\t
"
not
in
style
:
letter_list
=
[
letter
+
" "
for
letter
in
letter_list
]
letter_list
=
[
letter
+
" "
for
letter
in
letter_list
]
return
[
letter
+
choice
for
letter
,
choice
in
zip
(
letter_list
,
choices
)]
return
[
letter
+
choice
for
letter
,
choice
in
zip
(
letter_list
,
choices
)]
template_01
=
partial
(
doc_to_text_base
,
string
.
ascii_lowercase
,
"({})"
)
template_01
=
partial
(
doc_to_text_base
,
string
.
ascii_lowercase
,
"({})"
)
choice_01a
=
choice_A
choice_01a
=
choice_A
...
@@ -82,5 +87,3 @@ template_08 = partial(doc_to_text_base, string.ascii_uppercase, "{}\t")
...
@@ -82,5 +87,3 @@ template_08 = partial(doc_to_text_base, string.ascii_uppercase, "{}\t")
choice_08a
=
choice_A
choice_08a
=
choice_A
choice_08b
=
partial
(
choice_B
,
string
.
ascii_uppercase
,
"{}
\t
"
)
choice_08b
=
partial
(
choice_B
,
string
.
ascii_uppercase
,
"{}
\t
"
)
choice_08c
=
partial
(
choice_C
,
string
.
ascii_uppercase
,
"{}
\t
"
)
choice_08c
=
partial
(
choice_C
,
string
.
ascii_uppercase
,
"{}
\t
"
)
lm_eval/tasks/arithmetic/alternative_worlds/_template_yaml
View file @
c74e2761
lm_eval/tasks/arithmetic/alternative_worlds/arithmetic_alt.yaml
View file @
c74e2761
lm_eval/tasks/arithmetic/alternative_worlds/style_00/_template_00_yaml
View file @
c74e2761
lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_1dc.yaml
View file @
c74e2761
lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_2da.yaml
View file @
c74e2761
lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_2dm.yaml
View file @
c74e2761
lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_2ds.yaml
View file @
c74e2761
lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_3da.yaml
View file @
c74e2761
lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_3ds.yaml
View file @
c74e2761
lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_4da.yaml
View file @
c74e2761
lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_4ds.yaml
View file @
c74e2761
lm_eval/tasks/arithmetic/alternative_worlds/style_00/arithmetic_5da.yaml
View file @
c74e2761
Prev
1
2
3
4
5
6
…
8
Next
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment