Add tests to make sure task definitions don't change without the version also changing

main.py

@@ -81,6 +81,7 @@ def main():
     # todo: make latex table look good
     # print(latex_writer.dumps())
 
+    print(f"{args.model} ({args.model_args}), limit: {args.limit}, provide_description: {args.provide_description}, num_fewshot: {args.num_fewshot}, batch_size: {args.batch_size}")
     print(md_writer.dumps())
 
 if __name__ == "__main__":

tests/test_version_stable.py

+import lm_eval.tasks as tasks
+import lm_eval.models as models
+import lm_eval.evaluator as evaluator
+import random
+import pytest
+import os
+import json
+
+
+os.makedirs("tests/testdata", exist_ok=True)
+
+
+def assert_target(name, ob):
+    fname = f"tests/testdata/{name}.json"
+    if os.path.exists(fname):
+        with open(fname) as fh:
+            assert json.load(fh) == json.loads(json.dumps(ob))
+    else:
+        with open(fname, 'w') as fh:
+            json.dump(ob, fh)
+
+
+# make sure eval results for a task version are stable
+
+@pytest.mark.parametrize("taskname,Task", tasks.TASK_REGISTRY.items())
+def test_versions_stable(taskname, Task):
+    task_dict = tasks.get_task_dict([taskname])
+    lm = models.get_model('dummy')()
+
+    def ll_fn(reqs):
+        for ctx, cont in reqs:
+            if len(ctx) == 0: continue
+            # space convention
+            assert ctx[-1] != ' '
+            assert cont[0] == ' ' or ctx[-1] == '\n'
+        
+        assert_target(f"{taskname}-v{Task.VERSION}-loglikelihood", reqs)
+        res = []
+        
+        random.seed(42)
+        for _ in reqs:
+            res.append((-random.random(), False))
+
+        return res
+
+    def ll_perp_fn(reqs):
+        for string, in reqs:
+            assert isinstance(string, str)
+
+        assert_target(f"{taskname}-v{Task.VERSION}-loglikelihood_rolling", reqs)
+        res = []
+
+        random.seed(42)
+        for _ in reqs:
+            res.append(-random.random())
+
+        return res
+    
+    def greedy_until(reqs):
+        res = []
+        assert_target(f"{taskname}-v{Task.VERSION}-greedy_until", reqs)
+        
+        for ctx, _ in reqs:
+            res.append("lol")
+            assert ctx.strip() != ''
+
+        return res
+
+    lm.loglikelihood = ll_fn
+    lm.loglikelihood_rolling = ll_perp_fn
+    lm.greedy_until = greedy_until
+    res = evaluator.evaluate(lm, task_dict, False, 0, 10, bootstrap_iters=10)
+    assert_target(f"{taskname}-v{Task.VERSION}-res", res)

tests/testdata/anagrams1-v0-greedy_until.json

+[["\ntkhnis =", ["\n"]], ["\ndviel =", ["\n"]], ["\ncooalinl =", ["\n"]], ["\npupaa =", ["\n"]], ["\ncbmouila =", ["\n"]], ["\nimcfnaroits =", ["\n"]], ["\niagems =", ["\n"]], ["\nwithe =", ["\n"]], ["\nmoannta =", ["\n"]], ["\ngtiefd =", ["\n"]]]
\ No newline at end of file

tests/testdata/anagrams1-v0-res.json

+{"results": {"anagrams1": {"acc": 0.0, "acc_stderr": 0.0}}, "versions": {"anagrams1": 0}}
\ No newline at end of file

tests/testdata/anagrams2-v0-greedy_until.json

+[["\ncannon =", ["\n"]], ["\nknhigts =", ["\n"]], ["\nevloving =", ["\n"]], ["\nmoiton =", ["\n"]], ["\nautiiarhztoon =", ["\n"]], ["\nadtaped =", ["\n"]], ["\nantcsery =", ["\n"]], ["\nwondrfeul =", ["\n"]], ["\nsiimliarty =", ["\n"]], ["\ncobwoy =", ["\n"]]]
\ No newline at end of file

tests/testdata/anagrams2-v0-res.json

+{"results": {"anagrams2": {"acc": 0.0, "acc_stderr": 0.0}}, "versions": {"anagrams2": 0}}
\ No newline at end of file

tests/testdata/anli_r1-v0-loglikelihood.json

+[["Naser Mestarihi EP is the eponymous debut EP of Doha based hard rock guitarist Naser Mestarihi. The release of the album marked the first official release of a rock album out of Qatar. All the lyrics and music on the album were written by Mestarihi, who also plays all the instruments with the exception of the drums.\nQuestion: Mestarihi plays all the instruments on Naser Mestarihi True, False, or Neither?\nAnswer:", " True"], ["Naser Mestarihi EP is the eponymous debut EP of Doha based hard rock guitarist Naser Mestarihi. The release of the album marked the first official release of a rock album out of Qatar. All the lyrics and music on the album were written by Mestarihi, who also plays all the instruments with the exception of the drums.\nQuestion: Mestarihi plays all the instruments on Naser Mestarihi True, False, or Neither?\nAnswer:", " Neither"], ["Naser Mestarihi EP is the eponymous debut EP of Doha based hard rock guitarist Naser Mestarihi. The release of the album marked the first official release of a rock album out of Qatar. All the lyrics and music on the album were written by Mestarihi, who also plays all the instruments with the exception of the drums.\nQuestion: Mestarihi plays all the instruments on Naser Mestarihi True, False, or Neither?\nAnswer:", " False"], ["Fulletby is a village and a civil parish in the East Lindsey district of Lincolnshire, England. It is situated in the Lincolnshire Wolds, and 3 mi north-east from Horncastle, 9 mi south from Louth, and 8 mi north-west from Spilsby. The parish covers approximately 1950 acre . At the time of the 2011 census the population remained less than 100 and is included in the civil parish of Low Toynton.\nQuestion: Over one hundred people live in the tiny village of Fulletby True, False, or Neither?\nAnswer:", " True"], ["Fulletby is a village and a civil parish in the East Lindsey district of Lincolnshire, England. It is situated in the Lincolnshire Wolds, and 3 mi north-east from Horncastle, 9 mi south from Louth, and 8 mi north-west from Spilsby. The parish covers approximately 1950 acre . At the time of the 2011 census the population remained less than 100 and is included in the civil parish of Low Toynton.\nQuestion: Over one hundred people live in the tiny village of Fulletby True, False, or Neither?\nAnswer:", " Neither"], ["Fulletby is a village and a civil parish in the East Lindsey district of Lincolnshire, England. It is situated in the Lincolnshire Wolds, and 3 mi north-east from Horncastle, 9 mi south from Louth, and 8 mi north-west from Spilsby. The parish covers approximately 1950 acre . At the time of the 2011 census the population remained less than 100 and is included in the civil parish of Low Toynton.\nQuestion: Over one hundred people live in the tiny village of Fulletby True, False, or Neither?\nAnswer:", " False"], ["Gordon Robert \"Gordie\" Tapp, CM, O.Ont (June 4, 1922 \u2013 December 18, 2016), born in London, Ontario, was a Canadian entertainer, best known as a radio and television presenter, comedian and a CBS broadcaster. He was introduced to U.S. President Gerald Ford as the world's funniest storyteller.\nQuestion: Gordon Tapp was well known in Canada. True, False, or Neither?\nAnswer:", " True"], ["Gordon Robert \"Gordie\" Tapp, CM, O.Ont (June 4, 1922 \u2013 December 18, 2016), born in London, Ontario, was a Canadian entertainer, best known as a radio and television presenter, comedian and a CBS broadcaster. He was introduced to U.S. President Gerald Ford as the world's funniest storyteller.\nQuestion: Gordon Tapp was well known in Canada. True, False, or Neither?\nAnswer:", " Neither"], ["Gordon Robert \"Gordie\" Tapp, CM, O.Ont (June 4, 1922 \u2013 December 18, 2016), born in London, Ontario, was a Canadian entertainer, best known as a radio and television presenter, comedian and a CBS broadcaster. He was introduced to U.S. President Gerald Ford as the world's funniest storyteller.\nQuestion: Gordon Tapp was well known in Canada. True, False, or Neither?\nAnswer:", " False"], ["The Ten Commandments of Dog Ownership, also known as The Ten Commandments From a Pet's Point of View, or simply Ten Commandments For Dog Owners is a set of pet ownership rules. It was created in 1993 by Stan Rawlinson, and is written from the point of view of a dog.\nQuestion: The Ten Commandments of Dog Ownership were written by a dog. True, False, or Neither?\nAnswer:", " True"], ["The Ten Commandments of Dog Ownership, also known as The Ten Commandments From a Pet's Point of View, or simply Ten Commandments For Dog Owners is a set of pet ownership rules. It was created in 1993 by Stan Rawlinson, and is written from the point of view of a dog.\nQuestion: The Ten Commandments of Dog Ownership were written by a dog. True, False, or Neither?\nAnswer:", " Neither"], ["The Ten Commandments of Dog Ownership, also known as The Ten Commandments From a Pet's Point of View, or simply Ten Commandments For Dog Owners is a set of pet ownership rules. It was created in 1993 by Stan Rawlinson, and is written from the point of view of a dog.\nQuestion: The Ten Commandments of Dog Ownership were written by a dog. True, False, or Neither?\nAnswer:", " False"], ["Channels, released in 2008, is love story sometimes described as delving into thoughts of existentialism. The film was written, directed, produced and acted in by Nat Christian. The film stars Kim Oja, Nat Christian, Ed Asner, John Kassir, Joan Van Ark and Taylor Negron. Ann Marcus executive produced. The production company was Olliewood Films, Inc.\nQuestion: Channels was released before 2007. True, False, or Neither?\nAnswer:", " True"], ["Channels, released in 2008, is love story sometimes described as delving into thoughts of existentialism. The film was written, directed, produced and acted in by Nat Christian. The film stars Kim Oja, Nat Christian, Ed Asner, John Kassir, Joan Van Ark and Taylor Negron. Ann Marcus executive produced. The production company was Olliewood Films, Inc.\nQuestion: Channels was released before 2007. True, False, or Neither?\nAnswer:", " Neither"], ["Channels, released in 2008, is love story sometimes described as delving into thoughts of existentialism. The film was written, directed, produced and acted in by Nat Christian. The film stars Kim Oja, Nat Christian, Ed Asner, John Kassir, Joan Van Ark and Taylor Negron. Ann Marcus executive produced. The production company was Olliewood Films, Inc.\nQuestion: Channels was released before 2007. True, False, or Neither?\nAnswer:", " False"], ["Chris Shiflett & the Dead Peasants is the debut self-titled album from Chris Shiflett, lead guitarist of Foo Fighters. The Dead Peasants are his second side project, following Jackson United, and the backing band is composed of different musicians than that of Jackson United.\nQuestion: Chris Shiflett is the lead vocalist on his new side project True, False, or Neither?\nAnswer:", " True"], ["Chris Shiflett & the Dead Peasants is the debut self-titled album from Chris Shiflett, lead guitarist of Foo Fighters. The Dead Peasants are his second side project, following Jackson United, and the backing band is composed of different musicians than that of Jackson United.\nQuestion: Chris Shiflett is the lead vocalist on his new side project True, False, or Neither?\nAnswer:", " Neither"], ["Chris Shiflett & the Dead Peasants is the debut self-titled album from Chris Shiflett, lead guitarist of Foo Fighters. The Dead Peasants are his second side project, following Jackson United, and the backing band is composed of different musicians than that of Jackson United.\nQuestion: Chris Shiflett is the lead vocalist on his new side project True, False, or Neither?\nAnswer:", " False"], ["Phoebe Snow was a fictional character created by Earnest Elmo Calkins to promote the Delaware, Lackawanna and Western Railroad. The advertising campaign was one of the first to present a fictional character based on a live model amid impressionistic techniques.\nQuestion: Phoebe Snow's character was created by one person. True, False, or Neither?\nAnswer:", " True"], ["Phoebe Snow was a fictional character created by Earnest Elmo Calkins to promote the Delaware, Lackawanna and Western Railroad. The advertising campaign was one of the first to present a fictional character based on a live model amid impressionistic techniques.\nQuestion: Phoebe Snow's character was created by one person. True, False, or Neither?\nAnswer:", " Neither"], ["Phoebe Snow was a fictional character created by Earnest Elmo Calkins to promote the Delaware, Lackawanna and Western Railroad. The advertising campaign was one of the first to present a fictional character based on a live model amid impressionistic techniques.\nQuestion: Phoebe Snow's character was created by one person. True, False, or Neither?\nAnswer:", " False"], ["PDQ Chocolate was a popular drink mix in the 1960s and 1970s. It was manufactured by Ovaltine. It has been sold as \"PDQ Chocolate Flavor Beads\" and \"PDQ Choco Chips.\" \"Egg Nog Flavored PDQ\" and \"Strawberry PDQ\" were also available. These flavored beads and chips were used to mix with milk or to sprinkle over ice cream.\nQuestion: Ovaltine didn't manufacture any other drinks in this time period. True, False, or Neither?\nAnswer:", " True"], ["PDQ Chocolate was a popular drink mix in the 1960s and 1970s. It was manufactured by Ovaltine. It has been sold as \"PDQ Chocolate Flavor Beads\" and \"PDQ Choco Chips.\" \"Egg Nog Flavored PDQ\" and \"Strawberry PDQ\" were also available. These flavored beads and chips were used to mix with milk or to sprinkle over ice cream.\nQuestion: Ovaltine didn't manufacture any other drinks in this time period. True, False, or Neither?\nAnswer:", " Neither"], ["PDQ Chocolate was a popular drink mix in the 1960s and 1970s. It was manufactured by Ovaltine. It has been sold as \"PDQ Chocolate Flavor Beads\" and \"PDQ Choco Chips.\" \"Egg Nog Flavored PDQ\" and \"Strawberry PDQ\" were also available. These flavored beads and chips were used to mix with milk or to sprinkle over ice cream.\nQuestion: Ovaltine didn't manufacture any other drinks in this time period. True, False, or Neither?\nAnswer:", " False"], ["Collins Hickey Johnston (August 29, 1859 \u2013 December 29, 1936) was an American football player, medical doctor, surgeon, and civic leader in Grand Rapids, Michigan. He was a graduate of the University of Michigan where he played college football for the 1879 Michigan Wolverines football team, the first football team to represent the University of Michigan.\nQuestion: Collins Hickey Johnston would only listen to the Mozart when completing heart surgery, and rock and roll when completing brain surgery True, False, or Neither?\nAnswer:", " True"], ["Collins Hickey Johnston (August 29, 1859 \u2013 December 29, 1936) was an American football player, medical doctor, surgeon, and civic leader in Grand Rapids, Michigan. He was a graduate of the University of Michigan where he played college football for the 1879 Michigan Wolverines football team, the first football team to represent the University of Michigan.\nQuestion: Collins Hickey Johnston would only listen to the Mozart when completing heart surgery, and rock and roll when completing brain surgery True, False, or Neither?\nAnswer:", " Neither"], ["Collins Hickey Johnston (August 29, 1859 \u2013 December 29, 1936) was an American football player, medical doctor, surgeon, and civic leader in Grand Rapids, Michigan. He was a graduate of the University of Michigan where he played college football for the 1879 Michigan Wolverines football team, the first football team to represent the University of Michigan.\nQuestion: Collins Hickey Johnston would only listen to the Mozart when completing heart surgery, and rock and roll when completing brain surgery True, False, or Neither?\nAnswer:", " False"], ["Cody Martin Linley (born November 20, 1989) is an American actor and singer. He played a recurring role as Jake Ryan in the television series \"Hannah Montana\", and was a contestant on the seventh season of \"Dancing With The Stars\", in which he was partnered with Julianne Hough and finished fourth.\nQuestion: Cody Martin Linley did not win Dancing with the Stars True, False, or Neither?\nAnswer:", " True"], ["Cody Martin Linley (born November 20, 1989) is an American actor and singer. He played a recurring role as Jake Ryan in the television series \"Hannah Montana\", and was a contestant on the seventh season of \"Dancing With The Stars\", in which he was partnered with Julianne Hough and finished fourth.\nQuestion: Cody Martin Linley did not win Dancing with the Stars True, False, or Neither?\nAnswer:", " Neither"], ["Cody Martin Linley (born November 20, 1989) is an American actor and singer. He played a recurring role as Jake Ryan in the television series \"Hannah Montana\", and was a contestant on the seventh season of \"Dancing With The Stars\", in which he was partnered with Julianne Hough and finished fourth.\nQuestion: Cody Martin Linley did not win Dancing with the Stars True, False, or Neither?\nAnswer:", " False"]]
\ No newline at end of file

tests/testdata/anli_r1-v0-res.json

+{"results": {"anli_r1": {"acc": 0.2, "acc_stderr": 0.13333333333333333}}, "versions": {"anli_r1": 0}}
\ No newline at end of file

tests/testdata/anli_r2-v0-loglikelihood.json

+[["The 1992 Nutri-Metics Bendon Classic was a women's tennis tournament played on outdoor hard courts at the ASB Tennis Centre in Auckland in New Zealand that was part of Tier V of the 1992 WTA Tour. It was the seventh edition of the tournament and was held from 27 January February through 2 February 1992. Unseeded Robin White won the singles title.\nQuestion: The 1992 Nutri-Metics Bendon Classic had no loser. True, False, or Neither?\nAnswer:", " True"], ["The 1992 Nutri-Metics Bendon Classic was a women's tennis tournament played on outdoor hard courts at the ASB Tennis Centre in Auckland in New Zealand that was part of Tier V of the 1992 WTA Tour. It was the seventh edition of the tournament and was held from 27 January February through 2 February 1992. Unseeded Robin White won the singles title.\nQuestion: The 1992 Nutri-Metics Bendon Classic had no loser. True, False, or Neither?\nAnswer:", " Neither"], ["The 1992 Nutri-Metics Bendon Classic was a women's tennis tournament played on outdoor hard courts at the ASB Tennis Centre in Auckland in New Zealand that was part of Tier V of the 1992 WTA Tour. It was the seventh edition of the tournament and was held from 27 January February through 2 February 1992. Unseeded Robin White won the singles title.\nQuestion: The 1992 Nutri-Metics Bendon Classic had no loser. True, False, or Neither?\nAnswer:", " False"], ["In guitar music, especially electric guitar, a power chord    (also fifth chord) is a colloquial name for a chord that consists of the root note and the fifth. Power chords are commonly played on amplified guitars, especially on electric guitar with distortion. Power chords are a key element of many styles of rock and especially in heavy metal, and punk rock.\nQuestion: Power chords are played in more than amplified guitars  True, False, or Neither?\nAnswer:", " True"], ["In guitar music, especially electric guitar, a power chord    (also fifth chord) is a colloquial name for a chord that consists of the root note and the fifth. Power chords are commonly played on amplified guitars, especially on electric guitar with distortion. Power chords are a key element of many styles of rock and especially in heavy metal, and punk rock.\nQuestion: Power chords are played in more than amplified guitars  True, False, or Neither?\nAnswer:", " Neither"], ["In guitar music, especially electric guitar, a power chord    (also fifth chord) is a colloquial name for a chord that consists of the root note and the fifth. Power chords are commonly played on amplified guitars, especially on electric guitar with distortion. Power chords are a key element of many styles of rock and especially in heavy metal, and punk rock.\nQuestion: Power chords are played in more than amplified guitars  True, False, or Neither?\nAnswer:", " False"], ["Lakeshore Technical College (or LTC) is a technical and community college located in Cleveland, Wisconsin, which is centrally located between the lakeshore cities of Sheboygan and Manitowoc. It is a member of the 16 schools in the Wisconsin Technical College System. The college was originally named Lakeshore Technical Institute (LTI).\nQuestion: It is the only technical college in wisconsin True, False, or Neither?\nAnswer:", " True"], ["Lakeshore Technical College (or LTC) is a technical and community college located in Cleveland, Wisconsin, which is centrally located between the lakeshore cities of Sheboygan and Manitowoc. It is a member of the 16 schools in the Wisconsin Technical College System. The college was originally named Lakeshore Technical Institute (LTI).\nQuestion: It is the only technical college in wisconsin True, False, or Neither?\nAnswer:", " Neither"], ["Lakeshore Technical College (or LTC) is a technical and community college located in Cleveland, Wisconsin, which is centrally located between the lakeshore cities of Sheboygan and Manitowoc. It is a member of the 16 schools in the Wisconsin Technical College System. The college was originally named Lakeshore Technical Institute (LTI).\nQuestion: It is the only technical college in wisconsin True, False, or Neither?\nAnswer:", " False"], ["The Sandlot is a 1993 American coming-of-age baseball film co-written and directed by David M. Evans, which tells the story of a group of young baseball players during the summer of 1962. It stars Tom Guiry, Mike Vitar, Karen Allen, Denis Leary and James Earl Jones. The filming locations were in Glendale, Midvale, Salt Lake City, and Ogden, Utah.\nQuestion: The Sandlot takes place in Odgen, Utah.  True, False, or Neither?\nAnswer:", " True"], ["The Sandlot is a 1993 American coming-of-age baseball film co-written and directed by David M. Evans, which tells the story of a group of young baseball players during the summer of 1962. It stars Tom Guiry, Mike Vitar, Karen Allen, Denis Leary and James Earl Jones. The filming locations were in Glendale, Midvale, Salt Lake City, and Ogden, Utah.\nQuestion: The Sandlot takes place in Odgen, Utah.  True, False, or Neither?\nAnswer:", " Neither"], ["The Sandlot is a 1993 American coming-of-age baseball film co-written and directed by David M. Evans, which tells the story of a group of young baseball players during the summer of 1962. It stars Tom Guiry, Mike Vitar, Karen Allen, Denis Leary and James Earl Jones. The filming locations were in Glendale, Midvale, Salt Lake City, and Ogden, Utah.\nQuestion: The Sandlot takes place in Odgen, Utah.  True, False, or Neither?\nAnswer:", " False"], ["Taki's Magazine, called \"Takimag\" for short, is an online magazine of politics and culture published by the Greek paleoconservative journalist and socialite Taki Theodoracopulos and edited by his daughter Mandolyna Theodoracopulos. Initially called Taki's Top Drawer, the site was redesigned and relaunched under its current title in March 2008 with a subsequent redesign in 2010.\nQuestion: Taki's Magazine is the long version of Takimag True, False, or Neither?\nAnswer:", " True"], ["Taki's Magazine, called \"Takimag\" for short, is an online magazine of politics and culture published by the Greek paleoconservative journalist and socialite Taki Theodoracopulos and edited by his daughter Mandolyna Theodoracopulos. Initially called Taki's Top Drawer, the site was redesigned and relaunched under its current title in March 2008 with a subsequent redesign in 2010.\nQuestion: Taki's Magazine is the long version of Takimag True, False, or Neither?\nAnswer:", " Neither"], ["Taki's Magazine, called \"Takimag\" for short, is an online magazine of politics and culture published by the Greek paleoconservative journalist and socialite Taki Theodoracopulos and edited by his daughter Mandolyna Theodoracopulos. Initially called Taki's Top Drawer, the site was redesigned and relaunched under its current title in March 2008 with a subsequent redesign in 2010.\nQuestion: Taki's Magazine is the long version of Takimag True, False, or Neither?\nAnswer:", " False"], ["\"Hang On\" is a song by the band Weezer. The song impacted radio on February 15, 2011. It is the seventh track and second single from their eighth studio album, \"Hurley\". The album version of \"Hang On\" is co-written by Rick Nowels and features Canadian actor Michael Cera on backing vocals and pseudo-mandolin. The single version features no mandolin, and contains a harder sound.\nQuestion: If you listen carefully you can hear Michael Cera in on Weezers song True, False, or Neither?\nAnswer:", " True"], ["\"Hang On\" is a song by the band Weezer. The song impacted radio on February 15, 2011. It is the seventh track and second single from their eighth studio album, \"Hurley\". The album version of \"Hang On\" is co-written by Rick Nowels and features Canadian actor Michael Cera on backing vocals and pseudo-mandolin. The single version features no mandolin, and contains a harder sound.\nQuestion: If you listen carefully you can hear Michael Cera in on Weezers song True, False, or Neither?\nAnswer:", " Neither"], ["\"Hang On\" is a song by the band Weezer. The song impacted radio on February 15, 2011. It is the seventh track and second single from their eighth studio album, \"Hurley\". The album version of \"Hang On\" is co-written by Rick Nowels and features Canadian actor Michael Cera on backing vocals and pseudo-mandolin. The single version features no mandolin, and contains a harder sound.\nQuestion: If you listen carefully you can hear Michael Cera in on Weezers song True, False, or Neither?\nAnswer:", " False"], ["Destiny (Arabic: \u0627\u0644\u0645\u0635\u064a\u0631\u200e \u200e , translit.\u00a0Al-massir) is a 1997 French-Egyptian historical drama film directed by Youssef Chahine. It was screened out of competition at the 1997 Cannes Film Festival. The film was selected as the Egyptian entry for the Best Foreign Language Film at the 70th Academy Awards, but was not accepted as a nominee.\nQuestion: Despite being a French-Egyptian film, Destiny was filmed in neither France or Egypt. True, False, or Neither?\nAnswer:", " True"], ["Destiny (Arabic: \u0627\u0644\u0645\u0635\u064a\u0631\u200e \u200e , translit.\u00a0Al-massir) is a 1997 French-Egyptian historical drama film directed by Youssef Chahine. It was screened out of competition at the 1997 Cannes Film Festival. The film was selected as the Egyptian entry for the Best Foreign Language Film at the 70th Academy Awards, but was not accepted as a nominee.\nQuestion: Despite being a French-Egyptian film, Destiny was filmed in neither France or Egypt. True, False, or Neither?\nAnswer:", " Neither"], ["Destiny (Arabic: \u0627\u0644\u0645\u0635\u064a\u0631\u200e \u200e , translit.\u00a0Al-massir) is a 1997 French-Egyptian historical drama film directed by Youssef Chahine. It was screened out of competition at the 1997 Cannes Film Festival. The film was selected as the Egyptian entry for the Best Foreign Language Film at the 70th Academy Awards, but was not accepted as a nominee.\nQuestion: Despite being a French-Egyptian film, Destiny was filmed in neither France or Egypt. True, False, or Neither?\nAnswer:", " False"], ["Aleksandr Danilovich Aleksandrov (Russian: \u0410\u043b\u0435\u043a\u0441\u0430\u0301\u043d\u0434\u0440 \u0414\u0430\u043d\u0438\u0301\u043b\u043e\u0432\u0438\u0447 \u0410\u043b\u0435\u043a\u0441\u0430\u0301\u043d\u0434\u0440\u043e\u0432 , alternative transliterations: \"Alexandr\" or \"Alexander\" (first name), and \"Alexandrov\" (last name)) (August 4, 1912 \u2013 July 27, 1999), was a Soviet/Russian mathematician, physicist, philosopher and mountaineer.\nQuestion: Aleksandr was an intelligent person, and also adventurous.  True, False, or Neither?\nAnswer:", " True"], ["Aleksandr Danilovich Aleksandrov (Russian: \u0410\u043b\u0435\u043a\u0441\u0430\u0301\u043d\u0434\u0440 \u0414\u0430\u043d\u0438\u0301\u043b\u043e\u0432\u0438\u0447 \u0410\u043b\u0435\u043a\u0441\u0430\u0301\u043d\u0434\u0440\u043e\u0432 , alternative transliterations: \"Alexandr\" or \"Alexander\" (first name), and \"Alexandrov\" (last name)) (August 4, 1912 \u2013 July 27, 1999), was a Soviet/Russian mathematician, physicist, philosopher and mountaineer.\nQuestion: Aleksandr was an intelligent person, and also adventurous.  True, False, or Neither?\nAnswer:", " Neither"], ["Aleksandr Danilovich Aleksandrov (Russian: \u0410\u043b\u0435\u043a\u0441\u0430\u0301\u043d\u0434\u0440 \u0414\u0430\u043d\u0438\u0301\u043b\u043e\u0432\u0438\u0447 \u0410\u043b\u0435\u043a\u0441\u0430\u0301\u043d\u0434\u0440\u043e\u0432 , alternative transliterations: \"Alexandr\" or \"Alexander\" (first name), and \"Alexandrov\" (last name)) (August 4, 1912 \u2013 July 27, 1999), was a Soviet/Russian mathematician, physicist, philosopher and mountaineer.\nQuestion: Aleksandr was an intelligent person, and also adventurous.  True, False, or Neither?\nAnswer:", " False"], ["I Love Hong Kong is a 2011 Hong Kong comedy film produced and directed by Eric Tsang. Film stars Tsang, Tony Leung Ka-fai, Sandra Ng and a star-studded cast of Hong Kong stars. It was released in Chinese New Year Day. The sequel movies are I Love Hong Kong 2012 and I Love Hong Kong 2013.\nQuestion: Eric Tsang's I Love Hong Kong sequels were released on Chinese New Year Day. True, False, or Neither?\nAnswer:", " True"], ["I Love Hong Kong is a 2011 Hong Kong comedy film produced and directed by Eric Tsang. Film stars Tsang, Tony Leung Ka-fai, Sandra Ng and a star-studded cast of Hong Kong stars. It was released in Chinese New Year Day. The sequel movies are I Love Hong Kong 2012 and I Love Hong Kong 2013.\nQuestion: Eric Tsang's I Love Hong Kong sequels were released on Chinese New Year Day. True, False, or Neither?\nAnswer:", " Neither"], ["I Love Hong Kong is a 2011 Hong Kong comedy film produced and directed by Eric Tsang. Film stars Tsang, Tony Leung Ka-fai, Sandra Ng and a star-studded cast of Hong Kong stars. It was released in Chinese New Year Day. The sequel movies are I Love Hong Kong 2012 and I Love Hong Kong 2013.\nQuestion: Eric Tsang's I Love Hong Kong sequels were released on Chinese New Year Day. True, False, or Neither?\nAnswer:", " False"], ["Multnomah University (MU) is a non-denominational Christian university in Portland, Oregon, United States. Multnomah consists of a college, graduate school, seminary and Degree Completion Program, and the university offers bachelor's, master's and doctorate degrees in a wide range of fields.\nQuestion: Multnomah University is located on the West coast of the U.S. True, False, or Neither?\nAnswer:", " True"], ["Multnomah University (MU) is a non-denominational Christian university in Portland, Oregon, United States. Multnomah consists of a college, graduate school, seminary and Degree Completion Program, and the university offers bachelor's, master's and doctorate degrees in a wide range of fields.\nQuestion: Multnomah University is located on the West coast of the U.S. True, False, or Neither?\nAnswer:", " Neither"], ["Multnomah University (MU) is a non-denominational Christian university in Portland, Oregon, United States. Multnomah consists of a college, graduate school, seminary and Degree Completion Program, and the university offers bachelor's, master's and doctorate degrees in a wide range of fields.\nQuestion: Multnomah University is located on the West coast of the U.S. True, False, or Neither?\nAnswer:", " False"]]
\ No newline at end of file

tests/testdata/anli_r2-v0-res.json

+{"results": {"anli_r2": {"acc": 0.4, "acc_stderr": 0.16329931618554522}}, "versions": {"anli_r2": 0}}
\ No newline at end of file

tests/testdata/anli_r3-v0-loglikelihood.json

+[["How to keep your pets healthy<br>Consult with your pet's veterinarian concerning diets. Every pet needs to be well taken care of and the most important thing is to ensure that they have a proper diet. You might see yourself feeding a dog once a day, while birds and rabbits should always be provided with a fresh container of food and water at all times.\nQuestion: Rabbits always need food True, False, or Neither?\nAnswer:", " True"], ["How to keep your pets healthy<br>Consult with your pet's veterinarian concerning diets. Every pet needs to be well taken care of and the most important thing is to ensure that they have a proper diet. You might see yourself feeding a dog once a day, while birds and rabbits should always be provided with a fresh container of food and water at all times.\nQuestion: Rabbits always need food True, False, or Neither?\nAnswer:", " Neither"], ["How to keep your pets healthy<br>Consult with your pet's veterinarian concerning diets. Every pet needs to be well taken care of and the most important thing is to ensure that they have a proper diet. You might see yourself feeding a dog once a day, while birds and rabbits should always be provided with a fresh container of food and water at all times.\nQuestion: Rabbits always need food True, False, or Neither?\nAnswer:", " False"], ["Boys Republic () is a South Korean K-pop boy band consisting of five members: Onejunn, Sunwoo, Sungjun, Minsu, and Suwoong. They are Universal Music's first K-pop idol group (also managed by Happy Tribe Entertainment). Boys Republic debuted on June 5, 2013 with the single, \"Party Rock\". Their official fan club name is \"Royal Family\".\nQuestion: Boys Republic have been around for over six years. True, False, or Neither?\nAnswer:", " True"], ["Boys Republic () is a South Korean K-pop boy band consisting of five members: Onejunn, Sunwoo, Sungjun, Minsu, and Suwoong. They are Universal Music's first K-pop idol group (also managed by Happy Tribe Entertainment). Boys Republic debuted on June 5, 2013 with the single, \"Party Rock\". Their official fan club name is \"Royal Family\".\nQuestion: Boys Republic have been around for over six years. True, False, or Neither?\nAnswer:", " Neither"], ["Boys Republic () is a South Korean K-pop boy band consisting of five members: Onejunn, Sunwoo, Sungjun, Minsu, and Suwoong. They are Universal Music's first K-pop idol group (also managed by Happy Tribe Entertainment). Boys Republic debuted on June 5, 2013 with the single, \"Party Rock\". Their official fan club name is \"Royal Family\".\nQuestion: Boys Republic have been around for over six years. True, False, or Neither?\nAnswer:", " False"], ["How to find an airport career job<br>Find a job you would enjoy. The first step in finding an airport job will be to examine what jobs are available in the air travel industry. Since the field is rather large you will have plenty of positions to choose from.\nQuestion: to find an airport job have to start with the fast food restaurants True, False, or Neither?\nAnswer:", " True"], ["How to find an airport career job<br>Find a job you would enjoy. The first step in finding an airport job will be to examine what jobs are available in the air travel industry. Since the field is rather large you will have plenty of positions to choose from.\nQuestion: to find an airport job have to start with the fast food restaurants True, False, or Neither?\nAnswer:", " Neither"], ["How to find an airport career job<br>Find a job you would enjoy. The first step in finding an airport job will be to examine what jobs are available in the air travel industry. Since the field is rather large you will have plenty of positions to choose from.\nQuestion: to find an airport job have to start with the fast food restaurants True, False, or Neither?\nAnswer:", " False"], ["How to become highly motivated for your job<br>Vary your tasks throughout the day. People tend to lose motivation on the job out of sheer boredom. The brain functions best when it is subjected to stimulation and differentiated activity.\nQuestion: BRAIN NEEDS FUN TOO True, False, or Neither?\nAnswer:", " True"], ["How to become highly motivated for your job<br>Vary your tasks throughout the day. People tend to lose motivation on the job out of sheer boredom. The brain functions best when it is subjected to stimulation and differentiated activity.\nQuestion: BRAIN NEEDS FUN TOO True, False, or Neither?\nAnswer:", " Neither"], ["How to become highly motivated for your job<br>Vary your tasks throughout the day. People tend to lose motivation on the job out of sheer boredom. The brain functions best when it is subjected to stimulation and differentiated activity.\nQuestion: BRAIN NEEDS FUN TOO True, False, or Neither?\nAnswer:", " False"], ["And so they're looking for ways to satisfy their desire for a better life and still reduce pollution at the same time. I think that holding onto the old ways and the old argument that the environment and the economy are in conflict is really outdated.\nQuestion:  They are always looking for ways to have a better life, even if that means it will come at the expense of the environment True, False, or Neither?\nAnswer:", " True"], ["And so they're looking for ways to satisfy their desire for a better life and still reduce pollution at the same time. I think that holding onto the old ways and the old argument that the environment and the economy are in conflict is really outdated.\nQuestion:  They are always looking for ways to have a better life, even if that means it will come at the expense of the environment True, False, or Neither?\nAnswer:", " Neither"], ["And so they're looking for ways to satisfy their desire for a better life and still reduce pollution at the same time. I think that holding onto the old ways and the old argument that the environment and the economy are in conflict is really outdated.\nQuestion:  They are always looking for ways to have a better life, even if that means it will come at the expense of the environment True, False, or Neither?\nAnswer:", " False"], ["Thank you for visiting our website. Since you landed on this page you are looking for the solutions to Mirror Cryptic December 11th 2017 Crossword and you are in luck! We have just updated our site with the latest answers so without wasting time, scroll down and find the clue that you are looking for. Click any of the clues to reveal the answer and see the rest of the solutions from other publishers too.\nQuestion: This website is one stop shop. True, False, or Neither?\nAnswer:", " True"], ["Thank you for visiting our website. Since you landed on this page you are looking for the solutions to Mirror Cryptic December 11th 2017 Crossword and you are in luck! We have just updated our site with the latest answers so without wasting time, scroll down and find the clue that you are looking for. Click any of the clues to reveal the answer and see the rest of the solutions from other publishers too.\nQuestion: This website is one stop shop. True, False, or Neither?\nAnswer:", " Neither"], ["Thank you for visiting our website. Since you landed on this page you are looking for the solutions to Mirror Cryptic December 11th 2017 Crossword and you are in luck! We have just updated our site with the latest answers so without wasting time, scroll down and find the clue that you are looking for. Click any of the clues to reveal the answer and see the rest of the solutions from other publishers too.\nQuestion: This website is one stop shop. True, False, or Neither?\nAnswer:", " False"], ["Sullivan is a city in Hamilton Township, Sullivan County, Indiana, United States. The population was 4,249 at the 2010 census. It is part of the Terre Haute Metropolitan Statistical Area. The city is the county seat of Sullivan County. The town is served by the Southwest School Corporation, and students go to Sullivan High School.\nQuestion: Sullivan has a population of 4249. True, False, or Neither?\nAnswer:", " True"], ["Sullivan is a city in Hamilton Township, Sullivan County, Indiana, United States. The population was 4,249 at the 2010 census. It is part of the Terre Haute Metropolitan Statistical Area. The city is the county seat of Sullivan County. The town is served by the Southwest School Corporation, and students go to Sullivan High School.\nQuestion: Sullivan has a population of 4249. True, False, or Neither?\nAnswer:", " Neither"], ["Sullivan is a city in Hamilton Township, Sullivan County, Indiana, United States. The population was 4,249 at the 2010 census. It is part of the Terre Haute Metropolitan Statistical Area. The city is the county seat of Sullivan County. The town is served by the Southwest School Corporation, and students go to Sullivan High School.\nQuestion: Sullivan has a population of 4249. True, False, or Neither?\nAnswer:", " False"], ["How to determine the sex of a cat<br>Approach the cat or kitten with care. To ascertain the sex of a cat or kitten, it's necessary to pick the cat up. Some cats don't like to be handled, so take a little time to let him or her get comfortable around you.\nQuestion: If you pick up a cat you will know what gender it is. True, False, or Neither?\nAnswer:", " True"], ["How to determine the sex of a cat<br>Approach the cat or kitten with care. To ascertain the sex of a cat or kitten, it's necessary to pick the cat up. Some cats don't like to be handled, so take a little time to let him or her get comfortable around you.\nQuestion: If you pick up a cat you will know what gender it is. True, False, or Neither?\nAnswer:", " Neither"], ["How to determine the sex of a cat<br>Approach the cat or kitten with care. To ascertain the sex of a cat or kitten, it's necessary to pick the cat up. Some cats don't like to be handled, so take a little time to let him or her get comfortable around you.\nQuestion: If you pick up a cat you will know what gender it is. True, False, or Neither?\nAnswer:", " False"], ["How to clean a bed with baking soda<br>Remove the bedsheets and wash them in the washing machine. Start by stripping the bed of any sheets, blankets, or duvets. Put the bedding in the washing machine and clean them with detergent on the hottest water setting to kill germs.\nQuestion: Sheets and blankets are typically found on beds True, False, or Neither?\nAnswer:", " True"], ["How to clean a bed with baking soda<br>Remove the bedsheets and wash them in the washing machine. Start by stripping the bed of any sheets, blankets, or duvets. Put the bedding in the washing machine and clean them with detergent on the hottest water setting to kill germs.\nQuestion: Sheets and blankets are typically found on beds True, False, or Neither?\nAnswer:", " Neither"], ["How to clean a bed with baking soda<br>Remove the bedsheets and wash them in the washing machine. Start by stripping the bed of any sheets, blankets, or duvets. Put the bedding in the washing machine and clean them with detergent on the hottest water setting to kill germs.\nQuestion: Sheets and blankets are typically found on beds True, False, or Neither?\nAnswer:", " False"], ["Brian Helps Out<br>Five-year-old Brian was watching the news with his parents. When pictures of the Louisiana flood came on, the child cried. At dinner, he worried what children in Louisiana were eating. The next day, his teacher helped him write a letter to the school. Brian's letter raised 400 cans and boxes of food for The Red Cross!\nQuestion: Brian was more than 120 months old then. True, False, or Neither?\nAnswer:", " True"], ["Brian Helps Out<br>Five-year-old Brian was watching the news with his parents. When pictures of the Louisiana flood came on, the child cried. At dinner, he worried what children in Louisiana were eating. The next day, his teacher helped him write a letter to the school. Brian's letter raised 400 cans and boxes of food for The Red Cross!\nQuestion: Brian was more than 120 months old then. True, False, or Neither?\nAnswer:", " Neither"], ["Brian Helps Out<br>Five-year-old Brian was watching the news with his parents. When pictures of the Louisiana flood came on, the child cried. At dinner, he worried what children in Louisiana were eating. The next day, his teacher helped him write a letter to the school. Brian's letter raised 400 cans and boxes of food for The Red Cross!\nQuestion: Brian was more than 120 months old then. True, False, or Neither?\nAnswer:", " False"]]
\ No newline at end of file

tests/testdata/anli_r3-v0-res.json

+{"results": {"anli_r3": {"acc": 0.4, "acc_stderr": 0.16329931618554522}}, "versions": {"anli_r3": 0}}
\ No newline at end of file

tests/testdata/arc_challenge-v0-loglikelihood.json

+[["Question: Cities control the amount of pollution that is allowed to come from cars. How does this most likely help people?\nAnswer:", " The air stays cleaner."], ["Question: Cities control the amount of pollution that is allowed to come from cars. How does this most likely help people?\nAnswer:", " Cars can travel at faster speeds."], ["Question: Cities control the amount of pollution that is allowed to come from cars. How does this most likely help people?\nAnswer:", " The skills of the drivers improve."], ["Question: Cities control the amount of pollution that is allowed to come from cars. How does this most likely help people?\nAnswer:", " It becomes safer to drive on the roads."], ["Question: Which statement correctly describes a physical characteristic of the Moon?\nAnswer:", " The Moon is made of hot gases."], ["Question: Which statement correctly describes a physical characteristic of the Moon?\nAnswer:", " The Moon is covered with many craters."], ["Question: Which statement correctly describes a physical characteristic of the Moon?\nAnswer:", " The Moon has many bodies of liquid water."], ["Question: Which statement correctly describes a physical characteristic of the Moon?\nAnswer:", " The Moon has the ability to give off its own light."], ["Question: Which object in the solar system is orbited by a belt of asteroids?\nAnswer:", " Pluto"], ["Question: Which object in the solar system is orbited by a belt of asteroids?\nAnswer:", " Saturn"], ["Question: Which object in the solar system is orbited by a belt of asteroids?\nAnswer:", " the Sun"], ["Question: Which object in the solar system is orbited by a belt of asteroids?\nAnswer:", " the Moon"], ["Question: Light waves that cross from an air medium to a water medium will\nAnswer:", " be focused into a straight line."], ["Question: Light waves that cross from an air medium to a water medium will\nAnswer:", " lose energy and dissipate."], ["Question: Light waves that cross from an air medium to a water medium will\nAnswer:", " change length and direction."], ["Question: Light waves that cross from an air medium to a water medium will\nAnswer:", " reflect off the water's surface."], ["Question: How many valence electrons does selenium have?\nAnswer:", " 3"], ["Question: How many valence electrons does selenium have?\nAnswer:", " 5"], ["Question: How many valence electrons does selenium have?\nAnswer:", " 6"], ["Question: How many valence electrons does selenium have?\nAnswer:", " 8"], ["Question: As a warm moist air mass moving northward collides with a strong cold air mass moving southward, what observations will most likely be made?\nAnswer:", " Thick fog develops."], ["Question: As a warm moist air mass moving northward collides with a strong cold air mass moving southward, what observations will most likely be made?\nAnswer:", " Temperatures increase."], ["Question: As a warm moist air mass moving northward collides with a strong cold air mass moving southward, what observations will most likely be made?\nAnswer:", " Clouds begin to form."], ["Question: As a warm moist air mass moving northward collides with a strong cold air mass moving southward, what observations will most likely be made?\nAnswer:", " Winds die down."], ["Question: Chemical weathering occurs when minerals in rocks are changed chemically. Which of these will most likely change the rate of chemical weathering on a rock?\nAnswer:", " decrease in air temperature"], ["Question: Chemical weathering occurs when minerals in rocks are changed chemically. Which of these will most likely change the rate of chemical weathering on a rock?\nAnswer:", " increase in rainfall amounts"], ["Question: Chemical weathering occurs when minerals in rocks are changed chemically. Which of these will most likely change the rate of chemical weathering on a rock?\nAnswer:", " slow movement of a glacier"], ["Question: Chemical weathering occurs when minerals in rocks are changed chemically. Which of these will most likely change the rate of chemical weathering on a rock?\nAnswer:", " rapid growth of plant roots"], ["Question: A toy truck rolls over a smooth surface. If the surface is covered with sand, the truck will most likely roll\nAnswer:", " slower"], ["Question: A toy truck rolls over a smooth surface. If the surface is covered with sand, the truck will most likely roll\nAnswer:", " faster"], ["Question: A toy truck rolls over a smooth surface. If the surface is covered with sand, the truck will most likely roll\nAnswer:", " at the same speed"], ["Question: A town in northern Arkansas experienced colder than normal temperatures during part of the winter. Which change was most likely responsible for this?\nAnswer:", " A southward dip in the jet stream"], ["Question: A town in northern Arkansas experienced colder than normal temperatures during part of the winter. Which change was most likely responsible for this?\nAnswer:", " A northward movement of the jet stream"], ["Question: A town in northern Arkansas experienced colder than normal temperatures during part of the winter. Which change was most likely responsible for this?\nAnswer:", " The Coriolis effect creating a low pressure area"], ["Question: A town in northern Arkansas experienced colder than normal temperatures during part of the winter. Which change was most likely responsible for this?\nAnswer:", " The Coriolis effect creating a high pressure area"], ["Question: How much time is required for a bicycle to travel a distance of 100 m at an average speed of 2 m/s?\nAnswer:", " 0.02 s"], ["Question: How much time is required for a bicycle to travel a distance of 100 m at an average speed of 2 m/s?\nAnswer:", " 50 s"], ["Question: How much time is required for a bicycle to travel a distance of 100 m at an average speed of 2 m/s?\nAnswer:", " 100 s"], ["Question: How much time is required for a bicycle to travel a distance of 100 m at an average speed of 2 m/s?\nAnswer:", " 200 s"]]
\ No newline at end of file

tests/testdata/arc_challenge-v0-res.json

+{"results": {"arc_challenge": {"acc": 0.0, "acc_stderr": 0.0, "acc_norm": 0.0, "acc_norm_stderr": 0.0}}, "versions": {"arc_challenge": 0}}
\ No newline at end of file

tests/testdata/arc_easy-v0-loglikelihood.json

+[["Question: Which is the function of the gallbladder?\nAnswer:", " store bile"], ["Question: Which is the function of the gallbladder?\nAnswer:", " produce bile"], ["Question: Which is the function of the gallbladder?\nAnswer:", " store digestive enzymes"], ["Question: Which is the function of the gallbladder?\nAnswer:", " produce digestive enzymes"], ["Question: Which type of adaptation allows an animal to deceive its predator?\nAnswer:", " large size"], ["Question: Which type of adaptation allows an animal to deceive its predator?\nAnswer:", " protective coloration"], ["Question: Which type of adaptation allows an animal to deceive its predator?\nAnswer:", " scent glands"], ["Question: Which type of adaptation allows an animal to deceive its predator?\nAnswer:", " leathery skin"], ["Question: Which three things do animals need from their environment in order to survive?\nAnswer:", " soil, water, and food"], ["Question: Which three things do animals need from their environment in order to survive?\nAnswer:", " soil, light, and water"], ["Question: Which three things do animals need from their environment in order to survive?\nAnswer:", " air, food, and water"], ["Question: Which three things do animals need from their environment in order to survive?\nAnswer:", " air, water, and light"], ["Question: When atmospheric carbon dioxide increases, some of the carbon dioxide dissolves in the ocean causing the ocean to become\nAnswer:", " warmer."], ["Question: When atmospheric carbon dioxide increases, some of the carbon dioxide dissolves in the ocean causing the ocean to become\nAnswer:", " cooler."], ["Question: When atmospheric carbon dioxide increases, some of the carbon dioxide dissolves in the ocean causing the ocean to become\nAnswer:", " more acidic."], ["Question: When atmospheric carbon dioxide increases, some of the carbon dioxide dissolves in the ocean causing the ocean to become\nAnswer:", " more alkaline."], ["Question: Which is the most important safety measure to follow when using an acid in a classroom laboratory?\nAnswer:", " wearing protective clothing and goggles"], ["Question: Which is the most important safety measure to follow when using an acid in a classroom laboratory?\nAnswer:", " keeping the window open"], ["Question: Which is the most important safety measure to follow when using an acid in a classroom laboratory?\nAnswer:", " closing the door to the hall"], ["Question: Which is the most important safety measure to follow when using an acid in a classroom laboratory?\nAnswer:", " using bright fluorescent lights"], ["Question: Which is an effect that cities have on the weather?\nAnswer:", " Small numbers of plants cause rain to fall more often."], ["Question: Which is an effect that cities have on the weather?\nAnswer:", " Large numbers of houses cause the wind to be stronger."], ["Question: Which is an effect that cities have on the weather?\nAnswer:", " Small areas of open water cause fog to be more common."], ["Question: Which is an effect that cities have on the weather?\nAnswer:", " Large areas of pavement cause the temperature to be hotter."], ["Question: Which two terms are used to describe weather?\nAnswer:", " wind direction and amount of erosion"], ["Question: Which two terms are used to describe weather?\nAnswer:", " gravity and amount of rain"], ["Question: Which two terms are used to describe weather?\nAnswer:", " groundwater and cloud cover"], ["Question: Which two terms are used to describe weather?\nAnswer:", " air temperature and wind speed"], ["Question: A population of tree-climbing lizard lives on one bank of a large river. The other bank of the river is a treeless prairie. During a flood, 40 lizards were transferred to the prairie side of the river. After 200 generations, this transferred population of lizard lost the ability to climb.Which mechanism is most likely responsible for this loss of function within the transferred population?\nAnswer:", " gene flow"], ["Question: A population of tree-climbing lizard lives on one bank of a large river. The other bank of the river is a treeless prairie. During a flood, 40 lizards were transferred to the prairie side of the river. After 200 generations, this transferred population of lizard lost the ability to climb.Which mechanism is most likely responsible for this loss of function within the transferred population?\nAnswer:", " natural selection"], ["Question: A population of tree-climbing lizard lives on one bank of a large river. The other bank of the river is a treeless prairie. During a flood, 40 lizards were transferred to the prairie side of the river. After 200 generations, this transferred population of lizard lost the ability to climb.Which mechanism is most likely responsible for this loss of function within the transferred population?\nAnswer:", " genetic drift"], ["Question: A population of tree-climbing lizard lives on one bank of a large river. The other bank of the river is a treeless prairie. During a flood, 40 lizards were transferred to the prairie side of the river. After 200 generations, this transferred population of lizard lost the ability to climb.Which mechanism is most likely responsible for this loss of function within the transferred population?\nAnswer:", " mutation"], ["Question: In plants, what is responsible for passing traits from one generation to the next?\nAnswer:", " chloroplasts"], ["Question: In plants, what is responsible for passing traits from one generation to the next?\nAnswer:", " photosynthesis"], ["Question: In plants, what is responsible for passing traits from one generation to the next?\nAnswer:", " roots"], ["Question: In plants, what is responsible for passing traits from one generation to the next?\nAnswer:", " genes"], ["Question: In which way is a prokaryotic cell different from a eukaryotic cell?\nAnswer:", " A prokaryotic cell is larger."], ["Question: In which way is a prokaryotic cell different from a eukaryotic cell?\nAnswer:", " A prokaryotic cell has no nucleus."], ["Question: In which way is a prokaryotic cell different from a eukaryotic cell?\nAnswer:", " A prokaryotic cell can reproduce itself."], ["Question: In which way is a prokaryotic cell different from a eukaryotic cell?\nAnswer:", " A prokaryotic cell has a cell membrane."]]
\ No newline at end of file

tests/testdata/arc_easy-v0-res.json

+{"results": {"arc_easy": {"acc": 0.1, "acc_stderr": 0.09999999999999999, "acc_norm": 0.2, "acc_norm_stderr": 0.13333333333333333}}, "versions": {"arc_easy": 0}}
\ No newline at end of file

tests/testdata/arithmetic_1dc-v0-loglikelihood.json

+[["Question: What is (1 * 7) * 8?\nAnswer:", " 56"], ["Question: What is (2 - 6) * 3?\nAnswer:", " -12"], ["Question: What is (6 - 1) * 4?\nAnswer:", " 20"], ["Question: What is (8 + 4) * 9?\nAnswer:", " 108"], ["Question: What is (3 * 5) - 5?\nAnswer:", " 10"], ["Question: What is (8 * 5) * 8?\nAnswer:", " 320"], ["Question: What is (7 - 1) + 4?\nAnswer:", " 10"], ["Question: What is (1 - 9) * 1?\nAnswer:", " -8"], ["Question: What is (3 + 4) * 8?\nAnswer:", " 56"], ["Question: What is (8 - 3) - 3?\nAnswer:", " 2"]]
\ No newline at end of file

tests/testdata/arithmetic_1dc-v0-res.json

+{"results": {"arithmetic_1dc": {"acc": 0.0, "acc_stderr": 0.0}}, "versions": {"arithmetic_1dc": 0}}
\ No newline at end of file

tests/testdata/arithmetic_2da-v0-loglikelihood.json

+[["Question: What is 56 plus 58?\nAnswer:", " 114"], ["Question: What is 43 plus 40?\nAnswer:", " 83"], ["Question: What is 4 plus 80?\nAnswer:", " 84"], ["Question: What is 35 plus 64?\nAnswer:", " 99"], ["Question: What is 3 plus 28?\nAnswer:", " 31"], ["Question: What is 37 plus 72?\nAnswer:", " 109"], ["Question: What is 7 plus 45?\nAnswer:", " 52"], ["Question: What is 6 plus 50?\nAnswer:", " 56"], ["Question: What is 17 plus 48?\nAnswer:", " 65"], ["Question: What is 87 plus 44?\nAnswer:", " 131"]]
\ No newline at end of file

tests/testdata/arithmetic_2da-v0-res.json

+{"results": {"arithmetic_2da": {"acc": 0.0, "acc_stderr": 0.0}}, "versions": {"arithmetic_2da": 0}}
\ No newline at end of file