Merge branch 'main' into metrics

# Conflicts:
#	lm_eval/models/vllm_causallms.py
#	pyproject.toml

lm_eval/tasks/babilong/babilong.yaml

+group: babilong
+task:
+  - babilong_qa1
+  - babilong_qa2
+  - babilong_qa3
+  - babilong_qa4
+  - babilong_qa5
+  - babilong_qa6
+  - babilong_qa7
+  - babilong_qa8
+  - babilong_qa9
+  - babilong_qa10
+  - babilong_qa11
+  - babilong_qa12
+  - babilong_qa13
+  - babilong_qa14
+  - babilong_qa15
+  - babilong_qa16
+  - babilong_qa17
+  - babilong_qa18
+  - babilong_qa19
+  - babilong_qa20
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0

lm_eval/tasks/babilong/babilong_longctx.yaml

+group: babilong_longctx
+task:
+  - babilong_qa1
+  - babilong_qa2
+  - babilong_qa3
+  - babilong_qa4
+  - babilong_qa5
+aggregate_metric_list:
+  - metric: acc
+    weight_by_size: True
+metadata:
+  version: 0.0

lm_eval/tasks/babilong/babilong_qa1.yaml

+include: _babilong_common_yaml
+task: babilong_qa1
+test_split: qa1
+custom_dataset: !function common_utils.load_dataset
+dataset_kwargs:
+  qa_split: qa1
+description: "I will give you context with the facts about positions of different persons hidden in some random text and a question. You need to answer the question based only on the information from the facts. If a person was in different locations, use the latest location to answer the question.\nAlways return your answer in the following format:\nThe most recent location of 'person' is 'location'. Do not write anything else after that.\n\n"
+doc_to_text: "{{input.strip()}}\n{{question.strip()}}"
+
+fewshot_config:
+  sampler: first_n
+  samples:
+    - input: "Charlie went to the hallway. Judith come back to the kitchen. Charlie travelled to balcony."
+      question: "Where is Charlie?"
+      target: "The most recent location of Charlie is balcony."
+    - input: "Alan moved to the garage. Charlie went to the beach. Alan went to the shop. Rouse travelled to balcony."
+      question: "Where is Alan?"
+      target: "The most recent location of Alan is shop."

lm_eval/tasks/babilong/babilong_qa10.yaml

+include: _babilong_common_yaml
+task: babilong_qa10
+test_split: qa10
+custom_dataset: !function common_utils.load_dataset
+dataset_kwargs:
+  qa_split: qa10
+description: "I will give you context with the facts about people and their locations hidden in some random text and a question. You need to answer the question based only on the information from the facts.\nIf a person was in different locations, use the latest location the person was in to answer the question.\nYour answer should contain only one word - $yes$ or $no$ or $maybe$. Do not write anything else. Do not explain your answer.\n\n"
+doc_to_text: "{{input.strip()}}\n{{question.strip()}}"
+
+fewshot_config:
+  sampler: first_n
+  samples:
+    - input: "Bill is in the kitchen. Julie is either in the school or the cinema."
+      question: "Is Bill in the bedroom?"
+      target: "no"
+    - input: "Fred is in the bedroom. Mary is either in the school or the cinema."
+      question: "Is Mary in the school?"
+      target: "maybe"
+    - input: "Fred is either in the kitchen or the park. Bill moved to the cinema."
+      question: "Is Bill in the cinema?"
+      target: "yes"

lm_eval/tasks/babilong/babilong_qa11.yaml

+include: _babilong_common_yaml
+task: babilong_qa11
+test_split: qa11
+dataset_name: 0k
+description: "I will give you context with the facts about people and their locations hidden in some random text and a question. You need to answer the question based only on the information from the facts.\nIf a person was in different locations, use the latest location the person was in to answer the question.\nYour answer should contain only one word - location. Do not write anything else after that. Do not explain your answer.\n\n"
+doc_to_text: "{{input.strip()}}\n{{question.strip()}}"
+
+fewshot_config:
+  sampler: first_n
+  samples:
+    - input: "Daniel journeyed to the hallway. After that he journeyed to the garden."
+      question: "Where is Daniel?"
+      target: "garden"
+    - input: "Mary moved to the office. Afterwards she journeyed to the kitchen. Daniel went to the hallway. Then he journeyed to the garden."
+      question: "Where is Mary?"
+      target: "kitchen"
+    - input: "Sandra moved to the kitchen. After that she went back to the hallway. Sandra moved to the bedroom. Then she went to the hallway. Mary moved to the bedroom. Afterwards she travelled to the bathroom."
+      question: "Where is Sandra?"
+      target: "hallway"

lm_eval/tasks/babilong/babilong_qa12.yaml

+include: _babilong_common_yaml
+task: babilong_qa12
+test_split: qa12
+dataset_name: 0k
+description: "I will give you context with the facts about people and their locations hidden in some random text and a question. You need to answer the question based only on the information from the facts.\nIf a person was in different locations, use the latest location the person was in to answer the question.\nYour answer should contain only one word - location. Do not write anything else after that. Do not explain your answer.\n\n"
+doc_to_text: "{{input.strip()}}\n{{question.strip()}}"
+
+fewshot_config:
+  sampler: first_n
+  samples:
+    - input: "Mary and Daniel travelled to the bathroom. John and Daniel travelled to the office."
+      question: "Where is Daniel?"
+      target: "office"
+    - input: "Sandra and Mary went back to the office. Daniel and Sandra went to the bedroom. Sandra and Mary travelled to the hallway. John and Mary went to the kitchen."
+      question: "Where is Mary?"
+      target: "kitchen"
+    - input: "Daniel and Sandra went back to the hallway. Daniel and John moved to the office. Daniel and John moved to the garden. Daniel and Mary went back to the bathroom. Daniel and John went back to the kitchen. Daniel and Sandra went to the bathroom."
+      question: "Where is John?"
+      target: "kitchen"

lm_eval/tasks/babilong/babilong_qa13.yaml

+include: _babilong_common_yaml
+task: babilong_qa13
+test_split: qa13
+dataset_name: 0k
+description: "I will give you context with the facts about people and their locations hidden in some random text and a question. You need to answer the question based only on the information from the facts.\nIf a person was in different locations, use the latest location the person was in to answer the question.\nYour answer should contain only one word - location. Do not write anything else after that. Do not explain your answer.\n\n"
+doc_to_text: "{{input.strip()}}\n{{question.strip()}}"
+
+fewshot_config:
+  sampler: first_n
+  samples:
+    - input: "Mary and Daniel travelled to the bathroom. Then they journeyed to the hallway."
+      question: "Where is Daniel?"
+      target: "hallway"
+    - input: "Daniel and Sandra travelled to the kitchen. After that they journeyed to the hallway. Mary and Daniel travelled to the bedroom. After that they travelled to the hallway."
+      question: "Where is Sandra?"
+      target: "hallway"
+    - input: "John and Mary moved to the bathroom. Then they travelled to the office. John and Mary went to the kitchen. Afterwards they went to the bedroom. John and Sandra moved to the bathroom. Following that they went back to the kitchen."
+      question: "Where is Mary?"
+      target: "bedroom"

lm_eval/tasks/babilong/babilong_qa14.yaml

+include: _babilong_common_yaml
+task: babilong_qa14
+test_split: qa14
+dataset_name: 0k
+description: "I will give you context with the facts about people and their locations hidden in some random text and a question. You need to answer the question based only on the information from the facts.\nIf a person was in different locations, use the latest location the person was in to answer the question.\nYour answer should contain only one word - location. Do not write anything else after that. Do not explain your answer.\n\n"
+doc_to_text: "{{input.strip()}}\n{{question.strip()}}"
+
+fewshot_config:
+  sampler: first_n
+  samples:
+    - input: "Bill went back to the cinema yesterday. Julie went to the school this morning. Fred went to the park yesterday. Yesterday Julie went to the office."
+      question: "Where was Julie before the school?"
+      target: "office"
+    - input: "This morning Fred went to the kitchen. Fred journeyed to the bedroom yesterday. Mary travelled to the bedroom this morning. Yesterday Mary went to the cinema."
+      question: "Where was Mary before the bedroom?"
+      target: "cinema"
+    - input: "Yesterday Julie went back to the park. Julie went to the bedroom this morning. Bill journeyed to the cinema yesterday. This morning Bill went back to the park. This evening Julie went to the school. This afternoon Julie went back to the park."
+      question: "Where was Julie before the bedroom?"
+      target: "park"

lm_eval/tasks/babilong/babilong_qa15.yaml

+include: _babilong_common_yaml
+task: babilong_qa15
+test_split: qa15
+dataset_name: 0k
+description: "I will give you context with the facts about animals, their names and relations. The facts and a question are hidden in some random text. You need to answer the question based only on the information from the facts.\nYour answer should contain only one word - an animal species. Do not write anything else after that. Do not explain your answer.\n\n"
+doc_to_text: "{{input.strip()}}\n{{question.strip()}}"
+
+fewshot_config:
+  sampler: first_n
+  samples:
+    - input: "Mice are afraid of wolves. Gertrude is a mouse. Cats are afraid of sheep. Winona is a mouse. Sheep are afraid of wolves. Emily is a mouse. Jessica is a wolf."
+      question: "What is gertrude afraid of?"
+      target: "wolf"
+    - input: "Mice are afraid of wolves. Gertrude is a mouse. Cats are afraid of sheep. Winona is a mouse. Sheep are afraid of wolves. Emily is a mouse. Jessica is a wolf."
+      question: "What is jessica afraid of?"
+      target: "cat"
+    - input: "Mice are afraid of cats. Wolves are afraid of sheep. Emily is a wolf. Cats are afraid of sheep. Gertrude is a wolf. Sheep are afraid of cats. Winona is a wolf."
+      question: "What is emily afraid of?"
+      target: "sheep"

lm_eval/tasks/babilong/babilong_qa16.yaml

+include: _babilong_common_yaml
+task: babilong_qa16
+test_split: qa16
+dataset_name: 0k
+description: "I will give you context with the facts about animals, their names and colors. The facts and a question are hidden in some random text. You need to answer the question based only on the information from the facts.\nYour answer should contain only one word - a color. Do not write anything else after that.\nDo not explain your answer.\n\n"
+doc_to_text: "{{input.strip()}}\n{{question.strip()}}"
+
+fewshot_config:
+  sampler: first_n
+  samples:
+    - input: "Lily is a frog. Bernhard is a frog. Bernhard is green. Brian is a lion. Brian is white. Julius is a swan. Julius is green. Lily is green. Greg is a swan."
+      question: "What color is Greg?"
+      target: "green"
+    - input: "Julius is a lion. Lily is a rhino. Bernhard is a swan. Lily is white. Bernhard is green. Greg is a rhino. Greg is gray. Julius is white. Brian is a lion."
+      question: "What color is Brian?"
+      target: "white"
+    - input: "Brian is a rhino. Julius is a lion. Bernhard is a lion. Greg is a swan. Brian is gray. Greg is white. Lily is a rhino. Bernhard is yellow. Lily is gray."
+      question: "What color is Julius?"
+      target: "yellow"

lm_eval/tasks/babilong/babilong_qa17.yaml

+include: _babilong_common_yaml
+task: babilong_qa17
+test_split: qa17
+dataset_name: 0k
+description: "I will give you context with the facts about different figures, their location and colors, hidden in some random text and a question. You need to answer the question based only on the information from the facts.\nYour answer should contain only one word - $yes$ or $no$. Do not write anything else.\nDo not explain your answer.\n\n"
+doc_to_text: "{{input.strip()}}\n{{question.strip()}}"
+
+fewshot_config:
+  sampler: first_n
+  samples:
+    - input: "The triangle is above the pink rectangle. The blue square is to the left of the triangle."
+      question: "Is the pink rectangle to the right of the blue square?"
+      target: "yes"
+    - input: "The red sphere is to the left of the yellow square. The red sphere is below the pink rectangle."
+      question: "Is the pink rectangle to the left of the yellow square?"
+      target: "yes"
+    - input: "The red sphere is above the pink rectangle. The red sphere is to the right of the red square."
+      question: "Is the pink rectangle above the red square?"
+      target: "no"

lm_eval/tasks/babilong/babilong_qa18.yaml

+include: _babilong_common_yaml
+task: babilong_qa18
+test_split: qa18
+dataset_name: 0k
+description: "I will give you context with the facts about different objects and their sizes, hidden in some random text and a question. You need to answer the question based only on the information from the facts.\nYour answer should contain only one word - $yes$ or $no$. Do not write anything else.\nDo not explain your answer.\n\n"
+doc_to_text: "{{input.strip()}}\n{{question.strip()}}"
+
+fewshot_config:
+  sampler: first_n
+  samples:
+    - input: "The box of chocolates fits inside the chest. The box is bigger than the chest. The box is bigger than the suitcase. The suitcase fits inside the box. The container is bigger than the box of chocolates."
+      question: "Does the box fit in the box of chocolates?"
+      target: "no"
+    - input: "The suitcase is bigger than the container. The container fits inside the box. The chest is bigger than the chocolate. The suitcase fits inside the box. The chest fits inside the box."
+      question: "Does the chocolate fit in the box?"
+      target: "yes"
+    - input: "The chocolate fits inside the box of chocolates. The suitcase fits inside the box. The chocolate fits inside the box. The box is bigger than the box of chocolates. The suitcase is bigger than the box of chocolates."
+      question: "Is the chocolate bigger than the box?"
+      target: "no"

lm_eval/tasks/babilong/babilong_qa19.yaml

+include: _babilong_common_yaml
+task: babilong_qa19
+test_split: qa19
+dataset_name: 0k
+description: "I will give you context with the facts about different places and their locations, hidden in some random text and a question. You need to answer the question based only on the information from the facts.\nYour answer should contain only two letters, separated by a comma - ordinal directions. You can choose the letters from $n$, $s$, $e$ and $w$. Do not write anything else after that.\n\n"
+doc_to_text: "{{input.strip()}}\n{{question.strip()}}"
+
+fewshot_config:
+  sampler: first_n
+  samples:
+    - input: "The office is east of the hallway. The kitchen is north of the office. The garden is west of the bedroom. The office is west of the garden. The bathroom is north of the garden."
+      question: "How do you go from the kitchen to the garden?"
+      target: "s,e"
+    - input: "The bedroom is west of the hallway. The office is east of the garden. The garden is north of the kitchen. The kitchen is north of the bathroom. The hallway is west of the garden."
+      question: "How do you go from the kitchen to the hallway?"
+      target: "n,w"
+    - input: "The bedroom is south of the hallway. The bathroom is east of the office. The kitchen is west of the garden. The garden is south of the office. The office is south of the bedroom."
+      question: "How do you go from the garden to the bedroom?"
+      target: "n,n"

lm_eval/tasks/babilong/babilong_qa2.yaml

+include: _babilong_common_yaml
+task: babilong_qa2
+test_split: qa2
+custom_dataset: !function common_utils.load_dataset
+dataset_kwargs:
+  qa_split: qa2
+description: "I will give you context with the facts about locations and actions of different persons hidden in some random text and a question. You need to answer the question based only on the information from the facts. If a person got an item in the first location and travelled to the second location the item is also in the second location. If a person dropped an item in the first location and moved to the second location the item remains in the first location.\nAlways return your answer in the following format:\nThe 'item' is in 'location'. Do not write anything else after that.\n\n"
+doc_to_text: "{{input.strip()}}\n{{question.strip()}}"
+
+fewshot_config:
+  sampler: first_n
+  samples:
+    - input: "Charlie went to the kitchen. Charlie got a bottle. Charlie moved to the balcony."
+      question: "Where is the bottle?"
+      target: "The bottle is in the balcony."
+    - input: "Alan moved to the garage. Alan got a screw driver. Alan moved to the kitchen."
+      question: "Where is the screw driver?"
+      target: "The screw driver is in the kitchen."

lm_eval/tasks/babilong/babilong_qa20.yaml

+include: _babilong_common_yaml
+task: babilong_qa20
+test_split: qa20
+dataset_name: 0k
+description: "I will give you context with the facts about people, their locations and condition hidden in some random text and a question. You need to answer the question based only on the information from the facts.\nIf a person was in different locations, use the latest location the person was in to answer the question.\nYour answer should contain only one word - a person condition or a place. Do not write anything else after that. Do not explain your answer.\n\n"
+doc_to_text: "{{input.strip()}}\n{{question.strip()}}"
+
+fewshot_config:
+  sampler: first_n
+  samples:
+    - input: "Sumit is tired."
+      question: "Where will sumit go?"
+      target: "bedroom"
+    - input: "Yann is hungry. Yann journeyed to the kitchen."
+      question: "Why did yann go to the kitchen?"
+      target: "hungry"
+    - input: "Antoine is thirsty. Yann is tired. Yann went back to the bedroom. Yann picked up the pajamas there. Jason is thirsty. Antoine went back to the kitchen."
+      question: "Why did antoine go to the kitchen?"
+      target: "thirsty"

lm_eval/tasks/babilong/babilong_qa3.yaml

+include: _babilong_common_yaml
+task: babilong_qa3
+test_split: qa3
+custom_dataset: !function common_utils.load_dataset
+dataset_kwargs:
+  qa_split: qa3
+description: "I give you context with the facts about locations and actions of different persons hidden in some random text and a question. You need to answer the question based only on the information from the facts. If a person got an item in the first location and travelled to the second location the item is also in the second location. If a person dropped an item in the first location and moved to the second location the item remains in the first location.\nAlways return your answer in the following format:\nBefore the $location_1$ the $item$ was in the $location_2$. Do not write anything else after that.\n\n"
+doc_to_text: "{{input.strip()}}\n{{question.strip()}}"
+
+fewshot_config:
+  sampler: first_n
+  samples:
+    - input: "John journeyed to the bedroom. Mary grabbed the apple. Mary went back to the bathroom. Daniel journeyed to the bedroom. Daniel moved to the garden. Mary travelled to the kitchen."
+      question: "Where was the apple before the kitchen?"
+      target: "Before the kitchen the apple was in the bathroom."
+    - input: "John went back to the bedroom. John went back to the garden. John went back to the kitchen. Sandra took the football. Sandra travelled to the garden. Sandra journeyed to the bedroom."
+      question: "Where was the football before the bedroom?"
+      target: "Before the bedroom the football was in the garden."

lm_eval/tasks/babilong/babilong_qa4.yaml

+include: _babilong_common_yaml
+task: babilong_qa4
+test_split: qa4
+custom_dataset: !function common_utils.load_dataset
+dataset_kwargs:
+  qa_split: qa4
+description: "I will give you context with the facts about different people, their location and actions, hidden in some random text and a question. You need to answer the question based only on the information from the facts.\nYour answer should contain only one word - location. Do not write anything else after that.\n\n"
+doc_to_text: "{{input.strip()}}\n{{question.strip()}}"
+
+fewshot_config:
+  sampler: first_n
+  samples:
+    - input: "The hallway is south of the kitchen. The bedroom is north of the kitchen."
+      question: "What is the kitchen south of?"
+      target: "bedroom"
+    - input: "The garden is west of the bedroom. The bedroom is west of the kitchen."
+      question: "What is west of the bedroom?"
+      target: "garden"

lm_eval/tasks/babilong/babilong_qa5.yaml

+include: _babilong_common_yaml
+task: babilong_qa5
+test_split: qa5
+custom_dataset: !function common_utils.load_dataset
+dataset_kwargs:
+  qa_split: qa5
+description: "I will give you context with the facts about locations and their relations hidden in some random text and a question. You need to answer the question based only on the information from the facts.\nYour answer should contain only one word. Do not write anything else after that. Do not explain your answer.\n\n"
+doc_to_text: "{{input.strip()}}\n{{question.strip()}}"
+
+fewshot_config:
+  sampler: first_n
+  samples:
+    - input: "Mary picked up the apple there. Mary gave the apple to Fred. Mary moved to the bedroom. Bill took the milk there."
+      question: "Who did Mary give the apple to?"
+      target: "Fred"
+    - input: "Jeff took the football there. Jeff passed the football to Fred. Jeff got the milk there. Bill travelled to the bedroom."
+      question: "Who gave the football?"
+      target: "Jeff"
+    - input: "Fred picked up the apple there. Fred handed the apple to Bill. Bill journeyed to the bedroom. Jeff went back to the garden."
+      question: "What did Fred give to Bill?"
+      target: "apple"

lm_eval/tasks/babilong/babilong_qa6.yaml

+include: _babilong_common_yaml
+task: babilong_qa6
+test_split: qa6
+custom_dataset: !function common_utils.load_dataset
+dataset_kwargs:
+  qa_split: qa6
+description: "I will give you context with the facts about people and their locations hidden in some random text and a question. You need to answer the question based only on the information from the facts. If a person was in different locations, use the latest location the person was in to answer the question.\nYour answer should contain only one word - $yes$ or $no$. Do not write anything else after that.\nDo not explain your answer.\n\n"
+doc_to_text: "{{input.strip()}}\n{{question.strip()}}"
+
+fewshot_config:
+  sampler: first_n
+  samples:
+    - input: "John travelled to the hallway. John travelled to the garden."
+      question: "Is John in the garden?"
+      target: "yes"
+    - input: "Mary went to the office. Daniel journeyed to the hallway. Mary went to the bedroom. Sandra went to the garden."
+      question: "Is Mary in the office?"
+      target: "no"

lm_eval/tasks/babilong/babilong_qa7.yaml

+include: _babilong_common_yaml
+task: babilong_qa7
+test_split: qa7
+custom_dataset: !function common_utils.load_dataset
+dataset_kwargs:
+  qa_split: qa7
+description: "I will give you context with the facts about people and objects they carry, hidden in some random text and a question. You need to answer the question based only on the information from the facts.\nYour answer should contain only one word - $none$ or $number_of_objects$.\nDo not write anything else after that. Do not explain your answer.\n\n"
+doc_to_text: "{{input.strip()}}\n{{question.strip()}}"
+
+fewshot_config:
+  sampler: first_n
+  samples:
+    - input: "Daniel went to the bedroom. Daniel got the apple there."
+      question: "How many objects is Daniel carrying?"
+      target: "one"
+    - input: "Mary grabbed the apple there. Mary gave the apple to John."
+      question: "How many objects is Mary carrying?"
+      target: "none"
+    - input: "Sandra travelled to the hallway. Sandra picked up the milk there. Sandra took the apple there. Mary travelled to the garden."
+      question: "How many objects is Sandra carrying?"
+      target: "two"