Skip to content

GitLab

Commit 601be343 authored by Baber's avatar Baber
Browse files

Merge branch 'main' into feature/eval_from_config

parents d0884a96 68c3a811
Hide whitespace changes
Inline Side-by-side
Showing with 2809 additions and 168 deletions
lm_eval/models/vllm_vlms.py
View file @ 601be343
...@@ -12,6 +12,7 @@ from lm_eval.models.utils import ( ...@@ -12,6 +12,7 @@ from lm_eval.models.utils import (
Collator, Collator,
handle_stop_sequences, handle_stop_sequences,
replace_placeholders, replace_placeholders,
resize_image,
undistribute, undistribute,
) )
from lm_eval.models.vllm_causallms import VLLM from lm_eval.models.vllm_causallms import VLLM
...@@ -44,8 +45,20 @@ class VLLM_VLM(VLLM): ...@@ -44,8 +45,20 @@ class VLLM_VLM(VLLM):
interleave: bool = True, interleave: bool = True,
# TODO<baber>: handle max_images and limit_mm_per_prompt better # TODO<baber>: handle max_images and limit_mm_per_prompt better
max_images: int = 999, max_images: int = 999,
image_width: Optional[int] = None,
image_height: Optional[int] = None,
image_max_side: Optional[int] = None,
**kwargs, **kwargs,
): ):
self.image_width = image_width
self.image_height = image_height
self.image_max_side = image_max_side
if self.image_max_side and (self.image_width or self.image_height):
raise ValueError(
"Ambiguous config for image resize: you can not specify both "
"image_max_side and (image_width or image_height)"
)
if max_images != 999: if max_images != 999:
kwargs["limit_mm_per_prompt"] = {"image": max_images} kwargs["limit_mm_per_prompt"] = {"image": max_images}
eval_logger.info(f"Setting limit_mm_per_prompt[image] to {max_images}") eval_logger.info(f"Setting limit_mm_per_prompt[image] to {max_images}")
...@@ -93,7 +106,7 @@ class VLLM_VLM(VLLM): ...@@ -93,7 +106,7 @@ class VLLM_VLM(VLLM):
outputs.append(inputs) outputs.append(inputs)
return outputs return outputs
def _model_generate( def _multimodal_model_generate(
self, self,
requests: List[List[dict]] = None, requests: List[List[dict]] = None,
generate: bool = False, generate: bool = False,
...@@ -205,7 +218,10 @@ class VLLM_VLM(VLLM): ...@@ -205,7 +218,10 @@ class VLLM_VLM(VLLM):
def generate_until( def generate_until(
self, requests: List[Instance], disable_tqdm: bool = False self, requests: List[Instance], disable_tqdm: bool = False
) -> List[str]: ) -> List[str]:
# TODO: support text-only reqs if requests and len(requests[0].args) < 3:
# Fall back to non-multimodal generation.
return super().generate_until(requests=requests, disable_tqdm=disable_tqdm)
res = [] res = []
def _collate(x): def _collate(x):
...@@ -239,7 +255,15 @@ class VLLM_VLM(VLLM): ...@@ -239,7 +255,15 @@ class VLLM_VLM(VLLM):
for chunk in chunks: for chunk in chunks:
contexts, all_gen_kwargs, aux_arguments = zip(*chunk) contexts, all_gen_kwargs, aux_arguments = zip(*chunk)
visuals = [arg["visual"] for arg in aux_arguments] visuals = [
[
resize_image(
img, self.image_width, self.image_height, self.image_max_side
)
for img in arg["visual"]
]
for arg in aux_arguments
]
if not isinstance(contexts, list): if not isinstance(contexts, list):
contexts = list( contexts = list(
...@@ -272,7 +296,7 @@ class VLLM_VLM(VLLM): ...@@ -272,7 +296,7 @@ class VLLM_VLM(VLLM):
left_truncate_len=max_ctx_len, left_truncate_len=max_ctx_len,
) )
cont = self._model_generate( cont = self._multimodal_model_generate(
inputs, stop=until, generate=True, max_tokens=max_gen_toks, **kwargs inputs, stop=until, generate=True, max_tokens=max_gen_toks, **kwargs
) )
...@@ -288,3 +312,12 @@ class VLLM_VLM(VLLM): ...@@ -288,3 +312,12 @@ class VLLM_VLM(VLLM):
pbar.close() pbar.close()
return res return res
def loglikelihood_rolling(self, requests: List[Instance]) -> List[float]:
if requests and len(requests[0].args) < 3:
# Fall back to non-multimodal generation.
return super().loglikelihood_rolling(requests=requests)
raise NotImplementedError(
"model type `vllm-vlm` does not support loglikelihood_rolling. Use 'vlm' model type for text-only loglikelihood_rolling tasks ",
"this is because we do not support measuring the loglikelihood a model assigns to an image.",
)
lm_eval/tasks/README.md
View file @ 601be343
...@@ -5,165 +5,167 @@ ...@@ -5,165 +5,167 @@
For more information, including a full list of task names and their precise meanings or sources, follow the links provided to the individual README.md files for each subfolder. For more information, including a full list of task names and their precise meanings or sources, follow the links provided to the individual README.md files for each subfolder.
| Task Family | Description | Language(s) | | Task Family | Description | Language(s) |
|--------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------| |--------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------|
| [aclue](aclue/README.md) | Tasks focusing on ancient Chinese language understanding and cultural aspects. | Ancient Chinese | | [aclue](aclue/README.md) | Tasks focusing on ancient Chinese language understanding and cultural aspects. | Ancient Chinese |
| [acp_bench](acpbench/README.md) | Tasks evaluating the reasoning ability about Action, Change, and Planning | English | | [acp_bench](acpbench/README.md) | Tasks evaluating the reasoning ability about Action, Change, and Planning | English |
| [aexams](aexams/README.md) | Tasks in Arabic related to various academic exams covering a range of subjects. | Arabic | | [acp_bench_hard](acpbench/README.md) | Tasks evaluating the reasoning ability about Action, Change, and Planning | English |
| [agieval](agieval/README.md) | Tasks involving historical data or questions related to history and historical texts. | English, Chinese | | [aexams](aexams/README.md) | Tasks in Arabic related to various academic exams covering a range of subjects. | Arabic |
| [anli](anli/README.md) | Adversarial natural language inference tasks designed to test model robustness. | English | | [agieval](agieval/README.md) | Tasks involving historical data or questions related to history and historical texts. | English, Chinese |
| [arabic_leaderboard_complete](arabic_leaderboard_complete/README.md) | A full version of the tasks in the Open Arabic LLM Leaderboard, focusing on the evaluation of models that reflect the characteristics of Arabic language understanding and comprehension, culture, and heritage. Note that some of these tasks are machine-translated. | Arabic (Some MT) | | [anli](anli/README.md) | Adversarial natural language inference tasks designed to test model robustness. | English |
| [arabic_leaderboard_light](arabic_leaderboard_light/README.md) | A light version of the tasks in the Open Arabic LLM Leaderboard (i.e., 10% samples of the test set in the original benchmarks), focusing on the evaluation of models that reflect the characteristics of Arabic language understanding and comprehension, culture, and heritage. Note that some of these tasks are machine-translated. | Arabic (Some MT) | | [arabic_leaderboard_complete](arabic_leaderboard_complete/README.md) | A full version of the tasks in the Open Arabic LLM Leaderboard, focusing on the evaluation of models that reflect the characteristics of Arabic language understanding and comprehension, culture, and heritage. Note that some of these tasks are machine-translated. | Arabic (Some MT) |
| [arabicmmlu](arabicmmlu/README.md) | Localized Arabic version of MMLU with multiple-choice questions from 40 subjects. | Arabic | | [arabic_leaderboard_light](arabic_leaderboard_light/README.md) | A light version of the tasks in the Open Arabic LLM Leaderboard (i.e., 10% samples of the test set in the original benchmarks), focusing on the evaluation of models that reflect the characteristics of Arabic language understanding and comprehension, culture, and heritage. Note that some of these tasks are machine-translated. | Arabic (Some MT) |
| [AraDICE](aradice/README.md) | A collection of multiple tasks carefully designed to evaluate dialectal and cultural capabilities in large language models (LLMs). | Arabic | | [arabicmmlu](arabicmmlu/README.md) | Localized Arabic version of MMLU with multiple-choice questions from 40 subjects. | Arabic |
| [arc](arc/README.md) | Tasks involving complex reasoning over a diverse set of questions. | English | | [ArabCulture](arab_culture/README.md) | Benchmark for evaluating modeles' commonsense cultural knowledge across different 13 different Arab Countries. | Arabic |
| [arithmetic](arithmetic/README.md) | Tasks involving numerical computations and arithmetic reasoning. | English | [AraDICE](aradice/README.md) | A collection of multiple tasks carefully designed to evaluate dialectal and cultural capabilities in large language models (LLMs). | Arabic |
| [asdiv](asdiv/README.md) | Tasks involving arithmetic and mathematical reasoning challenges. | English | | [arc](arc/README.md) | Tasks involving complex reasoning over a diverse set of questions. | English |
| [babi](babi/README.md) | Tasks designed as question and answering challenges based on simulated stories. | English | | [arithmetic](arithmetic/README.md) | Tasks involving numerical computations and arithmetic reasoning. | English |
| [basque_bench](basque_bench/README.md) | Collection of tasks in Basque encompassing various evaluation areas. | Basque | | [asdiv](asdiv/README.md) | Tasks involving arithmetic and mathematical reasoning challenges. | English |
| [basqueglue](basqueglue/README.md) | Tasks designed to evaluate language understanding in Basque language. | Basque | | [babi](babi/README.md) | Tasks designed as question and answering challenges based on simulated stories. | English |
| [bbh](bbh/README.md) | Tasks focused on deep semantic understanding through hypothesization and reasoning. | English, German | | [basque_bench](basque_bench/README.md) | Collection of tasks in Basque encompassing various evaluation areas. | Basque |
| [bbq](bbq/README.md) | A question-answering benchmark designed to measure social biases in language models across various demographic categories and contexts. | English | | [basqueglue](basqueglue/README.md) | Tasks designed to evaluate language understanding in Basque language. | Basque |
| [belebele](belebele/README.md) | Language understanding tasks in a variety of languages and scripts. | Multiple (122 languages) | | [bbh](bbh/README.md) | Tasks focused on deep semantic understanding through hypothesization and reasoning. | English, German |
| benchmarks | General benchmarking tasks that test a wide range of language understanding capabilities. | | | [bbq](bbq/README.md) | A question-answering benchmark designed to measure social biases in language models across various demographic categories and contexts. | English |
| [bertaqa](bertaqa/README.md) | Local Basque cultural trivia QA tests in English and Basque languages. | English, Basque, Basque (MT) | | [belebele](belebele/README.md) | Language understanding tasks in a variety of languages and scripts. | Multiple (122 languages) |
| [bigbench](bigbench/README.md) | Broad tasks from the BIG-bench benchmark designed to push the boundaries of large models. | Multiple | | benchmarks | General benchmarking tasks that test a wide range of language understanding capabilities. | |
| [blimp](blimp/README.md) | Tasks testing grammatical phenomena to evaluate language model's linguistic capabilities. | English | | [bertaqa](bertaqa/README.md) | Local Basque cultural trivia QA tests in English and Basque languages. | English, Basque, Basque (MT) |
| [careqa](careqa/README.md) | Multiple choice and open-ended medical question answering based on the Spanish Specialised Healthcare Training (MIR) exams. | English, Spanish | | [bigbench](bigbench/README.md) | Broad tasks from the BIG-bench benchmark designed to push the boundaries of large models. | Multiple |
| [catalan_bench](catalan_bench/README.md) | Collection of tasks in Catalan encompassing various evaluation areas. | Catalan | | [blimp](blimp/README.md) | Tasks testing grammatical phenomena to evaluate language model's linguistic capabilities. | English |
| [ceval](ceval/README.md) | Tasks that evaluate language understanding and reasoning in an educational context. | Chinese | | [c4](c4/README.md) | Tasks based on a colossal, cleaned version of Common Crawl's web crawl corpus to assess models' language modeling capabilities. | English |
| [cmmlu](cmmlu/README.md) | Multi-subject multiple choice question tasks for comprehensive academic assessment. | Chinese | | [careqa](careqa/README.md) | Multiple choice and open-ended medical question answering based on the Spanish Specialised Healthcare Training (MIR) exams. | English, Spanish |
| code_x_glue | Tasks that involve understanding and generating code across multiple programming languages. | Go, Java, JS, PHP, Python, Ruby | | [catalan_bench](catalan_bench/README.md) | Collection of tasks in Catalan encompassing various evaluation areas. | Catalan |
| [commonsense_qa](commonsense_qa/README.md) | CommonsenseQA, a multiple-choice QA dataset for measuring commonsense knowledge. | English | | [ceval](ceval/README.md) | Tasks that evaluate language understanding and reasoning in an educational context. | Chinese |
| [copal_id](copal_id/README.md) United States | Indonesian causal commonsense reasoning dataset that captures local nuances. | Indonesian | | [cmmlu](cmmlu/README.md) | Multi-subject multiple choice question tasks for comprehensive academic assessment. | Chinese |
| [coqa](coqa/README.md) | Conversational question answering tasks to test dialog understanding. | English | | code_x_glue | Tasks that involve understanding and generating code across multiple programming languages. | Go, Java, JS, PHP, Python, Ruby |
| [crows_pairs](crows_pairs/README.md) | Tasks designed to test model biases in various sociodemographic groups. | English, French | | [commonsense_qa](commonsense_qa/README.md) | CommonsenseQA, a multiple-choice QA dataset for measuring commonsense knowledge. | English |
| csatqa | Tasks related to SAT and other standardized testing questions for academic assessment. | Korean | | [copal_id](copal_id/README.md) United States | Indonesian causal commonsense reasoning dataset that captures local nuances. | Indonesian |
| [darija_bench](darija_bench/README.md) | Traditional NLP tasks (Translation, Summariation, etc..) for Moroccan Darija | Moroccan Darija (some MT) | | [coqa](coqa/README.md) | Conversational question answering tasks to test dialog understanding. | English |
| [darijahellaswag](darijahellaswag/README.md) | Moroccan Darija version of HellaSwag. | Moroccan Darija (MT) | | [crows_pairs](crows_pairs/README.md) | Tasks designed to test model biases in various sociodemographic groups. | English, French |
| [darijammlu](darijammlu/README.md) | Multiple-choice QA in Moroccan Darija (an Arabic dialect). | Moroccan Darija (MT) | | csatqa | Tasks related to SAT and other standardized testing questions for academic assessment. | Korean |
| [drop](drop/README.md) | Tasks requiring numerical reasoning, reading comprehension, and question answering. | English | | [darija_bench](darija_bench/README.md) | Traditional NLP tasks (Translation, Summariation, etc..) for Moroccan Darija | Moroccan Darija (some MT) |
| [eq_bench](eq_bench/README.md) | Tasks focused on equality and ethics in question answering and decision-making. | English | | [darijahellaswag](darijahellaswag/README.md) | Moroccan Darija version of HellaSwag. | Moroccan Darija (MT) |
| [eus_exams](eus_exams/README.md) | Tasks based on various professional and academic exams in the Basque language. | Basque | | [darijammlu](darijammlu/README.md)| Multiple-choice QA in Moroccan Darija (an Arabic dialect). | Moroccan Darija (MT) |
| [eus_proficiency](eus_proficiency/README.md) | Tasks designed to test proficiency in the Basque language across various topics. | Basque | | [drop](drop/README.md) | Tasks requiring numerical reasoning, reading comprehension, and question answering. | English |
| [eus_reading](eus_reading/README.md) | Reading comprehension tasks specifically designed for the Basque language. | Basque | | [eq_bench](eq_bench/README.md) | Tasks focused on equality and ethics in question answering and decision-making. | English |
| [eus_trivia](eus_trivia/README.md) | Trivia and knowledge testing tasks in the Basque language. | Basque | | [eus_exams](eus_exams/README.md) | Tasks based on various professional and academic exams in the Basque language. | Basque |
| [evalita_LLM](evalita_llm/README.md) | A native Italian benchmark with diverse tasks formats and multiple prompts. | Italian | | [eus_proficiency](eus_proficiency/README.md) | Tasks designed to test proficiency in the Basque language across various topics. | Basque |
| [fda](fda/README.md) | Tasks for extracting key-value pairs from FDA documents to test information extraction. | English | | [eus_reading](eus_reading/README.md) | Reading comprehension tasks specifically designed for the Basque language. | Basque |
| [fld](fld/README.md) | Tasks involving free-form and directed dialogue understanding. | English | | [eus_trivia](eus_trivia/README.md) | Trivia and knowledge testing tasks in the Basque language. | Basque |
| [french_bench](french_bench/README.md) | Set of tasks designed to assess language model performance in French. | French | | [evalita_LLM](evalita_llm/README.md) | A native Italian benchmark with diverse tasks formats and multiple prompts. | Italian |
| [galician_bench](galician_bench/README.md) | Collection of tasks in Galician encompassing various evaluation areas. | Galician | | [fda](fda/README.md) | Tasks for extracting key-value pairs from FDA documents to test information extraction. | English |
| [global_mmlu](global_mmlu/README.md) | Collection of culturally sensitive and culturally agnostic MMLU tasks in 15 languages with human translations or post-edits. | Multiple (15 languages) | | [fld](fld/README.md) | Tasks involving free-form and directed dialogue understanding. | English |
| [glue](glue/README.md) | General Language Understanding Evaluation benchmark to test broad language abilities. | English | | [french_bench](french_bench/README.md) | Set of tasks designed to assess language model performance in French. | French |
| [gpqa](gpqa/README.md) | Tasks designed for general public question answering and knowledge verification. | English | | [galician_bench](galician_bench/README.md) | Collection of tasks in Galician encompassing various evaluation areas. | Galician |
| [gsm8k](gsm8k/README.md) | A benchmark of grade school math problems aimed at evaluating reasoning capabilities. | English | | [global_mmlu](global_mmlu/README.md) | Collection of culturally sensitive and culturally agnostic MMLU tasks in 15 languages with human translations or post-edits. | Multiple (15 languages) |
| [groundcocoa](groundcocoa/README.md) | A benchmark evaluating the conditional and compositional reasoning of language models using a grounding task. | English | | [glue](glue/README.md) | General Language Understanding Evaluation benchmark to test broad language abilities. | English |
| [haerae](haerae/README.md) | Tasks focused on assessing detailed factual and historical knowledge. | Korean | | [gpqa](gpqa/README.md) | Tasks designed for general public question answering and knowledge verification. | English |
| [headqa](headqa/README.md) | A high-level education-based question answering dataset to test specialized knowledge. | Spanish, English | | [gsm8k](gsm8k/README.md) | A benchmark of grade school math problems aimed at evaluating reasoning capabilities. | English |
| [hellaswag](hellaswag/README.md) | Tasks to predict the ending of stories or scenarios, testing comprehension and creativity. | English | | [groundcocoa](groundcocoa/README.md) | A benchmark evaluating the conditional and compositional reasoning of language models using a grounding task. | English |
| [hendrycks_ethics](hendrycks_ethics/README.md) | Tasks designed to evaluate the ethical reasoning capabilities of models. | English | | [haerae](haerae/README.md) | Tasks focused on assessing detailed factual and historical knowledge. | Korean |
| [hendrycks_math](hendrycks_math/README.md) | Mathematical problem-solving tasks to test numerical reasoning and problem-solving. | English | | [headqa](headqa/README.md) | A high-level education-based question answering dataset to test specialized knowledge. | Spanish, English |
| [histoires_morales](histoires_morales/README.md) | A dataset of structured narratives that describe normative and norm-divergent actions taken by individuals to accomplish certain intentions in concrete situations. | French (Some MT) | | [hellaswag](hellaswag/README.md) | Tasks to predict the ending of stories or scenarios, testing comprehension and creativity. | English |
| [hrm8k](hrm8k/README.md) | A challenging bilingual math reasoning benchmark for Korean and English. | Korean (Some MT), English (Some MT) | | [hendrycks_ethics](hendrycks_ethics/README.md) | Tasks designed to evaluate the ethical reasoning capabilities of models. | English |
| [humaneval](humaneval/README.md) | Code generation task that measure functional correctness for synthesizing programs from docstrings. | Python | | [hendrycks_math](hendrycks_math/README.md) | Mathematical problem-solving tasks to test numerical reasoning and problem-solving. | English |
| [ifeval](ifeval/README.md) | Interactive fiction evaluation tasks for narrative understanding and reasoning. | English | | [histoires_morales](histoires_morales/README.md) | A dataset of structured narratives that describe normative and norm-divergent actions taken by individuals to accomplish certain intentions in concrete situations. | French (Some MT) |
| [inverse_scaling](inverse_scaling/README.md) | Multiple-choice tasks from the Inverse Scaling Prize, designed to find settings where larger language models perform worse. | English | | [hrm8k](hrm8k/README.md) | A challenging bilingual math reasoning benchmark for Korean and English. | Korean (Some MT), English (Some MT) |
| [japanese_leaderboard](japanese_leaderboard/README.md) | Japanese language understanding tasks to benchmark model performance on various linguistic aspects. | Japanese | | [humaneval](humaneval/README.md) | Code generation task that measure functional correctness for synthesizing programs from docstrings. | Python |
| [jsonschema_bench](jsonschema_bench/README.md) | Evaluate the ability of LLMs to generate JSON objects that conform to a given JSON schema, including API, configuration files, and other structured data formats. | JSON | | [ifeval](ifeval/README.md) | Interactive fiction evaluation tasks for narrative understanding and reasoning. | English |
| [kbl](kbl/README.md) | Korean Benchmark for Legal Language Understanding. | Korean | | [inverse_scaling](inverse_scaling/README.md) | Multiple-choice tasks from the Inverse Scaling Prize, designed to find settings where larger language models perform worse. | English |
| [kmmlu](kmmlu/README.md) | Knowledge-based multi-subject multiple choice questions for academic evaluation. | Korean | | [japanese_leaderboard](japanese_leaderboard/README.md) | Japanese language understanding tasks to benchmark model performance on various linguistic aspects. | Japanese |
| [kobest](kobest/README.md) | A collection of tasks designed to evaluate understanding in Korean language. | Korean | | [jsonschema_bench](jsonschema_bench/README.md) | Evaluate the ability of LLMs to generate JSON objects that conform to a given JSON schema, including API, configuration files, and other structured data formats. | JSON |
| [kormedmcqa](kormedmcqa/README.md) | Medical question answering tasks in Korean to test specialized domain knowledge. | Korean | | [kbl](kbl/README.md) | Korean Benchmark for Legal Language Understanding. | Korean |
| [lambada](lambada/README.md) | Tasks designed to predict the endings of text passages, testing language prediction skills. | English | | [kmmlu](kmmlu/README.md) | Knowledge-based multi-subject multiple choice questions for academic evaluation. | Korean |
| [lambada_cloze](lambada_cloze/README.md) | Cloze-style LAMBADA dataset. | English | | [kobest](kobest/README.md) | A collection of tasks designed to evaluate understanding in Korean language. | Korean |
| [lambada_multilingual](lambada_multilingual/README.md) | Multilingual LAMBADA dataset. This is a legacy version of the multilingual dataset, and users should instead use `lambada_multilingual_stablelm`. | German, English, Spanish, French, Italian | | [kormedmcqa](kormedmcqa/README.md) | Medical question answering tasks in Korean to test specialized domain knowledge. | Korean |
| [lambada_multilingual_stablelm](lambada_multilingual_stablelm/README.md) | Multilingual LAMBADA dataset. Users should prefer evaluating on this version of the multilingual dataset instead of on `lambada_multilingual`. | German, English, Spanish, French, Italian, Dutch, Portuguese | | [lambada](lambada/README.md) | Tasks designed to predict the endings of text passages, testing language prediction skills. | English |
| [leaderboard](leaderboard/README.md) | Task group used by Hugging Face's [Open LLM Leaderboard v2](https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard). Those tasks are static and will not change through time | English | | [lambada_cloze](lambada_cloze/README.md) | Cloze-style LAMBADA dataset. | English |
| [lingoly](lingoly/README.md) | Challenging logical reasoning benchmark in low-resource languages with controls for memorization | English, Multilingual | | [lambada_multilingual](lambada_multilingual/README.md) | Multilingual LAMBADA dataset. This is a legacy version of the multilingual dataset, and users should instead use `lambada_multilingual_stablelm`. | German, English, Spanish, French, Italian |
| [llama3](llama3/README.md) | Evals reproducing those provided by the LLAMA team in the Hugging Face repo (instruct) | English, Multilingual | | [lambada_multilingual_stablelm](lambada_multilingual_stablelm/README.md) | Multilingual LAMBADA dataset. Users should prefer evaluating on this version of the multilingual dataset instead of on `lambada_multilingual`. | German, English, Spanish, French, Italian, Dutch, Portuguese |
| [logiqa](logiqa/README.md) | Logical reasoning tasks requiring advanced inference and deduction. | English, Chinese | | [leaderboard](leaderboard/README.md) | Task group used by Hugging Face's [Open LLM Leaderboard v2](https://huggingface.co/spaces/open-llm-leaderboard/open_llm_leaderboard). Those tasks are static and will not change through time | English |
| [logiqa2](logiqa2/README.md) | Large-scale logical reasoning dataset adapted from the Chinese Civil Service Examination. | English, Chinese | | [lingoly](lingoly/README.md) | Challenging logical reasoning benchmark in low-resource languages with controls for memorization | English, Multilingual |
| [longbench](longbench/README.md) | LongBench evaluates language models' ability to understand lengthy texts across multiple tasks and languages. | English, Chinese | | [logiqa](logiqa/README.md) | Logical reasoning tasks requiring advanced inference and deduction. | English, Chinese |
| [mastermind](mastermind/README.md) | Reasoning benchmark based on the board game of Mastermind. | English | | [logiqa2](logiqa2/README.md) | Large-scale logical reasoning dataset adapted from the Chinese Civil Service Examination. | English, Chinese |
| [mathqa](mathqa/README.md) | Question answering tasks involving mathematical reasoning and problem-solving. | English | | [mastermind](mastermind/README.md) | Reasoning benchmark based on the board game of Mastermind. | English |
| [mbpp](mbpp/README.md) | A benchmark designed to measure the ability to synthesize short Python programs from natural language descriptions. | Python | | [mathqa](mathqa/README.md) | Question answering tasks involving mathematical reasoning and problem-solving. | English |
| [meddialog](meddialog/README.md) | Medical open-ended QA and Question Entailment stemming from the MedDialog dataset. | English | | [mbpp](mbpp/README.md) | A benchmark designed to measure the ability to synthesize short Python programs from natural language descriptions. | Python |
| [medtext](medtext/README.md) | Medical open-ended QA from the MedText Clinical Notes dataset. | English | | [meddialog](meddialog/README.md) | Medical open-ended QA and Question Entailment stemming from the MedDialog dataset. | English |
| [mimic_repsum](mimic_repsum/README.md) | Medical report summarization from the MIMIC-III dataset. | English | | [medtext](medtext/README.md) | Medical open-ended QA from the MedText Clinical Notes dataset. | English |
| [mc_taco](mc_taco/README.md) | Question-answer pairs that require temporal commonsense comprehension. | English | | [mimic_repsum](mimic_repsum/README.md) | Medical report summarization from the MIMIC-III dataset. | English |
| [med_concepts_qa](med_concepts_qa/README.md) | Benchmark for evaluating LLMs on their abilities to interpret medical codes and distinguish between medical concept. | English | | [mc_taco](mc_taco/README.md) | Question-answer pairs that require temporal commonsense comprehension. | English |
| [metabench](metabench/README.md) | Distilled versions of six popular benchmarks which are highly predictive of overall benchmark performance and of a single general ability latent trait. | English | | [med_concepts_qa](med_concepts_qa/README.md) | Benchmark for evaluating LLMs on their abilities to interpret medical codes and distinguish between medical concept. | English |
| [mediqa_qa2019](mediqa_qa2019/README.md) | Open-ended healthcare question answering benchmark from the MEDIQA 2019 challenge. | English | | [metabench](metabench/README.md) | Distilled versions of six popular benchmarks which are highly predictive of overall benchmark performance and of a single general ability latent trait. | English |
| medmcqa | Medical multiple choice questions assessing detailed medical knowledge. | English | | [mediqa_qa2019](mediqa_qa2019/README.md) | Open-ended healthcare question answering benchmark from the MEDIQA 2019 challenge. | English |
| medqa | Multiple choice question answering based on the United States Medical License Exams. | | | medmcqa | Medical multiple choice questions assessing detailed medical knowledge. | English |
| [meqsum](meqsum/README.md) | Healtcare Question Entailment benchmark from the MeqSum dataset. | | | medqa | Multiple choice question answering based on the United States Medical License Exams. | |
| [mgsm](mgsm/README.md) | Benchmark of multilingual grade-school math problems. | Spanish, French, German, Russian, Chinese, Japanese, Thai, Swahili, Bengali, Telugu | | [meqsum](meqsum/README.md) | Healtcare Question Entailment benchmark from the MeqSum dataset. | |
| [minerva_math](minerva_math/README.md) | Mathematics-focused tasks requiring numerical reasoning and problem-solving skills. | English | | [mgsm](mgsm/README.md) | Benchmark of multilingual grade-school math problems. | Spanish, French, German, Russian, Chinese, Japanese, Thai, Swahili, Bengali, Telugu |
| [mlqa](mlqa/README.md) | MultiLingual Question Answering benchmark dataset for evaluating cross-lingual question answering performance. | English, Arabic, German, Spanish, Hindi, Vietnamese, Simplified Chinese | | [minerva_math](minerva_math/README.md) | Mathematics-focused tasks requiring numerical reasoning and problem-solving skills. | English |
| [mmlu](mmlu/README.md) | Massive Multitask Language Understanding benchmark for broad domain language evaluation. Several variants are supported. | English | | [mlqa](mlqa/README.md) | MultiLingual Question Answering benchmark dataset for evaluating cross-lingual question answering performance. | English, Arabic, German, Spanish, Hindi, Vietnamese, Simplified Chinese |
| [mmlu_pro](mmlu_pro/README.md) | A refined set of MMLU, integrating more challenging, reasoning-focused questions and expanding the choice set from four to ten options. | English | | [mmlu](mmlu/README.md) | Massive Multitask Language Understanding benchmark for broad domain language evaluation. Several variants are supported. | English |
| [mmlu-pro-plus](mmlu-pro-plus/README.md) | A new test set for evaluating shortcut learning and higher-order reasoning of LLMs. | English | | [mmlu_pro](mmlu_pro/README.md) | A refined set of MMLU, integrating more challenging, reasoning-focused questions and expanding the choice set from four to ten options. | English |
| [mmlu_prox](mmlu_prox/README.md) | A multilingual benchmark that extends MMLU-Pro to multiple typologically diverse languages with human validation. | English, Japanese, Chinese, Korean, French, German, Spanish, Portuguese, Swahili, Thai, Arabic, Hindi, Bengali | | [mmlu-pro-plus](mmlu-pro-plus/README.md) | A new test set for evaluating shortcut learning and higher-order reasoning of LLMs. | English |
| [mmlusr](mmlusr/README.md) | Variation of MMLU designed to be more rigorous. | English | | [mmlu_prox](mmlu_prox/README.md) | A multilingual benchmark that extends MMLU-Pro to multiple typologically diverse languages with human validation. | English, Japanese, Chinese, Korean, French, German, Spanish, Portuguese, Swahili, Thai, Arabic, Hindi, Bengali |
| model_written_evals | Evaluation tasks auto-generated for evaluating a collection of AI Safety concerns. | | | [mmlusr](mmlusr/README.md) | Variation of MMLU designed to be more rigorous. | English |
| [moral_stories](moral_stories/README.md) | A crowd-sourced dataset of structured narratives that describe normative and norm-divergent actions taken by individuals to accomplish certain intentions in concrete situations. | English | | model_written_evals | Evaluation tasks auto-generated for evaluating a collection of AI Safety concerns. | |
| [mts_dialog](mts_dialog/README.md) | Open-ended healthcare QA from the MTS-Dialog dataset. | English | | [moral_stories](moral_stories/README.md) | A crowd-sourced dataset of structured narratives that describe normative and norm-divergent actions taken by individuals to accomplish certain intentions in concrete situations. | English |
| [mutual](mutual/README.md) | A retrieval-based dataset for multi-turn dialogue reasoning. | English | | [mts_dialog](mts_dialog/README.md) | Open-ended healthcare QA from the MTS-Dialog dataset. | English |
| [nq_open](nq_open/README.md) | Open domain question answering tasks based on the Natural Questions dataset. | English | | [mutual](mutual/README.md) | A retrieval-based dataset for multi-turn dialogue reasoning. | English |
| [okapi/arc_multilingual](okapi/arc_multilingual/README.md) | Tasks that involve reading comprehension and information retrieval challenges. | Multiple (31 languages) **Machine Translated.** | | [noreval](noreval/README.md) | A human-created Norwegian language understanding and generation benchmark. | Norwegian (Bokmål and Nynorsk) |
| [okapi/hellaswag_multilingual](okapi/hellaswag_multilingual/README.md) | Tasks that involve reading comprehension and information retrieval challenges. | Multiple (30 languages) **Machine Translated.** | | [nq_open](nq_open/README.md) | Open domain question answering tasks based on the Natural Questions dataset. | English |
| okapi/mmlu_multilingual | Tasks that involve reading comprehension and information retrieval challenges. | Multiple (34 languages) **Machine Translated.** | | [okapi/arc_multilingual](okapi/arc_multilingual/README.md) | Tasks that involve reading comprehension and information retrieval challenges. | Multiple (31 languages) **Machine Translated.** |
| [okapi/truthfulqa_multilingual](okapi/truthfulqa_multilingual/README.md) | Tasks that involve reading comprehension and information retrieval challenges. | Multiple (31 languages) **Machine Translated.** | | [okapi/hellaswag_multilingual](okapi/hellaswag_multilingual/README.md) | Tasks that involve reading comprehension and information retrieval challenges. | Multiple (30 languages) **Machine Translated.** |
| [olaph](olaph/README.md) | Open-ended medical factuality Question Answering from the OLAPH dataset. | English | | okapi/mmlu_multilingual | Tasks that involve reading comprehension and information retrieval challenges. | Multiple (34 languages) **Machine Translated.** |
| [openbookqa](openbookqa/README.md) | Open-book question answering tasks that require external knowledge and reasoning. | English | | [okapi/truthfulqa_multilingual](okapi/truthfulqa_multilingual/README.md) | Tasks that involve reading comprehension and information retrieval challenges. | Multiple (31 languages) **Machine Translated.** |
| [paloma](paloma/README.md) | Paloma is a comprehensive benchmark designed to evaluate open language models across a wide range of domains, ranging from niche artist communities to mental health forums on Reddit. | English | | [olaph](olaph/README.md) | Open-ended medical factuality Question Answering from the OLAPH dataset. | English |
| [paws-x](paws-x/README.md) | Paraphrase Adversaries from Word Scrambling, focusing on cross-lingual capabilities. | English, French, Spanish, German, Chinese, Japanese, Korean | | [openbookqa](openbookqa/README.md) | Open-book question answering tasks that require external knowledge and reasoning. | English |
| [pile](pile/README.md) | Open source language modelling data set that consists of 22 smaller, high-quality datasets. | English | | [paloma](paloma/README.md) | Paloma is a comprehensive benchmark designed to evaluate open language models across a wide range of domains, ranging from niche artist communities to mental health forums on Reddit. | English |
| [pile_10k](pile_10k/README.md) | The first 10K elements of The Pile, useful for debugging models trained on it. | English | | [paws-x](paws-x/README.md) | Paraphrase Adversaries from Word Scrambling, focusing on cross-lingual capabilities. | English, French, Spanish, German, Chinese, Japanese, Korean |
| [piqa](piqa/README.md) | Physical Interaction Question Answering tasks to test physical commonsense reasoning. | English | | [pile](pile/README.md) | Open source language modelling data set that consists of 22 smaller, high-quality datasets. | English |
| [polemo2](polemo2/README.md) | Sentiment analysis and emotion detection tasks based on Polish language data. | Polish | | [pile_10k](pile_10k/README.md) | The first 10K elements of The Pile, useful for debugging models trained on it. | English |
| [portuguese_bench](portuguese_bench/README.md) | Collection of tasks in European Portuguese encompassing various evaluation areas. | Portuguese | | [piqa](piqa/README.md) | Physical Interaction Question Answering tasks to test physical commonsense reasoning. | English |
| [prost](prost/README.md) | Tasks requiring understanding of professional standards and ethics in various domains. | English | | [polemo2](polemo2/README.md) | Sentiment analysis and emotion detection tasks based on Polish language data. | Polish |
| [pubmedqa](pubmedqa/README.md) | Question answering tasks based on PubMed research articles for biomedical understanding. | English | | [portuguese_bench](portuguese_bench/README.md) | Collection of tasks in European Portuguese encompassing various evaluation areas. | Portuguese |
| [qa4mre](qa4mre/README.md) | Question Answering for Machine Reading Evaluation, assessing comprehension and reasoning. | English | | [prost](prost/README.md) | Tasks requiring understanding of professional standards and ethics in various domains. | English |
| [qasper](qasper/README.md) | Question Answering dataset based on academic papers, testing in-depth scientific knowledge. | English | | [pubmedqa](pubmedqa/README.md) | Question answering tasks based on PubMed research articles for biomedical understanding. | English |
| [race](race/README.md) | Reading comprehension assessment tasks based on English exams in China. | English | | [qa4mre](qa4mre/README.md) | Question Answering for Machine Reading Evaluation, assessing comprehension and reasoning. | English |
| realtoxicityprompts | Tasks to evaluate language models for generating text with potential toxicity. | | | [qasper](qasper/README.md) | Question Answering dataset based on academic papers, testing in-depth scientific knowledge. | English |
| [ruler](ruler/README.md) | RULER is a benchmark for testing how well language models handle long pieces of text. Requires custom arg (see readme) | English | | [race](race/README.md) | Reading comprehension assessment tasks based on English exams in China. | English |
| [sciq](sciq/README.md) | Science Question Answering tasks to assess understanding of scientific concepts. | English | | realtoxicityprompts | Tasks to evaluate language models for generating text with potential toxicity. | |
| [score](score/README.md) | Systematic consistency and robustness evaluation for LLMs on 3 datasets(MMLU-Pro, Agi Eval and MATH) | English | | [ruler](ruler/README.md) | RULER is a benchmark for testing how well language models handle long pieces of text. Requires custom arg (see readme) | English |
| [scrolls](scrolls/README.md) | Tasks that involve long-form reading comprehension across various domains. | English | | [sciq](sciq/README.md) | Science Question Answering tasks to assess understanding of scientific concepts. | English |
| [simple_cooccurrence_bias](simple_cooccurrence_bias/README.md) | A metric that evaluates language models for biases based on stereotypical word associations and co-occurrences in text. | English | | [score](score/README.md) | Systematic consistency and robustness evaluation for LLMs on 3 datasets(MMLU-Pro, Agi Eval and MATH) | English |
| [siqa](siqa/README.md) | Social Interaction Question Answering to evaluate common sense and social reasoning. | English | | [scrolls](scrolls/README.md) | Tasks that involve long-form reading comprehension across various domains. | English |
| [spanish_bench](spanish_bench/README.md) | Collection of tasks in Spanish encompassing various evaluation areas. | Spanish | | [simple_cooccurrence_bias](simple_cooccurrence_bias/README.md) | A metric that evaluates language models for biases based on stereotypical word associations and co-occurrences in text. | English |
| [squad_completion](squad_completion/README.md) | A variant of the SQuAD question answering task designed for zero-shot evaluation of small LMs. | English | | [siqa](siqa/README.md) | Social Interaction Question Answering to evaluate common sense and social reasoning. | English |
| [squadv2](squadv2/README.md) | Stanford Question Answering Dataset version 2, a reading comprehension benchmark. | English | | [spanish_bench](spanish_bench/README.md) | Collection of tasks in Spanish encompassing various evaluation areas. | Spanish |
| [storycloze](storycloze/README.md) | Tasks to predict story endings, focusing on narrative logic and coherence. | English | | [squad_completion](squad_completion/README.md) | A variant of the SQuAD question answering task designed for zero-shot evaluation of small LMs. | English |
| [super_glue](super_glue/README.md) | A suite of challenging tasks designed to test a range of language understanding skills. | English | | [squadv2](squadv2/README.md) | Stanford Question Answering Dataset version 2, a reading comprehension benchmark. | English |
| [swag](swag/README.md) | Situations With Adversarial Generations, predicting the next event in videos. | English | | [storycloze](storycloze/README.md) | Tasks to predict story endings, focusing on narrative logic and coherence. | English |
| [swde](swde/README.md) | Information extraction tasks from semi-structured web pages. | English | | [super_glue](super_glue/README.md) | A suite of challenging tasks designed to test a range of language understanding skills. | English |
| [tinyBenchmarks](tinyBenchmarks/README.md) | Evaluation of large language models with fewer examples using tiny versions of popular benchmarks. | English | | [swag](swag/README.md) | Situations With Adversarial Generations, predicting the next event in videos. | English |
| [tmmluplus](tmmluplus/README.md) | An extended set of tasks under the TMMLU framework for broader academic assessments. | Traditional Chinese | | [swde](swde/README.md) | Information extraction tasks from semi-structured web pages. | English |
| [toxigen](toxigen/README.md) | Tasks designed to evaluate language models on their propensity to generate toxic content. | English | | [tinyBenchmarks](tinyBenchmarks/README.md) | Evaluation of large language models with fewer examples using tiny versions of popular benchmarks. | English |
| [translation](translation/README.md) | Tasks focused on evaluating the language translation capabilities of models. | Arabic, English, Spanish, Basque, Hindi, Indonesian, Burmese, Russian, Swahili, Telugu, Chinese | | [tmmluplus](tmmluplus/README.md) | An extended set of tasks under the TMMLU framework for broader academic assessments. | Traditional Chinese |
| [triviaqa](triviaqa/README.md) | A large-scale dataset for trivia question answering to test general knowledge. | English | | [toxigen](toxigen/README.md) | Tasks designed to evaluate language models on their propensity to generate toxic content. | English |
| [truthfulqa](truthfulqa/README.md) | A QA task aimed at evaluating the truthfulness and factual accuracy of model responses. | English | | [translation](translation/README.md) | Tasks focused on evaluating the language translation capabilities of models. | Arabic, English, Spanish, Basque, Hindi, Indonesian, Burmese, Russian, Swahili, Telugu, Chinese |
| [turkishmmlu](turkishmmlu/README.md) | A multiple-choice QA test modeled after MMLU, written in Turkish based on Turkish high-school level exams. | Turkish | | [triviaqa](triviaqa/README.md) | A large-scale dataset for trivia question answering to test general knowledge. | English |
| [unitxt](unitxt/README.md) | A number of tasks implemented using the unitxt library for flexible, shareable, and reusable data preparation and evaluation for generative AI. | English | | [truthfulqa](truthfulqa/README.md) | A QA task aimed at evaluating the truthfulness and factual accuracy of model responses. | English |
| [unscramble](unscramble/README.md) | Tasks involving the rearrangement of scrambled sentences to test syntactic understanding. | English | | [turkishmmlu](turkishmmlu/README.md) | A multiple-choice QA test modeled after MMLU, written in Turkish based on Turkish high-school level exams. | Turkish |
| [webqs](webqs/README.md) | Web-based question answering tasks designed to evaluate internet search and retrieval. | English | | [unitxt](unitxt/README.md) | A number of tasks implemented using the unitxt library for flexible, shareable, and reusable data preparation and evaluation for generative AI. | English |
| [wikitext](wikitext/README.md) | Tasks based on text from Wikipedia articles to assess language modeling and generation. | English | | [unscramble](unscramble/README.md) | Tasks involving the rearrangement of scrambled sentences to test syntactic understanding. | English |
| [winogender](winogender/README.md) | A diagnostic dataset that tests for gender bias in coreference resolution by measuring how models associate pronouns with different occupations. | English | | [webqs](webqs/README.md) | Web-based question answering tasks designed to evaluate internet search and retrieval. | English |
| [winogrande](winogrande/README.md) | A large-scale dataset for coreference resolution, inspired by the Winograd Schema Challenge. | English | | [wikitext](wikitext/README.md) | Tasks based on text from Wikipedia articles to assess language modeling and generation. | English |
| [wmdp](wmdp/README.md) | A benchmark with the objective of minimizing performance, based on potentially-sensitive multiple-choice knowledge questions. | English | | [winogender](winogender/README.md) | A diagnostic dataset that tests for gender bias in coreference resolution by measuring how models associate pronouns with different occupations. | English |
| [wmt2016](wmt2016/README.md) | Tasks from the WMT 2016 shared task, focusing on translation between multiple languages. | English, Czech, German, Finnish, Russian, Romanian, Turkish | | [winogrande](winogrande/README.md) | A large-scale dataset for coreference resolution, inspired by the Winograd Schema Challenge. | English |
| [wsc273](wsc273/README.md) | The Winograd Schema Challenge, a test of commonsense reasoning and coreference resolution. | English | | [wmdp](wmdp/README.md) | A benchmark with the objective of minimizing performance, based on potentially-sensitive multiple-choice knowledge questions. | English |
| [xcopa](xcopa/README.md) | Cross-lingual Choice of Plausible Alternatives, testing reasoning in multiple languages. | Estonian, Haitian, Indonesian, Italian, Quechua, Swahili, Tamil, Thai, Turkish, Vietnamese, Chinese | | [wmt2016](wmt2016/README.md) | Tasks from the WMT 2016 shared task, focusing on translation between multiple languages. | English, Czech, German, Finnish, Russian, Romanian, Turkish |
| [wsc273](wsc273/README.md) | The Winograd Schema Challenge, a test of commonsense reasoning and coreference resolution. | English |
| [xcopa](xcopa/README.md) | Cross-lingual Choice of Plausible Alternatives, testing reasoning in multiple languages. | Estonian, Haitian, Indonesian, Italian, Quechua, Swahili, Tamil, Thai, Turkish, Vietnamese, Chinese |
| [xnli](xnli/README.md) | Cross-Lingual Natural Language Inference to test understanding across different languages. | Arabic, Bulgarian, German, Greek, English, Spanish, French, Hindi, Russian, Swahili, Thai, Turkish, Urdu, Vietnamese, Chinese | | [xnli](xnli/README.md) | Cross-Lingual Natural Language Inference to test understanding across different languages. | Arabic, Bulgarian, German, Greek, English, Spanish, French, Hindi, Russian, Swahili, Thai, Turkish, Urdu, Vietnamese, Chinese |
| [xnli_eu](xnli_eu/README.md) | Cross-lingual Natural Language Inference tasks in Basque. | Basque | | [xnli_eu](xnli_eu/README.md) | Cross-lingual Natural Language Inference tasks in Basque. | Basque |
| [xquad](xquad/README.md) | Cross-lingual Question Answering Dataset in multiple languages. | Arabic, German, Greek, English, Spanish, Hindi, Romanian, Russian, Thai, Turkish, Vietnamese, Chinese | | [xquad](xquad/README.md) | Cross-lingual Question Answering Dataset in multiple languages. | Arabic, German, Greek, English, Spanish, Hindi, Romanian, Russian, Thai, Turkish, Vietnamese, Chinese |
| [xstorycloze](xstorycloze/README.md) | Cross-lingual narrative understanding tasks to predict story endings in multiple languages. | Russian, Simplified Chinese, Spanish, Arabic, Hindi, Indonesian, Telugu, Swahili, Basque, Burmese | | [xstorycloze](xstorycloze/README.md) | Cross-lingual narrative understanding tasks to predict story endings in multiple languages. | Russian, Simplified Chinese, Spanish, Arabic, Hindi, Indonesian, Telugu, Swahili, Basque, Burmese |
| [xwinograd](xwinograd/README.md) | Cross-lingual Winograd schema tasks for coreference resolution in multiple languages. | English, French, Japanese, Portuguese, Russian, Chinese | | [xwinograd](xwinograd/README.md) | Cross-lingual Winograd schema tasks for coreference resolution in multiple languages. | English, French, Japanese, Portuguese, Russian, Chinese |
## Multimodal Tasks ## Multimodal Tasks
| Task Family | Description | Modality | | Task Family | Description | Modality |
......
lm_eval/tasks/aclue/_generate_configs.py
View file @ 601be343
...@@ -3,12 +3,14 @@ Take in a YAML, and output all other splits with this YAML ...@@ -3,12 +3,14 @@ Take in a YAML, and output all other splits with this YAML
""" """
import argparse import argparse
import logging
import os import os
import yaml import yaml
from tqdm import tqdm from tqdm import tqdm
from lm_eval.utils import eval_logger
eval_logger = logging.getLogger(__name__)
SUBJECTS = { SUBJECTS = {
......
lm_eval/tasks/acpbench/README.md
View file @ 601be343
# ACPBench # ACPBench
### Paper **Homepage:** https://ibm.github.io/ACPBench/
Title: ACPBench: Reasoning About Action, Change, and Planning ### Papers
Abstract: https://arxiv.org/pdf/2410.05669
**Title:** ACPBench: Reasoning About Action, Change, and Planning
**Pdf:** https://arxiv.org/pdf/2410.05669
**Task:** `acp_bench`
**Abstract:**
There is an increasing body of work using Large Language Models (LLMs) as agents for orchestrating workflows and making decisions in domains that require planning and multi-step reasoning. As a result, it is imperative to evaluate LMs on core skills required for planning. ACPBench is a benchmark for evaluating the reasoning tasks in the field of planning. The benchmark consists of 7 reasoning tasks over 13 planning domains. The collection is constructed from planning domains described in a formal language. This allows the synthesized problems to have provably correct solutions across many tasks and domains. Further, it allows the luxury to scale without additional human effort, i.e., many additional problems can be created automatically. There is an increasing body of work using Large Language Models (LLMs) as agents for orchestrating workflows and making decisions in domains that require planning and multi-step reasoning. As a result, it is imperative to evaluate LMs on core skills required for planning. ACPBench is a benchmark for evaluating the reasoning tasks in the field of planning. The benchmark consists of 7 reasoning tasks over 13 planning domains. The collection is constructed from planning domains described in a formal language. This allows the synthesized problems to have provably correct solutions across many tasks and domains. Further, it allows the luxury to scale without additional human effort, i.e., many additional problems can be created automatically.
Homepage: https://ibm.github.io/ACPBench/
**Title:** ACPBench Hard: Unrestrained Reasoning about Action, Change, and Planning
**Pdf:** https://arxiv.org/abs/2503.24378
**Task:** `acp_bench_hard`
**Abstract:**
We introduce ACPBench Hard, a dataset of generative, open-ended questions which LLM models needs to answer in order to plan. Models that perform well on these tasks could in principle be integrated into a planner or be used directly as a policy. We discuss the complexity of these tasks as well as the complexity of validating the correctness of their answers and present validation algorithms for each task. Equipped with these validators, we test the performance of a variety of models on our tasks and find that for most of these tasks, the performance of even the largest models is still subpar. Our experiments show that no model outperforms any other in these tasks, and with a few exceptions, all tested language models score below 65\%, indicating that even the current frontier language models as well as so-called reasoning models have a long way to go before they can reliably reason about planning.
The dataset is available on [HuggingFace](https://huggingface.co/datasets/ibm-research/acp_bench).
### Citation ### Citation
...@@ -23,6 +36,19 @@ Homepage: https://ibm.github.io/ACPBench/ ...@@ -23,6 +36,19 @@ Homepage: https://ibm.github.io/ACPBench/
publisher = {{AAAI} Press}, publisher = {{AAAI} Press},
year = {2025} year = {2025}
} }
@misc{KokelKSS25ACPHard,
title = {ACPBench Hard: Unrestrained Reasoning about Action, Change, and Planning},
author = {Harsha Kokel and
Michael Katz and
Kavitha Srinivas and
Shirin Sohrabi},
year = {2025},
eprint = {2503.24378},
archivePrefix = {arXiv},
primaryClass = {cs.AI},
url = {https://arxiv.org/abs/2503.24378},
}
``` ```
### Groups, Tags, and Tasks ### Groups, Tags, and Tasks
...@@ -33,9 +59,13 @@ Homepage: https://ibm.github.io/ACPBench/ ...@@ -33,9 +59,13 @@ Homepage: https://ibm.github.io/ACPBench/
#### Tags #### Tags
* `acp_bench` : Evaluates `acp_bool_cot_2shot` and `acp_mcq_cot_2shot` * `acp_bench` : Evaluates `acp_bool_cot_2shot` and `acp_mcq_cot_2shot` (Main variant for ACPBench paper)
* `acp_bool_cot_2shot` : Evaluates `acp_areach_bool`, `acp_app_bool`, `acp_just_bool`, `acp_land_bool`, `acp_prog_bool`, `acp_reach_bool`, `acp_val_bool` with chain-of-thought and 2 shots * `acp_bool_cot_2shot` : Evaluates `acp_areach_bool`, `acp_app_bool`, `acp_just_bool`, `acp_land_bool`, `acp_prog_bool`, `acp_reach_bool`, `acp_val_bool` with chain-of-thought and 2 shots
* `acp_mcq_cot_2shot` : Evaluates `acp_areach_mcq`, `acp_app_mcq`, `acp_just_mcq`, `acp_land_mcq`, `acp_prog_mcq`, `acp_reach_mcq`, `acp_val_mcq` with chain-of-thought and 2 shots * `acp_mcq_cot_2shot` : Evaluates `acp_areach_mcq`, `acp_app_mcq`, `acp_just_mcq`, `acp_land_mcq`, `acp_prog_mcq`, `acp_reach_mcq`, `acp_val_mcq` with chain-of-thought and 2 shots
* `acp_bench_hard` : Evaluates `acp_gen_2shot` (Main variant for ACPBench Hard paper)
* `acp_gen_2shot` : Evaluates `acp_areach_gen`, `acp_app_gen`, `acp_just_gen`, `acp_land_gen`, `acp_nexta_gen`, `acp_prog_gen`, `acp_reach_gen`, `acp_val_gen` with 2 shots
* `acp_bench_hard_with_pddl` : Evaluates `acp_gen_2shot_with_pddl`
* `acp_gen_2shot_with_pddl` : Evaluates `acp_areach_gen_with_pddl`, `acp_app_gen_with_pddl`, `acp_just_gen_with_pddl`, `acp_land_gen_with_pddl`, `acp_nexta_gen_with_pddl`, `acp_prog_gen_with_pddl`, `acp_reach_gen_with_pddl`, `acp_val_gen_with_pddl` with 2 shots
#### Tasks #### Tasks
...@@ -57,6 +87,26 @@ Homepage: https://ibm.github.io/ACPBench/ ...@@ -57,6 +87,26 @@ Homepage: https://ibm.github.io/ACPBench/
* `acp_reach_mcq` * `acp_reach_mcq`
* `acp_val_mcq` * `acp_val_mcq`
8 Generative tasks (with just natural language description in context)
* `acp_areach_gen`
* `acp_app_gen`
* `acp_just_gen`
* `acp_land_gen`
* `acp_nexta_gen`
* `acp_prog_gen`
* `acp_reach_gen`
* `acp_val_gen`
and the same 8 generative tasks with natural language as well as the PDDL description of the domain and problem in context.
* `acp_areach_gen_with_pddl`
* `acp_app_gen_with_pddl`
* `acp_just_gen_with_pddl`
* `acp_land_gen_with_pddl`
* `acp_nexta_gen_with_pddl`
* `acp_prog_gen_with_pddl`
* `acp_reach_gen_with_pddl`
* `acp_val_gen_with_pddl`
> ! The evaluation scripts are taken from original github https://github.com/IBM/ACPBench > ! The evaluation scripts are taken from original github https://github.com/IBM/ACPBench
...@@ -77,3 +127,4 @@ If other tasks on this dataset are already supported: ...@@ -77,3 +127,4 @@ If other tasks on this dataset are already supported:
### Change Log ### Change Log
* 03/17/2025 Initial Commit * 03/17/2025 Initial Commit
* 05/13/2025 Adding ACPBench Hard tasks (with and without PDDL)
lm_eval/tasks/acpbench/gen_2shot/_gen_yaml_2shot 0 → 100644
View file @ 601be343
tag:
- acp_gen_2shot
- acp_bench_hard
dataset_path: ibm-research/acp_bench
test_split: test
doc_to_target: "{{answer}}"
output_type: generate_until
num_fewshot: 2
generation_kwargs:
until:
- "\n\n\n\n"
- "\n\n"
- "**Question**:"
- "**Question:**"
- "Q:"
do_sample: false
max_gen_toks: 1000
temperature: 0.0
metadata:
version: 1.0
process_results: !function acp_utils.process_acp_results
metric_list:
- metric: "score"
aggregation: mean
higher_is_better: True
lm_eval/tasks/acpbench/gen_2shot/acp_grammar.lark 0 → 100644
View file @ 601be343
NAME: /[a-zA-Z][a-zA-Z0-9-_]*/
LPAR : "("
RPAR : ")"
LSPAR: "["
RSPAR: "]"
COMMA: ","
WS: /[ \n]/
action_none : "None"
action_name : LPAR NAME (WS NAME)* RPAR
action_list : (action_name WS?)*
prog_list : action_name* (COMMA action_name)*
progression_list : LSPAR prog_list RSPAR LSPAR prog_list RSPAR
act : action_name | action_none
index: /[0-9]+[0-9]*/
start: action_list
lm_eval/tasks/acpbench/gen_2shot/acp_utils.py 0 → 100644
View file @ 601be343
import json
import os
from abc import ABC, abstractmethod
from collections import defaultdict
from pathlib import Path
from lm_eval.api.registry import register_filter
from lm_eval.filters.extraction import RegexFilter
try:
import tempfile
import tarski
from kstar_planner import planners as kp
from lark import Lark
from lark.lexer import Token
from lark.visitors import Visitor
from pddl.core import Problem
from pddl.parser.domain import DomainParser
from pddl.parser.problem import ProblemParser
from tarski.grounding.common import StateVariableLite
from tarski.grounding.lp_grounding import LPGroundingStrategy
from tarski.io import PDDLReader
from tarski.io import fstrips as iofs
from tarski.syntax.formulas import is_atom
from tarski.syntax.transform.action_grounding import (
ground_schema_into_plain_operator_from_grounding,
)
from tarski.util import SymbolIndex
except ModuleNotFoundError:
raise ModuleNotFoundError(
"`lark>=1.1.9`, `tarski[clingo]==0.8.2`, `pddl==0.4.2` and `kstar-planner==1.4.2` are required for evaluating the generative tasks. \
Please install via pip install lm-eval[acpbench] or pip install -e .[acpbench]",
)
#########################################################################
# Grammar
GRAMMAR_FILE = os.path.join(
os.path.dirname(os.path.abspath(__file__)), "acp_grammar.lark"
)
class ACPBench_Visitor(Visitor):
def __init__(self) -> None:
super().__init__()
self.action_lists = None
self.action_names = None
self.progression_lists = None
self.prog_lists = None
self.indexes = None
def action_list(self, tree):
self.action_lists = []
def prog_list(self, tree):
if self.prog_lists is not None:
self.progression_lists.append(self.prog_lists)
self.prog_lists = []
def progression_list(self, tree):
self.progression_lists = []
def action_none(self, tree):
self.action_names = "None"
def action_name(self, tree):
act_name = "(" + "".join(tree.children[1:-1]) + ")"
self.action_names = act_name
if self.action_lists is not None:
self.action_lists.append(act_name)
if self.prog_lists is not None:
self.prog_lists.append(act_name)
def index(self, tree):
self.indexes = "".join(tree.children)
if not self.indexes.isnumeric():
self.indexes = None
class ACPGrammarParser(object):
def __init__(self, task) -> None:
self.task = task
with open(GRAMMAR_FILE) as f:
grammar = f.read()
self.acp_parser = Lark(grammar, start=task, parser="lalr")
def parse(self, input, debug=False):
def ignore_errors(e):
if hasattr(e, "token") and e.token.type == "$END":
for x in e.expected:
if x != "WS":
e.interactive_parser.feed_token(
Token(x, self.acp_parser.get_terminal(x).pattern.value)
)
return True
input = input.replace("\n", "")
input = input.strip()
try:
tree = self.acp_parser.parse(input, on_error=ignore_errors)
if debug:
print(tree)
visitor = ACPBench_Visitor()
visitor.visit_topdown(tree)
if self.task == "action_list":
return visitor.action_lists
elif self.task == "act":
return visitor.action_names
elif self.task == "action_name":
return visitor.action_names
elif self.task == "index":
return visitor.indexes
elif self.task == "progression_list":
if visitor.prog_lists not in visitor.progression_lists:
visitor.progression_lists.append(visitor.prog_lists)
return visitor.progression_lists
except Exception as e:
if debug:
print("exception")
print(e)
return None
##############################################################################
# Utils
# Used in next action
def is_on_optimal_plan(domain, problem, action, opt):
with (
tempfile.NamedTemporaryFile() as domain_temp,
tempfile.NamedTemporaryFile() as problem_temp,
):
with open(str(domain_temp.name), "w", encoding="utf8") as file:
file.write(domain.lower())
with open(str(problem_temp.name), "w", encoding="utf8") as file:
file.write(problem.lower())
# Here, we need to keep the temp files live until the end of the function
try:
P = STRIPS(str(domain_temp.name), str(problem_temp.name))
except Exception:
# Unsolvable
return False
a = P.get_action_or_none(action[1:-1])
if a is None:
return False
state = P.init
next_state = progress(state, a)
if opt is None:
# Get an optimal plan cost
plans = generate_optimal_plans_for_problem_state(
P, state, num_plans=1, timeout=5
)
opt = len(plans[0]["actions"])
else:
opt = int(opt)
# Getting an optimal plan for the next state
next_plans = generate_optimal_plans_for_problem_state(
P, next_state, num_plans=1, timeout=5
)
if next_plans is None:
return False
next_opt = len(next_plans[0]["actions"])
return next_opt + 1 == opt
# Used in justification
def is_plan(domain, problem, new_plan):
P = get_STRIPS(domain, problem)
if P is None:
# Unsolvable
return False
# Check if new_plan is a plan
current_state = P.init
for action in new_plan:
applicable_actions = P.get_applicable_actions(current_state)
app_actions_list = [f"({a.name.lower()})" for a in applicable_actions]
if action.lower() not in app_actions_list:
return False
a = applicable_actions[app_actions_list.index(action.lower())]
current_state = progress(current_state, a)
return entails(current_state, P.goal)
# Used in action reachability
def get_action_preconditions(domain, problem, action):
P = get_STRIPS(domain, problem)
assert P is not None, f"Domain\n{domain}\nProblem\n{problem}\nAction: {action}"
a = P.get_action_or_none(action[1:-1])
if a is None:
return a
return [f"({f})" for f in a.pres]
def generate_optimal_plans_for_problem_state(P, state, num_plans, timeout):
import tempfile
with (
tempfile.NamedTemporaryFile() as domain_temp,
tempfile.NamedTemporaryFile() as problem_temp,
):
create_tmp_dom_prob_replace_init(P, state, domain_temp, problem_temp)
plans = generate_top_q_plans(
domain=str(domain_temp.name),
problem=str(problem_temp.name),
num_plans=num_plans,
quality_bound=1.0,
timeout=timeout,
)
# print(plans)
if plans is None or len(plans["plans"]) == 0:
return None
return plans["plans"]
def generate_top_q_plans(domain, problem, num_plans=10, quality_bound=1.0, timeout=30):
# print("Running K* planner")
plans = kp.plan_unordered_topq(
domain_file=Path(domain),
problem_file=Path(problem),
number_of_plans_bound=num_plans,
quality_bound=quality_bound,
timeout=timeout,
)
return plans
# Used in (action) reachability
def is_unsolvable_new_goal(domain, problem, new_goal):
goal = extract_goal(problem)
new_problem = problem.replace(goal, f"(:goal {new_goal} )")
return is_unsolvable(domain, new_problem)
def is_unsolvable(domain, problem):
with (
tempfile.NamedTemporaryFile() as domain_temp,
tempfile.NamedTemporaryFile() as problem_temp,
):
with open(str(domain_temp.name), "w", encoding="utf8") as file:
file.write(str(domain))
with open(str(problem_temp.name), "w", encoding="utf8") as file:
file.write(str(problem))
plans = kp.plan_unordered_topq(
domain_file=Path(str(domain_temp.name)),
problem_file=Path(str(problem_temp.name)),
quality_bound=1.0,
number_of_plans_bound=1,
timeout=3,
)
if len(plans["planner_error"]) > 0:
fl = plans["planner_error"].split("\n")[0]
print(f"Planner error: {fl}")
return False
if plans is None or len(plans["plans"]) == 0:
return plans["unsolvable"]
return False
def extract_goal(prob):
a = prob.split("(:goal")[1]
cp = 1
for i, c in enumerate(a):
if c == ")":
cp -= 1
if c == "(":
cp += 1
if cp == 0:
return "(:goal" + a[: i + 1]
assert False
def entails(state, partialstate):
return partialstate <= state
def progress(state, act):
assert entails(state, act.pres), (
"Cannot progress with inconsistent state / action precondition:\n\t Action: "
+ act.name
+ "\n\t State: \n\t\t"
+ "\n\t\t".join(state)
)
return (state - act.dels) | act.adds
def regress(state, act):
assert len(state & act.dels) == 0, (
"Cannot regress with inconsistent state / action delete effect:\n\t Action: "
+ act.name
+ "\n\t State: \n\t\t"
+ "\n\t\t".join(state)
)
return (state - act.adds) | act.pres
def get_STRIPS(domain, problem):
with (
tempfile.NamedTemporaryFile() as domain_temp,
tempfile.NamedTemporaryFile() as problem_temp,
):
with open(str(domain_temp.name), "w", encoding="utf8") as file:
file.write(domain.lower())
with open(str(problem_temp.name), "w", encoding="utf8") as file:
file.write(problem.lower())
try:
P = STRIPS(str(domain_temp.name), str(problem_temp.name))
return P
except Exception as e:
print(f"||{e}||")
return None
def create_tmp_dom_prob_replace_init(P, state, result_domain_file, result_problem_file):
d, p = P.PDDL_replace_init_pddl_parser(state)
with open(str(result_domain_file.name), "w", encoding="utf8") as file:
file.write(str(d))
with open(str(result_problem_file.name), "w", encoding="utf8") as file:
file.write(str(p))
return d, p
def fix_name(s):
# (act param)
if "(" == s[0] and ")" == s[-1]:
return s[1:-1]
# make it space separated
s = s.replace(", ", " ").replace(",", " ")
# act(param)
if "(" in s:
assert ")" == s[-1], f"Broken name? {s}"
s = s.replace("(", " ").replace(")", "")
# act param
return s
def get_atoms_pddl(d, p, atoms):
objs = set()
preds = defaultdict(list)
for atom in atoms:
a = atom.lower().strip().split(" ")
args = a[1:]
preds[a[0]].append(args)
objs |= set(args)
constants = [o for o in p.objects | d.constants if o.name.lower() in objs]
constants_dict = {}
for c in constants:
constants_dict[c.name.lower()] = c
assert len(objs) == len(constants), (
f"Could not identify all objects: {objs - set(constants_dict.keys())} not found, {set(constants_dict.keys()) - objs} should not be there"
)
state = []
covered_preds = set()
for f in d.predicates:
name = f.name.lower()
if name in preds:
covered_preds.add(name)
assert len(preds[name][0]) == f.arity, (
f"The arity does not match: {preds[name]} vs {f.terms}"
)
# Going over the lists of objects, adding ground predicate for each
for ob in preds[name]:
c = [constants_dict[o] for o in ob]
state.append(f(*c))
assert len(covered_preds) == len(preds.keys()), (
f"Covered predicates: \n{sorted(list(covered_preds))} vs \n{sorted(list(preds.keys()))}"
)
return set(state)
class Action:
def __init__(self, name, pre, add, delete):
self.name = name
self.pres = pre
self.adds = add
self.dels = delete
def __str__(self):
pres = "{" + ", ".join([f"({a})" for a in self.pres]) + "}"
adds = "{" + ", ".join([f"({a})" for a in self.adds]) + "}"
dels = "{" + ", ".join([f"({a})" for a in self.dels]) + "}"
return f"< {self.name}, {pres}, {adds}, {dels} >"
def toJSON(self):
return json.dumps(
{
"name": self.name,
"preconditions": [f"({a})" for a in self.pres],
"add_effects": [f"({a})" for a in self.adds],
"delete_effects": [f"({a})" for a in self.dels],
},
sort_keys=True,
indent=4,
)
def __repr__(self):
return self.name
def __eq__(self, action):
return self.name == action.name
def __hash__(self):
return hash(self.name)
class STRIPS:
def __init__(self, domain, problem):
self.domain_file = domain
self.problem_file = problem
self.reader = PDDLReader(raise_on_error=True)
self.reader.parse_domain(domain)
self.problem = self.reader.parse_instance(problem)
(self.grounded_fluents, init, goal, self.operators, self.grounder) = (
self.ground_problem(self.problem)
)
self.fluents = set([fix_name(str(f)) for f in self.grounded_fluents])
self.fluents_map = dict()
for f in self.grounded_fluents:
self.fluents_map[fix_name(str(f))] = f
self.init = set([fix_name(str(f)) for f in init])
self.goal = set([fix_name(str(f)) for f in goal])
self.actions = set()
self.action_map = {}
self.init_fluents = [self.fluents_map[f] for f in self.init]
self.static_predicates = [i.name for i in self.grounder.static_symbols]
for op in self.operators:
act = self.operator_to_action(op)
self.actions.add(act)
self.action_map[act.name.lower()] = act
def __str__(self):
fluents = "P = {" + ", ".join([f"({a})" for a in self.fluents]) + "}"
init = "I = {" + ", ".join([f"({a})" for a in self.init]) + "}"
goal = "G = {" + ", ".join([f"({a})" for a in self.goal]) + "}"
actions = "A = {" + "\n ".join([a.__str__() for a in self.actions]) + "}"
return fluents + ",\n" + init + "\n" + goal + "\n" + actions
def toJSON(self):
actions = [a.toJSON() for a in self.actions]
return json.dumps(
{
"fluents": list(self.fluents),
"initial_state": list(self.init),
"goal": list(self.goal),
"actions": actions,
},
sort_keys=True,
indent=4,
)
def operator_to_action(self, op, check_fluents=True, check_static=False):
adds = {
fix_name(str(f.atom)) for f in op.effects if isinstance(f, iofs.AddEffect)
} & self.fluents
dels = {
fix_name(str(f.atom)) for f in op.effects if isinstance(f, iofs.DelEffect)
} & self.fluents
pre = self.fix_pre_name(op.precondition)
if check_fluents:
pre = pre & self.fluents
if check_static:
pre = {p for p in pre if p.split()[0] not in self.static_predicates}
act = Action(fix_name(str(op)), pre, adds, dels)
return act
def fix_pre_name(self, precondition):
if not is_atom(precondition):
return {fix_name(str(f)) for f in precondition.subformulas}
return {fix_name(str(precondition))}
def action(self, name):
return self.action_map[fix_name(name).lower()]
def get_action_or_none(self, name):
if "(" in name and ")" != name[-1]:
return None
return self.action_map.get(fix_name(name).lower(), None)
def fluent(self, name):
return fix_name(name)
def static_symbols(self):
return list(self.grounder.static_symbols)
def fluent_symbols(self):
return list(self.grounder.fluent_symbols)
def get_grounded_atoms(self, symbol):
variables = SymbolIndex()
lang = symbol.language
key = "atom_" + symbol.name
model = self.grounder._solve_lp()
if (
key in model
): # in case there is no reachable ground state variable from that fluent symbol
for binding in model[key]:
binding_with_constants = tuple(lang.get(c) for c in binding)
variables.add(StateVariableLite(symbol, binding_with_constants))
return variables
def get_applicable_actions(self, s):
return [a for a in self.actions if entails(s, a.pres)]
def ground_problem(self, problem):
grounder = LPGroundingStrategy(problem, include_variable_inequalities=True)
action_groundings = grounder.ground_actions()
operators = []
for action_name, groundings in action_groundings.items():
action = problem.get_action(action_name)
for grounding in groundings:
operators.append(
ground_schema_into_plain_operator_from_grounding(action, grounding)
)
grounded_fluents = set(
[
grounded_fluent.to_atom()
for grounded_fluent in grounder.ground_state_variables().objects
]
)
init = [f for f in problem.init.as_atoms() if f in grounded_fluents]
if isinstance(problem.goal, tarski.syntax.Atom):
goal = [problem.goal]
else:
goal = [f for f in problem.goal.subformulas if f in grounded_fluents]
return (grounded_fluents, init, goal, operators, grounder)
def get_static(self):
static_symbols = self.static_symbols()
ret = []
for symbol in static_symbols:
ret.extend(self.get_grounded_atoms(symbol))
return set([fix_name(str(x)) for x in ret])
def PDDL_replace_init_pddl_parser(self, s):
d = DomainParser()(open(self.domain_file, "r").read().lower())
p = ProblemParser()(open(self.problem_file, "r").read().lower())
new_state = get_atoms_pddl(d, p, s | self.get_static())
new_p = Problem(
p.name, domain=d, objects=p.objects, init=new_state, goal=p.goal
)
return d, new_p
def parse_ans(response: str, parser: ACPGrammarParser, task: str):
return [parser.parse(clean_answer(resp, task)) for resp in response]
# def parse_ans(response : str, parser : ACPGrammarParser, task : str):
# ans = [parser.parse(clean_answer(resp, task), debug=True) for resp in response]
# if any(elem is None for elem in ans) or any(elem is None for elem in ans[0]):
# return None
# return ans
def remove_garbage(s):
while True:
if s.endswith("."):
s = s[:-1]
elif s.endswith("\n"):
s = s[:-2]
else:
break
return s.rstrip()
def compare_str(s1, s2):
return remove_garbage(s1).lower() == remove_garbage(s2).lower()
def compare(l1, l2):
if not isinstance(l1, list):
return compare_str(l1, l2)
if not isinstance(l2, list):
return False
for i, v in enumerate(l1):
if not compare(v, l2[i]):
return False
return True
def check_prog_response(resp):
if (
"Positive Effects".lower() in resp.lower()
and "Negative Effects".lower() in resp.lower()
):
if "[" not in resp:
return True
return False
def clean_answer(resp, task):
# Minor cleanup
if "progression_gen" in task:
# Check for Positive Effects and Negative Effects instead of separation
if check_prog_response(resp):
# replace **Positive Effects** with "["
# replace **Negative Effects** with "] ["
# append "]" to the end
resp2 = resp.lower()
resp2 = resp2.replace("*", "")
resp2 = resp2.replace("positive effects", "[")
resp2 = resp2.replace("negative effects", "] [")
resp2 = resp2 + "]"
return resp2
if "action_justification_gen" in task:
# Check for "simplified plan:"
if "simplified plan:" in resp.lower():
resp2 = resp.lower()
resp2 = resp2.replace("*", "")
resp2 = resp2.split("simplified plan:")[1]
return resp2
return resp
def get_grammar_task(task):
# print(task)
if task == "reachable_atom_gen":
return "act"
elif task == "progression_gen":
return "progression_list"
elif task == "validation_gen":
return "index"
elif task == "reachable_action_gen":
return "act"
elif task == "action_justification_gen":
return "action_list"
elif task == "landmarks_gen":
return "act"
elif task == "goal_closer_gen":
return "action_name"
elif task == "applicable_actions_gen":
return "action_list"
##############################################################################
# Evaluators
def fix_action_name(a):
assert a.startswith("(") and a.endswith(")")
return "(" + " ".join([x.strip() for x in a[1:-1].split(" ") if len(x) > 0]) + ")"
def str_remove_before_first_parentheses(s):
if s.startswith("("):
return s
try:
return s[s.index("(") :]
except Exception:
return ""
def str_remove_after_last_parentheses(s):
if s.endswith(")"):
return s
i = s.rfind(")")
if i == -1:
return ""
return s[: i + 1]
def cleanup_answer(ans):
if isinstance(ans, str):
ans = str_remove_before_first_parentheses(ans)
ans = str_remove_after_last_parentheses(ans)
ans = ans.lower()
ans = (
ans.replace(")\n(", ")######(")
.replace("),(", ")######(")
.replace(") (", ")######(")
.split("######")
)
return ans
if isinstance(ans, list):
res = []
for x in ans:
res.extend(cleanup_answer(x))
return res
def set_equal(ans1, ans2):
return set(ans1) == set(ans2)
class BaseEvaluator(ABC):
def __init__(self) -> None:
self.scores = []
@abstractmethod
def get_score(self, ans, doc):
pass
def add_scores(self, scores):
self.scores.extend(scores)
def get_avg_score(self):
avg_score = sum(self.scores) / len(self.scores)
return avg_score
def get_evaluator(group):
if group == "applicable_actions_gen":
return ApplicabilityEvaluator()
elif group == "progression_gen":
return ProgressionEvaluator()
elif group == "validation_gen":
return ValidationEvaluator()
elif group == "reachable_atom_gen":
return ReachabilityEvaluator()
elif group == "goal_closer_gen":
return NextActionEvaluator()
elif group == "action_justification_gen":
return JustificationEvaluator()
elif group == "landmarks_gen":
return LandmarksEvaluator()
elif group == "reachable_action_gen":
return ActionReachabilityEvaluator()
assert True, f"Group {group} not found"
"""
Action Reachability task: generate a valid action that is not applicable to any reachable state.
answer: A subset of actions that are known to be unreachable (not an exhaustive set).
It is empty only when we *know* that there are no such actions.
"""
class ActionReachabilityEvaluator(BaseEvaluator):
def get_score(self, ans, doc):
real_answer = doc["answer"]
if not real_answer or len(real_answer) == 0:
# The correct answer is None
self.add_scores(
["none" == x.strip().lower() if x is not None else False for x in ans]
)
else:
for x in ans:
if x is None:
self.scores.append(False)
continue
action = x.strip().lower()
if action in real_answer:
# The answer is in the subset of stored correct answers
self.scores.append(True)
continue
prec = get_action_preconditions(
doc["PDDL_domain"].lower(), doc["PDDL_problem"].lower(), action
)
if prec is None:
# The answer does not correspond to a valid action
self.scores.append(False)
else:
# Need to run a planner on a task with the answer action preconditions as the new goal
prec = f"(and {' '.join(prec)})"
self.scores.append(
is_unsolvable_new_goal(
doc["PDDL_domain"].lower(),
doc["PDDL_problem"].lower(),
prec,
)
)
return self.get_avg_score()
"""
Action Applicability task: generate all actions that are applicable in the current state.
answer: A set of all applicable actions.
"""
class ApplicabilityEvaluator(BaseEvaluator):
def get_score(self, ans, doc):
real_answer = doc["answer"]
real_answer = [a.lower() for a in real_answer]
ans = [[fix_action_name(a) for a in x] if x is not None else None for x in ans]
# Check if the answer is equal (as a set) to the real stored answer
self.add_scores(
[
set_equal(real_answer, cleanup_answer(x)) if x is not None else False
for x in ans
]
)
return self.get_avg_score()
def is_subsequence(plan, new_plan):
i = 0
for a in plan:
if a == new_plan[i]:
i += 1
if len(new_plan) == i:
# Done
return True
return False
def is_subsequence_and_plan(domain, problem, plan, new_plan):
if len(plan) <= len(new_plan):
return False
if not is_subsequence(plan, new_plan):
return False
return is_plan(domain, problem, new_plan)
"""
Justification task: generate a proper subsequence of the given plan that is also a plan.
answer: A list of examples of actions that can be removed (ignored in evaluation).
"""
class JustificationEvaluator(BaseEvaluator):
def get_score(self, ans, doc):
# Sequence of actions (plan) from the question
if "inputs" in doc: # old field name
seq = doc["inputs"][19:-147]
else:
seq = doc["question"][19:-147]
seq = seq.replace(") (", ")######(").split("######")
for x in ans:
if x is None:
self.scores.append(False)
continue
# An answer plan candidate
x = [fix_action_name(a) for a in x]
if len(x) == 0:
# Wrong answer - never an empty sequence
self.scores.append(0)
continue
# Check if the plan candidate from the answer (a) is a proper subsequence of the plan in the question and (b) is a plan.
self.scores.append(
is_subsequence_and_plan(
doc["PDDL_domain"].lower(), doc["PDDL_problem"].lower(), seq, x
)
)
return self.get_avg_score()
"""
Landmarks task: generate a fact that is a non-trivial landmark for the current state.
answer: A list of facts that are found to be landmarks and a list of facts that are found to be non-landmarks.
The questions are generated only for cases where all facts either
(a) hold in the current state,
(b) true in goal,
(c) are found to be landmarks, or
(d) are found to be non-landmarks.
In such cases, the evaluation is simple, it does not require checking whether a fact is a landmark, it was
already done during question generation.
"""
class LandmarksEvaluator(BaseEvaluator):
def get_score(self, ans, doc):
# The set of facts that are found to be landmarks
real_answer = doc["answer"]
real_answer_yes = [a.lower() for a in real_answer["yes"]]
for x in ans:
if x is None:
self.scores.append(False)
continue
if x.strip().lower() in real_answer_yes:
# The answer fact is known to be landmark
self.scores.append(True)
elif x.strip().lower() == "none":
# The answer is none, correct only if there are no known landmarks,
# since we only generate questions when that means that there are no non-trivial landmarks
self.scores.append(len(real_answer_yes) == 0)
else:
# All other cases the answer is incorrect
self.scores.append(False)
return self.get_avg_score()
"""
Next Action task: generate an action that takes us closer to the goal.
answer:
(a) A list of applicable actions that are known to be correct answers
(b) A list of applicable actions that are known to be incorrect answers
(c) The rest of the applicable actions (maybe).
"""
class NextActionEvaluator(BaseEvaluator):
def get_score(self, ans, doc):
real_answer = doc["answer"]
real_answer_yes = [a.lower() for a in real_answer["yes"]]
real_answer_no = [a.lower() for a in real_answer["no"]]
real_answer_maybe = [a.lower() for a in real_answer["maybe"]]
# The cost of the optimal plan from the current state
opt = real_answer.get("opt", None)
for x in ans:
if x is None:
self.scores.append(False)
continue
action = x.strip().lower()
if action in real_answer_yes:
# Known to be correct
self.scores.append(True)
elif action in real_answer_no:
# Known to be incorrect
self.scores.append(False)
elif action not in real_answer_maybe:
# Not applicable, must be incorrect
self.scores.append(False)
else:
# Unknown, need to run a planner to check whether the state that results from applying the action is closer to the goal
# meaning has smaller optimal plan cost.
self.scores.append(
is_on_optimal_plan(
doc["PDDL_domain"].lower(),
doc["PDDL_problem"].lower(),
action,
opt,
)
)
return self.get_avg_score()
"""
Progression task: generate the positive and negative effects of an action in the current state.
answer:
(a) A list of facts that were false and become true, when the action is applied
(b) A list of facts that were true and become false, when the action is applied
"""
class ProgressionEvaluator(BaseEvaluator):
def get_score(self, ans, doc):
real_answer = doc["answer"]
real_answer_pos = [a.lower() for a in real_answer["pos"]]
real_answer_neg = [a.lower() for a in real_answer["neg"]]
for x in ans:
# The answer should be two lists. We allow for a single list and assume that the second one is empty (relaxed evaluation).
if x is None or len(x) > 2 or len(x) < 1:
self.scores.append(False)
else:
p = cleanup_answer(x[0])
if len(x) == 2:
n = cleanup_answer(x[1])
else:
# Assuming the last element is dropped because it is empty
n = []
# Check if the answer is equal as sets to the correct answers.
ans = [set_equal(real_answer_pos, p), set_equal(real_answer_neg, n)]
self.scores.append(all(ans))
return self.get_avg_score()
"""
Reachability task: generate a valid fact that will never become true in any reachable state.
answer: A subset of facts that are known to be unreachable (not an exhaustive set).
It is empty only when we *know* that there are no such facts.
"""
class ReachabilityEvaluator(BaseEvaluator):
def get_score(self, ans, doc):
real_answer = doc["answer"]
real_answer = [f"({x.strip().lower()})" for x in real_answer]
if len(real_answer) == 0:
# The correct answer is None
self.add_scores(
["none" == x.strip().lower() if x is not None else False for x in ans]
)
else:
for x in ans:
if x is None:
self.scores.append(False)
elif x.strip().lower() in real_answer:
# The answer is in the subset of stored correct answers
self.scores.append(True)
else:
# Need to run a planner on a task with the answer fact as the new goal
atom = x.strip().lower()
self.scores.append(
is_unsolvable_new_goal(
doc["PDDL_domain"].lower(),
doc["PDDL_problem"].lower(),
atom,
)
)
return self.get_avg_score()
"""
Validation task: generate an index of the first inapplicable action in the given sequence.
answer: the correct index.
"""
class ValidationEvaluator(BaseEvaluator):
def get_score(self, ans, doc):
real_answer = str(doc["answer"])
assert int(real_answer) >= 0, (
f"The index must be non-negative, received {real_answer}"
)
# Exact match
self.add_scores(
[
real_answer.lower() == x.strip().lower() if x is not None else False
for x in ans
]
)
return self.get_avg_score()
##############################################################################
def dump_item(item, **kwargs):
return json.dumps(item)
def parse_prediction(prediction):
try:
ans = json.loads(prediction.strip())
response = ans.get("answer", None)
return response
except Exception as e:
print(f"Exception occurred {e}")
return prediction
@register_filter("ACP_grammar_filter")
class ACPGrammarFilter(RegexFilter):
"""Filtering Index using"""
def __init__(self, *args, **kwargs):
self.parser = ACPGrammarParser(kwargs["grammar_task"])
self.clean = kwargs["clean"] if "clean" in kwargs else None
def clean_pos_neg(self, resp):
# Check for Positive Effects and Negative Effects instead of separation
if check_prog_response(resp):
resp2 = resp.lower()
resp2 = resp2.replace("*", "")
resp2 = resp2.replace("positive effects", "[")
resp2 = resp2.replace("negative effects", "] [")
resp2 = resp2 + "]"
return resp2
return resp
def clean_simplified_plan(self, resp):
# Check for "simplified plan:"
if "simplified plan:" in resp.lower():
resp2 = resp.lower()
resp2 = resp2.replace("*", "")
resp2 = resp2.split("simplified plan:")[1]
return resp2
return resp
def apply(self, resps, docs):
if self.clean == "pos_neg":
filtered_resps = [
[self.parser.parse(self.clean_pos_neg(r)) for r in resp]
for resp in resps
]
elif self.clean == "simplified plan":
filtered_resps = [
[self.parser.parse(self.clean_simplified_plan(r)) for r in resp]
for resp in resps
]
else:
filtered_resps = [[self.parser.parse(r) for r in resp] for resp in resps]
return filtered_resps
def process_acp_results(doc, results):
return {"score": get_evaluator(doc["group"]).get_score(results, doc)}
def get_score(references, predictions, **kwargs):
# print(f"References: {references}")
# print(f"Predictions: {predictions}")
data = json.loads(references[0].strip())
real_ans = data["answer"]
task = data["group"]
responses = [parse_prediction(prediction) for prediction in predictions]
print(f"Real answer: {real_ans}")
print(f"Model answers: {responses}")
parser = ACPGrammarParser(get_grammar_task(task))
ans = parse_ans(responses, parser, task)
print(f"Parsed model answers: {ans}")
score = get_evaluator(task).get_score(ans, data)
return {"get_score": score}
lm_eval/tasks/acpbench/gen_2shot/act_reach.yaml 0 → 100644
View file @ 601be343
task: acp_areach_gen
dataset_name: acp_areach_gen
include: _gen_yaml_2shot
fewshot_config:
sampler: first_n
samples:
- context: "A robot is in a grid and can only move to places that are connected to its current position. The grid size is 5x5, and the locations are of the form fi-jf (e.g., f3-2f or f0-1f). The grid cells are connected to their neighbors (e.g., f1-2f is connected to the four neighbors f0-2f, f2-2f, f1-1f, and f1-3f). Some positions on the grid are locked and can be opened with a key of a matching shape. The robot has an arm that can pick up a key when the key is in same location as the robot and the arm is empty. There are 2 keys in 0 different shapes: Key key0-0 is of shape shape0, Key key0-1 is of shape shape0. Currently, the robot is at position f2-2f and its arm is empty. All the positions are open except the following: f2-0f has shape0 shaped lock, f4-2f has shape0 shaped lock. Key key0-0 is at position f1-2f. Key key0-1 is at position f1-3f. The available actions are: (unlock ?curpos ?lockpos ?key ?shape) - unlock the place ?lockpos with key ?key of shape ?shape from the current position place ?curpos, (move ?curpos ?nextpos) - move from place ?curpos to place ?nextpos, (pickup ?curpos ?key) - retrieve the key ?key from its current position ?curpos, (pickup-and-loose ?curpos ?newkey ?oldkey) - pick up the key ?newkey from the current position ?curpos and loose the key ?oldkey which is being held, and (putdown ?curpos ?key) - put the key ?key at the current position place ?curpos."
question: "What action can never become applicable, in any state reachable from the current state?"
answer: "(unlock f0-3f f0-4f key0-0 shape0)"
- context: "There are several cities, each containing several locations, some of which are airports. There are also trucks, which can drive within a single city, and airplanes, which can fly between airports. The goal is to get some packages from various locations to various new locations. There are 2 trucks and 1 airplane, as well as 4 packages. There are 4 locations across 2 cities. The locations are in cities as follows: l1-1 and l1-0 are in c1; l0-0 and l0-1 are in c0. Currently, t1 is at l1-1, a0 is at l1-0, p0 is at l0-0, t0 is at l0-1, p2 is in a0, p1 is in t1, p3 is in t0. The available actions are: (load-truck ?obj ?truck ?loc) - load the object ?obj from location ?loc into the truck ?truck, (load-airplane ?obj ?airplane ?loc) - load object ?obj into airplane ?airplane at location ?loc, (unload-truck ?obj ?truck ?loc) - offload the object ?obj from the truck ?truck at location ?loc, (unload-airplane ?obj ?airplane ?loc) - offload the object ?obj from the airplane ?airplane at location ?loc, (drive-truck ?truck ?loc-from ?loc-to ?city) - navigate the truck ?truck from location ?loc-from in city ?city to location ?loc-to in the same city, and (fly-airplane ?airplane ?loc-from ?loc-to) - operate the airplane ?airplane from airport ?loc-from to airport ?loc-to."
question: "What action can never become applicable, in any state reachable from the current state?"
answer: "(drive-truck t0 l1-1 l0-0 c0)"
doc_to_text: "**Question**: {{context}} {{question}} Each action starts with an opening parenthesis and ends with closing parenthesis. Provide one action or None. **Final Answer**:"
filter_list:
- name: "acp_grammar_parse"
filter:
- function: "ACP_grammar_filter"
grammar_task: "act"
- function: "take_first"
lm_eval/tasks/acpbench/gen_2shot/app.yaml 0 → 100644
View file @ 601be343
task: acp_app_gen
dataset_name: acp_app_gen
include: _gen_yaml_2shot
fewshot_config:
sampler: first_n
samples:
- context: "A robot is in a grid and can only move to places that are connected to its current position. The grid size is 5x5, and the locations are of the form fi-jf (e.g., f3-2f or f0-1f). The grid cells are connected to their neighbors (e.g., f1-2f is connected to the four neighbors f0-2f, f2-2f, f1-1f, and f1-3f). Some positions on the grid are locked and can be opened with a key of a matching shape. The robot has an arm that can pick up a key when the key is in same location as the robot and the arm is empty. There are 2 keys in 1 different shapes: Key key0-1 is of shape shape0, Key key0-0 is of shape shape0. Currently, the robot is at position f3-2f and its arm is empty. All the positions are open except the following: f2-0f has shape0 shaped lock, f4-2f has shape0 shaped lock. Key key0-0 is at position f2-2f. Key key0-1 is at position f1-3f. The available actions are: (unlock ?curpos ?lockpos ?key ?shape) - unlock the place ?lockpos with key ?key of shape ?shape from the current position place ?curpos, (move ?curpos ?nextpos) - travel from the current position ?curpos to the next position ?nextpos, (pickup ?curpos ?key) - pick up key ?key from place ?curpos, (pickup-and-loose ?curpos ?newkey ?oldkey) - pick up the key ?newkey at the current position place ?curpos and loose the key ?oldkey being held, and (putdown ?curpos ?key) - put down key ?key at current position place ?curpos."
question: "Generate the list of all ground actions that are applicable in this state."
answer: "[(move f3-2f f3-1f), (move f3-2f f2-2f), (move f3-2f f3-3f)]"
- context: "There are several cities, each containing several locations, some of which are airports. There are also trucks, which can drive within a single city, and airplanes, which can fly between airports. The goal is to get some packages from various locations to various new locations. There are 2 trucks and 1 airplane, as well as 4 packages. There are 4 locations across 2 cities. The locations are in cities as follows: l0-0 and l0-1 are in c0; l1-0 and l1-1 are in c1. Currently, t1, p2, and p3 are at l1-0, a0 is at l0-0, t0 is at l0-1, p1 and p0 are in t1. The available actions are: (load-truck ?obj ?truck ?loc) - load object ?obj into truck ?truck at location ?loc, (load-airplane ?obj ?airplane ?loc) - load object ?obj into airplane ?airplane at location ?loc, (unload-truck ?obj ?truck ?loc) - unload object ?obj from truck ?truck at location ?loc, (unload-airplane ?obj ?airplane ?loc) - remove the object ?obj from the airplane ?airplane and place it on the location ?loc, (drive-truck ?truck ?loc-from ?loc-to ?city) - navigate the truck ?truck from its current location ?loc-from in city ?city to the new location ?loc-to within the same city, and (fly-airplane ?airplane ?loc-from ?loc-to) - fly airplane ?airplane from airport ?loc-from to airport ?loc-to."
question: "Generate the list of all ground actions that are applicable in this state."
answer: "[(drive-truck t1 l1-0 l1-0 c1), (drive-truck t0 l0-1 l0-0 c0), (load-truck p2 t1 l1-0), (unload-truck p0 t1 l1-0), (drive-truck t0 l0-1 l0-1 c0), (fly-airplane a0 l0-0 l1-0), (fly-airplane a0 l0-0 l0-0), (unload-truck p1 t1 l1-0), (drive-truck t1 l1-0 l1-1 c1), (load-truck p3 t1 l1-0)]"
doc_to_text: "**Question**: {{context}} {{question}} Each action starts with an opening parenthesis and ends with closing parenthesis. Provide only the actions. **Final Answer**:"
filter_list:
- name: "acp_grammar_parse"
filter:
- function: "ACP_grammar_filter"
grammar_task: "action_list"
- function: "take_first"
lm_eval/tasks/acpbench/gen_2shot/just.yaml 0 → 100644
View file @ 601be343
task: acp_just_gen
dataset_name: acp_just_gen
include: _gen_yaml_2shot
fewshot_config:
sampler: first_n
samples:
- context: "A robot is in a grid and can only move to places that are connected to its current position. The grid size is 5x5, and the locations are of the form fi-jf (e.g., f3-2f or f0-1f). The grid cells are connected to their neighbors (e.g., f1-2f is connected to the four neighbors f0-2f, f2-2f, f1-1f, and f1-3f). Some positions on the grid are locked and can be opened with a key of a matching shape. The robot has an arm that can pick up a key when the key is in same location as the robot and the arm is empty. There are 2 keys in 1 different shapes: Key key0-0 is of shape shape0, Key key0-1 is of shape shape0. Currently, the robot is at position f3-3f and its arm is empty. All the positions are open except the following: f4-2f has shape0 shaped lock, f2-0f has shape0 shaped lock. Key key0-0 is at position f2-2f. Key key0-1 is at position f1-3f. The available actions are: (unlock ?curpos ?lockpos ?key ?shape) - unlock place ?lockpos with key ?key of shape ?shape from current position place ?curpos, (move ?curpos ?nextpos) - move from ?curpos to ?nextpos, (pickup ?curpos ?key) - retrieve the key ?key from its current position ?curpos, (pickup-and-loose ?curpos ?newkey ?oldkey) - pick up key ?newkey at current position place ?curpos and loose key ?oldkey being held, and (putdown ?curpos ?key) - put down the key ?key at the current position ?curpos. The goal is to reach a state where the following facts hold: Key key0-0 is at f2-0f location and Key key0-1 is at f1-3f location."
question: "Simplify the plan [(move f3-3f f3-2f), (move f3-2f f2-2f), (pickup f2-2f key0-0), (move f2-2f f2-1f), (putdown f2-1f key0-0), (pickup f2-1f key0-0), (unlock f2-1f f2-0f key0-0 shape0), (move f2-1f f2-0f), (putdown f2-0f key0-0)] by removing either a single action or a pair of consecutive actions, while still maintaining a valid plan. Provide the resulting simplified plan."
answer: "[(move f3-3f f3-2f), (move f3-2f f2-2f), (pickup f2-2f key0-0), (move f2-2f f2-1f), (unlock f2-1f f2-0f key0-0 shape0), (move f2-1f f2-0f), (putdown f2-0f key0-0)]"
- context: "There are several cities, each containing several locations, some of which are airports. There are also trucks, which can drive within a single city, and airplanes, which can fly between airports. The goal is to get some packages from various locations to various new locations. There are 2 trucks and 1 airplane, as well as 4 packages. There are 4 locations across 2 cities. The locations are in cities as follows: l1-1 and l1-0 are in c1; l0-0 and l0-1 are in c0. Currently, p2, p1, and p3 are at l1-0, p0 and t1 are at l1-1, t0 is at l0-1, a0 is at l0-0. The available actions are: (load-truck ?obj ?truck ?loc) - load the object ?obj from location ?loc into the truck ?truck, (load-airplane ?obj ?airplane ?loc) - load the object ?obj from location ?loc onto the airplane ?airplane, (unload-truck ?obj ?truck ?loc) - unload the object ?obj from the truck ?truck at location ?loc, (unload-airplane ?obj ?airplane ?loc) - remove the object ?obj from the airplane ?airplane and place it on the location ?loc, (drive-truck ?truck ?loc-from ?loc-to ?city) - drive truck ?truck from location ?loc-from in city ?city to location ?loc-to in the same city, and (fly-airplane ?airplane ?loc-from ?loc-to) - fly the airplane ?airplane from location ?loc-from to location ?loc-to. The goal is to reach a state where the following facts hold: p3 is at l0-1, p2 is at l1-0, p0 is at l0-0, and p1 is at l1-0."
question: "Simplify the plan [(fly-airplane a0 l0-0 l1-0), (fly-airplane a0 l1-0 l0-0), (load-truck p0 t1 l1-1), (drive-truck t1 l1-1 l1-0 c1), (unload-truck p0 t1 l1-0), (fly-airplane a0 l0-0 l1-0), (load-airplane p0 a0 l1-0), (load-airplane p3 a0 l1-0), (fly-airplane a0 l1-0 l0-0), (unload-airplane p0 a0 l0-0), (unload-airplane p3 a0 l0-0), (drive-truck t0 l0-1 l0-0 c0), (load-truck p3 t0 l0-0), (drive-truck t0 l0-0 l0-1 c0), (unload-truck p3 t0 l0-1)] by removing either a single action or a pair of consecutive actions, while still maintaining a valid plan. Provide the resulting simplified plan."
answer: "[(load-truck p0 t1 l1-1), (drive-truck t1 l1-1 l1-0 c1), (unload-truck p0 t1 l1-0), (fly-airplane a0 l0-0 l1-0), (load-airplane p0 a0 l1-0), (load-airplane p3 a0 l1-0), (fly-airplane a0 l1-0 l0-0), (unload-airplane p0 a0 l0-0), (unload-airplane p3 a0 l0-0), (drive-truck t0 l0-1 l0-0 c0), (load-truck p3 t0 l0-0), (drive-truck t0 l0-0 l0-1 c0), (unload-truck p3 t0 l0-1)]"
doc_to_text: "**Question**: {{context}} {{question}} **Final Answer**:"
filter_list:
- name: "acp_grammar_parse"
filter:
- function: "ACP_grammar_filter"
grammar_task: "action_list"
clean: "simplified plan"
- function: "take_first"
lm_eval/tasks/acpbench/gen_2shot/land.yaml 0 → 100644
View file @ 601be343
task: acp_land_gen
dataset_name: acp_land_gen
include: _gen_yaml_2shot
fewshot_config:
sampler: first_n
samples:
- context: "A robot is in a grid and can only move to places that are connected to its current position. The grid size is 5x5, and the locations are of the form fi-jf (e.g., f3-2f or f0-1f). The grid cells are connected to their neighbors (e.g., f1-2f is connected to the four neighbors f0-2f, f2-2f, f1-1f, and f1-3f). Some positions on the grid are locked and can be opened with a key of a matching shape. The robot has an arm that can pick up a key when the key is in same location as the robot and the arm is empty. There are 2 keys in 1 different shapes: Key key0-1 is of shape shape0, Key key0-0 is of shape shape0. Currently, the robot is at position f3-0f and its arm is empty. All the positions are open except the following: f4-2f has shape0 shaped lock. Key key0-0 is at position f3-0f. Key key0-1 is at position f1-3f. The goal is to reach a state where the following facts hold: Key key0-0 is at f2-0f location and Key key0-1 is at f1-3f location. The available propositions are: (at ?r ?x) - Key ?r is at ?x location, (at-robot ?x) - Robot is at ?x location, (locked ?x) - Location ?x is locked, (holding ?k) - Robot is holding ?k, (open ?x) - Location ?x is open, and (arm-empty) - Robot's arm is empty."
question: "Generate a non-trivial fact landmark, one that does not hold in the initial state or goal."
answer: "(holding key0-0)"
- context: "There are several cities, each containing several locations, some of which are airports. There are also trucks, which can drive within a single city, and airplanes, which can fly between airports. The goal is to get some packages from various locations to various new locations. There are 2 trucks and 1 airplane, as well as 4 packages. There are 4 locations across 2 cities. The locations are in cities as follows: l1-0 and l1-1 are in c1; l0-1 and l0-0 are in c0. Currently, a0 and p2 are at l1-0, t0 is at l0-0, t1 is at l1-1, p3 and p1 are in a0, p0 is in t1. The goal is to reach a state where the following facts hold: p0 is at l0-0, p2 is at l1-0, p1 is at l1-0, and p3 is at l0-1. The available propositions are: (at ?obj ?loc) - ?obj is at ?loc and (in ?obj1 ?obj2) - ?obj1 is in ?obj2."
question: "Generate a non-trivial fact landmark, one that does not hold in the initial state or goal."
answer: "(in p3 t0)"
doc_to_text: "**Question**: {{context}} {{question}} Provide only the ground proposition or None. **Final Answer**:"
filter_list:
- name: "acp_grammar_parse"
filter:
- function: "ACP_grammar_filter"
grammar_task: "act"
- function: "take_first"
lm_eval/tasks/acpbench/gen_2shot/next_act.yaml 0 → 100644
View file @ 601be343
task: acp_nexta_gen
dataset_name: acp_nexta_gen
include: _gen_yaml_2shot
fewshot_config:
sampler: first_n
samples:
- context: "A robot is in a grid and can only move to places that are connected to its current position. The grid size is 5x5, and the locations are of the form fi-jf (e.g., f3-2f or f0-1f). The grid cells are connected to their neighbors (e.g., f1-2f is connected to the four neighbors f0-2f, f2-2f, f1-1f, and f1-3f). Some positions on the grid are locked and can be opened with a key of a matching shape. The robot has an arm that can pick up a key when the key is in same location as the robot and the arm is empty. There are 2 keys in 1 different shapes: Key key0-1 is of shape shape0, Key key0-0 is of shape shape0. Currently, the robot is at position f4-0f and its arm is empty. All the positions are open except the following: f4-2f has shape0 shaped lock. Key key0-0 is at position f3-0f. Key key0-1 is at position f1-3f. The goal is to reach a state where the following facts hold: Key key0-0 is at f2-0f location and Key key0-1 is at f1-3f location. The available actions are: (unlock ?curpos ?lockpos ?key ?shape) - unlock place ?lockpos with key ?key of shape ?shape from current position place ?curpos, (move ?curpos ?nextpos) - travel from the current position ?curpos to the next position ?nextpos, (pickup ?curpos ?key) - pick up key ?key from place ?curpos, (pickup-and-loose ?curpos ?newkey ?oldkey) - pick up the key ?newkey at the current position place ?curpos and loose the key ?oldkey being held, and (putdown ?curpos ?key) - put down the key ?key at the current position ?curpos."
question: "What is the next action that takes us towards the goal?"
answer: "(move f4-0f f3-0f)"
- context: "There are several cities, each containing several locations, some of which are airports. There are also trucks, which can drive within a single city, and airplanes, which can fly between airports. The goal is to get some packages from various locations to various new locations. There are 2 trucks and 1 airplane, as well as 4 packages. There are 4 locations across 2 cities. The locations are in cities as follows: l0-1 and l0-0 are in c0; l1-1 and l1-0 are in c1. Currently, t0 is at l0-1, a0 is at l0-0, t1 and p1 are at l1-0, p2, p0, and p3 are in t1. The goal is to reach a state where the following facts hold: p3 is at l0-1, p2 is at l1-0, p1 is at l1-0, and p0 is at l0-0. The available actions are: (load-truck ?obj ?truck ?loc) - load object ?obj into truck ?truck at location ?loc, (load-airplane ?obj ?airplane ?loc) - load the object ?obj from location ?loc onto the airplane ?airplane, (unload-truck ?obj ?truck ?loc) - unload the object ?obj from the truck ?truck at location ?loc, (unload-airplane ?obj ?airplane ?loc) - unload object ?obj from airplane ?airplane at location ?loc, (drive-truck ?truck ?loc-from ?loc-to ?city) - drive the truck ?truck in city ?city from location ?loc-from to location ?loc-to, and (fly-airplane ?airplane ?loc-from ?loc-to) - operate the airplane ?airplane from airport ?loc-from to airport ?loc-to."
question: "What is the next action that takes us towards the goal?"
answer: "(drive-truck t0 l0-1 l0-0 c0)"
doc_to_text: "**Question**: {{context}} {{question}} Each action starts with an opening parenthesis and ends with closing parenthesis. Provide only the action. **Final Answer**:"
filter_list:
- name: "acp_grammar_parse"
filter:
- function: "ACP_grammar_filter"
grammar_task: "action_name"
- function: "take_first"
lm_eval/tasks/acpbench/gen_2shot/prog.yaml 0 → 100644
View file @ 601be343
task: acp_prog_gen
dataset_name: acp_prog_gen
include: _gen_yaml_2shot
fewshot_config:
sampler: first_n
samples:
- context: "A robot is in a grid and can only move to places that are connected to its current position. \nThe grid size is 5x5, and the locations are of the form fi-jf (e.g., f3-2f or f0-1f). The grid cells are connected to their neighbors (e.g., f1-2f is connected to the four neighbors f0-2f, f2-2f, f1-1f, and f1-3f). Some positions on the grid are locked and can be opened with a key of a matching shape. The robot has an arm that can pick up a key when the key is in same location as the robot and the arm is empty. \nThere are 2 keys in 0 different shapes: Key key0-0 is of shape shape0, Key key0-1 is of shape shape0. \nCurrently, the robot is at position f0-1f and its arm is empty. All the positions are open except the following: f4-2f has shape0 shaped lock. Key key0-1 is at position f1-3f. Key key0-0 is at position f0-1f. The available propositions are: (at ?r ?x) - Key ?r is at ?x location, (at-robot ?x) - Robot is at ?x location, (locked ?x) - Location ?x is locked, (holding ?k) - Robot is holding ?k, (open ?x) - Location ?x is open, and (arm-empty) - Robot's arm is empty."
question: "Break down the outcomes of performing the action \"retrieve the key key0-0 from its current position f0-1f\" into two lists, positive effects and negative effects. Positive effects are the propositions that are false in the current state but will become true after performing the action. Negative effects are the propositions that are true in the current state and will become false after performing the action."
answer: "[(holding key0-0)] [(arm-empty), (at key0-0 f0-1f)]"
- context: "There are several cities, each containing several locations, some of which are airports. There are also trucks, which can drive within a single city, and airplanes, which can fly between airports. The goal is to get some packages from various locations to various new locations. There are 2 trucks and 1 airplane, as well as 4 packages. There are 4 locations across 2 cities. The locations are in cities as follows: l1-1 and l1-0 are in c1; l0-1 and l0-0 are in c0. Currently, p2, t1, p1, p3, a0, and p0 are at l1-0, t0 is at l0-1. The available propositions are: (at ?obj ?loc) - ?obj is at ?loc and (in ?obj1 ?obj2) - ?obj1 is in ?obj2."
question: "Break down the outcomes of performing the action \"load object p3 into truck t1 at location l1-0\" into two lists, positive effects and negative effects. Positive effects are the propositions that are false in the current state but will become true after performing the action. Negative effects are the propositions that are true in the current state and will become false after performing the action."
answer: "[(in p3 t1)] [(at p3 l1-0)]"
doc_to_text: "**Question**: {{context}} {{question}} Provide only the two lists with the ground propositions. **Final Answer**:"
filter_list:
- name: "acp_grammar_parse"
filter:
- function: "ACP_grammar_filter"
grammar_task: "progression_list"
clean: "pos_neg"
- function: "take_first"
lm_eval/tasks/acpbench/gen_2shot/reach.yaml 0 → 100644
View file @ 601be343
task: acp_reach_gen
dataset_name: acp_reach_gen
include: _gen_yaml_2shot
fewshot_config:
sampler: first_n
samples:
- context: "A robot is in a grid and can only move to places that are connected to its current position. The grid size is 5x5, and the locations are of the form fi-jf (e.g., f3-2f or f0-1f). The grid cells are connected to their neighbors (e.g., f1-2f is connected to the four neighbors f0-2f, f2-2f, f1-1f, and f1-3f). Some positions on the grid are locked and can be opened with a key of a matching shape. The robot has an arm that can pick up a key when the key is in same location as the robot and the arm is empty. There are 2 keys in 0 different shapes: Key key0-1 is of shape shape0, Key key0-0 is of shape shape0. Currently, the robot is at position f1-2f and its arm is empty. All the positions are open except the following: f4-2f has shape0 shaped lock. Key key0-0 is at position f1-0f. Key key0-1 is at position f1-3f. The available propositions are: (at ?r ?x) - Key ?r is at ?x location, (at-robot ?x) - Robot is at ?x location, (locked ?x) - Location ?x is locked, (holding ?k) - Robot is holding ?k, (open ?x) - Location ?x is open, and (arm-empty) - Robot's arm is empty."
question: "What proposition can never hold in any potentially reachable state?"
answer: "(locked f3-1f)"
- context: "There are several cities, each containing several locations, some of which are airports. There are also trucks, which can drive within a single city, and airplanes, which can fly between airports. The goal is to get some packages from various locations to various new locations. There are 2 trucks and 1 airplane, as well as 4 packages. There are 4 locations across 2 cities. The locations are in cities as follows: l0-0 and l0-1 are in c0; l1-0 and l1-1 are in c1. Currently, a0, p2, and t1 are at l1-0, p3 and p0 are at l0-0, t0 is at l0-1, p1 is in t1. The available propositions are: (at ?obj ?loc) - ?obj is at ?loc and (in ?obj1 ?obj2) - ?obj1 is in ?obj2."
question: "What proposition can never hold in any potentially reachable state?"
answer: "(at t0 l1-1)"
doc_to_text: "**Question**: {{context}} {{question}} Provide one proposition or None. **Final Answer**:"
filter_list:
- name: "acp_grammar_parse"
filter:
- function: "ACP_grammar_filter"
grammar_task: "act"
- function: "take_first"
lm_eval/tasks/acpbench/gen_2shot/val.yaml 0 → 100644
View file @ 601be343
task: acp_val_gen
dataset_name: acp_val_gen
include: _gen_yaml_2shot
fewshot_config:
sampler: first_n
samples:
- context: "A robot is in a grid and can only move to places that are connected to its current position. The grid size is 5x5, and the locations are of the form fi-jf (e.g., f3-2f or f0-1f). The grid cells are connected to their neighbors (e.g., f1-2f is connected to the four neighbors f0-2f, f2-2f, f1-1f, and f1-3f). Some positions on the grid are locked and can be opened with a key of a matching shape. The robot has an arm that can pick up a key when the key is in same location as the robot and the arm is empty. There are 2 keys in 1 different shapes: Key key0-0 is of shape shape0, Key key0-1 is of shape shape0. Currently, the robot is at position f3-3f and its arm is empty. All the positions are open except the following: f2-0f has shape0 shaped lock, f4-2f has shape0 shaped lock. Key key0-1 is at position f1-3f. Key key0-0 is at position f2-2f. The goal is to reach a state where the following facts hold: Key key0-0 is at f2-0f location and Key key0-1 is at f1-3f location. The available actions are: (unlock ?curpos ?lockpos ?key ?shape) - unlock the place ?lockpos with the key ?key of the shape ?shape from the current position place ?curpos, (move ?curpos ?nextpos) - travel from the current position ?curpos to the next position ?nextpos, (pickup ?curpos ?key) - pick up key ?key from place ?curpos, (pickup-and-loose ?curpos ?newkey ?oldkey) - pick up the key ?newkey from the current position ?curpos and loose the key ?oldkey which is being held, and (putdown ?curpos ?key) - put down key ?key at current position place ?curpos."
question: "What is the first inapplicable action in the next sequence of actions: [(move f3-3f f3-2f), (move f3-2f f2-2f), (pickup f2-2f key0-0), (pickup-and-loose f4-0f key0-0 key0-1), (unlock f2-1f f2-0f key0-0 shape0), (move f2-1f f2-0f), (putdown f2-0f key0-0), (move f2-0f f2-1f)]?"
answer: "3"
- context: "There are several cities, each containing several locations, some of which are airports. There are also trucks, which can drive within a single city, and airplanes, which can fly between airports. The goal is to get some packages from various locations to various new locations. There are 2 trucks and 1 airplane, as well as 4 packages. There are 4 locations across 2 cities. The locations are in cities as follows: l0-1 and l0-0 are in c0; l1-1 and l1-0 are in c1. Currently, t1 and p0 are at l1-1, t0 is at l0-1, p3, p2, and p1 are at l1-0, a0 is at l0-0. The goal is to reach a state where the following facts hold: p2 is at l1-0, p3 is at l0-1, p0 is at l0-0, and p1 is at l1-0. The available actions are: (load-truck ?obj ?truck ?loc) - load object ?obj into truck ?truck at location ?loc, (load-airplane ?obj ?airplane ?loc) - load the object ?obj from location ?loc onto the airplane ?airplane, (unload-truck ?obj ?truck ?loc) - unload the object ?obj from the truck ?truck at location ?loc, (unload-airplane ?obj ?airplane ?loc) - unload object ?obj from airplane ?airplane at location ?loc, (drive-truck ?truck ?loc-from ?loc-to ?city) - navigate the truck ?truck from its current location ?loc-from in city ?city to the new location ?loc-to within the same city, and (fly-airplane ?airplane ?loc-from ?loc-to) - fly the airplane ?airplane from location ?loc-from to location ?loc-to."
question: "What is the first inapplicable action in the next sequence of actions: [(load-truck p0 t1 l1-1), (drive-truck t1 l1-1 l1-0 c1), (unload-truck p0 t1 l1-0), (fly-airplane a0 l0-0 l1-0), (unload-truck p3 t0 l0-1), (load-airplane p3 a0 l1-0), (fly-airplane a0 l1-0 l0-0), (unload-airplane p0 a0 l0-0), (unload-airplane p3 a0 l0-0), (drive-truck t0 l0-1 l0-0 c0), (load-truck p3 t0 l0-0), (drive-truck t0 l0-0 l0-1 c0), (unload-truck p3 t0 l0-1)]?"
answer: "4"
doc_to_text: "**Question**: {{context}} {{question}} Provide only the index of the action. **Final Answer**:"
filter_list:
- name: "acp_grammar_parse"
filter:
- function: "ACP_grammar_filter"
grammar_task: "index"
- function: "take_first"
lm_eval/tasks/acpbench/gen_2shot_with_pddl/_gen_yaml_2shot 0 → 100644
View file @ 601be343
tag:
- acp_gen_2shot_with_pddl
- acp_bench_hard_with_pddl
dataset_path: ibm-research/acp_bench
test_split: test
description: "Answer the question based on the provided PDDL domain and PDDL problem. The current state is the initial state described in the PDDL problem below.\n\n"
doc_to_target: "{{answer}}"
output_type: generate_until
num_fewshot: 2
generation_kwargs:
until:
- "\n\n\n\n"
- "\n\n"
- "**Question**:"
- "**Question:**"
- "Q:"
do_sample: false
max_gen_toks: 1000
temperature: 0.0
metadata:
version: 1.0
process_results: !function acp_utils.process_acp_results
metric_list:
- metric: "score"
aggregation: mean
higher_is_better: True
lm_eval/tasks/acpbench/gen_2shot_with_pddl/acp_grammar.lark 0 → 100644
View file @ 601be343
NAME: /[a-zA-Z][a-zA-Z0-9-_]*/
LPAR : "("
RPAR : ")"
LSPAR: "["
RSPAR: "]"
COMMA: ","
WS: /[ \n]/
action_none : "None"
action_name : LPAR NAME (WS NAME)* RPAR
action_list : (action_name WS?)*
prog_list : action_name* (COMMA action_name)*
progression_list : LSPAR prog_list RSPAR LSPAR prog_list RSPAR
act : action_name | action_none
index: /[0-9]+[0-9]*/
start: action_list
lm_eval/tasks/acpbench/gen_2shot_with_pddl/acp_utils.py 0 → 100644
View file @ 601be343
import json
import os
from abc import ABC, abstractmethod
from collections import defaultdict
from pathlib import Path
from lm_eval.api.registry import register_filter
from lm_eval.filters.extraction import RegexFilter
try:
import tempfile
import tarski
from kstar_planner import planners as kp
from lark import Lark
from lark.lexer import Token
from lark.visitors import Visitor
from pddl.core import Problem
from pddl.parser.domain import DomainParser
from pddl.parser.problem import ProblemParser
from tarski.grounding.common import StateVariableLite
from tarski.grounding.lp_grounding import LPGroundingStrategy
from tarski.io import PDDLReader
from tarski.io import fstrips as iofs
from tarski.syntax.formulas import is_atom
from tarski.syntax.transform.action_grounding import (
ground_schema_into_plain_operator_from_grounding,
)
from tarski.util import SymbolIndex
except ModuleNotFoundError:
raise ModuleNotFoundError(
"`lark>=1.1.9`, `tarski[clingo]==0.8.2`, `pddl==0.4.2` and `kstar-planner==1.4.2` are required for evaluating the generative tasks. \
Please install via pip install lm-eval[acpbench] or pip install -e .[acpbench]",
)
#########################################################################
# Grammar
GRAMMAR_FILE = os.path.join(
os.path.dirname(os.path.abspath(__file__)), "acp_grammar.lark"
)
class ACPBench_Visitor(Visitor):
def __init__(self) -> None:
super().__init__()
self.action_lists = None
self.action_names = None
self.progression_lists = None
self.prog_lists = None
self.indexes = None
def action_list(self, tree):
self.action_lists = []
def prog_list(self, tree):
if self.prog_lists is not None:
self.progression_lists.append(self.prog_lists)
self.prog_lists = []
def progression_list(self, tree):
self.progression_lists = []
def action_none(self, tree):
self.action_names = "None"
def action_name(self, tree):
act_name = "(" + "".join(tree.children[1:-1]) + ")"
self.action_names = act_name
if self.action_lists is not None:
self.action_lists.append(act_name)
if self.prog_lists is not None:
self.prog_lists.append(act_name)
def index(self, tree):
self.indexes = "".join(tree.children)
if not self.indexes.isnumeric():
self.indexes = None
class ACPGrammarParser(object):
def __init__(self, task) -> None:
self.task = task
with open(GRAMMAR_FILE) as f:
grammar = f.read()
self.acp_parser = Lark(grammar, start=task, parser="lalr")
def parse(self, input, debug=False):
def ignore_errors(e):
if hasattr(e, "token") and e.token.type == "$END":
for x in e.expected:
if x != "WS":
e.interactive_parser.feed_token(
Token(x, self.acp_parser.get_terminal(x).pattern.value)
)
return True
input = input.replace("\n", "")
input = input.strip()
try:
tree = self.acp_parser.parse(input, on_error=ignore_errors)
if debug:
print(tree)
visitor = ACPBench_Visitor()
visitor.visit_topdown(tree)
if self.task == "action_list":
return visitor.action_lists
elif self.task == "act":
return visitor.action_names
elif self.task == "action_name":
return visitor.action_names
elif self.task == "index":
return visitor.indexes
elif self.task == "progression_list":
if visitor.prog_lists not in visitor.progression_lists:
visitor.progression_lists.append(visitor.prog_lists)
return visitor.progression_lists
except Exception as e:
if debug:
print("exception")
print(e)
return None
##############################################################################
# Utils
# Used in next action
def is_on_optimal_plan(domain, problem, action, opt):
with (
tempfile.NamedTemporaryFile() as domain_temp,
tempfile.NamedTemporaryFile() as problem_temp,
):
with open(str(domain_temp.name), "w", encoding="utf8") as file:
file.write(domain.lower())
with open(str(problem_temp.name), "w", encoding="utf8") as file:
file.write(problem.lower())
# Here, we need to keep the temp files live until the end of the function
try:
P = STRIPS(str(domain_temp.name), str(problem_temp.name))
except Exception:
# Unsolvable
return False
a = P.get_action_or_none(action[1:-1])
if a is None:
return False
state = P.init
next_state = progress(state, a)
if opt is None:
# Get an optimal plan cost
plans = generate_optimal_plans_for_problem_state(
P, state, num_plans=1, timeout=5
)
opt = len(plans[0]["actions"])
else:
opt = int(opt)
# Getting an optimal plan for the next state
next_plans = generate_optimal_plans_for_problem_state(
P, next_state, num_plans=1, timeout=5
)
if next_plans is None:
return False
next_opt = len(next_plans[0]["actions"])
return next_opt + 1 == opt
# Used in justification
def is_plan(domain, problem, new_plan):
P = get_STRIPS(domain, problem)
if P is None:
# Unsolvable
return False
# Check if new_plan is a plan
current_state = P.init
for action in new_plan:
applicable_actions = P.get_applicable_actions(current_state)
app_actions_list = [f"({a.name.lower()})" for a in applicable_actions]
if action.lower() not in app_actions_list:
return False
a = applicable_actions[app_actions_list.index(action.lower())]
current_state = progress(current_state, a)
return entails(current_state, P.goal)
# Used in action reachability
def get_action_preconditions(domain, problem, action):
P = get_STRIPS(domain, problem)
assert P is not None, f"Domain\n{domain}\nProblem\n{problem}\nAction: {action}"
a = P.get_action_or_none(action[1:-1])
if a is None:
return a
return [f"({f})" for f in a.pres]
def generate_optimal_plans_for_problem_state(P, state, num_plans, timeout):
import tempfile
with (
tempfile.NamedTemporaryFile() as domain_temp,
tempfile.NamedTemporaryFile() as problem_temp,
):
create_tmp_dom_prob_replace_init(P, state, domain_temp, problem_temp)
plans = generate_top_q_plans(
domain=str(domain_temp.name),
problem=str(problem_temp.name),
num_plans=num_plans,
quality_bound=1.0,
timeout=timeout,
)
# print(plans)
if plans is None or len(plans["plans"]) == 0:
return None
return plans["plans"]
def generate_top_q_plans(domain, problem, num_plans=10, quality_bound=1.0, timeout=30):
# print("Running K* planner")
plans = kp.plan_unordered_topq(
domain_file=Path(domain),
problem_file=Path(problem),
number_of_plans_bound=num_plans,
quality_bound=quality_bound,
timeout=timeout,
)
return plans
# Used in (action) reachability
def is_unsolvable_new_goal(domain, problem, new_goal):
goal = extract_goal(problem)
new_problem = problem.replace(goal, f"(:goal {new_goal} )")
return is_unsolvable(domain, new_problem)
def is_unsolvable(domain, problem):
with (
tempfile.NamedTemporaryFile() as domain_temp,
tempfile.NamedTemporaryFile() as problem_temp,
):
with open(str(domain_temp.name), "w", encoding="utf8") as file:
file.write(str(domain))
with open(str(problem_temp.name), "w", encoding="utf8") as file:
file.write(str(problem))
plans = kp.plan_unordered_topq(
domain_file=Path(str(domain_temp.name)),
problem_file=Path(str(problem_temp.name)),
quality_bound=1.0,
number_of_plans_bound=1,
timeout=3,
)
if len(plans["planner_error"]) > 0:
fl = plans["planner_error"].split("\n")[0]
print(f"Planner error: {fl}")
return False
if plans is None or len(plans["plans"]) == 0:
return plans["unsolvable"]
return False
def extract_goal(prob):
a = prob.split("(:goal")[1]
cp = 1
for i, c in enumerate(a):
if c == ")":
cp -= 1
if c == "(":
cp += 1
if cp == 0:
return "(:goal" + a[: i + 1]
assert False
def entails(state, partialstate):
return partialstate <= state
def progress(state, act):
assert entails(state, act.pres), (
"Cannot progress with inconsistent state / action precondition:\n\t Action: "
+ act.name
+ "\n\t State: \n\t\t"
+ "\n\t\t".join(state)
)
return (state - act.dels) | act.adds
def regress(state, act):
assert len(state & act.dels) == 0, (
"Cannot regress with inconsistent state / action delete effect:\n\t Action: "
+ act.name
+ "\n\t State: \n\t\t"
+ "\n\t\t".join(state)
)
return (state - act.adds) | act.pres
def get_STRIPS(domain, problem):
with (
tempfile.NamedTemporaryFile() as domain_temp,
tempfile.NamedTemporaryFile() as problem_temp,
):
with open(str(domain_temp.name), "w", encoding="utf8") as file:
file.write(domain.lower())
with open(str(problem_temp.name), "w", encoding="utf8") as file:
file.write(problem.lower())
try:
P = STRIPS(str(domain_temp.name), str(problem_temp.name))
return P
except Exception as e:
print(f"||{e}||")
return None
def create_tmp_dom_prob_replace_init(P, state, result_domain_file, result_problem_file):
d, p = P.PDDL_replace_init_pddl_parser(state)
with open(str(result_domain_file.name), "w", encoding="utf8") as file:
file.write(str(d))
with open(str(result_problem_file.name), "w", encoding="utf8") as file:
file.write(str(p))
return d, p
def fix_name(s):
# (act param)
if "(" == s[0] and ")" == s[-1]:
return s[1:-1]
# make it space separated
s = s.replace(", ", " ").replace(",", " ")
# act(param)
if "(" in s:
assert ")" == s[-1], f"Broken name? {s}"
s = s.replace("(", " ").replace(")", "")
# act param
return s
def get_atoms_pddl(d, p, atoms):
objs = set()
preds = defaultdict(list)
for atom in atoms:
a = atom.lower().strip().split(" ")
args = a[1:]
preds[a[0]].append(args)
objs |= set(args)
constants = [o for o in p.objects | d.constants if o.name.lower() in objs]
constants_dict = {}
for c in constants:
constants_dict[c.name.lower()] = c
assert len(objs) == len(constants), (
f"Could not identify all objects: {objs - set(constants_dict.keys())} not found, {set(constants_dict.keys()) - objs} should not be there"
)
state = []
covered_preds = set()
for f in d.predicates:
name = f.name.lower()
if name in preds:
covered_preds.add(name)
assert len(preds[name][0]) == f.arity, (
f"The arity does not match: {preds[name]} vs {f.terms}"
)
# Going over the lists of objects, adding ground predicate for each
for ob in preds[name]:
c = [constants_dict[o] for o in ob]
state.append(f(*c))
assert len(covered_preds) == len(preds.keys()), (
f"Covered predicates: \n{sorted(list(covered_preds))} vs \n{sorted(list(preds.keys()))}"
)
return set(state)
class Action:
def __init__(self, name, pre, add, delete):
self.name = name
self.pres = pre
self.adds = add
self.dels = delete
def __str__(self):
pres = "{" + ", ".join([f"({a})" for a in self.pres]) + "}"
adds = "{" + ", ".join([f"({a})" for a in self.adds]) + "}"
dels = "{" + ", ".join([f"({a})" for a in self.dels]) + "}"
return f"< {self.name}, {pres}, {adds}, {dels} >"
def toJSON(self):
return json.dumps(
{
"name": self.name,
"preconditions": [f"({a})" for a in self.pres],
"add_effects": [f"({a})" for a in self.adds],
"delete_effects": [f"({a})" for a in self.dels],
},
sort_keys=True,
indent=4,
)
def __repr__(self):
return self.name
def __eq__(self, action):
return self.name == action.name
def __hash__(self):
return hash(self.name)
class STRIPS:
def __init__(self, domain, problem):
self.domain_file = domain
self.problem_file = problem
self.reader = PDDLReader(raise_on_error=True)
self.reader.parse_domain(domain)
self.problem = self.reader.parse_instance(problem)
(self.grounded_fluents, init, goal, self.operators, self.grounder) = (
self.ground_problem(self.problem)
)
self.fluents = set([fix_name(str(f)) for f in self.grounded_fluents])
self.fluents_map = dict()
for f in self.grounded_fluents:
self.fluents_map[fix_name(str(f))] = f
self.init = set([fix_name(str(f)) for f in init])
self.goal = set([fix_name(str(f)) for f in goal])
self.actions = set()
self.action_map = {}
self.init_fluents = [self.fluents_map[f] for f in self.init]
self.static_predicates = [i.name for i in self.grounder.static_symbols]
for op in self.operators:
act = self.operator_to_action(op)
self.actions.add(act)
self.action_map[act.name.lower()] = act
def __str__(self):
fluents = "P = {" + ", ".join([f"({a})" for a in self.fluents]) + "}"
init = "I = {" + ", ".join([f"({a})" for a in self.init]) + "}"
goal = "G = {" + ", ".join([f"({a})" for a in self.goal]) + "}"
actions = "A = {" + "\n ".join([a.__str__() for a in self.actions]) + "}"
return fluents + ",\n" + init + "\n" + goal + "\n" + actions
def toJSON(self):
actions = [a.toJSON() for a in self.actions]
return json.dumps(
{
"fluents": list(self.fluents),
"initial_state": list(self.init),
"goal": list(self.goal),
"actions": actions,
},
sort_keys=True,
indent=4,
)
def operator_to_action(self, op, check_fluents=True, check_static=False):
adds = {
fix_name(str(f.atom)) for f in op.effects if isinstance(f, iofs.AddEffect)
} & self.fluents
dels = {
fix_name(str(f.atom)) for f in op.effects if isinstance(f, iofs.DelEffect)
} & self.fluents
pre = self.fix_pre_name(op.precondition)
if check_fluents:
pre = pre & self.fluents
if check_static:
pre = {p for p in pre if p.split()[0] not in self.static_predicates}
act = Action(fix_name(str(op)), pre, adds, dels)
return act
def fix_pre_name(self, precondition):
if not is_atom(precondition):
return {fix_name(str(f)) for f in precondition.subformulas}
return {fix_name(str(precondition))}
def action(self, name):
return self.action_map[fix_name(name).lower()]
def get_action_or_none(self, name):
if "(" in name and ")" != name[-1]:
return None
return self.action_map.get(fix_name(name).lower(), None)
def fluent(self, name):
return fix_name(name)
def static_symbols(self):
return list(self.grounder.static_symbols)
def fluent_symbols(self):
return list(self.grounder.fluent_symbols)
def get_grounded_atoms(self, symbol):
variables = SymbolIndex()
lang = symbol.language
key = "atom_" + symbol.name
model = self.grounder._solve_lp()
if (
key in model
): # in case there is no reachable ground state variable from that fluent symbol
for binding in model[key]:
binding_with_constants = tuple(lang.get(c) for c in binding)
variables.add(StateVariableLite(symbol, binding_with_constants))
return variables
def get_applicable_actions(self, s):
return [a for a in self.actions if entails(s, a.pres)]
def ground_problem(self, problem):
grounder = LPGroundingStrategy(problem, include_variable_inequalities=True)
action_groundings = grounder.ground_actions()
operators = []
for action_name, groundings in action_groundings.items():
action = problem.get_action(action_name)
for grounding in groundings:
operators.append(
ground_schema_into_plain_operator_from_grounding(action, grounding)
)
grounded_fluents = set(
[
grounded_fluent.to_atom()
for grounded_fluent in grounder.ground_state_variables().objects
]
)
init = [f for f in problem.init.as_atoms() if f in grounded_fluents]
if isinstance(problem.goal, tarski.syntax.Atom):
goal = [problem.goal]
else:
goal = [f for f in problem.goal.subformulas if f in grounded_fluents]
return (grounded_fluents, init, goal, operators, grounder)
def get_static(self):
static_symbols = self.static_symbols()
ret = []
for symbol in static_symbols:
ret.extend(self.get_grounded_atoms(symbol))
return set([fix_name(str(x)) for x in ret])
def PDDL_replace_init_pddl_parser(self, s):
d = DomainParser()(open(self.domain_file, "r").read().lower())
p = ProblemParser()(open(self.problem_file, "r").read().lower())
new_state = get_atoms_pddl(d, p, s | self.get_static())
new_p = Problem(
p.name, domain=d, objects=p.objects, init=new_state, goal=p.goal
)
return d, new_p
def parse_ans(response: str, parser: ACPGrammarParser, task: str):
return [parser.parse(clean_answer(resp, task)) for resp in response]
# def parse_ans(response : str, parser : ACPGrammarParser, task : str):
# ans = [parser.parse(clean_answer(resp, task), debug=True) for resp in response]
# if any(elem is None for elem in ans) or any(elem is None for elem in ans[0]):
# return None
# return ans
def remove_garbage(s):
while True:
if s.endswith("."):
s = s[:-1]
elif s.endswith("\n"):
s = s[:-2]
else:
break
return s.rstrip()
def compare_str(s1, s2):
return remove_garbage(s1).lower() == remove_garbage(s2).lower()
def compare(l1, l2):
if not isinstance(l1, list):
return compare_str(l1, l2)
if not isinstance(l2, list):
return False
for i, v in enumerate(l1):
if not compare(v, l2[i]):
return False
return True
def check_prog_response(resp):
if (
"Positive Effects".lower() in resp.lower()
and "Negative Effects".lower() in resp.lower()
):
if "[" not in resp:
return True
return False
def clean_answer(resp, task):
# Minor cleanup
if "progression_gen" in task:
# Check for Positive Effects and Negative Effects instead of separation
if check_prog_response(resp):
# replace **Positive Effects** with "["
# replace **Negative Effects** with "] ["
# append "]" to the end
resp2 = resp.lower()
resp2 = resp2.replace("*", "")
resp2 = resp2.replace("positive effects", "[")
resp2 = resp2.replace("negative effects", "] [")
resp2 = resp2 + "]"
return resp2
if "action_justification_gen" in task:
# Check for "simplified plan:"
if "simplified plan:" in resp.lower():
resp2 = resp.lower()
resp2 = resp2.replace("*", "")
resp2 = resp2.split("simplified plan:")[1]
return resp2
return resp
def get_grammar_task(task):
# print(task)
if task == "reachable_atom_gen":
return "act"
elif task == "progression_gen":
return "progression_list"
elif task == "validation_gen":
return "index"
elif task == "reachable_action_gen":
return "act"
elif task == "action_justification_gen":
return "action_list"
elif task == "landmarks_gen":
return "act"
elif task == "goal_closer_gen":
return "action_name"
elif task == "applicable_actions_gen":
return "action_list"
##############################################################################
# Evaluators
def fix_action_name(a):
assert a.startswith("(") and a.endswith(")")
return "(" + " ".join([x.strip() for x in a[1:-1].split(" ") if len(x) > 0]) + ")"
def str_remove_before_first_parentheses(s):
if s.startswith("("):
return s
try:
return s[s.index("(") :]
except Exception:
return ""
def str_remove_after_last_parentheses(s):
if s.endswith(")"):
return s
i = s.rfind(")")
if i == -1:
return ""
return s[: i + 1]
def cleanup_answer(ans):
if isinstance(ans, str):
ans = str_remove_before_first_parentheses(ans)
ans = str_remove_after_last_parentheses(ans)
ans = ans.lower()
ans = (
ans.replace(")\n(", ")######(")
.replace("),(", ")######(")
.replace(") (", ")######(")
.split("######")
)
return ans
if isinstance(ans, list):
res = []
for x in ans:
res.extend(cleanup_answer(x))
return res
def set_equal(ans1, ans2):
return set(ans1) == set(ans2)
class BaseEvaluator(ABC):
def __init__(self) -> None:
self.scores = []
@abstractmethod
def get_score(self, ans, doc):
pass
def add_scores(self, scores):
self.scores.extend(scores)
def get_avg_score(self):
avg_score = sum(self.scores) / len(self.scores)
return avg_score
def get_evaluator(group):
if group == "applicable_actions_gen":
return ApplicabilityEvaluator()
elif group == "progression_gen":
return ProgressionEvaluator()
elif group == "validation_gen":
return ValidationEvaluator()
elif group == "reachable_atom_gen":
return ReachabilityEvaluator()
elif group == "goal_closer_gen":
return NextActionEvaluator()
elif group == "action_justification_gen":
return JustificationEvaluator()
elif group == "landmarks_gen":
return LandmarksEvaluator()
elif group == "reachable_action_gen":
return ActionReachabilityEvaluator()
assert True, f"Group {group} not found"
"""
Action Reachability task: generate a valid action that is not applicable to any reachable state.
answer: A subset of actions that are known to be unreachable (not an exhaustive set).
It is empty only when we *know* that there are no such actions.
"""
class ActionReachabilityEvaluator(BaseEvaluator):
def get_score(self, ans, doc):
real_answer = doc["answer"]
if not real_answer or len(real_answer) == 0:
# The correct answer is None
self.add_scores(
["none" == x.strip().lower() if x is not None else False for x in ans]
)
else:
for x in ans:
if x is None:
self.scores.append(False)
continue
action = x.strip().lower()
if action in real_answer:
# The answer is in the subset of stored correct answers
self.scores.append(True)
continue
prec = get_action_preconditions(
doc["PDDL_domain"].lower(), doc["PDDL_problem"].lower(), action
)
if prec is None:
# The answer does not correspond to a valid action
self.scores.append(False)
else:
# Need to run a planner on a task with the answer action preconditions as the new goal
prec = f"(and {' '.join(prec)})"
self.scores.append(
is_unsolvable_new_goal(
doc["PDDL_domain"].lower(),
doc["PDDL_problem"].lower(),
prec,
)
)
return self.get_avg_score()
"""
Action Applicability task: generate all actions that are applicable in the current state.
answer: A set of all applicable actions.
"""
class ApplicabilityEvaluator(BaseEvaluator):
def get_score(self, ans, doc):
real_answer = doc["answer"]
real_answer = [a.lower() for a in real_answer]
ans = [[fix_action_name(a) for a in x] if x is not None else None for x in ans]
# Check if the answer is equal (as a set) to the real stored answer
self.add_scores(
[
set_equal(real_answer, cleanup_answer(x)) if x is not None else False
for x in ans
]
)
return self.get_avg_score()
def is_subsequence(plan, new_plan):
i = 0
for a in plan:
if a == new_plan[i]:
i += 1
if len(new_plan) == i:
# Done
return True
return False
def is_subsequence_and_plan(domain, problem, plan, new_plan):
if len(plan) <= len(new_plan):
return False
if not is_subsequence(plan, new_plan):
return False
return is_plan(domain, problem, new_plan)
"""
Justification task: generate a proper subsequence of the given plan that is also a plan.
answer: A list of examples of actions that can be removed (ignored in evaluation).
"""
class JustificationEvaluator(BaseEvaluator):
def get_score(self, ans, doc):
# Sequence of actions (plan) from the question
if "inputs" in doc: # old field name
seq = doc["inputs"][19:-147]
else:
seq = doc["question"][19:-147]
seq = seq.replace(") (", ")######(").split("######")
for x in ans:
if x is None:
self.scores.append(False)
continue
# An answer plan candidate
x = [fix_action_name(a) for a in x]
if len(x) == 0:
# Wrong answer - never an empty sequence
self.scores.append(0)
continue
# Check if the plan candidate from the answer (a) is a proper subsequence of the plan in the question and (b) is a plan.
self.scores.append(
is_subsequence_and_plan(
doc["PDDL_domain"].lower(), doc["PDDL_problem"].lower(), seq, x
)
)
return self.get_avg_score()
"""
Landmarks task: generate a fact that is a non-trivial landmark for the current state.
answer: A list of facts that are found to be landmarks and a list of facts that are found to be non-landmarks.
The questions are generated only for cases where all facts either
(a) hold in the current state,
(b) true in goal,
(c) are found to be landmarks, or
(d) are found to be non-landmarks.
In such cases, the evaluation is simple, it does not require checking whether a fact is a landmark, it was
already done during question generation.
"""
class LandmarksEvaluator(BaseEvaluator):
def get_score(self, ans, doc):
# The set of facts that are found to be landmarks
real_answer = doc["answer"]
real_answer_yes = [a.lower() for a in real_answer["yes"]]
for x in ans:
if x is None:
self.scores.append(False)
continue
if x.strip().lower() in real_answer_yes:
# The answer fact is known to be landmark
self.scores.append(True)
elif x.strip().lower() == "none":
# The answer is none, correct only if there are no known landmarks,
# since we only generate questions when that means that there are no non-trivial landmarks
self.scores.append(len(real_answer_yes) == 0)
else:
# All other cases the answer is incorrect
self.scores.append(False)
return self.get_avg_score()
"""
Next Action task: generate an action that takes us closer to the goal.
answer:
(a) A list of applicable actions that are known to be correct answers
(b) A list of applicable actions that are known to be incorrect answers
(c) The rest of the applicable actions (maybe).
"""
class NextActionEvaluator(BaseEvaluator):
def get_score(self, ans, doc):
real_answer = doc["answer"]
real_answer_yes = [a.lower() for a in real_answer["yes"]]
real_answer_no = [a.lower() for a in real_answer["no"]]
real_answer_maybe = [a.lower() for a in real_answer["maybe"]]
# The cost of the optimal plan from the current state
opt = real_answer.get("opt", None)
for x in ans:
if x is None:
self.scores.append(False)
continue
action = x.strip().lower()
if action in real_answer_yes:
# Known to be correct
self.scores.append(True)
elif action in real_answer_no:
# Known to be incorrect
self.scores.append(False)
elif action not in real_answer_maybe:
# Not applicable, must be incorrect
self.scores.append(False)
else:
# Unknown, need to run a planner to check whether the state that results from applying the action is closer to the goal
# meaning has smaller optimal plan cost.
self.scores.append(
is_on_optimal_plan(
doc["PDDL_domain"].lower(),
doc["PDDL_problem"].lower(),
action,
opt,
)
)
return self.get_avg_score()
"""
Progression task: generate the positive and negative effects of an action in the current state.
answer:
(a) A list of facts that were false and become true, when the action is applied
(b) A list of facts that were true and become false, when the action is applied
"""
class ProgressionEvaluator(BaseEvaluator):
def get_score(self, ans, doc):
real_answer = doc["answer"]
real_answer_pos = [a.lower() for a in real_answer["pos"]]
real_answer_neg = [a.lower() for a in real_answer["neg"]]
for x in ans:
# The answer should be two lists. We allow for a single list and assume that the second one is empty (relaxed evaluation).
if x is None or len(x) > 2 or len(x) < 1:
self.scores.append(False)
else:
p = cleanup_answer(x[0])
if len(x) == 2:
n = cleanup_answer(x[1])
else:
# Assuming the last element is dropped because it is empty
n = []
# Check if the answer is equal as sets to the correct answers.
ans = [set_equal(real_answer_pos, p), set_equal(real_answer_neg, n)]
self.scores.append(all(ans))
return self.get_avg_score()
"""
Reachability task: generate a valid fact that will never become true in any reachable state.
answer: A subset of facts that are known to be unreachable (not an exhaustive set).
It is empty only when we *know* that there are no such facts.
"""
class ReachabilityEvaluator(BaseEvaluator):
def get_score(self, ans, doc):
real_answer = doc["answer"]
real_answer = [f"({x.strip().lower()})" for x in real_answer]
if len(real_answer) == 0:
# The correct answer is None
self.add_scores(
["none" == x.strip().lower() if x is not None else False for x in ans]
)
else:
for x in ans:
if x is None:
self.scores.append(False)
elif x.strip().lower() in real_answer:
# The answer is in the subset of stored correct answers
self.scores.append(True)
else:
# Need to run a planner on a task with the answer fact as the new goal
atom = x.strip().lower()
self.scores.append(
is_unsolvable_new_goal(
doc["PDDL_domain"].lower(),
doc["PDDL_problem"].lower(),
atom,
)
)
return self.get_avg_score()
"""
Validation task: generate an index of the first inapplicable action in the given sequence.
answer: the correct index.
"""
class ValidationEvaluator(BaseEvaluator):
def get_score(self, ans, doc):
real_answer = str(doc["answer"])
assert int(real_answer) >= 0, (
f"The index must be non-negative, received {real_answer}"
)
# Exact match
self.add_scores(
[
real_answer.lower() == x.strip().lower() if x is not None else False
for x in ans
]
)
return self.get_avg_score()
##############################################################################
def dump_item(item, **kwargs):
return json.dumps(item)
def parse_prediction(prediction):
try:
ans = json.loads(prediction.strip())
response = ans.get("answer", None)
return response
except Exception as e:
print(f"Exception occurred {e}")
return prediction
@register_filter("ACP_grammar_filter")
class ACPGrammarFilter(RegexFilter):
"""Filtering Index using"""
def __init__(self, *args, **kwargs):
self.parser = ACPGrammarParser(kwargs["grammar_task"])
self.clean = kwargs["clean"] if "clean" in kwargs else None
def clean_pos_neg(self, resp):
# Check for Positive Effects and Negative Effects instead of separation
if check_prog_response(resp):
resp2 = resp.lower()
resp2 = resp2.replace("*", "")
resp2 = resp2.replace("positive effects", "[")
resp2 = resp2.replace("negative effects", "] [")
resp2 = resp2 + "]"
return resp2
return resp
def clean_simplified_plan(self, resp):
# Check for "simplified plan:"
if "simplified plan:" in resp.lower():
resp2 = resp.lower()
resp2 = resp2.replace("*", "")
resp2 = resp2.split("simplified plan:")[1]
return resp2
return resp
def apply(self, resps, docs):
if self.clean == "pos_neg":
filtered_resps = [
[self.parser.parse(self.clean_pos_neg(r)) for r in resp]
for resp in resps
]
elif self.clean == "simplified plan":
filtered_resps = [
[self.parser.parse(self.clean_simplified_plan(r)) for r in resp]
for resp in resps
]
else:
filtered_resps = [[self.parser.parse(r) for r in resp] for resp in resps]
return filtered_resps
def process_acp_results(doc, results):
return {"score": get_evaluator(doc["group"]).get_score(results, doc)}
def get_score(references, predictions, **kwargs):
# print(f"References: {references}")
# print(f"Predictions: {predictions}")
data = json.loads(references[0].strip())
real_ans = data["answer"]
task = data["group"]
responses = [parse_prediction(prediction) for prediction in predictions]
print(f"Real answer: {real_ans}")
print(f"Model answers: {responses}")
parser = ACPGrammarParser(get_grammar_task(task))
ans = parse_ans(responses, parser, task)
print(f"Parsed model answers: {ans}")
score = get_evaluator(task).get_score(ans, data)
return {"get_score": score}
lm_eval/tasks/acpbench/gen_2shot_with_pddl/act_reach.yaml 0 → 100644
View file @ 601be343
task: acp_areach_gen_with_pddl
dataset_name: acp_areach_gen
include: _gen_yaml_2shot
fewshot_config:
sampler: first_n
samples:
- context: "A robot is in a grid and can only move to places that are connected to its current position. \nThe grid size is 5x5, and the locations are of the form fi-jf (e.g., f3-2f or f0-1f). The grid cells are connected to their neighbors (e.g., f1-2f is connected to the four neighbors f0-2f, f2-2f, f1-1f, and f1-3f). Some positions on the grid are locked and can be opened with a key of a matching shape. The robot has an arm that can pick up a key when the key is in same location as the robot and the arm is empty. \nThere are 2 keys in 0 different shapes: Key key0-1 is of shape shape0, Key key0-0 is of shape shape0. \nCurrently, the robot is at position f3-2f and its arm is empty. All the positions are open except the following: f2-0f has shape0 shaped lock, f4-2f has shape0 shaped lock. Key key0-1 is at position f1-3f. Key key0-0 is at position f2-2f. The available actions are: (unlock ?curpos ?lockpos ?key ?shape) - unlock the place ?lockpos with key ?key of shape ?shape from the current position place ?curpos, (move ?curpos ?nextpos) - move to place ?nextpos from place ?curpos, (pickup ?curpos ?key) - acquire the key ?key from the place ?curpos, (pickup-and-loose ?curpos ?newkey ?oldkey) - pick up key ?newkey at current position place ?curpos and loose key ?oldkey being held, and (putdown ?curpos ?key) - put down key ?key at current position place ?curpos."
question: "What action can never become applicable, in any state reachable from the current state?"
answer: "(pickup-and-loose f0-1f key0-0 key0-0)"
PDDL_domain: "(define (domain grid)\n (:requirements :strips :typing)\n (:types key place shape - object)\n (:predicates (arm-empty) (at ?r - key ?x - place) (at-robot ?x - place) (conn ?x - place ?y - place) (holding ?k - key) (key-shape ?k - key ?s - shape) (lock-shape ?x - place ?s - shape) (locked ?x - place) (open ?x - place))\n (:action move\n :parameters (?curpos - place ?nextpos - place)\n :precondition (and (at-robot ?curpos) (conn ?curpos ?nextpos) (open ?nextpos))\n :effect (and (at-robot ?nextpos) (not (at-robot ?curpos)))\n )\n (:action pickup\n :parameters (?curpos - place ?key - key)\n :precondition (and (at-robot ?curpos) (at ?key ?curpos) (arm-empty))\n :effect (and (holding ?key) (not (at ?key ?curpos)) (not (arm-empty)))\n )\n (:action pickup-and-loose\n :parameters (?curpos - place ?newkey - key ?oldkey - key)\n :precondition (and (at-robot ?curpos) (holding ?oldkey) (at ?newkey ?curpos))\n :effect (and (holding ?newkey) (at ?oldkey ?curpos) (not (holding ?oldkey)) (not (at ?newkey ?curpos)))\n )\n (:action putdown\n :parameters (?curpos - place ?key - key)\n :precondition (and (at-robot ?curpos) (holding ?key))\n :effect (and (arm-empty) (at ?key ?curpos) (not (holding ?key)))\n )\n (:action unlock\n :parameters (?curpos - place ?lockpos - place ?key - key ?shape - shape)\n :precondition (and (conn ?curpos ?lockpos) (key-shape ?key ?shape) (lock-shape ?lockpos ?shape) (at-robot ?curpos) (locked ?lockpos) (holding ?key))\n :effect (and (open ?lockpos) (not (locked ?lockpos)))\n )\n)"
PDDL_problem: "(define (problem grid-x5-y5-t1-k2-l2-p100)\n (:domain grid)\n (:requirements :strips :typing)\n (:objects key0-0 key0-1 - key f0-0f f0-1f f0-2f f0-3f f0-4f f1-0f f1-1f f1-2f f1-3f f1-4f f2-0f f2-1f f2-2f f2-3f f2-4f f3-0f f3-1f f3-2f f3-3f f3-4f f4-0f f4-1f f4-2f f4-3f f4-4f - place shape0 - shape)\n (:init (arm-empty) (at key0-0 f2-2f) (at key0-1 f1-3f) (at-robot f3-2f) (conn f0-0f f0-1f) (conn f0-0f f1-0f) (conn f0-1f f0-0f) (conn f0-1f f0-2f) (conn f0-1f f1-1f) (conn f0-2f f0-1f) (conn f0-2f f0-3f) (conn f0-2f f1-2f) (conn f0-3f f0-2f) (conn f0-3f f0-4f) (conn f0-3f f1-3f) (conn f0-4f f0-3f) (conn f0-4f f1-4f) (conn f1-0f f0-0f) (conn f1-0f f1-1f) (conn f1-0f f2-0f) (conn f1-1f f0-1f) (conn f1-1f f1-0f) (conn f1-1f f1-2f) (conn f1-1f f2-1f) (conn f1-2f f0-2f) (conn f1-2f f1-1f) (conn f1-2f f1-3f) (conn f1-2f f2-2f) (conn f1-3f f0-3f) (conn f1-3f f1-2f) (conn f1-3f f1-4f) (conn f1-3f f2-3f) (conn f1-4f f0-4f) (conn f1-4f f1-3f) (conn f1-4f f2-4f) (conn f2-0f f1-0f) (conn f2-0f f2-1f) (conn f2-0f f3-0f) (conn f2-1f f1-1f) (conn f2-1f f2-0f) (conn f2-1f f2-2f) (conn f2-1f f3-1f) (conn f2-2f f1-2f) (conn f2-2f f2-1f) (conn f2-2f f2-3f) (conn f2-2f f3-2f) (conn f2-3f f1-3f) (conn f2-3f f2-2f) (conn f2-3f f2-4f) (conn f2-3f f3-3f) (conn f2-4f f1-4f) (conn f2-4f f2-3f) (conn f2-4f f3-4f) (conn f3-0f f2-0f) (conn f3-0f f3-1f) (conn f3-0f f4-0f) (conn f3-1f f2-1f) (conn f3-1f f3-0f) (conn f3-1f f3-2f) (conn f3-1f f4-1f) (conn f3-2f f2-2f) (conn f3-2f f3-1f) (conn f3-2f f3-3f) (conn f3-2f f4-2f) (conn f3-3f f2-3f) (conn f3-3f f3-2f) (conn f3-3f f3-4f) (conn f3-3f f4-3f) (conn f3-4f f2-4f) (conn f3-4f f3-3f) (conn f3-4f f4-4f) (conn f4-0f f3-0f) (conn f4-0f f4-1f) (conn f4-1f f3-1f) (conn f4-1f f4-0f) (conn f4-1f f4-2f) (conn f4-2f f3-2f) (conn f4-2f f4-1f) (conn f4-2f f4-3f) (conn f4-3f f3-3f) (conn f4-3f f4-2f) (conn f4-3f f4-4f) (conn f4-4f f3-4f) (conn f4-4f f4-3f) (key-shape key0-0 shape0) (key-shape key0-1 shape0) (lock-shape f2-0f shape0) (lock-shape f4-2f shape0) (locked f2-0f) (locked f4-2f) (open f0-0f) (open f0-1f) (open f0-2f) (open f0-3f) (open f0-4f) (open f1-0f) (open f1-1f) (open f1-2f) (open f1-3f) (open f1-4f) (open f2-1f) (open f2-2f) (open f2-3f) (open f2-4f) (open f3-0f) (open f3-1f) (open f3-2f) (open f3-3f) (open f3-4f) (open f4-0f) (open f4-1f) (open f4-3f) (open f4-4f))\n (:goal (and (at key0-0 f2-0f) (at key0-1 f1-3f)))\n)"
- context: "There are several cities, each containing several locations, some of which are airports. There are also trucks, which can drive within a single city, and airplanes, which can fly between airports. The goal is to get some packages from various locations to various new locations. \nThere are 2 trucks and 1 airplane, as well as 4 packages. There are 4 locations across 2 cities. \nThe locations are in cities as follows: l1-0 and l1-1 are in c1; l0-0 and l0-1 are in c0. \nCurrently, a0, p1, and p2 are at l1-0, t0 is at l0-1, p3 and p0 are at l0-0, t1 is at l1-1. The available actions are: (load-truck ?obj ?truck ?loc) - place the object ?obj into the truck ?truck at location ?loc, (load-airplane ?obj ?airplane ?loc) - load the object ?obj from location ?loc into the airplane ?airplane, (unload-truck ?obj ?truck ?loc) - unload object ?obj from truck ?truck at location ?loc, (unload-airplane ?obj ?airplane ?loc) - unload object ?obj from airplane ?airplane at location ?loc, (drive-truck ?truck ?loc-from ?loc-to ?city) - navigate the truck ?truck from its current location ?loc-from in city ?city to the new location ?loc-to within the same city, and (fly-airplane ?airplane ?loc-from ?loc-to) - fly airplane ?airplane from airport ?loc-from to airport ?loc-to."
question: "What action can never become applicable, in any state reachable from the current state??"
answer: "(load-truck p2 t0 l1-1)"
PDDL_domain: "(define (domain logistics-strips)\n (:requirements :strips :typing) \n\n (:types \n location locatable city - object \n package movable - locatable\n airport - location\n airplane truck - movable \n )\t\t\n \n (:predicates \t\n\t\t(at ?obj - locatable ?loc - location)\n\t\t(in ?obj1 - package ?obj2 - movable)\n\t\t(in-city ?obj - location ?city - city))\n\n\n(:action LOAD-TRUCK\n :parameters\n (?obj - package\n ?truck - truck\n ?loc - location)\n :precondition\n (and \n (at ?truck ?loc) (at ?obj ?loc))\n :effect\n (and (not (at ?obj ?loc)) (in ?obj ?truck)))\n\n(:action LOAD-AIRPLANE\n :parameters\n (?obj - package\n ?airplane - airplane\n ?loc - location)\n :precondition\n (and \n (at ?obj ?loc) (at ?airplane ?loc))\n :effect\n (and (not (at ?obj ?loc)) (in ?obj ?airplane)))\n\n\n\n(:action UNLOAD-TRUCK\n :parameters\n (?obj - package\n ?truck - truck\n ?loc - location)\n :precondition\n (and \n (at ?truck ?loc) (in ?obj ?truck))\n :effect\n (and (not (in ?obj ?truck)) (at ?obj ?loc)))\n\n(:action UNLOAD-AIRPLANE\n :parameters\n (?obj - package\n ?airplane - airplane\n ?loc - location)\n :precondition\n (and \n (in ?obj ?airplane) (at ?airplane ?loc))\n :effect\n (and (not (in ?obj ?airplane)) (at ?obj ?loc)))\n\n(:action DRIVE-TRUCK\n :parameters\n (?truck - truck\n ?loc-from - location\n ?loc-to - location\n ?city - city)\n :precondition\n (and \n (at ?truck ?loc-from)\n (in-city ?loc-from ?city)\n (in-city ?loc-to ?city))\n :effect\n (and (not (at ?truck ?loc-from)) (at ?truck ?loc-to)))\n\n(:action FLY-AIRPLANE\n :parameters\n (?airplane - airplane\n ?loc-from - airport\n ?loc-to - airport)\n :precondition\n (and \n\t(at ?airplane ?loc-from))\n :effect\n (and (not (at ?airplane ?loc-from)) (at ?airplane ?loc-to)))\n)"
PDDL_problem: "(define (problem logistics-c2-s2-p4-a1)\n (:domain logistics-strips)\n (:requirements :strips :typing)\n (:objects a0 - airplane l0-0 l1-0 - airport c0 c1 - city l0-1 l1-1 - location p0 p1 p2 p3 - package t0 t1 - truck)\n (:init (at a0 l1-0) (at p0 l0-0) (at p1 l1-0) (at p2 l1-0) (at p3 l0-0) (at t0 l0-1) (at t1 l1-1) (in-city l0-0 c0) (in-city l0-1 c0) (in-city l1-0 c1) (in-city l1-1 c1))\n (:goal (and (at p0 l0-0) (at p1 l1-0) (at p2 l1-0) (at p3 l0-1)))\n)"
doc_to_text: "# PDDL DOMAIN \n\n```\n{{PDDL_domain}}\n```\n\n# PDDL PROBLEM \n\n```\n{{PDDL_problem}}\n```\n\n**Question**: {{context}} {{question}} Each action starts with an opening parenthesis and ends with closing parenthesis. Provide one action or None. **Final Answer**:"
filter_list:
- name: "acp_grammar_parse"
filter:
- function: "ACP_grammar_filter"
grammar_task: "act"
- function: "take_first"
lm_eval/tasks/acpbench/gen_2shot_with_pddl/app.yaml 0 → 100644
View file @ 601be343
task: acp_app_gen_with_pddl
dataset_name: acp_app_gen
include: _gen_yaml_2shot
fewshot_config:
sampler: first_n
samples:
- context: "A robot is in a grid and can only move to places that are connected to its current position. \nThe grid size is 5x5, and the locations are of the form fi-jf (e.g., f3-2f or f0-1f). The grid cells are connected to their neighbors (e.g., f1-2f is connected to the four neighbors f0-2f, f2-2f, f1-1f, and f1-3f). Some positions on the grid are locked and can be opened with a key of a matching shape. The robot has an arm that can pick up a key when the key is in same location as the robot and the arm is empty. \nThere are 2 keys in 0 different shapes: Key key0-0 is of shape shape0, Key key0-1 is of shape shape0. \nCurrently, the robot is at position f4-3f and its arm is empty. All the positions are open except the following: f2-0f has shape0 shaped lock, f4-2f has shape0 shaped lock. Key key0-0 is at position f3-1f. Key key0-1 is at position f1-3f. The available actions are: (unlock ?curpos ?lockpos ?key ?shape) - use the key ?key of shape ?shape to unlock the place ?lockpos from the current position ?curpos, (move ?curpos ?nextpos) - transition from the current position ?curpos to the next position ?nextpos, (pickup ?curpos ?key) - pick up key ?key from place ?curpos, (pickup-and-loose ?curpos ?newkey ?oldkey) - pick up the key ?newkey from the current position ?curpos and loose the key ?oldkey which is being held, and (putdown ?curpos ?key) - place the key ?key at the current position ?curpos."
question: "Generate the list of all ground actions that are applicable in this state."
answer: "[(move f4-3f f3-3f), (move f4-3f f4-4f)]"
PDDL_domain: "(define (domain grid)\n (:requirements :strips :typing)\n (:types key place shape - object)\n (:predicates (arm-empty) (at ?r - key ?x - place) (at-robot ?x - place) (conn ?x - place ?y - place) (holding ?k - key) (key-shape ?k - key ?s - shape) (lock-shape ?x - place ?s - shape) (locked ?x - place) (open ?x - place))\n (:action move\n :parameters (?curpos - place ?nextpos - place)\n :precondition (and (at-robot ?curpos) (conn ?curpos ?nextpos) (open ?nextpos))\n :effect (and (at-robot ?nextpos) (not (at-robot ?curpos)))\n )\n (:action pickup\n :parameters (?curpos - place ?key - key)\n :precondition (and (at-robot ?curpos) (at ?key ?curpos) (arm-empty))\n :effect (and (holding ?key) (not (at ?key ?curpos)) (not (arm-empty)))\n )\n (:action pickup-and-loose\n :parameters (?curpos - place ?newkey - key ?oldkey - key)\n :precondition (and (at-robot ?curpos) (holding ?oldkey) (at ?newkey ?curpos))\n :effect (and (holding ?newkey) (at ?oldkey ?curpos) (not (holding ?oldkey)) (not (at ?newkey ?curpos)))\n )\n (:action putdown\n :parameters (?curpos - place ?key - key)\n :precondition (and (at-robot ?curpos) (holding ?key))\n :effect (and (arm-empty) (at ?key ?curpos) (not (holding ?key)))\n )\n (:action unlock\n :parameters (?curpos - place ?lockpos - place ?key - key ?shape - shape)\n :precondition (and (conn ?curpos ?lockpos) (key-shape ?key ?shape) (lock-shape ?lockpos ?shape) (at-robot ?curpos) (locked ?lockpos) (holding ?key))\n :effect (and (open ?lockpos) (not (locked ?lockpos)))\n )\n)"
PDDL_problem: "(define (problem grid-x5-y5-t1-k2-l2-p100)\n (:domain grid)\n (:requirements :strips :typing)\n (:objects key0-0 key0-1 - key f0-0f f0-1f f0-2f f0-3f f0-4f f1-0f f1-1f f1-2f f1-3f f1-4f f2-0f f2-1f f2-2f f2-3f f2-4f f3-0f f3-1f f3-2f f3-3f f3-4f f4-0f f4-1f f4-2f f4-3f f4-4f - place shape0 - shape)\n (:init (arm-empty) (at key0-0 f3-1f) (at key0-1 f1-3f) (at-robot f4-3f) (conn f0-0f f0-1f) (conn f0-0f f1-0f) (conn f0-1f f0-0f) (conn f0-1f f0-2f) (conn f0-1f f1-1f) (conn f0-2f f0-1f) (conn f0-2f f0-3f) (conn f0-2f f1-2f) (conn f0-3f f0-2f) (conn f0-3f f0-4f) (conn f0-3f f1-3f) (conn f0-4f f0-3f) (conn f0-4f f1-4f) (conn f1-0f f0-0f) (conn f1-0f f1-1f) (conn f1-0f f2-0f) (conn f1-1f f0-1f) (conn f1-1f f1-0f) (conn f1-1f f1-2f) (conn f1-1f f2-1f) (conn f1-2f f0-2f) (conn f1-2f f1-1f) (conn f1-2f f1-3f) (conn f1-2f f2-2f) (conn f1-3f f0-3f) (conn f1-3f f1-2f) (conn f1-3f f1-4f) (conn f1-3f f2-3f) (conn f1-4f f0-4f) (conn f1-4f f1-3f) (conn f1-4f f2-4f) (conn f2-0f f1-0f) (conn f2-0f f2-1f) (conn f2-0f f3-0f) (conn f2-1f f1-1f) (conn f2-1f f2-0f) (conn f2-1f f2-2f) (conn f2-1f f3-1f) (conn f2-2f f1-2f) (conn f2-2f f2-1f) (conn f2-2f f2-3f) (conn f2-2f f3-2f) (conn f2-3f f1-3f) (conn f2-3f f2-2f) (conn f2-3f f2-4f) (conn f2-3f f3-3f) (conn f2-4f f1-4f) (conn f2-4f f2-3f) (conn f2-4f f3-4f) (conn f3-0f f2-0f) (conn f3-0f f3-1f) (conn f3-0f f4-0f) (conn f3-1f f2-1f) (conn f3-1f f3-0f) (conn f3-1f f3-2f) (conn f3-1f f4-1f) (conn f3-2f f2-2f) (conn f3-2f f3-1f) (conn f3-2f f3-3f) (conn f3-2f f4-2f) (conn f3-3f f2-3f) (conn f3-3f f3-2f) (conn f3-3f f3-4f) (conn f3-3f f4-3f) (conn f3-4f f2-4f) (conn f3-4f f3-3f) (conn f3-4f f4-4f) (conn f4-0f f3-0f) (conn f4-0f f4-1f) (conn f4-1f f3-1f) (conn f4-1f f4-0f) (conn f4-1f f4-2f) (conn f4-2f f3-2f) (conn f4-2f f4-1f) (conn f4-2f f4-3f) (conn f4-3f f3-3f) (conn f4-3f f4-2f) (conn f4-3f f4-4f) (conn f4-4f f3-4f) (conn f4-4f f4-3f) (key-shape key0-0 shape0) (key-shape key0-1 shape0) (lock-shape f2-0f shape0) (lock-shape f4-2f shape0) (locked f2-0f) (locked f4-2f) (open f0-0f) (open f0-1f) (open f0-2f) (open f0-3f) (open f0-4f) (open f1-0f) (open f1-1f) (open f1-2f) (open f1-3f) (open f1-4f) (open f2-1f) (open f2-2f) (open f2-3f) (open f2-4f) (open f3-0f) (open f3-1f) (open f3-2f) (open f3-3f) (open f3-4f) (open f4-0f) (open f4-1f) (open f4-3f) (open f4-4f))\n (:goal (and (at key0-0 f2-0f) (at key0-1 f1-3f)))\n)"
- context: "There are several cities, each containing several locations, some of which are airports. There are also trucks, which can drive within a single city, and airplanes, which can fly between airports. The goal is to get some packages from various locations to various new locations. \nThere are 2 trucks and 1 airplane, as well as 4 packages. There are 4 locations across 2 cities. \nThe locations are in cities as follows: l0-1 and l0-0 are in c0; l1-1 and l1-0 are in c1. \nCurrently, t1 is at l1-0, p0, a0, t0, and p3 are at l0-0, p1 and p2 are in t1. The available actions are: (load-truck ?obj ?truck ?loc) - load the object ?obj from location ?loc into the truck ?truck, (load-airplane ?obj ?airplane ?loc) - place the object ?obj onto the airplane ?airplane at location ?loc, (unload-truck ?obj ?truck ?loc) - remove the object ?obj from the truck ?truck and place it on the location ?loc, (unload-airplane ?obj ?airplane ?loc) - unload object ?obj from airplane ?airplane at location ?loc, (drive-truck ?truck ?loc-from ?loc-to ?city) - navigate the truck ?truck from location ?loc-from in city ?city to location ?loc-to in the same city, and (fly-airplane ?airplane ?loc-from ?loc-to) - fly airplane ?airplane from airport ?loc-from to airport ?loc-to."
question: "Generate the list of all ground actions that are applicable in this state."
answer: "[(unload-truck p2 t1 l1-0), (drive-truck t0 l0-0 l0-0 c0), (load-airplane p0 a0 l0-0), (load-truck p0 t0 l0-0), (unload-truck p1 t1 l1-0), (drive-truck t1 l1-0 l1-0 c1), (drive-truck t0 l0-0 l0-1 c0), (drive-truck t1 l1-0 l1-1 c1), (fly-airplane a0 l0-0 l0-0), (load-truck p3 t0 l0-0), (fly-airplane a0 l0-0 l1-0), (load-airplane p3 a0 l0-0)]"
PDDL_domain: "(define (domain logistics-strips)\n (:requirements :strips :typing) \n\n (:types \n location locatable city - object \n package movable - locatable\n airport - location\n airplane truck - movable \n )\t\t\n \n (:predicates \t\n\t\t(at ?obj - locatable ?loc - location)\n\t\t(in ?obj1 - package ?obj2 - movable)\n\t\t(in-city ?obj - location ?city - city))\n\n\n(:action LOAD-TRUCK\n :parameters\n (?obj - package\n ?truck - truck\n ?loc - location)\n :precondition\n (and \n (at ?truck ?loc) (at ?obj ?loc))\n :effect\n (and (not (at ?obj ?loc)) (in ?obj ?truck)))\n\n(:action LOAD-AIRPLANE\n :parameters\n (?obj - package\n ?airplane - airplane\n ?loc - location)\n :precondition\n (and \n (at ?obj ?loc) (at ?airplane ?loc))\n :effect\n (and (not (at ?obj ?loc)) (in ?obj ?airplane)))\n\n\n\n(:action UNLOAD-TRUCK\n :parameters\n (?obj - package\n ?truck - truck\n ?loc - location)\n :precondition\n (and \n (at ?truck ?loc) (in ?obj ?truck))\n :effect\n (and (not (in ?obj ?truck)) (at ?obj ?loc)))\n\n(:action UNLOAD-AIRPLANE\n :parameters\n (?obj - package\n ?airplane - airplane\n ?loc - location)\n :precondition\n (and \n (in ?obj ?airplane) (at ?airplane ?loc))\n :effect\n (and (not (in ?obj ?airplane)) (at ?obj ?loc)))\n\n(:action DRIVE-TRUCK\n :parameters\n (?truck - truck\n ?loc-from - location\n ?loc-to - location\n ?city - city)\n :precondition\n (and \n (at ?truck ?loc-from)\n (in-city ?loc-from ?city)\n (in-city ?loc-to ?city))\n :effect\n (and (not (at ?truck ?loc-from)) (at ?truck ?loc-to)))\n\n(:action FLY-AIRPLANE\n :parameters\n (?airplane - airplane\n ?loc-from - airport\n ?loc-to - airport)\n :precondition\n (and \n\t(at ?airplane ?loc-from))\n :effect\n (and (not (at ?airplane ?loc-from)) (at ?airplane ?loc-to)))\n)"
PDDL_problem: "(define (problem logistics-c2-s2-p4-a1)\n (:domain logistics-strips)\n (:requirements :strips :typing)\n (:objects a0 - airplane l0-0 l1-0 - airport c0 c1 - city l0-1 l1-1 - location p0 p1 p2 p3 - package t0 t1 - truck)\n (:init (at a0 l0-0) (at p0 l0-0) (at p3 l0-0) (at t0 l0-0) (at t1 l1-0) (in p1 t1) (in p2 t1) (in-city l0-0 c0) (in-city l0-1 c0) (in-city l1-0 c1) (in-city l1-1 c1))\n (:goal (and (at p0 l0-0) (at p1 l1-0) (at p2 l1-0) (at p3 l0-1)))\n)"
doc_to_text: "# PDDL DOMAIN \n\n```\n{{PDDL_domain}}\n```\n\n# PDDL PROBLEM \n\n```\n{{PDDL_problem}}\n```\n\n**Question**: {{context}} {{question}} Each action starts with an opening parenthesis and ends with closing parenthesis. Provide only the actions. \n**Final Answer**:"
filter_list:
- name: "acp_grammar_parse"
filter:
- function: "ACP_grammar_filter"
grammar_task: "action_list"
- function: "take_first"
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment