[Refactor] Move fix_id_list to Retriever (#442)

* [Refactor] Move fix_id_list to Retriever

* update

* move to base

* fix

configs/datasets/GLUE_CoLA/GULE_CoLA_ppl_77d0df.py

@@ -23,8 +23,8 @@ CoLA_infer_cfg = dict(
         },
         ice_token='</E>',
     ),
-    retriever=dict(type=FixKRetriever),
-    inferencer=dict(type=PPLInferencer, fix_id_list=[17, 18, 19, 20, 21]))
+    retriever=dict(type=FixKRetriever, fix_id_list=[17, 18, 19, 20, 21]),
+    inferencer=dict(type=PPLInferencer))
 
 CoLA_eval_cfg = dict(evaluator=dict(type=AccEvaluator), )
 

configs/datasets/GLUE_QQP/GLUE_QQP_ppl_250d00.py

@@ -22,8 +22,8 @@ QQP_infer_cfg = dict(
         },
         ice_token='</E>',
     ),
-    retriever=dict(type=FixKRetriever),
-    inferencer=dict(type=PPLInferencer, fix_id_list=[0, 1, 2, 3, 4]))
+    retriever=dict(type=FixKRetriever, fix_id_list=[0, 1, 2, 3, 4]),
+    inferencer=dict(type=PPLInferencer))
 
 QQP_eval_cfg = dict(evaluator=dict(type=AccEvaluator), )
 

configs/datasets/ceval/ceval_gen_2daf24.py

@@ -161,8 +161,8 @@ for _split in ["val", "test"]:
                     ]),
                 ice_token="</E>",
             ),
-            retriever=dict(type=FixKRetriever),
-            inferencer=dict(type=GenInferencer, fix_id_list=[0, 1, 2, 3, 4]),
+            retriever=dict(type=FixKRetriever, fix_id_list=[0, 1, 2, 3, 4]),
+            inferencer=dict(type=GenInferencer),
         )
 
         ceval_eval_cfg = dict(

configs/datasets/ceval/ceval_gen_5f30c7.py

@@ -161,8 +161,8 @@ for _split in ["val"]:
                     ]),
                 ice_token="</E>",
             ),
-            retriever=dict(type=FixKRetriever),
-            inferencer=dict(type=GenInferencer, fix_id_list=[0, 1, 2, 3, 4]),
+            retriever=dict(type=FixKRetriever, fix_id_list=[0, 1, 2, 3, 4]),
+            inferencer=dict(type=GenInferencer),
         )
 
         ceval_eval_cfg = dict(

configs/datasets/ceval/ceval_ppl_578f8d.py

@@ -163,8 +163,8 @@ for _split in ["val"]:
                 },
                 ice_token="</E>",
             ),
-            retriever=dict(type=FixKRetriever),
-            inferencer=dict(type=PPLInferencer, fix_id_list=[0, 1, 2, 3, 4]),
+            retriever=dict(type=FixKRetriever, fix_id_list=[0, 1, 2, 3, 4]),
+            inferencer=dict(type=PPLInferencer),
         )
 
         ceval_eval_cfg = dict(evaluator=dict(type=AccEvaluator))

configs/datasets/ceval/ceval_ppl_93e5ce.py

@@ -163,8 +163,8 @@ for _split in ["val", "test"]:
                 },
                 ice_token="</E>",
             ),
-            retriever=dict(type=FixKRetriever),
-            inferencer=dict(type=PPLInferencer, fix_id_list=[0, 1, 2, 3, 4]),
+            retriever=dict(type=FixKRetriever, fix_id_list=[0, 1, 2, 3, 4]),
+            inferencer=dict(type=PPLInferencer),
         )
 
         ceval_eval_cfg = dict(evaluator=dict(type=AccEvaluator))

configs/datasets/cmb/cmb_gen_72cbb7.py

@@ -28,8 +28,8 @@ cmb_infer_cfg = dict(
         ),
         ice_token="</E>",
     ),
-    retriever=dict(type=FixKRetriever),
-    inferencer=dict(type=GenInferencer, fix_id_list=[0, 1, 2, 3, 4]),
+    retriever=dict(type=FixKRetriever, fix_id_list=[0, 1, 2, 3, 4]),
+    inferencer=dict(type=GenInferencer),
 )
 
 cmb_datasets.append(

configs/datasets/cmmlu/cmmlu_gen_c13365.py

@@ -96,8 +96,8 @@ for _name in cmmlu_all_sets:
                 ]),
             ice_token="</E>",
         ),
-        retriever=dict(type=FixKRetriever),
-        inferencer=dict(type=GenInferencer, fix_id_list=[0, 1, 2, 3, 4]),
+        retriever=dict(type=FixKRetriever, fix_id_list=[0, 1, 2, 3, 4]),
+        inferencer=dict(type=GenInferencer),
     )
 
     cmmlu_eval_cfg = dict(

configs/datasets/cmmlu/cmmlu_ppl_8b9c76.py

@@ -98,8 +98,8 @@ for _name in cmmlu_all_sets:
             },
             ice_token="</E>",
         ),
-        retriever=dict(type=FixKRetriever),
-        inferencer=dict(type=PPLInferencer, fix_id_list=[0, 1, 2, 3, 4]),
+        retriever=dict(type=FixKRetriever, fix_id_list=[0, 1, 2, 3, 4]),
+        inferencer=dict(type=PPLInferencer),
     )
 
     cmmlu_eval_cfg = dict(evaluator=dict(type=AccEvaluator))

configs/datasets/mmlu/mmlu_gen_23a9a9.py

@@ -29,8 +29,8 @@ mmlu_infer_cfg = dict(
             dict(role='BOT', prompt='{target}\n')
         ])),
     prompt_template=mmlu_prompt_template,
-    retriever=dict(type=FixKRetriever),
-    inferencer=dict(type=GenInferencer, fix_id_list=[0, 1, 2, 3, 4]))
+    retriever=dict(type=FixKRetriever, fix_id_list=[0, 1, 2, 3, 4]),
+    inferencer=dict(type=GenInferencer))
 
 mmlu_eval_cfg = dict(
     evaluator=dict(type=AccEvaluator),

configs/datasets/mmlu/mmlu_gen_5d1409.py

@@ -102,8 +102,8 @@ for _name in mmlu_all_sets:
             ),
             ice_token="</E>",
         ),
-        retriever=dict(type=FixKRetriever),
-        inferencer=dict(type=GenInferencer, fix_id_list=[0, 1, 2, 3, 4]),
+        retriever=dict(type=FixKRetriever, fix_id_list=[0, 1, 2, 3, 4]),
+        inferencer=dict(type=GenInferencer),
     )
 
     mmlu_eval_cfg = dict(

configs/datasets/mmlu/mmlu_gen_79e572.py

@@ -87,8 +87,8 @@ for _name in mmlu_all_sets:
             f"{_hint}</E>{{input}}\nA. {{A}}\nB. {{B}}\nC. {{C}}\nD. {{D}}\nAnswer:",
             ice_token="</E>",
         ),
-        retriever=dict(type=FixKRetriever),
-        inferencer=dict(type=GenInferencer, fix_id_list=[0, 1, 2, 3, 4]),
+        retriever=dict(type=FixKRetriever, fix_id_list=[0, 1, 2, 3, 4]),
+        inferencer=dict(type=GenInferencer),
     )
 
     mmlu_eval_cfg = dict(

configs/datasets/mmlu/mmlu_gen_a484b3.py

@@ -102,8 +102,8 @@ for _name in mmlu_all_sets:
             ),
             ice_token="</E>",
         ),
-        retriever=dict(type=FixKRetriever),
-        inferencer=dict(type=GenInferencer, fix_id_list=[0, 1, 2, 3, 4]),
+        retriever=dict(type=FixKRetriever, fix_id_list=[0, 1, 2, 3, 4]),
+        inferencer=dict(type=GenInferencer),
     )
 
     mmlu_eval_cfg = dict(

configs/datasets/mmlu/mmlu_ppl_ac766d.py

@@ -93,8 +93,8 @@ for _name in mmlu_all_sets:
             },
             ice_token="</E>",
         ),
-        retriever=dict(type=FixKRetriever),
-        inferencer=dict(type=PPLInferencer, fix_id_list=[0, 1, 2, 3, 4]),
+        retriever=dict(type=FixKRetriever, fix_id_list=[0, 1, 2, 3, 4]),
+        inferencer=dict(type=PPLInferencer),
     )
 
     mmlu_eval_cfg = dict(evaluator=dict(type=AccEvaluator), )

configs/datasets/nq/nq_gen_0356ec.py

@@ -44,8 +44,8 @@ for k in [0, 1, 5]:
                 ),
                 ice_token="</E>",
             ),
-            retriever=dict(type=FixKRetriever),
-            inferencer=dict(type=GenInferencer, max_out_len=50, fix_id_list=list(range(k))),
+            retriever=dict(type=FixKRetriever, fix_id_list=list(range(k))),
+            inferencer=dict(type=GenInferencer, max_out_len=50),
         )
 
     nq_eval_cfg = dict(evaluator=dict(type=NQEvaluator), pred_role="BOT")

configs/datasets/triviaqa/triviaqa_gen_0356ec.py

@@ -45,8 +45,8 @@ for k in [0, 1, 5]:
                 ),
                 ice_token="</E>",
             ),
-            retriever=dict(type=FixKRetriever),
-            inferencer=dict(type=GenInferencer, max_out_len=50, fix_id_list=list(range(k))),
+            retriever=dict(type=FixKRetriever, fix_id_list=list(range(k))),
+            inferencer=dict(type=GenInferencer, max_out_len=50),
         )
 
     triviaqa_eval_cfg = dict(evaluator=dict(type=TriviaQAEvaluator), pred_role="BOT")

docs/en/prompt/prompt_template.md

@@ -34,8 +34,8 @@ infer_cfg = dict(
         template='Solve the following questions.\n</E>{question}\n{answer}',
         ice_token="</E>"
     ),
-    retriever=dict(type=FixKRetriever),  # Definition of how to retrieve in-context examples.
-    inferencer=dict(type=GenInferencer, fix_id_list=[0, 1]),  # Method used to generate predictions.
+    retriever=dict(type=FixKRetriever, fix_id_list=[0, 1]),  # Definition of how to retrieve in-context examples.
+    inferencer=dict(type=GenInferencer),  # Method used to generate predictions.
 )
 ```
 

docs/zh_cn/prompt/prompt_template.md

@@ -34,8 +34,8 @@ infer_cfg=dict(
         template='Solve the following questions.\n</E>{question}\n{answer}',
         ice_token="</E>"
     ),
-    retriever=dict(type=FixKRetriever),  # 定义 in context example 的获取方式
-    inferencer=dict(type=GenInferencer, fix_id_list=[0, 1]),  # 使用何种方式推理得到 prediction
+    retriever=dict(type=FixKRetriever, fix_id_list=[0, 1]),  # 定义 in context example 的获取方式
+    inferencer=dict(type=GenInferencer),  # 使用何种方式推理得到 prediction
 )
 ```
 

opencompass/openicl/icl_inferencer/icl_agent_inferencer.py

@@ -55,9 +55,6 @@ class AgentInferencer(BaseInferencer):
             output_json_filename = self.output_json_filename
 
         # 2. Get results of retrieval process
-        if 'Fix' in retriever.__class__.__name__:
-            ice_idx_list = retriever.retrieve(self.fix_id_list)
-        else:
         ice_idx_list = retriever.retrieve()
 
         # Create tmp json file for saving intermediate results and future

opencompass/openicl/icl_inferencer/icl_attack_inferencer.py

@@ -59,7 +59,6 @@ class AttackInferencer(BaseInferencer):
             output_json_filepath: Optional[str] = './icl_inference_output',
             output_json_filename: Optional[str] = 'predictions',
             save_every: Optional[int] = None,
-            fix_id_list: Optional[List[int]] = None,
             dataset_cfg: Optional[List[int]] = None,
             **kwargs) -> None:
         super().__init__(
@@ -78,7 +77,6 @@ class AttackInferencer(BaseInferencer):
         self.output_column = dataset_cfg['reader_cfg']['output_column']
         self.gen_field_replace_token = gen_field_replace_token
         self.max_out_len = max_out_len
-        self.fix_id_list = fix_id_list
 
         if self.model.is_api and save_every is None:
             save_every = 1
@@ -94,9 +92,6 @@ class AttackInferencer(BaseInferencer):
         output_json_filename = self.output_json_filename
 
         # 2. Get results of retrieval process
-        if 'Fix' in self.retriever.__class__.__name__:
-            ice_idx_list = self.retriever.retrieve(self.fix_id_list)
-        else:
         ice_idx_list = self.retriever.retrieve()
 
         # 3. Generate prompts for testing input