Unverified Commit 80f831b4 authored by Fengzhe Zhou's avatar Fengzhe Zhou Committed by GitHub
Browse files

[Fix] use ProcessPoolExecutor during mbpp eval (#1159)

parent 8a8987be
...@@ -8,7 +8,7 @@ import re ...@@ -8,7 +8,7 @@ import re
import signal import signal
import tempfile import tempfile
from collections import defaultdict from collections import defaultdict
from concurrent.futures import ThreadPoolExecutor, as_completed from concurrent.futures import ProcessPoolExecutor, as_completed
from typing import List, Sequence, Union from typing import List, Sequence, Union
import numpy as np import numpy as np
...@@ -213,8 +213,7 @@ class MBPPEvaluator(BaseEvaluator): ...@@ -213,8 +213,7 @@ class MBPPEvaluator(BaseEvaluator):
if self.metric == 'MBPP': if self.metric == 'MBPP':
result = {'pass': 0, 'timeout': 0, 'failed': 0, 'wrong_answer': 0} result = {'pass': 0, 'timeout': 0, 'failed': 0, 'wrong_answer': 0}
details = {} details = {}
# change to thread pool for better killing blocked instance with ProcessPoolExecutor() as executor:
with ThreadPoolExecutor() as executor:
futures = [] futures = []
for i, (refer, pred) in enumerate(zip(references, for i, (refer, pred) in enumerate(zip(references,
predictions)): predictions)):
...@@ -439,7 +438,7 @@ class MBPPPassKEvaluator(MBPPEvaluator): ...@@ -439,7 +438,7 @@ class MBPPPassKEvaluator(MBPPEvaluator):
task_total = defaultdict(int) task_total = defaultdict(int)
result = {'pass': 0, 'timeout': 0, 'failed': 0, 'wrong_answer': 0} result = {'pass': 0, 'timeout': 0, 'failed': 0, 'wrong_answer': 0}
with ThreadPoolExecutor() as executor: with ProcessPoolExecutor() as executor:
futures = [] futures = []
for refer, preds in zip(references, predictions): for refer, preds in zip(references, predictions):
# suits for two case # suits for two case
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment