"tests/pipelines/kandinsky2_2/test_kandinsky_inpaint.py" did not exist on "03b7a84cbee11fa1cff98e5275050f284da168df"
Commit a1b665d9 authored by LucWeber's avatar LucWeber
Browse files

Add ordering of outputs for data-parallel

parent c6b5510c
...@@ -48,10 +48,14 @@ benchmark = 'hellaswag' # possible benchmarks: ...@@ -48,10 +48,14 @@ benchmark = 'hellaswag' # possible benchmarks:
# ['lb','mmlu','alpaca','helm_lite','truthfulqa', # ['lb','mmlu','alpaca','helm_lite','truthfulqa',
# 'gsm8k', 'winogrande', 'arc', 'hellaswag'] # 'gsm8k', 'winogrande', 'arc', 'hellaswag']
# Get score vector from output-file (the metric `acc_norm` depends on the benchmark) # Get score vector from output-file (the metric [here `acc_norm`] depends on the benchmark)
file_path = '<your-output-file.jsonl>' file_path = '<your-output-file.jsonl>'
with open(file_path, 'r') as file: with open(file_path, 'r') as file:
outputs = json.load(file) outputs = json.load(file)
# Ensuring correct order of outputs
outputs = sorted(outputs, key=lambda x: x['doc_id'])
y = np.array([float(item['acc_norm']) for item in outputs]) y = np.array([float(item['acc_norm']) for item in outputs])
### Evaluation ### Evaluation
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment