results_summary.py

from pathlib import Path

import fire
from numpy import column_stack
import pandas as pd


def read_kind_results(kind: str, dir: str):
    files = list(Path(dir).rglob(f"*{kind}.parquet"))
    df = pd.concat(
        [pd.read_parquet(file).assign(file=str(file).split("/")[-2]) for file in files],
        ignore_index=True,
    )
    return df


def summarize_results_per_file(metrics_df: pd.DataFrame):
    metrics_df_per_freq = metrics_df.groupby(["file", "metric", "model"]).mean(
        numeric_only=True
    )
    metrics_df_per_freq = metrics_df_per_freq.reset_index()
    metrics_df_per_freq = metrics_df_per_freq.query(
        "model in ['Prophet', 'SeasonalNaive', 'TimeGPT']"
    )
    models = metrics_df_per_freq["model"].unique()
    metrics_df_per_freq = pd.pivot(
        metrics_df_per_freq,
        index=["file", "metric"],
        columns="model",
        values="value",
    ).reset_index()
    for model in models:
        if model == "SeasonalNaive":
            continue
        metrics_df_per_freq[model] /= metrics_df_per_freq["SeasonalNaive"]
    metrics_df_per_freq["SeasonalNaive"] /= metrics_df_per_freq["SeasonalNaive"]
    return metrics_df_per_freq


def prepare_results(df: pd.DataFrame):
    def bold_best(row):
        row = row.round(3)
        models = row.drop(columns=["file", "metric"]).columns
        best_model = row[models].idxmin(axis=1).item()
        row[best_model] = "**" + str(row[best_model].item()) + "**"
        return row

    df_bolded = df.groupby(["file", "metric"]).apply(bold_best)
    df_bolded = df_bolded.reset_index(drop=True)
    return df_bolded


def write_to_readme(content: str):
    with open("README.md", "r") as file:
        readme_content = file.readlines()
    start_index = -1
    end_index = -1
    for i, line in enumerate(readme_content):
        if line.strip().lower() == "## results":
            start_index = i + 1
        if start_index != -1 and line.strip() == "<end>":
            end_index = i
            break

    if start_index != -1 and end_index != -1:
        readme_content = (
            readme_content[: start_index + 1]
            + [content + "\n"]
            + readme_content[end_index:]
        )
    else:
        print("Results section not found or improperly formatted")

    # Write the changes back to the README
    with open("README.md", "w") as file:
        file.writelines(readme_content)


def summarize_results(dir: str):
    metrics_df = read_kind_results("metrics", dir)
    summary_df = read_kind_results("summary", dir)
    summary_df = (
        summary_df.set_index(["file", "frequency"])
        .reset_index()
        .round(3)
        .sort_values("frequency")
    )
    no_int_cols = ["file", "frequency", "mean", "std"]
    for col in summary_df.columns:
        if col not in no_int_cols:
            summary_df[col] = summary_df[col].astype(int)
    summary_df = summary_df.to_markdown(index=False, intfmt=",", floatfmt=",.3f")
    time_df = read_kind_results("time", dir)
    time_df = time_df.assign(metric="time").rename(columns={"time": "value"})
    metrics_df_per_file = summarize_results_per_file(metrics_df)
    time_df = summarize_results_per_file(time_df)
    eval_df = pd.concat([metrics_df_per_file, time_df], ignore_index=True)
    eval_df = prepare_results(eval_df)[
        ["file", "metric", "TimeGPT", "Prophet", "SeasonalNaive"]
    ]
    n_files = eval_df["file"].nunique()
    eval_df = eval_df.to_markdown(
        index=False,
        colalign=2 * ["left"] + (eval_df.shape[1] - 2) * ["right"],
    )
    markdown_lines = eval_df.split("\n")
    custom_separator = markdown_lines[1].replace(":", "-")
    for i in range(4, len(markdown_lines) + n_files - 1, 4):
        markdown_lines.insert(i + 1, custom_separator)
    markdown_lines.insert(
        0,
        ("\n### Data Description\n\n" f"{summary_df}\n\n" "### Performance\n\n"),
    )
    eval_df = "\n".join(markdown_lines)
    write_to_readme(eval_df)


if __name__ == "__main__":
    fire.Fire(summarize_results)