init

Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/codexglue_code_to_text_bleu.py

+#!/usr/bin/python
+"""From https://github.com/microsoft/CodeXGLUE/tree/main/Code-Text/code-to-text (evaluator/evaluator.py)
+
+Call with:
+
+    python codexglue_bleu_evaluator.py ref_file < hyp_file
+"""
+
+"""
+This script was adapted from the original version by hieuhoang1972 which is part of MOSES. 
+"""
+
+# $Id: bleu.py 1307 2007-03-14 22:22:36Z hieuhoang1972 $
+
+"""Provides:
+
+cook_refs(refs, n=4): Transform a list of reference sentences as strings into a form usable by cook_test().
+cook_test(test, refs, n=4): Transform a test sentence as a string (together with the cooked reference sentences) into a form usable by score_cooked().
+score_cooked(alltest, n=4): Score a list of cooked test sentences.
+
+score_set(s, testid, refids, n=4): Interface with dataset.py; calculate BLEU score of testid against refids.
+
+The reason for breaking the BLEU computation into three phases cook_refs(), cook_test(), and score_cooked() is to allow the caller to calculate BLEU scores for multiple test sets as efficiently as possible.
+"""
+
+import math
+import os
+import re
+import subprocess
+import sys
+import xml.sax.saxutils
+
+# Added to bypass NIST-style pre-processing of hyp and ref files -- wade
+nonorm = 0
+
+preserve_case = False
+eff_ref_len = "shortest"
+
+normalize1 = [
+    ("<skipped>", ""),  # strip "skipped" tags
+    (r"-\n", ""),  # strip end-of-line hyphenation and join lines
+    (r"\n", " "),  # join lines
+    #    (r'(\d)\s+(?=\d)', r'\1'), # join digits
+]
+normalize1 = [(re.compile(pattern), replace) for (pattern, replace) in normalize1]
+
+normalize2 = [
+    (
+        r"([\{-\~\[-\` -\&\(-\+\:-\@\/])",
+        r" \1 ",
+    ),  # tokenize punctuation. apostrophe is missing
+    (
+        r"([^0-9])([\.,])",
+        r"\1 \2 ",
+    ),  # tokenize period and comma unless preceded by a digit
+    (
+        r"([\.,])([^0-9])",
+        r" \1 \2",
+    ),  # tokenize period and comma unless followed by a digit
+    (r"([0-9])(-)", r"\1 \2 "),  # tokenize dash when preceded by a digit
+]
+normalize2 = [(re.compile(pattern), replace) for (pattern, replace) in normalize2]
+
+
+def normalize(s):
+    """Normalize and tokenize text. This is lifted from NIST mteval-v11a.pl."""
+    # Added to bypass NIST-style pre-processing of hyp and ref files -- wade
+    if nonorm:
+        return s.split()
+    if type(s) is not str:
+        s = " ".join(s)
+    # language-independent part:
+    for (pattern, replace) in normalize1:
+        s = re.sub(pattern, replace, s)
+    s = xml.sax.saxutils.unescape(s, {"&quot;": '"'})
+    # language-dependent part (assuming Western languages):
+    s = " %s " % s
+    if not preserve_case:
+        s = s.lower()  # this might not be identical to the original
+    for (pattern, replace) in normalize2:
+        s = re.sub(pattern, replace, s)
+    return s.split()
+
+
+def count_ngrams(words, n=4):
+    counts = {}
+    for k in range(1, n + 1):
+        for i in range(len(words) - k + 1):
+            ngram = tuple(words[i : i + k])
+            counts[ngram] = counts.get(ngram, 0) + 1
+    return counts
+
+
+def cook_refs(refs, n=4):
+    """Takes a list of reference sentences for a single segment
+    and returns an object that encapsulates everything that BLEU
+    needs to know about them."""
+
+    refs = [normalize(ref) for ref in refs]
+    maxcounts = {}
+    for ref in refs:
+        counts = count_ngrams(ref, n)
+        for (ngram, count) in counts.items():
+            maxcounts[ngram] = max(maxcounts.get(ngram, 0), count)
+    return ([len(ref) for ref in refs], maxcounts)
+
+
+def cook_test(test, item, n=4):
+    """Takes a test sentence and returns an object that
+    encapsulates everything that BLEU needs to know about it."""
+    (reflens, refmaxcounts) = item
+    test = normalize(test)
+    result = {}
+    result["testlen"] = len(test)
+
+    # Calculate effective reference sentence length.
+
+    if eff_ref_len == "shortest":
+        result["reflen"] = min(reflens)
+    elif eff_ref_len == "average":
+        result["reflen"] = float(sum(reflens)) / len(reflens)
+    elif eff_ref_len == "closest":
+        min_diff = None
+        for reflen in reflens:
+            if min_diff is None or abs(reflen - len(test)) < min_diff:
+                min_diff = abs(reflen - len(test))
+                result["reflen"] = reflen
+
+    result["guess"] = [max(len(test) - k + 1, 0) for k in range(1, n + 1)]
+
+    result["correct"] = [0] * n
+    counts = count_ngrams(test, n)
+    for (ngram, count) in counts.items():
+        result["correct"][len(ngram) - 1] += min(refmaxcounts.get(ngram, 0), count)
+
+    return result
+
+
+def score_cooked(allcomps, n=4, ground=0, smooth=1):
+    totalcomps = {"testlen": 0, "reflen": 0, "guess": [0] * n, "correct": [0] * n}
+    for comps in allcomps:
+        for key in ["testlen", "reflen"]:
+            totalcomps[key] += comps[key]
+        for key in ["guess", "correct"]:
+            for k in range(n):
+                totalcomps[key][k] += comps[key][k]
+    logbleu = 0.0
+    all_bleus = []
+    for k in range(n):
+        correct = totalcomps["correct"][k]
+        guess = totalcomps["guess"][k]
+        addsmooth = 0
+        if smooth == 1 and k > 0:
+            addsmooth = 1
+        logbleu += math.log(correct + addsmooth + sys.float_info.min) - math.log(
+            guess + addsmooth + sys.float_info.min
+        )
+        if guess == 0:
+            all_bleus.append(-10000000)
+        else:
+            all_bleus.append(math.log(correct + sys.float_info.min) - math.log(guess))
+
+    logbleu /= float(n)
+    all_bleus.insert(0, logbleu)
+
+    brevPenalty = min(
+        0, 1 - float(totalcomps["reflen"] + 1) / (totalcomps["testlen"] + 1)
+    )
+    for i in range(len(all_bleus)):
+        if i == 0:
+            all_bleus[i] += brevPenalty
+        all_bleus[i] = math.exp(all_bleus[i])
+    return all_bleus
+
+
+def bleu(refs, candidate, ground=0, smooth=1):
+    refs = cook_refs(refs)
+    test = cook_test(candidate, refs)
+    return score_cooked([test], ground=ground, smooth=smooth)
+
+
+def splitPuncts(line):
+    return " ".join(re.findall(r"[\w]+|[^\s\w]", line))
+
+
+def computeMaps(predictions, goldfile):
+    predictionMap = {}
+    goldMap = {}
+    gf = open(goldfile, "r")
+
+    for row in predictions:
+        cols = row.strip().split("\t")
+        if len(cols) == 1:
+            (rid, pred) = (cols[0], "")
+        else:
+            (rid, pred) = (cols[0], cols[1])
+        predictionMap[rid] = [splitPuncts(pred.strip().lower())]
+
+    for i, row in enumerate(gf):
+        if len(row.split("\t")) != 2:
+            print(row)
+            print(i)
+        (rid, pred) = row.split("\t")
+        if rid in predictionMap:  # Only insert if the id exists for the method
+            if rid not in goldMap:
+                goldMap[rid] = []
+            goldMap[rid].append(splitPuncts(pred.strip().lower()))
+
+    sys.stderr.write("Total: " + str(len(goldMap)) + "\n")
+    return (goldMap, predictionMap)
+
+
+# m1 is the reference map
+# m2 is the prediction map
+def bleuFromMaps(m1, m2):
+    score = [0] * 5
+    num = 0.0
+
+    for key in m1:
+        if key in m2:
+            bl = bleu(m1[key], m2[key][0])
+            score = [score[i] + bl[i] for i in range(0, len(bl))]
+            num += 1
+    return [s * 100.0 / num for s in score]
+
+
+if __name__ == "__main__":
+    reference_file = sys.argv[1]
+    predictions = []
+    for row in sys.stdin:
+        predictions.append(row)
+    (goldMap, predictionMap) = computeMaps(predictions, reference_file)
+    print(bleuFromMaps(goldMap, predictionMap)[0])

Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/diff_eval.py

+# https://github.com/CarperAI/OpenELM/blob/e6402a0696096011572152334ccbe049f89c332e/src/openelm/utils/diff_eval.py #
+import re
+from enum import Enum
+from typing import Optional
+
+line_number_pattern = re.compile(
+    r"(?m)^@@ -(?P<l1>\d*),*?(?P<s1>\d*?) \+(?P<l2>\d*),*?(?P<s2>\d*?) @@"
+)
+diff_pattern = re.compile(
+    r"""<NME> (?P<name>.*?)
+<BEF> (?P<file>(.|\n)*?)
+<MSG> (?P<message>(.|\n)*?)
+<DFF> (?P<diff>(.|\n)*)"""
+)
+hunk_split_pattern = re.compile(r"(?m)^(@@ .*? @@).*\n")
+ignored = re.compile(r"(?m)^\\ No newline at end of file$\n?")
+
+
+class DiffState(Enum):
+    """
+    An Enum keeping track of the validity of the diff data. It is the return of the helper function `verify_diff`.
+    Binary codes help internally, as some errors are additive (e.g., can have both invalid text and invalid line num).
+    But we convert the binary codes into Enum for better readability.
+    """
+
+    VALID = 0b000  # valid diff
+
+    # The following are errors that can still be either ignored or fixed.
+    INVALID_TEXT = 0b001  # pre-diff texts cannot be found in the context.
+    INVALID_LINE_NUM = (
+        0b010  # the numbers in @@ -x,y +a,b @@ are invalid (but can be parsed).
+    )
+    INVALID_TEXT_AND_LINE_NUM = 0b011  # both 0b001 and 0b010.
+
+    # The following are format errors that cannot be ignored.
+    BAD_FORMAT = 0b100  # cannot be parsed according to <NME> ...\n<BEF> ...\n<MSG> ...\n<DFF> @@ ... @@\n...
+    BAD_DIFF_HUNK_FORMAT = 0b101  # diff hunk contains lines whose initial character is not one of ' ', '+', '-'
+    BAD_LINE_NUM_FORMAT = (
+        0b110  # the @@ ... @@ bracket can be found but numbers cannot be parsed.
+    )
+    BAD_HUNK_AND_LINE_FORMAT = 0b111  # both 0b110 and 0b101.
+
+
+def split_diff(content: str) -> dict:
+    """
+    Args:
+        content: the diff content.
+
+    Returns:
+        A dict with potentially 4 items:
+            name: the filename
+            file: the file content
+            message: the diff message
+            diff: the diff hunk
+        Any key could be missing. That would mean a failure in matching.
+    """
+    match = diff_pattern.match(content)
+    return {} if match is None else match.groupdict()
+
+
+def parse_line_info(content: str) -> tuple:
+    """
+    Parse @@ -x,y +a,b @@
+
+    Args:
+        the @@ ... @@ line
+    Returns:
+        (x, y, a, b) as integers
+    """
+    match = line_number_pattern.match(content)
+    if match is None:
+        return ()
+    match_dict = match.groupdict()
+    # line numbers are mandatory
+    if not match_dict["l1"] or not match_dict["l2"]:
+        return ()
+    for s in ["s1", "s2"]:
+        # line ranges are optional and default to 1
+        match_dict[s] = match_dict[s] if match_dict[s] else "1"
+    return (
+        int(match_dict["l1"]),
+        int(match_dict["s1"]),
+        int(match_dict["l2"]),
+        int(match_dict["s2"]),
+    )
+
+
+def parse_diff_content(
+    hunk: str, separate_lines=False, reject_invalid=False
+) -> Optional[tuple]:
+    """
+    Parse a diff content to turn it into (before_diff, after_diff) based on '+', '-' at the beginning of each line.
+
+    Args:
+        hunk: the diff content (without "@@ ... @@").
+        separate_lines: (Optional) True if return list of lines.
+        reject_invalid: (Optional) True if return None for invalid diff hunk (non-empty lines without starting
+            with ' ', '-', '+')
+    Returns:
+        (before_diff, after_diff);
+        None if reject_invalid==True and the diff hunk contains invalid format.
+    """
+    hunk = hunk.split("\n")
+    before_diff, after_diff = [], []
+    for line in hunk:
+        # Ignore invalid trailing '\n'. An empty line in the diff hunk should at least be '\n ' with the space.
+        if not line:
+            continue
+        if line[0] == "-" or line[0] == " ":
+            before_diff.append(line[1:])
+        if line[0] == "+" or line[0] == " ":
+            after_diff.append(line[1:])
+        if reject_invalid:
+            if all([line[0] != c for c in [" ", "-", "+"]]):
+                return None
+    if separate_lines:
+        return before_diff, after_diff
+    else:
+        return "\n".join(before_diff), "\n".join(after_diff)
+
+
+def replace_text(
+    text: str,
+    before: str,
+    after: str,
+    start_pointer: int,
+    reject_incomplete_line: bool = True,
+) -> tuple[str, int]:
+    """
+    Try to match `before` within `text` and replace the content into `after`.
+    If not found, return the original text.
+
+    Args:
+        text: the original text.
+        before: the text to be matched.
+        after: the text to be replaced into.
+        start_pointer: the index where we start to match (inclusive).
+        reject_incomplete_line: (Optional) reject the patch if `before` does not match till the end of a line.
+    Returns:
+        (diff_result, new_start_pointer)
+        the text after the match-and-replace and the new index at the end of the change.
+    """
+    idx = text[start_pointer:].find(before)
+    start_idx = start_pointer + idx
+
+    if reject_incomplete_line:
+        # If the end of the match is neither EOF nor \n, reject the patch.
+        if (
+            idx >= 0
+            and start_idx + len(before) < len(text)
+            and text[start_idx + len(before)] != "\n"
+        ):
+            return text, start_pointer
+
+    if idx < 0:
+        return text, start_pointer
+    else:
+        # Even if start_idx + len(before) is out-of-bound, the list slicing would return ""
+        return text[:start_idx] + after + text[
+            start_idx + len(before) :
+        ], start_idx + len(after)
+
+
+def apply_diff(file: str, diff: str, use_line_number=False, allow_add_file=True) -> str:
+    """
+    Apply the diff to the file content. We try to be lenient and keep applying the patch naively until we cannot.
+    (Note: use_line_number=True is somehow slightly slower.)
+    (Warning: if use_line_number==False, we could have some problematic cases like, if all lines in diff hunk
+        starts with "+", the pre-diff paragraphs relevant to the hunk is empty. Because we only use pre-diff
+        paragraphs to match, we would simply match the very beginning.)
+    Args:
+        file: the file content.
+        diff: the diff hunk (containing "@@ -x,y +a,b @@").
+        use_line_number: (Optional) use the line numbers in "@@ ... @@" faithfully.
+        allow_add_file: (Optional) when file is "ADDFILE" (meaning <BEF> ADDFILE\n... showed up in the diff text),
+            we automatically patch the diff by a direct replacement.
+    Return:
+        the maximally patched file content.
+    """
+    diff = hunk_split_pattern.split(ignored.sub("", diff))
+    # If we use the line numbers, we match-and-replace in a line-by-line fashion.
+    file_by_line = file.split("\n") if use_line_number else None
+    line_offset = 0  # the offset between pre-/post-patching line numbers
+
+    # If we do not use the line numbers, for multiple diff hunk, we only move forward in a greedy manner.
+    patch_pointer = 0
+
+    i = (
+        0 if diff[0] else 1
+    )  # We have delimiter at the beginning, causing empty initial string
+    while (
+        i < len(diff) - 1
+    ):  # Need at least a pair of '@@ ... @@' and diff hunk to continue
+        # Expect a string with '@@ ... @@' followed by a diff hunk
+        line_info = parse_line_info(diff[i])
+        diff_content = diff[i + 1]
+        i += 2
+
+        # Generate the pre-/post-diff string based on the first character being '+' or '-'
+        # (Note: parse_diff_content will ignore trailing \n at the beginning and at the end)
+        parsed_diff = parse_diff_content(diff_content, separate_lines=use_line_number)
+
+        # If we allow the recognition of "ADDFILE", special treatment is needed.
+        if allow_add_file and file == "ADDFILE":
+            if use_line_number:
+                # Immediately apply the first hunk but also check the partial validity of line numbers.
+                return parsed_diff[1] if line_info == (0, 0) else ""
+            else:
+                # Immediately apply the first hunk and ignore the rest.
+                return parsed_diff[1]
+
+        if use_line_number:
+            # If line numbers cannot be parsed, skip.
+            if not line_info:
+                continue
+
+            # Offset the starting line
+            start_idx = line_info[0] + line_offset
+
+            # Match the referred lines with the file context
+            referred_lines = file_by_line[start_idx - 1 : start_idx - 1 + line_info[1]]
+            valid = all([l1 == l2 for l1, l2 in zip(parsed_diff[0], referred_lines)])
+
+            # If lines fully match and the number of lines is consistent, apply the patch.
+            # We ignore the second pair "+a, b" just to be lenient.
+            if valid and len(parsed_diff[0]) == line_info[1]:
+                # Update the list of lines
+                if start_idx == 0:  # Add lines to the beginning.
+                    file_by_line = parsed_diff[1] + file_by_line
+                else:
+                    file_by_line = (
+                        file_by_line[: start_idx - 1]
+                        + parsed_diff[1]
+                        + file_by_line[start_idx - 1 + line_info[1] :]
+                    )
+                line_offset += len(parsed_diff[1]) - line_info[1]
+        else:
+            # CAUTION: this way of handling empty context is being very lenient and could lead to
+            # undesirable behaviors. Only do this when you want to be as tolerant as possible.
+            if parsed_diff[0] == "":
+                if (
+                    patch_pointer != 0
+                ):  # Lack of matching context can only happen at the beginning of file.
+                    continue
+                file = parsed_diff[1] + "\n" + file
+                patch_pointer = len(parsed_diff[0]) + 1
+            else:
+                # Directly (and naively) apply patch by match-and-replace.
+                file, patch_pointer = replace_text(
+                    file, parsed_diff[0], parsed_diff[1], patch_pointer
+                )
+
+    if use_line_number:
+        file = "\n".join(file_by_line)
+    return file
+
+
+def verify_diff(diff_text: str) -> DiffState:
+    """
+    Verify the validity of a complete diff text.
+
+    Args:
+        diff_text: the complete diff text.
+            The overall format conforms "<NME> ...\n<BEF> ...\n<MSG> ...\n<DFF> ..." and the text
+            after <DFF> has 1 or more lines of "@@ -x,y +a,b @@" followed by the corresponding hunk.
+    Returns:
+        A DiffState (see above).
+    """
+    diff_dict = split_diff(
+        ignored.sub("", diff_text)
+    )  # Ignore the GitHub warning on the end of file
+    line_offset = 0
+
+    keys = ["name", "file", "message", "diff"]
+    for key in keys:
+        if key not in diff_dict:
+            return DiffState(0b100)  # Invalid overall format
+
+    diff_parts = hunk_split_pattern.split(diff_dict["diff"])
+    if not diff_parts:
+        return DiffState(0b100)  # Invalid overall format
+
+    context_mismatch, line_number_mismatch = False, False
+    bad_diff_hunk, bad_line_number = False, False
+
+    i = 0 if diff_parts[0] else 1
+    while (
+        i < len(diff_parts) - 1
+    ):  # Need at least a pair of '@@ ... @@' and diff hunk to continue
+        line_info = parse_line_info(diff_parts[i])
+        diff_content = parse_diff_content(diff_parts[i + 1], reject_invalid=True)
+        i += 2
+
+        # Special treatment if we are adding a new file
+        if diff_dict["file"] == "ADDFILE":
+            if (
+                len(diff_parts) != i
+                or not line_info
+                or line_info[:3] != (0, 0, 1)
+                or line_info[3] != len(diff_content[1].split("\n"))
+                or diff_content[0]
+            ):
+                return DiffState(0b110)
+            else:
+                return DiffState(0b000)
+
+        if not line_info or len(line_info) != 4:
+            bad_line_number = True
+        if diff_content is None:
+            bad_diff_hunk = True
+
+        # Skip the diff matching checks if bad format already occurred
+        if bad_diff_hunk or bad_line_number:
+            continue
+
+        # Try to see if there is a match in the file context. Must match complete lines or till EOF.
+        match_idx = diff_dict["file"].find(diff_content[0])
+        if match_idx == -1 or (
+            match_idx + len(diff_content[0]) != len(diff_dict["file"])
+            and diff_dict["file"][match_idx + len(diff_content[0])] != "\n"
+        ):
+            context_mismatch = True
+
+        if line_info[0] <= 0:
+            # -0,0 only happens when we create a new file (in which case the context is <BEF> ADDFILE\n...).
+            if line_info[1] != 0 or diff_dict["file"] != "ADDFILE":
+                line_number_mismatch = True
+        else:
+            # Check the line numbers regardless of whether the context matches.
+            pre_diff_line_number = len(diff_content[0].split("\n"))
+            post_diff_line_number = len(diff_content[1].split("\n"))
+            if (pre_diff_line_number, post_diff_line_number) != (
+                line_info[1],
+                line_info[3],
+            ):
+                line_number_mismatch = True
+            else:
+                line_offset += len(diff_content[1]) - line_info[1]
+
+    if bad_diff_hunk or bad_line_number:
+        return DiffState(bad_diff_hunk * 0b001 + bad_line_number * 0b010 + 0b100)
+    else:
+        return DiffState(context_mismatch * 0b001 + line_number_mismatch * 0b010)
\ No newline at end of file

Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/execute.py

+# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This code is adapted from OpenAI's release
+# https://github.com/openai/human-eval/blob/master/human_eval/execution.py
+
+import contextlib
+import faulthandler
+import io
+import multiprocessing
+import os
+import platform
+import signal
+import tempfile
+
+
+def check_correctness(check_program, timeout, task_id, completion_id):
+    """
+    Evaluates the functional correctness of a completion by running the test
+    suite provided in the problem.
+
+    :param completion_id: an optional completion ID so we can match
+        the results later even if execution finishes asynchronously.
+    """
+    manager = multiprocessing.Manager()
+    result = manager.list()
+
+    p = multiprocessing.Process(target=unsafe_execute, args=(check_program, result, timeout))
+    p.start()
+    p.join(timeout=timeout + 1)
+    if p.is_alive():
+        p.kill()
+
+    if not result:
+        result.append("timed out")
+
+    return dict(
+        task_id=task_id,
+        passed=result[0] == "passed",
+        result=result[0],
+        completion_id=completion_id,
+    )
+
+
+def unsafe_execute(check_program, result, timeout):
+
+    with create_tempdir():
+
+        # These system calls are needed when cleaning up tempdir.
+        import os
+        import shutil
+
+        rmtree = shutil.rmtree
+        rmdir = os.rmdir
+        chdir = os.chdir
+
+        # Disable functionalities that can make destructive changes to the test.
+        reliability_guard()
+
+        # Run program.
+        try:
+            exec_globals = {}
+            with swallow_io():
+                with time_limit(timeout):
+                    exec(check_program, exec_globals)
+            result.append("passed")
+        except TimeoutException:
+            result.append("timed out")
+        except BaseException as e:
+            result.append(f"failed: {e}")
+
+        # Needed for cleaning up.
+        shutil.rmtree = rmtree
+        os.rmdir = rmdir
+        os.chdir = chdir
+
+
+@contextlib.contextmanager
+def time_limit(seconds):
+    def signal_handler(signum, frame):
+        raise TimeoutException("Timed out!")
+
+    signal.setitimer(signal.ITIMER_REAL, seconds)
+    signal.signal(signal.SIGALRM, signal_handler)
+    try:
+        yield
+    finally:
+        signal.setitimer(signal.ITIMER_REAL, 0)
+
+
+@contextlib.contextmanager
+def swallow_io():
+    stream = WriteOnlyStringIO()
+    with contextlib.redirect_stdout(stream):
+        with contextlib.redirect_stderr(stream):
+            with redirect_stdin(stream):
+                yield
+
+
+@contextlib.contextmanager
+def create_tempdir():
+    with tempfile.TemporaryDirectory() as dirname:
+        with chdir(dirname):
+            yield dirname
+
+
+class TimeoutException(Exception):
+    pass
+
+
+class WriteOnlyStringIO(io.StringIO):
+    """StringIO that throws an exception when it's read from"""
+
+    def read(self, *args, **kwargs):
+        raise OSError
+
+    def readline(self, *args, **kwargs):
+        raise OSError
+
+    def readlines(self, *args, **kwargs):
+        raise OSError
+
+    def readable(self, *args, **kwargs):
+        """Returns True if the IO object can be read."""
+        return False
+
+
+class redirect_stdin(contextlib._RedirectStream):  # type: ignore
+    _stream = "stdin"
+
+
+@contextlib.contextmanager
+def chdir(root):
+    if root == ".":
+        yield
+        return
+    cwd = os.getcwd()
+    os.chdir(root)
+    try:
+        yield
+    except BaseException as exc:
+        raise exc
+    finally:
+        os.chdir(cwd)
+
+
+def reliability_guard(maximum_memory_bytes=None):
+    """
+    This disables various destructive functions and prevents the generated code
+    from interfering with the test (e.g. fork bomb, killing other processes,
+    removing filesystem files, etc.)
+
+    WARNING
+    This function is NOT a security sandbox. Untrusted code, including, model-
+    generated code, should not be blindly executed outside of one. See the
+    Codex paper for more information about OpenAI's code sandbox, and proceed
+    with caution.
+    """
+
+    if maximum_memory_bytes is not None:
+        import resource
+
+        resource.setrlimit(resource.RLIMIT_AS, (maximum_memory_bytes, maximum_memory_bytes))
+        resource.setrlimit(resource.RLIMIT_DATA, (maximum_memory_bytes, maximum_memory_bytes))
+        if not platform.uname().system == "Darwin":
+            resource.setrlimit(resource.RLIMIT_STACK, (maximum_memory_bytes, maximum_memory_bytes))
+
+    faulthandler.disable()
+
+    import builtins
+
+    builtins.exit = None
+    builtins.quit = None
+
+    import os
+
+    os.environ["OMP_NUM_THREADS"] = "1"
+
+    os.kill = None
+    os.system = None
+    os.putenv = None
+    os.remove = None
+    os.removedirs = None
+    os.rmdir = None
+    os.fchdir = None
+    os.setuid = None
+    os.fork = None
+    os.forkpty = None
+    os.killpg = None
+    os.rename = None
+    os.renames = None
+    os.truncate = None
+    os.replace = None
+    os.unlink = None
+    os.fchmod = None
+    os.fchown = None
+    os.chmod = None
+    os.chown = None
+    os.chroot = None
+    os.fchdir = None
+    os.lchflags = None
+    os.lchmod = None
+    os.lchown = None
+    os.getcwd = None
+    os.chdir = None
+
+    import shutil
+
+    shutil.rmtree = None
+    shutil.move = None
+    shutil.chown = None
+
+    import subprocess
+
+    subprocess.Popen = None  # type: ignore
+
+    __builtins__["help"] = None
+
+    import sys
+
+    sys.modules["ipdb"] = None
+    sys.modules["joblib"] = None
+    sys.modules["resource"] = None
+    sys.modules["psutil"] = None
+    sys.modules["tkinter"] = None

Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/containerized_eval.py

+"""
+NOTE: Nothing containerized about this any more. This is just a helper
+for problem_evaluator.py.
+"""
+
+import tempfile
+from pathlib import Path
+
+from . import (eval_cpp, eval_dlang, eval_java, eval_javascript, eval_julia,
+               eval_lua, eval_php, eval_python, eval_r, eval_racket, eval_ruby,
+               eval_rust, eval_swift, eval_ts, eval_go, eval_pl, eval_sh, eval_scala, eval_cs)
+
+EVALUATORS = {
+    "rb": (eval_ruby.eval_script, ".rb"),
+    "lua": (eval_lua.eval_script, ".lua"),
+    "python": (eval_python.eval_script, ".py"),
+    "py": (eval_python.eval_script, ".py"),
+    "notypes.py": (eval_python.eval_script, ".py"),
+    "julia": (eval_julia.eval_script, ".jl"),
+    "java": (eval_java.eval_script, ".java"),
+    "rust": (eval_rust.eval_script, ".rs"),
+    "rs": (eval_rust.eval_script, ".rs"),
+    "swift": (eval_swift.eval_script, ".swift"),
+    "lua": (eval_lua.eval_script, ".lua"),
+    "racket": (eval_racket.eval_script, ".rkt"),
+    "rkt": (eval_racket.eval_script, ".rkt"),
+    "javascript": (eval_javascript.eval_script, ".js"),
+    "js": (eval_javascript.eval_script, ".js"),
+    "cpp": (eval_cpp.eval_script, ".cpp"),
+    "cs": (eval_cs.eval_script, ".cs"),
+    "php": (eval_php.eval_script, ".php"),
+    "humaneval_to_dlang.py": (eval_dlang.eval_script, ".d"),
+    "d": (eval_dlang.eval_script, ".d"),
+    "r": (eval_r.eval_script, ".r"),
+    "humaneval_to_r.py": (eval_r.eval_script, ".r"),
+    "jl": (eval_julia.eval_script, ".jl"),
+    "ts": (eval_ts.eval_script, ".ts"),
+    "go": (eval_go.eval_script, ".go"),
+    "pl": (eval_pl.eval_script, ".pl"),
+    "sh": (eval_sh.eval_script, ".sh"),
+    "scala": (eval_scala.eval_script, ".scala"),
+}
+
+
+def eval_string_script(language, program):
+    if language in EVALUATORS:
+        (eval_script, file_ext) = EVALUATORS[language]
+    else:
+        eval_module = __import__(
+            f"eval_{language}" if language != "go_test.go" else "eval_go"
+        )
+        eval_script = eval_module.eval_script
+        file_ext = f".{language}" if language != "go_test.go" else "_test.go"
+    with tempfile.NamedTemporaryFile(suffix=file_ext, delete=True) as f:
+        f.write(program.encode("utf-8"))
+        f.flush()
+        result = eval_script(Path(f.name))
+        # Only save the first 2K of output from the running program. Any futher
+        # output is very likely an exceptionally long stack trace or a long
+        # series of prints.
+        if type(result["stdout"]) == bytes:
+            result["stdout"] = result["stdout"].decode("utf-8", errors="ignore")
+        if result["stdout"] is None:
+            result["stdout"] = ""
+        if result["stderr"] is None:
+            result["stderr"] = ""
+        if type(result["stderr"]) == bytes:
+            result["stderr"] = result["stderr"].decode("utf-8", errors="ignore")
+        assert type(result["stdout"]) == str
+        assert type(result["stderr"]) == str
+        return {
+            "program": program,
+            "stdout": result["stdout"].replace("!!int", "")[:2048],
+            "stderr": result["stderr"][:2048],
+            "exit_code": result["exit_code"],
+            "status": result["status"],
+        }

Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/eval_cpp.py

+from pathlib import Path
+
+from .generic_eval import main
+from .safe_subprocess import run
+
+LANG_NAME = "C++"
+LANG_EXT = ".cpp"
+
+
+def eval_script(path: Path):
+    basename = ".".join(str(path).split(".")[:-1])
+    build_result = run(["g++", path, "-o", basename, "-std=c++17"])
+    if build_result.exit_code != 0:
+        return {
+            "status": "SyntaxError",
+            "exit_code": build_result.exit_code,
+            "stdout": build_result.stdout,
+            "stderr": build_result.stderr,
+        }
+
+    run_result = run([basename])
+    if "In file included from /shared/centos7/gcc/9.2.0-skylake/" in run_result.stderr:
+        raise Exception("Skylake bug encountered")
+    if "/4.8.2" in run_result.stderr:
+        raise Exception("Ancient compiler encountered")
+    if run_result.timeout:
+        status = "Timeout"
+    elif run_result.exit_code != 0:
+        status = "Exception"
+    else:
+        status = "OK"
+    return {
+        "status": status,
+        "exit_code": run_result.exit_code,
+        "stdout": run_result.stdout,
+        "stderr": run_result.stderr,
+    }
+
+
+if __name__ == "__main__":
+    main(eval_script, LANG_NAME, LANG_EXT)

Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/eval_cs.py

+import os
+import subprocess
+
+from .generic_eval import main
+
+LANG_NAME = "CSharp"
+LANG_EXT = ".cs"
+
+# Following files have problems:
+# 137,
+# 22: Any
+# 148: Elipsis
+
+
+def eval_script(path: str):
+    if ".cs" not in path.name:
+        return
+    basename = ".".join(str(path).split(".")[:-1])
+    binaryname = basename + ".exe"
+    build = subprocess.run(
+        ["csc", "/d:DEBUG", "-r:System.Numerics.dll", path, f"/out:{binaryname}"],
+        capture_output=True,
+    )
+    status = None
+    returncode = -1
+    output = None
+    if build.returncode != 0:
+        # Well, it's a compile error. May be a type error or
+        # something. But, why break the set convention
+        status = "SyntaxError"
+        returncode = build.returncode
+        output = build
+    else:
+        try:
+            output = subprocess.run(
+                ["mono", binaryname],
+                env={"PATH": os.getenv("PATH"), "MONO_TRACE_LISTENER": "Console.Error"},
+                capture_output=True,
+                timeout=5,
+            )
+            returncode = output.returncode
+            output.stderr = str(output.stderr, "utf-8")
+            # mono return 0 even when failing
+            fail = (
+                "System.Diagnostics.DefaultTraceListener.Fail" in output.stderr
+                or "Unhandled Exception" in output.stderr
+            )
+            output.returncode = 1 if fail else 0
+            if output.returncode == 0:
+                status = "OK"
+            else:
+                # Well, it's a panic
+                status = "Exception"
+        except subprocess.TimeoutExpired as exc:
+            status = "Timeout"
+            output = exc
+        os.remove(binaryname)
+
+    if output.stdout is not None:
+        output.stdout = output.stdout.decode("utf-8")
+    else:
+        output.stdout = "None"
+
+    if output.stderr == "":
+        output.stderr = "None"
+
+    return {
+        "status": status,
+        "exit_code": returncode,
+        "stdout": output.stdout,
+        "stderr": output.stderr,
+    }
+
+
+if __name__ == "__main__":
+    main(eval_script, LANG_NAME, LANG_EXT)

Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/eval_dlang.py

+import os
+import re
+from pathlib import Path
+
+from .safe_subprocess import run
+
+ENABLE_SYNTAX_CHECK = False
+
+
+def eval_script(path: Path):
+    result = run(["rdmd", "-unittest", str(path)], timeout_seconds=15)
+    if "might not be correctly installed" in result.stderr:
+        raise Exception("D is not correctly installed")
+
+    if result.timeout:
+        status = "Timeout"
+    elif result.exit_code == 0:
+        status = "OK"
+    elif "Error:" in result.stderr:
+        status = "SyntaxError"
+    else:
+        status = "Exception"
+
+    return {
+        "status": status,
+        "exit_code": result.exit_code,
+        "stdout": result.stdout,
+        "stderr": result.stderr,
+    }
+
+
+DIR = "d-keep-code_davinci_001_temp_0.2"
+
+
+def main():
+    directory = Path(Path(__file__).parent, "..", "datasets", DIR).resolve()
+
+    count = {"OK": 0, "Timeout": 0, "Exception": 0, "SyntaxError": 0}
+    for filename in os.listdir(directory):
+        path = Path.joinpath(directory, filename)
+        r = eval_script(path)
+        status = r["status"]
+        count[status] += 1
+
+        if ENABLE_SYNTAX_CHECK and status == "SyntaxError":
+            error_msgs = r["stderr"].split("\n")
+            with open(path) as source_file:
+                lines = source_file.readlines()
+                unittest_line_start = lines.index("unittest\n")
+                unittest_line_end = len(lines)
+                for err_msg_line in error_msgs:
+                    matched_parts = re.match(
+                        r"(\/?.*?\.[\w:]+\/.*.d)\(([0-9]+)\): Error: (.*)",
+                        err_msg_line[2:-1],
+                    )
+                    _file, line_num = matched_parts[1], int(matched_parts[2])
+                    if (
+                        unittest_line_start <= line_num
+                        and line_num <= unittest_line_end
+                    ):
+                        print("===============")
+                        print(path, "contains error in unit test part")
+                        print(error_msgs)
+                        print("===============")
+
+        filename = filename.split(".")[0]
+        print(f"Dlang,{filename},{status}")
+
+    print(DIR + ":" + str(count))
+
+
+if __name__ == "__main__":
+    main()

Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/eval_go.py

+import argparse
+import subprocess
+from pathlib import Path
+from sys import exit
+
+from .generic_eval import main as gmain
+
+
+def eval_script(path: Path):
+    status = None
+    stdout = None
+    stderr = None
+    exit_code = None
+    try:
+        build = subprocess.run(
+            ["go", "test", path],
+            timeout=30,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+
+        stdout = build.stdout.decode("utf-8", errors="ignore")
+        stderr = build.stderr.decode("utf-8", errors="ignore")
+        exit_code = build.returncode
+        # write to stderr just so that we can redirect stdout to a csv
+
+        if "[setup failed]" in stdout or "[build failed]" in stdout:
+            status = "SyntaxError"
+        elif "FAIL" in stdout:
+            status = "Exception"
+        else:
+            status = "OK"
+    except subprocess.TimeoutExpired:
+        status = "Timeout"
+
+    return {
+        "status": status,
+        "exit_code": exit_code,
+        "stdout": stdout,
+        "stderr": stderr,
+    }
+
+
+if __name__ == "__main__":
+    gmain(eval_script, "Go", ".go")

Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/eval_java.py

+import os
+import tempfile
+from pathlib import Path
+
+from .generic_eval import main
+from .safe_subprocess import run
+
+LANG_NAME = "Java"
+LANG_EXT = ".java"
+
+# Following files have problems:
+# 137,
+# 22: Any
+# 148: Elipsis
+
+
+def eval_script(path: Path):
+
+    sys_env = os.environ.copy()
+    javatuples_path = Path("/usr/multiple/javatuples-1.2.jar")
+
+    sys_env["CLASSPATH"] = f"{javatuples_path}"
+
+    with tempfile.TemporaryDirectory() as outdir:
+        # Each Java file contains the class with same name `JAVA_CLASS_NAME`
+        # Hence, javac will same JAVA_CLASS_NAME.class file for each problem
+        # Write class for each problem to a different temp dir
+        # Use UTF8 encoding with javac
+        result = run(["javac", "-encoding", "UTF8", "-d", outdir, path], env=sys_env)
+
+        if result.exit_code != 0:
+            # Well, it's a compile error. May be a type error or
+            # something. But, why break the set convention
+            status = "SyntaxError"
+        else:
+            result = run(["java", "-ea", "-cp", f"{outdir}", "Problem"], env=sys_env)
+            if result.timeout:
+                status = "Timeout"
+            elif result.exit_code == 0:
+                status = "OK"
+            else:
+                status = "Exception"
+
+    return {
+        "status": status,
+        "exit_code": result.exit_code,
+        "stdout": result.stdout,
+        "stderr": result.stderr,
+    }
+
+
+if __name__ == "__main__":
+    main(eval_script, LANG_NAME, LANG_EXT)

Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/eval_javascript.py

+import os
+import subprocess
+from pathlib import Path
+
+
+def eval_script(path: Path):
+    try:
+        # Assumes exit-code 0 is all okay
+        output = subprocess.run(["node", str(path)], capture_output=True, timeout=5)
+
+        if output.returncode == 0:
+            status = "OK"
+        else:
+            outmessage = str(output)
+            if "ERR_ASSERTION" in outmessage:
+                status = "AssertionError"
+            elif "SyntaxError" in outmessage:
+                status = "SyntaxError"
+            elif "ReferenceError" in outmessage:
+                status = "ReferenceError"
+            else:
+                status = "Exception"
+        returncode = output.returncode
+    except subprocess.TimeoutExpired as exc:
+        status = "Timeout"
+        output = exc
+        returncode = -1
+    except subprocess.CalledProcessError as exc:
+        status = "Exception"
+        returncode = exc.returncode
+        output = exc
+    return {
+        "status": status,
+        "exit_code": returncode,
+        "stdout": "" if output.stdout is None else output.stdout.decode("utf-8"),
+        "stderr": "" if output.stderr is None else output.stderr.decode("utf-8"),
+    }
+
+
+def main():
+    directory = Path(
+        Path(__file__).parent, "..", "datasets", "js-keep-code_davinci_001_temp_0.2"
+    ).resolve()
+
+    for filename in os.listdir(directory):
+        r = eval_script(Path.joinpath(directory, filename))
+        filename = filename.split(".")[0]
+        print(f"JavaScript,{filename},{r['status']}")
+
+
+if __name__ == "__main__":
+    main()

Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/eval_julia.py

+from pathlib import Path
+
+from .safe_subprocess import run
+
+
+def eval_script(path: Path):
+    result = run(["julia", str(path)], timeout_seconds=5)
+    if result.timeout:
+        status = "Timeout"
+    elif result.exit_code == 0:
+        status = "OK"
+    # TODO(arjun): I would like this to be reviewed more carefully by John.
+    elif len(result.stderr) < 1:
+        status = "Exception"
+    else:
+        status = "SyntaxError"
+
+    return {
+        "status": status,
+        "exit_code": result.exit_code,
+        "stdout": result.stdout,
+        "stderr": result.stderr,
+    }

Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/eval_lua.py

+from pathlib import Path
+
+from .safe_subprocess import run
+
+
+def eval_script(path: Path):
+    r = run(["lua", str(path)])
+    if r.timeout:
+        status = "Timeout"
+    elif r.exit_code == 0:
+        status = "OK"
+    else:
+        status = "Exception"
+    return {
+        "status": status,
+        "exit_code": r.exit_code,
+        "stdout": r.stdout,
+        "stderr": r.stderr,
+    }

Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/eval_php.py

+from pathlib import Path
+
+from .safe_subprocess import run
+
+LANG_NAME = "PHP"
+LANG_EXT = ".php"
+
+
+def eval_script(path: Path):
+    r = run(["php", path])
+    if "PHP Parse error" in r.stdout:
+        status = "SyntaxError"
+    elif r.exit_code != 0:
+        status = "Exception"
+    else:
+        status = "OK"
+    return {
+        "status": status,
+        "exit_code": r.exit_code,
+        "stdout": r.stdout,
+        "stderr": r.stderr,
+    }

Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/eval_pl.py

+from pathlib import Path
+
+from .safe_subprocess import run
+
+
+def eval_script(path: Path):
+    r = run(["perl", path])
+
+    if r.timeout:
+        status = "Timeout"
+    elif r.exit_code != 0:
+        status = "Exception"
+    elif "ERROR" in r.stdout or "ERROR" in r.stderr:
+        status = "Exception"
+    else:
+        status = "OK"
+    return {
+        "status": status,
+        "exit_code": r.exit_code,
+        "stdout": r.stdout,
+        "stderr": r.stderr,
+    }

Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/eval_python.py

+from pathlib import Path
+
+from .safe_subprocess import run
+
+
+def eval_script(path: Path):
+    r = run(["python3", str(path)])
+    if r.timeout:
+        status = "Timeout"
+    elif r.exit_code == 0:
+        status = "OK"
+    elif "SyntaxError" in r.stderr:
+        status = "SyntaxError"
+    else:
+        status = "Exception"
+    return {
+        "status": status,
+        "exit_code": r.exit_code,
+        "stdout": r.stdout,
+        "stderr": r.stderr,
+    }

Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/eval_r.py

+import os
+import subprocess
+from pathlib import Path
+
+
+def eval_script(path: Path):
+    try:
+        # Assumes exit-code 0 is all okay
+        # Run R on the file, capturing stderr
+        output = subprocess.run(["Rscript", str(path)], capture_output=True, timeout=5)
+        if output.returncode == 0:
+            status = "OK"
+        else:
+            outmessage = str(output)
+            if "unexpected" in outmessage:
+                status = "SyntaxError"
+            elif "err=b''" in outmessage:
+                status = "AssertionError"
+            else:
+                status = "Exception"
+        returncode = output.returncode
+    except subprocess.TimeoutExpired as exc:
+        status = "Timeout"
+        output = exc
+        returncode = -1
+    except subprocess.CalledProcessError as exc:
+        status = "Exception"
+        returncode = exc.returncode
+        output = exc
+    return {
+        "status": status,
+        "exit_code": returncode,
+        "stdout": output.stdout,
+        "stderr": output.stderr,
+    }
+
+
+def main():
+    directory = Path(
+        Path(__file__).parent, "..", "datasets", "R-keep-code_davinci_001_temp_0.2"
+    ).resolve()
+
+    for filename in os.listdir(directory):
+        r = eval_script(Path.joinpath(directory, filename))
+        filename = filename.split(".")[0]
+        print(f"R,{filename},{r['status']}")
+
+
+if __name__ == "__main__":
+    main()

Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/eval_racket.py

+"""
+Evaluates a generated Racket program (.rkt).
+"""
+import os
+from pathlib import Path
+
+from .safe_subprocess import run
+
+
+def eval_script(path: Path):
+    result = run(["racket", str(path)])
+
+    if (
+        "standard-module-name-resolver: collection not found\n  for module path: rackunit"
+        in result.stderr
+    ):
+        print(f"Failed to run evaluation for {path}: rackunit is not installed")
+        return None
+
+    # rackunit produces exit code 0 even if tests fail.
+    if len(result.stderr) > 0 or result.exit_code != 0:
+        if "read-syntax" in result.stderr:
+            status = "SyntaxError"
+        else:
+            status = "Exception"
+    else:
+        status = "OK"
+
+    return {
+        "status": status,
+        "exit_code": result.exit_code,
+        "stdout": result.stdout,
+        "stderr": result.stderr,
+    }
+
+
+def main():
+    directory = Path(
+        Path(__file__).parent, "..", "datasets", "racket-keep-code_davinci_001_temp_0.2"
+    ).resolve()
+
+    for filename in os.listdir(directory):
+        r = eval_script(Path.joinpath(directory, filename))
+        filename = filename.split(".")[0]
+        print(f"Racket,{filename},{r['status']}")
+
+
+if __name__ == "__main__":
+    main()

Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/eval_ruby.py

+import argparse
+import subprocess
+from pathlib import Path
+
+from .generic_eval import main as gmain
+
+
+def eval_script(path: Path):
+    try:
+        # Assumes exit-code 0 is all okay
+        # Need check=True for Ruby to pass errors to CalledProcessError
+        output = subprocess.run(
+            ["ruby", path], check=True, capture_output=True, timeout=5
+        )
+        if output.returncode == 0:
+            status = "OK"
+            out = output.stderr
+            error = output.stdout
+            returncode = 0
+        else:
+            raise Exception("there's an issue with check = True for Ruby, INVESTIGATE!")
+    except subprocess.TimeoutExpired as exc:
+        status = "Timeout"
+        out = exc.stdout
+        error = exc.stderr
+        returncode = -1
+    except subprocess.CalledProcessError as exc:
+        returncode = exc.returncode
+        out = exc.stdout
+        error = exc.stderr
+        # failure with code 1 but no error message is an Exception from Failed tests
+        if len(error) < 1:
+            status = "Exception"
+        else:  # everything that prints out an error message is a SyntaxError
+            status = "SyntaxError"
+    return {
+        "status": status,
+        "exit_code": returncode,
+        "stdout": out,
+        "stderr": error,
+    }
+
+
+if __name__ == "__main__":
+    gmain(eval_script, "Ruby", ".rb")

Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/eval_rust.py

+import os
+import subprocess
+from pathlib import Path
+
+from .generic_eval import main
+
+LANG_NAME = "Rust"
+LANG_EXT = ".rs"
+
+
+def eval_script(path: Path):
+    basename = ".".join(str(path).split(".")[:-1])
+    try:
+        build = subprocess.run(
+            ["rustc", path, "-o", basename], capture_output=True, timeout=15
+        )
+    except subprocess.TimeoutExpired as exc:
+        return {
+            "status": "Timeout",
+            "exit_code": -1,
+            "stdout": "Compiler timeout",
+            "stderr": "Compiler timeout",
+        }
+    status = None
+    returncode = -1
+    output = None
+    if build.returncode != 0:
+        # Well, it's a compile error. May be a type error or
+        # something. But, why break the set convention
+        status = "SyntaxError"
+        returncode = build.returncode
+        output = build
+    else:
+        try:
+            # Assumes exit-code 0 is all okay
+            output = subprocess.run([basename], capture_output=True, timeout=5)
+            returncode = output.returncode
+            if output.returncode == 0:
+                status = "OK"
+            else:
+                # Well, it's a panic
+                status = "Exception"
+        except subprocess.TimeoutExpired as exc:
+            status = "Timeout"
+            output = exc
+        os.remove(basename)
+    return {
+        "status": status,
+        "exit_code": returncode,
+        "stdout": "" if output.stdout is None else output.stdout.decode("utf-8"),
+        "stderr": "" if output.stderr is None else output.stderr.decode("utf-8"),
+    }
+
+
+if __name__ == "__main__":
+    main(eval_script, LANG_NAME, LANG_EXT)