Skip to content

GitLab

Commit 5add46aa authored by hepj's avatar hepj
Browse files

添加Megatron项目

parent deb8370c
Pipeline #2199 failed with stages
in 0 seconds
Show whitespace changes
Inline Side-by-side
Showing with 1342 additions and 0 deletions
Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/diff_eval.py 0 → 100644
View file @ 5add46aa
# https://github.com/CarperAI/OpenELM/blob/e6402a0696096011572152334ccbe049f89c332e/src/openelm/utils/diff_eval.py #
import re
from enum import Enum
from typing import Optional
line_number_pattern = re.compile(
r"(?m)^@@ -(?P<l1>\d*),*?(?P<s1>\d*?) \+(?P<l2>\d*),*?(?P<s2>\d*?) @@"
)
diff_pattern = re.compile(
r"""<NME> (?P<name>.*?)
<BEF> (?P<file>(.|\n)*?)
<MSG> (?P<message>(.|\n)*?)
<DFF> (?P<diff>(.|\n)*)"""
)
hunk_split_pattern = re.compile(r"(?m)^(@@ .*? @@).*\n")
ignored = re.compile(r"(?m)^\\ No newline at end of file$\n?")
class DiffState(Enum):
"""
An Enum keeping track of the validity of the diff data. It is the return of the helper function `verify_diff`.
Binary codes help internally, as some errors are additive (e.g., can have both invalid text and invalid line num).
But we convert the binary codes into Enum for better readability.
"""
VALID = 0b000 # valid diff
# The following are errors that can still be either ignored or fixed.
INVALID_TEXT = 0b001 # pre-diff texts cannot be found in the context.
INVALID_LINE_NUM = (
0b010 # the numbers in @@ -x,y +a,b @@ are invalid (but can be parsed).
)
INVALID_TEXT_AND_LINE_NUM = 0b011 # both 0b001 and 0b010.
# The following are format errors that cannot be ignored.
BAD_FORMAT = 0b100 # cannot be parsed according to <NME> ...\n<BEF> ...\n<MSG> ...\n<DFF> @@ ... @@\n...
BAD_DIFF_HUNK_FORMAT = 0b101 # diff hunk contains lines whose initial character is not one of ' ', '+', '-'
BAD_LINE_NUM_FORMAT = (
0b110 # the @@ ... @@ bracket can be found but numbers cannot be parsed.
)
BAD_HUNK_AND_LINE_FORMAT = 0b111 # both 0b110 and 0b101.
def split_diff(content: str) -> dict:
"""
Args:
content: the diff content.
Returns:
A dict with potentially 4 items:
name: the filename
file: the file content
message: the diff message
diff: the diff hunk
Any key could be missing. That would mean a failure in matching.
"""
match = diff_pattern.match(content)
return {} if match is None else match.groupdict()
def parse_line_info(content: str) -> tuple:
"""
Parse @@ -x,y +a,b @@
Args:
the @@ ... @@ line
Returns:
(x, y, a, b) as integers
"""
match = line_number_pattern.match(content)
if match is None:
return ()
match_dict = match.groupdict()
# line numbers are mandatory
if not match_dict["l1"] or not match_dict["l2"]:
return ()
for s in ["s1", "s2"]:
# line ranges are optional and default to 1
match_dict[s] = match_dict[s] if match_dict[s] else "1"
return (
int(match_dict["l1"]),
int(match_dict["s1"]),
int(match_dict["l2"]),
int(match_dict["s2"]),
)
def parse_diff_content(
hunk: str, separate_lines=False, reject_invalid=False
) -> Optional[tuple]:
"""
Parse a diff content to turn it into (before_diff, after_diff) based on '+', '-' at the beginning of each line.
Args:
hunk: the diff content (without "@@ ... @@").
separate_lines: (Optional) True if return list of lines.
reject_invalid: (Optional) True if return None for invalid diff hunk (non-empty lines without starting
with ' ', '-', '+')
Returns:
(before_diff, after_diff);
None if reject_invalid==True and the diff hunk contains invalid format.
"""
hunk = hunk.split("\n")
before_diff, after_diff = [], []
for line in hunk:
# Ignore invalid trailing '\n'. An empty line in the diff hunk should at least be '\n ' with the space.
if not line:
continue
if line[0] == "-" or line[0] == " ":
before_diff.append(line[1:])
if line[0] == "+" or line[0] == " ":
after_diff.append(line[1:])
if reject_invalid:
if all([line[0] != c for c in [" ", "-", "+"]]):
return None
if separate_lines:
return before_diff, after_diff
else:
return "\n".join(before_diff), "\n".join(after_diff)
def replace_text(
text: str,
before: str,
after: str,
start_pointer: int,
reject_incomplete_line: bool = True,
) -> tuple[str, int]:
"""
Try to match `before` within `text` and replace the content into `after`.
If not found, return the original text.
Args:
text: the original text.
before: the text to be matched.
after: the text to be replaced into.
start_pointer: the index where we start to match (inclusive).
reject_incomplete_line: (Optional) reject the patch if `before` does not match till the end of a line.
Returns:
(diff_result, new_start_pointer)
the text after the match-and-replace and the new index at the end of the change.
"""
idx = text[start_pointer:].find(before)
start_idx = start_pointer + idx
if reject_incomplete_line:
# If the end of the match is neither EOF nor \n, reject the patch.
if (
idx >= 0
and start_idx + len(before) < len(text)
and text[start_idx + len(before)] != "\n"
):
return text, start_pointer
if idx < 0:
return text, start_pointer
else:
# Even if start_idx + len(before) is out-of-bound, the list slicing would return ""
return text[:start_idx] + after + text[
start_idx + len(before) :
], start_idx + len(after)
def apply_diff(file: str, diff: str, use_line_number=False, allow_add_file=True) -> str:
"""
Apply the diff to the file content. We try to be lenient and keep applying the patch naively until we cannot.
(Note: use_line_number=True is somehow slightly slower.)
(Warning: if use_line_number==False, we could have some problematic cases like, if all lines in diff hunk
starts with "+", the pre-diff paragraphs relevant to the hunk is empty. Because we only use pre-diff
paragraphs to match, we would simply match the very beginning.)
Args:
file: the file content.
diff: the diff hunk (containing "@@ -x,y +a,b @@").
use_line_number: (Optional) use the line numbers in "@@ ... @@" faithfully.
allow_add_file: (Optional) when file is "ADDFILE" (meaning <BEF> ADDFILE\n... showed up in the diff text),
we automatically patch the diff by a direct replacement.
Return:
the maximally patched file content.
"""
diff = hunk_split_pattern.split(ignored.sub("", diff))
# If we use the line numbers, we match-and-replace in a line-by-line fashion.
file_by_line = file.split("\n") if use_line_number else None
line_offset = 0 # the offset between pre-/post-patching line numbers
# If we do not use the line numbers, for multiple diff hunk, we only move forward in a greedy manner.
patch_pointer = 0
i = (
0 if diff[0] else 1
) # We have delimiter at the beginning, causing empty initial string
while (
i < len(diff) - 1
): # Need at least a pair of '@@ ... @@' and diff hunk to continue
# Expect a string with '@@ ... @@' followed by a diff hunk
line_info = parse_line_info(diff[i])
diff_content = diff[i + 1]
i += 2
# Generate the pre-/post-diff string based on the first character being '+' or '-'
# (Note: parse_diff_content will ignore trailing \n at the beginning and at the end)
parsed_diff = parse_diff_content(diff_content, separate_lines=use_line_number)
# If we allow the recognition of "ADDFILE", special treatment is needed.
if allow_add_file and file == "ADDFILE":
if use_line_number:
# Immediately apply the first hunk but also check the partial validity of line numbers.
return parsed_diff[1] if line_info == (0, 0) else ""
else:
# Immediately apply the first hunk and ignore the rest.
return parsed_diff[1]
if use_line_number:
# If line numbers cannot be parsed, skip.
if not line_info:
continue
# Offset the starting line
start_idx = line_info[0] + line_offset
# Match the referred lines with the file context
referred_lines = file_by_line[start_idx - 1 : start_idx - 1 + line_info[1]]
valid = all([l1 == l2 for l1, l2 in zip(parsed_diff[0], referred_lines)])
# If lines fully match and the number of lines is consistent, apply the patch.
# We ignore the second pair "+a, b" just to be lenient.
if valid and len(parsed_diff[0]) == line_info[1]:
# Update the list of lines
if start_idx == 0: # Add lines to the beginning.
file_by_line = parsed_diff[1] + file_by_line
else:
file_by_line = (
file_by_line[: start_idx - 1]
+ parsed_diff[1]
+ file_by_line[start_idx - 1 + line_info[1] :]
)
line_offset += len(parsed_diff[1]) - line_info[1]
else:
# CAUTION: this way of handling empty context is being very lenient and could lead to
# undesirable behaviors. Only do this when you want to be as tolerant as possible.
if parsed_diff[0] == "":
if (
patch_pointer != 0
): # Lack of matching context can only happen at the beginning of file.
continue
file = parsed_diff[1] + "\n" + file
patch_pointer = len(parsed_diff[0]) + 1
else:
# Directly (and naively) apply patch by match-and-replace.
file, patch_pointer = replace_text(
file, parsed_diff[0], parsed_diff[1], patch_pointer
)
if use_line_number:
file = "\n".join(file_by_line)
return file
def verify_diff(diff_text: str) -> DiffState:
"""
Verify the validity of a complete diff text.
Args:
diff_text: the complete diff text.
The overall format conforms "<NME> ...\n<BEF> ...\n<MSG> ...\n<DFF> ..." and the text
after <DFF> has 1 or more lines of "@@ -x,y +a,b @@" followed by the corresponding hunk.
Returns:
A DiffState (see above).
"""
diff_dict = split_diff(
ignored.sub("", diff_text)
) # Ignore the GitHub warning on the end of file
line_offset = 0
keys = ["name", "file", "message", "diff"]
for key in keys:
if key not in diff_dict:
return DiffState(0b100) # Invalid overall format
diff_parts = hunk_split_pattern.split(diff_dict["diff"])
if not diff_parts:
return DiffState(0b100) # Invalid overall format
context_mismatch, line_number_mismatch = False, False
bad_diff_hunk, bad_line_number = False, False
i = 0 if diff_parts[0] else 1
while (
i < len(diff_parts) - 1
): # Need at least a pair of '@@ ... @@' and diff hunk to continue
line_info = parse_line_info(diff_parts[i])
diff_content = parse_diff_content(diff_parts[i + 1], reject_invalid=True)
i += 2
# Special treatment if we are adding a new file
if diff_dict["file"] == "ADDFILE":
if (
len(diff_parts) != i
or not line_info
or line_info[:3] != (0, 0, 1)
or line_info[3] != len(diff_content[1].split("\n"))
or diff_content[0]
):
return DiffState(0b110)
else:
return DiffState(0b000)
if not line_info or len(line_info) != 4:
bad_line_number = True
if diff_content is None:
bad_diff_hunk = True
# Skip the diff matching checks if bad format already occurred
if bad_diff_hunk or bad_line_number:
continue
# Try to see if there is a match in the file context. Must match complete lines or till EOF.
match_idx = diff_dict["file"].find(diff_content[0])
if match_idx == -1 or (
match_idx + len(diff_content[0]) != len(diff_dict["file"])
and diff_dict["file"][match_idx + len(diff_content[0])] != "\n"
):
context_mismatch = True
if line_info[0] <= 0:
# -0,0 only happens when we create a new file (in which case the context is <BEF> ADDFILE\n...).
if line_info[1] != 0 or diff_dict["file"] != "ADDFILE":
line_number_mismatch = True
else:
# Check the line numbers regardless of whether the context matches.
pre_diff_line_number = len(diff_content[0].split("\n"))
post_diff_line_number = len(diff_content[1].split("\n"))
if (pre_diff_line_number, post_diff_line_number) != (
line_info[1],
line_info[3],
):
line_number_mismatch = True
else:
line_offset += len(diff_content[1]) - line_info[1]
if bad_diff_hunk or bad_line_number:
return DiffState(bad_diff_hunk * 0b001 + bad_line_number * 0b010 + 0b100)
else:
return DiffState(context_mismatch * 0b001 + line_number_mismatch * 0b010)
\ No newline at end of file
Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/execute.py 0 → 100644
View file @ 5add46aa
# Copyright 2020 The HuggingFace Datasets Authors and the current dataset script contributor.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This code is adapted from OpenAI's release
# https://github.com/openai/human-eval/blob/master/human_eval/execution.py
import contextlib
import faulthandler
import io
import multiprocessing
import os
import platform
import signal
import tempfile
def check_correctness(check_program, timeout, task_id, completion_id):
"""
Evaluates the functional correctness of a completion by running the test
suite provided in the problem.
:param completion_id: an optional completion ID so we can match
the results later even if execution finishes asynchronously.
"""
manager = multiprocessing.Manager()
result = manager.list()
p = multiprocessing.Process(target=unsafe_execute, args=(check_program, result, timeout))
p.start()
p.join(timeout=timeout + 1)
if p.is_alive():
p.kill()
if not result:
result.append("timed out")
return dict(
task_id=task_id,
passed=result[0] == "passed",
result=result[0],
completion_id=completion_id,
)
def unsafe_execute(check_program, result, timeout):
with create_tempdir():
# These system calls are needed when cleaning up tempdir.
import os
import shutil
rmtree = shutil.rmtree
rmdir = os.rmdir
chdir = os.chdir
# Disable functionalities that can make destructive changes to the test.
reliability_guard()
# Run program.
try:
exec_globals = {}
with swallow_io():
with time_limit(timeout):
exec(check_program, exec_globals)
result.append("passed")
except TimeoutException:
result.append("timed out")
except BaseException as e:
result.append(f"failed: {e}")
# Needed for cleaning up.
shutil.rmtree = rmtree
os.rmdir = rmdir
os.chdir = chdir
@contextlib.contextmanager
def time_limit(seconds):
def signal_handler(signum, frame):
raise TimeoutException("Timed out!")
signal.setitimer(signal.ITIMER_REAL, seconds)
signal.signal(signal.SIGALRM, signal_handler)
try:
yield
finally:
signal.setitimer(signal.ITIMER_REAL, 0)
@contextlib.contextmanager
def swallow_io():
stream = WriteOnlyStringIO()
with contextlib.redirect_stdout(stream):
with contextlib.redirect_stderr(stream):
with redirect_stdin(stream):
yield
@contextlib.contextmanager
def create_tempdir():
with tempfile.TemporaryDirectory() as dirname:
with chdir(dirname):
yield dirname
class TimeoutException(Exception):
pass
class WriteOnlyStringIO(io.StringIO):
"""StringIO that throws an exception when it's read from"""
def read(self, *args, **kwargs):
raise OSError
def readline(self, *args, **kwargs):
raise OSError
def readlines(self, *args, **kwargs):
raise OSError
def readable(self, *args, **kwargs):
"""Returns True if the IO object can be read."""
return False
class redirect_stdin(contextlib._RedirectStream): # type: ignore
_stream = "stdin"
@contextlib.contextmanager
def chdir(root):
if root == ".":
yield
return
cwd = os.getcwd()
os.chdir(root)
try:
yield
except BaseException as exc:
raise exc
finally:
os.chdir(cwd)
def reliability_guard(maximum_memory_bytes=None):
"""
This disables various destructive functions and prevents the generated code
from interfering with the test (e.g. fork bomb, killing other processes,
removing filesystem files, etc.)
WARNING
This function is NOT a security sandbox. Untrusted code, including, model-
generated code, should not be blindly executed outside of one. See the
Codex paper for more information about OpenAI's code sandbox, and proceed
with caution.
"""
if maximum_memory_bytes is not None:
import resource
resource.setrlimit(resource.RLIMIT_AS, (maximum_memory_bytes, maximum_memory_bytes))
resource.setrlimit(resource.RLIMIT_DATA, (maximum_memory_bytes, maximum_memory_bytes))
if not platform.uname().system == "Darwin":
resource.setrlimit(resource.RLIMIT_STACK, (maximum_memory_bytes, maximum_memory_bytes))
faulthandler.disable()
import builtins
builtins.exit = None
builtins.quit = None
import os
os.environ["OMP_NUM_THREADS"] = "1"
os.kill = None
os.system = None
os.putenv = None
os.remove = None
os.removedirs = None
os.rmdir = None
os.fchdir = None
os.setuid = None
os.fork = None
os.forkpty = None
os.killpg = None
os.rename = None
os.renames = None
os.truncate = None
os.replace = None
os.unlink = None
os.fchmod = None
os.fchown = None
os.chmod = None
os.chown = None
os.chroot = None
os.fchdir = None
os.lchflags = None
os.lchmod = None
os.lchown = None
os.getcwd = None
os.chdir = None
import shutil
shutil.rmtree = None
shutil.move = None
shutil.chown = None
import subprocess
subprocess.Popen = None # type: ignore
__builtins__["help"] = None
import sys
sys.modules["ipdb"] = None
sys.modules["joblib"] = None
sys.modules["resource"] = None
sys.modules["psutil"] = None
sys.modules["tkinter"] = None
Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/containerized_eval.py 0 → 100644
View file @ 5add46aa
"""
NOTE: Nothing containerized about this any more. This is just a helper
for problem_evaluator.py.
"""
import tempfile
from pathlib import Path
from . import (eval_cpp, eval_dlang, eval_java, eval_javascript, eval_julia,
eval_lua, eval_php, eval_python, eval_r, eval_racket, eval_ruby,
eval_rust, eval_swift, eval_ts, eval_go, eval_pl, eval_sh, eval_scala, eval_cs)
EVALUATORS = {
"rb": (eval_ruby.eval_script, ".rb"),
"lua": (eval_lua.eval_script, ".lua"),
"python": (eval_python.eval_script, ".py"),
"py": (eval_python.eval_script, ".py"),
"notypes.py": (eval_python.eval_script, ".py"),
"julia": (eval_julia.eval_script, ".jl"),
"java": (eval_java.eval_script, ".java"),
"rust": (eval_rust.eval_script, ".rs"),
"rs": (eval_rust.eval_script, ".rs"),
"swift": (eval_swift.eval_script, ".swift"),
"lua": (eval_lua.eval_script, ".lua"),
"racket": (eval_racket.eval_script, ".rkt"),
"rkt": (eval_racket.eval_script, ".rkt"),
"javascript": (eval_javascript.eval_script, ".js"),
"js": (eval_javascript.eval_script, ".js"),
"cpp": (eval_cpp.eval_script, ".cpp"),
"cs": (eval_cs.eval_script, ".cs"),
"php": (eval_php.eval_script, ".php"),
"humaneval_to_dlang.py": (eval_dlang.eval_script, ".d"),
"d": (eval_dlang.eval_script, ".d"),
"r": (eval_r.eval_script, ".r"),
"humaneval_to_r.py": (eval_r.eval_script, ".r"),
"jl": (eval_julia.eval_script, ".jl"),
"ts": (eval_ts.eval_script, ".ts"),
"go": (eval_go.eval_script, ".go"),
"pl": (eval_pl.eval_script, ".pl"),
"sh": (eval_sh.eval_script, ".sh"),
"scala": (eval_scala.eval_script, ".scala"),
}
def eval_string_script(language, program):
if language in EVALUATORS:
(eval_script, file_ext) = EVALUATORS[language]
else:
eval_module = __import__(
f"eval_{language}" if language != "go_test.go" else "eval_go"
)
eval_script = eval_module.eval_script
file_ext = f".{language}" if language != "go_test.go" else "_test.go"
with tempfile.NamedTemporaryFile(suffix=file_ext, delete=True) as f:
f.write(program.encode("utf-8"))
f.flush()
result = eval_script(Path(f.name))
# Only save the first 2K of output from the running program. Any futher
# output is very likely an exceptionally long stack trace or a long
# series of prints.
if type(result["stdout"]) == bytes:
result["stdout"] = result["stdout"].decode("utf-8", errors="ignore")
if result["stdout"] is None:
result["stdout"] = ""
if result["stderr"] is None:
result["stderr"] = ""
if type(result["stderr"]) == bytes:
result["stderr"] = result["stderr"].decode("utf-8", errors="ignore")
assert type(result["stdout"]) == str
assert type(result["stderr"]) == str
return {
"program": program,
"stdout": result["stdout"].replace("!!int", "")[:2048],
"stderr": result["stderr"][:2048],
"exit_code": result["exit_code"],
"status": result["status"],
}
Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/eval_cpp.py 0 → 100644
View file @ 5add46aa
from pathlib import Path
from .generic_eval import main
from .safe_subprocess import run
LANG_NAME = "C++"
LANG_EXT = ".cpp"
def eval_script(path: Path):
basename = ".".join(str(path).split(".")[:-1])
build_result = run(["g++", path, "-o", basename, "-std=c++17"])
if build_result.exit_code != 0:
return {
"status": "SyntaxError",
"exit_code": build_result.exit_code,
"stdout": build_result.stdout,
"stderr": build_result.stderr,
}
run_result = run([basename])
if "In file included from /shared/centos7/gcc/9.2.0-skylake/" in run_result.stderr:
raise Exception("Skylake bug encountered")
if "/4.8.2" in run_result.stderr:
raise Exception("Ancient compiler encountered")
if run_result.timeout:
status = "Timeout"
elif run_result.exit_code != 0:
status = "Exception"
else:
status = "OK"
return {
"status": status,
"exit_code": run_result.exit_code,
"stdout": run_result.stdout,
"stderr": run_result.stderr,
}
if __name__ == "__main__":
main(eval_script, LANG_NAME, LANG_EXT)
Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/eval_cs.py 0 → 100644
View file @ 5add46aa
import os
import subprocess
from .generic_eval import main
LANG_NAME = "CSharp"
LANG_EXT = ".cs"
# Following files have problems:
# 137,
# 22: Any
# 148: Elipsis
def eval_script(path: str):
if ".cs" not in path.name:
return
basename = ".".join(str(path).split(".")[:-1])
binaryname = basename + ".exe"
build = subprocess.run(
["csc", "/d:DEBUG", "-r:System.Numerics.dll", path, f"/out:{binaryname}"],
capture_output=True,
)
status = None
returncode = -1
output = None
if build.returncode != 0:
# Well, it's a compile error. May be a type error or
# something. But, why break the set convention
status = "SyntaxError"
returncode = build.returncode
output = build
else:
try:
output = subprocess.run(
["mono", binaryname],
env={"PATH": os.getenv("PATH"), "MONO_TRACE_LISTENER": "Console.Error"},
capture_output=True,
timeout=5,
)
returncode = output.returncode
output.stderr = str(output.stderr, "utf-8")
# mono return 0 even when failing
fail = (
"System.Diagnostics.DefaultTraceListener.Fail" in output.stderr
or "Unhandled Exception" in output.stderr
)
output.returncode = 1 if fail else 0
if output.returncode == 0:
status = "OK"
else:
# Well, it's a panic
status = "Exception"
except subprocess.TimeoutExpired as exc:
status = "Timeout"
output = exc
os.remove(binaryname)
if output.stdout is not None:
output.stdout = output.stdout.decode("utf-8")
else:
output.stdout = "None"
if output.stderr == "":
output.stderr = "None"
return {
"status": status,
"exit_code": returncode,
"stdout": output.stdout,
"stderr": output.stderr,
}
if __name__ == "__main__":
main(eval_script, LANG_NAME, LANG_EXT)
Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/eval_dlang.py 0 → 100644
View file @ 5add46aa
import os
import re
from pathlib import Path
from .safe_subprocess import run
ENABLE_SYNTAX_CHECK = False
def eval_script(path: Path):
result = run(["rdmd", "-unittest", str(path)], timeout_seconds=15)
if "might not be correctly installed" in result.stderr:
raise Exception("D is not correctly installed")
if result.timeout:
status = "Timeout"
elif result.exit_code == 0:
status = "OK"
elif "Error:" in result.stderr:
status = "SyntaxError"
else:
status = "Exception"
return {
"status": status,
"exit_code": result.exit_code,
"stdout": result.stdout,
"stderr": result.stderr,
}
DIR = "d-keep-code_davinci_001_temp_0.2"
def main():
directory = Path(Path(__file__).parent, "..", "datasets", DIR).resolve()
count = {"OK": 0, "Timeout": 0, "Exception": 0, "SyntaxError": 0}
for filename in os.listdir(directory):
path = Path.joinpath(directory, filename)
r = eval_script(path)
status = r["status"]
count[status] += 1
if ENABLE_SYNTAX_CHECK and status == "SyntaxError":
error_msgs = r["stderr"].split("\n")
with open(path) as source_file:
lines = source_file.readlines()
unittest_line_start = lines.index("unittest\n")
unittest_line_end = len(lines)
for err_msg_line in error_msgs:
matched_parts = re.match(
r"(\/?.*?\.[\w:]+\/.*.d)\(([0-9]+)\): Error: (.*)",
err_msg_line[2:-1],
)
_file, line_num = matched_parts[1], int(matched_parts[2])
if (
unittest_line_start <= line_num
and line_num <= unittest_line_end
):
print("===============")
print(path, "contains error in unit test part")
print(error_msgs)
print("===============")
filename = filename.split(".")[0]
print(f"Dlang,{filename},{status}")
print(DIR + ":" + str(count))
if __name__ == "__main__":
main()
Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/eval_go.py 0 → 100644
View file @ 5add46aa
import argparse
import subprocess
from pathlib import Path
from sys import exit
from .generic_eval import main as gmain
def eval_script(path: Path):
status = None
stdout = None
stderr = None
exit_code = None
try:
build = subprocess.run(
["go", "test", path],
timeout=30,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout = build.stdout.decode("utf-8", errors="ignore")
stderr = build.stderr.decode("utf-8", errors="ignore")
exit_code = build.returncode
# write to stderr just so that we can redirect stdout to a csv
if "[setup failed]" in stdout or "[build failed]" in stdout:
status = "SyntaxError"
elif "FAIL" in stdout:
status = "Exception"
else:
status = "OK"
except subprocess.TimeoutExpired:
status = "Timeout"
return {
"status": status,
"exit_code": exit_code,
"stdout": stdout,
"stderr": stderr,
}
if __name__ == "__main__":
gmain(eval_script, "Go", ".go")
Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/eval_java.py 0 → 100644
View file @ 5add46aa
import os
import tempfile
from pathlib import Path
from .generic_eval import main
from .safe_subprocess import run
LANG_NAME = "Java"
LANG_EXT = ".java"
# Following files have problems:
# 137,
# 22: Any
# 148: Elipsis
def eval_script(path: Path):
sys_env = os.environ.copy()
javatuples_path = Path("/usr/multiple/javatuples-1.2.jar")
sys_env["CLASSPATH"] = f"{javatuples_path}"
with tempfile.TemporaryDirectory() as outdir:
# Each Java file contains the class with same name `JAVA_CLASS_NAME`
# Hence, javac will same JAVA_CLASS_NAME.class file for each problem
# Write class for each problem to a different temp dir
# Use UTF8 encoding with javac
result = run(["javac", "-encoding", "UTF8", "-d", outdir, path], env=sys_env)
if result.exit_code != 0:
# Well, it's a compile error. May be a type error or
# something. But, why break the set convention
status = "SyntaxError"
else:
result = run(["java", "-ea", "-cp", f"{outdir}", "Problem"], env=sys_env)
if result.timeout:
status = "Timeout"
elif result.exit_code == 0:
status = "OK"
else:
status = "Exception"
return {
"status": status,
"exit_code": result.exit_code,
"stdout": result.stdout,
"stderr": result.stderr,
}
if __name__ == "__main__":
main(eval_script, LANG_NAME, LANG_EXT)
Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/eval_javascript.py 0 → 100644
View file @ 5add46aa
import os
import subprocess
from pathlib import Path
def eval_script(path: Path):
try:
# Assumes exit-code 0 is all okay
output = subprocess.run(["node", str(path)], capture_output=True, timeout=5)
if output.returncode == 0:
status = "OK"
else:
outmessage = str(output)
if "ERR_ASSERTION" in outmessage:
status = "AssertionError"
elif "SyntaxError" in outmessage:
status = "SyntaxError"
elif "ReferenceError" in outmessage:
status = "ReferenceError"
else:
status = "Exception"
returncode = output.returncode
except subprocess.TimeoutExpired as exc:
status = "Timeout"
output = exc
returncode = -1
except subprocess.CalledProcessError as exc:
status = "Exception"
returncode = exc.returncode
output = exc
return {
"status": status,
"exit_code": returncode,
"stdout": "" if output.stdout is None else output.stdout.decode("utf-8"),
"stderr": "" if output.stderr is None else output.stderr.decode("utf-8"),
}
def main():
directory = Path(
Path(__file__).parent, "..", "datasets", "js-keep-code_davinci_001_temp_0.2"
).resolve()
for filename in os.listdir(directory):
r = eval_script(Path.joinpath(directory, filename))
filename = filename.split(".")[0]
print(f"JavaScript,{filename},{r['status']}")
if __name__ == "__main__":
main()
Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/eval_julia.py 0 → 100644
View file @ 5add46aa
from pathlib import Path
from .safe_subprocess import run
def eval_script(path: Path):
result = run(["julia", str(path)], timeout_seconds=5)
if result.timeout:
status = "Timeout"
elif result.exit_code == 0:
status = "OK"
# TODO(arjun): I would like this to be reviewed more carefully by John.
elif len(result.stderr) < 1:
status = "Exception"
else:
status = "SyntaxError"
return {
"status": status,
"exit_code": result.exit_code,
"stdout": result.stdout,
"stderr": result.stderr,
}
Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/eval_lua.py 0 → 100644
View file @ 5add46aa
from pathlib import Path
from .safe_subprocess import run
def eval_script(path: Path):
r = run(["lua", str(path)])
if r.timeout:
status = "Timeout"
elif r.exit_code == 0:
status = "OK"
else:
status = "Exception"
return {
"status": status,
"exit_code": r.exit_code,
"stdout": r.stdout,
"stderr": r.stderr,
}
Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/eval_php.py 0 → 100644
View file @ 5add46aa
from pathlib import Path
from .safe_subprocess import run
LANG_NAME = "PHP"
LANG_EXT = ".php"
def eval_script(path: Path):
r = run(["php", path])
if "PHP Parse error" in r.stdout:
status = "SyntaxError"
elif r.exit_code != 0:
status = "Exception"
else:
status = "OK"
return {
"status": status,
"exit_code": r.exit_code,
"stdout": r.stdout,
"stderr": r.stderr,
}
Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/eval_pl.py 0 → 100644
View file @ 5add46aa
from pathlib import Path
from .safe_subprocess import run
def eval_script(path: Path):
r = run(["perl", path])
if r.timeout:
status = "Timeout"
elif r.exit_code != 0:
status = "Exception"
elif "ERROR" in r.stdout or "ERROR" in r.stderr:
status = "Exception"
else:
status = "OK"
return {
"status": status,
"exit_code": r.exit_code,
"stdout": r.stdout,
"stderr": r.stderr,
}
Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/eval_python.py 0 → 100644
View file @ 5add46aa
from pathlib import Path
from .safe_subprocess import run
def eval_script(path: Path):
r = run(["python3", str(path)])
if r.timeout:
status = "Timeout"
elif r.exit_code == 0:
status = "OK"
elif "SyntaxError" in r.stderr:
status = "SyntaxError"
else:
status = "Exception"
return {
"status": status,
"exit_code": r.exit_code,
"stdout": r.stdout,
"stderr": r.stderr,
}
Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/eval_r.py 0 → 100644
View file @ 5add46aa
import os
import subprocess
from pathlib import Path
def eval_script(path: Path):
try:
# Assumes exit-code 0 is all okay
# Run R on the file, capturing stderr
output = subprocess.run(["Rscript", str(path)], capture_output=True, timeout=5)
if output.returncode == 0:
status = "OK"
else:
outmessage = str(output)
if "unexpected" in outmessage:
status = "SyntaxError"
elif "err=b''" in outmessage:
status = "AssertionError"
else:
status = "Exception"
returncode = output.returncode
except subprocess.TimeoutExpired as exc:
status = "Timeout"
output = exc
returncode = -1
except subprocess.CalledProcessError as exc:
status = "Exception"
returncode = exc.returncode
output = exc
return {
"status": status,
"exit_code": returncode,
"stdout": output.stdout,
"stderr": output.stderr,
}
def main():
directory = Path(
Path(__file__).parent, "..", "datasets", "R-keep-code_davinci_001_temp_0.2"
).resolve()
for filename in os.listdir(directory):
r = eval_script(Path.joinpath(directory, filename))
filename = filename.split(".")[0]
print(f"R,{filename},{r['status']}")
if __name__ == "__main__":
main()
Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/eval_racket.py 0 → 100644
View file @ 5add46aa
"""
Evaluates a generated Racket program (.rkt).
"""
import os
from pathlib import Path
from .safe_subprocess import run
def eval_script(path: Path):
result = run(["racket", str(path)])
if (
"standard-module-name-resolver: collection not found\n for module path: rackunit"
in result.stderr
):
print(f"Failed to run evaluation for {path}: rackunit is not installed")
return None
# rackunit produces exit code 0 even if tests fail.
if len(result.stderr) > 0 or result.exit_code != 0:
if "read-syntax" in result.stderr:
status = "SyntaxError"
else:
status = "Exception"
else:
status = "OK"
return {
"status": status,
"exit_code": result.exit_code,
"stdout": result.stdout,
"stderr": result.stderr,
}
def main():
directory = Path(
Path(__file__).parent, "..", "datasets", "racket-keep-code_davinci_001_temp_0.2"
).resolve()
for filename in os.listdir(directory):
r = eval_script(Path.joinpath(directory, filename))
filename = filename.split(".")[0]
print(f"Racket,{filename},{r['status']}")
if __name__ == "__main__":
main()
Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/eval_ruby.py 0 → 100644
View file @ 5add46aa
import argparse
import subprocess
from pathlib import Path
from .generic_eval import main as gmain
def eval_script(path: Path):
try:
# Assumes exit-code 0 is all okay
# Need check=True for Ruby to pass errors to CalledProcessError
output = subprocess.run(
["ruby", path], check=True, capture_output=True, timeout=5
)
if output.returncode == 0:
status = "OK"
out = output.stderr
error = output.stdout
returncode = 0
else:
raise Exception("there's an issue with check = True for Ruby, INVESTIGATE!")
except subprocess.TimeoutExpired as exc:
status = "Timeout"
out = exc.stdout
error = exc.stderr
returncode = -1
except subprocess.CalledProcessError as exc:
returncode = exc.returncode
out = exc.stdout
error = exc.stderr
# failure with code 1 but no error message is an Exception from Failed tests
if len(error) < 1:
status = "Exception"
else: # everything that prints out an error message is a SyntaxError
status = "SyntaxError"
return {
"status": status,
"exit_code": returncode,
"stdout": out,
"stderr": error,
}
if __name__ == "__main__":
gmain(eval_script, "Ruby", ".rb")
Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/eval_rust.py 0 → 100644
View file @ 5add46aa
import os
import subprocess
from pathlib import Path
from .generic_eval import main
LANG_NAME = "Rust"
LANG_EXT = ".rs"
def eval_script(path: Path):
basename = ".".join(str(path).split(".")[:-1])
try:
build = subprocess.run(
["rustc", path, "-o", basename], capture_output=True, timeout=15
)
except subprocess.TimeoutExpired as exc:
return {
"status": "Timeout",
"exit_code": -1,
"stdout": "Compiler timeout",
"stderr": "Compiler timeout",
}
status = None
returncode = -1
output = None
if build.returncode != 0:
# Well, it's a compile error. May be a type error or
# something. But, why break the set convention
status = "SyntaxError"
returncode = build.returncode
output = build
else:
try:
# Assumes exit-code 0 is all okay
output = subprocess.run([basename], capture_output=True, timeout=5)
returncode = output.returncode
if output.returncode == 0:
status = "OK"
else:
# Well, it's a panic
status = "Exception"
except subprocess.TimeoutExpired as exc:
status = "Timeout"
output = exc
os.remove(basename)
return {
"status": status,
"exit_code": returncode,
"stdout": "" if output.stdout is None else output.stdout.decode("utf-8"),
"stderr": "" if output.stderr is None else output.stderr.decode("utf-8"),
}
if __name__ == "__main__":
main(eval_script, LANG_NAME, LANG_EXT)
Bigcode-Evaluation-Harness-240327/bigcode_eval/tasks/custom_metrics/multiple_metrics/eval_scala.py 0 → 100644
View file @ 5add46aa
import tempfile
from pathlib import Path
from .safe_subprocess import run
LANG_NAME = "Scala"
LANG_EXT = ".scala"
def eval_script(path: Path):
with tempfile.TemporaryDirectory() as outdir:
# Each Scala file contains the class with same name `JAVA_CLASS_NAME`
# Hence, scalac will same JAVA_CLASS_NAME.class file for each problem
# Write class for each problem to a different temp dir
build = run(["scalac", "-d", outdir, path], timeout_seconds=45)
if build.exit_code != 0:
# Well, it's a compile error. May be a type error or
# something. But, why break the set convention
return {
"status": "SyntaxError",
"exit_code": build.exit_code,
"stdout": build.stdout,
"stderr": build.stderr,
}
# "Problem" is the name of the class we emit.
r = run(["scala", "-cp", f"{outdir}", "Problem"])
if r.timeout:
status = "Timeout"
elif r.exit_code == 0 and r.stderr == "":
status = "OK"
else:
# Well, it's a panic
status = "Exception"
return {
"status": status,
"exit_code": r.exit_code,
"stdout": r.stdout,
"stderr": r.stderr,
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment