"...ops/git@developer.sourcefind.cn:OpenDAS/mmdetection3d.git" did not exist on "79a8299cf8a939f69929db1e543443252fbb6629"
Unverified Commit 2a358f45 authored by Sam Shleifer's avatar Sam Shleifer Committed by GitHub
Browse files

[s2s] fix nltk pytest race condition with FileLock (#7515)

parent 72d363d9
import re
from filelock import FileLock
try:
import nltk
......@@ -9,13 +11,12 @@ except (ImportError, ModuleNotFoundError):
NLTK_AVAILABLE = False
if NLTK_AVAILABLE:
try:
with FileLock("a_random_string") as lock:
nltk.download("punkt", quiet=True)
except FileExistsError: # multiprocessing race condition
pass
def add_newline_to_end_of_each_sentence(x: str) -> str:
"""This was added to get rougeLsum scores matching published rougeL scores for BART and PEGASUS."""
re.sub("<n>", "", x) # remove pegasus newline char
assert NLTK_AVAILABLE, "nltk must be installed to separate newlines betwee sentences. (pip install nltk)"
assert NLTK_AVAILABLE, "nltk must be installed to separate newlines between sentences. (pip install nltk)"
return "\n".join(nltk.sent_tokenize(x))
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment