Unverified Commit e3669375 authored by Yih-Dar's avatar Yih-Dar Committed by GitHub
Browse files

Fix 2 tests in `FillMaskPipelineTests` (#27889)



* fix

* fix

* fix

* fix

* fix

---------
Co-authored-by: default avatarydshieh <ydshieh@users.noreply.github.com>
parent 79e76559
...@@ -216,15 +216,24 @@ class FillMaskPipelineTests(unittest.TestCase): ...@@ -216,15 +216,24 @@ class FillMaskPipelineTests(unittest.TestCase):
], ],
) )
dummy_str = "Lorem ipsum dolor sit amet, consectetur adipiscing elit," * 100
outputs = unmasker( outputs = unmasker(
"My name is <mask>" + "Lorem ipsum dolor sit amet, consectetur adipiscing elit," * 100, "My name is <mask>" + dummy_str,
tokenizer_kwargs={"truncation": True}, tokenizer_kwargs={"truncation": True},
) )
simplified = nested_simplify(outputs, decimals=4)
self.assertEqual( self.assertEqual(
nested_simplify(outputs, decimals=6), [{"sequence": x["sequence"][:100]} for x in simplified],
[ [
{"sequence": "My name is grouped", "score": 2.2e-05, "token": 38015, "token_str": " grouped"}, {"sequence": f"My name is,{dummy_str}"[:100]},
{"sequence": "My name is accuser", "score": 2.1e-05, "token": 25506, "token_str": " accuser"}, {"sequence": f"My name is:,{dummy_str}"[:100]},
],
)
self.assertEqual(
[{k: x[k] for k in x if k != "sequence"} for x in simplified],
[
{"score": 0.2819, "token": 6, "token_str": ","},
{"score": 0.0954, "token": 46686, "token_str": ":,"},
], ],
) )
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment