Skip to content

GitLab

Commit 99c9752e authored by Tianqi Liu's avatar Tianqi Liu Committed by A. Unique TensorFlower
Browse files

Internal cleanup. 

PiperOrigin-RevId: 318851715
parent 2284f823
Hide whitespace changes
Inline Side-by-side
Showing with 9 additions and 8 deletions
official/nlp/data/classifier_data_lib.py
View file @ 99c9752e
...@@ -302,14 +302,15 @@ class PawsxProcessor(DataProcessor): ...@@ -302,14 +302,15 @@ class PawsxProcessor(DataProcessor):
"""See base class.""" """See base class."""
lines = [] lines = []
for lang in PawsxProcessor.supported_languages: for lang in PawsxProcessor.supported_languages:
lines.extend(self._read_tsv(os.path.join(data_dir, f"dev-{lang}.tsv"))) lines.extend(
self._read_tsv(os.path.join(data_dir, lang, "dev_2k.tsv"))[1:])
examples = [] examples = []
for (i, line) in enumerate(lines): for (i, line) in enumerate(lines):
guid = "dev-%d" % i guid = "dev-%d" % i
text_a = self.process_text_fn(line[0]) text_a = self.process_text_fn(line[1])
text_b = self.process_text_fn(line[1]) text_b = self.process_text_fn(line[2])
label = self.process_text_fn(line[2]) label = self.process_text_fn(line[3])
examples.append( examples.append(
InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label))
return examples return examples
...@@ -318,12 +319,12 @@ class PawsxProcessor(DataProcessor): ...@@ -318,12 +319,12 @@ class PawsxProcessor(DataProcessor):
"""See base class.""" """See base class."""
examples_by_lang = {k: [] for k in self.supported_languages} examples_by_lang = {k: [] for k in self.supported_languages}
for lang in self.supported_languages: for lang in self.supported_languages:
lines = self._read_tsv(os.path.join(data_dir, f"test-{lang}.tsv")) lines = self._read_tsv(os.path.join(data_dir, lang, "test_2k.tsv"))[1:]
for (i, line) in enumerate(lines): for (i, line) in enumerate(lines):
guid = "test-%d" % i guid = "test-%d" % i
text_a = self.process_text_fn(line[0]) text_a = self.process_text_fn(line[1])
text_b = self.process_text_fn(line[1]) text_b = self.process_text_fn(line[2])
label = self.process_text_fn(line[2]) label = self.process_text_fn(line[3])
examples_by_lang[lang].append( examples_by_lang[lang].append(
InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label)) InputExample(guid=guid, text_a=text_a, text_b=text_b, label=label))
return examples_by_lang return examples_by_lang
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment