Skip to content

GitLab

Commit c5783656 authored by A. Unique TensorFlower's avatar A. Unique TensorFlower
Browse files

Internal change 

PiperOrigin-RevId: 380190962
parent 44550d37
Show whitespace changes
Inline Side-by-side
Showing with 2 additions and 5 deletions
official/nlp/data/classifier_data_lib.py
View file @ c5783656
...@@ -181,10 +181,6 @@ class AxProcessor(DataProcessor): ...@@ -181,10 +181,6 @@ class AxProcessor(DataProcessor):
class ColaProcessor(DataProcessor): class ColaProcessor(DataProcessor):
"""Processor for the CoLA data set (GLUE version).""" """Processor for the CoLA data set (GLUE version)."""
def __init__(self, process_text_fn=tokenization.convert_to_unicode):
super(ColaProcessor, self).__init__(process_text_fn)
self.dataset = tfds.load("glue/cola", try_gcs=True)
def get_train_examples(self, data_dir): def get_train_examples(self, data_dir):
"""See base class.""" """See base class."""
return self._create_examples_tfds("train") return self._create_examples_tfds("train")
...@@ -208,7 +204,8 @@ class ColaProcessor(DataProcessor): ...@@ -208,7 +204,8 @@ class ColaProcessor(DataProcessor):
def _create_examples_tfds(self, set_type): def _create_examples_tfds(self, set_type):
"""Creates examples for the training/dev/test sets.""" """Creates examples for the training/dev/test sets."""
dataset = self.dataset[set_type].as_numpy_iterator() dataset = tfds.load(
"glue/cola", split=set_type, try_gcs=True).as_numpy_iterator()
examples = [] examples = []
for i, example in enumerate(dataset): for i, example in enumerate(dataset):
guid = "%s-%s" % (set_type, i) guid = "%s-%s" % (set_type, i)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment