Commit 5c3e08b7 authored by A. Unique TensorFlower's avatar A. Unique TensorFlower
Browse files

Merge pull request #9670 from ChristopherBrix:nlp_data_tagging_quadratic

PiperOrigin-RevId: 364726354
parents 23ef9155 723d96c5
......@@ -224,10 +224,10 @@ def _tokenize_example(example, max_length, tokenizer, text_preprocessing=None):
max_length = max_length - 2
new_examples = []
new_example = InputExample(sentence_id=example.sentence_id, sub_sentence_id=0)
for i, word in enumerate(example.words):
if any([x < 0 for x in example.label_ids]):
raise ValueError("Unexpected negative label_id: %s" % example.label_ids)
if any([x < 0 for x in example.label_ids]):
raise ValueError("Unexpected negative label_id: %s" % example.label_ids)
for i, word in enumerate(example.words):
if text_preprocessing:
word = text_preprocessing(word)
subwords = tokenizer.tokenize(word)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment