Skip to content

GitLab

"...git@developer.sourcefind.cn:chenpangpang/transformers.git" did not exist on "8472a224fbbe05dd3f7f304f39d19cbc66a90c9d"
Unverified Commit 3263b343 authored by Sanchit Gandhi's avatar Sanchit Gandhi Committed by GitHub
Browse files

Revert "Incorrect Whisper long-form decoding timestamps " (#32148) 

Revert "Incorrect Whisper long-form decoding timestamps  (#32003)"

This reverts commit cd48553f.
parent 034b4778
Hide whitespace changes
Inline Side-by-side
Showing with 25 additions and 133 deletions
src/transformers/models/clvp/processing_clvp.py
View file @ 3263b343
......@@ -73,6 +73,7 @@ class ClvpProcessor(ProcessorMixin):
inputs["attention_mask"] = encodings["attention_mask"]
return inputs
# Copied from transformers.models.whisper.processing_whisper.WhisperProcessor.batch_decode with Whisper->Clvp
def batch_decode(self, *args, **kwargs):
"""
This method forwards all its arguments to ClvpTokenizer's [`~PreTrainedTokenizer.batch_decode`]. Please
......
src/transformers/models/whisper/processing_whisper.py
View file @ 3263b343
......@@ -84,13 +84,6 @@ class WhisperProcessor(ProcessorMixin):
This method forwards all its arguments to WhisperTokenizer's [`~PreTrainedTokenizer.batch_decode`]. Please
refer to the docstring of this method for more information.
"""
# If segments are present in args, we are performing long-form generation and need to return long form timestamps.
# The long-form timestamps are already present in segments and should be passed as kwargs to batch_decode.
if isinstance(args[0], dict) and "segments" in args[0]:
kwargs["longform_timestamps"] = args[0].pop("segments")
args = tuple(args[0]["sequences"].unsqueeze(0))
return self.tokenizer.batch_decode(*args, **kwargs)
def decode(self, *args, **kwargs):
......
src/transformers/models/whisper/tokenization_whisper.py
View file @ 3263b343
......@@ -558,7 +558,7 @@ class WhisperTokenizer(PreTrainedTokenizer):
]
return "".join(outputs)
def _compute_offsets(self, token_ids, time_precision=0.02, longform_timestamps=None):
def _compute_offsets(self, token_ids, time_precision=0.02):
"""
Compute offsets for a given tokenized input
......@@ -567,8 +567,6 @@ class WhisperTokenizer(PreTrainedTokenizer):
List of tokenized input ids. Can be obtained using the `__call__` method.
time_precision (`float`, `optional`, defaults to 0.02):
The time ratio to convert from token to time.
longform_timestamps (List[dict], *optional*):
Timestamps obtained using long form generation in Whisper, to be used to replace predicted timestamps in token_ids.
"""
offsets = []
# ensure torch tensor of token ids is placed on cpu
......@@ -589,7 +587,7 @@ class WhisperTokenizer(PreTrainedTokenizer):
consecutive = np.append(consecutive, np.where(timestamp_tokens)[0][-1] + 1)
last_slice = np.where(timestamp_tokens)[0][0]
for i, current_slice in enumerate(consecutive):
for current_slice in consecutive:
sliced_tokens = token_ids[last_slice:current_slice]
if len(sliced_tokens) > 1:
start_timestamp_position = sliced_tokens[0].item() - timestamp_begin
......@@ -598,27 +596,15 @@ class WhisperTokenizer(PreTrainedTokenizer):
sliced_tokens = self._preprocess_token_ids(sliced_tokens)
text = self._decode(sliced_tokens)
text = self._filter_timestamp_ids(text)
if longform_timestamps is not None:
offsets.append(
{
"text": text,
"timestamp": (
longform_timestamps[0][i]["start"].item(),
longform_timestamps[0][i]["end"].item(),
),
}
)
else:
offsets.append(
{
"text": text,
"timestamp": (
start_timestamp_position * time_precision,
end_timestamp_position * time_precision,
),
}
)
offsets.append(
{
"text": text,
"timestamp": (
start_timestamp_position * time_precision,
end_timestamp_position * time_precision,
),
}
)
last_slice = current_slice
return offsets
......@@ -727,11 +713,7 @@ class WhisperTokenizer(PreTrainedTokenizer):
# retrieve offsets
if output_offsets:
longform_timestamps = kwargs.get("longform_timestamps")
offsets = self._compute_offsets(
token_ids, time_precision=time_precision, longform_timestamps=longform_timestamps
)
offsets = self._compute_offsets(token_ids, time_precision=time_precision)
return {"text": text, "offsets": offsets}
return text
......
src/transformers/models/whisper/tokenization_whisper_fast.py
View file @ 3263b343
......@@ -200,7 +200,7 @@ class WhisperTokenizerFast(PreTrainedTokenizerFast):
return "".join(outputs)
# Copied from transformers.models.whisper.tokenization_whisper.WhisperTokenizer._compute_offsets
def _compute_offsets(self, token_ids, time_precision=0.02, longform_timestamps=None):
def _compute_offsets(self, token_ids, time_precision=0.02):
"""
Compute offsets for a given tokenized input
......@@ -209,8 +209,6 @@ class WhisperTokenizerFast(PreTrainedTokenizerFast):
List of tokenized input ids. Can be obtained using the `__call__` method.
time_precision (`float`, `optional`, defaults to 0.02):
The time ratio to convert from token to time.
longform_timestamps (List[dict], *optional*):
Timestamps obtained using long form generation in Whisper, to be used to replace predicted timestamps in token_ids.
"""
offsets = []
# ensure torch tensor of token ids is placed on cpu
......@@ -231,7 +229,7 @@ class WhisperTokenizerFast(PreTrainedTokenizerFast):
consecutive = np.append(consecutive, np.where(timestamp_tokens)[0][-1] + 1)
last_slice = np.where(timestamp_tokens)[0][0]
for i, current_slice in enumerate(consecutive):
for current_slice in consecutive:
sliced_tokens = token_ids[last_slice:current_slice]
if len(sliced_tokens) > 1:
start_timestamp_position = sliced_tokens[0].item() - timestamp_begin
......@@ -240,27 +238,15 @@ class WhisperTokenizerFast(PreTrainedTokenizerFast):
sliced_tokens = self._preprocess_token_ids(sliced_tokens)
text = self._decode(sliced_tokens)
text = self._filter_timestamp_ids(text)
if longform_timestamps is not None:
offsets.append(
{
"text": text,
"timestamp": (
longform_timestamps[0][i]["start"].item(),
longform_timestamps[0][i]["end"].item(),
),
}
)
else:
offsets.append(
{
"text": text,
"timestamp": (
start_timestamp_position * time_precision,
end_timestamp_position * time_precision,
),
}
)
offsets.append(
{
"text": text,
"timestamp": (
start_timestamp_position * time_precision,
end_timestamp_position * time_precision,
),
}
)
last_slice = current_slice
return offsets
......@@ -373,11 +359,7 @@ class WhisperTokenizerFast(PreTrainedTokenizerFast):
# retrieve offsets
if output_offsets:
longform_timestamps = kwargs.get("longform_timestamps")
offsets = self._compute_offsets(
token_ids, time_precision=time_precision, longform_timestamps=longform_timestamps
)
offsets = self._compute_offsets(token_ids, time_precision=time_precision)
return {"text": text, "offsets": offsets}
return text
......
tests/models/whisper/test_modeling_whisper.py
View file @ 3263b343
......@@ -2243,72 +2243,6 @@ class WhisperModelIntegrationTests(unittest.TestCase):
transcript = processor.batch_decode(generated_ids, skip_special_tokens=True, output_offsets=True)
self.assertEqual(transcript, EXPECTED_TRANSCRIPT)
@slow
def test_tiny_longform_timestamps_generation(self):
set_seed(0)
processor = WhisperProcessor.from_pretrained("openai/whisper-tiny")
model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-tiny")
model.to(torch_device)
sample = self._load_datasamples(1)
input_speech = np.concatenate(sample * 10)
input_features = processor(input_speech, return_tensors="pt", truncation=False, sampling_rate=16_000)
input_features = input_features.to(torch_device)
generated_ids = model.generate(**input_features, return_timestamps=True, return_segments=True)
EXPECTED_TRANSCRIPT = [
{
"text": " Mr. Quilter is the apostle of the middle classes and we are glad to welcome his gospel. Mr. Quilter is the apostle of the middle classes and we are glad to welcome his gospel. Mr. Quilter is the apostle of the middle classes and we are glad to welcome his gospel. Mr. Quilter is the apostle of the middle classes and we are glad to welcome his gospel. Mr. Quilter is the apostle of the middle classes and we are glad to welcome his gospel. Mr. Quilter is the apostle of the middle classes and we are glad to welcome his gospel. Mr. Quilter is the apostle of the middle classes and we are glad to welcome his gospel. Mr. Quilter is the apostle of the middle classes and we are glad to welcome his gospel. Mr. Quilter is the apostle of the middle classes and we are glad to welcome his gospel. Mr. Quilter is the apostle of the middle classes and we are glad to welcome his gospel.",
"offsets": [
{
"text": " Mr. Quilter is the apostle of the middle classes and we are glad to welcome his gospel.",
"timestamp": (0.0, 6.0),
},
{
"text": " Mr. Quilter is the apostle of the middle classes and we are glad to welcome his gospel.",
"timestamp": (6.0, 12.0),
},
{
"text": " Mr. Quilter is the apostle of the middle classes and we are glad to welcome his gospel.",
"timestamp": (12.0, 18.0),
},
{
"text": " Mr. Quilter is the apostle of the middle classes and we are glad to welcome his gospel.",
"timestamp": (18.0, 24.0),
},
{
"text": " Mr. Quilter is the apostle of the middle classes and we are glad to welcome his gospel.",
"timestamp": (24.0, 29.0),
},
{
"text": " Mr. Quilter is the apostle of the middle classes and we are glad to welcome his gospel.",
"timestamp": (29.0, 35.0),
},
{
"text": " Mr. Quilter is the apostle of the middle classes and we are glad to welcome his gospel.",
"timestamp": (35.0, 41.0),
},
{
"text": " Mr. Quilter is the apostle of the middle classes and we are glad to welcome his gospel.",
"timestamp": (41.0, 47.0),
},
{
"text": " Mr. Quilter is the apostle of the middle classes and we are glad to welcome his gospel.",
"timestamp": (47.0, 53.0),
},
{
"text": " Mr. Quilter is the apostle of the middle classes and we are glad to welcome his gospel.",
"timestamp": (53.0, 58.20000076293945),
},
],
}
]
transcript = processor.batch_decode(generated_ids, skip_special_tokens=True, output_offsets=True)
self.assertEqual(transcript, EXPECTED_TRANSCRIPT)
@slow
def test_large_timestamp_generation(self):
set_seed(0)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment