Patch recursive import (#11812)

src/transformers/convert_slow_tokenizer.py

@@ -25,7 +25,6 @@ from tokenizers import Regex, Tokenizer, decoders, normalizers, pre_tokenizers,
 from tokenizers.models import BPE, Unigram, WordPiece
 
 from .file_utils import requires_backends
-from .models.roformer.tokenization_utils import JiebaPreTokenizer
 
 
 class SentencePieceExtractor:
@@ -299,6 +298,8 @@ class RobertaConverter(Converter):
 
 class RoFormerConverter(Converter):
     def converted(self) -> Tokenizer:
+        from .models.roformer.tokenization_utils import JiebaPreTokenizer
+
         vocab = self.original_tokenizer.vocab
         tokenizer = Tokenizer(WordPiece(vocab, unk_token=str(self.original_tokenizer.unk_token)))
 

tests/test_tokenization_utils.py

@@ -12,6 +12,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+"""
+isort:skip_file
+"""
 import os
 import pickle
 import tempfile
@@ -20,12 +23,14 @@ from typing import Callable, Optional
 
 import numpy as np
 
+# Ensure there are no circular imports when importing the parent class
+from transformers import PreTrainedTokenizerFast
+
 from transformers import (
     BatchEncoding,
     BertTokenizer,
     BertTokenizerFast,
     PreTrainedTokenizer,
-    PreTrainedTokenizerFast,
     TensorType,
     TokenSpan,
     is_tokenizers_available,