fix tokenization

transformers/tests/tokenization_ctrl_test.py

@@ -55,7 +55,7 @@ class CTRLTokenizationTest(CommonTestCases.CommonTokenizerTester):
         tokenizer = CTRLTokenizer(self.vocab_file, self.merges_file, **self.special_tokens_map)
         text = "adapt react readapt apt"
         bpe_tokens = 'adapt re@@ a@@ c@@ t re@@ adapt apt'.split()
-        tokens = tokenizer.tokenize(text, add_prefix_space=True)
+        tokens = tokenizer.tokenize(text)
         self.assertListEqual(tokens, bpe_tokens)
 
         input_tokens = tokens + [tokenizer.unk_token]

transformers/tokenization_ctrl.py

@@ -205,7 +205,7 @@ class CTRLTokenizer(PreTrainedTokenizer):
 
     def convert_tokens_to_string(self, tokens):
         """ Converts a sequence of tokens (string) in a single string. """
-        out_string = ''.join(tokens).replace('@@', ' ').strip()
+        out_string = ' '.join(tokens).replace('@@ ', '').strip()
         return out_string
 
     def save_vocabulary(self, save_directory):