[`Llava`] Fix convert_llava_weights_to_hf.py script (#28570)

* Update convert_llava_weights_to_hf.py

Fix call to `tokenizer.add_tokens`

* Add special_tokens to tokenizer.add_tokens in convert_vipllava_weights_to_hf.py

src/transformers/models/llava/convert_llava_weights_to_hf.py

@@ -55,7 +55,7 @@ def convert_llava_llama_to_hf(text_model_id, vision_model_id, output_hub_path, o
     text_config = AutoConfig.from_pretrained(text_model_id)
 
     tokenizer = AutoTokenizer.from_pretrained(text_model_id)
-    tokenizer.add_tokens(AddedToken("<image>", special=True, normalized=False), special=True)
+    tokenizer.add_tokens(AddedToken("<image>", special=True, normalized=False), special_tokens=True)
     tokenizer.add_special_tokens({"pad_token": "<pad>"})
 
     image_processor = CLIPImageProcessor.from_pretrained(vision_model_id)

src/transformers/models/vipllava/convert_vipllava_weights_to_hf.py

@@ -58,7 +58,7 @@ def convert_vipllava_llama_to_hf(text_model_id, vision_model_id, output_hub_path
     text_config = AutoConfig.from_pretrained(text_model_id)
 
     tokenizer = AutoTokenizer.from_pretrained(text_model_id)
-    tokenizer.add_tokens(AddedToken("<image>", special=True, normalized=False))
+    tokenizer.add_tokens(AddedToken("<image>", special=True, normalized=False), special_tokens=True)
     tokenizer.add_special_tokens({"pad_token": "<pad>"})
 
     image_processor = CLIPImageProcessor.from_pretrained(vision_model_id)