fix loading custom text encoder when using `from_single_file` (#4571)

fix loading custom text encoder when using from_single_file

src/diffusers/pipelines/stable_diffusion/convert_from_ckpt.py

@@ -783,6 +783,8 @@ def convert_ldm_clip_checkpoint(checkpoint, local_files_only=False, text_encoder
         ctx = init_empty_weights if is_accelerate_available() else nullcontext
         with ctx():
             text_model = CLIPTextModel(config)
+    else:
+        text_model = text_encoder
 
     keys = list(checkpoint.keys())