[from_pretrained] only load config one time (#2131)

src/diffusers/pipelines/pipeline_utils.py

@@ -531,8 +531,7 @@ class DiffusionPipeline(ConfigMixin):
             )
         else:
             cached_folder = pretrained_model_name_or_path
-
-        config_dict = cls.load_config(cached_folder)
+            config_dict = cls.load_config(cached_folder)
 
         # 2. Load the pipeline class, if using custom module then load it from the hub
         # if we load from explicit class, let's use it