Make style

examples/community/clip_guided_images_mixing_stable_diffusion.py

@@ -32,8 +32,7 @@ def preprocess(image, w, h):
         image = [image]
 
     if isinstance(image[0], PIL.Image.Image):
-        image = [np.array(i.resize((w, h), resample=PIL_INTERPOLATION['lanczos']))[
-            None, :] for i in image]
+        image = [np.array(i.resize((w, h), resample=PIL_INTERPOLATION["lanczos"]))[None, :] for i in image]
         image = np.concatenate(image, axis=0)
         image = np.array(image).astype(np.float32) / 255.0
         image = image.transpose(0, 3, 1, 2)
@@ -45,7 +44,6 @@ def preprocess(image, w, h):
 
 
 def slerp(t, v0, v1, DOT_THRESHOLD=0.9995):
-
     if not isinstance(v0, np.ndarray):
         inputs_are_torch = True
         input_device = v0.device
@@ -82,7 +80,6 @@ def set_requires_grad(model, value):
 
 
 class CLIPGuidedImagesMixingStableDiffusion(DiffusionPipeline):
-
     def __init__(
         self,
         vae: AutoencoderKL,
@@ -112,15 +109,14 @@ class CLIPGuidedImagesMixingStableDiffusion(DiffusionPipeline):
         self.feature_extractor_size = (
             feature_extractor.size
             if isinstance(feature_extractor.size, int)
-            else feature_extractor.size['shortest_edge']
+            else feature_extractor.size["shortest_edge"]
         )
-        self.normalize = transforms.Normalize(
-            mean=feature_extractor.image_mean, std=feature_extractor.image_std)
+        self.normalize = transforms.Normalize(mean=feature_extractor.image_mean, std=feature_extractor.image_std)
         set_requires_grad(self.text_encoder, False)
         set_requires_grad(self.clip_model, False)
 
-    def enable_attention_slicing(self, slice_size: Optional[Union[str, int]] = 'auto'):
-        if slice_size == 'auto':
+    def enable_attention_slicing(self, slice_size: Optional[Union[str, int]] = "auto"):
+        if slice_size == "auto":
             # half the attention head size is usually a good trade-off between
             # speed and memory
             slice_size = self.unet.config.attention_head_dim // 2
@@ -143,8 +139,7 @@ class CLIPGuidedImagesMixingStableDiffusion(DiffusionPipeline):
 
     def get_timesteps(self, num_inference_steps, strength, device):
         # get the original timestep using init_timestep
-        init_timestep = min(
-            int(num_inference_steps * strength), num_inference_steps)
+        init_timestep = min(int(num_inference_steps * strength), num_inference_steps)
 
         t_start = max(num_inference_steps - init_timestep, 0)
         timesteps = self.scheduler.timesteps[t_start:]
@@ -153,15 +148,13 @@ class CLIPGuidedImagesMixingStableDiffusion(DiffusionPipeline):
 
     def prepare_latents(self, image, timestep, batch_size, dtype, device, generator=None):
         if not isinstance(image, torch.Tensor):
-            raise ValueError(
-                f'`image` has to be of type `torch.Tensor` but is {type(image)}'
-            )
+            raise ValueError(f"`image` has to be of type `torch.Tensor` but is {type(image)}")
 
         image = image.to(device=device, dtype=dtype)
 
         if isinstance(generator, list):
             init_latents = [
-                self.vae.encode(image[i: i + 1]).latent_dist.sample(generator[i]) for i in range(batch_size)
+                self.vae.encode(image[i : i + 1]).latent_dist.sample(generator[i]) for i in range(batch_size)
             ]
             init_latents = torch.cat(init_latents, dim=0)
         else:
@@ -171,8 +164,7 @@ class CLIPGuidedImagesMixingStableDiffusion(DiffusionPipeline):
         init_latents = 0.18215 * init_latents
         init_latents = init_latents.repeat_interleave(batch_size, dim=0)
 
-        noise = randn_tensor(init_latents.shape,
-                             generator=generator, device=device, dtype=dtype)
+        noise = randn_tensor(init_latents.shape, generator=generator, device=device, dtype=dtype)
 
         # get latents
         init_latents = self.scheduler.add_noise(init_latents, noise, timestep)
@@ -183,21 +175,16 @@ class CLIPGuidedImagesMixingStableDiffusion(DiffusionPipeline):
     def get_image_description(self, image):
         transformed_image = self.coca_transform(image).unsqueeze(0)
         with torch.no_grad(), torch.cuda.amp.autocast():
-            generated = self.coca_model.generate(transformed_image.to(
-                device=self.device, dtype=self.coca_model.dtype))
+            generated = self.coca_model.generate(transformed_image.to(device=self.device, dtype=self.coca_model.dtype))
         generated = self.coca_tokenizer.decode(generated[0].cpu().numpy())
-        return generated.split('<end_of_text>')[0].replace('<start_of_text>', '').rstrip(' .,')
+        return generated.split("<end_of_text>")[0].replace("<start_of_text>", "").rstrip(" .,")
 
     def get_clip_image_embeddings(self, image, batch_size):
         clip_image_input = self.feature_extractor.preprocess(image)
-        clip_image_features = torch.from_numpy(
-            clip_image_input['pixel_values'][0]).unsqueeze(0).to(self.device).half()
-        image_embeddings_clip = self.clip_model.get_image_features(
-            clip_image_features)
-        image_embeddings_clip = image_embeddings_clip / \
-            image_embeddings_clip.norm(p=2, dim=-1, keepdim=True)
-        image_embeddings_clip = image_embeddings_clip.repeat_interleave(
-            batch_size, dim=0)
+        clip_image_features = torch.from_numpy(clip_image_input["pixel_values"][0]).unsqueeze(0).to(self.device).half()
+        image_embeddings_clip = self.clip_model.get_image_features(clip_image_features)
+        image_embeddings_clip = image_embeddings_clip / image_embeddings_clip.norm(p=2, dim=-1, keepdim=True)
+        image_embeddings_clip = image_embeddings_clip.repeat_interleave(batch_size, dim=0)
         return image_embeddings_clip
 
     @torch.enable_grad()
@@ -213,20 +200,17 @@ class CLIPGuidedImagesMixingStableDiffusion(DiffusionPipeline):
     ):
         latents = latents.detach().requires_grad_()
 
-        latent_model_input = self.scheduler.scale_model_input(
-            latents, timestep)
+        latent_model_input = self.scheduler.scale_model_input(latents, timestep)
 
         # predict the noise residual
-        noise_pred = self.unet(latent_model_input, timestep,
-                               encoder_hidden_states=text_embeddings).sample
+        noise_pred = self.unet(latent_model_input, timestep, encoder_hidden_states=text_embeddings).sample
 
         if isinstance(self.scheduler, (PNDMScheduler, DDIMScheduler, DPMSolverMultistepScheduler)):
             alpha_prod_t = self.scheduler.alphas_cumprod[timestep]
             beta_prod_t = 1 - alpha_prod_t
             # compute predicted original sample from predicted noise also called
             # "predicted x_0" of formula (12) from https://arxiv.org/pdf/2010.02502.pdf
-            pred_original_sample = (
-                latents - beta_prod_t ** (0.5) * noise_pred) / alpha_prod_t ** (0.5)
+            pred_original_sample = (latents - beta_prod_t ** (0.5) * noise_pred) / alpha_prod_t ** (0.5)
 
             fac = torch.sqrt(beta_prod_t)
             sample = pred_original_sample * (fac) + latents * (1 - fac)
@@ -234,8 +218,7 @@ class CLIPGuidedImagesMixingStableDiffusion(DiffusionPipeline):
             sigma = self.scheduler.sigmas[index]
             sample = latents - sigma * noise_pred
         else:
-            raise ValueError(
-                f'scheduler type {type(self.scheduler)} not supported')
+            raise ValueError(f"scheduler type {type(self.scheduler)} not supported")
 
         # Hardcode 0.18215 because stable-diffusion-2-base has not self.vae.config.scaling_factor
         sample = 1 / 0.18215 * sample
@@ -246,11 +229,9 @@ class CLIPGuidedImagesMixingStableDiffusion(DiffusionPipeline):
         image = self.normalize(image).to(latents.dtype)
 
         image_embeddings_clip = self.clip_model.get_image_features(image)
-        image_embeddings_clip = image_embeddings_clip / \
-            image_embeddings_clip.norm(p=2, dim=-1, keepdim=True)
+        image_embeddings_clip = image_embeddings_clip / image_embeddings_clip.norm(p=2, dim=-1, keepdim=True)
 
-        loss = spherical_dist_loss(
-            image_embeddings_clip, original_image_embeddings_clip).mean() * clip_guidance_scale
+        loss = spherical_dist_loss(image_embeddings_clip, original_image_embeddings_clip).mean() * clip_guidance_scale
 
         grads = -torch.autograd.grad(loss, latents)[0]
 
@@ -277,121 +258,101 @@ class CLIPGuidedImagesMixingStableDiffusion(DiffusionPipeline):
         eta: float = 0.0,
         clip_guidance_scale: Optional[float] = 100,
         generator: Optional[torch.Generator] = None,
-        output_type: Optional[str] = 'pil',
+        output_type: Optional[str] = "pil",
         return_dict: bool = True,
         slerp_latent_style_strength: float = 0.8,
         slerp_prompt_style_strength: float = 0.1,
         slerp_clip_image_style_strength: float = 0.1,
     ):
-
         if isinstance(generator, list) and len(generator) != batch_size:
-            raise ValueError(
-                f'You have passed {batch_size} batch_size, but only {len(generator)} generators.')
+            raise ValueError(f"You have passed {batch_size} batch_size, but only {len(generator)} generators.")
 
         if height % 8 != 0 or width % 8 != 0:
-            raise ValueError(
-                f'`height` and `width` have to be divisible by 8 but are {height} and {width}.')
+            raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.")
 
         if isinstance(generator, torch.Generator) and batch_size > 1:
             generator = [generator] + [None] * (batch_size - 1)
 
         coca_is_none = [
-            ('model', self.coca_model is None),
-            ('tokenizer', self.coca_tokenizer is None),
-            ('transform', self.coca_transform is None)
+            ("model", self.coca_model is None),
+            ("tokenizer", self.coca_tokenizer is None),
+            ("transform", self.coca_transform is None),
         ]
         coca_is_none = [x[0] for x in coca_is_none if x[1]]
-        coca_is_none_str = ', '.join(coca_is_none)
+        coca_is_none_str = ", ".join(coca_is_none)
         # generate prompts with coca model if prompt is None
         if content_prompt is None:
             if len(coca_is_none):
                 raise ValueError(
-                    f'Content prompt is None and CoCa [{coca_is_none_str}] is None.'
-                    f'Set prompt or pass Coca [{coca_is_none_str}] to DiffusionPipeline.'
+                    f"Content prompt is None and CoCa [{coca_is_none_str}] is None."
+                    f"Set prompt or pass Coca [{coca_is_none_str}] to DiffusionPipeline."
                 )
             content_prompt = self.get_image_description(content_image)
         if style_prompt is None:
             if len(coca_is_none):
                 raise ValueError(
-                    f'Style prompt is None and CoCa [{coca_is_none_str}] is None.'
-                    f' Set prompt or pass Coca [{coca_is_none_str}] to DiffusionPipeline.'
+                    f"Style prompt is None and CoCa [{coca_is_none_str}] is None."
+                    f" Set prompt or pass Coca [{coca_is_none_str}] to DiffusionPipeline."
                 )
             style_prompt = self.get_image_description(style_image)
 
         # get prompt text embeddings for content and style
         content_text_input = self.tokenizer(
             content_prompt,
-            padding='max_length',
+            padding="max_length",
             max_length=self.tokenizer.model_max_length,
             truncation=True,
-            return_tensors='pt',
+            return_tensors="pt",
         )
-        content_text_embeddings = self.text_encoder(
-            content_text_input.input_ids.to(self.device))[0]
+        content_text_embeddings = self.text_encoder(content_text_input.input_ids.to(self.device))[0]
 
         style_text_input = self.tokenizer(
             style_prompt,
-            padding='max_length',
+            padding="max_length",
             max_length=self.tokenizer.model_max_length,
             truncation=True,
-            return_tensors='pt',
+            return_tensors="pt",
         )
-        style_text_embeddings = self.text_encoder(
-            style_text_input.input_ids.to(self.device))[0]
+        style_text_embeddings = self.text_encoder(style_text_input.input_ids.to(self.device))[0]
 
-        text_embeddings = slerp(
-            slerp_prompt_style_strength, content_text_embeddings, style_text_embeddings)
+        text_embeddings = slerp(slerp_prompt_style_strength, content_text_embeddings, style_text_embeddings)
 
         # duplicate text embeddings for each generation per prompt
         text_embeddings = text_embeddings.repeat_interleave(batch_size, dim=0)
 
         # set timesteps
-        accepts_offset = 'offset' in set(inspect.signature(
-            self.scheduler.set_timesteps).parameters.keys())
+        accepts_offset = "offset" in set(inspect.signature(self.scheduler.set_timesteps).parameters.keys())
         extra_set_kwargs = {}
         if accepts_offset:
-            extra_set_kwargs['offset'] = 1
+            extra_set_kwargs["offset"] = 1
 
         self.scheduler.set_timesteps(num_inference_steps, **extra_set_kwargs)
         # Some schedulers like PNDM have timesteps as arrays
         # It's more optimized to move all timesteps to correct device beforehand
         self.scheduler.timesteps.to(self.device)
 
-        timesteps, num_inference_steps = self.get_timesteps(
-            num_inference_steps, noise_strength, self.device)
+        timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, noise_strength, self.device)
         latent_timestep = timesteps[:1].repeat(batch_size)
 
         # Preprocess image
         preprocessed_content_image = preprocess(content_image, width, height)
         content_latents = self.prepare_latents(
-            preprocessed_content_image,
-            latent_timestep,
-            batch_size,
-            text_embeddings.dtype,
-            self.device,
-            generator
+            preprocessed_content_image, latent_timestep, batch_size, text_embeddings.dtype, self.device, generator
         )
 
         preprocessed_style_image = preprocess(style_image, width, height)
         style_latents = self.prepare_latents(
-            preprocessed_style_image,
-            latent_timestep,
-            batch_size,
-            text_embeddings.dtype,
-            self.device,
-            generator
+            preprocessed_style_image, latent_timestep, batch_size, text_embeddings.dtype, self.device, generator
         )
 
-        latents = slerp(slerp_latent_style_strength,
-                        content_latents, style_latents)
+        latents = slerp(slerp_latent_style_strength, content_latents, style_latents)
 
         if clip_guidance_scale > 0:
-            content_clip_image_embedding = self.get_clip_image_embeddings(
-                content_image, batch_size)
-            style_clip_image_embedding = self.get_clip_image_embeddings(
-                style_image, batch_size)
+            content_clip_image_embedding = self.get_clip_image_embeddings(content_image, batch_size)
+            style_clip_image_embedding = self.get_clip_image_embeddings(style_image, batch_size)
             clip_image_embeddings = slerp(
-                slerp_clip_image_style_strength, content_clip_image_embedding, style_clip_image_embedding)
+                slerp_clip_image_style_strength, content_clip_image_embedding, style_clip_image_embedding
+            )
 
         # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2)
         # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`
@@ -400,13 +361,10 @@ class CLIPGuidedImagesMixingStableDiffusion(DiffusionPipeline):
         # get unconditional embeddings for classifier free guidance
         if do_classifier_free_guidance:
             max_length = content_text_input.input_ids.shape[-1]
-            uncond_input = self.tokenizer(
-                [''], padding='max_length', max_length=max_length, return_tensors='pt')
-            uncond_embeddings = self.text_encoder(
-                uncond_input.input_ids.to(self.device))[0]
+            uncond_input = self.tokenizer([""], padding="max_length", max_length=max_length, return_tensors="pt")
+            uncond_embeddings = self.text_encoder(uncond_input.input_ids.to(self.device))[0]
             # duplicate unconditional embeddings for each generation per prompt
-            uncond_embeddings = uncond_embeddings.repeat_interleave(
-                batch_size, dim=0)
+            uncond_embeddings = uncond_embeddings.repeat_interleave(batch_size, dim=0)
 
             # For classifier free guidance, we need to do two forward passes.
             # Here we concatenate the unconditional and text embeddings into a single batch
@@ -418,25 +376,19 @@ class CLIPGuidedImagesMixingStableDiffusion(DiffusionPipeline):
         # Unlike in other pipelines, latents need to be generated in the target device
         # for 1-to-1 results reproducibility with the CompVis implementation.
         # However this currently doesn't work in `mps`.
-        latents_shape = (
-            batch_size, self.unet.config.in_channels, height // 8, width // 8)
+        latents_shape = (batch_size, self.unet.config.in_channels, height // 8, width // 8)
         latents_dtype = text_embeddings.dtype
         if latents is None:
-            if self.device.type == 'mps':
+            if self.device.type == "mps":
                 # randn does not work reproducibly on mps
-                latents = torch.randn(
-                    latents_shape,
-                    generator=generator,
-                    device='cpu',
-                    dtype=latents_dtype
-                ).to(self.device)
+                latents = torch.randn(latents_shape, generator=generator, device="cpu", dtype=latents_dtype).to(
+                    self.device
+                )
             else:
-                latents = torch.randn(
-                    latents_shape, generator=generator, device=self.device, dtype=latents_dtype)
+                latents = torch.randn(latents_shape, generator=generator, device=self.device, dtype=latents_dtype)
         else:
             if latents.shape != latents_shape:
-                raise ValueError(
-                    f'Unexpected latents shape, got {latents.shape}, expected {latents_shape}')
+                raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {latents_shape}")
             latents = latents.to(self.device)
 
         # scale the initial noise by the standard deviation required by the scheduler
@@ -446,41 +398,34 @@ class CLIPGuidedImagesMixingStableDiffusion(DiffusionPipeline):
         # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers.
         # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502
         # and should be between [0, 1]
-        accepts_eta = 'eta' in set(inspect.signature(
-            self.scheduler.step).parameters.keys())
+        accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys())
         extra_step_kwargs = {}
         if accepts_eta:
-            extra_step_kwargs['eta'] = eta
+            extra_step_kwargs["eta"] = eta
 
         # check if the scheduler accepts generator
-        accepts_generator = 'generator' in set(
-            inspect.signature(self.scheduler.step).parameters.keys())
+        accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys())
         if accepts_generator:
-            extra_step_kwargs['generator'] = generator
+            extra_step_kwargs["generator"] = generator
 
         with self.progress_bar(total=num_inference_steps):
             for i, t in enumerate(timesteps):
                 # expand the latents if we are doing classifier free guidance
-                latent_model_input = torch.cat(
-                    [latents] * 2) if do_classifier_free_guidance else latents
-                latent_model_input = self.scheduler.scale_model_input(
-                    latent_model_input, t)
+                latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents
+                latent_model_input = self.scheduler.scale_model_input(latent_model_input, t)
 
                 # predict the noise residual
-                noise_pred = self.unet(
-                    latent_model_input, t, encoder_hidden_states=text_embeddings).sample
+                noise_pred = self.unet(latent_model_input, t, encoder_hidden_states=text_embeddings).sample
 
                 # perform classifier free guidance
                 if do_classifier_free_guidance:
                     noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
-                    noise_pred = noise_pred_uncond + guidance_scale * \
-                        (noise_pred_text - noise_pred_uncond)
+                    noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)
 
                 # perform clip guidance
                 if clip_guidance_scale > 0:
                     text_embeddings_for_guidance = (
-                        text_embeddings.chunk(
-                            2)[1] if do_classifier_free_guidance else text_embeddings
+                        text_embeddings.chunk(2)[1] if do_classifier_free_guidance else text_embeddings
                     )
                     noise_pred, latents = self.cond_fn(
                         latents,
@@ -493,8 +438,7 @@ class CLIPGuidedImagesMixingStableDiffusion(DiffusionPipeline):
                     )
 
                 # compute the previous noisy sample x_t -> x_t-1
-                latents = self.scheduler.step(
-                    noise_pred, t, latents, **extra_step_kwargs).prev_sample
+                latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample
 
         # Hardcode 0.18215 because stable-diffusion-2-base has not self.vae.config.scaling_factor
         latents = 1 / 0.18215 * latents
@@ -503,7 +447,7 @@ class CLIPGuidedImagesMixingStableDiffusion(DiffusionPipeline):
         image = (image / 2 + 0.5).clamp(0, 1)
         image = image.cpu().permute(0, 2, 3, 1).numpy()
 
-        if output_type == 'pil':
+        if output_type == "pil":
             image = self.numpy_to_pil(image)
 
         if not return_dict: