From d35dedede50213c8ae835a27cc2049048f2d8ab0 Mon Sep 17 00:00:00 2001 From: andres Date: Fri, 24 Nov 2023 11:13:49 +0100 Subject: [PATCH 1/2] bug in MultiAdapter for Inpainting --- ..._stable_diffusion_xl_controlnet_adapter_inpaint.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/examples/community/pipeline_stable_diffusion_xl_controlnet_adapter_inpaint.py b/examples/community/pipeline_stable_diffusion_xl_controlnet_adapter_inpaint.py index bc612edbc20e..2229b1b03aab 100644 --- a/examples/community/pipeline_stable_diffusion_xl_controlnet_adapter_inpaint.py +++ b/examples/community/pipeline_stable_diffusion_xl_controlnet_adapter_inpaint.py @@ -1644,9 +1644,14 @@ def denoising_value_valid(dnv): # 10. Prepare added time ids & embeddings & adapter features adapter_input = adapter_input.type(latents.dtype) - adapter_state = adapter(adapter_input) - for k, v in enumerate(adapter_state): - adapter_state[k] = v * adapter_conditioning_scale + if isinstance(adapter, MultiAdapter): + adapter_state = adapter(adapter_input, adapter_conditioning_scale) + for k, v in enumerate(adapter_state): + adapter_state[k] = v + else: + adapter_state = adapter(adapter_input) + for k, v in enumerate(adapter_state): + adapter_state[k] = v * adapter_conditioning_scale if num_images_per_prompt > 1: for k, v in enumerate(adapter_state): adapter_state[k] = v.repeat(num_images_per_prompt, 1, 1, 1) From ca131b2bb49a2136af4eeba457280643fc6a09b4 Mon Sep 17 00:00:00 2001 From: Andres Romero Date: Fri, 24 Nov 2023 12:56:42 +0100 Subject: [PATCH 2/2] adapter_input is a list for MultiAdapter --- ..._stable_diffusion_xl_controlnet_adapter_inpaint.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/examples/community/pipeline_stable_diffusion_xl_controlnet_adapter_inpaint.py b/examples/community/pipeline_stable_diffusion_xl_controlnet_adapter_inpaint.py index 2229b1b03aab..ca2a1521d3d5 100644 --- a/examples/community/pipeline_stable_diffusion_xl_controlnet_adapter_inpaint.py +++ b/examples/community/pipeline_stable_diffusion_xl_controlnet_adapter_inpaint.py @@ -1470,7 +1470,15 @@ def __call__( height, width = self._default_height_width(height, width, adapter_image) device = self._execution_device - adapter_input = _preprocess_adapter_image(adapter_image, height, width).to(device) + if isinstance(adapter, MultiAdapter): + adapter_input = [] + for one_image in adapter_image: + one_image = _preprocess_adapter_image(one_image, height, width) + one_image = one_image.to(device=device, dtype=adapter.dtype) + adapter_input.append(one_image) + else: + adapter_input = _preprocess_adapter_image(adapter_image, height, width) + adapter_input = adapter_input.to(device=device, dtype=adapter.dtype) original_size = original_size or (height, width) target_size = target_size or (height, width) @@ -1643,7 +1651,6 @@ def denoising_value_valid(dnv): extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) # 10. Prepare added time ids & embeddings & adapter features - adapter_input = adapter_input.type(latents.dtype) if isinstance(adapter, MultiAdapter): adapter_state = adapter(adapter_input, adapter_conditioning_scale) for k, v in enumerate(adapter_state):