From e29375e23c9971d5faa49c3a5b0a7d4bf5e89071 Mon Sep 17 00:00:00 2001 From: Pedro Cuenca Date: Wed, 21 Dec 2022 13:07:16 +0100 Subject: [PATCH 1/6] Make safety_checker optional in more pipelines. --- .../pipelines/alt_diffusion/pipeline_alt_diffusion_img2img.py | 2 +- .../pipelines/paint_by_example/pipeline_paint_by_example.py | 2 ++ .../pipeline_stable_diffusion_image_variation.py | 2 ++ .../stable_diffusion/pipeline_stable_diffusion_img2img.py | 2 +- .../stable_diffusion/pipeline_stable_diffusion_inpaint.py | 4 +++- 5 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/diffusers/pipelines/alt_diffusion/pipeline_alt_diffusion_img2img.py b/src/diffusers/pipelines/alt_diffusion/pipeline_alt_diffusion_img2img.py index 79c29a98c31c..6d712cac9eea 100644 --- a/src/diffusers/pipelines/alt_diffusion/pipeline_alt_diffusion_img2img.py +++ b/src/diffusers/pipelines/alt_diffusion/pipeline_alt_diffusion_img2img.py @@ -92,7 +92,7 @@ class AltDiffusionImg2ImgPipeline(DiffusionPipeline): feature_extractor ([`CLIPFeatureExtractor`]): Model that extracts features from generated images to be used as inputs for the `safety_checker`. """ - _optional_components = ["safety_checker"] + _optional_components = ["safety_checker", "feature_extractor"] def __init__( self, diff --git a/src/diffusers/pipelines/paint_by_example/pipeline_paint_by_example.py b/src/diffusers/pipelines/paint_by_example/pipeline_paint_by_example.py index 332165e1fadc..f9de7d911068 100644 --- a/src/diffusers/pipelines/paint_by_example/pipeline_paint_by_example.py +++ b/src/diffusers/pipelines/paint_by_example/pipeline_paint_by_example.py @@ -161,6 +161,8 @@ class PaintByExamplePipeline(DiffusionPipeline): feature_extractor ([`CLIPFeatureExtractor`]): Model that extracts features from generated images to be used as inputs for the `safety_checker`. """ + # TODO: feature_extractor is required to encode initial images (if they are in PIL format), + # we should give a descriptive message if the pipeline doesn't have one. _optional_components = ["safety_checker"] def __init__( diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_image_variation.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_image_variation.py index 3bc0a8cfa96f..975afddfe003 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_image_variation.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_image_variation.py @@ -65,6 +65,8 @@ class StableDiffusionImageVariationPipeline(DiffusionPipeline): feature_extractor ([`CLIPFeatureExtractor`]): Model that extracts features from generated images to be used as inputs for the `safety_checker`. """ + # TODO: feature_extractor is required to encode images (if they are in PIL format), + # we should give a descriptive message if the pipeline doesn't have one. _optional_components = ["safety_checker"] def __init__( diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py index 2a7def75fa83..857ad6f3d507 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py @@ -90,7 +90,7 @@ class StableDiffusionImg2ImgPipeline(DiffusionPipeline): feature_extractor ([`CLIPFeatureExtractor`]): Model that extracts features from generated images to be used as inputs for the `safety_checker`. """ - _optional_components = ["safety_checker"] + _optional_components = ["safety_checker", "feature_extractor"] # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.__init__ def __init__( diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py index deec66a97640..4e4526453433 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py @@ -166,7 +166,9 @@ class StableDiffusionInpaintPipeline(DiffusionPipeline): feature_extractor ([`CLIPFeatureExtractor`]): Model that extracts features from generated images to be used as inputs for the `safety_checker`. """ - _optional_components = ["safety_checker"] + # TODO: feature_extractor is required to encode initial images (if they are in PIL format), + # we should give a descriptive message if the pipeline doesn't have one. + _optional_components = ["safety_checker", "feature_extractor"] def __init__( self, From 49491a4a8a3fca9e62064c1c4b0dc52896845cd6 Mon Sep 17 00:00:00 2001 From: Pedro Cuenca Date: Wed, 21 Dec 2022 13:11:47 +0100 Subject: [PATCH 2/6] Remove inappropriate comment in inpaint pipeline. --- .../stable_diffusion/pipeline_stable_diffusion_inpaint.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py index 4e4526453433..985b138dc0ab 100644 --- a/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py +++ b/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py @@ -166,8 +166,6 @@ class StableDiffusionInpaintPipeline(DiffusionPipeline): feature_extractor ([`CLIPFeatureExtractor`]): Model that extracts features from generated images to be used as inputs for the `safety_checker`. """ - # TODO: feature_extractor is required to encode initial images (if they are in PIL format), - # we should give a descriptive message if the pipeline doesn't have one. _optional_components = ["safety_checker", "feature_extractor"] def __init__( From f2c4b116ea804cec1d9a9bfd63ac3a75ad854cb1 Mon Sep 17 00:00:00 2001 From: Pedro Cuenca Date: Sat, 24 Dec 2022 19:52:03 +0100 Subject: [PATCH 3/6] InPaint Test: set feature_extractor to None. --- .../stable_diffusion/test_stable_diffusion_inpaint.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint.py b/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint.py index bd4c0369340a..21f2500cf9a4 100644 --- a/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint.py +++ b/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint.py @@ -79,7 +79,6 @@ def get_dummy_components(self): ) text_encoder = CLIPTextModel(text_encoder_config) tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") - feature_extractor = CLIPImageProcessor(crop_size=32, size=32) components = { "unet": unet, @@ -88,7 +87,7 @@ def get_dummy_components(self): "text_encoder": text_encoder, "tokenizer": tokenizer, "safety_checker": None, - "feature_extractor": feature_extractor, + "feature_extractor": None, } return components From 728a17460533837807475d96730acfe7c191a663 Mon Sep 17 00:00:00 2001 From: Pedro Cuenca Date: Sat, 24 Dec 2022 20:01:33 +0100 Subject: [PATCH 4/6] Remove import --- .../pipelines/stable_diffusion/test_stable_diffusion_inpaint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint.py b/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint.py index 21f2500cf9a4..c763eff2b32d 100644 --- a/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint.py +++ b/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint.py @@ -32,7 +32,7 @@ from diffusers.utils import floats_tensor, load_image, load_numpy, nightly, slow, torch_device from diffusers.utils.testing_utils import require_torch_gpu from PIL import Image -from transformers import CLIPImageProcessor, CLIPTextConfig, CLIPTextModel, CLIPTokenizer +from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer from ...test_pipelines_common import PipelineTesterMixin From 34147a102785bfa16641e09977a6342a4cbfa002 Mon Sep 17 00:00:00 2001 From: Pedro Cuenca Date: Sun, 25 Dec 2022 21:32:10 +0100 Subject: [PATCH 5/6] img2img test: set feature_extractor to None. --- .../stable_diffusion/test_stable_diffusion_img2img.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py b/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py index 53df7f8571b4..fe6dc729e63e 100644 --- a/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py +++ b/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py @@ -31,7 +31,7 @@ ) from diffusers.utils import floats_tensor, load_image, load_numpy, nightly, slow, torch_device from diffusers.utils.testing_utils import require_torch_gpu -from transformers import CLIPImageProcessor, CLIPTextConfig, CLIPTextModel, CLIPTokenizer +from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer from ...test_pipelines_common import PipelineTesterMixin @@ -78,7 +78,6 @@ def get_dummy_components(self): ) text_encoder = CLIPTextModel(text_encoder_config) tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") - feature_extractor = CLIPImageProcessor(crop_size=32, size=32) components = { "unet": unet, @@ -87,7 +86,7 @@ def get_dummy_components(self): "text_encoder": text_encoder, "tokenizer": tokenizer, "safety_checker": None, - "feature_extractor": feature_extractor, + "feature_extractor": None, } return components From 5b4c4990f4cd2bfb759a03dbf0cdc2213560daa6 Mon Sep 17 00:00:00 2001 From: Pedro Cuenca Date: Sun, 25 Dec 2022 21:51:05 +0100 Subject: [PATCH 6/6] inpaint sd2 test: set feature_extractor to None. --- .../stable_diffusion_2/test_stable_diffusion_inpaint.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/pipelines/stable_diffusion_2/test_stable_diffusion_inpaint.py b/tests/pipelines/stable_diffusion_2/test_stable_diffusion_inpaint.py index 29e14264c091..88157f22de6b 100644 --- a/tests/pipelines/stable_diffusion_2/test_stable_diffusion_inpaint.py +++ b/tests/pipelines/stable_diffusion_2/test_stable_diffusion_inpaint.py @@ -24,7 +24,7 @@ from diffusers.utils import floats_tensor, load_image, load_numpy, torch_device from diffusers.utils.testing_utils import require_torch_gpu, slow from PIL import Image -from transformers import CLIPImageProcessor, CLIPTextConfig, CLIPTextModel, CLIPTokenizer +from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer from ...test_pipelines_common import PipelineTesterMixin @@ -78,7 +78,6 @@ def get_dummy_components(self): ) text_encoder = CLIPTextModel(text_encoder_config) tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") - feature_extractor = CLIPImageProcessor(crop_size=32, size=32) components = { "unet": unet, @@ -87,7 +86,7 @@ def get_dummy_components(self): "text_encoder": text_encoder, "tokenizer": tokenizer, "safety_checker": None, - "feature_extractor": feature_extractor, + "feature_extractor": None, } return components