From 314678ed11a57be17ca85c606e3736423b3fb059 Mon Sep 17 00:00:00 2001
From: Parag Ekbote <thecoolekbote189@gmail.com>
Date: Wed, 19 Mar 2025 18:01:30 +0000
Subject: [PATCH 1/3] Add 4 Notebooks for diffusers community pipelines.

---
 diffusers/composable_stable_diffusion.ipynb   | 155 ++++++++++++
 ...to_image_inpainting_stable_diffusion.ipynb | 116 +++++++++
 diffusers/prompt_2_prompt_pipeline.ipynb      | 161 ++++++++++++
 ...text2image_stable_diffusion_pipeline.ipynb | 230 ++++++++++++++++++
 4 files changed, 662 insertions(+)
 create mode 100644 diffusers/composable_stable_diffusion.ipynb
 create mode 100644 diffusers/image_to_image_inpainting_stable_diffusion.ipynb
 create mode 100644 diffusers/prompt_2_prompt_pipeline.ipynb
 create mode 100644 diffusers/tensorrt_text2image_stable_diffusion_pipeline.ipynb

diff --git a/diffusers/composable_stable_diffusion.ipynb b/diffusers/composable_stable_diffusion.ipynb
new file mode 100644
index 00000000..b33abef0
--- /dev/null
+++ b/diffusers/composable_stable_diffusion.ipynb
@@ -0,0 +1,155 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Composable Stable diffusion\n",
+    "\n",
+    "[Composable Stable Diffusion](https://energy-based-model.github.io/Compositional-Visual-Generation-with-Composable-Diffusion-Models/) proposes conjunction and negation (negative prompts) operators for compositional generation with conditional diffusion models. This script was contributed by [MarkRich](https://github.com/MarkRich) and the notebook by [Parag Ekbote](https://github.com/ParagEkbote)."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pip install torch numpy torchvision diffusers"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "3dadcf1262e0492cafe9556f62ba3a9f",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "composable_stable_diffusion.py:   0%|          | 0.00/27.6k [00:00<?, ?B/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "279a467d562041ec935edacbf177caba",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Loading pipeline components...:   0%|          | 0/7 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "composing ['mystical trees', 'A magical pond', 'dark']...\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "3717298308004b648b65d6c1b1e02dbe",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Image saved successfully!\n"
+     ]
+    }
+   ],
+   "source": [
+    "import torch as th\n",
+    "import numpy as np\n",
+    "import torchvision.utils as tvu\n",
+    "from diffusers import DiffusionPipeline\n",
+    "import argparse\n",
+    "import sys\n",
+    "\n",
+    "# Simulate passing arguments explicitly (bypassing Jupyter's arguments)\n",
+    "sys.argv = [\n",
+    "    \"ipykernel_launcher.py\", \n",
+    "    \"--prompt\", \"mystical trees | A magical pond | dark\",\n",
+    "    \"--steps\", \"50\",\n",
+    "    \"--scale\", \"7.5\",\n",
+    "    \"--weights\", \"7.5 | 7.5 | -7.5\",\n",
+    "    \"--seed\", \"2\",\n",
+    "    \"--model_path\", \"CompVis/stable-diffusion-v1-4\",\n",
+    "    \"--num_images\", \"1\"\n",
+    "]\n",
+    "\n",
+    "parser = argparse.ArgumentParser()\n",
+    "parser.add_argument(\"--prompt\", type=str, default=\"mystical trees | A magical pond | dark\",\n",
+    "                    help=\"use '|' as the delimiter to compose separate sentences.\")\n",
+    "parser.add_argument(\"--steps\", type=int, default=50)\n",
+    "parser.add_argument(\"--scale\", type=float, default=7.5)\n",
+    "parser.add_argument(\"--weights\", type=str, default=\"7.5 | 7.5 | -7.5\")\n",
+    "parser.add_argument(\"--seed\", type=int, default=2)\n",
+    "parser.add_argument(\"--model_path\", type=str, default=\"CompVis/stable-diffusion-v1-4\")\n",
+    "parser.add_argument(\"--num_images\", type=int, default=1)\n",
+    "args = parser.parse_args()\n",
+    "\n",
+    "# CUDA Setup\n",
+    "has_cuda = th.cuda.is_available()\n",
+    "device = th.device('cpu' if not has_cuda else 'cuda')\n",
+    "\n",
+    "# Assign parameters\n",
+    "prompt = args.prompt\n",
+    "scale = args.scale\n",
+    "steps = args.steps\n",
+    "\n",
+    "# Load pipeline\n",
+    "pipe = DiffusionPipeline.from_pretrained(\n",
+    "    args.model_path,\n",
+    "    custom_pipeline=\"composable_stable_diffusion\",\n",
+    ").to(device)\n",
+    "\n",
+    "# Disable safety checker (if intentional for internal use)\n",
+    "pipe.safety_checker = None\n",
+    "\n",
+    "# Generate images\n",
+    "images = []\n",
+    "generator = th.Generator(\"cuda\").manual_seed(args.seed)\n",
+    "for i in range(args.num_images):\n",
+    "    image = pipe(prompt, guidance_scale=scale, num_inference_steps=steps,\n",
+    "                 weights=args.weights, generator=generator).images[0]\n",
+    "    images.append(th.from_numpy(np.array(image)).permute(2, 0, 1) / 255.)\n",
+    "\n",
+    "# Create and save image grid\n",
+    "grid = tvu.make_grid(th.stack(images, dim=0), nrow=4, padding=0)\n",
+    "tvu.save_image(grid, f'{prompt}_{args.weights}.png')\n",
+    "\n",
+    "print(\"Image saved successfully!\")\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "language_info": {
+   "name": "python"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/diffusers/image_to_image_inpainting_stable_diffusion.ipynb b/diffusers/image_to_image_inpainting_stable_diffusion.ipynb
new file mode 100644
index 00000000..dbfa8315
--- /dev/null
+++ b/diffusers/image_to_image_inpainting_stable_diffusion.ipynb
@@ -0,0 +1,116 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Image to Image Inpainting Stable Diffusion\n",
+    "\n",
+    "Similar to the standard stable diffusion inpainting example, except with the addition of an `inner_image` argument.\n",
+    "\n",
+    "`image`, `inner_image`, and `mask` should have the same dimensions. `inner_image` should have an alpha (transparency) channel.\n",
+    "\n",
+    "The aim is to overlay two images, then mask out the boundary between `image` and `inner_image` to allow stable diffusion to make the connection more seamless. For example, this could be used to place a logo on a shirt and make it blend seamlessly.This script was contributed by [Alex McKinney](https://github.com/vvvm23) and the notebook by [Parag Ekbote](https://github.com/ParagEkbote)."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pip install diffusers torch"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "3988967ef7f8488abaf43c4a712e27da",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Loading pipeline components...:   0%|          | 0/7 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "An error occurred while trying to fetch /home/zeus/.cache/huggingface/hub/models--runwayml--stable-diffusion-inpainting/snapshots/8a4288a76071f7280aedbdb3253bdb9e9d5d84bb/vae: Error no file named diffusion_pytorch_model.safetensors found in directory /home/zeus/.cache/huggingface/hub/models--runwayml--stable-diffusion-inpainting/snapshots/8a4288a76071f7280aedbdb3253bdb9e9d5d84bb/vae.\n",
+      "Defaulting to unsafe serialization. Pass `allow_pickle=False` to raise an error instead.\n",
+      "An error occurred while trying to fetch /home/zeus/.cache/huggingface/hub/models--runwayml--stable-diffusion-inpainting/snapshots/8a4288a76071f7280aedbdb3253bdb9e9d5d84bb/unet: Error no file named diffusion_pytorch_model.safetensors found in directory /home/zeus/.cache/huggingface/hub/models--runwayml--stable-diffusion-inpainting/snapshots/8a4288a76071f7280aedbdb3253bdb9e9d5d84bb/unet.\n",
+      "Defaulting to unsafe serialization. Pass `allow_pickle=False` to raise an error instead.\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "b393bb433c3f46ba9a9ed392f534742d",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "import torch\n",
+    "import requests\n",
+    "from PIL import Image\n",
+    "from io import BytesIO\n",
+    "from diffusers import DiffusionPipeline\n",
+    "\n",
+    "# Correct image URLs\n",
+    "image_url = \"https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo.png\"\n",
+    "inner_image_url = \"https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo_mask.png\"\n",
+    "mask_url = \"https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo_mask.png\"\n",
+    "\n",
+    "# Function to load image from URL\n",
+    "def load_image(url, mode=\"RGB\"):\n",
+    "    response = requests.get(url)\n",
+    "    if response.status_code == 200:\n",
+    "        return Image.open(BytesIO(response.content)).convert(mode).resize((512, 512))\n",
+    "    else:\n",
+    "        raise FileNotFoundError(f\"Could not retrieve image from {url}\")\n",
+    "\n",
+    "# Load images\n",
+    "init_image = load_image(image_url, mode=\"RGB\")\n",
+    "inner_image = load_image(inner_image_url, mode=\"RGBA\")\n",
+    "mask_image = load_image(mask_url, mode=\"RGB\")\n",
+    "\n",
+    "# Load the pipeline\n",
+    "pipe = DiffusionPipeline.from_pretrained(\n",
+    "    \"runwayml/stable-diffusion-inpainting\",\n",
+    "    torch_dtype=torch.float16\n",
+    ")\n",
+    "pipe = pipe.to(\"cuda\")\n",
+    "\n",
+    "# Inpainting\n",
+    "prompt = \"a mecha robot sitting on a bench\"\n",
+    "image = pipe(prompt=prompt, image=init_image, mask_image=mask_image).images[0]\n",
+    "\n",
+    "image.save(\"output.png\")\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "language_info": {
+   "name": "python"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/diffusers/prompt_2_prompt_pipeline.ipynb b/diffusers/prompt_2_prompt_pipeline.ipynb
new file mode 100644
index 00000000..1756f53d
--- /dev/null
+++ b/diffusers/prompt_2_prompt_pipeline.ipynb
@@ -0,0 +1,161 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Prompt2Prompt Pipeline\n",
+    "\n",
+    "Prompt2Prompt allows the following edits:\n",
+    "\n",
+    "1. ReplaceEdit (change words in prompt)\n",
+    "2. ReplaceEdit with local blend (change words in prompt, keep image part unrelated to changes constant)\n",
+    "3. RefineEdit (add words to prompt)\n",
+    "4. RefineEdit with local blend (add words to prompt, keep image part unrelated to changes constant)\n",
+    "5. ReweightEdit (modulate importance of words)\n",
+    "\n",
+    "Abbreviated examples for the other edits:\n",
+    "\n",
+    "ReplaceEdit with local blend\n",
+    "\n",
+    "prompts = [\"A turtle playing with a ball\",\n",
+    "           \"A monkey playing with a ball\"]\n",
+    "\n",
+    "cross_attention_kwargs = {\n",
+    "    \"edit_type\": \"replace\",\n",
+    "    \"cross_replace_steps\": 0.4,\n",
+    "    \"self_replace_steps\": 0.4,\n",
+    "    \"local_blend_words\": [\"turtle\", \"monkey\"]\n",
+    "}\n",
+    "RefineEdit\n",
+    "\n",
+    "prompts = [\"A turtle\",\n",
+    "           \"A turtle in a forest\"]\n",
+    "\n",
+    "cross_attention_kwargs = {\n",
+    "    \"edit_type\": \"refine\",\n",
+    "    \"cross_replace_steps\": 0.4,\n",
+    "    \"self_replace_steps\": 0.4,\n",
+    "}\n",
+    "RefineEdit with local blend\n",
+    "\n",
+    "prompts = [\"A turtle\",\n",
+    "           \"A turtle in a forest\"]\n",
+    "\n",
+    "cross_attention_kwargs = {\n",
+    "    \"edit_type\": \"refine\",\n",
+    "    \"cross_replace_steps\": 0.4,\n",
+    "    \"self_replace_steps\": 0.4,\n",
+    "    \"local_blend_words\": [\"in\", \"a\" , \"forest\"]\n",
+    "}\n",
+    "ReweightEdit\n",
+    "\n",
+    "prompts = [\"A smiling turtle\"] * 2\n",
+    "\n",
+    "edit_kcross_attention_kwargswargs = {\n",
+    "    \"edit_type\": \"reweight\",\n",
+    "    \"cross_replace_steps\": 0.4,\n",
+    "    \"self_replace_steps\": 0.4,\n",
+    "    \"equalizer_words\": [\"smiling\"],\n",
+    "    \"equalizer_strengths\": [5]\n",
+    "}\n",
+    "\n",
+    "Side note: See this [GitHub gist](https://gist.github.com/UmerHA/b65bb5fb9626c9c73f3ade2869e36164) if you want to visualize the attention maps. This script was contributed by [Umer Adil](https://github.com/UmerHA) and the notebook by [Parag Ekbote](https://github.com/ParagEkbote)."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "cannot get type annotation for Parameter vae of <class 'diffusers_modules.git.pipeline_prompt2prompt.Prompt2PromptPipeline'>.\n",
+      "cannot get type annotation for Parameter text_encoder of <class 'diffusers_modules.git.pipeline_prompt2prompt.Prompt2PromptPipeline'>.\n",
+      "cannot get type annotation for Parameter tokenizer of <class 'diffusers_modules.git.pipeline_prompt2prompt.Prompt2PromptPipeline'>.\n",
+      "cannot get type annotation for Parameter unet of <class 'diffusers_modules.git.pipeline_prompt2prompt.Prompt2PromptPipeline'>.\n",
+      "cannot get type annotation for Parameter scheduler of <class 'diffusers_modules.git.pipeline_prompt2prompt.Prompt2PromptPipeline'>.\n",
+      "cannot get type annotation for Parameter safety_checker of <class 'diffusers_modules.git.pipeline_prompt2prompt.Prompt2PromptPipeline'>.\n",
+      "cannot get type annotation for Parameter feature_extractor of <class 'diffusers_modules.git.pipeline_prompt2prompt.Prompt2PromptPipeline'>.\n",
+      "cannot get type annotation for Parameter image_encoder of <class 'diffusers_modules.git.pipeline_prompt2prompt.Prompt2PromptPipeline'>.\n",
+      "cannot get type annotation for Parameter requires_safety_checker of <class 'diffusers_modules.git.pipeline_prompt2prompt.Prompt2PromptPipeline'>.\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "3e47bb32a31d4d6fb2e0fc2a197ac074",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Loading pipeline components...:   0%|          | 0/7 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "431ad3ea3ee04839bba22a7c67a883c4",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "  0%|          | 0/50 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "import torch\n",
+    "from diffusers import DiffusionPipeline\n",
+    "import numpy as np\n",
+    "from PIL import Image\n",
+    "\n",
+    "# Load the pipeline with custom prompt-to-prompt\n",
+    "pipe = DiffusionPipeline.from_pretrained(\n",
+    "    \"CompVis/stable-diffusion-v1-4\", \n",
+    "    custom_pipeline=\"pipeline_prompt2prompt\"\n",
+    ").to(\"cuda\")\n",
+    "\n",
+    "# Prompts for image generation\n",
+    "prompts = [\n",
+    "    \"A turtle playing with a ball\",\n",
+    "    \"A monkey playing with a ball\"\n",
+    "]\n",
+    "\n",
+    "# Custom attention settings\n",
+    "cross_attention_kwargs = {\n",
+    "    \"edit_type\": \"replace\",\n",
+    "    \"cross_replace_steps\": 0.4,\n",
+    "    \"self_replace_steps\": 0.4\n",
+    "}\n",
+    "\n",
+    "# Generate images\n",
+    "outputs = pipe(\n",
+    "    prompt=prompts,\n",
+    "    height=512,\n",
+    "    width=512,\n",
+    "    num_inference_steps=50,\n",
+    "    cross_attention_kwargs=cross_attention_kwargs\n",
+    ")\n",
+    "\n",
+    "# Save each image explicitly without a loop\n",
+    "outputs.images[0].save(\"output_image_0.png\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "language_info": {
+   "name": "python"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/diffusers/tensorrt_text2image_stable_diffusion_pipeline.ipynb b/diffusers/tensorrt_text2image_stable_diffusion_pipeline.ipynb
new file mode 100644
index 00000000..3ed7fc0e
--- /dev/null
+++ b/diffusers/tensorrt_text2image_stable_diffusion_pipeline.ipynb
@@ -0,0 +1,230 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "TensorRT Text2Image Stable Diffusion Pipeline\n",
+    "\n",
+    "The TensorRT Pipeline can be used to accelerate the Text2Image Stable Diffusion Inference run.\n",
+    "\n",
+    "NOTE: The ONNX conversions and TensorRT engine build may take up to 30 minutes. This script was contributed by [Asfiya Baig](https://github.com/asfiyab-nvidia) and the notebook by [Parag Ekbote](https://github.com/ParagEkbote)."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Requirement already satisfied: polygraphy in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (0.49.20)\n",
+      "Requirement already satisfied: onnx in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (1.17.0)\n",
+      "Requirement already satisfied: cuda-python in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (12.8.0)\n",
+      "Requirement already satisfied: onnx-graphsurgeon in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (0.5.6)\n",
+      "Requirement already satisfied: tensorrt in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (10.9.0.34)\n",
+      "Requirement already satisfied: onnxruntime-gpu in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (1.21.0)\n",
+      "Collecting colored\n",
+      "  Downloading colored-2.3.0-py3-none-any.whl.metadata (3.6 kB)\n",
+      "Requirement already satisfied: numpy>=1.20 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from onnx) (1.26.4)\n",
+      "Requirement already satisfied: protobuf>=3.20.2 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from onnx) (6.30.1)\n",
+      "Requirement already satisfied: cuda-bindings~=12.8.0 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from cuda-python) (12.8.0)\n",
+      "Requirement already satisfied: tensorrt_cu12==10.9.0.34 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from tensorrt) (10.9.0.34)\n",
+      "Requirement already satisfied: tensorrt_cu12_libs==10.9.0.34 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from tensorrt_cu12==10.9.0.34->tensorrt) (10.9.0.34)\n",
+      "Requirement already satisfied: tensorrt_cu12_bindings==10.9.0.34 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from tensorrt_cu12==10.9.0.34->tensorrt) (10.9.0.34)\n",
+      "Requirement already satisfied: nvidia-cuda-runtime-cu12 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from tensorrt_cu12_libs==10.9.0.34->tensorrt_cu12==10.9.0.34->tensorrt) (12.4.127)\n",
+      "Requirement already satisfied: coloredlogs in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from onnxruntime-gpu) (15.0.1)\n",
+      "Requirement already satisfied: flatbuffers in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from onnxruntime-gpu) (25.2.10)\n",
+      "Requirement already satisfied: packaging in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from onnxruntime-gpu) (24.2)\n",
+      "Requirement already satisfied: sympy in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from onnxruntime-gpu) (1.13.1)\n",
+      "Requirement already satisfied: humanfriendly>=9.1 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from coloredlogs->onnxruntime-gpu) (10.0)\n",
+      "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /system/conda/miniconda3/envs/cloudspace/lib/python3.10/site-packages (from sympy->onnxruntime-gpu) (1.3.0)\n",
+      "Downloading colored-2.3.0-py3-none-any.whl (18 kB)\n",
+      "Installing collected packages: colored\n",
+      "Successfully installed colored-2.3.0\n",
+      "Note: you may need to restart the kernel to use updated packages.\n"
+     ]
+    }
+   ],
+   "source": [
+    "pip install polygraphy onnx cuda-python onnx-graphsurgeon tensorrt onnxruntime-gpu colored"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "e388ba8b82364f27be70166a7d74439c",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Loading pipeline components...:   0%|          | 0/6 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "6295b4e5c12346cca961ac9f4b67363b",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Fetching 28 files:   0%|          | 0/28 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Running inference on device: cuda:0\n",
+      "Building Engines...\n",
+      "Engine build can take a while to complete\n",
+      "Building Engines...\n",
+      "Engine build can take a while to complete\n",
+      "Building TensorRT engine for /home/zeus/.cache/huggingface/hub/models--stabilityai--stable-diffusion-2-1/snapshots/5cae40e6a2745ae2b01ad92ae5043f95f23644d6/onnx/unet.opt.onnx: /home/zeus/.cache/huggingface/hub/models--stabilityai--stable-diffusion-2-1/snapshots/5cae40e6a2745ae2b01ad92ae5043f95f23644d6/engine/unet.plan\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[I] TF32 is disabled by default. Turn on TF32 for better performance with minor accuracy differences.\n",
+      "[I] Configuring with profiles:[\n",
+      "        Profile 0:\n",
+      "            {sample [min=(2, 4, 96, 96), opt=(2, 4, 96, 96), max=(8, 4, 96, 96)],\n",
+      "             encoder_hidden_states [min=(2, 77, 1024), opt=(2, 77, 1024), max=(8, 77, 1024)],\n",
+      "             timestep [min=[1], opt=[1], max=[1]]}\n",
+      "    ]\n",
+      "[I] Loading tactic timing cache from /home/zeus/.cache/huggingface/hub/models--stabilityai--stable-diffusion-2-1/snapshots/5cae40e6a2745ae2b01ad92ae5043f95f23644d6/timing_cache\n",
+      "\u001b[38;5;11m[W] profileSharing0806 is on by default in TensorRT 10.0. This flag is deprecated and has no effect.\u001b[0m\n",
+      "\u001b[38;5;14m[I] Building engine with configuration:\n",
+      "    Flags                  | [FP16]\n",
+      "    Engine Capability      | EngineCapability.STANDARD\n",
+      "    Memory Pools           | [WORKSPACE: 22699.88 MiB, TACTIC_DRAM: 22699.88 MiB, TACTIC_SHARED_MEMORY: 1024.00 MiB]\n",
+      "    Tactic Sources         | []\n",
+      "    Profiling Verbosity    | ProfilingVerbosity.DETAILED\n",
+      "    Preview Features       | [PROFILE_SHARING_0806]\u001b[0m\n",
+      "\u001b[38;5;11m[W] UNSUPPORTED_STATE: Skipping tactic 0 due to insufficient memory on requested size of 27843792896 detected for tactic 0x0000000000000000.\u001b[0m\n",
+      "\u001b[38;5;10m[I] Finished engine building in 254.971 seconds\u001b[0m\n",
+      "[I] Saving tactic timing cache to /home/zeus/.cache/huggingface/hub/models--stabilityai--stable-diffusion-2-1/snapshots/5cae40e6a2745ae2b01ad92ae5043f95f23644d6/timing_cache\n",
+      "[I] Saving engine to /home/zeus/.cache/huggingface/hub/models--stabilityai--stable-diffusion-2-1/snapshots/5cae40e6a2745ae2b01ad92ae5043f95f23644d6/engine/unet.plan\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Building TensorRT engine for /home/zeus/.cache/huggingface/hub/models--stabilityai--stable-diffusion-2-1/snapshots/5cae40e6a2745ae2b01ad92ae5043f95f23644d6/onnx/vae.opt.onnx: /home/zeus/.cache/huggingface/hub/models--stabilityai--stable-diffusion-2-1/snapshots/5cae40e6a2745ae2b01ad92ae5043f95f23644d6/engine/vae.plan\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[I] TF32 is disabled by default. Turn on TF32 for better performance with minor accuracy differences.\n",
+      "[I] Configuring with profiles:[\n",
+      "        Profile 0:\n",
+      "            {latent [min=(1, 4, 96, 96), opt=(1, 4, 96, 96), max=(4, 4, 96, 96)]}\n",
+      "    ]\n",
+      "[I] Loading tactic timing cache from /home/zeus/.cache/huggingface/hub/models--stabilityai--stable-diffusion-2-1/snapshots/5cae40e6a2745ae2b01ad92ae5043f95f23644d6/timing_cache\n",
+      "\u001b[38;5;14m[I] Building engine with configuration:\n",
+      "    Flags                  | [FP16]\n",
+      "    Engine Capability      | EngineCapability.STANDARD\n",
+      "    Memory Pools           | [WORKSPACE: 22699.88 MiB, TACTIC_DRAM: 22699.88 MiB, TACTIC_SHARED_MEMORY: 1024.00 MiB]\n",
+      "    Tactic Sources         | []\n",
+      "    Profiling Verbosity    | ProfilingVerbosity.DETAILED\n",
+      "    Preview Features       | [PROFILE_SHARING_0806]\u001b[0m\n",
+      "\u001b[38;5;10m[I] Finished engine building in 172.855 seconds\u001b[0m\n",
+      "[I] Saving tactic timing cache to /home/zeus/.cache/huggingface/hub/models--stabilityai--stable-diffusion-2-1/snapshots/5cae40e6a2745ae2b01ad92ae5043f95f23644d6/timing_cache\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Loading TensorRT engine: /home/zeus/.cache/huggingface/hub/models--stabilityai--stable-diffusion-2-1/snapshots/5cae40e6a2745ae2b01ad92ae5043f95f23644d6/engine/clip.plan\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[I] Saving engine to /home/zeus/.cache/huggingface/hub/models--stabilityai--stable-diffusion-2-1/snapshots/5cae40e6a2745ae2b01ad92ae5043f95f23644d6/engine/vae.plan\n",
+      "[I] Loading bytes from /home/zeus/.cache/huggingface/hub/models--stabilityai--stable-diffusion-2-1/snapshots/5cae40e6a2745ae2b01ad92ae5043f95f23644d6/engine/clip.plan\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Loading TensorRT engine: /home/zeus/.cache/huggingface/hub/models--stabilityai--stable-diffusion-2-1/snapshots/5cae40e6a2745ae2b01ad92ae5043f95f23644d6/engine/unet.plan\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[I] Loading bytes from /home/zeus/.cache/huggingface/hub/models--stabilityai--stable-diffusion-2-1/snapshots/5cae40e6a2745ae2b01ad92ae5043f95f23644d6/engine/unet.plan\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Loading TensorRT engine: /home/zeus/.cache/huggingface/hub/models--stabilityai--stable-diffusion-2-1/snapshots/5cae40e6a2745ae2b01ad92ae5043f95f23644d6/engine/vae.plan\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[I] Loading bytes from /home/zeus/.cache/huggingface/hub/models--stabilityai--stable-diffusion-2-1/snapshots/5cae40e6a2745ae2b01ad92ae5043f95f23644d6/engine/vae.plan\n"
+     ]
+    }
+   ],
+   "source": [
+    "import torch\n",
+    "from diffusers import DDIMScheduler\n",
+    "from diffusers.pipelines import DiffusionPipeline\n",
+    "\n",
+    "# Use the DDIMScheduler scheduler here instead\n",
+    "scheduler = DDIMScheduler.from_pretrained(\"stabilityai/stable-diffusion-2-1\", subfolder=\"scheduler\")\n",
+    "\n",
+    "pipe = DiffusionPipeline.from_pretrained(\"stabilityai/stable-diffusion-2-1\",\n",
+    "    custom_pipeline=\"stable_diffusion_tensorrt_txt2img\",\n",
+    "    variant='fp16',\n",
+    "    torch_dtype=torch.float16,\n",
+    "    scheduler=scheduler,)\n",
+    "\n",
+    "# re-use cached folder to save ONNX models and TensorRT Engines\n",
+    "pipe.set_cached_folder(\"stabilityai/stable-diffusion-2-1\", variant='fp16',)\n",
+    "\n",
+    "pipe = pipe.to(\"cuda\")\n",
+    "\n",
+    "prompt = \"a beautiful photograph of Mt. Fuji during cherry blossom\"\n",
+    "image = pipe(prompt).images[0]\n",
+    "image.save('tensorrt_mt_fuji.png')"
+   ]
+  }
+ ],
+ "metadata": {
+  "language_info": {
+   "name": "python"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

From 60e4b1e40e6516e19903ddc5715ba3d6e4d47af6 Mon Sep 17 00:00:00 2001
From: Parag Ekbote <thecoolekbote189@gmail.com>
Date: Thu, 20 Mar 2025 04:00:24 +0000
Subject: [PATCH 2/3] update as per code review.

---
 ...to_image_inpainting_stable_diffusion.ipynb | 23 ++++++++++++-------
 diffusers/prompt_2_prompt_pipeline.ipynb      | 10 +++++++-
 2 files changed, 24 insertions(+), 9 deletions(-)

diff --git a/diffusers/image_to_image_inpainting_stable_diffusion.ipynb b/diffusers/image_to_image_inpainting_stable_diffusion.ipynb
index dbfa8315..f2326966 100644
--- a/diffusers/image_to_image_inpainting_stable_diffusion.ipynb
+++ b/diffusers/image_to_image_inpainting_stable_diffusion.ipynb
@@ -24,13 +24,13 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "3988967ef7f8488abaf43c4a712e27da",
+       "model_id": "0b56a945eb5145598c4fd153bc658786",
        "version_major": 2,
        "version_minor": 0
       },
@@ -45,16 +45,22 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "An error occurred while trying to fetch /home/zeus/.cache/huggingface/hub/models--runwayml--stable-diffusion-inpainting/snapshots/8a4288a76071f7280aedbdb3253bdb9e9d5d84bb/vae: Error no file named diffusion_pytorch_model.safetensors found in directory /home/zeus/.cache/huggingface/hub/models--runwayml--stable-diffusion-inpainting/snapshots/8a4288a76071f7280aedbdb3253bdb9e9d5d84bb/vae.\n",
-      "Defaulting to unsafe serialization. Pass `allow_pickle=False` to raise an error instead.\n",
-      "An error occurred while trying to fetch /home/zeus/.cache/huggingface/hub/models--runwayml--stable-diffusion-inpainting/snapshots/8a4288a76071f7280aedbdb3253bdb9e9d5d84bb/unet: Error no file named diffusion_pytorch_model.safetensors found in directory /home/zeus/.cache/huggingface/hub/models--runwayml--stable-diffusion-inpainting/snapshots/8a4288a76071f7280aedbdb3253bdb9e9d5d84bb/unet.\n",
+      "An error occurred while trying to fetch /home/zeus/.cache/huggingface/hub/models--stable-diffusion-v1-5--stable-diffusion-inpainting/snapshots/8a4288a76071f7280aedbdb3253bdb9e9d5d84bb/unet: Error no file named diffusion_pytorch_model.safetensors found in directory /home/zeus/.cache/huggingface/hub/models--stable-diffusion-v1-5--stable-diffusion-inpainting/snapshots/8a4288a76071f7280aedbdb3253bdb9e9d5d84bb/unet.\n",
+      "Defaulting to unsafe serialization. Pass `allow_pickle=False` to raise an error instead.\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "An error occurred while trying to fetch /home/zeus/.cache/huggingface/hub/models--stable-diffusion-v1-5--stable-diffusion-inpainting/snapshots/8a4288a76071f7280aedbdb3253bdb9e9d5d84bb/vae: Error no file named diffusion_pytorch_model.safetensors found in directory /home/zeus/.cache/huggingface/hub/models--stable-diffusion-v1-5--stable-diffusion-inpainting/snapshots/8a4288a76071f7280aedbdb3253bdb9e9d5d84bb/vae.\n",
       "Defaulting to unsafe serialization. Pass `allow_pickle=False` to raise an error instead.\n"
      ]
     },
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "b393bb433c3f46ba9a9ed392f534742d",
+       "model_id": "cfa34d5822784b449f9014f3f1b0e4ef",
        "version_major": 2,
        "version_minor": 0
       },
@@ -93,14 +99,15 @@
     "\n",
     "# Load the pipeline\n",
     "pipe = DiffusionPipeline.from_pretrained(\n",
-    "    \"runwayml/stable-diffusion-inpainting\",\n",
+    "    \"stable-diffusion-v1-5/stable-diffusion-inpainting\",\n",
+    "    custom_pipeline=\"img2img_inpainting\",\n",
     "    torch_dtype=torch.float16\n",
     ")\n",
     "pipe = pipe.to(\"cuda\")\n",
     "\n",
     "# Inpainting\n",
     "prompt = \"a mecha robot sitting on a bench\"\n",
-    "image = pipe(prompt=prompt, image=init_image, mask_image=mask_image).images[0]\n",
+    "image = pipe(prompt=prompt, image=init_image, inner_image=inner_image, mask_image=mask_image).images[0]\n",
     "\n",
     "image.save(\"output.png\")\n"
    ]
diff --git a/diffusers/prompt_2_prompt_pipeline.ipynb b/diffusers/prompt_2_prompt_pipeline.ipynb
index 1756f53d..2e892235 100644
--- a/diffusers/prompt_2_prompt_pipeline.ipynb
+++ b/diffusers/prompt_2_prompt_pipeline.ipynb
@@ -18,6 +18,7 @@
     "\n",
     "ReplaceEdit with local blend\n",
     "\n",
+    "```python\n",
     "prompts = [\"A turtle playing with a ball\",\n",
     "           \"A monkey playing with a ball\"]\n",
     "\n",
@@ -27,8 +28,10 @@
     "    \"self_replace_steps\": 0.4,\n",
     "    \"local_blend_words\": [\"turtle\", \"monkey\"]\n",
     "}\n",
+    "```\n",
     "RefineEdit\n",
     "\n",
+    "```python\n",
     "prompts = [\"A turtle\",\n",
     "           \"A turtle in a forest\"]\n",
     "\n",
@@ -37,8 +40,10 @@
     "    \"cross_replace_steps\": 0.4,\n",
     "    \"self_replace_steps\": 0.4,\n",
     "}\n",
-    "RefineEdit with local blend\n",
+    "```\n",
     "\n",
+    "RefineEdit with local blend\n",
+    "```python\n",
     "prompts = [\"A turtle\",\n",
     "           \"A turtle in a forest\"]\n",
     "\n",
@@ -48,8 +53,10 @@
     "    \"self_replace_steps\": 0.4,\n",
     "    \"local_blend_words\": [\"in\", \"a\" , \"forest\"]\n",
     "}\n",
+    "```\n",
     "ReweightEdit\n",
     "\n",
+    "```python\n",
     "prompts = [\"A smiling turtle\"] * 2\n",
     "\n",
     "edit_kcross_attention_kwargswargs = {\n",
@@ -59,6 +66,7 @@
     "    \"equalizer_words\": [\"smiling\"],\n",
     "    \"equalizer_strengths\": [5]\n",
     "}\n",
+    "```\n",
     "\n",
     "Side note: See this [GitHub gist](https://gist.github.com/UmerHA/b65bb5fb9626c9c73f3ade2869e36164) if you want to visualize the attention maps. This script was contributed by [Umer Adil](https://github.com/UmerHA) and the notebook by [Parag Ekbote](https://github.com/ParagEkbote)."
    ]

From 18168244d8f6fb57c3f7241155645e7fd7237061 Mon Sep 17 00:00:00 2001
From: Parag Ekbote <thecoolekbote189@gmail.com>
Date: Thu, 20 Mar 2025 13:11:11 +0000
Subject: [PATCH 3/3] Update prompt_2_prompt_pipeline with proper subheading.

---
 diffusers/prompt_2_prompt_pipeline.ipynb | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/diffusers/prompt_2_prompt_pipeline.ipynb b/diffusers/prompt_2_prompt_pipeline.ipynb
index 2e892235..ed59af12 100644
--- a/diffusers/prompt_2_prompt_pipeline.ipynb
+++ b/diffusers/prompt_2_prompt_pipeline.ipynb
@@ -29,6 +29,7 @@
     "    \"local_blend_words\": [\"turtle\", \"monkey\"]\n",
     "}\n",
     "```\n",
+    "\n",
     "RefineEdit\n",
     "\n",
     "```python\n",
@@ -43,6 +44,7 @@
     "```\n",
     "\n",
     "RefineEdit with local blend\n",
+    "\n",
     "```python\n",
     "prompts = [\"A turtle\",\n",
     "           \"A turtle in a forest\"]\n",
@@ -54,6 +56,7 @@
     "    \"local_blend_words\": [\"in\", \"a\" , \"forest\"]\n",
     "}\n",
     "```\n",
+    "\n",
     "ReweightEdit\n",
     "\n",
     "```python\n",