Skip to content

feat(diffusers): update, add autopipeline, controlnet #1432

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Dec 13, 2023
Merged

feat(diffusers): update, add autopipeline, controlnet #1432

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8720a03
feat(diffusers): update, add autopipeline, controlenet
mudler Dec 12, 2023
f8ee74d
tests with AutoPipeline
mudler Dec 12, 2023
c303312
simplify logic
mudler Dec 13, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -383,7 +383,7 @@ help: ## Show this help.
protogen: protogen-go protogen-python

protogen-go:
protoc --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative \
protoc -Ibackend/ --go_out=pkg/grpc/proto/ --go_opt=paths=source_relative --go-grpc_out=pkg/grpc/proto/ --go-grpc_opt=paths=source_relative \
backend/backend.proto

protogen-python:
Expand Down
1 change: 1 addition & 0 deletions api/backend/image.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ func ImageGeneration(height, width, mode, step, seed int, positive_prompt, negat
CLIPModel: c.Diffusers.ClipModel,
CLIPSubfolder: c.Diffusers.ClipSubFolder,
CLIPSkip: int32(c.Diffusers.ClipSkip),
ControlNet: c.Diffusers.ControlNet,
}),
})

Expand Down
4 changes: 2 additions & 2 deletions api/config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,7 @@ type Config struct {

// Diffusers
Diffusers Diffusers `yaml:"diffusers"`

Step int `yaml:"step"`
Step int `yaml:"step"`

// GRPC Options
GRPC GRPC `yaml:"grpc"`
Expand Down Expand Up @@ -77,6 +76,7 @@ type Diffusers struct {
ClipSkip int `yaml:"clip_skip"` // Skip every N frames
ClipModel string `yaml:"clip_model"` // Clip model to use
ClipSubFolder string `yaml:"clip_subfolder"` // Subfolder to use for clip model
ControlNet string `yaml:"control_net"`
}

type LLMConfig struct {
Expand Down
1 change: 1 addition & 0 deletions backend/backend.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,7 @@ message ModelOptions {
string CLIPModel = 31;
string CLIPSubfolder = 32;
int32 CLIPSkip = 33;
string ControlNet = 48;

// RWKV
string Tokenizer = 34;
Expand Down
65 changes: 33 additions & 32 deletions backend/python/autogptq/backend_pb2.py
View file
Open in desktop

Large diffs are not rendered by default.

65 changes: 33 additions & 32 deletions backend/python/bark/backend_pb2.py
View file
Open in desktop

Large diffs are not rendered by default.

65 changes: 43 additions & 22 deletions backend/python/diffusers/backend_diffusers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,9 @@
import grpc

from diffusers import StableDiffusionXLPipeline, StableDiffusionDepth2ImgPipeline, DPMSolverMultistepScheduler, StableDiffusionPipeline, DiffusionPipeline, EulerAncestralDiscreteScheduler
from diffusers import StableDiffusionImg2ImgPipeline
from diffusers import StableDiffusionImg2ImgPipeline, AutoPipelineForText2Image, ControlNetModel
from diffusers.pipelines.stable_diffusion import safety_checker

from diffusers.utils import load_image
from compel import Compel

from transformers import CLIPTextModel
Expand All @@ -30,6 +30,7 @@
_ONE_DAY_IN_SECONDS = 60 * 60 * 24
COMPEL=os.environ.get("COMPEL", "1") == "1"
CLIPSKIP=os.environ.get("CLIPSKIP", "1") == "1"
SAFETENSORS=os.environ.get("SAFETENSORS", "1") == "1"

# If MAX_WORKERS are specified in the environment use it, otherwise default to 1
MAX_WORKERS = int(os.environ.get('PYTHON_GRPC_MAX_WORKERS', '1'))
Expand Down Expand Up @@ -135,8 +136,11 @@ def LoadModel(self, request, context):
print(f"Loading model {request.Model}...", file=sys.stderr)
print(f"Request {request}", file=sys.stderr)
torchType = torch.float32
variant = None

if request.F16Memory:
torchType = torch.float16
variant="fp16"

local = False
modelFile = request.Model
Expand All @@ -160,14 +164,8 @@ def LoadModel(self, request, context):

fromSingleFile = request.Model.startswith("http") or request.Model.startswith("/") or local

if request.IMG2IMG and request.PipelineType == "":
request.PipelineType == "StableDiffusionImg2ImgPipeline"

if request.PipelineType == "":
request.PipelineType == "StableDiffusionPipeline"

## img2img
if request.PipelineType == "StableDiffusionImg2ImgPipeline":
if (request.PipelineType == "StableDiffusionImg2ImgPipeline") or (request.IMG2IMG and request.PipelineType == ""):
if fromSingleFile:
self.pipe = StableDiffusionImg2ImgPipeline.from_single_file(modelFile,
torch_dtype=torchType,
Expand All @@ -177,12 +175,18 @@ def LoadModel(self, request, context):
torch_dtype=torchType,
guidance_scale=cfg_scale)

if request.PipelineType == "StableDiffusionDepth2ImgPipeline":
elif request.PipelineType == "StableDiffusionDepth2ImgPipeline":
self.pipe = StableDiffusionDepth2ImgPipeline.from_pretrained(request.Model,
torch_dtype=torchType,
guidance_scale=cfg_scale)
## text2img
if request.PipelineType == "StableDiffusionPipeline":
elif request.PipelineType == "AutoPipelineForText2Image" or request.PipelineType == "":
self.pipe = AutoPipelineForText2Image.from_pretrained(request.Model,
torch_dtype=torchType,
use_safetensors=SAFETENSORS,
variant=variant,
guidance_scale=cfg_scale)
elif request.PipelineType == "StableDiffusionPipeline":
if fromSingleFile:
self.pipe = StableDiffusionPipeline.from_single_file(modelFile,
torch_dtype=torchType,
Expand All @@ -191,13 +195,11 @@ def LoadModel(self, request, context):
self.pipe = StableDiffusionPipeline.from_pretrained(request.Model,
torch_dtype=torchType,
guidance_scale=cfg_scale)

if request.PipelineType == "DiffusionPipeline":
elif request.PipelineType == "DiffusionPipeline":
self.pipe = DiffusionPipeline.from_pretrained(request.Model,
torch_dtype=torchType,
guidance_scale=cfg_scale)

if request.PipelineType == "StableDiffusionXLPipeline":
elif request.PipelineType == "StableDiffusionXLPipeline":
if fromSingleFile:
self.pipe = StableDiffusionXLPipeline.from_single_file(modelFile,
torch_dtype=torchType, use_safetensors=True,
Expand All @@ -207,21 +209,34 @@ def LoadModel(self, request, context):
request.Model,
torch_dtype=torchType,
use_safetensors=True,
# variant="fp16"
variant=variant,
guidance_scale=cfg_scale)
# https://github.com/huggingface/diffusers/issues/4446
# do not use text_encoder in the constructor since then
# https://github.com/huggingface/diffusers/issues/3212#issuecomment-1521841481

if CLIPSKIP and request.CLIPSkip != 0:
text_encoder = CLIPTextModel.from_pretrained(clipmodel, num_hidden_layers=request.CLIPSkip, subfolder=clipsubfolder, torch_dtype=torchType)
self.pipe.text_encoder=text_encoder
self.clip_skip = request.CLIPSkip
else:
self.clip_skip = 0

# torch_dtype needs to be customized. float16 for GPU, float32 for CPU
# TODO: this needs to be customized
if request.SchedulerType != "":
self.pipe.scheduler = get_scheduler(request.SchedulerType, self.pipe.scheduler.config)

self.compel = Compel(tokenizer=self.pipe.tokenizer, text_encoder=self.pipe.text_encoder)


if request.ControlNet:
self.controlnet = ControlNetModel.from_pretrained(
request.ControlNet, torch_dtype=torchType, variant=variant
)
self.pipe.controlnet = self.controlnet
else:
self.controlnet = None

if request.CUDA:
self.pipe.to('cuda')
if self.controlnet:
self.controlnet.to('cuda')
# Assume directory from request.ModelFile.
# Only if request.LoraAdapter it's not an absolute path
if request.LoraAdapter and request.ModelFile != "" and not os.path.isabs(request.LoraAdapter) and request.LoraAdapter:
Expand Down Expand Up @@ -316,9 +331,15 @@ def GenerateImage(self, request, context):
"num_inference_steps": steps,
}

if request.src != "":
if request.src != "" and not self.controlnet:
image = Image.open(request.src)
options["image"] = image
elif self.controlnet and request.src:
pose_image = load_image(request.src)
options["image"] = pose_image

if CLIPSKIP and self.clip_skip != 0:
options["clip_skip"]=self.clip_skip

# Get the keys that we will build the args for our pipe for
keys = options.keys()
Expand Down
Loading