From bbba74646486fd51420cba07792937776cf239b0 Mon Sep 17 00:00:00 2001 From: Won-Kyu Park Date: Sat, 31 Aug 2024 16:34:33 +0900 Subject: [PATCH 01/29] import Flux from https://github.com/black-forest-labs/flux/ License: Apache 2.0 original author: Tim Dockhorn @timudk --- modules/models/flux/math.py | 30 +++ modules/models/flux/model.py | 112 ++++++++++++ modules/models/flux/modules/layers.py | 253 ++++++++++++++++++++++++++ modules/models/flux/util.py | 201 ++++++++++++++++++++ 4 files changed, 596 insertions(+) create mode 100644 modules/models/flux/math.py create mode 100644 modules/models/flux/model.py create mode 100644 modules/models/flux/modules/layers.py create mode 100644 modules/models/flux/util.py diff --git a/modules/models/flux/math.py b/modules/models/flux/math.py new file mode 100644 index 00000000000..0156bb6a205 --- /dev/null +++ b/modules/models/flux/math.py @@ -0,0 +1,30 @@ +import torch +from einops import rearrange +from torch import Tensor + + +def attention(q: Tensor, k: Tensor, v: Tensor, pe: Tensor) -> Tensor: + q, k = apply_rope(q, k, pe) + + x = torch.nn.functional.scaled_dot_product_attention(q, k, v) + x = rearrange(x, "B H L D -> B L (H D)") + + return x + + +def rope(pos: Tensor, dim: int, theta: int) -> Tensor: + assert dim % 2 == 0 + scale = torch.arange(0, dim, 2, dtype=torch.float64, device=pos.device) / dim + omega = 1.0 / (theta**scale) + out = torch.einsum("...n,d->...nd", pos, omega) + out = torch.stack([torch.cos(out), -torch.sin(out), torch.sin(out), torch.cos(out)], dim=-1) + out = rearrange(out, "b n d (i j) -> b n d i j", i=2, j=2) + return out.float() + + +def apply_rope(xq: Tensor, xk: Tensor, freqs_cis: Tensor) -> tuple[Tensor, Tensor]: + xq_ = xq.float().reshape(*xq.shape[:-1], -1, 1, 2) + xk_ = xk.float().reshape(*xk.shape[:-1], -1, 1, 2) + xq_out = freqs_cis[..., 0] * xq_[..., 0] + freqs_cis[..., 1] * xq_[..., 1] + xk_out = freqs_cis[..., 0] * xk_[..., 0] + freqs_cis[..., 1] * xk_[..., 1] + return xq_out.reshape(*xq.shape).type_as(xq), xk_out.reshape(*xk.shape).type_as(xk) diff --git a/modules/models/flux/model.py b/modules/models/flux/model.py new file mode 100644 index 00000000000..f33ab83233a --- /dev/null +++ b/modules/models/flux/model.py @@ -0,0 +1,112 @@ +from dataclasses import dataclass + +import torch +from torch import Tensor, nn + +from flux.modules.layers import (DoubleStreamBlock, EmbedND, LastLayer, + MLPEmbedder, SingleStreamBlock, + timestep_embedding) + + +@dataclass +class FluxParams: + in_channels: int + vec_in_dim: int + context_in_dim: int + hidden_size: int + mlp_ratio: float + num_heads: int + depth: int + depth_single_blocks: int + axes_dim: list[int] + theta: int + qkv_bias: bool + guidance_embed: bool + + +class Flux(nn.Module): + """ + Transformer model for flow matching on sequences. + """ + + def __init__(self, params: FluxParams): + super().__init__() + + self.params = params + self.in_channels = params.in_channels + self.out_channels = self.in_channels + if params.hidden_size % params.num_heads != 0: + raise ValueError( + f"Hidden size {params.hidden_size} must be divisible by num_heads {params.num_heads}" + ) + pe_dim = params.hidden_size // params.num_heads + if sum(params.axes_dim) != pe_dim: + raise ValueError(f"Got {params.axes_dim} but expected positional dim {pe_dim}") + self.hidden_size = params.hidden_size + self.num_heads = params.num_heads + self.pe_embedder = EmbedND(dim=pe_dim, theta=params.theta, axes_dim=params.axes_dim) + self.img_in = nn.Linear(self.in_channels, self.hidden_size, bias=True) + self.time_in = MLPEmbedder(in_dim=256, hidden_dim=self.hidden_size) + self.vector_in = MLPEmbedder(params.vec_in_dim, self.hidden_size) + self.guidance_in = ( + MLPEmbedder(in_dim=256, hidden_dim=self.hidden_size) if params.guidance_embed else nn.Identity() + ) + self.txt_in = nn.Linear(params.context_in_dim, self.hidden_size) + + self.double_blocks = nn.ModuleList( + [ + DoubleStreamBlock( + self.hidden_size, + self.num_heads, + mlp_ratio=params.mlp_ratio, + qkv_bias=params.qkv_bias, + ) + for _ in range(params.depth) + ] + ) + + self.single_blocks = nn.ModuleList( + [ + SingleStreamBlock(self.hidden_size, self.num_heads, mlp_ratio=params.mlp_ratio) + for _ in range(params.depth_single_blocks) + ] + ) + + self.final_layer = LastLayer(self.hidden_size, 1, self.out_channels) + + def forward( + self, + img: Tensor, + img_ids: Tensor, + txt: Tensor, + txt_ids: Tensor, + timesteps: Tensor, + y: Tensor, + guidance: Tensor | None = None, + ) -> Tensor: + if img.ndim != 3 or txt.ndim != 3: + raise ValueError("Input img and txt tensors must have 3 dimensions.") + + # running on sequences img + img = self.img_in(img) + vec = self.time_in(timestep_embedding(timesteps, 256)) + if self.params.guidance_embed: + if guidance is None: + raise ValueError("Didn't get guidance strength for guidance distilled model.") + vec = vec + self.guidance_in(timestep_embedding(guidance, 256)) + vec = vec + self.vector_in(y) + txt = self.txt_in(txt) + + ids = torch.cat((txt_ids, img_ids), dim=1) + pe = self.pe_embedder(ids) + + for block in self.double_blocks: + img, txt = block(img=img, txt=txt, vec=vec, pe=pe) + + img = torch.cat((txt, img), 1) + for block in self.single_blocks: + img = block(img, vec=vec, pe=pe) + img = img[:, txt.shape[1] :, ...] + + img = self.final_layer(img, vec) # (N, T, patch_size ** 2 * out_channels) + return img diff --git a/modules/models/flux/modules/layers.py b/modules/models/flux/modules/layers.py new file mode 100644 index 00000000000..091ddf62415 --- /dev/null +++ b/modules/models/flux/modules/layers.py @@ -0,0 +1,253 @@ +import math +from dataclasses import dataclass + +import torch +from einops import rearrange +from torch import Tensor, nn + +from flux.math import attention, rope + + +class EmbedND(nn.Module): + def __init__(self, dim: int, theta: int, axes_dim: list[int]): + super().__init__() + self.dim = dim + self.theta = theta + self.axes_dim = axes_dim + + def forward(self, ids: Tensor) -> Tensor: + n_axes = ids.shape[-1] + emb = torch.cat( + [rope(ids[..., i], self.axes_dim[i], self.theta) for i in range(n_axes)], + dim=-3, + ) + + return emb.unsqueeze(1) + + +def timestep_embedding(t: Tensor, dim, max_period=10000, time_factor: float = 1000.0): + """ + Create sinusoidal timestep embeddings. + :param t: a 1-D Tensor of N indices, one per batch element. + These may be fractional. + :param dim: the dimension of the output. + :param max_period: controls the minimum frequency of the embeddings. + :return: an (N, D) Tensor of positional embeddings. + """ + t = time_factor * t + half = dim // 2 + freqs = torch.exp(-math.log(max_period) * torch.arange(start=0, end=half, dtype=torch.float32) / half).to( + t.device + ) + + args = t[:, None].float() * freqs[None] + embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1) + if dim % 2: + embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1) + if torch.is_floating_point(t): + embedding = embedding.to(t) + return embedding + + +class MLPEmbedder(nn.Module): + def __init__(self, in_dim: int, hidden_dim: int): + super().__init__() + self.in_layer = nn.Linear(in_dim, hidden_dim, bias=True) + self.silu = nn.SiLU() + self.out_layer = nn.Linear(hidden_dim, hidden_dim, bias=True) + + def forward(self, x: Tensor) -> Tensor: + return self.out_layer(self.silu(self.in_layer(x))) + + +class RMSNorm(torch.nn.Module): + def __init__(self, dim: int): + super().__init__() + self.scale = nn.Parameter(torch.ones(dim)) + + def forward(self, x: Tensor): + x_dtype = x.dtype + x = x.float() + rrms = torch.rsqrt(torch.mean(x**2, dim=-1, keepdim=True) + 1e-6) + return (x * rrms).to(dtype=x_dtype) * self.scale + + +class QKNorm(torch.nn.Module): + def __init__(self, dim: int): + super().__init__() + self.query_norm = RMSNorm(dim) + self.key_norm = RMSNorm(dim) + + def forward(self, q: Tensor, k: Tensor, v: Tensor) -> tuple[Tensor, Tensor]: + q = self.query_norm(q) + k = self.key_norm(k) + return q.to(v), k.to(v) + + +class SelfAttention(nn.Module): + def __init__(self, dim: int, num_heads: int = 8, qkv_bias: bool = False): + super().__init__() + self.num_heads = num_heads + head_dim = dim // num_heads + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.norm = QKNorm(head_dim) + self.proj = nn.Linear(dim, dim) + + def forward(self, x: Tensor, pe: Tensor) -> Tensor: + qkv = self.qkv(x) + q, k, v = rearrange(qkv, "B L (K H D) -> K B H L D", K=3, H=self.num_heads) + q, k = self.norm(q, k, v) + x = attention(q, k, v, pe=pe) + x = self.proj(x) + return x + + +@dataclass +class ModulationOut: + shift: Tensor + scale: Tensor + gate: Tensor + + +class Modulation(nn.Module): + def __init__(self, dim: int, double: bool): + super().__init__() + self.is_double = double + self.multiplier = 6 if double else 3 + self.lin = nn.Linear(dim, self.multiplier * dim, bias=True) + + def forward(self, vec: Tensor) -> tuple[ModulationOut, ModulationOut | None]: + out = self.lin(nn.functional.silu(vec))[:, None, :].chunk(self.multiplier, dim=-1) + + return ( + ModulationOut(*out[:3]), + ModulationOut(*out[3:]) if self.is_double else None, + ) + + +class DoubleStreamBlock(nn.Module): + def __init__(self, hidden_size: int, num_heads: int, mlp_ratio: float, qkv_bias: bool = False): + super().__init__() + + mlp_hidden_dim = int(hidden_size * mlp_ratio) + self.num_heads = num_heads + self.hidden_size = hidden_size + self.img_mod = Modulation(hidden_size, double=True) + self.img_norm1 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6) + self.img_attn = SelfAttention(dim=hidden_size, num_heads=num_heads, qkv_bias=qkv_bias) + + self.img_norm2 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6) + self.img_mlp = nn.Sequential( + nn.Linear(hidden_size, mlp_hidden_dim, bias=True), + nn.GELU(approximate="tanh"), + nn.Linear(mlp_hidden_dim, hidden_size, bias=True), + ) + + self.txt_mod = Modulation(hidden_size, double=True) + self.txt_norm1 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6) + self.txt_attn = SelfAttention(dim=hidden_size, num_heads=num_heads, qkv_bias=qkv_bias) + + self.txt_norm2 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6) + self.txt_mlp = nn.Sequential( + nn.Linear(hidden_size, mlp_hidden_dim, bias=True), + nn.GELU(approximate="tanh"), + nn.Linear(mlp_hidden_dim, hidden_size, bias=True), + ) + + def forward(self, img: Tensor, txt: Tensor, vec: Tensor, pe: Tensor) -> tuple[Tensor, Tensor]: + img_mod1, img_mod2 = self.img_mod(vec) + txt_mod1, txt_mod2 = self.txt_mod(vec) + + # prepare image for attention + img_modulated = self.img_norm1(img) + img_modulated = (1 + img_mod1.scale) * img_modulated + img_mod1.shift + img_qkv = self.img_attn.qkv(img_modulated) + img_q, img_k, img_v = rearrange(img_qkv, "B L (K H D) -> K B H L D", K=3, H=self.num_heads) + img_q, img_k = self.img_attn.norm(img_q, img_k, img_v) + + # prepare txt for attention + txt_modulated = self.txt_norm1(txt) + txt_modulated = (1 + txt_mod1.scale) * txt_modulated + txt_mod1.shift + txt_qkv = self.txt_attn.qkv(txt_modulated) + txt_q, txt_k, txt_v = rearrange(txt_qkv, "B L (K H D) -> K B H L D", K=3, H=self.num_heads) + txt_q, txt_k = self.txt_attn.norm(txt_q, txt_k, txt_v) + + # run actual attention + q = torch.cat((txt_q, img_q), dim=2) + k = torch.cat((txt_k, img_k), dim=2) + v = torch.cat((txt_v, img_v), dim=2) + + attn = attention(q, k, v, pe=pe) + txt_attn, img_attn = attn[:, : txt.shape[1]], attn[:, txt.shape[1] :] + + # calculate the img bloks + img = img + img_mod1.gate * self.img_attn.proj(img_attn) + img = img + img_mod2.gate * self.img_mlp((1 + img_mod2.scale) * self.img_norm2(img) + img_mod2.shift) + + # calculate the txt bloks + txt = txt + txt_mod1.gate * self.txt_attn.proj(txt_attn) + txt = txt + txt_mod2.gate * self.txt_mlp((1 + txt_mod2.scale) * self.txt_norm2(txt) + txt_mod2.shift) + return img, txt + + +class SingleStreamBlock(nn.Module): + """ + A DiT block with parallel linear layers as described in + https://arxiv.org/abs/2302.05442 and adapted modulation interface. + """ + + def __init__( + self, + hidden_size: int, + num_heads: int, + mlp_ratio: float = 4.0, + qk_scale: float | None = None, + ): + super().__init__() + self.hidden_dim = hidden_size + self.num_heads = num_heads + head_dim = hidden_size // num_heads + self.scale = qk_scale or head_dim**-0.5 + + self.mlp_hidden_dim = int(hidden_size * mlp_ratio) + # qkv and mlp_in + self.linear1 = nn.Linear(hidden_size, hidden_size * 3 + self.mlp_hidden_dim) + # proj and mlp_out + self.linear2 = nn.Linear(hidden_size + self.mlp_hidden_dim, hidden_size) + + self.norm = QKNorm(head_dim) + + self.hidden_size = hidden_size + self.pre_norm = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6) + + self.mlp_act = nn.GELU(approximate="tanh") + self.modulation = Modulation(hidden_size, double=False) + + def forward(self, x: Tensor, vec: Tensor, pe: Tensor) -> Tensor: + mod, _ = self.modulation(vec) + x_mod = (1 + mod.scale) * self.pre_norm(x) + mod.shift + qkv, mlp = torch.split(self.linear1(x_mod), [3 * self.hidden_size, self.mlp_hidden_dim], dim=-1) + + q, k, v = rearrange(qkv, "B L (K H D) -> K B H L D", K=3, H=self.num_heads) + q, k = self.norm(q, k, v) + + # compute attention + attn = attention(q, k, v, pe=pe) + # compute activation in mlp stream, cat again and run second linear layer + output = self.linear2(torch.cat((attn, self.mlp_act(mlp)), 2)) + return x + mod.gate * output + + +class LastLayer(nn.Module): + def __init__(self, hidden_size: int, patch_size: int, out_channels: int): + super().__init__() + self.norm_final = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6) + self.linear = nn.Linear(hidden_size, patch_size * patch_size * out_channels, bias=True) + self.adaLN_modulation = nn.Sequential(nn.SiLU(), nn.Linear(hidden_size, 2 * hidden_size, bias=True)) + + def forward(self, x: Tensor, vec: Tensor) -> Tensor: + shift, scale = self.adaLN_modulation(vec).chunk(2, dim=1) + x = (1 + scale[:, None, :]) * self.norm_final(x) + shift[:, None, :] + x = self.linear(x) + return x diff --git a/modules/models/flux/util.py b/modules/models/flux/util.py new file mode 100644 index 00000000000..77fc76c099c --- /dev/null +++ b/modules/models/flux/util.py @@ -0,0 +1,201 @@ +import os +from dataclasses import dataclass + +import torch +from einops import rearrange +from huggingface_hub import hf_hub_download +from imwatermark import WatermarkEncoder +from safetensors.torch import load_file as load_sft + +from flux.model import Flux, FluxParams +from flux.modules.autoencoder import AutoEncoder, AutoEncoderParams +from flux.modules.conditioner import HFEmbedder + + +@dataclass +class ModelSpec: + params: FluxParams + ae_params: AutoEncoderParams + ckpt_path: str | None + ae_path: str | None + repo_id: str | None + repo_flow: str | None + repo_ae: str | None + + +configs = { + "flux-dev": ModelSpec( + repo_id="black-forest-labs/FLUX.1-dev", + repo_flow="flux1-dev.safetensors", + repo_ae="ae.safetensors", + ckpt_path=os.getenv("FLUX_DEV"), + params=FluxParams( + in_channels=64, + vec_in_dim=768, + context_in_dim=4096, + hidden_size=3072, + mlp_ratio=4.0, + num_heads=24, + depth=19, + depth_single_blocks=38, + axes_dim=[16, 56, 56], + theta=10_000, + qkv_bias=True, + guidance_embed=True, + ), + ae_path=os.getenv("AE"), + ae_params=AutoEncoderParams( + resolution=256, + in_channels=3, + ch=128, + out_ch=3, + ch_mult=[1, 2, 4, 4], + num_res_blocks=2, + z_channels=16, + scale_factor=0.3611, + shift_factor=0.1159, + ), + ), + "flux-schnell": ModelSpec( + repo_id="black-forest-labs/FLUX.1-schnell", + repo_flow="flux1-schnell.safetensors", + repo_ae="ae.safetensors", + ckpt_path=os.getenv("FLUX_SCHNELL"), + params=FluxParams( + in_channels=64, + vec_in_dim=768, + context_in_dim=4096, + hidden_size=3072, + mlp_ratio=4.0, + num_heads=24, + depth=19, + depth_single_blocks=38, + axes_dim=[16, 56, 56], + theta=10_000, + qkv_bias=True, + guidance_embed=False, + ), + ae_path=os.getenv("AE"), + ae_params=AutoEncoderParams( + resolution=256, + in_channels=3, + ch=128, + out_ch=3, + ch_mult=[1, 2, 4, 4], + num_res_blocks=2, + z_channels=16, + scale_factor=0.3611, + shift_factor=0.1159, + ), + ), +} + + +def print_load_warning(missing: list[str], unexpected: list[str]) -> None: + if len(missing) > 0 and len(unexpected) > 0: + print(f"Got {len(missing)} missing keys:\n\t" + "\n\t".join(missing)) + print("\n" + "-" * 79 + "\n") + print(f"Got {len(unexpected)} unexpected keys:\n\t" + "\n\t".join(unexpected)) + elif len(missing) > 0: + print(f"Got {len(missing)} missing keys:\n\t" + "\n\t".join(missing)) + elif len(unexpected) > 0: + print(f"Got {len(unexpected)} unexpected keys:\n\t" + "\n\t".join(unexpected)) + + +def load_flow_model(name: str, device: str | torch.device = "cuda", hf_download: bool = True): + # Loading Flux + print("Init model") + ckpt_path = configs[name].ckpt_path + if ( + ckpt_path is None + and configs[name].repo_id is not None + and configs[name].repo_flow is not None + and hf_download + ): + ckpt_path = hf_hub_download(configs[name].repo_id, configs[name].repo_flow) + + with torch.device("meta" if ckpt_path is not None else device): + model = Flux(configs[name].params).to(torch.bfloat16) + + if ckpt_path is not None: + print("Loading checkpoint") + # load_sft doesn't support torch.device + sd = load_sft(ckpt_path, device=str(device)) + missing, unexpected = model.load_state_dict(sd, strict=False, assign=True) + print_load_warning(missing, unexpected) + return model + + +def load_t5(device: str | torch.device = "cuda", max_length: int = 512) -> HFEmbedder: + # max length 64, 128, 256 and 512 should work (if your sequence is short enough) + return HFEmbedder("google/t5-v1_1-xxl", max_length=max_length, torch_dtype=torch.bfloat16).to(device) + + +def load_clip(device: str | torch.device = "cuda") -> HFEmbedder: + return HFEmbedder("openai/clip-vit-large-patch14", max_length=77, torch_dtype=torch.bfloat16).to(device) + + +def load_ae(name: str, device: str | torch.device = "cuda", hf_download: bool = True) -> AutoEncoder: + ckpt_path = configs[name].ae_path + if ( + ckpt_path is None + and configs[name].repo_id is not None + and configs[name].repo_ae is not None + and hf_download + ): + ckpt_path = hf_hub_download(configs[name].repo_id, configs[name].repo_ae) + + # Loading the autoencoder + print("Init AE") + with torch.device("meta" if ckpt_path is not None else device): + ae = AutoEncoder(configs[name].ae_params) + + if ckpt_path is not None: + sd = load_sft(ckpt_path, device=str(device)) + missing, unexpected = ae.load_state_dict(sd, strict=False, assign=True) + print_load_warning(missing, unexpected) + return ae + + +class WatermarkEmbedder: + def __init__(self, watermark): + self.watermark = watermark + self.num_bits = len(WATERMARK_BITS) + self.encoder = WatermarkEncoder() + self.encoder.set_watermark("bits", self.watermark) + + def __call__(self, image: torch.Tensor) -> torch.Tensor: + """ + Adds a predefined watermark to the input image + + Args: + image: ([N,] B, RGB, H, W) in range [-1, 1] + + Returns: + same as input but watermarked + """ + image = 0.5 * image + 0.5 + squeeze = len(image.shape) == 4 + if squeeze: + image = image[None, ...] + n = image.shape[0] + image_np = rearrange((255 * image).detach().cpu(), "n b c h w -> (n b) h w c").numpy()[:, :, :, ::-1] + # torch (b, c, h, w) in [0, 1] -> numpy (b, h, w, c) [0, 255] + # watermarking libary expects input as cv2 BGR format + for k in range(image_np.shape[0]): + image_np[k] = self.encoder.encode(image_np[k], "dwtDct") + image = torch.from_numpy(rearrange(image_np[:, :, :, ::-1], "(n b) h w c -> n b c h w", n=n)).to( + image.device + ) + image = torch.clamp(image / 255, min=0.0, max=1.0) + if squeeze: + image = image[0] + image = 2 * image - 1 + return image + + +# A fixed 48-bit message that was chosen at random +WATERMARK_MESSAGE = 0b001010101111111010000111100111001111010100101110 +# bin(x)[2:] gives bits of x as str, use int to convert them to 0/1 +WATERMARK_BITS = [int(bit) for bit in bin(WATERMARK_MESSAGE)[2:]] +embed_watermark = WatermarkEmbedder(WATERMARK_BITS) From 8e847ee801749669afb5811405ba52b889e97cf7 Mon Sep 17 00:00:00 2001 From: Won-Kyu Park Date: Sat, 31 Aug 2024 16:40:28 +0900 Subject: [PATCH 02/29] fix for A1111 webui --- modules/models/flux/__init__.py | 5 + modules/models/flux/flux.py | 338 ++++++++++++++++++++++++++++++++ 2 files changed, 343 insertions(+) create mode 100644 modules/models/flux/__init__.py create mode 100644 modules/models/flux/flux.py diff --git a/modules/models/flux/__init__.py b/modules/models/flux/__init__.py new file mode 100644 index 00000000000..1cc52a00bef --- /dev/null +++ b/modules/models/flux/__init__.py @@ -0,0 +1,5 @@ +from .flux import FLUX1Inferencer + +__all__ = [ + "FLUX1Inferencer", +] diff --git a/modules/models/flux/flux.py b/modules/models/flux/flux.py new file mode 100644 index 00000000000..d17febc684f --- /dev/null +++ b/modules/models/flux/flux.py @@ -0,0 +1,338 @@ +import contextlib + +import os +import safetensors +import torch +import math + +import k_diffusion +from transformers import CLIPTokenizer + +from modules import shared, devices, modelloader, sd_hijack_clip + +from modules.models.sd3.sd3_impls import SDVAE +from modules.models.sd3.sd3_cond import CLIPL_CONFIG, T5_CONFIG, CLIPL_URL, T5_URL, SafetensorsMapping, Sd3T5 +from modules.models.sd3.other_impls import SDClipModel, T5XXLModel, SDTokenizer, T5XXLTokenizer +from PIL import Image + +from .model import Flux + + +class FluxTokenizer: + def __init__(self): + clip_tokenizer = CLIPTokenizer.from_pretrained("openai/clip-vit-large-patch14") + self.clip_l = SDTokenizer(tokenizer=clip_tokenizer) + self.t5xxl = T5XXLTokenizer() + + def tokenize_with_weights(self, text:str): + out = {} + out["l"] = self.clip_l.tokenize_with_weights(text) + out["t5xxl"] = self.t5xxl.tokenize_with_weights(text) + return out + + +class Flux1ClipL(sd_hijack_clip.TextConditionalModel): + def __init__(self, clip_l): + super().__init__() + + self.clip_l = clip_l + + self.tokenizer = CLIPTokenizer.from_pretrained("openai/clip-vit-large-patch14") + + empty = self.tokenizer('')["input_ids"] + self.id_start = empty[0] + self.id_end = empty[1] + self.id_pad = empty[1] + + self.return_pooled = True + + def tokenize(self, texts): + return self.tokenizer(texts, truncation=False, add_special_tokens=False)["input_ids"] + + def encode_with_transformers(self, tokens): + tokens_g = tokens.clone() + + for batch_pos in range(tokens_g.shape[0]): + index = tokens_g[batch_pos].cpu().tolist().index(self.id_end) + tokens_g[batch_pos, index+1:tokens_g.shape[1]] = 0 + + l_out, l_pooled = self.clip_l(tokens) + l_out = torch.cat([l_out], dim=-1) + l_out = torch.nn.functional.pad(l_out, (0, 4096 - l_out.shape[-1])) + + vector_out = torch.cat([l_pooled], dim=-1) + + l_out.pooled = vector_out + + return l_out + + def encode_embedding_init_text(self, init_text, nvpt): + return torch.zeros((nvpt, 768+1280), device=devices.device) # XXX + + + +class FluxCond(torch.nn.Module): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.tokenizer = FluxTokenizer() + + with torch.no_grad(): + self.clip_l = SDClipModel(layer="hidden", layer_idx=-2, device="cpu", dtype=devices.dtype, layer_norm_hidden_state=False, return_projected_pooled=False, textmodel_json_config=CLIPL_CONFIG) + + if shared.opts.sd3_enable_t5: + self.t5xxl = T5XXLModel(T5_CONFIG, device="cpu", dtype=devices.dtype) + else: + self.t5xxl = None + + self.model_l = Flux1ClipL(self.clip_l) + self.model_t5 = Sd3T5(self.t5xxl) + + def forward(self, prompts: list[str]): + with devices.without_autocast(): + l_out, vector_out = self.model_l(prompts) + t5_out = self.model_t5(prompts, token_count=l_out.shape[1]) + lt_out = torch.cat([l_out, t5_out], dim=-2) + + return { + 'crossattn': lt_out, + 'vector': vector_out, + } + + def before_load_weights(self, state_dict): + clip_path = os.path.join(shared.models_path, "CLIP") + + if 'text_encoders.clip_l.transformer.text_model.embeddings.position_embedding.weight' not in state_dict: + clip_l_file = modelloader.load_file_from_url(CLIPL_URL, model_dir=clip_path, file_name="clip_l.safetensors") + with safetensors.safe_open(clip_l_file, framework="pt") as file: + self.clip_l.transformer.load_state_dict(SafetensorsMapping(file), strict=False) + + if self.t5xxl and 'text_encoders.t5xxl.transformer.encoder.embed_tokens.weight' not in state_dict: + t5_file = modelloader.load_file_from_url(T5_URL, model_dir=clip_path, file_name="t5xxl_fp16.safetensors") + with safetensors.safe_open(t5_file, framework="pt") as file: + self.t5xxl.transformer.load_state_dict(SafetensorsMapping(file), strict=False) + + def encode_embedding_init_text(self, init_text, nvpt): + return self.model_l.encode_embedding_init_text(init_text, nvpt) + + def tokenize(self, texts): + return self.model_l.tokenize(texts) + + def medvram_modules(self): + return [self.clip_l, self.t5xxl] + + def get_token_count(self, text): + _, token_count = self.model_l.process_texts([text]) + + return token_count + + def get_target_prompt_token_count(self, token_count): + return self.model_l.get_target_prompt_token_count(token_count) + +def flux_time_shift(mu: float, sigma: float, t): + return math.exp(mu) / (math.exp(mu) + (1 / t - 1) ** sigma) + +class ModelSamplingFlux(torch.nn.Module): + def __init__(self, model_config=None): + super().__init__() + if model_config is not None: + sampling_settings = model_config.sampling_settings + else: + sampling_settings = {} + + self.set_parameters(shift=sampling_settings.get("shift", 1.15)) + + def set_parameters(self, shift=1.15, timesteps=10000): + self.shift = shift + ts = self.sigma((torch.arange(1, timesteps + 1, 1) / timesteps)) + self.register_buffer('sigmas', ts) + + @property + def sigma_min(self): + return self.sigmas[0] + + @property + def sigma_max(self): + return self.sigmas[-1] + + def timestep(self, sigma): + return sigma + + def sigma(self, timestep): + return flux_time_shift(self.shift, 1.0, timestep) + + def percent_to_sigma(self, percent): + if percent <= 0.0: + return 1.0 + if percent >= 1.0: + return 0.0 + return 1.0 - percent + + def calculate_denoised(self, sigma, model_output, model_input): + sigma = sigma.view(sigma.shape[:1] + (1,) * (model_output.ndim - 1)) + return model_input - model_output * sigma + + +class BaseModel(torch.nn.Module): + """Wrapper around the core FLUX model""" + def __init__(self, shift=1.0, device=None, dtype=torch.float16, state_dict=None, prefix=""): + super().__init__() + + params = dict( + image_model="flux", + in_channels=16, + vec_in_dim=768, + context_in_dim=4096, + hidden_size=3072, + mlp_ratio=4.0, + num_heads=24, + depth=19, + depth_single_blocks=38, + axes_dim=[16, 56, 56], + theta=10000, + qkv_bias=True, + guidance_embed=True, + ) + + self.diffusion_model = Flux(device=device, dtype=devices.dtype, **params) + self.model_sampling = ModelSamplingFlux() + self.depth = 19 + + def apply_model(self, x, sigma, c_crossattn=None, y=None): + dtype = self.get_dtype() + timestep = self.model_sampling.timestep(sigma).float() + guidance = torch.FloatTensor([3.5]).to(device=devices.device, dtype=dtype) + model_output = self.diffusion_model(x.to(dtype), timestep, context=c_crossattn.to(dtype), y=y.to(dtype), guidance=guidance).float() + return self.model_sampling.calculate_denoised(sigma, model_output, x) + + def forward(self, *args, **kwargs): + return self.apply_model(*args, **kwargs) + + def get_dtype(self): + return self.diffusion_model.dtype + + +class FLUX1LatentFormat: + """Latents are slightly shifted from center - this class must be called after VAE Decode to correct for the shift""" + def __init__(self): + self.scale_factor = 0.3611 + self.shift_factor = 0.1159 + + def process_in(self, latent): + return (latent - self.shift_factor) * self.scale_factor + + def process_out(self, latent): + return (latent / self.scale_factor) + self.shift_factor + + def decode_latent_to_preview(self, x0): + """Quick RGB approximate preview of sd3 latents""" + factors = torch.tensor([ + [-0.0404, 0.0159, 0.0609], [ 0.0043, 0.0298, 0.0850], + [ 0.0328, -0.0749, -0.0503], [-0.0245, 0.0085, 0.0549], + [ 0.0966, 0.0894, 0.0530], [ 0.0035, 0.0399, 0.0123], + [ 0.0583, 0.1184, 0.1262], [-0.0191, -0.0206, -0.0306], + [-0.0324, 0.0055, 0.1001], [ 0.0955, 0.0659, -0.0545], + [-0.0504, 0.0231, -0.0013], [ 0.0500, -0.0008, -0.0088], + [ 0.0982, 0.0941, 0.0976], [-0.1233, -0.0280, -0.0897], + [-0.0005, -0.0530, -0.0020], [-0.1273, -0.0932, -0.0680], + ], device="cpu") + latent_image = x0[0].permute(1, 2, 0).cpu() @ factors + + latents_ubyte = (((latent_image + 1) / 2) + .clamp(0, 1) # change scale from -1..1 to 0..1 + .mul(0xFF) # to 0..255 + .byte()).cpu() + + return Image.fromarray(latents_ubyte.numpy()) + + +class FLUX1Denoiser(k_diffusion.external.DiscreteSchedule): + def __init__(self, inner_model, sigmas): + super().__init__(sigmas, quantize=shared.opts.enable_quantization) + self.inner_model = inner_model + + def forward(self, input, sigma, **kwargs): + return self.inner_model.apply_model(input, sigma, **kwargs) + + +class FLUX1Inferencer(torch.nn.Module): + def __init__(self, state_dict, use_ema=False): + super().__init__() + + # detect model_prefix + diffusion_model_prefix = "model.diffusion_model." + if "model.diffusion_model.double_blocks.0.img_attn.norm.key_norm.scale" in state_dict: + diffusion_model_prefix = "model.diffusion_model." + elif "double_blocks.0.img_attn.norm.key_norm.scale" in state_dict: + diffusion_model_prefix = "" + + with torch.no_grad(): + self.model = BaseModel(state_dict=state_dict, prefix=diffusion_model_prefix, device="cpu", dtype=devices.dtype) + self.first_stage_model = SDVAE(device="cpu", dtype=devices.dtype_vae) + self.first_stage_model.dtype = self.model.diffusion_model.dtype + + self.alphas_cumprod = 1 / (self.model.model_sampling.sigmas ** 2 + 1) + + self.text_encoders = FluxCond() + self.cond_stage_key = 'txt' + + self.parameterization = "eps" + self.model.conditioning_key = "crossattn" + + self.latent_format = FLUX1LatentFormat() + self.latent_channels = 16 + + @property + def cond_stage_model(self): + return self.text_encoders + + def before_load_weights(self, state_dict): + self.cond_stage_model.before_load_weights(state_dict) + + def ema_scope(self): + return contextlib.nullcontext() + + def get_learned_conditioning(self, batch: list[str]): + return self.cond_stage_model(batch) + + def apply_model(self, x, t, cond): + return self.model(x, t, c_crossattn=cond['crossattn'], y=cond['vector']) + + def decode_first_stage(self, latent): + latent = self.latent_format.process_out(latent) + return self.first_stage_model.decode(latent) + + def encode_first_stage(self, image): + latent = self.first_stage_model.encode(image) + return self.latent_format.process_in(latent) + + def get_first_stage_encoding(self, x): + return x + + def create_denoiser(self): + return FLUX1Denoiser(self, self.model.model_sampling.sigmas) + + def medvram_fields(self): + return [ + (self, 'first_stage_model'), + (self, 'text_encoders'), + (self, 'model'), + ] + + def add_noise_to_latent(self, x, noise, amount): + return x * (1 - amount) + noise * amount + + def fix_dimensions(self, width, height): + return width // 16 * 16, height // 16 * 16 + + def diffusers_weight_mapping(self): + for i in range(self.model.depth): + yield f"transformer.transformer_blocks.{i}.attn.to_q", f"diffusion_model_joint_blocks_{i}_x_block_attn_qkv_q_proj" + yield f"transformer.transformer_blocks.{i}.attn.to_k", f"diffusion_model_joint_blocks_{i}_x_block_attn_qkv_k_proj" + yield f"transformer.transformer_blocks.{i}.attn.to_v", f"diffusion_model_joint_blocks_{i}_x_block_attn_qkv_v_proj" + yield f"transformer.transformer_blocks.{i}.attn.to_out.0", f"diffusion_model_joint_blocks_{i}_x_block_attn_proj" + + yield f"transformer.transformer_blocks.{i}.attn.add_q_proj", f"diffusion_model_joint_blocks_{i}_context_block.attn_qkv_q_proj" + yield f"transformer.transformer_blocks.{i}.attn.add_k_proj", f"diffusion_model_joint_blocks_{i}_context_block.attn_qkv_k_proj" + yield f"transformer.transformer_blocks.{i}.attn.add_v_proj", f"diffusion_model_joint_blocks_{i}_context_block.attn_qkv_v_proj" + yield f"transformer.transformer_blocks.{i}.attn.add_out_proj.0", f"diffusion_model_joint_blocks_{i}_context_block_attn_proj" From c0d4ab0fdd0a207a224539eeddca189049504775 Mon Sep 17 00:00:00 2001 From: Won-Kyu Park Date: Sat, 31 Aug 2024 16:46:34 +0900 Subject: [PATCH 03/29] support Flux1 --- modules/models/sd3/other_impls.py | 27 +++++++++++++++++---------- modules/sd_models.py | 12 ++++++++++-- modules/sd_models_config.py | 4 ++++ modules/sd_models_types.py | 3 +++ modules/sd_vae_taesd.py | 8 ++++++-- 5 files changed, 40 insertions(+), 14 deletions(-) diff --git a/modules/models/sd3/other_impls.py b/modules/models/sd3/other_impls.py index 78c1dc68758..695d356cbfd 100644 --- a/modules/models/sd3/other_impls.py +++ b/modules/models/sd3/other_impls.py @@ -24,6 +24,11 @@ class AutocastLinear(nn.Linear): def forward(self, x): return torch.nn.functional.linear(x, self.weight.to(x.dtype), self.bias.to(x.dtype) if self.bias is not None else None) +class AutocastLayerNorm(nn.LayerNorm): + def forward(self, x): + return torch.nn.functional.layer_norm( + x, self.normalized_shape, self.weight.to(x.dtype), self.bias.to(x.dtype) if self.bias is not None else None, self.eps) + def attention(q, k, v, heads, mask=None): """Convenience wrapper around a basic attention operation""" @@ -41,9 +46,9 @@ def __init__(self, in_features, hidden_features=None, out_features=None, act_lay out_features = out_features or in_features hidden_features = hidden_features or in_features - self.fc1 = nn.Linear(in_features, hidden_features, bias=bias, dtype=dtype, device=device) + self.fc1 = AutocastLinear(in_features, hidden_features, bias=bias, dtype=dtype, device=device) self.act = act_layer - self.fc2 = nn.Linear(hidden_features, out_features, bias=bias, dtype=dtype, device=device) + self.fc2 = AutocastLinear(hidden_features, out_features, bias=bias, dtype=dtype, device=device) def forward(self, x): x = self.fc1(x) @@ -61,10 +66,10 @@ class CLIPAttention(torch.nn.Module): def __init__(self, embed_dim, heads, dtype, device): super().__init__() self.heads = heads - self.q_proj = nn.Linear(embed_dim, embed_dim, bias=True, dtype=dtype, device=device) - self.k_proj = nn.Linear(embed_dim, embed_dim, bias=True, dtype=dtype, device=device) - self.v_proj = nn.Linear(embed_dim, embed_dim, bias=True, dtype=dtype, device=device) - self.out_proj = nn.Linear(embed_dim, embed_dim, bias=True, dtype=dtype, device=device) + self.q_proj = AutocastLinear(embed_dim, embed_dim, bias=True, dtype=dtype, device=device) + self.k_proj = AutocastLinear(embed_dim, embed_dim, bias=True, dtype=dtype, device=device) + self.v_proj = AutocastLinear(embed_dim, embed_dim, bias=True, dtype=dtype, device=device) + self.out_proj = AutocastLinear(embed_dim, embed_dim, bias=True, dtype=dtype, device=device) def forward(self, x, mask=None): q = self.q_proj(x) @@ -82,9 +87,11 @@ def forward(self, x, mask=None): class CLIPLayer(torch.nn.Module): def __init__(self, embed_dim, heads, intermediate_size, intermediate_activation, dtype, device): super().__init__() - self.layer_norm1 = nn.LayerNorm(embed_dim, dtype=dtype, device=device) + #self.layer_norm1 = nn.LayerNorm(embed_dim, dtype=dtype, device=device) + self.layer_norm1 = AutocastLayerNorm(embed_dim, dtype=dtype, device=device) self.self_attn = CLIPAttention(embed_dim, heads, dtype, device) - self.layer_norm2 = nn.LayerNorm(embed_dim, dtype=dtype, device=device) + self.layer_norm2 = AutocastLayerNorm(embed_dim, dtype=dtype, device=device) + #self.layer_norm2 = nn.LayerNorm(embed_dim, dtype=dtype, device=device) #self.mlp = CLIPMLP(embed_dim, intermediate_size, intermediate_activation, dtype, device) self.mlp = Mlp(embed_dim, intermediate_size, embed_dim, act_layer=ACTIVATIONS[intermediate_activation], dtype=dtype, device=device) @@ -131,7 +138,7 @@ def __init__(self, config_dict, dtype, device): super().__init__() self.embeddings = CLIPEmbeddings(embed_dim, dtype=torch.float32, device=device, textual_inversion_key=config_dict.get('textual_inversion_key', 'clip_l')) self.encoder = CLIPEncoder(num_layers, embed_dim, heads, intermediate_size, intermediate_activation, dtype, device) - self.final_layer_norm = nn.LayerNorm(embed_dim, dtype=dtype, device=device) + self.final_layer_norm = AutocastLayerNorm(embed_dim, dtype=dtype, device=device) def forward(self, input_tokens, intermediate_output=None, final_layer_norm_intermediate=True): x = self.embeddings(input_tokens) @@ -150,7 +157,7 @@ def __init__(self, config_dict, dtype, device): self.num_layers = config_dict["num_hidden_layers"] self.text_model = CLIPTextModel_(config_dict, dtype, device) embed_dim = config_dict["hidden_size"] - self.text_projection = nn.Linear(embed_dim, embed_dim, bias=False, dtype=dtype, device=device) + self.text_projection = AutocastLinear(embed_dim, embed_dim, bias=False, dtype=dtype, device=device) self.text_projection.weight.copy_(torch.eye(embed_dim)) self.dtype = dtype diff --git a/modules/sd_models.py b/modules/sd_models.py index 55bd9ca5e43..a0e7233b590 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -33,6 +33,7 @@ class ModelType(enum.Enum): SDXL = 3 SSD = 4 SD3 = 5 + FLUX1 = 6 def replace_key(d, key, new_key, value): @@ -369,7 +370,7 @@ def check_fp8(model): enable_fp8 = False elif shared.opts.fp8_storage == "Enable": enable_fp8 = True - elif getattr(model, "is_sdxl", False) and shared.opts.fp8_storage == "Enable for SDXL": + elif any(getattr(model, attr, False) for attr in ("is_sdxl", "is_flux1")) and shared.opts.fp8_storage == "Enable for SDXL": enable_fp8 = True else: enable_fp8 = False @@ -382,10 +383,14 @@ def set_model_type(model, state_dict): model.is_sdxl = False model.is_ssd = False model.is_sd3 = False + model.is_flux1 = False if "model.diffusion_model.x_embedder.proj.weight" in state_dict: model.is_sd3 = True model.model_type = ModelType.SD3 + elif "model.diffusion_model.double_blocks.0.img_attn.norm.key_norm.scale" in state_dict: + model.is_flux1 = True + model.model_type = ModelType.FLUX1 elif hasattr(model, 'conditioner'): model.is_sdxl = True @@ -671,6 +676,9 @@ def apply_alpha_schedule_override(sd_model, p=None): sd2_clip_weight = 'cond_stage_model.model.transformer.resblocks.0.attn.in_proj_weight' sdxl_clip_weight = 'conditioner.embedders.1.model.ln_final.weight' sdxl_refiner_clip_weight = 'conditioner.embedders.0.model.ln_final.weight' +clip_l_clip_weight = 'text_encoders.clip_l.transformer.text_model.final_layer_norm.weight' +clip_g_clip_weight = 'text_encoders.clip_g.transformer.text_model.final_layer_norm.weight' +t5xxl_clip_weight = 'text_encoders.t5xxl.transformer.encoder.final_layer_norm.weight' class SdModelData: @@ -802,7 +810,7 @@ def load_model(checkpoint_info=None, already_loaded_state_dict=None): state_dict = get_checkpoint_state_dict(checkpoint_info, timer) checkpoint_config = sd_models_config.find_checkpoint_config(state_dict, checkpoint_info) - clip_is_included_into_sd = any(x for x in [sd1_clip_weight, sd2_clip_weight, sdxl_clip_weight, sdxl_refiner_clip_weight] if x in state_dict) + clip_is_included_into_sd = any(x for x in [sd1_clip_weight, sd2_clip_weight, sdxl_clip_weight, sdxl_refiner_clip_weight, clip_l_clip_weight, clip_g_clip_weight ] if x in state_dict) timer.record("find config") diff --git a/modules/sd_models_config.py b/modules/sd_models_config.py index fb44c5a8d98..57b22b595e2 100644 --- a/modules/sd_models_config.py +++ b/modules/sd_models_config.py @@ -24,6 +24,7 @@ config_alt_diffusion = os.path.join(sd_configs_path, "alt-diffusion-inference.yaml") config_alt_diffusion_m18 = os.path.join(sd_configs_path, "alt-diffusion-m18-inference.yaml") config_sd3 = os.path.join(sd_configs_path, "sd3-inference.yaml") +config_flux1 = os.path.join(sd_configs_path, "flux1-inference.yaml") def is_using_v_parameterization_for_sd2(state_dict): @@ -77,6 +78,9 @@ def guess_model_config_from_state_dict(sd, filename): if "model.diffusion_model.x_embedder.proj.weight" in sd: return config_sd3 + if "model.diffusion_model.double_blocks.0.img_attn.norm.key_norm.scale" in sd: + return config_flux1 + if sd.get('conditioner.embedders.1.model.ln_final.weight', None) is not None: if diffusion_model_input.shape[1] == 9: return config_sdxl_inpainting diff --git a/modules/sd_models_types.py b/modules/sd_models_types.py index 2fce2777b2f..867f8b6e20a 100644 --- a/modules/sd_models_types.py +++ b/modules/sd_models_types.py @@ -36,5 +36,8 @@ class WebuiSdModel(LatentDiffusion): is_sd3: bool """True if the model's architecture is SD 3""" + is_flux1: bool + """True if the model's architecture is FLUX 1""" + latent_channels: int """number of layer in latent image representation; will be 16 in SD3 and 4 in other version""" diff --git a/modules/sd_vae_taesd.py b/modules/sd_vae_taesd.py index d06253d2a88..76771e95e55 100644 --- a/modules/sd_vae_taesd.py +++ b/modules/sd_vae_taesd.py @@ -63,7 +63,7 @@ def __init__(self, decoder_path="taesd_decoder.pth", latent_channels=None): super().__init__() if latent_channels is None: - latent_channels = 16 if "taesd3" in str(decoder_path) else 4 + latent_channels = 16 if any(typ in str(decoder_path) for typ in ("taesd3", "taef1")) else 4 self.decoder = decoder(latent_channels) self.decoder.load_state_dict( @@ -79,7 +79,7 @@ def __init__(self, encoder_path="taesd_encoder.pth", latent_channels=None): super().__init__() if latent_channels is None: - latent_channels = 16 if "taesd3" in str(encoder_path) else 4 + latent_channels = 16 if any(typ in str(encoder_path) for typ in ("taesd3", "taef1")) else 4 self.encoder = encoder(latent_channels) self.encoder.load_state_dict( @@ -97,6 +97,8 @@ def download_model(model_path, model_url): def decoder_model(): if shared.sd_model.is_sd3: model_name = "taesd3_decoder.pth" + elif shared.sd_model.is_flux1: + model_name = "taef1_decoder.pth" elif shared.sd_model.is_sdxl: model_name = "taesdxl_decoder.pth" else: @@ -122,6 +124,8 @@ def decoder_model(): def encoder_model(): if shared.sd_model.is_sd3: model_name = "taesd3_encoder.pth" + elif shared.sd_model.is_flux1: + model_name = "taef1_encoder.pth" elif shared.sd_model.is_sdxl: model_name = "taesdxl_encoder.pth" else: From 793233531474755393f431dbf8be6ac142a574ab Mon Sep 17 00:00:00 2001 From: Won-Kyu Park Date: Sat, 31 Aug 2024 16:47:17 +0900 Subject: [PATCH 04/29] fix for flux --- configs/flux1-inference.yaml | 4 ++ modules/models/flux/math.py | 2 +- modules/models/flux/model.py | 67 ++++++++++++----- modules/models/flux/modules/layers.py | 100 ++++++++++++++------------ modules/models/flux/util.py | 6 +- 5 files changed, 113 insertions(+), 66 deletions(-) create mode 100644 configs/flux1-inference.yaml diff --git a/configs/flux1-inference.yaml b/configs/flux1-inference.yaml new file mode 100644 index 00000000000..f9bbe9073f3 --- /dev/null +++ b/configs/flux1-inference.yaml @@ -0,0 +1,4 @@ +model: + target: modules.models.flux.FLUX1Inferencer + params: + state_dict: null diff --git a/modules/models/flux/math.py b/modules/models/flux/math.py index 0156bb6a205..4ad99818e2c 100644 --- a/modules/models/flux/math.py +++ b/modules/models/flux/math.py @@ -22,7 +22,7 @@ def rope(pos: Tensor, dim: int, theta: int) -> Tensor: return out.float() -def apply_rope(xq: Tensor, xk: Tensor, freqs_cis: Tensor) -> tuple[Tensor, Tensor]: +def apply_rope(xq: Tensor, xk: Tensor, freqs_cis: Tensor): xq_ = xq.float().reshape(*xq.shape[:-1], -1, 1, 2) xk_ = xk.float().reshape(*xk.shape[:-1], -1, 1, 2) xq_out = freqs_cis[..., 0] * xq_[..., 0] + freqs_cis[..., 1] * xq_[..., 1] diff --git a/modules/models/flux/model.py b/modules/models/flux/model.py index f33ab83233a..c873978277d 100644 --- a/modules/models/flux/model.py +++ b/modules/models/flux/model.py @@ -1,9 +1,13 @@ +# original code from https://github.com/black-forest-labs/flux +# from dataclasses import dataclass import torch + +from einops import rearrange, repeat from torch import Tensor, nn -from flux.modules.layers import (DoubleStreamBlock, EmbedND, LastLayer, +from .modules.layers import (DoubleStreamBlock, EmbedND, LastLayer, MLPEmbedder, SingleStreamBlock, timestep_embedding) @@ -18,7 +22,7 @@ class FluxParams: num_heads: int depth: int depth_single_blocks: int - axes_dim: list[int] + axes_dim: list theta: int qkv_bias: bool guidance_embed: bool @@ -29,11 +33,13 @@ class Flux(nn.Module): Transformer model for flow matching on sequences. """ - def __init__(self, params: FluxParams): + def __init__(self, image_model=None, final_layer=True, dtype=None, device=None, **kwargs): super().__init__() + self.dtype = dtype + params = FluxParams(**kwargs) self.params = params - self.in_channels = params.in_channels + self.in_channels = params.in_channels * 2 * 2 self.out_channels = self.in_channels if params.hidden_size % params.num_heads != 0: raise ValueError( @@ -45,13 +51,13 @@ def __init__(self, params: FluxParams): self.hidden_size = params.hidden_size self.num_heads = params.num_heads self.pe_embedder = EmbedND(dim=pe_dim, theta=params.theta, axes_dim=params.axes_dim) - self.img_in = nn.Linear(self.in_channels, self.hidden_size, bias=True) - self.time_in = MLPEmbedder(in_dim=256, hidden_dim=self.hidden_size) - self.vector_in = MLPEmbedder(params.vec_in_dim, self.hidden_size) + self.img_in = nn.Linear(self.in_channels, self.hidden_size, bias=True, dtype=dtype, device=device) + self.time_in = MLPEmbedder(in_dim=256, hidden_dim=self.hidden_size, dtype=dtype, device=device) + self.vector_in = MLPEmbedder(params.vec_in_dim, self.hidden_size, dtype=dtype, device=device) self.guidance_in = ( - MLPEmbedder(in_dim=256, hidden_dim=self.hidden_size) if params.guidance_embed else nn.Identity() + MLPEmbedder(in_dim=256, hidden_dim=self.hidden_size, dtype=dtype, device=device) if params.guidance_embed else nn.Identity() ) - self.txt_in = nn.Linear(params.context_in_dim, self.hidden_size) + self.txt_in = nn.Linear(params.context_in_dim, self.hidden_size, dtype=dtype, device=device) self.double_blocks = nn.ModuleList( [ @@ -60,6 +66,7 @@ def __init__(self, params: FluxParams): self.num_heads, mlp_ratio=params.mlp_ratio, qkv_bias=params.qkv_bias, + dtype=dtype, device=device, ) for _ in range(params.depth) ] @@ -67,14 +74,15 @@ def __init__(self, params: FluxParams): self.single_blocks = nn.ModuleList( [ - SingleStreamBlock(self.hidden_size, self.num_heads, mlp_ratio=params.mlp_ratio) + SingleStreamBlock(self.hidden_size, self.num_heads, mlp_ratio=params.mlp_ratio, dtype=dtype, device=device) for _ in range(params.depth_single_blocks) ] ) - self.final_layer = LastLayer(self.hidden_size, 1, self.out_channels) + if final_layer: + self.final_layer = LastLayer(self.hidden_size, 1, self.out_channels, dtype=dtype, device=device) - def forward( + def forward_orig( self, img: Tensor, img_ids: Tensor, @@ -82,18 +90,18 @@ def forward( txt_ids: Tensor, timesteps: Tensor, y: Tensor, - guidance: Tensor | None = None, + guidance: Tensor = None, ) -> Tensor: if img.ndim != 3 or txt.ndim != 3: raise ValueError("Input img and txt tensors must have 3 dimensions.") # running on sequences img img = self.img_in(img) - vec = self.time_in(timestep_embedding(timesteps, 256)) + vec = self.time_in(timestep_embedding(timesteps, 256).to(img.dtype)) if self.params.guidance_embed: if guidance is None: raise ValueError("Didn't get guidance strength for guidance distilled model.") - vec = vec + self.guidance_in(timestep_embedding(guidance, 256)) + vec = vec + self.guidance_in(timestep_embedding(guidance, 256).to(img.dtype)) vec = vec + self.vector_in(y) txt = self.txt_in(txt) @@ -108,5 +116,32 @@ def forward( img = block(img, vec=vec, pe=pe) img = img[:, txt.shape[1] :, ...] - img = self.final_layer(img, vec) # (N, T, patch_size ** 2 * out_channels) + if self.final_layer: + img = self.final_layer(img, vec) # (N, T, patch_size ** 2 * out_channels) return img + + def forward(self, x, timestep, context, y, guidance, **kwargs): + # from comfy/ldm/common_dit.py + def pad_to_patch_size(img, patch_size=(2, 2), padding_mode="circular"): + if padding_mode == "circular" and torch.jit.is_tracing() or torch.jit.is_scripting(): + padding_mode = "reflect" + pad_h = (patch_size[0] - img.shape[-2] % patch_size[0]) % patch_size[0] + pad_w = (patch_size[1] - img.shape[-1] % patch_size[1]) % patch_size[1] + return torch.nn.functional.pad(img, (0, pad_w, 0, pad_h), mode=padding_mode) + + bs, c, h, w = x.shape + patch_size = 2 + x = pad_to_patch_size(x, (patch_size, patch_size)) + + img = rearrange(x, "b c (h ph) (w pw) -> b (h w) (c ph pw)", ph=patch_size, pw=patch_size) + + h_len = ((h + (patch_size // 2)) // patch_size) + w_len = ((w + (patch_size // 2)) // patch_size) + img_ids = torch.zeros((h_len, w_len, 3), device=x.device, dtype=x.dtype) + img_ids[..., 1] = img_ids[..., 1] + torch.linspace(0, h_len - 1, steps=h_len, device=x.device, dtype=x.dtype)[:, None] + img_ids[..., 2] = img_ids[..., 2] + torch.linspace(0, w_len - 1, steps=w_len, device=x.device, dtype=x.dtype)[None, :] + img_ids = repeat(img_ids, "h w c -> b (h w) c", b=bs) + + txt_ids = torch.zeros((bs, context.shape[1], 3), device=x.device, dtype=x.dtype) + out = self.forward_orig(img, img_ids, context, txt_ids, timestep, y, guidance) + return rearrange(out, "b (h w) (c ph pw) -> b c (h ph) (w pw)", h=h_len, w=w_len, ph=2, pw=2)[:,:,:h,:w] diff --git a/modules/models/flux/modules/layers.py b/modules/models/flux/modules/layers.py index 091ddf62415..aa830849eae 100644 --- a/modules/models/flux/modules/layers.py +++ b/modules/models/flux/modules/layers.py @@ -5,11 +5,11 @@ from einops import rearrange from torch import Tensor, nn -from flux.math import attention, rope +from ..math import attention, rope class EmbedND(nn.Module): - def __init__(self, dim: int, theta: int, axes_dim: list[int]): + def __init__(self, dim: int, theta: int, axes_dim: list): super().__init__() self.dim = dim self.theta = theta @@ -36,9 +36,7 @@ def timestep_embedding(t: Tensor, dim, max_period=10000, time_factor: float = 10 """ t = time_factor * t half = dim // 2 - freqs = torch.exp(-math.log(max_period) * torch.arange(start=0, end=half, dtype=torch.float32) / half).to( - t.device - ) + freqs = torch.exp(-math.log(max_period) * torch.arange(start=0, end=half, dtype=torch.float32, device=t.device) / half) args = t[:, None].float() * freqs[None] embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1) @@ -50,20 +48,20 @@ def timestep_embedding(t: Tensor, dim, max_period=10000, time_factor: float = 10 class MLPEmbedder(nn.Module): - def __init__(self, in_dim: int, hidden_dim: int): + def __init__(self, in_dim: int, hidden_dim: int, dtype=None, device=None): super().__init__() - self.in_layer = nn.Linear(in_dim, hidden_dim, bias=True) + self.in_layer = nn.Linear(in_dim, hidden_dim, bias=True, dtype=dtype, device=device) self.silu = nn.SiLU() - self.out_layer = nn.Linear(hidden_dim, hidden_dim, bias=True) + self.out_layer = nn.Linear(hidden_dim, hidden_dim, bias=True, dtype=dtype, device=device) def forward(self, x: Tensor) -> Tensor: return self.out_layer(self.silu(self.in_layer(x))) class RMSNorm(torch.nn.Module): - def __init__(self, dim: int): + def __init__(self, dim: int, dtype=None, device=None): super().__init__() - self.scale = nn.Parameter(torch.ones(dim)) + self.scale = nn.Parameter(torch.empty((dim), dtype=dtype, device=device)) def forward(self, x: Tensor): x_dtype = x.dtype @@ -73,26 +71,26 @@ def forward(self, x: Tensor): class QKNorm(torch.nn.Module): - def __init__(self, dim: int): + def __init__(self, dim: int, dtype=None, device=None): super().__init__() - self.query_norm = RMSNorm(dim) - self.key_norm = RMSNorm(dim) + self.query_norm = RMSNorm(dim, dtype=dtype, device=device) + self.key_norm = RMSNorm(dim, dtype=dtype, device=device) - def forward(self, q: Tensor, k: Tensor, v: Tensor) -> tuple[Tensor, Tensor]: + def forward(self, q: Tensor, k: Tensor, v: Tensor) -> tuple: q = self.query_norm(q) k = self.key_norm(k) return q.to(v), k.to(v) class SelfAttention(nn.Module): - def __init__(self, dim: int, num_heads: int = 8, qkv_bias: bool = False): + def __init__(self, dim: int, num_heads: int = 8, qkv_bias: bool = False, dtype=None, device=None): super().__init__() self.num_heads = num_heads head_dim = dim // num_heads - self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) - self.norm = QKNorm(head_dim) - self.proj = nn.Linear(dim, dim) + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias, dtype=dtype, device=device) + self.norm = QKNorm(head_dim, dtype=dtype, device=device) + self.proj = nn.Linear(dim, dim, dtype=dtype, device=device) def forward(self, x: Tensor, pe: Tensor) -> Tensor: qkv = self.qkv(x) @@ -111,13 +109,13 @@ class ModulationOut: class Modulation(nn.Module): - def __init__(self, dim: int, double: bool): + def __init__(self, dim: int, double: bool, dtype=None, device=None): super().__init__() self.is_double = double self.multiplier = 6 if double else 3 - self.lin = nn.Linear(dim, self.multiplier * dim, bias=True) + self.lin = nn.Linear(dim, self.multiplier * dim, bias=True, dtype=dtype, device=device) - def forward(self, vec: Tensor) -> tuple[ModulationOut, ModulationOut | None]: + def forward(self, vec: Tensor) -> tuple: out = self.lin(nn.functional.silu(vec))[:, None, :].chunk(self.multiplier, dim=-1) return ( @@ -127,35 +125,35 @@ def forward(self, vec: Tensor) -> tuple[ModulationOut, ModulationOut | None]: class DoubleStreamBlock(nn.Module): - def __init__(self, hidden_size: int, num_heads: int, mlp_ratio: float, qkv_bias: bool = False): + def __init__(self, hidden_size: int, num_heads: int, mlp_ratio: float, qkv_bias: bool = False, dtype=None, device=None): super().__init__() mlp_hidden_dim = int(hidden_size * mlp_ratio) self.num_heads = num_heads self.hidden_size = hidden_size - self.img_mod = Modulation(hidden_size, double=True) - self.img_norm1 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6) - self.img_attn = SelfAttention(dim=hidden_size, num_heads=num_heads, qkv_bias=qkv_bias) + self.img_mod = Modulation(hidden_size, double=True, dtype=dtype, device=device) + self.img_norm1 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6, dtype=dtype, device=device) + self.img_attn = SelfAttention(dim=hidden_size, num_heads=num_heads, qkv_bias=qkv_bias, dtype=dtype, device=device) - self.img_norm2 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6) + self.img_norm2 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6, dtype=dtype, device=device) self.img_mlp = nn.Sequential( - nn.Linear(hidden_size, mlp_hidden_dim, bias=True), + nn.Linear(hidden_size, mlp_hidden_dim, bias=True, dtype=dtype, device=device), nn.GELU(approximate="tanh"), - nn.Linear(mlp_hidden_dim, hidden_size, bias=True), + nn.Linear(mlp_hidden_dim, hidden_size, bias=True, dtype=dtype, device=device), ) - self.txt_mod = Modulation(hidden_size, double=True) - self.txt_norm1 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6) - self.txt_attn = SelfAttention(dim=hidden_size, num_heads=num_heads, qkv_bias=qkv_bias) + self.txt_mod = Modulation(hidden_size, double=True, dtype=dtype, device=device) + self.txt_norm1 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6, dtype=dtype, device=device) + self.txt_attn = SelfAttention(dim=hidden_size, num_heads=num_heads, qkv_bias=qkv_bias, dtype=dtype, device=device) - self.txt_norm2 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6) + self.txt_norm2 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6, dtype=dtype, device=device) self.txt_mlp = nn.Sequential( - nn.Linear(hidden_size, mlp_hidden_dim, bias=True), + nn.Linear(hidden_size, mlp_hidden_dim, bias=True, dtype=dtype, device=device), nn.GELU(approximate="tanh"), - nn.Linear(mlp_hidden_dim, hidden_size, bias=True), + nn.Linear(mlp_hidden_dim, hidden_size, bias=True, dtype=dtype, device=device), ) - def forward(self, img: Tensor, txt: Tensor, vec: Tensor, pe: Tensor) -> tuple[Tensor, Tensor]: + def forward(self, img: Tensor, txt: Tensor, vec: Tensor, pe: Tensor): img_mod1, img_mod2 = self.img_mod(vec) txt_mod1, txt_mod2 = self.txt_mod(vec) @@ -188,6 +186,11 @@ def forward(self, img: Tensor, txt: Tensor, vec: Tensor, pe: Tensor) -> tuple[Te # calculate the txt bloks txt = txt + txt_mod1.gate * self.txt_attn.proj(txt_attn) txt = txt + txt_mod2.gate * self.txt_mlp((1 + txt_mod2.scale) * self.txt_norm2(txt) + txt_mod2.shift) + + if txt.dtype == torch.float16: + txt = torch.nan_to_num(txt, nan=0.0, posinf=65504, neginf=-65504) + + return img, txt @@ -202,7 +205,9 @@ def __init__( hidden_size: int, num_heads: int, mlp_ratio: float = 4.0, - qk_scale: float | None = None, + qk_scale: float = None, + dtype=None, + device=None, ): super().__init__() self.hidden_dim = hidden_size @@ -212,17 +217,17 @@ def __init__( self.mlp_hidden_dim = int(hidden_size * mlp_ratio) # qkv and mlp_in - self.linear1 = nn.Linear(hidden_size, hidden_size * 3 + self.mlp_hidden_dim) + self.linear1 = nn.Linear(hidden_size, hidden_size * 3 + self.mlp_hidden_dim, dtype=dtype, device=device) # proj and mlp_out - self.linear2 = nn.Linear(hidden_size + self.mlp_hidden_dim, hidden_size) + self.linear2 = nn.Linear(hidden_size + self.mlp_hidden_dim, hidden_size, dtype=dtype, device=device) - self.norm = QKNorm(head_dim) + self.norm = QKNorm(head_dim, dtype=dtype, device=device) self.hidden_size = hidden_size - self.pre_norm = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6) + self.pre_norm = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6, dtype=dtype, device=device) self.mlp_act = nn.GELU(approximate="tanh") - self.modulation = Modulation(hidden_size, double=False) + self.modulation = Modulation(hidden_size, double=False, dtype=dtype, device=device) def forward(self, x: Tensor, vec: Tensor, pe: Tensor) -> Tensor: mod, _ = self.modulation(vec) @@ -236,15 +241,18 @@ def forward(self, x: Tensor, vec: Tensor, pe: Tensor) -> Tensor: attn = attention(q, k, v, pe=pe) # compute activation in mlp stream, cat again and run second linear layer output = self.linear2(torch.cat((attn, self.mlp_act(mlp)), 2)) - return x + mod.gate * output + x += mod.gate * output + if x.dtype == torch.float16: + x = torch.nan_to_num(x, nan=0.0, posinf=65504, neginf=-65504) + return x class LastLayer(nn.Module): - def __init__(self, hidden_size: int, patch_size: int, out_channels: int): + def __init__(self, hidden_size: int, patch_size: int, out_channels: int, dtype=None, device=None): super().__init__() - self.norm_final = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6) - self.linear = nn.Linear(hidden_size, patch_size * patch_size * out_channels, bias=True) - self.adaLN_modulation = nn.Sequential(nn.SiLU(), nn.Linear(hidden_size, 2 * hidden_size, bias=True)) + self.norm_final = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6, dtype=dtype, device=device) + self.linear = nn.Linear(hidden_size, patch_size * patch_size * out_channels, bias=True, dtype=dtype, device=device) + self.adaLN_modulation = nn.Sequential(nn.SiLU(), nn.Linear(hidden_size, 2 * hidden_size, bias=True, dtype=dtype, device=device)) def forward(self, x: Tensor, vec: Tensor) -> Tensor: shift, scale = self.adaLN_modulation(vec).chunk(2, dim=1) diff --git a/modules/models/flux/util.py b/modules/models/flux/util.py index 77fc76c099c..9303eb7cffe 100644 --- a/modules/models/flux/util.py +++ b/modules/models/flux/util.py @@ -7,9 +7,9 @@ from imwatermark import WatermarkEncoder from safetensors.torch import load_file as load_sft -from flux.model import Flux, FluxParams -from flux.modules.autoencoder import AutoEncoder, AutoEncoderParams -from flux.modules.conditioner import HFEmbedder +from .model import Flux, FluxParams +from .modules.autoencoder import AutoEncoder, AutoEncoderParams +from .modules.conditioner import HFEmbedder @dataclass From 4eee381b0b0791317765e22bc9fbc29cc9849839 Mon Sep 17 00:00:00 2001 From: Won-Kyu Park Date: Sat, 31 Aug 2024 18:07:57 +0900 Subject: [PATCH 05/29] add cheap approximation for flux --- modules/sd_vae_approx.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/modules/sd_vae_approx.py b/modules/sd_vae_approx.py index c5dda7431f1..48ffd672d5c 100644 --- a/modules/sd_vae_approx.py +++ b/modules/sd_vae_approx.py @@ -44,6 +44,8 @@ def model(): model_name = "vaeapprox-sd3.pt" elif shared.sd_model.is_sdxl: model_name = "vaeapprox-sdxl.pt" + elif shared.sd_model.is_flux1: + model_name = "vaeapprox-sd3.pt" else: model_name = "model.pt" @@ -81,6 +83,18 @@ def cheap_approximation(sample): [ 0.0815, 0.0846, 0.1207], [-0.0120, -0.0055, -0.0867], [-0.0749, -0.0634, -0.0456], [-0.1418, -0.1457, -0.1259], ] + elif shared.sd_model.is_flux1: + coeffs = [ + # from comfy + [-0.0404, 0.0159, 0.0609], [ 0.0043, 0.0298, 0.0850], + [ 0.0328, -0.0749, -0.0503], [-0.0245, 0.0085, 0.0549], + [ 0.0966, 0.0894, 0.0530], [ 0.0035, 0.0399, 0.0123], + [ 0.0583, 0.1184, 0.1262], [-0.0191, -0.0206, -0.0306], + [-0.0324, 0.0055, 0.1001], [ 0.0955, 0.0659, -0.0545], + [-0.0504, 0.0231, -0.0013], [ 0.0500, -0.0008, -0.0088], + [ 0.0982, 0.0941, 0.0976], [-0.1233, -0.0280, -0.0897], + [-0.0005, -0.0530, -0.0020], [-0.1273, -0.0932, -0.0680], + ] elif shared.sd_model.is_sdxl: coeffs = [ [ 0.3448, 0.4168, 0.4395], From 67bf106502b02dcd83551e5cb1a102885f382ef5 Mon Sep 17 00:00:00 2001 From: Won-Kyu Park Date: Sat, 31 Aug 2024 23:22:43 +0900 Subject: [PATCH 06/29] fix for float8_* --- modules/sd_disable_initialization.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/modules/sd_disable_initialization.py b/modules/sd_disable_initialization.py index 273a7edd8b4..17ee4e8b3ee 100644 --- a/modules/sd_disable_initialization.py +++ b/modules/sd_disable_initialization.py @@ -160,7 +160,7 @@ def __init__(self, state_dict, device, weight_dtype_conversion=None): self.state_dict = state_dict self.device = device self.weight_dtype_conversion = weight_dtype_conversion or {} - self.default_dtype = self.weight_dtype_conversion.get('') + self.default_dtype = self.weight_dtype_conversion.get('', None) def get_weight_dtype(self, key): key_first_term, _ = key.split('.', 1) @@ -183,12 +183,20 @@ def load_from_state_dict(original, module, state_dict, prefix, *args, **kwargs): key = prefix + name sd_param = sd.pop(key, None) if sd_param is not None: - state_dict[key] = sd_param.to(dtype=self.get_weight_dtype(key)) + dtype = self.get_weight_dtype(key) + if dtype is None: + state_dict[key] = sd_param + else: + state_dict[key] = sd_param.to(dtype=dtype) used_param_keys.append(key) if param.is_meta: dtype = sd_param.dtype if sd_param is not None else param.dtype - module._parameters[name] = torch.nn.parameter.Parameter(torch.zeros_like(param, device=device, dtype=dtype), requires_grad=param.requires_grad) + if dtype in (torch.float8_e4m3fn, torch.float8_e5m2): + zeros = torch.zeros_like(param, device=device, dtype=torch.float32) + else: + zeros = torch.zeros_like(param, device=device, dtype=dtype) + module._parameters[name] = torch.nn.parameter.Parameter(zeros.to(dtype=dtype), requires_grad=param.requires_grad) for name in module._buffers: key = prefix + name From 47a601cbe39ae822d5304916662434d694172f20 Mon Sep 17 00:00:00 2001 From: Won-Kyu Park Date: Thu, 5 Sep 2024 08:57:42 +0900 Subject: [PATCH 07/29] fix for t5xxl --- modules/models/flux/flux.py | 11 +++++++---- modules/shared_options.py | 3 +++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/modules/models/flux/flux.py b/modules/models/flux/flux.py index d17febc684f..dab2bd177b8 100644 --- a/modules/models/flux/flux.py +++ b/modules/models/flux/flux.py @@ -80,7 +80,7 @@ def __init__(self, *args, **kwargs): with torch.no_grad(): self.clip_l = SDClipModel(layer="hidden", layer_idx=-2, device="cpu", dtype=devices.dtype, layer_norm_hidden_state=False, return_projected_pooled=False, textmodel_json_config=CLIPL_CONFIG) - if shared.opts.sd3_enable_t5: + if shared.opts.flux_enable_t5: self.t5xxl = T5XXLModel(T5_CONFIG, device="cpu", dtype=devices.dtype) else: self.t5xxl = None @@ -107,7 +107,7 @@ def before_load_weights(self, state_dict): with safetensors.safe_open(clip_l_file, framework="pt") as file: self.clip_l.transformer.load_state_dict(SafetensorsMapping(file), strict=False) - if self.t5xxl and 'text_encoders.t5xxl.transformer.encoder.embed_tokens.weight' not in state_dict: + if self.t5xxl and 'text_encoders.t5xxl.transformer.encoder.block.0.layer.0.SelfAttention.k.weight' not in state_dict: t5_file = modelloader.load_file_from_url(T5_URL, model_dir=clip_path, file_name="t5xxl_fp16.safetensors") with safetensors.safe_open(t5_file, framework="pt") as file: self.t5xxl.transformer.load_state_dict(SafetensorsMapping(file), strict=False) @@ -194,7 +194,7 @@ def __init__(self, shift=1.0, device=None, dtype=torch.float16, state_dict=None, guidance_embed=True, ) - self.diffusion_model = Flux(device=device, dtype=devices.dtype, **params) + self.diffusion_model = Flux(device=device, dtype=dtype, **params) self.model_sampling = ModelSamplingFlux() self.depth = 19 @@ -300,7 +300,10 @@ def apply_model(self, x, t, cond): def decode_first_stage(self, latent): latent = self.latent_format.process_out(latent) - return self.first_stage_model.decode(latent) + x = self.first_stage_model.decode(latent) + if x.dtype == torch.float16: + x = torch.nan_to_num(x, nan=0.0, posinf=65504, neginf=-65504) + return x def encode_first_stage(self, image): latent = self.first_stage_model.encode(image) diff --git a/modules/shared_options.py b/modules/shared_options.py index 9f4520274b1..01d8f2065ab 100644 --- a/modules/shared_options.py +++ b/modules/shared_options.py @@ -195,6 +195,9 @@ options_templates.update(options_section(('sd3', "Stable Diffusion 3", "sd"), { "sd3_enable_t5": OptionInfo(False, "Enable T5").info("load T5 text encoder; increases VRAM use by a lot, potentially improving quality of generation; requires model reload to apply"), })) +options_templates.update(options_section(('flux', "Stable Diffusion FLUX", "sd"), { + "flux_enable_t5": OptionInfo(False, "Enable T5").info("load T5 text encoder; increases VRAM use by a lot, potentially improving quality of generation; requires model reload to apply"), +})) options_templates.update(options_section(('vae', "VAE", "sd"), { "sd_vae_explanation": OptionHTML(""" From 495212dd4d338af8c83d5cc122411abc591c6917 Mon Sep 17 00:00:00 2001 From: Won-Kyu Park Date: Thu, 5 Sep 2024 09:17:26 +0900 Subject: [PATCH 08/29] fix misc * check supported dtypes * detect non_blocking * update autocast() to use non_blocking, target_device and current_dtype --- modules/devices.py | 82 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 63 insertions(+), 19 deletions(-) diff --git a/modules/devices.py b/modules/devices.py index ee679141ad7..556e72d2eae 100644 --- a/modules/devices.py +++ b/modules/devices.py @@ -128,6 +128,26 @@ def enable_tf32(): dtype_inference: torch.dtype = torch.float16 unet_needs_upcast = False +supported_vae_dtypes = [torch.float16, torch.float32] + + +# prepare available dtypes +if torch.version.cuda: + if torch.cuda.is_bf16_supported() and torch.cuda.get_device_properties(torch.cuda.current_device()).major >= 8: + supported_vae_dtypes = [torch.bfloat16] + supported_vae_dtypes + if has_xpu(): + supported_vae_dtypes = [torch.bfloat16] + supported_vae_dtypes + + +def supports_non_blocking(): + if has_mps() or has_xpu(): + return False + + if npu_specific.has_npu: + return False + + return True + def cond_cast_unet(input): if force_fp16: @@ -149,14 +169,23 @@ def cond_cast_float(input): ] -def manual_cast_forward(target_dtype): +def manual_cast_forward(target_dtype, target_device=None): + params = dict() + if supports_non_blocking(): + params['non_blocking'] = True + def forward_wrapper(self, *args, **kwargs): - if any( - isinstance(arg, torch.Tensor) and arg.dtype != target_dtype - for arg in args - ): - args = [arg.to(target_dtype) if isinstance(arg, torch.Tensor) else arg for arg in args] - kwargs = {k: v.to(target_dtype) if isinstance(v, torch.Tensor) else v for k, v in kwargs.items()} + if target_device is not None: + params['device'] = target_device + params['dtype'] = target_dtype + + args = list(args) + for j in (i for i, arg in enumerate(args) if isinstance(arg, torch.Tensor) and arg.dtype != target_dtype): + args[j] = args[j].to(**params) + args = tuple(args) + + for key in (k for k, v in kwargs.items() if isinstance(v, torch.Tensor) and v.dtype != target_dtype): + kwargs[key] = kwargs[key].to(**params) org_dtype = target_dtype for param in self.parameters(): @@ -165,37 +194,41 @@ def forward_wrapper(self, *args, **kwargs): break if org_dtype != target_dtype: - self.to(target_dtype) + self.to(**params) result = self.org_forward(*args, **kwargs) + if org_dtype != target_dtype: - self.to(org_dtype) + params['dtype'] = org_dtype + self.to(**params) if target_dtype != dtype_inference: + params['dtype'] = dtype_inference if isinstance(result, tuple): result = tuple( - i.to(dtype_inference) + i.to(**params) if isinstance(i, torch.Tensor) else i for i in result ) elif isinstance(result, torch.Tensor): - result = result.to(dtype_inference) + result = result.to(**params) return result return forward_wrapper @contextlib.contextmanager -def manual_cast(target_dtype): +def manual_cast(target_dtype, target_device=None): applied = False + for module_type in patch_module_list: if hasattr(module_type, "org_forward"): continue applied = True org_forward = module_type.forward if module_type == torch.nn.MultiheadAttention: - module_type.forward = manual_cast_forward(torch.float32) + module_type.forward = manual_cast_forward(torch.float32, target_device) else: - module_type.forward = manual_cast_forward(target_dtype) + module_type.forward = manual_cast_forward(target_dtype, target_device) module_type.org_forward = org_forward try: yield None @@ -207,26 +240,37 @@ def manual_cast(target_dtype): delattr(module_type, "org_forward") -def autocast(disable=False): +def autocast(disable=False, current_dtype=None, target_dtype=None, target_device=None): if disable: return contextlib.nullcontext() + if target_dtype is None: + target_dtype = dtype + if target_device is None: + target_device = device + if force_fp16: # No casting during inference if force_fp16 is enabled. # All tensor dtype conversion happens before inference. return contextlib.nullcontext() - if fp8 and device==cpu: + if fp8 and target_device==cpu: return torch.autocast("cpu", dtype=torch.bfloat16, enabled=True) if fp8 and dtype_inference == torch.float32: - return manual_cast(dtype) + return manual_cast(target_dtype, target_device) + + if target_dtype != dtype_inference: + return manual_cast(target_dtype, target_device) + + if current_dtype is not None and current_dtype != target_dtype: + return manual_cast(target_dtype, target_device) - if dtype == torch.float32 or dtype_inference == torch.float32: + if target_dtype == torch.float32 or dtype_inference == torch.float32: return contextlib.nullcontext() if has_xpu() or has_mps() or cuda_no_autocast(): - return manual_cast(dtype) + return manual_cast(target_dtype, target_device) return torch.autocast("cuda") From 4dc5c902d5e51239fbfd32efc0b5dec455480381 Mon Sep 17 00:00:00 2001 From: Won-Kyu Park Date: Thu, 5 Sep 2024 09:34:08 +0900 Subject: [PATCH 09/29] check Unet/VAE and load as is - check float8 unet dtype to save memory - check vae dtype --- modules/sd_models.py | 128 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 120 insertions(+), 8 deletions(-) diff --git a/modules/sd_models.py b/modules/sd_models.py index a0e7233b590..0e57b297918 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -412,6 +412,88 @@ def set_model_fields(model): model.latent_channels = 4 +def get_state_dict_dtype(state_dict): + # detect dtypes of state_dict + state_dict_dtype = {} + + known_prefixes = ("model.diffusion_model.", "first_stage_model.", "cond_stage_model.", "conditioner", "vae.", "text_encoders.") + + for k in state_dict.keys(): + found = [prefix for prefix in known_prefixes if k.startswith(prefix)] + if len(found) > 0: + prefix = found[0] + dtype = state_dict[k].dtype + dtypes = state_dict_dtype.get(prefix, {}) + if dtype in dtypes: + dtypes[dtype] += 1 + else: + dtypes[dtype] = 1 + state_dict_dtype[prefix] = dtypes + + for prefix in state_dict_dtype: + dtypes = state_dict_dtype[prefix] + # sort by count + state_dict_dtype[prefix] = dict(sorted(dtypes.items(), key=lambda item: item[1], reverse=True)) + + print("Detected dtypes:", state_dict_dtype) + return state_dict_dtype + + +def get_loadable_dtype(prefix="model.diffusion_model.", dtype=None, state_dict=None, state_dict_dtype=None, count=490): + if state_dict is not None: + state_dict_dtype = get_state_dict_dtype(state_dict) + + aliases = { + "FP8": "F8", + "FP16": "F16", + "FP32": "F32", + } + + loadables = { + "F8": (torch.float8_e4m3fn,), + "F16": (torch.float16,), + "F32": (torch.float32,), + "BF16": (torch.bfloat16,), + } + + if dtype is None: + # get the first dtype + if prefix in state_dict_dtype: + return list(state_dict_dtype[prefix])[0] + return None + + + if dtype in aliases: + dtype = aliases[dtype] + loadable = loadables[dtype] + + if prefix in state_dict_dtype: + dtypes = [d for d in state_dict_dtype[prefix].keys() if d in loadable] + if len(dtypes) > 0 and state_dict_dtype[prefix][dtypes[0]] >= count: + # mostly dtype weights. + return dtypes[0] + + return None + + +def get_vae_dtype(state_dict=None, state_dict_dtype=None): + if state_dict is not None: + state_dict_dtype = get_state_dict_dtype(state_dict) + + if state_dict_dtype is None: + raise ValueError("fail to get vae dtype") + + + vae_prefix = [prefix for prefix in ("vae.", "first_stage_model.") if prefix in state_dict_dtype][0] + + for dtype in state_dict_dtype[vae_prefix]: + if state_dict_dtype[vae_prefix][dtype] > 240 and dtype in (torch.float16, torch.float32, torch.bfloat16): + # vae items: 248 for SD1, SDXL 245 for flux + return dtype + + return None + + def load_model_weights(model, checkpoint_info: CheckpointInfo, state_dict, timer): sd_model_hash = checkpoint_info.calculate_shorthash() timer.record("calculate hash") @@ -442,6 +524,9 @@ def load_model_weights(model, checkpoint_info: CheckpointInfo, state_dict, timer if hasattr(model, "before_load_weights"): model.before_load_weights(state_dict) + # get all dtypes of state_dict + state_dict_dtype = get_state_dict_dtype(state_dict) + model.load_state_dict(state_dict, strict=False) timer.record("apply weights to model") @@ -467,7 +552,9 @@ def load_model_weights(model, checkpoint_info: CheckpointInfo, state_dict, timer model.to(memory_format=torch.channels_last) timer.record("apply channels_last") - if shared.cmd_opts.no_half: + unet_has_float = [f for f in state_dict_dtype["model.diffusion_model."].keys() if f in (torch.float16, torch.float32, torch.bfloat16)] + + if unet_has_float and shared.cmd_opts.no_half: model.float() model.alphas_cumprod_original = model.alphas_cumprod devices.dtype_unet = torch.float32 @@ -486,15 +573,29 @@ def load_model_weights(model, checkpoint_info: CheckpointInfo, state_dict, timer alphas_cumprod = model.alphas_cumprod model.alphas_cumprod = None - model.half() + + found_unet_dtype = get_loadable_dtype("model.diffusion_model.", state_dict_dtype=state_dict_dtype) + + if found_unet_dtype in (torch.float16, torch.float32, torch.bfloat16): + model.half() + elif found_unet_dtype in (torch.float8_e4m3fn,): + pass + else: + raise ValueError("fail to get a vaild UNet dtype") + model.alphas_cumprod = alphas_cumprod model.alphas_cumprod_original = alphas_cumprod model.first_stage_model = vae if depth_model: model.depth_model = depth_model - devices.dtype_unet = torch.float16 - timer.record("apply half()") + if found_unet_dtype in (torch.float16, torch.float32): + devices.dtype_unet = torch.float16 + timer.record("apply half()") + else: + print(f"load Unet {found_unet_dtype} as is ...") + devices.dtype_unet = found_unet_dtype + timer.record("load UNet") apply_alpha_schedule_override(model) @@ -504,7 +605,7 @@ def load_model_weights(model, checkpoint_info: CheckpointInfo, state_dict, timer if hasattr(module, 'fp16_bias'): del module.fp16_bias - if check_fp8(model): + if found_unet_dtype not in (torch.float8_e4m3fn,) and check_fp8(model): devices.fp8 = True first_stage = model.first_stage_model model.first_stage_model = None @@ -522,8 +623,16 @@ def load_model_weights(model, checkpoint_info: CheckpointInfo, state_dict, timer devices.unet_needs_upcast = shared.cmd_opts.upcast_sampling and devices.dtype == torch.float16 and devices.dtype_unet == torch.float16 - model.first_stage_model.to(devices.dtype_vae) - timer.record("apply dtype to VAE") + # check supported vae dtype + dtype_vae = get_vae_dtype(state_dict_dtype=state_dict_dtype) + if dtype_vae == torch.bfloat16 and dtype_vae in devices.supported_vae_dtypes: + devices.dtype_vae = torch.bfloat16 + print(f"VAE dtype {dtype_vae} detected. load as is.") + else: + devices.dtype_vae = torch.float16 + model.first_stage_model.to(devices.dtype_vae) + print(f"force VAE dtype {devices.dtype_vae}") + timer.record("apply dtype to VAE") # clean up cache if limit is reached while len(checkpoints_loaded) > shared.opts.sd_checkpoint_cache: @@ -821,6 +930,9 @@ def load_model(checkpoint_info=None, already_loaded_state_dict=None): print(f"Creating model from config: {checkpoint_config}") + # check loadable unet dtype before loading + loadable_unet_dtype = get_loadable_dtype("model.diffusion_model.", state_dict=state_dict) + sd_model = None try: with sd_disable_initialization.DisableInitialization(disable_clip=clip_is_included_into_sd or shared.cmd_opts.do_not_download_clip): @@ -846,7 +958,7 @@ def load_model(checkpoint_info=None, already_loaded_state_dict=None): weight_dtype_conversion = { 'first_stage_model': None, 'alphas_cumprod': None, - '': torch.float16, + '': torch.float16 if loadable_unet_dtype in (torch.float16, torch.float32, torch.bfloat16) else None, } with sd_disable_initialization.LoadStateDictOnMeta(state_dict, device=model_target_device(sd_model), weight_dtype_conversion=weight_dtype_conversion): From 90ff0520ebf2b15e5b86c1304c9d75c5a8fab8c4 Mon Sep 17 00:00:00 2001 From: Won-Kyu Park Date: Thu, 5 Sep 2024 09:18:09 +0900 Subject: [PATCH 10/29] patch reset_parameters --- modules/sd_hijack.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/modules/sd_hijack.py b/modules/sd_hijack.py index 0de83054186..1cefe050e45 100644 --- a/modules/sd_hijack.py +++ b/modules/sd_hijack.py @@ -1,4 +1,5 @@ import torch +import torch.nn as nn from torch.nn.functional import silu from types import MethodType @@ -45,6 +46,15 @@ sgm_original_forward = patches.patch(__file__, sgm.modules.diffusionmodules.openaimodel.UNetModel, "forward", sgm_patched_forward) +# patch reset_parameters() to speed up +for module in [ + nn.Linear, nn.Conv1d, nn.Conv2d, nn.Conv3d, + nn.GroupNorm, nn.LayerNorm, nn.Embedding, + nn.ConvTranspose1d, nn.ConvTranspose2d, + ]: + patches.patch(__name__, module, "reset_parameters", lambda x: None) + + def list_optimizers(): new_optimizers = script_callbacks.list_optimizers_callback() From 233e05f18863b8acc7ae708edd021151f8339716 Mon Sep 17 00:00:00 2001 From: Won-Kyu Park Date: Thu, 5 Sep 2024 09:19:04 +0900 Subject: [PATCH 11/29] preserve detected dtype_inference --- modules/shared_init.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/shared_init.py b/modules/shared_init.py index a6ad0433d6f..2d58c637450 100644 --- a/modules/shared_init.py +++ b/modules/shared_init.py @@ -29,7 +29,7 @@ def initialize(): devices.dtype = torch.float32 if cmd_opts.no_half else torch.float16 devices.dtype_vae = torch.float32 if cmd_opts.no_half or cmd_opts.no_half_vae else torch.float16 - devices.dtype_inference = torch.float32 if cmd_opts.precision == 'full' else devices.dtype + devices.dtype_inference = torch.float32 if cmd_opts.precision == 'full' else devices.dtype_inference if cmd_opts.precision == "half": msg = "--no-half and --no-half-vae conflict with --precision half" From 0124ec30ed4083c1d1eb3e82388a72435c070aab Mon Sep 17 00:00:00 2001 From: Won-Kyu Park Date: Thu, 5 Sep 2024 20:15:37 +0900 Subject: [PATCH 12/29] add diffusers weight mapping for flux lora * add QkvLinear class for Flux lora --- modules/models/flux/flux.py | 41 ++++++++++++++++++++------- modules/models/flux/modules/layers.py | 8 ++++-- 2 files changed, 37 insertions(+), 12 deletions(-) diff --git a/modules/models/flux/flux.py b/modules/models/flux/flux.py index dab2bd177b8..eb17a31c739 100644 --- a/modules/models/flux/flux.py +++ b/modules/models/flux/flux.py @@ -196,7 +196,8 @@ def __init__(self, shift=1.0, device=None, dtype=torch.float16, state_dict=None, self.diffusion_model = Flux(device=device, dtype=dtype, **params) self.model_sampling = ModelSamplingFlux() - self.depth = 19 + self.depth = params['depth'] + self.depth_single_block = params['depth_single_blocks'] def apply_model(self, x, sigma, c_crossattn=None, y=None): dtype = self.get_dtype() @@ -329,13 +330,33 @@ def fix_dimensions(self, width, height): return width // 16 * 16, height // 16 * 16 def diffusers_weight_mapping(self): + # https://github.com/kohya-ss/sd-scripts/blob/a61cf73a5cb5209c3f4d1a3688dd276a4dfd1ecb/networks/convert_flux_lora.py + # please see also https://github.com/huggingface/diffusers/blob/main/src/diffusers/loaders/lora_conversion_utils.py for i in range(self.model.depth): - yield f"transformer.transformer_blocks.{i}.attn.to_q", f"diffusion_model_joint_blocks_{i}_x_block_attn_qkv_q_proj" - yield f"transformer.transformer_blocks.{i}.attn.to_k", f"diffusion_model_joint_blocks_{i}_x_block_attn_qkv_k_proj" - yield f"transformer.transformer_blocks.{i}.attn.to_v", f"diffusion_model_joint_blocks_{i}_x_block_attn_qkv_v_proj" - yield f"transformer.transformer_blocks.{i}.attn.to_out.0", f"diffusion_model_joint_blocks_{i}_x_block_attn_proj" - - yield f"transformer.transformer_blocks.{i}.attn.add_q_proj", f"diffusion_model_joint_blocks_{i}_context_block.attn_qkv_q_proj" - yield f"transformer.transformer_blocks.{i}.attn.add_k_proj", f"diffusion_model_joint_blocks_{i}_context_block.attn_qkv_k_proj" - yield f"transformer.transformer_blocks.{i}.attn.add_v_proj", f"diffusion_model_joint_blocks_{i}_context_block.attn_qkv_v_proj" - yield f"transformer.transformer_blocks.{i}.attn.add_out_proj.0", f"diffusion_model_joint_blocks_{i}_context_block_attn_proj" + yield f"transformer.transformer_blocks.{i}.attn.add_k_proj", f"diffusion_model_double_blocks_{i}_txt_attn_qkv_k_proj" + yield f"transformer.transformer_blocks.{i}.attn.add_q_proj", f"diffusion_model_double_blocks_{i}_txt_attn_qkv_q_proj" + yield f"transformer.transformer_blocks.{i}.attn.add_v_proj", f"diffusion_model_double_blocks_{i}_txt_attn_qkv_v_proj" + + yield f"transformer.transformer_blocks.{i}.attn.to_add_out", f"diffusion_model_double_blocks_{i}_txt_attn_proj" + + yield f"transformer.transformer_blocks.{i}.attn.to_k", f"diffusion_model_double_blocks_{i}_img_attn_qkv_k_proj" + yield f"transformer.transformer_blocks.{i}.attn.to_q", f"diffusion_model_double_blocks_{i}_img_attn_qkv_q_proj" + yield f"transformer.transformer_blocks.{i}.attn.to_v", f"diffusion_model_double_blocks_{i}_img_attn_qkv_v_proj" + + yield f"transformer.transformer_blocks.{i}.attn.to_out.0", f"diffusion_model_double_blocks_{i}_img_attn_proj" + + yield f"transformer.transformer_blocks.{i}.ff.net.0.proj", f"diffusion_model_double_blocks_{i}_img_mlp_0" + yield f"transformer.transformer_blocks.{i}.ff.net.2", f"diffusion_model_double_blocks_{i}_img_mlp_2" + yield f"transformer.transformer_blocks.{i}.ff_context.net.0.proj", f"diffusion_model_double_blocks_{i}_txt_mlp_0" + yield f"transformer.transformer_blocks.{i}.ff_context.net.2", f"diffusion_model_double_blocks_{i}_txt_mlp_2" + yield f"transformer.transformer_blocks.{i}.norm1.linear", f"diffusion_model_double_blocks_{i}_img_mod_lin" + yield f"transformer.transformer_blocks.{i}.norm1_context.linear", f"diffusion_model_double_blocks_{i}_txt_mod_lin" + + for i in range(self.model.depth_single_block): + yield f"transformer.single_transformer_blocks.{i}.attn.to_q", f"diffusion_model_single_blocks_{i}_linear1_q_proj" + yield f"transformer.single_transformer_blocks.{i}.attn.to_k", f"diffusion_model_single_blocks_{i}_linear1_k_proj" + yield f"transformer.single_transformer_blocks.{i}.attn.to_v", f"diffusion_model_single_blocks_{i}_linear1_v_proj" + yield f"transformer.single_transformer_blocks.{i}.proj_mlp", f"diffusion_model_single_blocks_{i}_linear1_mlp_proj" + + yield f"transformer.single_transformer_blocks.{i}.proj_out", f"diffusion_model_single_blocks_{i}_linear2" + yield f"transformer.single_transformer_blocks.{i}.morm.linear", f"diffusion_model_single_blocks_{i}_modulation_lin" diff --git a/modules/models/flux/modules/layers.py b/modules/models/flux/modules/layers.py index aa830849eae..2202f5dcb35 100644 --- a/modules/models/flux/modules/layers.py +++ b/modules/models/flux/modules/layers.py @@ -82,13 +82,17 @@ def forward(self, q: Tensor, k: Tensor, v: Tensor) -> tuple: return q.to(v), k.to(v) +class QkvLinear(torch.nn.Linear): + pass + + class SelfAttention(nn.Module): def __init__(self, dim: int, num_heads: int = 8, qkv_bias: bool = False, dtype=None, device=None): super().__init__() self.num_heads = num_heads head_dim = dim // num_heads - self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias, dtype=dtype, device=device) + self.qkv = QkvLinear(dim, dim * 3, bias=qkv_bias, dtype=dtype, device=device) self.norm = QKNorm(head_dim, dtype=dtype, device=device) self.proj = nn.Linear(dim, dim, dtype=dtype, device=device) @@ -217,7 +221,7 @@ def __init__( self.mlp_hidden_dim = int(hidden_size * mlp_ratio) # qkv and mlp_in - self.linear1 = nn.Linear(hidden_size, hidden_size * 3 + self.mlp_hidden_dim, dtype=dtype, device=device) + self.linear1 = QkvLinear(hidden_size, hidden_size * 3 + self.mlp_hidden_dim, dtype=dtype, device=device) # proj and mlp_out self.linear2 = nn.Linear(hidden_size + self.mlp_hidden_dim, hidden_size, dtype=dtype, device=device) From a5e057d5cc93b81013c398757440b01260982575 Mon Sep 17 00:00:00 2001 From: Won-Kyu Park Date: Fri, 6 Sep 2024 14:07:31 +0900 Subject: [PATCH 13/29] fix for Lora flux --- extensions-builtin/Lora/network_lora.py | 3 ++- extensions-builtin/Lora/networks.py | 28 ++++++++++++++++++++++--- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/extensions-builtin/Lora/network_lora.py b/extensions-builtin/Lora/network_lora.py index a7a088949ea..2bc6af5d204 100644 --- a/extensions-builtin/Lora/network_lora.py +++ b/extensions-builtin/Lora/network_lora.py @@ -2,6 +2,7 @@ import lyco_helpers import modules.models.sd3.mmdit +import modules.models.flux.modules.layers import network from modules import devices @@ -37,7 +38,7 @@ def create_module(self, weights, key, none_ok=False): if weight is None and none_ok: return None - is_linear = type(self.sd_module) in [torch.nn.Linear, torch.nn.modules.linear.NonDynamicallyQuantizableLinear, torch.nn.MultiheadAttention, modules.models.sd3.mmdit.QkvLinear] + is_linear = type(self.sd_module) in [torch.nn.Linear, torch.nn.modules.linear.NonDynamicallyQuantizableLinear, torch.nn.MultiheadAttention, modules.models.sd3.mmdit.QkvLinear, modules.models.flux.modules.layers.QkvLinear ] is_conv = type(self.sd_module) in [torch.nn.Conv2d] if is_linear: diff --git a/extensions-builtin/Lora/networks.py b/extensions-builtin/Lora/networks.py index 67f9abe2a37..22e0e33591f 100644 --- a/extensions-builtin/Lora/networks.py +++ b/extensions-builtin/Lora/networks.py @@ -37,7 +37,7 @@ re_digits = re.compile(r"\d+") -re_x_proj = re.compile(r"(.*)_([qkv]_proj)$") +re_x_proj = re.compile(r"(.*)_((?:[qkv]|mlp)_proj)$") re_compiled = {} suffix_conversion = { @@ -455,7 +455,7 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn for net in loaded_networks: module = net.modules.get(network_layer_name, None) - if module is not None and hasattr(self, 'weight') and not isinstance(module, modules.models.sd3.mmdit.QkvLinear): + if module is not None and hasattr(self, 'weight') and not all(isinstance(module, linear) for linear in (modules.models.sd3.mmdit.QkvLinear, modules.models.flux.modules.layers.QkvLinear)): try: with torch.no_grad(): if getattr(self, 'fp16_weight', None) is None: @@ -515,7 +515,9 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn continue - if isinstance(self, modules.models.sd3.mmdit.QkvLinear) and module_q and module_k and module_v: + module_mlp = net.modules.get(network_layer_name + "_mlp_proj", None) + + if any(isinstance(self, linear) for linear in (modules.models.sd3.mmdit.QkvLinear, modules.models.flux.modules.layers.QkvLinear)) and module_q and module_k and module_v and module_mlp is None: try: with torch.no_grad(): # Send "real" orig_weight into MHA's lora module @@ -526,6 +528,26 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn del qw, kw, vw updown_qkv = torch.vstack([updown_q, updown_k, updown_v]) self.weight += updown_qkv + del updown_qkv + + except RuntimeError as e: + logging.debug(f"Network {net.name} layer {network_layer_name}: {e}") + extra_network_lora.errors[net.name] = extra_network_lora.errors.get(net.name, 0) + 1 + + continue + + if any(isinstance(self, linear) for linear in (modules.models.flux.modules.layers.QkvLinear,)) and module_q and module_k and module_v and module_mlp: + try: + with torch.no_grad(): + qw, kw, vw, mlp = torch.tensor_split(self.weight, (3072, 6144, 9216,), 0) + updown_q, _ = module_q.calc_updown(qw) + updown_k, _ = module_k.calc_updown(kw) + updown_v, _ = module_v.calc_updown(vw) + updown_mlp, _ = module_v.calc_updown(mlp) + del qw, kw, vw, mlp + updown_qkv_mlp = torch.vstack([updown_q, updown_k, updown_v, updown_mlp]) + self.weight += updown_qkv_mlp + del updown_qkv_mlp except RuntimeError as e: logging.debug(f"Network {net.name} layer {network_layer_name}: {e}") From 45a3dae48d350dd451a55338e0a5eab356342b31 Mon Sep 17 00:00:00 2001 From: Won-Kyu Park Date: Sat, 7 Sep 2024 01:06:05 +0900 Subject: [PATCH 14/29] misc fixes to support float8 dtype_unet * devices.dtype_unet, dtype_vae could be considered as storage dtypes * use devices.dtype_inference as computational dtype * misc fixes to support float8 unet storage --- modules/processing.py | 2 +- modules/sd_hijack_unet.py | 14 +++++++------- modules/sd_samplers_common.py | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/modules/processing.py b/modules/processing.py index 7535b56e18c..28003eacb94 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -984,7 +984,7 @@ def process_images_inner(p: StableDiffusionProcessing) -> Processed: sd_models.apply_alpha_schedule_override(p.sd_model, p) - with devices.without_autocast() if devices.unet_needs_upcast else devices.autocast(): + with devices.without_autocast() if devices.unet_needs_upcast else devices.autocast(target_dtype=devices.dtype_inference, current_dtype=devices.dtype_unet): samples_ddim = p.sample(conditioning=p.c, unconditional_conditioning=p.uc, seeds=p.seeds, subseeds=p.subseeds, subseed_strength=p.subseed_strength, prompts=p.prompts) if p.scripts is not None: diff --git a/modules/sd_hijack_unet.py b/modules/sd_hijack_unet.py index b4f03b138a4..842030be849 100644 --- a/modules/sd_hijack_unet.py +++ b/modules/sd_hijack_unet.py @@ -42,12 +42,12 @@ def apply_model(orig_func, self, x_noisy, t, cond, **kwargs): if isinstance(cond, dict): for y in cond.keys(): if isinstance(cond[y], list): - cond[y] = [x.to(devices.dtype_unet) if isinstance(x, torch.Tensor) else x for x in cond[y]] + cond[y] = [x.to(devices.dtype_inference) if isinstance(x, torch.Tensor) else x for x in cond[y]] else: - cond[y] = cond[y].to(devices.dtype_unet) if isinstance(cond[y], torch.Tensor) else cond[y] + cond[y] = cond[y].to(devices.dtype_inference) if isinstance(cond[y], torch.Tensor) else cond[y] with devices.autocast(): - result = orig_func(self, x_noisy.to(devices.dtype_unet), t.to(devices.dtype_unet), cond, **kwargs) + result = orig_func(self, x_noisy.to(devices.dtype_inference), t.to(devices.dtype_inference), cond, **kwargs) if devices.unet_needs_upcast: return result.float() else: @@ -107,7 +107,7 @@ def __init__(self, *args, **kwargs): torch.nn.GELU.__init__(self, *args, **kwargs) def forward(self, x): if devices.unet_needs_upcast: - return torch.nn.GELU.forward(self.float(), x.float()).to(devices.dtype_unet) + return torch.nn.GELU.forward(self.float(), x.float()).to(devices.dtype_inference) else: return torch.nn.GELU.forward(self, x) @@ -125,11 +125,11 @@ def hijack_ddpm_edit(): CondFunc('ldm.models.diffusion.ddpm.LatentDiffusion.apply_model', apply_model, unet_needs_upcast) CondFunc('ldm.modules.diffusionmodules.openaimodel.timestep_embedding', timestep_embedding) CondFunc('ldm.modules.attention.SpatialTransformer.forward', spatial_transformer_forward) -CondFunc('ldm.modules.diffusionmodules.openaimodel.timestep_embedding', lambda orig_func, timesteps, *args, **kwargs: orig_func(timesteps, *args, **kwargs).to(torch.float32 if timesteps.dtype == torch.int64 else devices.dtype_unet), unet_needs_upcast) +CondFunc('ldm.modules.diffusionmodules.openaimodel.timestep_embedding', lambda orig_func, timesteps, *args, **kwargs: orig_func(timesteps, *args, **kwargs).to(torch.float32 if timesteps.dtype == torch.int64 else devices.dtype_inference), unet_needs_upcast) if version.parse(torch.__version__) <= version.parse("1.13.2") or torch.cuda.is_available(): CondFunc('ldm.modules.diffusionmodules.util.GroupNorm32.forward', lambda orig_func, self, *args, **kwargs: orig_func(self.float(), *args, **kwargs), unet_needs_upcast) - CondFunc('ldm.modules.attention.GEGLU.forward', lambda orig_func, self, x: orig_func(self.float(), x.float()).to(devices.dtype_unet), unet_needs_upcast) + CondFunc('ldm.modules.attention.GEGLU.forward', lambda orig_func, self, x: orig_func(self.float(), x.float()).to(devices.dtype_inference), unet_needs_upcast) CondFunc('open_clip.transformer.ResidualAttentionBlock.__init__', lambda orig_func, *args, **kwargs: kwargs.update({'act_layer': GELUHijack}) and False or orig_func(*args, **kwargs), lambda _, *args, **kwargs: kwargs.get('act_layer') is None or kwargs['act_layer'] == torch.nn.GELU) first_stage_cond = lambda _, self, *args, **kwargs: devices.unet_needs_upcast and self.model.diffusion_model.dtype == torch.float16 @@ -146,7 +146,7 @@ def timestep_embedding_cast_result(orig_func, timesteps, *args, **kwargs): if devices.unet_needs_upcast and timesteps.dtype == torch.int64: dtype = torch.float32 else: - dtype = devices.dtype_unet + dtype = devices.dtype_inference return orig_func(timesteps, *args, **kwargs).to(dtype=dtype) diff --git a/modules/sd_samplers_common.py b/modules/sd_samplers_common.py index c060cccb24b..28b8bd820b7 100644 --- a/modules/sd_samplers_common.py +++ b/modules/sd_samplers_common.py @@ -54,7 +54,7 @@ def samples_to_images_tensor(sample, approximation=None, model=None): else: if model is None: model = shared.sd_model - with torch.no_grad(), devices.without_autocast(): # fixes an issue with unstable VAEs that are flaky even in fp32 + with torch.no_grad(), devices.manual_cast(devices.dtype_vae): # fixes an issue with unstable VAEs that are flaky even in fp32 x_sample = model.decode_first_stage(sample.to(model.first_stage_model.dtype)) return x_sample @@ -64,7 +64,7 @@ def single_sample_to_image(sample, approximation=None): x_sample = samples_to_images_tensor(sample.unsqueeze(0), approximation)[0] * 0.5 + 0.5 x_sample = torch.clamp(x_sample, min=0.0, max=1.0) - x_sample = 255. * np.moveaxis(x_sample.cpu().numpy(), 0, 2) + x_sample = 255. * np.moveaxis(x_sample.to(dtype=devices.dtype).cpu().numpy(), 0, 2) x_sample = x_sample.astype(np.uint8) return Image.fromarray(x_sample) From 673e6650f351db211b2a917c3a8e1357f3beffb1 Mon Sep 17 00:00:00 2001 From: Won-Kyu Park Date: Sat, 7 Sep 2024 12:31:00 +0900 Subject: [PATCH 15/29] add shared.opts.lora_without_backup_weight option to reduce ram usage --- extensions-builtin/Lora/networks.py | 12 +++++++++++- modules/shared_options.py | 1 + 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/extensions-builtin/Lora/networks.py b/extensions-builtin/Lora/networks.py index 22e0e33591f..75c12c6c400 100644 --- a/extensions-builtin/Lora/networks.py +++ b/extensions-builtin/Lora/networks.py @@ -377,6 +377,8 @@ def store_weights_backup(weight): if weight is None: return None + if shared.opts.lora_without_backup_weight: + return True return weight.to(devices.cpu, copy=True) @@ -395,6 +397,9 @@ def network_restore_weights_from_backup(self: Union[torch.nn.Conv2d, torch.nn.Li if weights_backup is None and bias_backup is None: return + if weights_backup is True or weights_backup == (True, True): # fake backup + return + if weights_backup is not None: if isinstance(self, torch.nn.MultiheadAttention): restore_weights_backup(self, 'in_proj_weight', weights_backup[0]) @@ -561,7 +566,12 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn logging.debug(f"Network {net.name} layer {network_layer_name}: couldn't find supported operation") extra_network_lora.errors[net.name] = extra_network_lora.errors.get(net.name, 0) + 1 - self.network_current_names = wanted_names + + if weights_backup is True or weights_backup == (True, True): # fake backup + setattr(self, "network_weights_backup", None) + setattr(self, "network_bias_backup", None) + else: + self.network_current_names = wanted_names def network_forward(org_module, input, original_forward): diff --git a/modules/shared_options.py b/modules/shared_options.py index 01d8f2065ab..3f78c26d968 100644 --- a/modules/shared_options.py +++ b/modules/shared_options.py @@ -245,6 +245,7 @@ "batch_cond_uncond": OptionInfo(True, "Batch cond/uncond").info("do both conditional and unconditional denoising in one batch; uses a bit more VRAM during sampling, but improves speed; previously this was controlled by --always-batch-cond-uncond commandline argument"), "fp8_storage": OptionInfo("Disable", "FP8 weight", gr.Radio, {"choices": ["Disable", "Enable for SDXL", "Enable"]}).info("Use FP8 to store Linear/Conv layers' weight. Require pytorch>=2.1.0."), "cache_fp16_weight": OptionInfo(False, "Cache FP16 weight for LoRA").info("Cache fp16 weight when enabling FP8, will increase the quality of LoRA. Use more system ram."), + "lora_without_backup_weight": OptionInfo(False, "LoRA without backup weights").info("LoRA without backup weights to save RAM."), })) options_templates.update(options_section(('compatibility', "Compatibility", "sd"), { From 0e627de265b5c3af3fbf6dac9e1b4fb2b0065dc1 Mon Sep 17 00:00:00 2001 From: Won-Kyu Park Date: Sat, 7 Sep 2024 12:23:03 +0900 Subject: [PATCH 16/29] support copy option to reduce ram usage --- extensions-builtin/Lora/networks.py | 4 ++-- modules/devices.py | 30 ++++++++++++++++++++--------- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/extensions-builtin/Lora/networks.py b/extensions-builtin/Lora/networks.py index 75c12c6c400..46c9ac1ffae 100644 --- a/extensions-builtin/Lora/networks.py +++ b/extensions-builtin/Lora/networks.py @@ -568,8 +568,8 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn if weights_backup is True or weights_backup == (True, True): # fake backup - setattr(self, "network_weights_backup", None) - setattr(self, "network_bias_backup", None) + self.network_weights_backup = None + self.network_bias_backup = None else: self.network_current_names = wanted_names diff --git a/modules/devices.py b/modules/devices.py index 556e72d2eae..866b6ab161b 100644 --- a/modules/devices.py +++ b/modules/devices.py @@ -1,5 +1,6 @@ import sys import contextlib +from copy import deepcopy from functools import lru_cache import torch @@ -169,7 +170,7 @@ def cond_cast_float(input): ] -def manual_cast_forward(target_dtype, target_device=None): +def manual_cast_forward(target_dtype, target_device=None, copy=False): params = dict() if supports_non_blocking(): params['non_blocking'] = True @@ -193,13 +194,22 @@ def forward_wrapper(self, *args, **kwargs): org_dtype = param.dtype break - if org_dtype != target_dtype: - self.to(**params) - result = self.org_forward(*args, **kwargs) + if copy: + copied = deepcopy(self) + if org_dtype != target_dtype: + copied.to(**params) - if org_dtype != target_dtype: - params['dtype'] = org_dtype - self.to(**params) + result = copied.org_forward(*args, **kwargs) + del copied + else: + if org_dtype != target_dtype: + self.to(**params) + + result = self.org_forward(*args, **kwargs) + + if org_dtype != target_dtype: + params['dtype'] = org_dtype + self.to(**params) if target_dtype != dtype_inference: params['dtype'] = dtype_inference @@ -220,15 +230,17 @@ def forward_wrapper(self, *args, **kwargs): def manual_cast(target_dtype, target_device=None): applied = False + copy = shared.opts.lora_without_backup_weight + for module_type in patch_module_list: if hasattr(module_type, "org_forward"): continue applied = True org_forward = module_type.forward if module_type == torch.nn.MultiheadAttention: - module_type.forward = manual_cast_forward(torch.float32, target_device) + module_type.forward = manual_cast_forward(torch.float32, target_device, copy) else: - module_type.forward = manual_cast_forward(target_dtype, target_device) + module_type.forward = manual_cast_forward(target_dtype, target_device, copy) module_type.org_forward = org_forward try: yield None From c64dd9a02764821c0f4800e18ac6e3867bad0ec4 Mon Sep 17 00:00:00 2001 From: Won-Kyu Park Date: Sun, 8 Sep 2024 15:07:05 +0900 Subject: [PATCH 17/29] optimize * replace rearrange to view https://github.com/AUTOMATIC1111/stable-diffusion-webui/pull/15804 * see also https://github.com/lllyasviel/stable-diffusion-webui-forge/commit/79adfa8998c7b6c448d188cc469eda3540f17b53 * conditional use torch.rms_norm for torch 2.4 * fix RMSNorm() for clear: use torch.ones() --- modules/models/flux/math.py | 13 ++++++--- modules/models/flux/modules/layers.py | 39 ++++++++++++++++++++------- 2 files changed, 38 insertions(+), 14 deletions(-) diff --git a/modules/models/flux/math.py b/modules/models/flux/math.py index 4ad99818e2c..6fbafbeead6 100644 --- a/modules/models/flux/math.py +++ b/modules/models/flux/math.py @@ -1,5 +1,4 @@ import torch -from einops import rearrange from torch import Tensor @@ -7,7 +6,9 @@ def attention(q: Tensor, k: Tensor, v: Tensor, pe: Tensor) -> Tensor: q, k = apply_rope(q, k, pe) x = torch.nn.functional.scaled_dot_product_attention(q, k, v) - x = rearrange(x, "B H L D -> B L (H D)") + #x = rearrange(x, "B H L D -> B L (H D)") + B, H, L, D = x.shape + x = x.permute(0, 2, 1, 3).contiguous().view(B, L, H * D) return x @@ -17,9 +18,13 @@ def rope(pos: Tensor, dim: int, theta: int) -> Tensor: scale = torch.arange(0, dim, 2, dtype=torch.float64, device=pos.device) / dim omega = 1.0 / (theta**scale) out = torch.einsum("...n,d->...nd", pos, omega) + #out = pos.unsqueeze(-1) * omega.unsqueeze(0) + out = torch.stack([torch.cos(out), -torch.sin(out), torch.sin(out), torch.cos(out)], dim=-1) - out = rearrange(out, "b n d (i j) -> b n d i j", i=2, j=2) - return out.float() + #out = rearrange(out, "b n d (i j) -> b n d i j", i=2, j=2) + b, n, d, _ = out.shape + out = out.view(b, n, d, 2, 2) + return out.to(dtype=torch.float32, device=pos.device) def apply_rope(xq: Tensor, xk: Tensor, freqs_cis: Tensor): diff --git a/modules/models/flux/modules/layers.py b/modules/models/flux/modules/layers.py index 2202f5dcb35..3ffaeb037d3 100644 --- a/modules/models/flux/modules/layers.py +++ b/modules/models/flux/modules/layers.py @@ -2,7 +2,6 @@ from dataclasses import dataclass import torch -from einops import rearrange from torch import Tensor, nn from ..math import attention, rope @@ -58,16 +57,28 @@ def forward(self, x: Tensor) -> Tensor: return self.out_layer(self.silu(self.in_layer(x))) +def rms_norm(x, normalized_shape, weight, eps): + if hasattr(torch, 'rms_norm'): # torch 2.4 + return torch.rms_norm(x, normalized_shape, weight, eps) + + if x.dtype in [torch.bfloat16, torch.float32]: + n = torch.rsqrt(torch.mean(x ** 2, dim=-1, keepdim=True) + eps) * weight + else: + n = torch.rsqrt(torch.mean(x.float() ** 2, dim=-1, keepdim=True) + eps).to(x.dtype) * weight + return x * n + + class RMSNorm(torch.nn.Module): def __init__(self, dim: int, dtype=None, device=None): super().__init__() - self.scale = nn.Parameter(torch.empty((dim), dtype=dtype, device=device)) + self.scale = nn.Parameter(torch.ones((dim), dtype=dtype, device=device)) + self.normalized_shape = [dim] def forward(self, x: Tensor): - x_dtype = x.dtype - x = x.float() - rrms = torch.rsqrt(torch.mean(x**2, dim=-1, keepdim=True) + 1e-6) - return (x * rrms).to(dtype=x_dtype) * self.scale + if self.scale.dtype != x.dtype: + self.scale = nn.Parameter(self.scale.to(dtype=x.dtype), requires_grad=x.requires_grad) + + return rms_norm(x, self.normalized_shape, self.scale, 1e-6) class QKNorm(torch.nn.Module): @@ -98,7 +109,9 @@ def __init__(self, dim: int, num_heads: int = 8, qkv_bias: bool = False, dtype=N def forward(self, x: Tensor, pe: Tensor) -> Tensor: qkv = self.qkv(x) - q, k, v = rearrange(qkv, "B L (K H D) -> K B H L D", K=3, H=self.num_heads) + #q, k, v = rearrange(qkv, "B L (K H D) -> K B H L D", K=3, H=self.num_heads) + B, L, _ = qkv.shape + q, k, v = qkv.view(B, L, 3, self.num_heads, -1).permute(2, 0, 3, 1, 4) q, k = self.norm(q, k, v) x = attention(q, k, v, pe=pe) x = self.proj(x) @@ -165,14 +178,18 @@ def forward(self, img: Tensor, txt: Tensor, vec: Tensor, pe: Tensor): img_modulated = self.img_norm1(img) img_modulated = (1 + img_mod1.scale) * img_modulated + img_mod1.shift img_qkv = self.img_attn.qkv(img_modulated) - img_q, img_k, img_v = rearrange(img_qkv, "B L (K H D) -> K B H L D", K=3, H=self.num_heads) + #img_q, img_k, img_v = rearrange(img_qkv, "B L (K H D) -> K B H L D", K=3, H=self.num_heads) + B, L, _ = img_qkv.shape + img_q, img_k, img_v = img_qkv.view(B, L, 3, self.num_heads, -1).permute(2, 0, 3, 1, 4) img_q, img_k = self.img_attn.norm(img_q, img_k, img_v) # prepare txt for attention txt_modulated = self.txt_norm1(txt) txt_modulated = (1 + txt_mod1.scale) * txt_modulated + txt_mod1.shift txt_qkv = self.txt_attn.qkv(txt_modulated) - txt_q, txt_k, txt_v = rearrange(txt_qkv, "B L (K H D) -> K B H L D", K=3, H=self.num_heads) + #txt_q, txt_k, txt_v = rearrange(txt_qkv, "B L (K H D) -> K B H L D", K=3, H=self.num_heads) + B, L, _ = txt_qkv.shape + txt_q, txt_k, txt_v = txt_qkv.view(B, L, 3, self.num_heads, -1).permute(2, 0, 3, 1, 4) txt_q, txt_k = self.txt_attn.norm(txt_q, txt_k, txt_v) # run actual attention @@ -238,7 +255,9 @@ def forward(self, x: Tensor, vec: Tensor, pe: Tensor) -> Tensor: x_mod = (1 + mod.scale) * self.pre_norm(x) + mod.shift qkv, mlp = torch.split(self.linear1(x_mod), [3 * self.hidden_size, self.mlp_hidden_dim], dim=-1) - q, k, v = rearrange(qkv, "B L (K H D) -> K B H L D", K=3, H=self.num_heads) + #q, k, v = rearrange(qkv, "B L (K H D) -> K B H L D", K=3, H=self.num_heads) + B, L, _ = qkv.shape + q, k, v = qkv.view(B, L, 3, self.num_heads, -1).permute(2, 0, 3, 1, 4) q, k = self.norm(q, k, v) # compute attention From c2c44c60ddfcad5d329c39bbf41fdfaba5dd0370 Mon Sep 17 00:00:00 2001 From: Won-Kyu Park Date: Sun, 8 Sep 2024 15:14:01 +0900 Subject: [PATCH 18/29] reduce memort usage --- modules/models/flux/model.py | 4 ++++ modules/models/flux/modules/layers.py | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/modules/models/flux/model.py b/modules/models/flux/model.py index c873978277d..ece5674a6fc 100644 --- a/modules/models/flux/model.py +++ b/modules/models/flux/model.py @@ -107,6 +107,7 @@ def forward_orig( ids = torch.cat((txt_ids, img_ids), dim=1) pe = self.pe_embedder(ids) + del ids for block in self.double_blocks: img, txt = block(img=img, txt=txt, vec=vec, pe=pe) @@ -115,9 +116,11 @@ def forward_orig( for block in self.single_blocks: img = block(img, vec=vec, pe=pe) img = img[:, txt.shape[1] :, ...] + del pe, txt if self.final_layer: img = self.final_layer(img, vec) # (N, T, patch_size ** 2 * out_channels) + del vec return img def forward(self, x, timestep, context, y, guidance, **kwargs): @@ -144,4 +147,5 @@ def pad_to_patch_size(img, patch_size=(2, 2), padding_mode="circular"): txt_ids = torch.zeros((bs, context.shape[1], 3), device=x.device, dtype=x.dtype) out = self.forward_orig(img, img_ids, context, txt_ids, timestep, y, guidance) + del img, img_ids, txt_ids return rearrange(out, "b (h w) (c ph pw) -> b c (h ph) (w pw)", h=h_len, w=w_len, ph=2, pw=2)[:,:,:h,:w] diff --git a/modules/models/flux/modules/layers.py b/modules/models/flux/modules/layers.py index 3ffaeb037d3..9849e2d92e5 100644 --- a/modules/models/flux/modules/layers.py +++ b/modules/models/flux/modules/layers.py @@ -38,7 +38,9 @@ def timestep_embedding(t: Tensor, dim, max_period=10000, time_factor: float = 10 freqs = torch.exp(-math.log(max_period) * torch.arange(start=0, end=half, dtype=torch.float32, device=t.device) / half) args = t[:, None].float() * freqs[None] + del freqs embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1) + del args if dim % 2: embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1) if torch.is_floating_point(t): @@ -178,34 +180,45 @@ def forward(self, img: Tensor, txt: Tensor, vec: Tensor, pe: Tensor): img_modulated = self.img_norm1(img) img_modulated = (1 + img_mod1.scale) * img_modulated + img_mod1.shift img_qkv = self.img_attn.qkv(img_modulated) + del img_modulated #img_q, img_k, img_v = rearrange(img_qkv, "B L (K H D) -> K B H L D", K=3, H=self.num_heads) B, L, _ = img_qkv.shape img_q, img_k, img_v = img_qkv.view(B, L, 3, self.num_heads, -1).permute(2, 0, 3, 1, 4) + del img_qkv img_q, img_k = self.img_attn.norm(img_q, img_k, img_v) # prepare txt for attention txt_modulated = self.txt_norm1(txt) txt_modulated = (1 + txt_mod1.scale) * txt_modulated + txt_mod1.shift txt_qkv = self.txt_attn.qkv(txt_modulated) + del txt_modulated #txt_q, txt_k, txt_v = rearrange(txt_qkv, "B L (K H D) -> K B H L D", K=3, H=self.num_heads) B, L, _ = txt_qkv.shape txt_q, txt_k, txt_v = txt_qkv.view(B, L, 3, self.num_heads, -1).permute(2, 0, 3, 1, 4) + del txt_qkv txt_q, txt_k = self.txt_attn.norm(txt_q, txt_k, txt_v) # run actual attention q = torch.cat((txt_q, img_q), dim=2) k = torch.cat((txt_k, img_k), dim=2) v = torch.cat((txt_v, img_v), dim=2) + del txt_q, img_q + del txt_k, img_k + del txt_v, img_v attn = attention(q, k, v, pe=pe) + del q, k, v, pe txt_attn, img_attn = attn[:, : txt.shape[1]], attn[:, txt.shape[1] :] + del attn # calculate the img bloks img = img + img_mod1.gate * self.img_attn.proj(img_attn) img = img + img_mod2.gate * self.img_mlp((1 + img_mod2.scale) * self.img_norm2(img) + img_mod2.shift) + del img_attn # calculate the txt bloks txt = txt + txt_mod1.gate * self.txt_attn.proj(txt_attn) + del txt_attn txt = txt + txt_mod2.gate * self.txt_mlp((1 + txt_mod2.scale) * self.txt_norm2(txt) + txt_mod2.shift) if txt.dtype == torch.float16: @@ -254,17 +267,23 @@ def forward(self, x: Tensor, vec: Tensor, pe: Tensor) -> Tensor: mod, _ = self.modulation(vec) x_mod = (1 + mod.scale) * self.pre_norm(x) + mod.shift qkv, mlp = torch.split(self.linear1(x_mod), [3 * self.hidden_size, self.mlp_hidden_dim], dim=-1) + del x_mod #q, k, v = rearrange(qkv, "B L (K H D) -> K B H L D", K=3, H=self.num_heads) B, L, _ = qkv.shape q, k, v = qkv.view(B, L, 3, self.num_heads, -1).permute(2, 0, 3, 1, 4) + del qkv q, k = self.norm(q, k, v) # compute attention attn = attention(q, k, v, pe=pe) + del q, k, v, pe # compute activation in mlp stream, cat again and run second linear layer output = self.linear2(torch.cat((attn, self.mlp_act(mlp)), 2)) + del attn, mlp x += mod.gate * output + del mod, output + if x.dtype == torch.float16: x = torch.nan_to_num(x, nan=0.0, posinf=65504, neginf=-65504) return x @@ -280,5 +299,6 @@ def __init__(self, hidden_size: int, patch_size: int, out_channels: int, dtype=N def forward(self, x: Tensor, vec: Tensor) -> Tensor: shift, scale = self.adaLN_modulation(vec).chunk(2, dim=1) x = (1 + scale[:, None, :]) * self.norm_final(x) + shift[:, None, :] + del shift, scale x = self.linear(x) return x From 9a5088057dbbddbd0f4a0efc32dcbb07cf181756 Mon Sep 17 00:00:00 2001 From: Won-Kyu Park Date: Sun, 8 Sep 2024 20:21:48 +0900 Subject: [PATCH 19/29] vae fix for flux --- modules/models/flux/flux.py | 4 +++- modules/sd_models.py | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/modules/models/flux/flux.py b/modules/models/flux/flux.py index eb17a31c739..bb96d7ecbd7 100644 --- a/modules/models/flux/flux.py +++ b/modules/models/flux/flux.py @@ -270,7 +270,8 @@ def __init__(self, state_dict, use_ema=False): with torch.no_grad(): self.model = BaseModel(state_dict=state_dict, prefix=diffusion_model_prefix, device="cpu", dtype=devices.dtype) self.first_stage_model = SDVAE(device="cpu", dtype=devices.dtype_vae) - self.first_stage_model.dtype = self.model.diffusion_model.dtype + self.first_stage_model.dtype = devices.dtype_vae + self.vae = self.first_stage_model # real vae self.alphas_cumprod = 1 / (self.model.model_sampling.sigmas ** 2 + 1) @@ -319,6 +320,7 @@ def create_denoiser(self): def medvram_fields(self): return [ (self, 'first_stage_model'), + (self, 'vae'), (self, 'text_encoders'), (self, 'model'), ] diff --git a/modules/sd_models.py b/modules/sd_models.py index 0e57b297918..7404c22fb9b 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -957,6 +957,8 @@ def load_model(checkpoint_info=None, already_loaded_state_dict=None): else: weight_dtype_conversion = { 'first_stage_model': None, + 'text_encoders': None, + 'vae': None, 'alphas_cumprod': None, '': torch.float16 if loadable_unet_dtype in (torch.float16, torch.float32, torch.bfloat16) else None, } From c5d84c4b5afc1f8238c557e6a3a46180f7a60a71 Mon Sep 17 00:00:00 2001 From: Won-Kyu Park Date: Sun, 8 Sep 2024 23:13:43 +0900 Subject: [PATCH 20/29] check vae/ text_encoders dtype and use as intended --- modules/sd_models.py | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/modules/sd_models.py b/modules/sd_models.py index 7404c22fb9b..b4a2aeec6b8 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -553,6 +553,8 @@ def load_model_weights(model, checkpoint_info: CheckpointInfo, state_dict, timer timer.record("apply channels_last") unet_has_float = [f for f in state_dict_dtype["model.diffusion_model."].keys() if f in (torch.float16, torch.float32, torch.bfloat16)] + # check dtype of vae + dtype_vae = get_vae_dtype(state_dict_dtype=state_dict_dtype) if unet_has_float and shared.cmd_opts.no_half: model.float() @@ -564,8 +566,11 @@ def load_model_weights(model, checkpoint_info: CheckpointInfo, state_dict, timer vae = model.first_stage_model depth_model = getattr(model, 'depth_model', None) + if dtype_vae == torch.bfloat16 and dtype_vae in devices.supported_vae_dtypes: + # preserve bfloat16 if it supported + model.first_stage_model = None # with --no-half-vae, remove VAE from model when doing half() to prevent its weights from being converted to float16 - if shared.cmd_opts.no_half_vae: + elif shared.cmd_opts.no_half_vae: model.first_stage_model = None # with --upcast-sampling, don't convert the depth model weights to float16 if shared.cmd_opts.upcast_sampling and depth_model: @@ -607,8 +612,16 @@ def load_model_weights(model, checkpoint_info: CheckpointInfo, state_dict, timer if found_unet_dtype not in (torch.float8_e4m3fn,) and check_fp8(model): devices.fp8 = True + + # do not convert vae, text_encoders.clip_l, clip_g, t5xxl first_stage = model.first_stage_model model.first_stage_model = None + vae = getattr(model, 'vae', None) + if vae is not None: + model.vae = None + text_encoders = getattr(model, 'text_encoders', None) + if text_encoders is not None: + model.text_encoders = None for module in model.modules(): if isinstance(module, (torch.nn.Conv2d, torch.nn.Linear)): if shared.opts.cache_fp16_weight: @@ -616,6 +629,10 @@ def load_model_weights(model, checkpoint_info: CheckpointInfo, state_dict, timer if module.bias is not None: module.fp16_bias = module.bias.data.clone().cpu().half() module.to(torch.float8_e4m3fn) + if text_encoders is not None: + model.text_encoders = text_encoders + if vae is not None: + model.vae = vae model.first_stage_model = first_stage timer.record("apply fp8") else: @@ -930,8 +947,17 @@ def load_model(checkpoint_info=None, already_loaded_state_dict=None): print(f"Creating model from config: {checkpoint_config}") + # get all dtypes of state_dict + state_dict_dtype = get_state_dict_dtype(state_dict) + # check loadable unet dtype before loading - loadable_unet_dtype = get_loadable_dtype("model.diffusion_model.", state_dict=state_dict) + loadable_unet_dtype = get_loadable_dtype("model.diffusion_model.", state_dict_dtype=state_dict_dtype) + + # check dtype of vae + dtype_vae = get_vae_dtype(state_dict_dtype=state_dict_dtype) + if dtype_vae == torch.bfloat16 and dtype_vae in devices.supported_vae_dtypes: + devices.dtype_vae = torch.bfloat16 + print(f"VAE dtype {dtype_vae} detected.") sd_model = None try: From f7444623b255c4721b9836f2a67e43160c20c689 Mon Sep 17 00:00:00 2001 From: Won-Kyu Park Date: Tue, 10 Sep 2024 19:05:41 +0900 Subject: [PATCH 21/29] minor update --- modules/models/flux/flux.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/models/flux/flux.py b/modules/models/flux/flux.py index bb96d7ecbd7..cc91f123d2d 100644 --- a/modules/models/flux/flux.py +++ b/modules/models/flux/flux.py @@ -78,10 +78,10 @@ def __init__(self, *args, **kwargs): self.tokenizer = FluxTokenizer() with torch.no_grad(): - self.clip_l = SDClipModel(layer="hidden", layer_idx=-2, device="cpu", dtype=devices.dtype, layer_norm_hidden_state=False, return_projected_pooled=False, textmodel_json_config=CLIPL_CONFIG) + self.clip_l = SDClipModel(layer="hidden", layer_idx=-2, device="cpu", dtype=devices.dtype_inference, layer_norm_hidden_state=False, return_projected_pooled=False, textmodel_json_config=CLIPL_CONFIG) if shared.opts.flux_enable_t5: - self.t5xxl = T5XXLModel(T5_CONFIG, device="cpu", dtype=devices.dtype) + self.t5xxl = T5XXLModel(T5_CONFIG, device="cpu", dtype=devices.dtype_inference) else: self.t5xxl = None @@ -202,8 +202,8 @@ def __init__(self, shift=1.0, device=None, dtype=torch.float16, state_dict=None, def apply_model(self, x, sigma, c_crossattn=None, y=None): dtype = self.get_dtype() timestep = self.model_sampling.timestep(sigma).float() - guidance = torch.FloatTensor([3.5]).to(device=devices.device, dtype=dtype) - model_output = self.diffusion_model(x.to(dtype), timestep, context=c_crossattn.to(dtype), y=y.to(dtype), guidance=guidance).float() + guidance = torch.FloatTensor([3.5]).to(device=devices.device, dtype=torch.float32) + model_output = self.diffusion_model(x.to(dtype), timestep, context=c_crossattn.to(dtype), y=y.to(dtype), guidance=guidance).to(x.dtype) return self.model_sampling.calculate_denoised(sigma, model_output, x) def forward(self, *args, **kwargs): @@ -268,7 +268,7 @@ def __init__(self, state_dict, use_ema=False): diffusion_model_prefix = "" with torch.no_grad(): - self.model = BaseModel(state_dict=state_dict, prefix=diffusion_model_prefix, device="cpu", dtype=devices.dtype) + self.model = BaseModel(state_dict=state_dict, prefix=diffusion_model_prefix, device="cpu", dtype=devices.dtype_inference) self.first_stage_model = SDVAE(device="cpu", dtype=devices.dtype_vae) self.first_stage_model.dtype = devices.dtype_vae self.vae = self.first_stage_model # real vae From 88135af7da56f0a8394b4194fab58ab564c63076 Mon Sep 17 00:00:00 2001 From: Won-Kyu Park Date: Tue, 10 Sep 2024 20:41:19 +0900 Subject: [PATCH 22/29] reduce intermediate steps and optimize --- modules/models/flux/math.py | 11 ++++++----- modules/models/flux/modules/layers.py | 12 ++++-------- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/modules/models/flux/math.py b/modules/models/flux/math.py index 6fbafbeead6..be4359f6478 100644 --- a/modules/models/flux/math.py +++ b/modules/models/flux/math.py @@ -17,13 +17,14 @@ def rope(pos: Tensor, dim: int, theta: int) -> Tensor: assert dim % 2 == 0 scale = torch.arange(0, dim, 2, dtype=torch.float64, device=pos.device) / dim omega = 1.0 / (theta**scale) - out = torch.einsum("...n,d->...nd", pos, omega) - #out = pos.unsqueeze(-1) * omega.unsqueeze(0) + #out = torch.einsum("...n,d->...nd", pos, omega) + out = pos.unsqueeze(-1) * omega.unsqueeze(0) - out = torch.stack([torch.cos(out), -torch.sin(out), torch.sin(out), torch.cos(out)], dim=-1) + cos_out, sin_out = torch.cos(out), torch.sin(out) + + out = torch.stack([cos_out, -sin_out, sin_out, cos_out], dim=-1) #out = rearrange(out, "b n d (i j) -> b n d i j", i=2, j=2) - b, n, d, _ = out.shape - out = out.view(b, n, d, 2, 2) + out = out.view(*out.shape[:-1], 2, 2) return out.to(dtype=torch.float32, device=pos.device) diff --git a/modules/models/flux/modules/layers.py b/modules/models/flux/modules/layers.py index 9849e2d92e5..c6b109249f3 100644 --- a/modules/models/flux/modules/layers.py +++ b/modules/models/flux/modules/layers.py @@ -112,8 +112,7 @@ def __init__(self, dim: int, num_heads: int = 8, qkv_bias: bool = False, dtype=N def forward(self, x: Tensor, pe: Tensor) -> Tensor: qkv = self.qkv(x) #q, k, v = rearrange(qkv, "B L (K H D) -> K B H L D", K=3, H=self.num_heads) - B, L, _ = qkv.shape - q, k, v = qkv.view(B, L, 3, self.num_heads, -1).permute(2, 0, 3, 1, 4) + q, k, v = qkv.view(*qkv.shape[:-1], 3, self.num_heads, -1).permute(2, 0, 3, 1, 4) q, k = self.norm(q, k, v) x = attention(q, k, v, pe=pe) x = self.proj(x) @@ -182,8 +181,7 @@ def forward(self, img: Tensor, txt: Tensor, vec: Tensor, pe: Tensor): img_qkv = self.img_attn.qkv(img_modulated) del img_modulated #img_q, img_k, img_v = rearrange(img_qkv, "B L (K H D) -> K B H L D", K=3, H=self.num_heads) - B, L, _ = img_qkv.shape - img_q, img_k, img_v = img_qkv.view(B, L, 3, self.num_heads, -1).permute(2, 0, 3, 1, 4) + img_q, img_k, img_v = img_qkv.view(*img_qkv.shape[:-1], 3, self.num_heads, -1).permute(2, 0, 3, 1, 4) del img_qkv img_q, img_k = self.img_attn.norm(img_q, img_k, img_v) @@ -193,8 +191,7 @@ def forward(self, img: Tensor, txt: Tensor, vec: Tensor, pe: Tensor): txt_qkv = self.txt_attn.qkv(txt_modulated) del txt_modulated #txt_q, txt_k, txt_v = rearrange(txt_qkv, "B L (K H D) -> K B H L D", K=3, H=self.num_heads) - B, L, _ = txt_qkv.shape - txt_q, txt_k, txt_v = txt_qkv.view(B, L, 3, self.num_heads, -1).permute(2, 0, 3, 1, 4) + txt_q, txt_k, txt_v = txt_qkv.view(*txt_qkv.shape[:-1], 3, self.num_heads, -1).permute(2, 0, 3, 1, 4) del txt_qkv txt_q, txt_k = self.txt_attn.norm(txt_q, txt_k, txt_v) @@ -270,8 +267,7 @@ def forward(self, x: Tensor, vec: Tensor, pe: Tensor) -> Tensor: del x_mod #q, k, v = rearrange(qkv, "B L (K H D) -> K B H L D", K=3, H=self.num_heads) - B, L, _ = qkv.shape - q, k, v = qkv.view(B, L, 3, self.num_heads, -1).permute(2, 0, 3, 1, 4) + q, k, v = qkv.view(*qkv.shape[:-1], 3, self.num_heads, -1).permute(2, 0, 3, 1, 4) del qkv q, k = self.norm(q, k, v) From aaacdbc22b172a38083f3c98140ebc86e43bf2c8 Mon Sep 17 00:00:00 2001 From: Won-Kyu Park Date: Thu, 12 Sep 2024 00:23:00 +0900 Subject: [PATCH 23/29] fix to support dtype_inference != dtype case --- modules/processing.py | 4 ++-- modules/sd_models.py | 2 +- modules/sd_models_xl.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/processing.py b/modules/processing.py index 28003eacb94..3db7b8debd6 100644 --- a/modules/processing.py +++ b/modules/processing.py @@ -484,7 +484,7 @@ def get_conds_with_caching(self, function, required_prompts, steps, caches, extr cache = caches[0] - with devices.autocast(): + with devices.autocast(target_dtype=devices.dtype_inference): cache[1] = function(shared.sd_model, required_prompts, steps, hires_steps, shared.opts.use_old_scheduling) cache[0] = cached_params @@ -1436,7 +1436,7 @@ def save_intermediate(image, index): with devices.autocast(): extra_networks.activate(self, self.hr_extra_network_data) - with devices.autocast(): + with devices.autocast(target_dtype=devices.dtype_inference): self.calculate_hr_conds() sd_models.apply_token_merging(self.sd_model, self.get_token_merging_ratio(for_hr=True)) diff --git a/modules/sd_models.py b/modules/sd_models.py index b4a2aeec6b8..e00b0e6cb20 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -1013,7 +1013,7 @@ def load_model(checkpoint_info=None, already_loaded_state_dict=None): timer.record("scripts callbacks") - with devices.autocast(), torch.no_grad(): + with devices.autocast(target_dtype=devices.dtype_inference), torch.no_grad(): sd_model.cond_stage_model_empty_prompt = get_empty_cond(sd_model) timer.record("calculate empty prompt") diff --git a/modules/sd_models_xl.py b/modules/sd_models_xl.py index 1242a59369f..5abb75d1f02 100644 --- a/modules/sd_models_xl.py +++ b/modules/sd_models_xl.py @@ -18,7 +18,7 @@ def get_learned_conditioning(self: sgm.models.diffusion.DiffusionEngine, batch: is_negative_prompt = getattr(batch, 'is_negative_prompt', False) aesthetic_score = shared.opts.sdxl_refiner_low_aesthetic_score if is_negative_prompt else shared.opts.sdxl_refiner_high_aesthetic_score - devices_args = dict(device=devices.device, dtype=devices.dtype) + devices_args = dict(device=devices.device, dtype=devices.dtype_inference) sdxl_conds = { "txt": batch, From 5075552c730ed355df3b69218b930d498f747e26 Mon Sep 17 00:00:00 2001 From: Won-Kyu Park Date: Fri, 13 Sep 2024 14:03:40 +0900 Subject: [PATCH 24/29] use empty_like() and partial revert for speed --- modules/sd_disable_initialization.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/modules/sd_disable_initialization.py b/modules/sd_disable_initialization.py index 17ee4e8b3ee..00441a6fd28 100644 --- a/modules/sd_disable_initialization.py +++ b/modules/sd_disable_initialization.py @@ -192,11 +192,7 @@ def load_from_state_dict(original, module, state_dict, prefix, *args, **kwargs): if param.is_meta: dtype = sd_param.dtype if sd_param is not None else param.dtype - if dtype in (torch.float8_e4m3fn, torch.float8_e5m2): - zeros = torch.zeros_like(param, device=device, dtype=torch.float32) - else: - zeros = torch.zeros_like(param, device=device, dtype=dtype) - module._parameters[name] = torch.nn.parameter.Parameter(zeros.to(dtype=dtype), requires_grad=param.requires_grad) + module._parameters[name] = torch.nn.parameter.Parameter(torch.empty_like(param, device=device, dtype=dtype), requires_grad=param.requires_grad) for name in module._buffers: key = prefix + name From a6c55b2d6babfaf96182a5db6965eb93f2dee0ac Mon Sep 17 00:00:00 2001 From: Won-Kyu Park Date: Fri, 13 Sep 2024 17:58:12 +0900 Subject: [PATCH 25/29] fix for pytest --- modules/sd_models.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/modules/sd_models.py b/modules/sd_models.py index e00b0e6cb20..a856997d0ce 100644 --- a/modules/sd_models.py +++ b/modules/sd_models.py @@ -484,12 +484,14 @@ def get_vae_dtype(state_dict=None, state_dict_dtype=None): raise ValueError("fail to get vae dtype") - vae_prefix = [prefix for prefix in ("vae.", "first_stage_model.") if prefix in state_dict_dtype][0] + vae_prefixes = [prefix for prefix in ("vae.", "first_stage_model.") if prefix in state_dict_dtype] - for dtype in state_dict_dtype[vae_prefix]: - if state_dict_dtype[vae_prefix][dtype] > 240 and dtype in (torch.float16, torch.float32, torch.bfloat16): - # vae items: 248 for SD1, SDXL 245 for flux - return dtype + if len(vae_prefixes) > 0: + vae_prefix = vae_prefixes[0] + for dtype in state_dict_dtype[vae_prefix]: + if state_dict_dtype[vae_prefix][dtype] > 240 and dtype in (torch.float16, torch.float32, torch.bfloat16): + # vae items: 248 for SD1, SDXL 245 for flux + return dtype return None @@ -552,11 +554,13 @@ def load_model_weights(model, checkpoint_info: CheckpointInfo, state_dict, timer model.to(memory_format=torch.channels_last) timer.record("apply channels_last") - unet_has_float = [f for f in state_dict_dtype["model.diffusion_model."].keys() if f in (torch.float16, torch.float32, torch.bfloat16)] # check dtype of vae dtype_vae = get_vae_dtype(state_dict_dtype=state_dict_dtype) + found_unet_dtype = get_loadable_dtype("model.diffusion_model.", state_dict_dtype=state_dict_dtype) + unet_has_float = found_unet_dtype in (torch.float16, torch.float32, torch.bfloat16) - if unet_has_float and shared.cmd_opts.no_half: + if (found_unet_dtype is None or unet_has_float) and shared.cmd_opts.no_half: + # unet type is not detected or unet has float dtypes model.float() model.alphas_cumprod_original = model.alphas_cumprod devices.dtype_unet = torch.float32 @@ -579,14 +583,13 @@ def load_model_weights(model, checkpoint_info: CheckpointInfo, state_dict, timer alphas_cumprod = model.alphas_cumprod model.alphas_cumprod = None - found_unet_dtype = get_loadable_dtype("model.diffusion_model.", state_dict_dtype=state_dict_dtype) if found_unet_dtype in (torch.float16, torch.float32, torch.bfloat16): model.half() elif found_unet_dtype in (torch.float8_e4m3fn,): pass else: - raise ValueError("fail to get a vaild UNet dtype") + print("Fail to get a vaild UNet dtype. ignore...") model.alphas_cumprod = alphas_cumprod model.alphas_cumprod_original = alphas_cumprod @@ -599,7 +602,7 @@ def load_model_weights(model, checkpoint_info: CheckpointInfo, state_dict, timer timer.record("apply half()") else: print(f"load Unet {found_unet_dtype} as is ...") - devices.dtype_unet = found_unet_dtype + devices.dtype_unet = found_unet_dtype if found_unet_dtype else torch.float16 timer.record("load UNet") apply_alpha_schedule_override(model) @@ -646,9 +649,9 @@ def load_model_weights(model, checkpoint_info: CheckpointInfo, state_dict, timer devices.dtype_vae = torch.bfloat16 print(f"VAE dtype {dtype_vae} detected. load as is.") else: - devices.dtype_vae = torch.float16 + # use default devices.dtype_vae model.first_stage_model.to(devices.dtype_vae) - print(f"force VAE dtype {devices.dtype_vae}") + print(f"Use VAE dtype {devices.dtype_vae}") timer.record("apply dtype to VAE") # clean up cache if limit is reached From 5cb6200735b338e717107e95185520e7aa9898c5 Mon Sep 17 00:00:00 2001 From: Won-Kyu Park Date: Fri, 13 Sep 2024 20:00:49 +0900 Subject: [PATCH 26/29] pytest with --precision full --no-half --- .github/workflows/run_tests.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/run_tests.yaml b/.github/workflows/run_tests.yaml index 0610f4f5436..eccf15e4684 100644 --- a/.github/workflows/run_tests.yaml +++ b/.github/workflows/run_tests.yaml @@ -51,6 +51,7 @@ jobs: --test-server --do-not-download-clip --no-half + --precision full --disable-opt-split-attention --use-cpu all --api-server-stop From 44993847a294a17b71da9b78d3a56ed3e2cedcff Mon Sep 17 00:00:00 2001 From: Won-Kyu Park Date: Fri, 13 Sep 2024 20:32:13 +0900 Subject: [PATCH 27/29] minor fixes --- modules/devices.py | 6 ++++-- modules/models/flux/flux.py | 1 - 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/modules/devices.py b/modules/devices.py index 866b6ab161b..6b20c144f03 100644 --- a/modules/devices.py +++ b/modules/devices.py @@ -193,6 +193,8 @@ def forward_wrapper(self, *args, **kwargs): if param.dtype != target_dtype: org_dtype = param.dtype break + else: + break if copy: copied = deepcopy(self) @@ -227,10 +229,10 @@ def forward_wrapper(self, *args, **kwargs): @contextlib.contextmanager -def manual_cast(target_dtype, target_device=None): +def manual_cast(target_dtype, target_device=None, copy=None): applied = False - copy = shared.opts.lora_without_backup_weight + copy = copy if copy is not None else shared.opts.lora_without_backup_weight for module_type in patch_module_list: if hasattr(module_type, "org_forward"): diff --git a/modules/models/flux/flux.py b/modules/models/flux/flux.py index cc91f123d2d..46fd568a095 100644 --- a/modules/models/flux/flux.py +++ b/modules/models/flux/flux.py @@ -320,7 +320,6 @@ def create_denoiser(self): def medvram_fields(self): return [ (self, 'first_stage_model'), - (self, 'vae'), (self, 'text_encoders'), (self, 'model'), ] From 16e85906f3fca931e052c87cde3a4f4b1aeda674 Mon Sep 17 00:00:00 2001 From: Won-Kyu Park Date: Sun, 15 Sep 2024 23:29:45 +0900 Subject: [PATCH 28/29] fix lora without backup --- extensions-builtin/Lora/networks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extensions-builtin/Lora/networks.py b/extensions-builtin/Lora/networks.py index 46c9ac1ffae..dfdf3c7e6e5 100644 --- a/extensions-builtin/Lora/networks.py +++ b/extensions-builtin/Lora/networks.py @@ -397,7 +397,7 @@ def network_restore_weights_from_backup(self: Union[torch.nn.Conv2d, torch.nn.Li if weights_backup is None and bias_backup is None: return - if weights_backup is True or weights_backup == (True, True): # fake backup + if shared.opts.lora_without_backup_weight: return if weights_backup is not None: @@ -567,7 +567,7 @@ def network_apply_weights(self: Union[torch.nn.Conv2d, torch.nn.Linear, torch.nn extra_network_lora.errors[net.name] = extra_network_lora.errors.get(net.name, 0) + 1 - if weights_backup is True or weights_backup == (True, True): # fake backup + if shared.opts.lora_without_backup_weight: self.network_weights_backup = None self.network_bias_backup = None else: From 517c395fdf748651a8f58949f34aea9364e2aa5b Mon Sep 17 00:00:00 2001 From: Won-Kyu Park Date: Mon, 16 Sep 2024 06:33:46 +0900 Subject: [PATCH 29/29] revert to use without_autocast() --- modules/sd_samplers_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/sd_samplers_common.py b/modules/sd_samplers_common.py index 28b8bd820b7..b312c41d8e6 100644 --- a/modules/sd_samplers_common.py +++ b/modules/sd_samplers_common.py @@ -54,7 +54,7 @@ def samples_to_images_tensor(sample, approximation=None, model=None): else: if model is None: model = shared.sd_model - with torch.no_grad(), devices.manual_cast(devices.dtype_vae): # fixes an issue with unstable VAEs that are flaky even in fp32 + with torch.no_grad(), devices.without_autocast(): # fixes an issue with unstable VAEs that are flaky even in fp32 x_sample = model.decode_first_stage(sample.to(model.first_stage_model.dtype)) return x_sample