From 017a3d68fde01dc0098bed0d761a29e37609d830 Mon Sep 17 00:00:00 2001 From: Qing Date: Thu, 25 Apr 2024 22:21:33 +0800 Subject: [PATCH] remove freeu support --- iopaint/model_manager.py | 17 -- iopaint/schema.py | 25 --- iopaint/tests/test_brushnet.py | 4 +- iopaint/tests/test_low_mem.py | 31 +-- iopaint/tests/test_sd_model.py | 31 +-- iopaint/tests/test_sdxl.py | 56 +----- .../components/SidePanel/DiffusionOptions.tsx | 179 +++--------------- web_app/src/lib/api.ts | 2 - web_app/src/lib/states.ts | 6 - web_app/src/lib/types.ts | 8 - 10 files changed, 33 insertions(+), 326 deletions(-) diff --git a/iopaint/model_manager.py b/iopaint/model_manager.py index d66b8fe..40c66c2 100644 --- a/iopaint/model_manager.py +++ b/iopaint/model_manager.py @@ -103,7 +103,6 @@ class ModelManager: self.switch_brushnet_method(config) self.enable_disable_powerpaint_v2(config) - self.enable_disable_freeu(config) self.enable_disable_lcm_lora(config) return self.model(image, mask, config).astype(np.uint8) @@ -240,22 +239,6 @@ class ModelManager: else: logger.info("Disable PowerPaintV2") - def enable_disable_freeu(self, config: InpaintRequest): - if str(self.model.device) == "mps": - return - - if self.available_models[self.name].support_freeu: - if config.sd_freeu: - freeu_config = config.sd_freeu_config - self.model.model.enable_freeu( - s1=freeu_config.s1, - s2=freeu_config.s2, - b1=freeu_config.b1, - b2=freeu_config.b2, - ) - else: - self.model.model.disable_freeu() - def enable_disable_lcm_lora(self, config: InpaintRequest): if self.available_models[self.name].support_lcm_lora: # TODO: change this if load other lora is supported diff --git a/iopaint/schema.py b/iopaint/schema.py index 1b87f14..7d3d743 100644 --- a/iopaint/schema.py +++ b/iopaint/schema.py @@ -126,16 +126,6 @@ class ModelInfo(BaseModel): ModelType.DIFFUSERS_SD, ] - @computed_field - @property - def support_freeu(self) -> bool: - return self.model_type in [ - ModelType.DIFFUSERS_SD, - ModelType.DIFFUSERS_SDXL, - ModelType.DIFFUSERS_SD_INPAINT, - ModelType.DIFFUSERS_SDXL_INPAINT, - ] or self.name in [INSTRUCT_PIX2PIX_NAME] - class Choices(str, Enum): @classmethod @@ -225,12 +215,6 @@ class SDSampler(str, Enum): lcm = "LCM" -class FREEUConfig(BaseModel): - s1: float = 0.9 - s2: float = 0.2 - b1: float = 1.2 - b2: float = 1.4 - class PowerPaintTask(Choices): text_guided = "text-guided" @@ -352,12 +336,6 @@ class InpaintRequest(BaseModel): sd_outpainting_softness: float = Field(20.0) sd_outpainting_space: float = Field(20.0) - sd_freeu: bool = Field( - False, - description="Enable freeu mode. https://huggingface.co/docs/diffusers/main/en/using-diffusers/freeu", - ) - sd_freeu_config: FREEUConfig = FREEUConfig() - sd_lcm_lora: bool = Field( False, description="Enable lcm-lora mode. https://huggingface.co/docs/diffusers/main/en/using-diffusers/inference_with_lcm#texttoimage", @@ -433,9 +411,6 @@ class InpaintRequest(BaseModel): if values.sd_lcm_lora: logger.info("BrushNet is enabled, set sd_lcm_lora=False") values.sd_lcm_lora = False - if values.sd_freeu: - logger.info("BrushNet is enabled, set sd_freeu=False") - values.sd_freeu = False if values.enable_controlnet: logger.info("ControlNet is enabled, set enable_brushnet=False") diff --git a/iopaint/tests/test_brushnet.py b/iopaint/tests/test_brushnet.py index 8da4aa1..73394c0 100644 --- a/iopaint/tests/test_brushnet.py +++ b/iopaint/tests/test_brushnet.py @@ -10,7 +10,7 @@ import pytest import torch from iopaint.model_manager import ModelManager -from iopaint.schema import HDStrategy, SDSampler, FREEUConfig, PowerPaintTask +from iopaint.schema import HDStrategy, SDSampler, PowerPaintTask current_dir = Path(__file__).parent.absolute().resolve() save_dir = current_dir / "result" @@ -32,8 +32,6 @@ def test_runway_brushnet(device, sampler): prompt="face of a fox, sitting on a bench", sd_steps=sd_steps, sd_guidance_scale=7.5, - sd_freeu=True, - sd_freeu_config=FREEUConfig(), enable_brushnet=True, brushnet_method=SD_BRUSHNET_CHOICES[0], ) diff --git a/iopaint/tests/test_low_mem.py b/iopaint/tests/test_low_mem.py index 70e8801..e3c1b91 100644 --- a/iopaint/tests/test_low_mem.py +++ b/iopaint/tests/test_low_mem.py @@ -10,7 +10,7 @@ import pytest import torch from iopaint.model_manager import ModelManager -from iopaint.schema import HDStrategy, SDSampler, FREEUConfig +from iopaint.schema import HDStrategy, SDSampler @pytest.mark.parametrize("device", ["cuda", "mps"]) @@ -75,35 +75,6 @@ def test_runway_sd_lcm_lora_low_mem(device, sampler): ) -@pytest.mark.parametrize("device", ["cuda", "mps", "cpu"]) -@pytest.mark.parametrize("sampler", [SDSampler.ddim]) -def test_runway_sd_freeu(device, sampler): - sd_steps = check_device(device) - model = ModelManager( - name="runwayml/stable-diffusion-inpainting", - device=torch.device(device), - disable_nsfw=True, - sd_cpu_textencoder=False, - low_mem=True, - ) - cfg = get_config( - strategy=HDStrategy.ORIGINAL, - prompt="face of a fox, sitting on a bench", - sd_steps=sd_steps, - sd_guidance_scale=7.5, - sd_freeu=True, - sd_freeu_config=FREEUConfig(), - ) - cfg.sd_sampler = sampler - - assert_equal( - model, - cfg, - f"runway_sd_1_5_freeu_device_{device}_low_mem.png", - img_p=current_dir / "overture-creations-5sI6fQgYIuo.png", - mask_p=current_dir / "overture-creations-5sI6fQgYIuo_mask.png", - ) - @pytest.mark.parametrize("device", ["cuda", "mps", "cpu"]) @pytest.mark.parametrize("strategy", [HDStrategy.ORIGINAL]) diff --git a/iopaint/tests/test_sd_model.py b/iopaint/tests/test_sd_model.py index aa26c71..89eedb5 100644 --- a/iopaint/tests/test_sd_model.py +++ b/iopaint/tests/test_sd_model.py @@ -11,7 +11,7 @@ import pytest import torch from iopaint.model_manager import ModelManager -from iopaint.schema import HDStrategy, SDSampler, FREEUConfig +from iopaint.schema import HDStrategy, SDSampler current_dir = Path(__file__).parent.absolute().resolve() save_dir = current_dir / "result" @@ -90,35 +90,6 @@ def test_runway_sd_lcm_lora(device, sampler): ) -@pytest.mark.parametrize("device", ["cuda", "mps", "cpu"]) -@pytest.mark.parametrize("sampler", [SDSampler.ddim]) -def test_runway_sd_freeu(device, sampler): - sd_steps = check_device(device) - model = ModelManager( - name="runwayml/stable-diffusion-inpainting", - device=torch.device(device), - disable_nsfw=True, - sd_cpu_textencoder=False, - ) - cfg = get_config( - strategy=HDStrategy.ORIGINAL, - prompt="face of a fox, sitting on a bench", - sd_steps=sd_steps, - sd_guidance_scale=7.5, - sd_freeu=True, - sd_freeu_config=FREEUConfig(), - ) - cfg.sd_sampler = sampler - - assert_equal( - model, - cfg, - f"runway_sd_1_5_freeu_device_{device}.png", - img_p=current_dir / "overture-creations-5sI6fQgYIuo.png", - mask_p=current_dir / "overture-creations-5sI6fQgYIuo_mask.png", - ) - - @pytest.mark.parametrize("device", ["cuda", "mps"]) @pytest.mark.parametrize("strategy", [HDStrategy.ORIGINAL]) @pytest.mark.parametrize("sampler", [SDSampler.ddim]) diff --git a/iopaint/tests/test_sdxl.py b/iopaint/tests/test_sdxl.py index e236948..8d5e5ab 100644 --- a/iopaint/tests/test_sdxl.py +++ b/iopaint/tests/test_sdxl.py @@ -8,7 +8,7 @@ import pytest import torch from iopaint.model_manager import ModelManager -from iopaint.schema import HDStrategy, SDSampler, FREEUConfig +from iopaint.schema import HDStrategy, SDSampler from iopaint.tests.test_model import get_config, assert_equal @@ -76,60 +76,6 @@ def test_sdxl_cpu_text_encoder(device, strategy, sampler): ) -@pytest.mark.parametrize("device", ["cuda", "mps"]) -@pytest.mark.parametrize("strategy", [HDStrategy.ORIGINAL]) -@pytest.mark.parametrize("sampler", [SDSampler.ddim]) -def test_sdxl_lcm_lora_and_freeu(device, strategy, sampler): - sd_steps = check_device(device) - - model = ModelManager( - name="diffusers/stable-diffusion-xl-1.0-inpainting-0.1", - device=torch.device(device), - disable_nsfw=True, - sd_cpu_textencoder=False, - ) - cfg = get_config( - strategy=strategy, - prompt="face of a fox, sitting on a bench", - sd_steps=sd_steps, - sd_strength=1.0, - sd_guidance_scale=2.0, - sd_lcm_lora=True, - ) - cfg.sd_sampler = sampler - - name = f"device_{device}_{sampler}" - - assert_equal( - model, - cfg, - f"sdxl_{name}_lcm_lora.png", - img_p=current_dir / "overture-creations-5sI6fQgYIuo.png", - mask_p=current_dir / "overture-creations-5sI6fQgYIuo_mask.png", - fx=2, - fy=2, - ) - - cfg = get_config( - strategy=strategy, - prompt="face of a fox, sitting on a bench", - sd_steps=sd_steps, - sd_guidance_scale=7.5, - sd_freeu=True, - sd_freeu_config=FREEUConfig(), - ) - - assert_equal( - model, - cfg, - f"sdxl_{name}_freeu_device_{device}.png", - img_p=current_dir / "overture-creations-5sI6fQgYIuo.png", - mask_p=current_dir / "overture-creations-5sI6fQgYIuo_mask.png", - fx=2, - fy=2, - ) - - @pytest.mark.parametrize("device", ["cuda", "mps"]) @pytest.mark.parametrize( "rect", diff --git a/web_app/src/components/SidePanel/DiffusionOptions.tsx b/web_app/src/components/SidePanel/DiffusionOptions.tsx index c0760d7..62e8d2a 100644 --- a/web_app/src/components/SidePanel/DiffusionOptions.tsx +++ b/web_app/src/components/SidePanel/DiffusionOptions.tsx @@ -316,129 +316,6 @@ const DiffusionOptions = () => { ) } - const renderFreeu = () => { - if (!settings.model.support_freeu) { - return null - } - - let toolTip = - "FreeU is a technique for improving image quality. Different models may require different FreeU-specific hyperparameters, which can be viewed in the more info section." - if (settings.enableBrushNet) { - toolTip = "BrushNet is enabled, FreeU is disabled." - } - - let disable = settings.enableBrushNet || !settings.enableFreeu - - return ( -
- - - { - updateSettings({ enableFreeu: value }) - }} - disabled={settings.enableBrushNet} - /> - -
-
-
- - { - updateSettings({ - freeuConfig: { ...settings.freeuConfig, s1: value }, - }) - }} - /> -
-
- - { - updateSettings({ - freeuConfig: { ...settings.freeuConfig, s2: value }, - }) - }} - /> -
-
- -
-
- - { - updateSettings({ - freeuConfig: { ...settings.freeuConfig, b1: value }, - }) - }} - /> -
-
- - { - updateSettings({ - freeuConfig: { ...settings.freeuConfig, b2: value }, - }) - }} - /> -
-
-
- -
- ) - } - const renderNegativePrompt = () => { if (!settings.model.need_prompt) { return null @@ -851,30 +728,33 @@ const DiffusionOptions = () => { const renderMaskBlur = () => { return ( - - - updateSettings({ sdMaskBlur: vals[0] })} - /> - { - updateSettings({ sdMaskBlur: value }) - }} - /> - + <> + + + updateSettings({ sdMaskBlur: vals[0] })} + /> + { + updateSettings({ sdMaskBlur: value }) + }} + /> + + + ) } @@ -985,7 +865,9 @@ const DiffusionOptions = () => {
{renderCropper()} {renderExtender()} + {renderMaskBlur()} {renderMaskAdjuster()} + {renderMatchHistograms()} {renderPowerPaintTaskType()} {renderSteps()} {renderGuidanceScale()} @@ -998,9 +880,6 @@ const DiffusionOptions = () => { {renderBrushNetSetting()} {renderConterNetSetting()} {renderLCMLora()} - {renderMaskBlur()} - {renderMatchHistograms()} - {renderFreeu()} {renderPaintByExample()}
) diff --git a/web_app/src/lib/api.ts b/web_app/src/lib/api.ts index 21e72dd..df4a112 100644 --- a/web_app/src/lib/api.ts +++ b/web_app/src/lib/api.ts @@ -68,8 +68,6 @@ export default async function inpaint( sd_sampler: settings.sdSampler, sd_seed: settings.seedFixed ? settings.seed : -1, sd_match_histograms: settings.sdMatchHistograms, - sd_freeu: settings.enableFreeu, - sd_freeu_config: settings.freeuConfig, sd_lcm_lora: settings.enableLCMLora, paint_by_example_example_image: exampleImageBase64, p2p_image_guidance_scale: settings.p2pImageGuidanceScale, diff --git a/web_app/src/lib/states.ts b/web_app/src/lib/states.ts index aa3adf2..9541623 100644 --- a/web_app/src/lib/states.ts +++ b/web_app/src/lib/states.ts @@ -7,7 +7,6 @@ import { AdjustMaskOperate, CV2Flag, ExtenderDirection, - FreeuConfig, LDMSampler, Line, LineGroup, @@ -105,8 +104,6 @@ export type Settings = { brushnetConditioningScale: number enableLCMLora: boolean - enableFreeu: boolean - freeuConfig: FreeuConfig // PowerPaint powerpaintTask: PowerPaintTask @@ -316,7 +313,6 @@ const defaultValues: AppState = { support_outpainting: false, controlnets: [], brushnets: [], - support_freeu: false, support_lcm_lora: false, is_single_file_diffusers: false, need_prompt: false, @@ -352,8 +348,6 @@ const defaultValues: AppState = { brushnetMethod: "random_mask", brushnetConditioningScale: 1.0, enableLCMLora: false, - enableFreeu: false, - freeuConfig: { s1: 0.9, s2: 0.2, b1: 1.2, b2: 1.4 }, powerpaintTask: PowerPaintTask.text_guided, adjustMaskKernelSize: 12, }, diff --git a/web_app/src/lib/types.ts b/web_app/src/lib/types.ts index 70e436b..4e7c23a 100644 --- a/web_app/src/lib/types.ts +++ b/web_app/src/lib/types.ts @@ -51,7 +51,6 @@ export interface ModelInfo { support_brushnet: boolean controlnets: string[] brushnets: string[] - support_freeu: boolean support_lcm_lora: boolean need_prompt: boolean is_single_file_diffusers: boolean @@ -98,13 +97,6 @@ export interface Rect { height: number } -export interface FreeuConfig { - s1: number - s2: number - b1: number - b2: number -} - export interface Point { x: number y: number