kamp/IOPaint
1
0
Fork 0
Code Issues Pull Requests Releases 1 Activity

update

Browse Source
This commit is contained in:
Qing 2023-12-16 13:34:56 +08:00
parent cbe6577890
commit 24e95daac1
16 changed files with 381 additions and 323 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
lama_cleaner/benchmark.py
View File

@ -101,7 +101,6 @@ if __name__ == "__main__":
model = ModelManager( model = ModelManager(
name=args.name, name=args.name,
device=device, device=device,
sd_run_local=True,
disable_nsfw=True, disable_nsfw=True,
sd_cpu_textencoder=True, sd_cpu_textencoder=True,
hf_access_token="123" hf_access_token="123"

2
lama_cleaner/const.py
View File

@ -65,7 +65,7 @@ Run Stable Diffusion text encoder model on CPU to save GPU memory.
SD_CONTROLNET_HELP = """ SD_CONTROLNET_HELP = """
Run Stable Diffusion normal or inpainting model with ControlNet. Run Stable Diffusion normal or inpainting model with ControlNet.
""" """
DEFAULT_SD_CONTROLNET_METHOD = "thibaud/controlnet-sd21-openpose-diffusers" DEFAULT_SD_CONTROLNET_METHOD = "lllyasviel/control_v11p_sd15_canny"
SD_CONTROLNET_CHOICES = [ SD_CONTROLNET_CHOICES = [
"lllyasviel/control_v11p_sd15_canny", "lllyasviel/control_v11p_sd15_canny",
# "lllyasviel/control_v11p_sd15_seg", # "lllyasviel/control_v11p_sd15_seg",

4
lama_cleaner/model/instruct_pix2pix.py
View File

@ -66,9 +66,9 @@ class InstructPix2Pix(DiffusionInpaintModel):
image=PIL.Image.fromarray(image), image=PIL.Image.fromarray(image),
prompt=config.prompt, prompt=config.prompt,
negative_prompt=config.negative_prompt, negative_prompt=config.negative_prompt,
num_inference_steps=config.p2p_steps, num_inference_steps=config.sd_steps,
image_guidance_scale=config.p2p_image_guidance_scale, image_guidance_scale=config.p2p_image_guidance_scale,
guidance_scale=config.p2p_guidance_scale, guidance_scale=config.sd_guidance_scale,
output_type="np", output_type="np",
generator=torch.manual_seed(config.sd_seed), generator=torch.manual_seed(config.sd_seed),
).images[0] ).images[0]

17
lama_cleaner/model/paint_by_example.py
View File

@ -19,7 +19,7 @@ class PaintByExample(DiffusionInpaintModel):
fp16 = not kwargs.get("no_half", False) fp16 = not kwargs.get("no_half", False)
use_gpu = device == torch.device("cuda") and torch.cuda.is_available() use_gpu = device == torch.device("cuda") and torch.cuda.is_available()
torch_dtype = torch.float16 if use_gpu and fp16 else torch.float32 torch_dtype = torch.float16 if use_gpu and fp16 else torch.float32
model_kwargs = {"local_files_only": kwargs.get("local_files_only", False)} model_kwargs = {}
if kwargs["disable_nsfw"] or kwargs.get("cpu_offload", False): if kwargs["disable_nsfw"] or kwargs.get("cpu_offload", False):
logger.info("Disable Paint By Example Model NSFW checker") logger.info("Disable Paint By Example Model NSFW checker")
@ -58,24 +58,17 @@ class PaintByExample(DiffusionInpaintModel):
image=PIL.Image.fromarray(image), image=PIL.Image.fromarray(image),
mask_image=PIL.Image.fromarray(mask[:, :, -1], mode="L"), mask_image=PIL.Image.fromarray(mask[:, :, -1], mode="L"),
example_image=config.paint_by_example_example_image, example_image=config.paint_by_example_example_image,
num_inference_steps=config.paint_by_example_steps, num_inference_steps=config.sd_steps,
guidance_scale=config.sd_guidance_scale,
negative_prompt="out of frame, lowres, error, cropped, worst quality, low quality, jpeg artifacts, ugly, duplicate, morbid, mutilated, out of frame, mutation, deformed, blurry, dehydrated, bad anatomy, bad proportions, extra limbs, disfigured, gross proportions, malformed limbs, watermark, signature",
output_type="np.array", output_type="np.array",
generator=torch.manual_seed(config.paint_by_example_seed), generator=torch.manual_seed(config.sd_seed),
).images[0] ).images[0]
output = (output * 255).round().astype("uint8") output = (output * 255).round().astype("uint8")
output = cv2.cvtColor(output, cv2.COLOR_RGB2BGR) output = cv2.cvtColor(output, cv2.COLOR_RGB2BGR)
return output return output
def forward_post_process(self, result, image, mask, config):
if config.paint_by_example_match_histograms:
result = self._match_histograms(result, image[:, :, ::-1], mask)
if config.paint_by_example_mask_blur != 0:
k = 2 * config.paint_by_example_mask_blur + 1
mask = cv2.GaussianBlur(mask, (k, k), 0)
return result, image, mask
@staticmethod @staticmethod
def is_downloaded() -> bool: def is_downloaded() -> bool:
# model will be downloaded when app start, and can't switch in frontend settings # model will be downloaded when app start, and can't switch in frontend settings

2
lama_cleaner/model/utils.py
View File

@ -1,3 +1,4 @@
import gc
import math import math
import random import random
from typing import Any from typing import Any
@ -913,6 +914,7 @@ def torch_gc():
if torch.cuda.is_available(): if torch.cuda.is_available():
torch.cuda.empty_cache() torch.cuda.empty_cache()
torch.cuda.ipc_collect() torch.cuda.ipc_collect()
gc.collect()
def set_seed(seed: int): def set_seed(seed: int):

0
lama_cleaner/tests/test_model_switch.py Normal file
View File

3
web_app/index.html
View File

@ -2,9 +2,8 @@
<html lang="en"> <html lang="en">
<head> <head>
<meta charset="UTF-8" /> <meta charset="UTF-8" />
<link rel="icon" type="image/svg+xml" href="/vite.svg" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Vite + React + TS</title> <title>Lama Cleaner</title>
</head> </head>
<body> <body>
<div id="root"></div> <div id="root"></div>

58
web_app/src/components/Editor.tsx
View File

@ -50,22 +50,12 @@ export default function Editor(props: EditorProps) {
imageHeight, imageHeight,
settings, settings,
enableAutoSaving, enableAutoSaving,
cropperRect,
enableManualInpainting,
setImageSize, setImageSize,
setBaseBrushSize, setBaseBrushSize,
setIsInpainting,
setSeed,
interactiveSegState, interactiveSegState,
updateInteractiveSegState, updateInteractiveSegState,
resetInteractiveSegState,
isPluginRunning,
setIsPluginRunning,
handleCanvasMouseDown, handleCanvasMouseDown,
handleCanvasMouseMove, handleCanvasMouseMove,
cleanCurLineGroup,
updateEditorState,
resetRedoState,
undo, undo,
redo, redo,
undoDisabled, undoDisabled,
@ -82,22 +72,12 @@ export default function Editor(props: EditorProps) {
state.imageHeight, state.imageHeight,
state.settings, state.settings,
state.serverConfig.enableAutoSaving, state.serverConfig.enableAutoSaving,
state.cropperState,
state.settings.enableManualInpainting,
state.setImageSize, state.setImageSize,
state.setBaseBrushSize, state.setBaseBrushSize,
state.setIsInpainting,
state.setSeed,
state.interactiveSegState, state.interactiveSegState,
state.updateInteractiveSegState, state.updateInteractiveSegState,
state.resetInteractiveSegState,
state.isPluginRunning,
state.setIsPluginRunning,
state.handleCanvasMouseDown, state.handleCanvasMouseDown,
state.handleCanvasMouseMove, state.handleCanvasMouseMove,
state.cleanCurLineGroup,
state.updateEditorState,
state.resetRedoState,
state.undo, state.undo,
state.redo, state.redo,
state.undoDisabled(), state.undoDisabled(),
@ -112,9 +92,7 @@ export default function Editor(props: EditorProps) {
const renders = useStore((state) => state.editorState.renders) const renders = useStore((state) => state.editorState.renders)
const extraMasks = useStore((state) => state.editorState.extraMasks) const extraMasks = useStore((state) => state.editorState.extraMasks)
const lineGroups = useStore((state) => state.editorState.lineGroups) const lineGroups = useStore((state) => state.editorState.lineGroups)
const lastLineGroup = useStore((state) => state.editorState.lastLineGroup)
const curLineGroup = useStore((state) => state.editorState.curLineGroup) const curLineGroup = useStore((state) => state.editorState.curLineGroup)
const redoLineGroups = useStore((state) => state.editorState.redoLineGroups)
// Local State // Local State
const [showOriginal, setShowOriginal] = useState(false) const [showOriginal, setShowOriginal] = useState(false)
@ -338,8 +316,6 @@ export default function Editor(props: EditorProps) {
if (isDraging) { if (isDraging) {
setIsDraging(false) setIsDraging(false)
// setCurLineGroup([])
// drawOnCurrentRender([])
} else { } else {
resetZoom() resetZoom()
} }
@ -434,22 +410,6 @@ export default function Editor(props: EditorProps) {
} }
} }
const isOutsideCroper = (clickPnt: { x: number; y: number }) => {
if (clickPnt.x < cropperRect.x) {
return true
}
if (clickPnt.y < cropperRect.y) {
return true
}
if (clickPnt.x > cropperRect.x + cropperRect.width) {
return true
}
if (clickPnt.y > cropperRect.y + cropperRect.height) {
return true
}
return false
}
const onCanvasMouseUp = (ev: SyntheticEvent) => { const onCanvasMouseUp = (ev: SyntheticEvent) => {
if (interactiveSegState.isInteractiveSeg) { if (interactiveSegState.isInteractiveSeg) {
const xy = mouseXY(ev) const xy = mouseXY(ev)
@ -491,15 +451,6 @@ export default function Editor(props: EditorProps) {
return return
} }
// if (
// isDiffusionModels &&
// settings.showCroper &&
// isOutsideCroper(mouseXY(ev))
// ) {
// // TODO: 去掉这个逻辑，在 cropper 层截断 click 点击？
// return
// }
setIsDraging(true) setIsDraging(true)
handleCanvasMouseDown(mouseXY(ev)) handleCanvasMouseDown(mouseXY(ev))
} }
@ -850,15 +801,6 @@ export default function Editor(props: EditorProps) {
) )
} }
// const onInteractiveAccept = () => {
// setInteractiveSegMask(tmpInteractiveSegMask)
// setTmpInteractiveSegMask(null)
// if (!enableManualInpainting && tmpInteractiveSegMask) {
// runInpainting(false, undefined, tmpInteractiveSegMask)
// }
// }
return ( return (
<div <div
className="flex w-screen h-screen justify-center items-center" className="flex w-screen h-screen justify-center items-center"

528
web_app/src/components/SidePanel.tsx
View File

@ -1,5 +1,5 @@
import { FormEvent, useState } from "react" import { FormEvent } from "react"
import { useToggle, useWindowSize } from "react-use" import { useToggle } from "react-use"
import { useStore } from "@/lib/states" import { useStore } from "@/lib/states"
import { Switch } from "./ui/switch" import { Switch } from "./ui/switch"
import { Label } from "./ui/label" import { Label } from "./ui/label"
@ -16,39 +16,45 @@ import { Textarea } from "./ui/textarea"
import { SDSampler } from "@/lib/types" import { SDSampler } from "@/lib/types"
import { Separator } from "./ui/separator" import { Separator } from "./ui/separator"
import { ScrollArea } from "./ui/scroll-area" import { ScrollArea } from "./ui/scroll-area"
import { import { Sheet, SheetContent, SheetHeader, SheetTrigger } from "./ui/sheet"
Sheet, import { ChevronLeft, ChevronRight, Upload } from "lucide-react"
SheetContent, import { Button, ImageUploadButton } from "./ui/button"
SheetHeader,
SheetTitle,
SheetTrigger,
} from "./ui/sheet"
import { ChevronLeft, ChevronRight } from "lucide-react"
import { Button } from "./ui/button"
import useHotKey from "@/hooks/useHotkey" import useHotKey from "@/hooks/useHotkey"
import { Slider } from "./ui/slider" import { Slider } from "./ui/slider"
import { useImage } from "@/hooks/useImage"
import { INSTRUCT_PIX2PIX, PAINT_BY_EXAMPLE } from "@/lib/const"
const RowContainer = ({ children }: { children: React.ReactNode }) => ( const RowContainer = ({ children }: { children: React.ReactNode }) => (
<div className="flex justify-between items-center pr-2">{children}</div> <div className="flex justify-between items-center pr-2">{children}</div>
) )
const SidePanel = () => { const SidePanel = () => {
const [settings, updateSettings, showSidePanel, runInpainting] = useStore( const [
(state) => [ settings,
state.settings, windowSize,
state.updateSettings, paintByExampleFile,
state.showSidePanel(), isProcessing,
state.runInpainting, updateSettings,
] showSidePanel,
) runInpainting,
updateAppState,
] = useStore((state) => [
state.settings,
state.windowSize,
state.paintByExampleFile,
state.getIsProcessing(),
state.updateSettings,
state.showSidePanel(),
state.runInpainting,
state.updateAppState,
])
const [exampleImage, isExampleImageLoaded] = useImage(paintByExampleFile)
const [open, toggleOpen] = useToggle(false) const [open, toggleOpen] = useToggle(false)
useHotKey("c", () => { useHotKey("c", () => {
toggleOpen() toggleOpen()
}) })
const windowSize = useWindowSize()
if (!showSidePanel) { if (!showSidePanel) {
return null return null
} }
@ -72,20 +78,47 @@ const SidePanel = () => {
<Label htmlFor="controlnet">Controlnet</Label> <Label htmlFor="controlnet">Controlnet</Label>
<Switch <Switch
id="controlnet" id="controlnet"
checked={settings.enableControlNet} checked={settings.enableControlnet}
onCheckedChange={(value) => { onCheckedChange={(value) => {
updateSettings({ enableControlNet: value }) updateSettings({ enableControlnet: value })
}} }}
/> />
</div> </div>
<div className="pl-1 pr-2"> <div className="flex flex-col gap-1">
<RowContainer>
<Slider
className="w-[180px]"
defaultValue={[100]}
min={1}
max={100}
step={1}
disabled={!settings.enableControlnet}
value={[Math.floor(settings.controlnetConditioningScale * 100)]}
onValueChange={(vals) =>
updateSettings({ controlnetConditioningScale: vals[0] / 100 })
}
/>
<NumberInput
id="controlnet-weight"
className="w-[60px] rounded-full"
disabled={!settings.enableControlnet}
numberValue={settings.controlnetConditioningScale}
allowFloat={false}
onNumberValueChange={(val) => {
updateSettings({ controlnetConditioningScale: val })
}}
/>
</RowContainer>
</div>
<div className="pr-2">
<Select <Select
value={settings.controlnetMethod} value={settings.controlnetMethod}
onValueChange={(value) => { onValueChange={(value) => {
updateSettings({ controlnetMethod: value }) updateSettings({ controlnetMethod: value })
}} }}
disabled={!settings.enableControlNet} disabled={!settings.enableControlnet}
> >
<SelectTrigger> <SelectTrigger>
<SelectValue placeholder="Select control method" /> <SelectValue placeholder="Select control method" />
@ -102,26 +135,6 @@ const SidePanel = () => {
</Select> </Select>
</div> </div>
</div> </div>
<RowContainer>
<Label
htmlFor="controlnet-weight"
disabled={!settings.enableControlNet}
>
weight
</Label>
<NumberInput
id="controlnet-weight"
className="w-14"
disabled={!settings.enableControlNet}
numberValue={settings.controlnetConditioningScale}
allowFloat
onNumberValueChange={(value) => {
updateSettings({ controlnetConditioningScale: value })
}}
/>
</RowContainer>
<Separator /> <Separator />
</div> </div>
) )
@ -166,74 +179,79 @@ const SidePanel = () => {
}} }}
/> />
</div> </div>
<div className="flex gap-3"> <div className="flex flex-col gap-4">
<div className="flex flex-col gap-2 items-start"> <div className="flex justify-center gap-6">
<Label htmlFor="freeu-s1" disabled={!settings.enableFreeu}> <div className="flex gap-2 items-center justify-center">
s1 <Label htmlFor="freeu-s1" disabled={!settings.enableFreeu}>
</Label> s1
<NumberInput </Label>
id="freeu-s1" <NumberInput
className="w-14" id="freeu-s1"
disabled={!settings.enableFreeu} className="w-14"
numberValue={settings.freeuConfig.s1} disabled={!settings.enableFreeu}
allowFloat numberValue={settings.freeuConfig.s1}
onNumberValueChange={(value) => { allowFloat
updateSettings({ onNumberValueChange={(value) => {
freeuConfig: { ...settings.freeuConfig, s1: value }, updateSettings({
}) freeuConfig: { ...settings.freeuConfig, s1: value },
}} })
/> }}
/>
</div>
<div className="flex gap-2 items-center justify-center">
<Label htmlFor="freeu-s2" disabled={!settings.enableFreeu}>
s2
</Label>
<NumberInput
id="freeu-s2"
className="w-14"
disabled={!settings.enableFreeu}
numberValue={settings.freeuConfig.s2}
allowFloat
onNumberValueChange={(value) => {
updateSettings({
freeuConfig: { ...settings.freeuConfig, s2: value },
})
}}
/>
</div>
</div> </div>
<div className="flex flex-col gap-2 items-start">
<Label htmlFor="freeu-s2" disabled={!settings.enableFreeu}> <div className="flex justify-center gap-6">
s2 <div className="flex gap-2 items-center justify-center">
</Label> <Label htmlFor="freeu-b1" disabled={!settings.enableFreeu}>
<NumberInput b1
id="freeu-s2" </Label>
className="w-14" <NumberInput
disabled={!settings.enableFreeu} id="freeu-b1"
numberValue={settings.freeuConfig.s2} className="w-14"
allowFloat disabled={!settings.enableFreeu}
onNumberValueChange={(value) => { numberValue={settings.freeuConfig.b1}
updateSettings({ allowFloat
freeuConfig: { ...settings.freeuConfig, s2: value }, onNumberValueChange={(value) => {
}) updateSettings({
}} freeuConfig: { ...settings.freeuConfig, b1: value },
/> })
</div> }}
<div className="flex flex-col gap-2 items-start"> />
<Label htmlFor="freeu-b1" disabled={!settings.enableFreeu}> </div>
b1 <div className="flex gap-2 items-center justify-center">
</Label> <Label htmlFor="freeu-b2" disabled={!settings.enableFreeu}>
<NumberInput b2
id="freeu-b1" </Label>
className="w-14" <NumberInput
disabled={!settings.enableFreeu} id="freeu-b2"
numberValue={settings.freeuConfig.b1} className="w-14"
allowFloat disabled={!settings.enableFreeu}
onNumberValueChange={(value) => { numberValue={settings.freeuConfig.b2}
updateSettings({ allowFloat
freeuConfig: { ...settings.freeuConfig, b1: value }, onNumberValueChange={(value) => {
}) updateSettings({
}} freeuConfig: { ...settings.freeuConfig, b2: value },
/> })
</div> }}
<div className="flex flex-col gap-2 items-start"> />
<Label htmlFor="freeu-b2" disabled={!settings.enableFreeu}> </div>
b2
</Label>
<NumberInput
id="freeu-b2"
className="w-14"
disabled={!settings.enableFreeu}
numberValue={settings.freeuConfig.b2}
allowFloat
onNumberValueChange={(value) => {
updateSettings({
freeuConfig: { ...settings.freeuConfig, b2: value },
})
}}
/>
</div> </div>
</div> </div>
<Separator /> <Separator />
@ -241,6 +259,110 @@ const SidePanel = () => {
) )
} }
const renderNegativePrompt = () => {
if (!settings.model.need_prompt) {
return null
}
return (
<div className="flex flex-col gap-4">
<Label htmlFor="negative-prompt">Negative prompt</Label>
<div className="pl-2 pr-4">
<Textarea
rows={4}
onKeyUp={onKeyUp}
className="max-h-[8rem] overflow-y-auto mb-2"
placeholder=""
id="negative-prompt"
value={settings.negativePrompt}
onInput={(evt: FormEvent<HTMLTextAreaElement>) => {
evt.preventDefault()
evt.stopPropagation()
const target = evt.target as HTMLTextAreaElement
updateSettings({ negativePrompt: target.value })
}}
/>
</div>
</div>
)
}
const renderPaintByExample = () => {
if (settings.model.name !== PAINT_BY_EXAMPLE) {
return null
}
return (
<div>
<RowContainer>
<div>Example Image</div>
<ImageUploadButton
tooltip="Upload example image"
onFileUpload={(file) => {
updateAppState({ paintByExampleFile: file })
}}
>
<Upload />
</ImageUploadButton>
</RowContainer>
{isExampleImageLoaded ? (
<div className="flex justify-center items-center">
<img
src={exampleImage.src}
alt="example"
className="max-w-[200px] max-h-[200px] m-3"
/>
</div>
) : (
<></>
)}
<Button
variant="default"
className="w-full"
disabled={isProcessing || !isExampleImageLoaded}
onClick={() => {
runInpainting()
}}
>
Paint
</Button>
</div>
)
}
const renderP2PImageGuidanceScale = () => {
if (settings.model.name !== INSTRUCT_PIX2PIX) {
return null
}
return (
<div className="flex flex-col gap-1">
<Label htmlFor="image-guidance-scale">Image guidance scale</Label>
<RowContainer>
<Slider
className="w-[180px]"
defaultValue={[150]}
min={100}
max={1000}
step={1}
value={[Math.floor(settings.p2pImageGuidanceScale * 100)]}
onValueChange={(vals) =>
updateSettings({ p2pImageGuidanceScale: vals[0] / 100 })
}
/>
<NumberInput
id="image-guidance-scale"
className="w-[60px] rounded-full"
numberValue={settings.p2pImageGuidanceScale}
allowFloat
onNumberValueChange={(val) => {
updateSettings({ p2pImageGuidanceScale: val })
}}
/>
</RowContainer>
</div>
)
}
return ( return (
<Sheet open={open} modal={false}> <Sheet open={open} modal={false}>
<SheetTrigger <SheetTrigger
@ -263,7 +385,7 @@ const SidePanel = () => {
onOpenAutoFocus={(event) => event.preventDefault()} onOpenAutoFocus={(event) => event.preventDefault()}
onPointerDownOutside={(event) => event.preventDefault()} onPointerDownOutside={(event) => event.preventDefault()}
> >
<SheetHeader className="mb-4"> <SheetHeader>
<RowContainer> <RowContainer>
<div className="overflow-hidden mr-8"> <div className="overflow-hidden mr-8">
{ {
@ -287,7 +409,7 @@ const SidePanel = () => {
style={{ height: windowSize.height - 160 }} style={{ height: windowSize.height - 160 }}
className="pr-3" className="pr-3"
> >
<div className="flex flex-col gap-3"> <div className="flex flex-col gap-4 mt-4">
<RowContainer> <RowContainer>
<Label htmlFor="cropper">Cropper</Label> <Label htmlFor="cropper">Cropper</Label>
<Switch <Switch
@ -299,50 +421,83 @@ const SidePanel = () => {
/> />
</RowContainer> </RowContainer>
<RowContainer> <div className="flex flex-col gap-1">
<Label htmlFor="steps">Steps</Label> <Label htmlFor="steps">Steps</Label>
<NumberInput <RowContainer>
id="steps" <Slider
className="w-14" className="w-[180px]"
numberValue={settings.sdSteps} defaultValue={[30]}
allowFloat={false} min={1}
onNumberValueChange={(value) => { max={100}
updateSettings({ sdSteps: value }) step={1}
}} value={[Math.floor(settings.sdSteps)]}
/> onValueChange={(vals) => updateSettings({ sdSteps: vals[0] })}
</RowContainer> />
<NumberInput
id="steps"
className="w-[60px] rounded-full"
numberValue={settings.sdSteps}
allowFloat={false}
onNumberValueChange={(val) => {
updateSettings({ sdSteps: val })
}}
/>
</RowContainer>
</div>
<RowContainer> <div className="flex flex-col gap-1">
<Label htmlFor="guidance-scale">Guidance scale</Label> <Label htmlFor="guidance-scale">Guidance scale</Label>
<NumberInput <RowContainer>
id="guidance-scale" <Slider
className="w-14" className="w-[180px]"
numberValue={settings.sdGuidanceScale} defaultValue={[750]}
allowFloat min={100}
onNumberValueChange={(value) => { max={1500}
updateSettings({ sdGuidanceScale: value }) step={1}
}} value={[Math.floor(settings.sdGuidanceScale * 100)]}
/> onValueChange={(vals) =>
</RowContainer> updateSettings({ sdGuidanceScale: vals[0] / 100 })
}
/>
<NumberInput
id="guidance-scale"
className="w-[60px] rounded-full"
numberValue={settings.sdGuidanceScale}
allowFloat
onNumberValueChange={(val) => {
updateSettings({ sdGuidanceScale: val })
}}
/>
</RowContainer>
</div>
<RowContainer> {renderP2PImageGuidanceScale()}
<div className="flex gap-2 items-center">
<Label htmlFor="strength">Strength</Label> <div className="flex flex-col gap-1">
<div className="text-sm">({settings.sdStrength})</div> <Label htmlFor="strength">Strength</Label>
</div> <RowContainer>
<Slider <Slider
className="w-24" className="w-[180px]"
defaultValue={[100]} defaultValue={[100]}
min={10} min={10}
max={100} max={100}
step={1} step={1}
tabIndex={-1} value={[Math.floor(settings.sdStrength * 100)]}
value={[Math.floor(settings.sdStrength * 100)]} onValueChange={(vals) =>
onValueChange={(vals) => updateSettings({ sdStrength: vals[0] / 100 })
updateSettings({ sdStrength: vals[0] / 100 }) }
} />
/> <NumberInput
</RowContainer> id="strength"
className="w-[60px] rounded-full"
numberValue={settings.sdStrength}
allowFloat
onNumberValueChange={(val) => {
updateSettings({ sdStrength: val })
}}
/>
</RowContainer>
</div>
<RowContainer> <RowContainer>
<Label htmlFor="sampler">Sampler</Label> <Label htmlFor="sampler">Sampler</Label>
@ -383,7 +538,7 @@ const SidePanel = () => {
}} }}
/> />
<NumberInput <NumberInput
title="Seed" id="seed"
className="w-[100px]" className="w-[100px]"
disabled={!settings.seedFixed} disabled={!settings.seedFixed}
numberValue={settings.seed} numberValue={settings.seed}
@ -395,46 +550,39 @@ const SidePanel = () => {
</div> </div>
</RowContainer> </RowContainer>
<div className="flex flex-col gap-4"> {renderNegativePrompt()}
<Label htmlFor="negative-prompt">Negative prompt</Label>
<div className="pl-2 pr-4">
<Textarea
rows={4}
onKeyUp={onKeyUp}
className="max-h-[8rem] overflow-y-auto mb-2"
placeholder=""
id="negative-prompt"
value={settings.negativePrompt}
onInput={(evt: FormEvent<HTMLTextAreaElement>) => {
evt.preventDefault()
evt.stopPropagation()
const target = evt.target as HTMLTextAreaElement
updateSettings({ negativePrompt: target.value })
}}
/>
</div>
</div>
<Separator /> <Separator />
<div className="flex flex-col gap-4"> {renderConterNetSetting()}
{renderConterNetSetting()}
</div>
{renderFreeu()} {renderFreeu()}
{renderLCMLora()} {renderLCMLora()}
<RowContainer> <div className="flex flex-col gap-1">
<Label htmlFor="mask-blur">Mask blur</Label> <Label htmlFor="mask-blur">Mask blur</Label>
<NumberInput <RowContainer>
id="mask-blur" <Slider
className="w-14" className="w-[180px]"
numberValue={settings.sdMaskBlur} defaultValue={[5]}
allowFloat={false} min={0}
onNumberValueChange={(value) => { max={35}
updateSettings({ sdMaskBlur: value }) step={1}
}} value={[Math.floor(settings.sdMaskBlur)]}
/> onValueChange={(vals) =>
</RowContainer> updateSettings({ sdMaskBlur: vals[0] })
}
/>
<NumberInput
id="mask-blur"
className="w-[60px] rounded-full"
numberValue={settings.sdMaskBlur}
allowFloat={false}
onNumberValueChange={(value) => {
updateSettings({ sdMaskBlur: value })
}}
/>
</RowContainer>
</div>
<RowContainer> <RowContainer>
<Label htmlFor="match-histograms">Match histograms</Label> <Label htmlFor="match-histograms">Match histograms</Label>
@ -446,6 +594,10 @@ const SidePanel = () => {
}} }}
/> />
</RowContainer> </RowContainer>
<Separator />
{renderPaintByExample()}
</div> </div>
</ScrollArea> </ScrollArea>
</SheetContent> </SheetContent>

2
web_app/src/components/ui/select.tsx
View File

@ -25,6 +25,7 @@ const SelectTrigger = React.forwardRef<
"flex h-9 w-full items-center justify-between whitespace-nowrap rounded-md border border-input bg-transparent px-3 py-2 text-sm shadow-sm ring-offset-background placeholder:text-muted-foreground focus:outline-none focus:ring-1 focus:ring-ring disabled:cursor-not-allowed disabled:opacity-50 [&>span]:line-clamp-1", "flex h-9 w-full items-center justify-between whitespace-nowrap rounded-md border border-input bg-transparent px-3 py-2 text-sm shadow-sm ring-offset-background placeholder:text-muted-foreground focus:outline-none focus:ring-1 focus:ring-ring disabled:cursor-not-allowed disabled:opacity-50 [&>span]:line-clamp-1",
className className
)} )}
tabIndex={-1}
{...props} {...props}
> >
{children} {children}
@ -84,6 +85,7 @@ const SelectContent = React.forwardRef<
className className
)} )}
position={position} position={position}
onCloseAutoFocus={(event) => event.preventDefault()}
{...props} {...props}
> >
<SelectScrollUpButton /> <SelectScrollUpButton />

7
web_app/src/components/ui/slider.tsx
View File

@ -13,14 +13,15 @@ const Slider = React.forwardRef<
"relative flex w-full touch-none select-none items-center", "relative flex w-full touch-none select-none items-center",
className className
)} )}
tabIndex={-1}
{...props} {...props}
> >
<SliderPrimitive.Track className="relative h-1.5 w-full grow overflow-hidden rounded-full bg-primary/20"> <SliderPrimitive.Track className="relative h-1.5 w-full grow overflow-hidden rounded-full bg-primary/20 data-[disabled]:cursor-not-allowed">
<SliderPrimitive.Range className="absolute h-full bg-primary" /> <SliderPrimitive.Range className="absolute h-full bg-primary data-[disabled]:cursor-not-allowed " />
</SliderPrimitive.Track> </SliderPrimitive.Track>
<SliderPrimitive.Thumb <SliderPrimitive.Thumb
tabIndex={-1} tabIndex={-1}
className="block h-4 w-4 rounded-full border border-primary/60 bg-background shadow transition-colors focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring disabled:pointer-events-none disabled:opacity-50" className="block h-4 w-4 rounded-full border border-primary/60 bg-background shadow transition-colors focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring data-[disabled]:cursor-not-allowed"
/> />
</SliderPrimitive.Root> </SliderPrimitive.Root>
)) ))

1
web_app/src/components/ui/switch.tsx
View File

@ -12,6 +12,7 @@ const Switch = React.forwardRef<
"peer inline-flex h-5 w-9 shrink-0 cursor-pointer items-center rounded-full border-2 border-transparent shadow-sm transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 focus-visible:ring-offset-background disabled:cursor-not-allowed disabled:opacity-50 data-[state=checked]:bg-primary data-[state=unchecked]:bg-input", "peer inline-flex h-5 w-9 shrink-0 cursor-pointer items-center rounded-full border-2 border-transparent shadow-sm transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 focus-visible:ring-offset-background disabled:cursor-not-allowed disabled:opacity-50 data-[state=checked]:bg-primary data-[state=unchecked]:bg-input",
className className
)} )}
tabIndex={-1}
{...props} {...props}
ref={ref} ref={ref}
> >

3
web_app/src/components/ui/textarea.tsx
View File

@ -11,7 +11,8 @@ const Textarea = React.forwardRef<HTMLTextAreaElement, TextareaProps>(
<textarea <textarea
className={cn( className={cn(
"flex min-h-[60px] w-full rounded-md border border-input bg-transparent px-3 py-2 text-sm shadow-sm placeholder:text-muted-foreground focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring disabled:cursor-not-allowed disabled:opacity-50", "flex min-h-[60px] w-full rounded-md border border-input bg-transparent px-3 py-2 text-sm shadow-sm placeholder:text-muted-foreground focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring disabled:cursor-not-allowed disabled:opacity-50",
className className,
"overflow-auto"
)} )}
tabIndex={-1} tabIndex={-1}
ref={ref} ref={ref}

16
web_app/src/lib/api.ts
View File

@ -51,20 +51,6 @@ export default async function inpaint(
fd.append("cv2Radius", settings.cv2Radius.toString()) fd.append("cv2Radius", settings.cv2Radius.toString())
fd.append("cv2Flag", settings.cv2Flag.toString()) fd.append("cv2Flag", settings.cv2Flag.toString())
fd.append("paintByExampleSteps", settings.paintByExampleSteps.toString())
fd.append(
"paintByExampleGuidanceScale",
settings.paintByExampleGuidanceScale.toString()
)
fd.append("paintByExampleSeed", settings.seed.toString())
fd.append(
"paintByExampleMaskBlur",
settings.paintByExampleMaskBlur.toString()
)
fd.append(
"paintByExampleMatchHistograms",
settings.paintByExampleMatchHistograms ? "true" : "false"
)
// TODO: resize image's shortest_edge to 224 before pass to backend, save network time? // TODO: resize image's shortest_edge to 224 before pass to backend, save network time?
// https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPImageProcessor // https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPImageProcessor
if (paintByExampleImage) { if (paintByExampleImage) {
@ -72,9 +58,7 @@ export default async function inpaint(
} }
// InstructPix2Pix // InstructPix2Pix
fd.append("p2pSteps", settings.p2pSteps.toString())
fd.append("p2pImageGuidanceScale", settings.p2pImageGuidanceScale.toString()) fd.append("p2pImageGuidanceScale", settings.p2pImageGuidanceScale.toString())
fd.append("p2pGuidanceScale", settings.p2pGuidanceScale.toString())
// ControlNet // ControlNet
fd.append( fd.append(

2
web_app/src/lib/const.ts
View File

@ -11,3 +11,5 @@ export const BRUSH_COLOR = "#ffcc00bb"
export const PAINT_BY_EXAMPLE = "Fantasy-Studio/Paint-by-Example" export const PAINT_BY_EXAMPLE = "Fantasy-Studio/Paint-by-Example"
export const INSTRUCT_PIX2PIX = "timbrooks/instruct-pix2pix" export const INSTRUCT_PIX2PIX = "timbrooks/instruct-pix2pix"
export const KANDINSKY_2_2 = "kandinsky-community/kandinsky-2-2-decoder-inpaint" export const KANDINSKY_2_2 = "kandinsky-community/kandinsky-2-2-decoder-inpaint"
export const DEFAULT_NEGATIVE_PROMPT =
"out of frame, lowres, error, cropped, worst quality, low quality, jpeg artifacts, ugly, duplicate, morbid, mutilated, out of frame, mutation, deformed, blurry, dehydrated, bad anatomy, bad proportions, extra limbs, disfigured, gross proportions, malformed limbs, watermark, signature"

58
web_app/src/lib/states.ts
View File

@ -2,7 +2,6 @@ import { persist } from "zustand/middleware"
import { shallow } from "zustand/shallow" import { shallow } from "zustand/shallow"
import { immer } from "zustand/middleware/immer" import { immer } from "zustand/middleware/immer"
import { castDraft } from "immer" import { castDraft } from "immer"
import { nanoid } from "nanoid"
import { createWithEqualityFn } from "zustand/traditional" import { createWithEqualityFn } from "zustand/traditional"
import { import {
CV2Flag, CV2Flag,
@ -20,14 +19,13 @@ import {
} from "./types" } from "./types"
import { import {
DEFAULT_BRUSH_SIZE, DEFAULT_BRUSH_SIZE,
INSTRUCT_PIX2PIX, DEFAULT_NEGATIVE_PROMPT,
MODEL_TYPE_INPAINT, MODEL_TYPE_INPAINT,
MODEL_TYPE_OTHER,
PAINT_BY_EXAMPLE, PAINT_BY_EXAMPLE,
} from "./const" } from "./const"
import { dataURItoBlob, generateMask, loadImage, srcToFile } from "./utils" import { dataURItoBlob, generateMask, loadImage, srcToFile } from "./utils"
import inpaint, { runPlugin } from "./api" import inpaint, { runPlugin } from "./api"
import { toast, useToast } from "@/components/ui/use-toast" import { toast } from "@/components/ui/use-toast"
type FileManagerState = { type FileManagerState = {
sortBy: SortBy sortBy: SortBy
@ -78,19 +76,11 @@ export type Settings = {
sdMatchHistograms: boolean sdMatchHistograms: boolean
sdScale: number sdScale: number
// Paint by Example // Pix2Pix
paintByExampleSteps: number
paintByExampleGuidanceScale: number
paintByExampleMaskBlur: number
paintByExampleMatchHistograms: boolean
// InstructPix2Pix
p2pSteps: number
p2pImageGuidanceScale: number p2pImageGuidanceScale: number
p2pGuidanceScale: number
// ControlNet // ControlNet
enableControlNet: boolean enableControlnet: boolean
controlnetConditioningScale: number controlnetConditioningScale: number
controlnetMethod: string controlnetMethod: string
@ -103,6 +93,8 @@ type ServerConfig = {
plugins: string[] plugins: string[]
enableFileManager: boolean enableFileManager: boolean
enableAutoSaving: boolean enableAutoSaving: boolean
enableControlnet: boolean
controlnetMethod: string
} }
type InteractiveSegState = { type InteractiveSegState = {
@ -117,7 +109,6 @@ type EditorState = {
baseBrushSize: number baseBrushSize: number
brushSizeScale: number brushSizeScale: number
renders: HTMLImageElement[] renders: HTMLImageElement[]
paintByExampleImage: File | null
lineGroups: LineGroup[] lineGroups: LineGroup[]
lastLineGroup: LineGroup lastLineGroup: LineGroup
curLineGroup: LineGroup curLineGroup: LineGroup
@ -130,6 +121,7 @@ type EditorState = {
type AppState = { type AppState = {
file: File | null file: File | null
paintByExampleFile: File | null
customMask: File | null customMask: File | null
imageHeight: number imageHeight: number
imageWidth: number imageWidth: number
@ -195,6 +187,7 @@ type AppAction = {
const defaultValues: AppState = { const defaultValues: AppState = {
file: null, file: null,
paintByExampleFile: null,
customMask: null, customMask: null,
imageHeight: 0, imageHeight: 0,
imageWidth: 0, imageWidth: 0,
@ -210,7 +203,6 @@ const defaultValues: AppState = {
baseBrushSize: DEFAULT_BRUSH_SIZE, baseBrushSize: DEFAULT_BRUSH_SIZE,
brushSizeScale: 1, brushSizeScale: 1,
renders: [], renders: [],
paintByExampleImage: null,
extraMasks: [], extraMasks: [],
lineGroups: [], lineGroups: [],
lastLineGroup: [], lastLineGroup: [],
@ -246,6 +238,8 @@ const defaultValues: AppState = {
plugins: [], plugins: [],
enableFileManager: false, enableFileManager: false,
enableAutoSaving: false, enableAutoSaving: false,
enableControlnet: false,
controlnetMethod: "lllyasviel/control_v11p_sd15_canny",
}, },
settings: { settings: {
model: { model: {
@ -259,7 +253,7 @@ const defaultValues: AppState = {
is_single_file_diffusers: false, is_single_file_diffusers: false,
need_prompt: false, need_prompt: false,
}, },
enableControlNet: false, enableControlnet: false,
showCroper: false, showCroper: false,
enableDownloadMask: false, enableDownloadMask: false,
enableManualInpainting: false, enableManualInpainting: false,
@ -270,7 +264,7 @@ const defaultValues: AppState = {
cv2Radius: 5, cv2Radius: 5,
cv2Flag: CV2Flag.INPAINT_NS, cv2Flag: CV2Flag.INPAINT_NS,
prompt: "", prompt: "",
negativePrompt: "", negativePrompt: DEFAULT_NEGATIVE_PROMPT,
seed: 42, seed: 42,
seedFixed: false, seedFixed: false,
sdMaskBlur: 5, sdMaskBlur: 5,
@ -280,13 +274,7 @@ const defaultValues: AppState = {
sdSampler: SDSampler.uni_pc, sdSampler: SDSampler.uni_pc,
sdMatchHistograms: false, sdMatchHistograms: false,
sdScale: 100, sdScale: 100,
paintByExampleSteps: 50,
paintByExampleGuidanceScale: 7.5,
paintByExampleMaskBlur: 5,
paintByExampleMatchHistograms: false,
p2pSteps: 50,
p2pImageGuidanceScale: 1.5, p2pImageGuidanceScale: 1.5,
p2pGuidanceScale: 7.5,
controlnetConditioningScale: 0.4, controlnetConditioningScale: 0.4,
controlnetMethod: "lllyasviel/control_v11p_sd15_canny", controlnetMethod: "lllyasviel/control_v11p_sd15_canny",
enableLCMLora: false, enableLCMLora: false,
@ -320,6 +308,7 @@ export const useStore = createWithEqualityFn<AppState & AppAction>()(
const { const {
isInpainting, isInpainting,
file, file,
paintByExampleFile,
imageWidth, imageWidth,
imageHeight, imageHeight,
settings, settings,
@ -332,13 +321,8 @@ export const useStore = createWithEqualityFn<AppState & AppAction>()(
if (file === null) { if (file === null) {
return return
} }
const { const { lastLineGroup, curLineGroup, lineGroups, renders } =
lastLineGroup, get().editorState
curLineGroup,
lineGroups,
renders,
paintByExampleImage,
} = get().editorState
const { interactiveSegMask, prevInteractiveSegMask } = const { interactiveSegMask, prevInteractiveSegMask } =
get().interactiveSegState get().interactiveSegState
@ -413,7 +397,7 @@ export const useStore = createWithEqualityFn<AppState & AppAction>()(
settings, settings,
cropperState, cropperState,
dataURItoBlob(maskCanvas.toDataURL()), dataURItoBlob(maskCanvas.toDataURL()),
paintByExampleImage paintByExampleFile
) )
if (!res) { if (!res) {
@ -687,6 +671,8 @@ export const useStore = createWithEqualityFn<AppState & AppAction>()(
setServerConfig: (newValue: ServerConfig) => { setServerConfig: (newValue: ServerConfig) => {
set((state) => { set((state) => {
state.serverConfig = newValue state.serverConfig = newValue
state.settings.enableControlnet = newValue.enableControlnet
state.settings.controlnetMethod = newValue.controlnetMethod
}) })
}, },
@ -804,7 +790,7 @@ export const useStore = createWithEqualityFn<AppState & AppAction>()(
})), })),
{ {
name: "ZUSTAND_STATE", // name of the item in the storage (must be unique) name: "ZUSTAND_STATE", // name of the item in the storage (must be unique)
version: 0, version: 1,
partialize: (state) => partialize: (state) =>
Object.fromEntries( Object.fromEntries(
Object.entries(state).filter(([key]) => Object.entries(state).filter(([key]) =>
@ -815,9 +801,3 @@ export const useStore = createWithEqualityFn<AppState & AppAction>()(
), ),
shallow shallow
) )
// export const useStore = <U>(selector: (state: AppState & AppAction) => U) => {
// return createWithEqualityFn(selector, shallow)
// }
// export const useStore = createWithEqualityFn(useBaseStore, shallow)