add negative prompt

lama_cleaner/app/src/adapters/inpainting.ts

@@ -9,6 +9,7 @@ export default async function inpaint(
   settings: Settings,
   croperRect: Rect,
   prompt?: string,
+  negativePrompt?: string,
   sizeLimit?: string,
   seed?: number
 ) {
@@ -34,6 +35,10 @@ export default async function inpaint(
   )
 
   fd.append('prompt', prompt === undefined ? '' : prompt)
+  fd.append(
+    'negativePrompt',
+    negativePrompt === undefined ? '' : negativePrompt
+  )
   fd.append('croperX', croperRect.x.toString())
   fd.append('croperY', croperRect.y.toString())
   fd.append('croperHeight', croperRect.height.toString())

lama_cleaner/app/src/components/Editor/Editor.tsx

@@ -36,6 +36,7 @@ import {
   fileState,
   isInpaintingState,
   isSDState,
+  negativePropmtState,
   propmtState,
   runManuallyState,
   seedState,
@@ -88,6 +89,7 @@ function mouseXY(ev: SyntheticEvent) {
 export default function Editor() {
   const [file, setFile] = useRecoilState(fileState)
   const promptVal = useRecoilValue(propmtState)
+  const negativePromptVal = useRecoilValue(negativePropmtState)
   const settings = useRecoilValue(settingState)
   const [seedVal, setSeed] = useRecoilState(seedState)
   const croperRect = useRecoilValue(croperState)
@@ -261,6 +263,7 @@ export default function Editor() {
           settings,
           croperRect,
           prompt,
+          negativePromptVal,
           sizeLimit.toString(),
           sdSeed
         )
@@ -311,6 +314,7 @@ export default function Editor() {
       croperRect,
       sizeLimit,
       promptVal,
+      negativePromptVal,
       drawOnCurrentRender,
       hadDrawSomething,
       drawLinesOnMask,

lama_cleaner/app/src/components/Settings/SettingBlock.scss

@@ -25,6 +25,14 @@
   gap: 12rem;
 }
 
+.setting-block-content-v {
+  display: flex;
+  flex-direction: column;
+  justify-content: flex-start;
+  align-items: flex-start;
+  gap: 1rem;
+}
+
 .setting-block-content-title {
   display: flex;
   flex-direction: row;

lama_cleaner/app/src/components/Settings/SettingBlock.tsx

@@ -8,13 +8,18 @@ interface SettingBlockProps {
   input: ReactNode
   optionDesc?: ReactNode
   className?: string
+  layout?: string
 }
 
 function SettingBlock(props: SettingBlockProps) {
-  const { title, titleSuffix, desc, input, optionDesc, className } = props
+  const { title, titleSuffix, desc, input, optionDesc, className, layout } =
+    props
+  const contentClass =
+    layout === 'h' ? 'setting-block-content' : 'setting-block-content-v'
+
   return (
     <div className={`setting-block ${className}`}>
-      <div className="setting-block-content">
+      <div className={contentClass}>
         <div className="setting-block-content-title">
           <div style={{ display: 'flex', alignItems: 'center', gap: '12px' }}>
             {desc ? (
@@ -34,4 +39,8 @@ function SettingBlock(props: SettingBlockProps) {
   )
 }
 
+SettingBlock.defaultProps = {
+  layout: 'h',
+}
+
 export default SettingBlock

lama_cleaner/app/src/components/SidePanel/SidePanel.scss

@@ -55,3 +55,31 @@
   //   // border-radius: 4px;
   // }
 }
+
+.negative-prompt {
+  all: unset;
+  border-width: 0;
+  border-radius: 0.5rem;
+  min-height: 150px;
+  max-width: 200px;
+  width: 100%;
+  padding: 12px 0.8rem;
+  outline: 1px solid var(--border-color);
+
+  &:focus-visible {
+    border-width: 0;
+    outline: 1px solid var(--yellow-accent);
+  }
+
+  &:-webkit-input-placeholder {
+    padding-top: 10px;
+  }
+
+  &:-moz-input-placeholder {
+    padding-top: 10px;
+  }
+
+  &:-ms-input-placeholder {
+    padding-top: 10px;
+  }
+}

lama_cleaner/app/src/components/SidePanel/SidePanel.tsx

@@ -1,12 +1,13 @@
-import React, { useState } from 'react'
+import React, { FormEvent, useState } from 'react'
 import { useRecoilState } from 'recoil'
 import * as PopoverPrimitive from '@radix-ui/react-popover'
 import { useToggle } from 'react-use'
-import { SDSampler, settingState } from '../../store/Atoms'
+import { negativePropmtState, SDSampler, settingState } from '../../store/Atoms'
 import NumberInputSetting from '../Settings/NumberInputSetting'
 import SettingBlock from '../Settings/SettingBlock'
 import Selector from '../shared/Selector'
 import { Switch, SwitchThumb } from '../shared/Switch'
+import TextAreaInput from '../shared/Textarea'
 
 const INPUT_WIDTH = 30
 
@@ -14,6 +15,15 @@ const INPUT_WIDTH = 30
 const SidePanel = () => {
   const [open, toggleOpen] = useToggle(true)
   const [setting, setSettingState] = useRecoilState(settingState)
+  const [negativePrompt, setNegativePrompt] =
+    useRecoilState(negativePropmtState)
+
+  const handleOnInput = (evt: FormEvent<HTMLTextAreaElement>) => {
+    evt.preventDefault()
+    evt.stopPropagation()
+    const target = evt.target as HTMLTextAreaElement
+    setNegativePrompt(target.value)
+  }
 
   return (
     <div className="side-panel">
@@ -59,7 +69,7 @@ const SidePanel = () => {
               title="Steps"
               width={INPUT_WIDTH}
               value={`${setting.sdSteps}`}
-              desc="Large steps result in better result, but more time-consuming"
+              desc="The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference."
               onValue={value => {
                 const val = value.length === 0 ? 0 : parseInt(value, 10)
                 setSettingState(old => {
@@ -88,7 +98,7 @@ const SidePanel = () => {
               width={INPUT_WIDTH}
               allowFloat
               value={`${setting.sdGuidanceScale}`}
-              desc="TODO"
+              desc="Higher guidance scale encourages to generate images that are closely linked to the text prompt, usually at the expense of lower image quality."
               onValue={value => {
                 const val = value.length === 0 ? 0 : parseFloat(value)
                 setSettingState(old => {
@@ -101,7 +111,7 @@ const SidePanel = () => {
               title="Mask Blur"
               width={INPUT_WIDTH}
               value={`${setting.sdMaskBlur}`}
-              desc="TODO"
+              desc="Blur the edge of mask area. The higher the number the smoother blend with the original image"
               onValue={value => {
                 const val = value.length === 0 ? 0 : parseInt(value, 10)
                 setSettingState(old => {
@@ -167,6 +177,20 @@ const SidePanel = () => {
                 </div>
               }
             />
+
+            <SettingBlock
+              className="sub-setting-block"
+              title="Negative prompt"
+              layout="v"
+              input={
+                <TextAreaInput
+                  className="negative-prompt"
+                  value={negativePrompt}
+                  onInput={handleOnInput}
+                  placeholder=""
+                />
+              }
+            />
           </PopoverPrimitive.Content>
         </PopoverPrimitive.Portal>
       </PopoverPrimitive.Root>

lama_cleaner/app/src/components/shared/Input.tsx

@@ -1,5 +1,4 @@
 import React, { FocusEvent, InputHTMLAttributes, RefObject } from 'react'
-import { useClickAway } from 'react-use'
 import { useRecoilState } from 'recoil'
 import { appState } from '../../store/Atoms'
 

lama_cleaner/app/src/components/shared/Textarea.tsx

@@ -0,0 +1,45 @@
+import React, { FocusEvent, TextareaHTMLAttributes } from 'react'
+import { useRecoilState } from 'recoil'
+import { appState } from '../../store/Atoms'
+
+const TextAreaInput = React.forwardRef<
+  HTMLTextAreaElement,
+  TextareaHTMLAttributes<HTMLTextAreaElement>
+>((props, ref) => {
+  const { onFocus, onBlur, ...itemProps } = props
+  const [_, setAppState] = useRecoilState(appState)
+
+  const handleOnFocus = (evt: FocusEvent<any>) => {
+    setAppState(old => {
+      return { ...old, disableShortCuts: true }
+    })
+    onFocus?.(evt)
+  }
+
+  const handleOnBlur = (evt: FocusEvent<any>) => {
+    setAppState(old => {
+      return { ...old, disableShortCuts: false }
+    })
+    onBlur?.(evt)
+  }
+
+  return (
+    <textarea
+      {...itemProps}
+      ref={ref}
+      onFocus={handleOnFocus}
+      onBlur={handleOnBlur}
+      onPaste={evt => evt.stopPropagation()}
+      onKeyDown={e => {
+        if (e.key === 'Escape') {
+          e.currentTarget.blur()
+        }
+        if ((e.ctrlKey || e.metaKey) && e.key === 'z') {
+          e.stopPropagation()
+        }
+      }}
+    />
+  )
+})
+
+export default TextAreaInput

lama_cleaner/app/src/store/Atoms.tsx

@@ -43,6 +43,11 @@ export const propmtState = atom<string>({
   default: '',
 })
 
+export const negativePropmtState = atom<string>({
+  key: 'negativePromptState',
+  default: '',
+})
+
 export const isInpaintingState = selector({
   key: 'isInpainting',
   get: ({ get }) => {

lama_cleaner/model/sd.py

@@ -140,6 +140,7 @@ class SD(InpaintModel):
 
         output = self.model(
             prompt=config.prompt,
+            negative_prompt=config.negative_prompt,
             mask_image=PIL.Image.fromarray(mask[:, :, -1], mode="L"),
             strength=config.sd_strength,
             num_inference_steps=config.sd_steps,

lama_cleaner/schema.py

@@ -30,6 +30,7 @@ class Config(BaseModel):
     hd_strategy_resize_limit: int
 
     prompt: str = ""
+    negative_prompt: str = ""
     # 始终是在原图尺度上的值
     use_croper: bool = False
     croper_x: int = None

lama_cleaner/server.py

@@ -113,6 +113,7 @@ def process():
         hd_strategy_crop_trigger_size=form["hdStrategyCropTrigerSize"],
         hd_strategy_resize_limit=form["hdStrategyResizeLimit"],
         prompt=form["prompt"],
+        negative_prompt=form["negativePrompt"],
         use_croper=form["useCroper"],
         croper_x=form["croperX"],
         croper_y=form["croperY"],

lama_cleaner/tests/test_model.py

@@ -195,6 +195,44 @@ def test_runway_sd_1_5(sd_device, strategy, sampler, cpu_textencoder, disable_ns
     )
 
 
+@pytest.mark.parametrize("sd_device", ['cuda'])
+@pytest.mark.parametrize("strategy", [HDStrategy.ORIGINAL])
+@pytest.mark.parametrize("sampler", [SDSampler.ddim])
+def test_runway_sd_1_5_negative_prompt(sd_device, strategy, sampler):
+    def callback(i, t, latents):
+        pass
+
+    if sd_device == 'cuda' and not torch.cuda.is_available():
+        return
+
+    sd_steps = 50
+    model = ModelManager(name="sd1.5",
+                         device=sd_device,
+                         hf_access_token="",
+                         sd_run_local=True,
+                         sd_disable_nsfw=True,
+                         sd_cpu_textencoder=True,
+                         callback=callback)
+    cfg = get_config(
+        strategy,
+        sd_steps=sd_steps,
+        prompt='Face of a fox, high resolution, sitting on a park bench',
+        negative_prompt='orange, yellow, small',
+        sd_sampler=sampler
+    )
+
+    name = f"{sampler}_negative_prompt"
+
+    assert_equal(
+        model,
+        cfg,
+        f"runway_sd_{strategy.capitalize()}_{name}.png",
+        img_p=current_dir / "overture-creations-5sI6fQgYIuo.png",
+        mask_p=current_dir / "overture-creations-5sI6fQgYIuo_mask.png",
+        fx=1
+    )
+
+
 @pytest.mark.parametrize(
     "strategy", [HDStrategy.ORIGINAL, HDStrategy.RESIZE, HDStrategy.CROP]
 )