kamp/anything-llm
1
0
Fork 0
mirror of https://github.com/Mintplex-Labs/anything-llm.git synced 2024-07-02 15:20:37 +02:00
Code Issues Packages Projects Releases Wiki Activity

improve citations to show all text chunks referred and expand the citation to view full referenced text (#161)

Browse Source 
* improve citations to show all text chunks referred and expand the citation to view full referenced text
chunk text of same document together

* remove debug
This commit is contained in:
Timothy Carambat 2023-07-27 22:33:27 -07:00 committed by GitHub
parent ab7837068b
commit 0a2f837fb2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 146 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
frontend/src/components/Sidebar/index.jsx
View File

@ -129,7 +129,7 @@ export default function Sidebar() {
</div>
<a
href={paths.mailToMintplex()}
className="transition-all duration-300 text-xs text-slate-200 dark:text-slate-600 hover:text-blue-600 dark:hover:text-blue-400"
className="transition-all duration-300 text-xs text-slate-500 dark:text-slate-600 hover:text-blue-600 dark:hover:text-blue-400"
>
@MintplexLabs
</a>
@ -295,7 +295,7 @@ export function SidebarMobileHeader() {
</div>
<a
href={paths.mailToMintplex()}
className="transition-all duration-300 text-xs text-slate-200 dark:text-slate-600 hover:text-blue-600 dark:hover:text-blue-400"
className="transition-all duration-300 text-xs text-slate-500 dark:text-slate-600 hover:text-blue-600 dark:hover:text-blue-400"
>
@MintplexLabs
</a>

99
frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/Citation/index.jsx Normal file
View File

@ -0,0 +1,99 @@
import { memo, useState } from "react";
import { Maximize2, Minimize2 } from "react-feather";
import { v4 } from "uuid";
import { decode as HTMLDecode } from "he";
function combineLikeSources(sources) {
const combined = {};
sources.forEach((source) => {
const { id, title, text } = source;
if (combined.hasOwnProperty(title)) {
combined[title].text += `\n\n ---- Chunk ${id || ""} ---- \n\n${text}`;
combined[title].references += 1;
} else {
combined[title] = { title, text, references: 1 };
}
});
return Object.values(combined);
}
export default function Citations({ sources = [] }) {
if (sources.length === 0) return null;
return (
<div className="flex flex-col mt-4 justify-left">
<div className="no-scroll flex flex-col justify-left overflow-x-scroll ">
<div className="w-full flex overflow-x-scroll items-center gap-4 mt-1 doc__source">
{combineLikeSources(sources).map((source) => (
<Citation id={source?.id || v4()} source={source} />
))}
</div>
</div>
<p className="w-fit text-gray-700 dark:text-stone-400 text-xs mt-1">
*citations may not be relevant to end result.
</p>
</div>
);
}
const Citation = memo(({ source, id }) => {
const [maximized, setMaximized] = useState(false);
const { references = 0, title, text } = source;
if (title?.length === 0 || text?.length === 0) return null;
const handleMinMax = () => {
setMaximized(!maximized);
Array.from(
document?.querySelectorAll(
`div[data-citation]:not([data-citation="${id}"])`
)
).forEach((el) => {
const func = maximized ? "remove" : "add";
el.classList[func]("hidden");
});
};
return (
<div
key={id || v4()}
data-citation={id || v4()}
className={`transition-all duration-300 relative flex flex-col w-full md:w-80 h-40 bg-gray-100 dark:bg-stone-800 border border-gray-700 dark:border-stone-800 rounded-lg shrink-0 ${
maximized ? "md:w-full h-fit pb-4" : ""
}`}
>
<div className="rounded-t-lg bg-gray-300 dark:bg-stone-900 px-4 py-2 w-full h-fit flex items-center justify-between">
<p className="text-base text-gray-800 dark:text-slate-400 italic truncate w-3/4">
{title}
</p>
<button
onClick={handleMinMax}
className="hover:dark:bg-stone-800 hover:bg-gray-200 dark:text-slate-400 text-gray-800 rounded-full p-1"
>
{maximized ? (
<Minimize2 className="h-4 w-4" />
) : (
<Maximize2 className="h-4 w-4" />
)}
</button>
</div>
<div
className={`overflow-hidden relative w-full ${
maximized ? "overflow-y-scroll" : ""
}`}
>
<p className="px-2 py-1 text-xs whitespace-pre-line text-gray-800 dark:text-slate-300 italic">
{references > 1 && (
<p className="text-xs text-gray-500 dark:text-slate-500 mb-2">
referenced {references} times.
</p>
)}
{HTMLDecode(text)}
</p>
<div
className={`absolute bottom-0 flex w-full h-[20px] fade-up-border rounded-b-lg ${
maximized ? "hidden" : ""
}`}
/>
</div>
</div>
);
});

49
frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/index.jsx
View File

@ -1,10 +1,9 @@
import { useEffect, useRef, memo, useState } from "react";
import { useEffect, useRef, memo } from "react";
import { AlertTriangle } from "react-feather";
import Jazzicon from "../../../../UserIcon";
import { v4 } from "uuid";
import { decode as HTMLDecode } from "he";
import renderMarkdown from "../../../../../utils/chat/markdown";
import { userFromStorage } from "../../../../../utils/request";
import Citations from "../Citation";
function HistoricalMessage({
message,
@ -55,7 +54,7 @@ function HistoricalMessage({
<Jazzicon size={30} user={{ uid: workspace.slug }} />
<div className="ml-2 py-3 px-4 overflow-x-scroll w-fit md:max-w-[75%] bg-orange-100 dark:bg-stone-700 rounded-t-2xl rounded-br-2xl rounded-bl-sm">
<span
className="whitespace-pre-line text-slate-800 dark:text-slate-200 font-[500] md:font-semibold text-sm md:text-base flex flex-col gap-y-1"
className="no-scroll whitespace-pre-line text-slate-800 dark:text-slate-200 font-[500] md:font-semibold text-sm md:text-base flex flex-col gap-y-1"
dangerouslySetInnerHTML={{ __html: renderMarkdown(message) }}
/>
<Citations sources={sources} />
@ -64,46 +63,4 @@ function HistoricalMessage({
);
}
const Citations = ({ sources = [] }) => {
const [show, setShow] = useState(false);
if (sources.length === 0) return null;
return (
<div className="flex flex-col mt-4 justify-left">
<button
type="button"
onClick={() => setShow(!show)}
className="w-fit text-gray-700 dark:text-stone-400 italic text-xs"
>
{show ? "hide" : "show"} citations{show && "*"}
</button>
{show && (
<>
<div className="w-full flex flex-wrap items-center gap-4 mt-1 doc__source">
{sources.map((source) => {
const { id = null, title, url } = source;
const handleClick = () => {
if (!url) return false;
window.open(url, "_blank");
};
return (
<button
key={id || v4()}
onClick={handleClick}
className="italic transition-all duration-300 w-fit bg-gray-400 text-gray-900 py-[1px] hover:text-slate-200 hover:bg-gray-500 hover:dark:text-gray-900 dark:bg-stone-400 dark:hover:bg-stone-300 rounded-full px-2 text-xs leading-tight"
>
"{HTMLDecode(title)}"
</button>
);
})}
</div>
<p className="w-fit text-gray-700 dark:text-stone-400 text-xs mt-1">
*citation may not be relevant to end result.
</p>
</>
)}
</div>
);
};
export default memo(HistoricalMessage);

47
frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/PromptReply/index.jsx
View File

@ -1,9 +1,8 @@
import { memo, useEffect, useRef, useState } from "react";
import { memo, useEffect, useRef } from "react";
import { AlertTriangle } from "react-feather";
import Jazzicon from "../../../../UserIcon";
import { v4 } from "uuid";
import { decode as HTMLDecode } from "he";
import renderMarkdown from "../../../../../utils/chat/markdown";
import Citations from "../Citation";
function PromptReply({
uuid,
@ -69,46 +68,4 @@ function PromptReply({
);
}
const Citations = ({ sources = [] }) => {
const [show, setShow] = useState(false);
if (sources.length === 0) return null;
return (
<div className="flex flex-col mt-4 justify-left">
<button
type="button"
onClick={() => setShow(!show)}
className="w-fit text-gray-700 dark:text-stone-400 italic text-xs"
>
{show ? "hide" : "show"} citations{show && "*"}
</button>
{show && (
<>
<div className="w-full flex flex-wrap items-center gap-4 mt-1 doc__source">
{sources.map((source) => {
const { id = null, title, url } = source;
const handleClick = () => {
if (!url) return false;
window.open(url, "_blank");
};
return (
<button
key={id || v4()}
onClick={handleClick}
className="italic transition-all duration-300 w-fit bg-gray-400 text-gray-900 py-[1px] hover:text-slate-200 hover:bg-gray-500 hover:dark:text-gray-900 dark:bg-stone-400 dark:hover:bg-stone-300 rounded-full px-2 text-xs leading-tight"
>
"{HTMLDecode(title)}"
</button>
);
})}
</div>
<p className="w-fit text-gray-700 dark:text-stone-400 text-xs mt-1">
*citation may not be relevant to end result.
</p>
</>
)}
</div>
);
};
export default memo(PromptReply);

24
frontend/src/components/WorkspaceChat/ChatContainer/PromptInput/index.jsx
View File

@ -65,7 +65,7 @@ export default function PromptInput({
<button
onClick={() => setShowMenu(!showMenu)}
type="button"
className="p-2 text-slate-200 bg-transparent rounded-md hover:bg-gray-50 dark:hover:bg-stone-500"
className="p-2 text-slate-500 bg-transparent rounded-md hover:bg-gray-200 dark:hover:bg-stone-500 dark:hover:text-slate-200"
>
<Menu className="w-4 h-4 md:h-6 md:w-6" />
</button>
@ -93,14 +93,14 @@ export default function PromptInput({
ref={formRef}
type="submit"
disabled={buttonDisabled}
className="inline-flex justify-center p-0 md:p-2 rounded-full cursor-pointer text-black-900 dark:text-slate-200 hover:bg-gray-600 dark:hover:bg-stone-500"
className="inline-flex justify-center p-0 md:p-2 rounded-full cursor-pointer text-black-900 dark:text-slate-200 hover:bg-gray-200 dark:hover:bg-stone-500 group"
>
{buttonDisabled ? (
<Loader className="w-6 h-6 animate-spin" />
) : (
<svg
aria-hidden="true"
className="w-6 h-6 rotate-45"
className="w-6 h-6 rotate-45 fill-gray-500 dark:fill-slate-500 group-hover:dark:fill-slate-200"
fill="currentColor"
viewBox="0 0 20 20"
xmlns="http://www.w3.org/2000/svg"
@ -138,7 +138,7 @@ const Tracking = memo(({ workspaceSlug }) => {
return (
<div className="flex flex-col md:flex-row w-full justify-center items-center gap-2 mb-2 px-4 mx:px-0">
<p className="bg-stone-600 text-slate-400 text-xs px-2 rounded-lg font-mono text-center">
<p className="bg-gray-200 dark:bg-stone-600 text-gray-800 dark:text-slate-400 text-xs px-2 rounded-lg font-mono text-center">
Chat mode: {chatMode}
</p>
<p className="text-slate-400 text-xs text-center">
@ -164,13 +164,13 @@ function CommandMenu({ workspace, show, handleClick, hide }) {
];
return (
<div className="absolute top-[-25vh] md:top-[-23vh] min-h-[200px] flex flex-col rounded-lg border border-slate-400 p-2 pt-4 bg-stone-600">
<div className="absolute top-[-25vh] md:top-[-23vh] min-h-[200px] flex flex-col rounded-lg border border-slate-400 p-2 pt-4 bg-gray-50 dark:bg-stone-600">
<div className="flex justify-between items-center border-b border-slate-400 px-2 py-1 ">
<p className="text-slate-200">Available Commands</p>
<p className="text-gray-800 dark:text-slate-200">Available Commands</p>
<button
type="button"
onClick={hide}
className="p-2 rounded-lg hover:bg-slate-500 rounded-full text-slate-400"
className="p-2 rounded-lg hover:bg-gray-200 hover:dark:bg-slate-500 rounded-full text-gray-800 dark:text-slate-400"
>
<X className="h-4 w-4" />
</button>
@ -188,10 +188,14 @@ function CommandMenu({ workspace, show, handleClick, hide }) {
handleClick(cmd);
hide();
}}
className="w-full px-4 py-2 flex items-center rounded-lg hover:bg-slate-500 gap-x-1 disabled:cursor-not-allowed"
className="w-full px-4 py-2 flex items-center rounded-lg hover:bg-gray-300 hover:dark:bg-slate-500 gap-x-1 disabled:cursor-not-allowed"
>
<p className="text-slate-200 font-semibold">{cmd}</p>
<p className="text-slate-400 text-sm">{description}</p>
<p className="text-gray-800 dark:text-slate-200 font-semibold">
{cmd}
</p>
<p className="text-gray-800 dark:text-slate-300 text-sm">
{description}
</p>
</button>
</div>
);

20
frontend/src/index.css
View File

@ -133,6 +133,26 @@ a {
}
}
@media (prefers-color-scheme: light) {
.fade-up-border {
background: linear-gradient(
to bottom,
rgba(220, 221, 223, 10%),
rgb(220, 221, 223) 89%
);
}
}
@media (prefers-color-scheme: dark) {
.fade-up-border {
background: linear-gradient(
to bottom,
rgba(41, 37, 36, 50%),
rgb(41 37 36) 90%
);
}
}
/**
* ==============================================
* Dot Falling

2
frontend/tailwind.config.js
View File

@ -5,7 +5,7 @@ export default {
extend: {
colors: {
'black-900': '#141414',
}
},
},
},
plugins: [],

10
server/utils/helpers/index.js
View File

@ -23,7 +23,6 @@ function toChunks(arr, size) {
}
function curateSources(sources = []) {
const knownDocs = [];
const documents = [];
// Sometimes the source may or may not have a metadata property
@ -32,17 +31,12 @@ function curateSources(sources = []) {
for (const source of sources) {
if (source.hasOwnProperty("metadata")) {
const { metadata = {} } = source;
if (
Object.keys(metadata).length > 0 &&
!knownDocs.includes(metadata.title)
) {
if (Object.keys(metadata).length > 0) {
documents.push({ ...metadata });
knownDocs.push(metadata.title);
}
} else {
if (Object.keys(source).length > 0 && !knownDocs.includes(source.title)) {
if (Object.keys(source).length > 0) {
documents.push({ ...source });
knownDocs.push(source.title);
}
}
}

11
server/utils/vectorDbProviders/lance/index.js
View File

@ -10,16 +10,11 @@ const { chatPrompt } = require("../../chats");
// Since we roll our own results for prompting we
// have to manually curate sources as well.
function curateLanceSources(sources = []) {
const knownDocs = [];
const documents = [];
for (const source of sources) {
const { text: _t, vector: _v, score: _s, ...metadata } = source;
if (
Object.keys(metadata).length > 0 &&
!knownDocs.includes(metadata.title)
) {
documents.push({ ...metadata });
knownDocs.push(metadata.title);
const { text, vector: _v, score: _s, ...metadata } = source;
if (Object.keys(metadata).length > 0) {
documents.push({ ...metadata, text });
}
}