mirror of
https://github.com/Mintplex-Labs/anything-llm.git
synced 2024-10-02 08:50:11 +02:00
Enable ability to do full-text query on documents (#758)
* Enable ability to do full-text query on documents Show alert modal on first pin for client Add ability to use pins in stream/chat/embed * typo and copy update * simplify spread of context and sources
This commit is contained in:
parent
e63c426223
commit
791c0ee9dc
@ -1,10 +1,10 @@
|
|||||||
import UploadFile from "../UploadFile";
|
import UploadFile from "../UploadFile";
|
||||||
import PreLoader from "@/components/Preloader";
|
import PreLoader from "@/components/Preloader";
|
||||||
import { useEffect, useState } from "react";
|
import { memo, useEffect, useState } from "react";
|
||||||
import FolderRow from "./FolderRow";
|
import FolderRow from "./FolderRow";
|
||||||
import pluralize from "pluralize";
|
import pluralize from "pluralize";
|
||||||
|
|
||||||
export default function Directory({
|
function Directory({
|
||||||
files,
|
files,
|
||||||
loading,
|
loading,
|
||||||
setLoading,
|
setLoading,
|
||||||
@ -146,3 +146,5 @@ export default function Directory({
|
|||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export default memo(Directory);
|
||||||
|
@ -1,12 +1,19 @@
|
|||||||
import { useState } from "react";
|
import { memo, useState } from "react";
|
||||||
import {
|
import {
|
||||||
formatDate,
|
formatDate,
|
||||||
getFileExtension,
|
getFileExtension,
|
||||||
middleTruncate,
|
middleTruncate,
|
||||||
} from "@/utils/directories";
|
} from "@/utils/directories";
|
||||||
import { ArrowUUpLeft, File } from "@phosphor-icons/react";
|
import {
|
||||||
|
ArrowUUpLeft,
|
||||||
|
File,
|
||||||
|
PushPin,
|
||||||
|
PushPinSlash,
|
||||||
|
} from "@phosphor-icons/react";
|
||||||
import Workspace from "@/models/workspace";
|
import Workspace from "@/models/workspace";
|
||||||
import debounce from "lodash.debounce";
|
import debounce from "lodash.debounce";
|
||||||
|
import { Tooltip } from "react-tooltip";
|
||||||
|
import showToast from "@/utils/toast";
|
||||||
|
|
||||||
export default function WorkspaceFileRow({
|
export default function WorkspaceFileRow({
|
||||||
item,
|
item,
|
||||||
@ -80,21 +87,105 @@ export default function WorkspaceFileRow({
|
|||||||
<p className="col-span-2 pl-2 uppercase overflow-x-hidden">
|
<p className="col-span-2 pl-2 uppercase overflow-x-hidden">
|
||||||
{getFileExtension(item.url)}
|
{getFileExtension(item.url)}
|
||||||
</p>
|
</p>
|
||||||
<div className="col-span-2 flex justify-end items-center">
|
<div className="col-span-2 flex justify-center items-center">
|
||||||
{item?.cached && (
|
|
||||||
<div className="bg-white/10 rounded-3xl">
|
|
||||||
<p className="text-xs px-2 py-0.5">Cached</p>
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
{hasChanges ? (
|
{hasChanges ? (
|
||||||
<div className="w-4 h-4 ml-2 flex-shrink-0" />
|
<div className="w-4 h-4 ml-2 flex-shrink-0" />
|
||||||
) : (
|
) : (
|
||||||
<ArrowUUpLeft
|
<div className="flex gap-x-2 items-center">
|
||||||
onClick={onRemoveClick}
|
<PinItemToWorkspace
|
||||||
className="text-base font-bold w-4 h-4 ml-2 flex-shrink-0 cursor-pointer"
|
workspace={workspace}
|
||||||
|
docPath={`${folderName}/${item.name}`} // how to find documents during pin/unpin
|
||||||
|
item={item}
|
||||||
/>
|
/>
|
||||||
|
<RemoveItemFromWorkspace item={item} onClick={onRemoveClick} />
|
||||||
|
</div>
|
||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const PinItemToWorkspace = memo(({ workspace, docPath, item }) => {
|
||||||
|
const [pinned, setPinned] = useState(
|
||||||
|
item?.pinnedWorkspaces?.includes(workspace.id) || false
|
||||||
|
);
|
||||||
|
const [hover, setHover] = useState(false);
|
||||||
|
const pinEvent = new CustomEvent("pinned_document");
|
||||||
|
|
||||||
|
const updatePinStatus = async () => {
|
||||||
|
try {
|
||||||
|
if (!pinned) window.dispatchEvent(pinEvent);
|
||||||
|
const success = await Workspace.setPinForDocument(
|
||||||
|
workspace.slug,
|
||||||
|
docPath,
|
||||||
|
!pinned
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!success) {
|
||||||
|
showToast(`Failed to ${!pinned ? "pin" : "unpin"} document.`, "error", {
|
||||||
|
clear: true,
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
showToast(
|
||||||
|
`Document ${!pinned ? "pinned to" : "unpinned from"} workspace`,
|
||||||
|
"success",
|
||||||
|
{ clear: true }
|
||||||
|
);
|
||||||
|
setPinned(!pinned);
|
||||||
|
} catch (error) {
|
||||||
|
showToast(`Failed to pin document. ${error.message}`, "error", {
|
||||||
|
clear: true,
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
if (!item) return <div />;
|
||||||
|
|
||||||
|
const PinIcon = pinned ? PushPinSlash : PushPin;
|
||||||
|
return (
|
||||||
|
<div
|
||||||
|
onMouseEnter={() => setHover(true)}
|
||||||
|
onMouseLeave={() => setHover(false)}
|
||||||
|
>
|
||||||
|
<PinIcon
|
||||||
|
data-tooltip-id={`pin-${item.id}`}
|
||||||
|
data-tooltip-content={
|
||||||
|
pinned ? "Unpin document from workspace" : "Pin document to workspace"
|
||||||
|
}
|
||||||
|
onClick={updatePinStatus}
|
||||||
|
weight={hover ? "fill" : "regular"}
|
||||||
|
className={`outline-none text-base font-bold w-4 h-4 ml-2 flex-shrink-0 cursor-pointer ${
|
||||||
|
pinned ? "hover:text-red-300" : ""
|
||||||
|
}`}
|
||||||
|
/>
|
||||||
|
<Tooltip
|
||||||
|
id={`pin-${item.id}`}
|
||||||
|
place="bottom"
|
||||||
|
delayShow={300}
|
||||||
|
className="tooltip !text-xs"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
const RemoveItemFromWorkspace = ({ item, onClick }) => {
|
||||||
|
return (
|
||||||
|
<div>
|
||||||
|
<ArrowUUpLeft
|
||||||
|
data-tooltip-id={`remove-${item.id}`}
|
||||||
|
data-tooltip-content="Remove document from workspace"
|
||||||
|
onClick={onClick}
|
||||||
|
className="text-base font-bold w-4 h-4 ml-2 flex-shrink-0 cursor-pointer"
|
||||||
|
/>
|
||||||
|
<Tooltip
|
||||||
|
id={`remove-${item.id}`}
|
||||||
|
place="bottom"
|
||||||
|
delayShow={300}
|
||||||
|
className="tooltip !text-xs"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
};
|
||||||
|
@ -1,8 +1,12 @@
|
|||||||
import PreLoader from "@/components/Preloader";
|
import PreLoader from "@/components/Preloader";
|
||||||
import { dollarFormat } from "@/utils/numbers";
|
import { dollarFormat } from "@/utils/numbers";
|
||||||
import WorkspaceFileRow from "./WorkspaceFileRow";
|
import WorkspaceFileRow from "./WorkspaceFileRow";
|
||||||
|
import { memo, useEffect, useState } from "react";
|
||||||
|
import ModalWrapper from "@/components/ModalWrapper";
|
||||||
|
import { PushPin } from "@phosphor-icons/react";
|
||||||
|
import { SEEN_DOC_PIN_ALERT } from "@/utils/constants";
|
||||||
|
|
||||||
export default function WorkspaceDirectory({
|
function WorkspaceDirectory({
|
||||||
workspace,
|
workspace,
|
||||||
files,
|
files,
|
||||||
highlightWorkspace,
|
highlightWorkspace,
|
||||||
@ -29,7 +33,7 @@ export default function WorkspaceDirectory({
|
|||||||
<p className="col-span-5">Name</p>
|
<p className="col-span-5">Name</p>
|
||||||
<p className="col-span-3">Date</p>
|
<p className="col-span-3">Date</p>
|
||||||
<p className="col-span-2">Kind</p>
|
<p className="col-span-2">Kind</p>
|
||||||
<p className="col-span-2">Cached</p>
|
<p className="col-span-2" />
|
||||||
</div>
|
</div>
|
||||||
<div className="w-full h-full flex items-center justify-center flex-col gap-y-5">
|
<div className="w-full h-full flex items-center justify-center flex-col gap-y-5">
|
||||||
<PreLoader />
|
<PreLoader />
|
||||||
@ -43,6 +47,7 @@ export default function WorkspaceDirectory({
|
|||||||
}
|
}
|
||||||
|
|
||||||
return (
|
return (
|
||||||
|
<>
|
||||||
<div className="px-8">
|
<div className="px-8">
|
||||||
<div className="flex items-center justify-start w-[560px]">
|
<div className="flex items-center justify-start w-[560px]">
|
||||||
<h3 className="text-white text-base font-bold ml-5">
|
<h3 className="text-white text-base font-bold ml-5">
|
||||||
@ -58,7 +63,7 @@ export default function WorkspaceDirectory({
|
|||||||
<p className="col-span-5">Name</p>
|
<p className="col-span-5">Name</p>
|
||||||
<p className="col-span-3">Date</p>
|
<p className="col-span-3">Date</p>
|
||||||
<p className="col-span-2">Kind</p>
|
<p className="col-span-2">Kind</p>
|
||||||
<p className="col-span-2">Cached</p>
|
<p className="col-span-2" />
|
||||||
</div>
|
</div>
|
||||||
<div className="w-full h-full flex flex-col z-0">
|
<div className="w-full h-full flex flex-col z-0">
|
||||||
{Object.values(files.items).some(
|
{Object.values(files.items).some(
|
||||||
@ -116,5 +121,71 @@ export default function WorkspaceDirectory({
|
|||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
|
<PinAlert />
|
||||||
|
</>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const PinAlert = memo(() => {
|
||||||
|
const [showAlert, setShowAlert] = useState(false);
|
||||||
|
function dismissAlert() {
|
||||||
|
setShowAlert(false);
|
||||||
|
window.localStorage.setItem(SEEN_DOC_PIN_ALERT, "1");
|
||||||
|
window.removeEventListener(handlePinEvent);
|
||||||
|
}
|
||||||
|
|
||||||
|
function handlePinEvent() {
|
||||||
|
if (!!window?.localStorage?.getItem(SEEN_DOC_PIN_ALERT)) return;
|
||||||
|
setShowAlert(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!window || !!window?.localStorage?.getItem(SEEN_DOC_PIN_ALERT)) return;
|
||||||
|
window?.addEventListener("pinned_document", handlePinEvent);
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
return (
|
||||||
|
<ModalWrapper isOpen={showAlert}>
|
||||||
|
<div className="relative w-full max-w-2xl max-h-full">
|
||||||
|
<div className="relative bg-main-gradient rounded-lg shadow">
|
||||||
|
<div className="flex items-start justify-between p-4 border-b rounded-t border-gray-500/50">
|
||||||
|
<div className="flex items-center gap-2">
|
||||||
|
<PushPin className="text-red-600 text-lg w-6 h-6" weight="fill" />
|
||||||
|
<h3 className="text-xl font-semibold text-white">
|
||||||
|
What is document pinning?
|
||||||
|
</h3>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div className="w-full p-6 text-white text-md flex flex-col gap-y-2">
|
||||||
|
<p>
|
||||||
|
When you <b>pin</b> a document in AnythingLLM we will inject the
|
||||||
|
entire content of the document into your prompt window for your
|
||||||
|
LLM to fully comprehend.
|
||||||
|
</p>
|
||||||
|
<p>
|
||||||
|
This works best with <b>large-context models</b> or small files
|
||||||
|
that are critical to its knowledge-base.
|
||||||
|
</p>
|
||||||
|
<p>
|
||||||
|
If you are not getting the answers you desire from AnythingLLM by
|
||||||
|
default then pinning is a great way to get higher quality answers
|
||||||
|
in a click.
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="flex w-full justify-between items-center p-6 space-x-2 border-t rounded-b border-gray-500/50">
|
||||||
|
<button disabled={true} className="invisible" />
|
||||||
|
<button
|
||||||
|
onClick={dismissAlert}
|
||||||
|
className="transition-all duration-300 border border-slate-200 px-4 py-2 rounded-lg text-white text-sm items-center flex gap-x-2 hover:bg-slate-200 hover:text-slate-800 focus:ring-gray-800"
|
||||||
|
>
|
||||||
|
Okay, got it
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</ModalWrapper>
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
export default memo(WorkspaceDirectory);
|
||||||
|
@ -2,8 +2,8 @@ import { ArrowsDownUp } from "@phosphor-icons/react";
|
|||||||
import { useEffect, useState } from "react";
|
import { useEffect, useState } from "react";
|
||||||
import Workspace from "../../../../models/workspace";
|
import Workspace from "../../../../models/workspace";
|
||||||
import System from "../../../../models/system";
|
import System from "../../../../models/system";
|
||||||
import Directory from "./Directory";
|
|
||||||
import showToast from "../../../../utils/toast";
|
import showToast from "../../../../utils/toast";
|
||||||
|
import Directory from "./Directory";
|
||||||
import WorkspaceDirectory from "./WorkspaceDirectory";
|
import WorkspaceDirectory from "./WorkspaceDirectory";
|
||||||
|
|
||||||
// OpenAI Cost per token
|
// OpenAI Cost per token
|
||||||
|
@ -218,6 +218,25 @@ const Workspace = {
|
|||||||
return { success: false, error: e.message };
|
return { success: false, error: e.message };
|
||||||
});
|
});
|
||||||
},
|
},
|
||||||
|
setPinForDocument: async function (slug, docPath, pinStatus) {
|
||||||
|
return fetch(`${API_BASE}/workspace/${slug}/update-pin`, {
|
||||||
|
method: "POST",
|
||||||
|
headers: baseHeaders(),
|
||||||
|
body: JSON.stringify({ docPath, pinStatus }),
|
||||||
|
})
|
||||||
|
.then((res) => {
|
||||||
|
if (!res.ok) {
|
||||||
|
throw new Error(
|
||||||
|
res.statusText || "Error setting pin status for document."
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
})
|
||||||
|
.catch((e) => {
|
||||||
|
console.error(e);
|
||||||
|
return false;
|
||||||
|
});
|
||||||
|
},
|
||||||
threads: WorkspaceThread,
|
threads: WorkspaceThread,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -4,6 +4,7 @@ export const AUTH_USER = "anythingllm_user";
|
|||||||
export const AUTH_TOKEN = "anythingllm_authToken";
|
export const AUTH_TOKEN = "anythingllm_authToken";
|
||||||
export const AUTH_TIMESTAMP = "anythingllm_authTimestamp";
|
export const AUTH_TIMESTAMP = "anythingllm_authTimestamp";
|
||||||
export const COMPLETE_QUESTIONNAIRE = "anythingllm_completed_questionnaire";
|
export const COMPLETE_QUESTIONNAIRE = "anythingllm_completed_questionnaire";
|
||||||
|
export const SEEN_DOC_PIN_ALERT = "anythingllm_pinned_document_alert";
|
||||||
|
|
||||||
export const USER_BACKGROUND_COLOR = "bg-historical-msg-user";
|
export const USER_BACKGROUND_COLOR = "bg-historical-msg-user";
|
||||||
export const AI_BACKGROUND_COLOR = "bg-historical-msg-system";
|
export const AI_BACKGROUND_COLOR = "bg-historical-msg-system";
|
||||||
|
@ -395,6 +395,33 @@ function workspaceEndpoints(app) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
|
||||||
|
app.post(
|
||||||
|
"/workspace/:slug/update-pin",
|
||||||
|
[
|
||||||
|
validatedRequest,
|
||||||
|
flexUserRoleValid([ROLES.admin, ROLES.manager]),
|
||||||
|
validWorkspaceSlug,
|
||||||
|
],
|
||||||
|
async (request, response) => {
|
||||||
|
try {
|
||||||
|
const { docPath, pinStatus = false } = reqBody(request);
|
||||||
|
const workspace = response.locals.workspace;
|
||||||
|
|
||||||
|
const document = await Document.get({
|
||||||
|
workspaceId: workspace.id,
|
||||||
|
docpath: docPath,
|
||||||
|
});
|
||||||
|
if (!document) return response.sendStatus(404).end();
|
||||||
|
|
||||||
|
await Document.update(document.id, { pinned: pinStatus });
|
||||||
|
return response.status(200).end();
|
||||||
|
} catch (error) {
|
||||||
|
console.error("Error processing the pin status update:", error);
|
||||||
|
return response.status(500).end();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = { workspaceEndpoints };
|
module.exports = { workspaceEndpoints };
|
||||||
|
@ -1,4 +1,3 @@
|
|||||||
const { fileData } = require("../utils/files");
|
|
||||||
const { v4: uuidv4 } = require("uuid");
|
const { v4: uuidv4 } = require("uuid");
|
||||||
const { getVectorDbClass } = require("../utils/helpers");
|
const { getVectorDbClass } = require("../utils/helpers");
|
||||||
const prisma = require("../utils/prisma");
|
const prisma = require("../utils/prisma");
|
||||||
@ -6,6 +5,8 @@ const { Telemetry } = require("./telemetry");
|
|||||||
const { EventLogs } = require("./eventLogs");
|
const { EventLogs } = require("./eventLogs");
|
||||||
|
|
||||||
const Document = {
|
const Document = {
|
||||||
|
writable: ["pinned"],
|
||||||
|
|
||||||
forWorkspace: async function (workspaceId = null) {
|
forWorkspace: async function (workspaceId = null) {
|
||||||
if (!workspaceId) return [];
|
if (!workspaceId) return [];
|
||||||
return await prisma.workspace_documents.findMany({
|
return await prisma.workspace_documents.findMany({
|
||||||
@ -23,7 +24,7 @@ const Document = {
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
firstWhere: async function (clause = {}) {
|
get: async function (clause = {}) {
|
||||||
try {
|
try {
|
||||||
const document = await prisma.workspace_documents.findFirst({
|
const document = await prisma.workspace_documents.findFirst({
|
||||||
where: clause,
|
where: clause,
|
||||||
@ -35,9 +36,39 @@ const Document = {
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
|
getPins: async function (clause = {}) {
|
||||||
|
try {
|
||||||
|
const workspaceIds = await prisma.workspace_documents.findMany({
|
||||||
|
where: clause,
|
||||||
|
select: {
|
||||||
|
workspaceId: true,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
return workspaceIds.map((pin) => pin.workspaceId) || [];
|
||||||
|
} catch (error) {
|
||||||
|
console.error(error.message);
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
},
|
||||||
|
|
||||||
|
where: async function (clause = {}, limit = null, orderBy = null) {
|
||||||
|
try {
|
||||||
|
const results = await prisma.workspace_documents.findMany({
|
||||||
|
where: clause,
|
||||||
|
...(limit !== null ? { take: limit } : {}),
|
||||||
|
...(orderBy !== null ? { orderBy } : {}),
|
||||||
|
});
|
||||||
|
return results;
|
||||||
|
} catch (error) {
|
||||||
|
console.error(error.message);
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
},
|
||||||
|
|
||||||
addDocuments: async function (workspace, additions = [], userId = null) {
|
addDocuments: async function (workspace, additions = [], userId = null) {
|
||||||
const VectorDb = getVectorDbClass();
|
const VectorDb = getVectorDbClass();
|
||||||
if (additions.length === 0) return { failed: [], embedded: [] };
|
if (additions.length === 0) return { failed: [], embedded: [] };
|
||||||
|
const { fileData } = require("../utils/files");
|
||||||
const embedded = [];
|
const embedded = [];
|
||||||
const failedToEmbed = [];
|
const failedToEmbed = [];
|
||||||
const errors = new Set();
|
const errors = new Set();
|
||||||
@ -101,7 +132,7 @@ const Document = {
|
|||||||
if (removals.length === 0) return;
|
if (removals.length === 0) return;
|
||||||
|
|
||||||
for (const path of removals) {
|
for (const path of removals) {
|
||||||
const document = await this.firstWhere({
|
const document = await this.get({
|
||||||
docpath: path,
|
docpath: path,
|
||||||
workspaceId: workspace.id,
|
workspaceId: workspace.id,
|
||||||
});
|
});
|
||||||
@ -151,6 +182,26 @@ const Document = {
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
update: async function (id = null, data = {}) {
|
||||||
|
if (!id) throw new Error("No workspace document id provided for update");
|
||||||
|
|
||||||
|
const validKeys = Object.keys(data).filter((key) =>
|
||||||
|
this.writable.includes(key)
|
||||||
|
);
|
||||||
|
if (validKeys.length === 0)
|
||||||
|
return { document: { id }, message: "No valid fields to update!" };
|
||||||
|
|
||||||
|
try {
|
||||||
|
const document = await prisma.workspace_documents.update({
|
||||||
|
where: { id },
|
||||||
|
data,
|
||||||
|
});
|
||||||
|
return { document, message: null };
|
||||||
|
} catch (error) {
|
||||||
|
console.error(error.message);
|
||||||
|
return { document: null, message: error.message };
|
||||||
|
}
|
||||||
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
module.exports = { Document };
|
module.exports = { Document };
|
||||||
|
@ -0,0 +1,2 @@
|
|||||||
|
-- AlterTable
|
||||||
|
ALTER TABLE "workspace_documents" ADD COLUMN "pinned" BOOLEAN DEFAULT false;
|
@ -30,6 +30,7 @@ model workspace_documents {
|
|||||||
docpath String
|
docpath String
|
||||||
workspaceId Int
|
workspaceId Int
|
||||||
metadata String?
|
metadata String?
|
||||||
|
pinned Boolean? @default(false)
|
||||||
createdAt DateTime @default(now())
|
createdAt DateTime @default(now())
|
||||||
lastUpdatedAt DateTime @default(now())
|
lastUpdatedAt DateTime @default(now())
|
||||||
workspace workspaces @relation(fields: [workspaceId], references: [id])
|
workspace workspaces @relation(fields: [workspaceId], references: [id])
|
||||||
|
@ -195,10 +195,14 @@ class OpenAiLLM {
|
|||||||
`OpenAI chat: ${this.model} is not valid for chat completion!`
|
`OpenAI chat: ${this.model} is not valid for chat completion!`
|
||||||
);
|
);
|
||||||
|
|
||||||
const { data } = await this.openai.createChatCompletion({
|
const { data } = await this.openai
|
||||||
|
.createChatCompletion({
|
||||||
model: this.model,
|
model: this.model,
|
||||||
messages,
|
messages,
|
||||||
temperature,
|
temperature,
|
||||||
|
})
|
||||||
|
.catch((e) => {
|
||||||
|
throw new Error(e.response.data.error.message);
|
||||||
});
|
});
|
||||||
|
|
||||||
if (!data.hasOwnProperty("choices")) return null;
|
if (!data.hasOwnProperty("choices")) return null;
|
||||||
|
72
server/utils/DocumentManager/index.js
Normal file
72
server/utils/DocumentManager/index.js
Normal file
@ -0,0 +1,72 @@
|
|||||||
|
const fs = require("fs");
|
||||||
|
const path = require("path");
|
||||||
|
|
||||||
|
const documentsPath =
|
||||||
|
process.env.NODE_ENV === "development"
|
||||||
|
? path.resolve(__dirname, `../../storage/documents`)
|
||||||
|
: path.resolve(process.env.STORAGE_DIR, `documents`);
|
||||||
|
|
||||||
|
class DocumentManager {
|
||||||
|
constructor({ workspace = null, maxTokens = null }) {
|
||||||
|
this.workspace = workspace;
|
||||||
|
this.maxTokens = maxTokens || Number.POSITIVE_INFINITY;
|
||||||
|
this.documentStoragePath = documentsPath;
|
||||||
|
}
|
||||||
|
|
||||||
|
log(text, ...args) {
|
||||||
|
console.log(`\x1b[36m[DocumentManager]\x1b[0m ${text}`, ...args);
|
||||||
|
}
|
||||||
|
|
||||||
|
async pinnedDocuments() {
|
||||||
|
if (!this.workspace) return [];
|
||||||
|
const { Document } = require("../../models/documents");
|
||||||
|
return await Document.where({
|
||||||
|
workspaceId: Number(this.workspace.id),
|
||||||
|
pinned: true,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async pinnedDocs() {
|
||||||
|
if (!this.workspace) return [];
|
||||||
|
const docPaths = (await this.pinnedDocuments()).map((doc) => doc.docpath);
|
||||||
|
if (docPaths.length === 0) return [];
|
||||||
|
|
||||||
|
let tokens = 0;
|
||||||
|
const pinnedDocs = [];
|
||||||
|
for await (const docPath of docPaths) {
|
||||||
|
try {
|
||||||
|
const filePath = path.resolve(this.documentStoragePath, docPath);
|
||||||
|
const data = JSON.parse(
|
||||||
|
fs.readFileSync(filePath, { encoding: "utf-8" })
|
||||||
|
);
|
||||||
|
|
||||||
|
if (
|
||||||
|
!data.hasOwnProperty("pageContent") ||
|
||||||
|
!data.hasOwnProperty("token_count_estimate")
|
||||||
|
) {
|
||||||
|
this.log(
|
||||||
|
`Skipping document - Could not find page content or token_count_estimate in pinned source.`
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (tokens >= this.maxTokens) {
|
||||||
|
this.log(
|
||||||
|
`Skipping document - Token limit of ${this.maxTokens} has already been exceeded by pinned documents.`
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
pinnedDocs.push(data);
|
||||||
|
tokens += data.token_count_estimate || 0;
|
||||||
|
} catch {}
|
||||||
|
}
|
||||||
|
|
||||||
|
this.log(
|
||||||
|
`Found ${pinnedDocs.length} pinned sources - prepending to content with ~${tokens} tokens of content.`
|
||||||
|
);
|
||||||
|
return pinnedDocs;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports.DocumentManager = DocumentManager;
|
@ -6,6 +6,7 @@ const {
|
|||||||
convertToPromptHistory,
|
convertToPromptHistory,
|
||||||
writeResponseChunk,
|
writeResponseChunk,
|
||||||
} = require("../helpers/chat/responses");
|
} = require("../helpers/chat/responses");
|
||||||
|
const { DocumentManager } = require("../DocumentManager");
|
||||||
|
|
||||||
async function streamChatWithForEmbed(
|
async function streamChatWithForEmbed(
|
||||||
response,
|
response,
|
||||||
@ -64,6 +65,8 @@ async function streamChatWithForEmbed(
|
|||||||
}
|
}
|
||||||
|
|
||||||
let completeText;
|
let completeText;
|
||||||
|
let contextTexts = [];
|
||||||
|
let sources = [];
|
||||||
const { rawHistory, chatHistory } = await recentEmbedChatHistory(
|
const { rawHistory, chatHistory } = await recentEmbedChatHistory(
|
||||||
sessionId,
|
sessionId,
|
||||||
embed,
|
embed,
|
||||||
@ -71,11 +74,28 @@ async function streamChatWithForEmbed(
|
|||||||
chatMode
|
chatMode
|
||||||
);
|
);
|
||||||
|
|
||||||
const {
|
// Look for pinned documents and see if the user decided to use this feature. We will also do a vector search
|
||||||
contextTexts = [],
|
// as pinning is a supplemental tool but it should be used with caution since it can easily blow up a context window.
|
||||||
sources = [],
|
await new DocumentManager({
|
||||||
message: error,
|
workspace: embed.workspace,
|
||||||
} = embeddingsCount !== 0 // if there no embeddings don't bother searching.
|
maxTokens: LLMConnector.limits.system,
|
||||||
|
})
|
||||||
|
.pinnedDocs()
|
||||||
|
.then((pinnedDocs) => {
|
||||||
|
pinnedDocs.forEach((doc) => {
|
||||||
|
const { pageContent, ...metadata } = doc;
|
||||||
|
contextTexts.push(doc.pageContent);
|
||||||
|
sources.push({
|
||||||
|
text:
|
||||||
|
pageContent.slice(0, 1_000) +
|
||||||
|
"...continued on in source document...",
|
||||||
|
...metadata,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
const vectorSearchResults =
|
||||||
|
embeddingsCount !== 0
|
||||||
? await VectorDb.performSimilaritySearch({
|
? await VectorDb.performSimilaritySearch({
|
||||||
namespace: embed.workspace.slug,
|
namespace: embed.workspace.slug,
|
||||||
input: message,
|
input: message,
|
||||||
@ -89,8 +109,8 @@ async function streamChatWithForEmbed(
|
|||||||
message: null,
|
message: null,
|
||||||
};
|
};
|
||||||
|
|
||||||
// Failed similarity search.
|
// Failed similarity search if it was run at all and failed.
|
||||||
if (!!error) {
|
if (!!vectorSearchResults.message) {
|
||||||
writeResponseChunk(response, {
|
writeResponseChunk(response, {
|
||||||
id: uuid,
|
id: uuid,
|
||||||
type: "abort",
|
type: "abort",
|
||||||
@ -102,6 +122,9 @@ async function streamChatWithForEmbed(
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
contextTexts = [...contextTexts, ...vectorSearchResults.contextTexts];
|
||||||
|
sources = [...sources, ...vectorSearchResults.sources];
|
||||||
|
|
||||||
// If in query mode and no sources are found, do not
|
// If in query mode and no sources are found, do not
|
||||||
// let the LLM try to hallucinate a response or use general knowledge
|
// let the LLM try to hallucinate a response or use general knowledge
|
||||||
if (chatMode === "query" && sources.length === 0) {
|
if (chatMode === "query" && sources.length === 0) {
|
||||||
|
@ -3,6 +3,7 @@ const { WorkspaceChats } = require("../../models/workspaceChats");
|
|||||||
const { resetMemory } = require("./commands/reset");
|
const { resetMemory } = require("./commands/reset");
|
||||||
const { getVectorDbClass, getLLMProvider } = require("../helpers");
|
const { getVectorDbClass, getLLMProvider } = require("../helpers");
|
||||||
const { convertToPromptHistory } = require("../helpers/chat/responses");
|
const { convertToPromptHistory } = require("../helpers/chat/responses");
|
||||||
|
const { DocumentManager } = require("../DocumentManager");
|
||||||
|
|
||||||
const VALID_COMMANDS = {
|
const VALID_COMMANDS = {
|
||||||
"/reset": resetMemory,
|
"/reset": resetMemory,
|
||||||
@ -73,6 +74,8 @@ async function chatWithWorkspace(
|
|||||||
// If we are here we know that we are in a workspace that is:
|
// If we are here we know that we are in a workspace that is:
|
||||||
// 1. Chatting in "chat" mode and may or may _not_ have embeddings
|
// 1. Chatting in "chat" mode and may or may _not_ have embeddings
|
||||||
// 2. Chatting in "query" mode and has at least 1 embedding
|
// 2. Chatting in "query" mode and has at least 1 embedding
|
||||||
|
let contextTexts = [];
|
||||||
|
let sources = [];
|
||||||
const { rawHistory, chatHistory } = await recentChatHistory({
|
const { rawHistory, chatHistory } = await recentChatHistory({
|
||||||
user,
|
user,
|
||||||
workspace,
|
workspace,
|
||||||
@ -81,11 +84,28 @@ async function chatWithWorkspace(
|
|||||||
chatMode,
|
chatMode,
|
||||||
});
|
});
|
||||||
|
|
||||||
const {
|
// Look for pinned documents and see if the user decided to use this feature. We will also do a vector search
|
||||||
contextTexts = [],
|
// as pinning is a supplemental tool but it should be used with caution since it can easily blow up a context window.
|
||||||
sources = [],
|
await new DocumentManager({
|
||||||
message: error,
|
workspace,
|
||||||
} = embeddingsCount !== 0 // if there no embeddings don't bother searching.
|
maxTokens: LLMConnector.limits.system,
|
||||||
|
})
|
||||||
|
.pinnedDocs()
|
||||||
|
.then((pinnedDocs) => {
|
||||||
|
pinnedDocs.forEach((doc) => {
|
||||||
|
const { pageContent, ...metadata } = doc;
|
||||||
|
contextTexts.push(doc.pageContent);
|
||||||
|
sources.push({
|
||||||
|
text:
|
||||||
|
pageContent.slice(0, 1_000) +
|
||||||
|
"...continued on in source document...",
|
||||||
|
...metadata,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
const vectorSearchResults =
|
||||||
|
embeddingsCount !== 0
|
||||||
? await VectorDb.performSimilaritySearch({
|
? await VectorDb.performSimilaritySearch({
|
||||||
namespace: workspace.slug,
|
namespace: workspace.slug,
|
||||||
input: message,
|
input: message,
|
||||||
@ -100,17 +120,20 @@ async function chatWithWorkspace(
|
|||||||
};
|
};
|
||||||
|
|
||||||
// Failed similarity search if it was run at all and failed.
|
// Failed similarity search if it was run at all and failed.
|
||||||
if (!!error) {
|
if (!!vectorSearchResults.message) {
|
||||||
return {
|
return {
|
||||||
id: uuid,
|
id: uuid,
|
||||||
type: "abort",
|
type: "abort",
|
||||||
textResponse: null,
|
textResponse: null,
|
||||||
sources: [],
|
sources: [],
|
||||||
close: true,
|
close: true,
|
||||||
error,
|
error: vectorSearchResults.message,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
contextTexts = [...contextTexts, ...vectorSearchResults.contextTexts];
|
||||||
|
sources = [...sources, ...vectorSearchResults.sources];
|
||||||
|
|
||||||
// If in query mode and no sources are found, do not
|
// If in query mode and no sources are found, do not
|
||||||
// let the LLM try to hallucinate a response or use general knowledge and exit early
|
// let the LLM try to hallucinate a response or use general knowledge and exit early
|
||||||
if (chatMode === "query" && sources.length === 0) {
|
if (chatMode === "query" && sources.length === 0) {
|
||||||
|
@ -1,4 +1,5 @@
|
|||||||
const { v4: uuidv4 } = require("uuid");
|
const { v4: uuidv4 } = require("uuid");
|
||||||
|
const { DocumentManager } = require("../DocumentManager");
|
||||||
const { WorkspaceChats } = require("../../models/workspaceChats");
|
const { WorkspaceChats } = require("../../models/workspaceChats");
|
||||||
const { getVectorDbClass, getLLMProvider } = require("../helpers");
|
const { getVectorDbClass, getLLMProvider } = require("../helpers");
|
||||||
const { writeResponseChunk } = require("../helpers/chat/responses");
|
const { writeResponseChunk } = require("../helpers/chat/responses");
|
||||||
@ -74,6 +75,8 @@ async function streamChatWithWorkspace(
|
|||||||
// 1. Chatting in "chat" mode and may or may _not_ have embeddings
|
// 1. Chatting in "chat" mode and may or may _not_ have embeddings
|
||||||
// 2. Chatting in "query" mode and has at least 1 embedding
|
// 2. Chatting in "query" mode and has at least 1 embedding
|
||||||
let completeText;
|
let completeText;
|
||||||
|
let contextTexts = [];
|
||||||
|
let sources = [];
|
||||||
const { rawHistory, chatHistory } = await recentChatHistory({
|
const { rawHistory, chatHistory } = await recentChatHistory({
|
||||||
user,
|
user,
|
||||||
workspace,
|
workspace,
|
||||||
@ -82,11 +85,28 @@ async function streamChatWithWorkspace(
|
|||||||
chatMode,
|
chatMode,
|
||||||
});
|
});
|
||||||
|
|
||||||
const {
|
// Look for pinned documents and see if the user decided to use this feature. We will also do a vector search
|
||||||
contextTexts = [],
|
// as pinning is a supplemental tool but it should be used with caution since it can easily blow up a context window.
|
||||||
sources = [],
|
await new DocumentManager({
|
||||||
message: error,
|
workspace,
|
||||||
} = embeddingsCount !== 0 // if there no embeddings don't bother searching.
|
maxTokens: LLMConnector.limits.system,
|
||||||
|
})
|
||||||
|
.pinnedDocs()
|
||||||
|
.then((pinnedDocs) => {
|
||||||
|
pinnedDocs.forEach((doc) => {
|
||||||
|
const { pageContent, ...metadata } = doc;
|
||||||
|
contextTexts.push(doc.pageContent);
|
||||||
|
sources.push({
|
||||||
|
text:
|
||||||
|
pageContent.slice(0, 1_000) +
|
||||||
|
"...continued on in source document...",
|
||||||
|
...metadata,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
const vectorSearchResults =
|
||||||
|
embeddingsCount !== 0
|
||||||
? await VectorDb.performSimilaritySearch({
|
? await VectorDb.performSimilaritySearch({
|
||||||
namespace: workspace.slug,
|
namespace: workspace.slug,
|
||||||
input: message,
|
input: message,
|
||||||
@ -101,18 +121,21 @@ async function streamChatWithWorkspace(
|
|||||||
};
|
};
|
||||||
|
|
||||||
// Failed similarity search if it was run at all and failed.
|
// Failed similarity search if it was run at all and failed.
|
||||||
if (!!error) {
|
if (!!vectorSearchResults.message) {
|
||||||
writeResponseChunk(response, {
|
writeResponseChunk(response, {
|
||||||
id: uuid,
|
id: uuid,
|
||||||
type: "abort",
|
type: "abort",
|
||||||
textResponse: null,
|
textResponse: null,
|
||||||
sources: [],
|
sources: [],
|
||||||
close: true,
|
close: true,
|
||||||
error,
|
error: vectorSearchResults.message,
|
||||||
});
|
});
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
contextTexts = [...contextTexts, ...vectorSearchResults.contextTexts];
|
||||||
|
sources = [...sources, ...vectorSearchResults.sources];
|
||||||
|
|
||||||
// If in query mode and no sources are found, do not
|
// If in query mode and no sources are found, do not
|
||||||
// let the LLM try to hallucinate a response or use general knowledge and exit early
|
// let the LLM try to hallucinate a response or use general knowledge and exit early
|
||||||
if (chatMode === "query" && sources.length === 0) {
|
if (chatMode === "query" && sources.length === 0) {
|
||||||
|
@ -1,6 +1,7 @@
|
|||||||
const fs = require("fs");
|
const fs = require("fs");
|
||||||
const path = require("path");
|
const path = require("path");
|
||||||
const { v5: uuidv5 } = require("uuid");
|
const { v5: uuidv5 } = require("uuid");
|
||||||
|
const { Document } = require("../../models/documents");
|
||||||
const documentsPath =
|
const documentsPath =
|
||||||
process.env.NODE_ENV === "development"
|
process.env.NODE_ENV === "development"
|
||||||
? path.resolve(__dirname, `../../storage/documents`)
|
? path.resolve(__dirname, `../../storage/documents`)
|
||||||
@ -55,6 +56,10 @@ async function viewLocalFiles() {
|
|||||||
type: "file",
|
type: "file",
|
||||||
...metadata,
|
...metadata,
|
||||||
cached: await cachedVectorInformation(cachefilename, true),
|
cached: await cachedVectorInformation(cachefilename, true),
|
||||||
|
pinnedWorkspaces: await Document.getPins({
|
||||||
|
docpath: cachefilename,
|
||||||
|
pinned: true,
|
||||||
|
}),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
directory.items.push(subdocs);
|
directory.items.push(subdocs);
|
||||||
|
@ -85,11 +85,35 @@ async function messageArrayCompressor(llm, messages = [], rawHistory = []) {
|
|||||||
// Split context from system prompt - cannonball since its over the window.
|
// Split context from system prompt - cannonball since its over the window.
|
||||||
// We assume the context + user prompt is enough tokens to fit.
|
// We assume the context + user prompt is enough tokens to fit.
|
||||||
const [prompt, context = ""] = system.content.split("Context:");
|
const [prompt, context = ""] = system.content.split("Context:");
|
||||||
system.content = `${cannonball({
|
let compressedPrompt;
|
||||||
|
let compressedContext;
|
||||||
|
|
||||||
|
// If the user system prompt contribution's to the system prompt is more than
|
||||||
|
// 25% of the system limit, we will cannonball it - this favors the context
|
||||||
|
// over the instruction from the user.
|
||||||
|
if (tokenManager.countFromString(prompt) >= llm.limits.system * 0.25) {
|
||||||
|
compressedPrompt = cannonball({
|
||||||
input: prompt,
|
input: prompt,
|
||||||
targetTokenSize: llm.limits.system,
|
targetTokenSize: llm.limits.system * 0.25,
|
||||||
tiktokenInstance: tokenManager,
|
tiktokenInstance: tokenManager,
|
||||||
})}${context ? `\nContext: ${context}` : ""}`;
|
});
|
||||||
|
} else {
|
||||||
|
compressedPrompt = prompt;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (tokenManager.countFromString(context) >= llm.limits.system * 0.75) {
|
||||||
|
compressedContext = cannonball({
|
||||||
|
input: context,
|
||||||
|
targetTokenSize: llm.limits.system * 0.75,
|
||||||
|
tiktokenInstance: tokenManager,
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
compressedContext = context;
|
||||||
|
}
|
||||||
|
|
||||||
|
system.content = `${compressedPrompt}${
|
||||||
|
compressedContext ? `\nContext: ${compressedContext}` : ""
|
||||||
|
}`;
|
||||||
resolve(system);
|
resolve(system);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user