mirror of
https://github.com/Mintplex-Labs/anything-llm.git
synced 2024-07-02 15:20:37 +02:00
Implement Workspace-specific setting configs + other technical features (#58)
* 1. Define LLM Temperature as a workspace setting 2. Implement rudimentry table migration code for both new and existing repos to bring tables up to date 3. Trigger for workspace on update to update timestamp 4. Always fallback temp to 0.7 5. Extract WorkspaceModal into Tabbed content 6. Remove workspace name UNIQUE constraint (cannot be migrated :() 7. Add slug +seed when existing slug is already take 8. Seperate name from slug so display names can be changed * remove blocking test return
This commit is contained in:
Timothy Carambat
GitHub
parent
4118c9dcf3
commit
2a556c275c
|
|
@ -1,5 +1,5 @@
|
|||
import React from "react";
|
||||
import { dollarFormat } from "../../../../utils/numbers";
|
||||
import { dollarFormat } from "../../../../../utils/numbers";
|
||||
|
||||
export default function ConfirmationModal({
|
||||
directories,
|
||||
|
|
@ -7,7 +7,7 @@ import {
|
|||
FolderPlus,
|
||||
Zap,
|
||||
} from "react-feather";
|
||||
import { nFormatter } from "../../../../utils/numbers";
|
||||
import { nFormatter } from "../../../../../utils/numbers";
|
||||
|
||||
export default function Directory({
|
||||
files,
|
||||
|
|
@ -0,0 +1,205 @@
|
|||
import React, { useState, useEffect } from "react";
|
||||
import System from "../../../../models/system";
|
||||
import Workspace from "../../../../models/workspace";
|
||||
import paths from "../../../../utils/paths";
|
||||
import { useParams } from "react-router-dom";
|
||||
import Directory from "./Directory";
|
||||
import ConfirmationModal from "./ConfirmationModal";
|
||||
import CannotRemoveModal from "./CannotRemoveModal";
|
||||
|
||||
export default function DocumentSettings({ workspace }) {
|
||||
const { slug } = useParams();
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [saving, setSaving] = useState(false);
|
||||
const [showConfirmation, setShowConfirmation] = useState(false);
|
||||
const [directories, setDirectories] = useState(null);
|
||||
const [originalDocuments, setOriginalDocuments] = useState([]);
|
||||
const [selectedFiles, setSelectFiles] = useState([]);
|
||||
const [vectordb, setVectorDB] = useState(null);
|
||||
const [showingNoRemovalModal, setShowingNoRemovalModal] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
async function fetchKeys() {
|
||||
const localFiles = await System.localFiles();
|
||||
const settings = await System.keys();
|
||||
const originalDocs = workspace.documents.map((doc) => doc.docpath) || [];
|
||||
setDirectories(localFiles);
|
||||
setOriginalDocuments([...originalDocs]);
|
||||
setSelectFiles([...originalDocs]);
|
||||
setVectorDB(settings?.VectorDB);
|
||||
setLoading(false);
|
||||
}
|
||||
fetchKeys();
|
||||
}, []);
|
||||
|
||||
const deleteWorkspace = async () => {
|
||||
if (
|
||||
!window.confirm(
|
||||
`You are about to delete your entire ${workspace.name} workspace. This will remove all vector embeddings on your vector database.\n\nThe original source files will remain untouched. This action is irreversible.`
|
||||
)
|
||||
)
|
||||
return false;
|
||||
await Workspace.delete(workspace.slug);
|
||||
workspace.slug === slug
|
||||
? (window.location = paths.home())
|
||||
: window.location.reload();
|
||||
};
|
||||
|
||||
const docChanges = () => {
|
||||
const changes = {
|
||||
adds: [],
|
||||
deletes: [],
|
||||
};
|
||||
|
||||
selectedFiles.map((doc) => {
|
||||
const inOriginal = !!originalDocuments.find((oDoc) => oDoc === doc);
|
||||
if (!inOriginal) {
|
||||
changes.adds.push(doc);
|
||||
}
|
||||
});
|
||||
|
||||
originalDocuments.map((doc) => {
|
||||
const selected = !!selectedFiles.find((oDoc) => oDoc === doc);
|
||||
if (!selected) {
|
||||
changes.deletes.push(doc);
|
||||
}
|
||||
});
|
||||
|
||||
return changes;
|
||||
};
|
||||
|
||||
const confirmChanges = (e) => {
|
||||
e.preventDefault();
|
||||
const changes = docChanges();
|
||||
changes.adds.length > 0 ? setShowConfirmation(true) : updateWorkspace(e);
|
||||
};
|
||||
|
||||
const updateWorkspace = async (e) => {
|
||||
e.preventDefault();
|
||||
setSaving(true);
|
||||
setShowConfirmation(false);
|
||||
const changes = docChanges();
|
||||
await Workspace.modifyEmbeddings(workspace.slug, changes);
|
||||
setSaving(false);
|
||||
window.location.reload();
|
||||
};
|
||||
|
||||
const isSelected = (filepath) => {
|
||||
const isFolder = !filepath.includes("/");
|
||||
return isFolder
|
||||
? selectedFiles.some((doc) => doc.includes(filepath.split("/")[0]))
|
||||
: selectedFiles.some((doc) => doc.includes(filepath));
|
||||
};
|
||||
|
||||
const isOriginalDoc = (filepath) => {
|
||||
const isFolder = !filepath.includes("/");
|
||||
return isFolder
|
||||
? originalDocuments.some((doc) => doc.includes(filepath.split("/")[0]))
|
||||
: originalDocuments.some((doc) => doc.includes(filepath));
|
||||
};
|
||||
|
||||
const toggleSelection = (filepath) => {
|
||||
const isFolder = !filepath.includes("/");
|
||||
const parent = isFolder ? filepath : filepath.split("/")[0];
|
||||
|
||||
if (isSelected(filepath)) {
|
||||
// Certain vector DBs do not contain the ability to delete vectors
|
||||
// so we cannot remove from these. The user will have to clear the entire workspace.
|
||||
if (["lancedb"].includes(vectordb) && isOriginalDoc(filepath)) {
|
||||
setShowingNoRemovalModal(true);
|
||||
return false;
|
||||
}
|
||||
|
||||
const updatedDocs = isFolder
|
||||
? selectedFiles.filter((doc) => !doc.includes(parent))
|
||||
: selectedFiles.filter((doc) => !doc.includes(filepath));
|
||||
setSelectFiles([...new Set(updatedDocs)]);
|
||||
} else {
|
||||
var newDocs = [];
|
||||
var parentDirs = directories.items.find((item) => item.name === parent);
|
||||
if (isFolder && parentDirs) {
|
||||
const folderItems = parentDirs.items;
|
||||
newDocs = folderItems.map((item) => parent + "/" + item.name);
|
||||
} else {
|
||||
newDocs = [filepath];
|
||||
}
|
||||
|
||||
const combined = [...selectedFiles, ...newDocs];
|
||||
setSelectFiles([...new Set(combined)]);
|
||||
}
|
||||
};
|
||||
|
||||
if (loading) {
|
||||
return (
|
||||
<>
|
||||
<div className="p-6 flex h-full w-full max-h-[80vh] overflow-y-scroll">
|
||||
<div className="flex flex-col gap-y-1 w-full">
|
||||
<p className="text-slate-200 dark:text-stone-300 text-center">
|
||||
loading workspace files
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex items-center p-6 space-x-2 border-t border-gray-200 rounded-b dark:border-gray-600"></div>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<>
|
||||
{showConfirmation && (
|
||||
<ConfirmationModal
|
||||
directories={directories}
|
||||
hideConfirm={() => setShowConfirmation(false)}
|
||||
additions={docChanges().adds}
|
||||
updateWorkspace={updateWorkspace}
|
||||
/>
|
||||
)}
|
||||
{showingNoRemovalModal && (
|
||||
<CannotRemoveModal
|
||||
hideModal={() => setShowingNoRemovalModal(false)}
|
||||
vectordb={vectordb}
|
||||
/>
|
||||
)}
|
||||
<div className="p-6 flex h-full w-full max-h-[80vh] overflow-y-scroll">
|
||||
<div className="flex flex-col gap-y-1 w-full">
|
||||
<div className="flex flex-col mb-2">
|
||||
<p className="text-gray-800 dark:text-stone-200 text-base ">
|
||||
Select folders to add or remove from workspace.
|
||||
</p>
|
||||
<p className="text-gray-800 dark:text-stone-400 text-xs italic">
|
||||
{selectedFiles.length} documents in workspace selected.
|
||||
</p>
|
||||
</div>
|
||||
<div className="w-full h-auto border border-slate-200 dark:border-stone-600 rounded-lg px-4 py-2">
|
||||
{!!directories && (
|
||||
<Directory
|
||||
files={directories}
|
||||
toggleSelection={toggleSelection}
|
||||
isSelected={isSelected}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex items-center justify-between p-6 space-x-2 border-t border-gray-200 rounded-b dark:border-gray-600">
|
||||
<button
|
||||
onClick={deleteWorkspace}
|
||||
type="button"
|
||||
className="border border-transparent text-gray-500 bg-white hover:bg-red-100 rounded-lg text-sm font-medium px-5 py-2.5 hover:text-red-900 focus:z-10 dark:bg-transparent dark:text-gray-300 dark:hover:text-white dark:hover:bg-red-600"
|
||||
>
|
||||
Delete Workspace
|
||||
</button>
|
||||
<div className="flex items-center">
|
||||
<button
|
||||
disabled={saving}
|
||||
onClick={confirmChanges}
|
||||
type="submit"
|
||||
className="text-slate-200 bg-black-900 px-4 py-2 rounded-lg hover:bg-gray-900"
|
||||
>
|
||||
{saving ? "Saving..." : "Confirm Changes"}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
}
|
||||
176
frontend/src/components/Modals/MangeWorkspace/Settings/index.jsx
Normal file
176
frontend/src/components/Modals/MangeWorkspace/Settings/index.jsx
Normal file
|
|
@ -0,0 +1,176 @@
|
|||
import React, { useState, useRef, useEffect } from "react";
|
||||
import Workspace from "../../../../models/workspace";
|
||||
import paths from "../../../../utils/paths";
|
||||
|
||||
export default function WorkspaceSettings({ workspace }) {
|
||||
const formEl = useRef(null);
|
||||
const [saving, setSaving] = useState(false);
|
||||
const [hasChanges, setHasChanges] = useState(false);
|
||||
const [error, setError] = useState(null);
|
||||
const [success, setSuccess] = useState(null);
|
||||
|
||||
useEffect(() => {
|
||||
function setTimer() {
|
||||
if (success !== null) {
|
||||
setTimeout(() => {
|
||||
setSuccess(null);
|
||||
}, 3_000);
|
||||
}
|
||||
|
||||
if (error !== null) {
|
||||
setTimeout(() => {
|
||||
setError(null);
|
||||
}, 3_000);
|
||||
}
|
||||
}
|
||||
setTimer();
|
||||
}, [success, error]);
|
||||
|
||||
const handleUpdate = async (e) => {
|
||||
setError(null);
|
||||
setSuccess(null);
|
||||
setSaving(true);
|
||||
e.preventDefault();
|
||||
const data = {};
|
||||
const form = new FormData(formEl.current);
|
||||
for (var [key, value] of form.entries()) data[key] = value;
|
||||
const { workspace: updatedWorkspace, message } = await Workspace.update(
|
||||
workspace.slug,
|
||||
data
|
||||
);
|
||||
if (!!updatedWorkspace) {
|
||||
setSuccess("Workspace updated!");
|
||||
} else {
|
||||
setError(message);
|
||||
}
|
||||
setSaving(false);
|
||||
};
|
||||
|
||||
const deleteWorkspace = async () => {
|
||||
if (
|
||||
!window.confirm(
|
||||
`You are about to delete your entire ${workspace.name} workspace. This will remove all vector embeddings on your vector database.\n\nThe original source files will remain untouched. This action is irreversible.`
|
||||
)
|
||||
)
|
||||
return false;
|
||||
await Workspace.delete(workspace.slug);
|
||||
workspace.slug === slug
|
||||
? (window.location = paths.home())
|
||||
: window.location.reload();
|
||||
};
|
||||
|
||||
return (
|
||||
<form ref={formEl} onSubmit={handleUpdate}>
|
||||
<div className="p-6 flex h-full w-full max-h-[80vh] overflow-y-scroll">
|
||||
<div className="flex flex-col gap-y-1 w-full">
|
||||
<div className="flex flex-col mb-2">
|
||||
<p className="text-gray-800 dark:text-stone-200 text-base ">
|
||||
Edit your workspace's settings
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div className="w-full flex flex-col gap-y-4">
|
||||
<div>
|
||||
<input
|
||||
type="text"
|
||||
disabled={true}
|
||||
defaultValue={workspace?.slug}
|
||||
className="bg-gray-50 border disabled:bg-gray-400 disabled:text-gray-700 disabled:border-gray-400 disabled:dark:bg-stone-800 disabled:dark:border-stone-900 disabled:dark:text-stone-600 disabled:cursor-not-allowed border-gray-300 text-gray-900 text-sm rounded-lg focus:ring-blue-500 focus:border-blue-500 block w-full p-2.5 dark:bg-stone-600 dark:border-stone-600 dark:placeholder-gray-400 dark:text-white dark:focus:ring-blue-500 dark:focus:border-blue-500"
|
||||
required={true}
|
||||
autoComplete="off"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<div className="flex flex-col gap-y-1 mb-4">
|
||||
<label
|
||||
htmlFor="name"
|
||||
className="block text-sm font-medium text-gray-900 dark:text-white"
|
||||
>
|
||||
Workspace Name
|
||||
</label>
|
||||
<p className="text-xs text-gray-600 dark:text-stone-400">
|
||||
This will only change the display name of your workspace.
|
||||
</p>
|
||||
</div>
|
||||
<input
|
||||
name="name"
|
||||
type="text"
|
||||
minLength={2}
|
||||
maxLength={80}
|
||||
defaultValue={workspace?.name}
|
||||
className="bg-gray-50 border border-gray-300 text-gray-900 text-sm rounded-lg focus:ring-blue-500 focus:border-blue-500 block w-full p-2.5 dark:bg-stone-600 dark:border-stone-600 dark:placeholder-gray-400 dark:text-white dark:focus:ring-blue-500 dark:focus:border-blue-500"
|
||||
placeholder="My Workspace"
|
||||
required={true}
|
||||
autoComplete="off"
|
||||
onChange={() => setHasChanges(true)}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<div className="flex flex-col gap-y-1 mb-4">
|
||||
<label
|
||||
htmlFor="name"
|
||||
className="block text-sm font-medium text-gray-900 dark:text-white"
|
||||
>
|
||||
LLM Temperature
|
||||
</label>
|
||||
<p className="text-xs text-gray-600 dark:text-stone-400">
|
||||
This setting controls how "random" or dynamic your chat
|
||||
responses will be.
|
||||
<br />
|
||||
The higher the number (2.0 maximum) the more random and
|
||||
incoherent.
|
||||
<br />
|
||||
Recommended: 0.7
|
||||
</p>
|
||||
</div>
|
||||
<input
|
||||
name="openAiTemp"
|
||||
type="number"
|
||||
min={0.0}
|
||||
max={2.0}
|
||||
step={0.1}
|
||||
onWheel={(e) => e.target.blur()}
|
||||
defaultValue={workspace?.openAiTemp ?? 0.7}
|
||||
className="bg-gray-50 border border-gray-300 text-gray-900 text-sm rounded-lg focus:ring-blue-500 focus:border-blue-500 block w-full p-2.5 dark:bg-stone-600 dark:border-stone-600 dark:placeholder-gray-400 dark:text-white dark:focus:ring-blue-500 dark:focus:border-blue-500"
|
||||
placeholder="0.7"
|
||||
required={true}
|
||||
autoComplete="off"
|
||||
onChange={() => setHasChanges(true)}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{error && (
|
||||
<p className="text-red-600 dark:text-red-400 text-sm">
|
||||
Error: {error}
|
||||
</p>
|
||||
)}
|
||||
{success && (
|
||||
<p className="text-green-600 dark:text-green-400 text-sm">
|
||||
Success: {success}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex items-center justify-between p-6 space-x-2 border-t border-gray-200 rounded-b dark:border-gray-600">
|
||||
<button
|
||||
onClick={deleteWorkspace}
|
||||
type="button"
|
||||
className="border border-transparent text-gray-500 bg-white hover:bg-red-100 rounded-lg text-sm font-medium px-5 py-2.5 hover:text-red-900 focus:z-10 dark:bg-transparent dark:text-gray-300 dark:hover:text-white dark:hover:bg-red-600"
|
||||
>
|
||||
Delete Workspace
|
||||
</button>
|
||||
{hasChanges && (
|
||||
<button
|
||||
type="submit"
|
||||
className="text-gray-500 bg-white hover:bg-gray-100 focus:ring-4 focus:outline-none focus:ring-blue-300 rounded-lg border border-gray-200 text-sm font-medium px-5 py-2.5 hover:text-gray-900 focus:z-10 dark:bg-black dark:text-slate-200 dark:border-transparent dark:hover:text-slate-200 dark:hover:bg-gray-900 dark:focus:ring-gray-800"
|
||||
>
|
||||
{saving ? "Updating..." : "Update workspace"}
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
</form>
|
||||
);
|
||||
}
|
||||
|
|
@ -1,199 +1,47 @@
|
|||
import React, { useState, useEffect } from "react";
|
||||
import { X } from "react-feather";
|
||||
import System from "../../../models/system";
|
||||
import Workspace from "../../../models/workspace";
|
||||
import paths from "../../../utils/paths";
|
||||
import { Archive, Sliders, X } from "react-feather";
|
||||
import DocumentSettings from "./Documents";
|
||||
import WorkspaceSettings from "./Settings";
|
||||
import { useParams } from "react-router-dom";
|
||||
import Directory from "./Directory";
|
||||
import ConfirmationModal from "./ConfirmationModal";
|
||||
import CannotRemoveModal from "./CannotRemoveModal";
|
||||
import Workspace from "../../../models/workspace";
|
||||
|
||||
const TABS = {
|
||||
documents: DocumentSettings,
|
||||
settings: WorkspaceSettings,
|
||||
};
|
||||
|
||||
const noop = () => false;
|
||||
export default function ManageWorkspace({ hideModal = noop, workspace }) {
|
||||
export default function ManageWorkspace({
|
||||
hideModal = noop,
|
||||
providedSlug = null,
|
||||
}) {
|
||||
const { slug } = useParams();
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [saving, setSaving] = useState(false);
|
||||
const [showConfirmation, setShowConfirmation] = useState(false);
|
||||
const [directories, setDirectories] = useState(null);
|
||||
const [originalDocuments, setOriginalDocuments] = useState([]);
|
||||
const [selectedFiles, setSelectFiles] = useState([]);
|
||||
const [vectordb, setVectorDB] = useState(null);
|
||||
const [showingNoRemovalModal, setShowingNoRemovalModal] = useState(false);
|
||||
const [selectedTab, setSelectedTab] = useState("documents");
|
||||
const [workspace, setWorkspace] = useState(null);
|
||||
|
||||
useEffect(() => {
|
||||
async function fetchKeys() {
|
||||
const _workspace = await Workspace.bySlug(workspace.slug);
|
||||
const localFiles = await System.localFiles();
|
||||
const settings = await System.keys();
|
||||
const originalDocs = _workspace.documents.map((doc) => doc.docpath) || [];
|
||||
setDirectories(localFiles);
|
||||
setOriginalDocuments([...originalDocs]);
|
||||
setSelectFiles([...originalDocs]);
|
||||
setVectorDB(settings?.VectorDB);
|
||||
setLoading(false);
|
||||
async function fetchWorkspace() {
|
||||
const workspace = await Workspace.bySlug(providedSlug ?? slug);
|
||||
setWorkspace(workspace);
|
||||
}
|
||||
fetchKeys();
|
||||
}, []);
|
||||
fetchWorkspace();
|
||||
}, [selectedTab, slug]);
|
||||
|
||||
const deleteWorkspace = async () => {
|
||||
if (
|
||||
!window.confirm(
|
||||
`You are about to delete your entire ${workspace.name} workspace. This will remove all vector embeddings on your vector database.\n\nThe original source files will remain untouched. This action is irreversible.`
|
||||
)
|
||||
)
|
||||
return false;
|
||||
await Workspace.delete(workspace.slug);
|
||||
workspace.slug === slug
|
||||
? (window.location = paths.home())
|
||||
: window.location.reload();
|
||||
};
|
||||
|
||||
const docChanges = () => {
|
||||
const changes = {
|
||||
adds: [],
|
||||
deletes: [],
|
||||
};
|
||||
|
||||
selectedFiles.map((doc) => {
|
||||
const inOriginal = !!originalDocuments.find((oDoc) => oDoc === doc);
|
||||
if (!inOriginal) {
|
||||
changes.adds.push(doc);
|
||||
}
|
||||
});
|
||||
|
||||
originalDocuments.map((doc) => {
|
||||
const selected = !!selectedFiles.find((oDoc) => oDoc === doc);
|
||||
if (!selected) {
|
||||
changes.deletes.push(doc);
|
||||
}
|
||||
});
|
||||
|
||||
return changes;
|
||||
};
|
||||
|
||||
const confirmChanges = (e) => {
|
||||
e.preventDefault();
|
||||
const changes = docChanges();
|
||||
changes.adds.length > 0 ? setShowConfirmation(true) : updateWorkspace(e);
|
||||
};
|
||||
|
||||
const updateWorkspace = async (e) => {
|
||||
e.preventDefault();
|
||||
setSaving(true);
|
||||
setShowConfirmation(false);
|
||||
const changes = docChanges();
|
||||
await Workspace.modifyEmbeddings(workspace.slug, changes);
|
||||
setSaving(false);
|
||||
window.location.reload();
|
||||
};
|
||||
|
||||
const isSelected = (filepath) => {
|
||||
const isFolder = !filepath.includes("/");
|
||||
return isFolder
|
||||
? selectedFiles.some((doc) => doc.includes(filepath.split("/")[0]))
|
||||
: selectedFiles.some((doc) => doc.includes(filepath));
|
||||
};
|
||||
|
||||
const isOriginalDoc = (filepath) => {
|
||||
const isFolder = !filepath.includes("/");
|
||||
return isFolder
|
||||
? originalDocuments.some((doc) => doc.includes(filepath.split("/")[0]))
|
||||
: originalDocuments.some((doc) => doc.includes(filepath));
|
||||
};
|
||||
|
||||
const toggleSelection = (filepath) => {
|
||||
const isFolder = !filepath.includes("/");
|
||||
const parent = isFolder ? filepath : filepath.split("/")[0];
|
||||
|
||||
if (isSelected(filepath)) {
|
||||
// Certain vector DBs do not contain the ability to delete vectors
|
||||
// so we cannot remove from these. The user will have to clear the entire workspace.
|
||||
if (["lancedb"].includes(vectordb) && isOriginalDoc(filepath)) {
|
||||
setShowingNoRemovalModal(true);
|
||||
return false;
|
||||
}
|
||||
|
||||
const updatedDocs = isFolder
|
||||
? selectedFiles.filter((doc) => !doc.includes(parent))
|
||||
: selectedFiles.filter((doc) => !doc.includes(filepath));
|
||||
setSelectFiles([...new Set(updatedDocs)]);
|
||||
} else {
|
||||
var newDocs = [];
|
||||
var parentDirs = directories.items.find((item) => item.name === parent);
|
||||
if (isFolder && parentDirs) {
|
||||
const folderItems = parentDirs.items;
|
||||
newDocs = folderItems.map((item) => parent + "/" + item.name);
|
||||
} else {
|
||||
newDocs = [filepath];
|
||||
}
|
||||
|
||||
const combined = [...selectedFiles, ...newDocs];
|
||||
setSelectFiles([...new Set(combined)]);
|
||||
}
|
||||
};
|
||||
|
||||
if (loading) {
|
||||
return (
|
||||
<div className="fixed top-0 left-0 right-0 z-50 w-full p-4 overflow-x-hidden overflow-y-auto md:inset-0 h-[calc(100%-1rem)] h-full bg-black bg-opacity-50 flex items-center justify-center">
|
||||
<div
|
||||
className="flex fixed top-0 left-0 right-0 w-full h-full"
|
||||
onClick={hideModal}
|
||||
/>
|
||||
<div className="relative w-full max-w-2xl max-h-full">
|
||||
<div className="relative bg-white rounded-lg shadow dark:bg-stone-700">
|
||||
<div className="flex items-start justify-between p-4 border-b rounded-t dark:border-gray-600">
|
||||
<h3 className="text-xl font-semibold text-gray-900 dark:text-white">
|
||||
{workspace.name} Settings
|
||||
</h3>
|
||||
<button
|
||||
onClick={hideModal}
|
||||
type="button"
|
||||
className="text-gray-400 bg-transparent hover:bg-gray-200 hover:text-gray-900 rounded-lg text-sm p-1.5 ml-auto inline-flex items-center dark:hover:bg-gray-600 dark:hover:text-white"
|
||||
data-modal-hide="staticModal"
|
||||
>
|
||||
<X className="text-gray-300 text-lg" />
|
||||
</button>
|
||||
</div>
|
||||
<div className="p-6 flex h-full w-full max-h-[80vh] overflow-y-scroll">
|
||||
<div className="flex flex-col gap-y-1 w-full">
|
||||
<p className="text-slate-200 dark:text-stone-300 text-center">
|
||||
loading workspace files
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex items-center p-6 space-x-2 border-t border-gray-200 rounded-b dark:border-gray-600"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
if (!workspace) return null;
|
||||
|
||||
const Component = TABS[selectedTab || "documents"];
|
||||
return (
|
||||
<>
|
||||
{showConfirmation && (
|
||||
<ConfirmationModal
|
||||
directories={directories}
|
||||
hideConfirm={() => setShowConfirmation(false)}
|
||||
additions={docChanges().adds}
|
||||
updateWorkspace={updateWorkspace}
|
||||
/>
|
||||
)}
|
||||
{showingNoRemovalModal && (
|
||||
<CannotRemoveModal
|
||||
hideModal={() => setShowingNoRemovalModal(false)}
|
||||
vectordb={vectordb}
|
||||
/>
|
||||
)}
|
||||
<div className="fixed top-0 left-0 right-0 z-50 w-full p-4 overflow-x-hidden overflow-y-auto md:inset-0 h-[calc(100%-1rem)] h-full bg-black bg-opacity-50 flex items-center justify-center">
|
||||
<div
|
||||
className="flex fixed top-0 left-0 right-0 w-full h-full"
|
||||
onClick={hideModal}
|
||||
/>
|
||||
<div className="relative w-full max-w-2xl max-h-full">
|
||||
<div className="relative bg-white rounded-lg shadow dark:bg-stone-700">
|
||||
<div className="flex items-start justify-between p-4 border-b rounded-t dark:border-gray-600">
|
||||
<div className="fixed top-0 left-0 right-0 z-50 w-full p-4 overflow-x-hidden overflow-y-auto md:inset-0 h-[calc(100%-1rem)] h-full bg-black bg-opacity-50 flex items-center justify-center">
|
||||
<div
|
||||
className="flex fixed top-0 left-0 right-0 w-full h-full"
|
||||
onClick={hideModal}
|
||||
/>
|
||||
<div className="relative w-full max-w-2xl max-h-full">
|
||||
<div className="relative bg-white rounded-lg shadow dark:bg-stone-700">
|
||||
<div className="flex flex-col gap-y-1 border-b dark:border-gray-600 px-4 pt-4 ">
|
||||
<div className="flex items-start justify-between rounded-t ">
|
||||
<h3 className="text-xl font-semibold text-gray-900 dark:text-white">
|
||||
"{workspace.name}" workspace settings
|
||||
Update "{workspace.name}"
|
||||
</h3>
|
||||
<button
|
||||
onClick={hideModal}
|
||||
|
|
@ -204,51 +52,64 @@ export default function ManageWorkspace({ hideModal = noop, workspace }) {
|
|||
<X className="text-gray-300 text-lg" />
|
||||
</button>
|
||||
</div>
|
||||
<div className="p-6 flex h-full w-full max-h-[80vh] overflow-y-scroll">
|
||||
<div className="flex flex-col gap-y-1 w-full">
|
||||
<div className="flex flex-col mb-2">
|
||||
<p className="text-gray-800 dark:text-stone-200 text-base ">
|
||||
Select folders to add or remove from workspace.
|
||||
</p>
|
||||
<p className="text-gray-800 dark:text-stone-400 text-xs italic">
|
||||
{selectedFiles.length} documents in workspace selected.
|
||||
</p>
|
||||
</div>
|
||||
<div className="w-full h-auto border border-slate-200 dark:border-stone-600 rounded-lg px-4 py-2">
|
||||
{!!directories && (
|
||||
<Directory
|
||||
files={directories}
|
||||
toggleSelection={toggleSelection}
|
||||
isSelected={isSelected}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="flex items-center justify-between p-6 space-x-2 border-t border-gray-200 rounded-b dark:border-gray-600">
|
||||
<button
|
||||
onClick={deleteWorkspace}
|
||||
type="button"
|
||||
className="border border-transparent text-gray-500 bg-white hover:bg-red-100 rounded-lg text-sm font-medium px-5 py-2.5 hover:text-red-900 focus:z-10 dark:bg-transparent dark:text-gray-300 dark:hover:text-white dark:hover:bg-red-600"
|
||||
>
|
||||
Delete Workspace
|
||||
</button>
|
||||
<div className="flex items-center">
|
||||
<button
|
||||
disabled={saving}
|
||||
onClick={confirmChanges}
|
||||
type="submit"
|
||||
className="text-slate-200 bg-black-900 px-4 py-2 rounded-lg hover:bg-gray-900"
|
||||
>
|
||||
{saving ? "Saving..." : "Confirm Changes"}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
<WorkspaceSettingTabs
|
||||
selectedTab={selectedTab}
|
||||
changeTab={setSelectedTab}
|
||||
/>
|
||||
</div>
|
||||
<Component hideModal={hideModal} workspace={workspace} />
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function WorkspaceSettingTabs({ selectedTab, changeTab }) {
|
||||
return (
|
||||
<div>
|
||||
<ul className="flex flex-wrap -mb-px text-sm gap-x-2 font-medium text-center text-gray-500 dark:text-gray-400">
|
||||
<WorkspaceTab
|
||||
active={selectedTab === "documents"}
|
||||
displayName="Documents"
|
||||
tabName="documents"
|
||||
icon={<Archive className="h-4 w-4" />}
|
||||
onClick={changeTab}
|
||||
/>
|
||||
<WorkspaceTab
|
||||
active={selectedTab === "settings"}
|
||||
displayName="Settings"
|
||||
tabName="settings"
|
||||
icon={<Sliders className="h-4 w-4" />}
|
||||
onClick={changeTab}
|
||||
/>
|
||||
</ul>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function WorkspaceTab({
|
||||
active = false,
|
||||
displayName,
|
||||
tabName,
|
||||
icon = "",
|
||||
onClick,
|
||||
}) {
|
||||
const classes = active
|
||||
? "text-blue-600 border-blue-600 active dark:text-blue-400 dark:border-blue-400 bg-blue-500 bg-opacity-5"
|
||||
: "border-transparent hover:text-gray-600 hover:border-gray-300 dark:hover:text-gray-300";
|
||||
return (
|
||||
<li className="mr-2">
|
||||
<button
|
||||
disabled={active}
|
||||
onClick={() => onClick(tabName)}
|
||||
className={
|
||||
"flex items-center gap-x-1 p-4 border-b-2 rounded-t-lg group " +
|
||||
classes
|
||||
}
|
||||
>
|
||||
{icon} {displayName}
|
||||
</button>
|
||||
</li>
|
||||
);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -18,33 +18,33 @@ export default function NewWorkspaceModal({ hideModal = noop }) {
|
|||
};
|
||||
|
||||
return (
|
||||
<div class="fixed top-0 left-0 right-0 z-50 w-full p-4 overflow-x-hidden overflow-y-auto md:inset-0 h-[calc(100%-1rem)] h-full bg-black bg-opacity-50 flex items-center justify-center">
|
||||
<div className="fixed top-0 left-0 right-0 z-50 w-full p-4 overflow-x-hidden overflow-y-auto md:inset-0 h-[calc(100%-1rem)] h-full bg-black bg-opacity-50 flex items-center justify-center">
|
||||
<div
|
||||
className="flex fixed top-0 left-0 right-0 w-full h-full"
|
||||
onClick={hideModal}
|
||||
/>
|
||||
<div class="relative w-full max-w-2xl max-h-full">
|
||||
<div class="relative bg-white rounded-lg shadow dark:bg-stone-700">
|
||||
<div class="flex items-start justify-between p-4 border-b rounded-t dark:border-gray-600">
|
||||
<h3 class="text-xl font-semibold text-gray-900 dark:text-white">
|
||||
<div className="relative w-full max-w-2xl max-h-full">
|
||||
<div className="relative bg-white rounded-lg shadow dark:bg-stone-700">
|
||||
<div className="flex items-start justify-between p-4 border-b rounded-t dark:border-gray-600">
|
||||
<h3 className="text-xl font-semibold text-gray-900 dark:text-white">
|
||||
Create a New Workspace
|
||||
</h3>
|
||||
<button
|
||||
onClick={hideModal}
|
||||
type="button"
|
||||
class="text-gray-400 bg-transparent hover:bg-gray-200 hover:text-gray-900 rounded-lg text-sm p-1.5 ml-auto inline-flex items-center dark:hover:bg-gray-600 dark:hover:text-white"
|
||||
className="text-gray-400 bg-transparent hover:bg-gray-200 hover:text-gray-900 rounded-lg text-sm p-1.5 ml-auto inline-flex items-center dark:hover:bg-gray-600 dark:hover:text-white"
|
||||
data-modal-hide="staticModal"
|
||||
>
|
||||
<X className="text-gray-300 text-lg" />
|
||||
</button>
|
||||
</div>
|
||||
<form ref={formEl} onSubmit={handleCreate}>
|
||||
<div class="p-6 space-y-6 flex h-full w-full">
|
||||
<div className="p-6 space-y-6 flex h-full w-full">
|
||||
<div className="w-full flex flex-col gap-y-4">
|
||||
<div>
|
||||
<label
|
||||
htmlFor="name"
|
||||
class="block mb-2 text-sm font-medium text-gray-900 dark:text-white"
|
||||
className="block mb-2 text-sm font-medium text-gray-900 dark:text-white"
|
||||
>
|
||||
Workspace Name
|
||||
</label>
|
||||
|
|
@ -52,7 +52,7 @@ export default function NewWorkspaceModal({ hideModal = noop }) {
|
|||
name="name"
|
||||
type="text"
|
||||
id="name"
|
||||
class="bg-gray-50 border border-gray-300 text-gray-900 text-sm rounded-lg focus:ring-blue-500 focus:border-blue-500 block w-full p-2.5 dark:bg-stone-600 dark:border-stone-600 dark:placeholder-gray-400 dark:text-white dark:focus:ring-blue-500 dark:focus:border-blue-500"
|
||||
className="bg-gray-50 border border-gray-300 text-gray-900 text-sm rounded-lg focus:ring-blue-500 focus:border-blue-500 block w-full p-2.5 dark:bg-stone-600 dark:border-stone-600 dark:placeholder-gray-400 dark:text-white dark:focus:ring-blue-500 dark:focus:border-blue-500"
|
||||
placeholder="My Workspace"
|
||||
required={true}
|
||||
autoComplete="off"
|
||||
|
|
@ -69,7 +69,7 @@ export default function NewWorkspaceModal({ hideModal = noop }) {
|
|||
</p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="flex w-full justify-between items-center p-6 space-x-2 border-t border-gray-200 rounded-b dark:border-gray-600">
|
||||
<div className="flex w-full justify-between items-center p-6 space-x-2 border-t border-gray-200 rounded-b dark:border-gray-600">
|
||||
<button
|
||||
onClick={hideModal}
|
||||
type="button"
|
||||
|
|
@ -79,7 +79,7 @@ export default function NewWorkspaceModal({ hideModal = noop }) {
|
|||
</button>
|
||||
<button
|
||||
type="submit"
|
||||
class="text-gray-500 bg-white hover:bg-gray-100 focus:ring-4 focus:outline-none focus:ring-blue-300 rounded-lg border border-gray-200 text-sm font-medium px-5 py-2.5 hover:text-gray-900 focus:z-10 dark:bg-black dark:text-slate-200 dark:border-transparent dark:hover:text-slate-200 dark:hover:bg-gray-900 dark:focus:ring-gray-800"
|
||||
className="text-gray-500 bg-white hover:bg-gray-100 focus:ring-4 focus:outline-none focus:ring-blue-300 rounded-lg border border-gray-200 text-sm font-medium px-5 py-2.5 hover:text-gray-900 focus:z-10 dark:bg-black dark:text-slate-200 dark:border-transparent dark:hover:text-slate-200 dark:hover:bg-gray-900 dark:focus:ring-gray-800"
|
||||
>
|
||||
Create Workspace
|
||||
</button>
|
||||
|
|
|
|||
|
|
@ -25,22 +25,22 @@ export default function PasswordModal() {
|
|||
};
|
||||
|
||||
return (
|
||||
<div class="fixed top-0 left-0 right-0 z-50 w-full p-4 overflow-x-hidden overflow-y-auto md:inset-0 h-[calc(100%-1rem)] h-full bg-black bg-opacity-50 flex items-center justify-center">
|
||||
<div className="fixed top-0 left-0 right-0 z-50 w-full p-4 overflow-x-hidden overflow-y-auto md:inset-0 h-[calc(100%-1rem)] h-full bg-black bg-opacity-50 flex items-center justify-center">
|
||||
<div className="flex fixed top-0 left-0 right-0 w-full h-full" />
|
||||
<div class="relative w-full max-w-2xl max-h-full">
|
||||
<div className="relative w-full max-w-2xl max-h-full">
|
||||
<form ref={formEl} onSubmit={handleLogin}>
|
||||
<div class="relative bg-white rounded-lg shadow dark:bg-stone-700">
|
||||
<div class="flex items-start justify-between p-4 border-b rounded-t dark:border-gray-600">
|
||||
<h3 class="text-xl font-semibold text-gray-900 dark:text-white">
|
||||
<div className="relative bg-white rounded-lg shadow dark:bg-stone-700">
|
||||
<div className="flex items-start justify-between p-4 border-b rounded-t dark:border-gray-600">
|
||||
<h3 className="text-xl font-semibold text-gray-900 dark:text-white">
|
||||
This workspace is password protected.
|
||||
</h3>
|
||||
</div>
|
||||
<div class="p-6 space-y-6 flex h-full w-full">
|
||||
<div className="p-6 space-y-6 flex h-full w-full">
|
||||
<div className="w-full flex flex-col gap-y-4">
|
||||
<div>
|
||||
<label
|
||||
htmlFor="password"
|
||||
class="block mb-2 text-sm font-medium text-gray-900 dark:text-white"
|
||||
className="block mb-2 text-sm font-medium text-gray-900 dark:text-white"
|
||||
>
|
||||
Workspace Password
|
||||
</label>
|
||||
|
|
@ -48,7 +48,7 @@ export default function PasswordModal() {
|
|||
name="password"
|
||||
type="password"
|
||||
id="password"
|
||||
class="bg-gray-50 border border-gray-300 text-gray-900 text-sm rounded-lg focus:ring-blue-500 focus:border-blue-500 block w-full p-2.5 dark:bg-stone-600 dark:border-stone-600 dark:placeholder-gray-400 dark:text-white dark:focus:ring-blue-500 dark:focus:border-blue-500"
|
||||
className="bg-gray-50 border border-gray-300 text-gray-900 text-sm rounded-lg focus:ring-blue-500 focus:border-blue-500 block w-full p-2.5 dark:bg-stone-600 dark:border-stone-600 dark:placeholder-gray-400 dark:text-white dark:focus:ring-blue-500 dark:focus:border-blue-500"
|
||||
required={true}
|
||||
autoComplete="off"
|
||||
/>
|
||||
|
|
@ -64,11 +64,11 @@ export default function PasswordModal() {
|
|||
</p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="flex items-center justify-end p-6 space-x-2 border-t border-gray-200 rounded-b dark:border-gray-600">
|
||||
<div className="flex items-center justify-end p-6 space-x-2 border-t border-gray-200 rounded-b dark:border-gray-600">
|
||||
<button
|
||||
disabled={loading}
|
||||
type="submit"
|
||||
class="text-gray-500 bg-white hover:bg-gray-100 focus:ring-4 focus:outline-none focus:ring-blue-300 rounded-lg border border-gray-200 text-sm font-medium px-5 py-2.5 hover:text-gray-900 focus:z-10 dark:bg-gray-700 dark:text-gray-300 dark:border-gray-500 dark:hover:text-white dark:hover:bg-gray-600 dark:focus:ring-gray-600"
|
||||
className="text-gray-500 bg-white hover:bg-gray-100 focus:ring-4 focus:outline-none focus:ring-blue-300 rounded-lg border border-gray-200 text-sm font-medium px-5 py-2.5 hover:text-gray-900 focus:z-10 dark:bg-gray-700 dark:text-gray-300 dark:border-gray-500 dark:hover:text-white dark:hover:bg-gray-600 dark:focus:ring-gray-600"
|
||||
>
|
||||
{loading ? "Validating..." : "Submit"}
|
||||
</button>
|
||||
|
|
|
|||
|
|
@ -75,7 +75,7 @@ export default function ActiveWorkspaces() {
|
|||
);
|
||||
})}
|
||||
{showing && !!selectedWs && (
|
||||
<ManageWorkspace hideModal={hideModal} workspace={selectedWs} />
|
||||
<ManageWorkspace hideModal={hideModal} providedSlug={selectedWs.slug} />
|
||||
)}
|
||||
</>
|
||||
);
|
||||
|
|
|
|||
|
|
@ -15,6 +15,22 @@ const Workspace = {
|
|||
|
||||
return { workspace, message };
|
||||
},
|
||||
update: async function (slug, data = {}) {
|
||||
const { workspace, message } = await fetch(
|
||||
`${API_BASE}/workspace/${slug}/update`,
|
||||
{
|
||||
method: "POST",
|
||||
body: JSON.stringify(data),
|
||||
headers: baseHeaders(),
|
||||
}
|
||||
)
|
||||
.then((res) => res.json())
|
||||
.catch((e) => {
|
||||
return { workspace: null, message: e.message };
|
||||
});
|
||||
|
||||
return { workspace, message };
|
||||
},
|
||||
modifyEmbeddings: async function (slug, changes = {}) {
|
||||
const { workspace, message } = await fetch(
|
||||
`${API_BASE}/workspace/${slug}/update-embeddings`,
|
||||
|
|
|
|||
|
|
@ -31,6 +31,6 @@ window.copySnippet = function () {
|
|||
}, 5000);
|
||||
};
|
||||
|
||||
export default function renderMarkdown(text) {
|
||||
export default function renderMarkdown(text = "") {
|
||||
return markdown.render(text);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,6 +20,28 @@ function workspaceEndpoints(app) {
|
|||
}
|
||||
});
|
||||
|
||||
app.post("/workspace/:slug/update", async (request, response) => {
|
||||
try {
|
||||
const { slug = null } = request.params;
|
||||
const data = reqBody(request);
|
||||
const currWorkspace = await Workspace.get(`slug = '${slug}'`);
|
||||
|
||||
if (!currWorkspace) {
|
||||
response.sendStatus(400).end();
|
||||
return;
|
||||
}
|
||||
|
||||
const { workspace, message } = await Workspace.update(
|
||||
currWorkspace.id,
|
||||
data
|
||||
);
|
||||
response.status(200).json({ workspace, message });
|
||||
} catch (e) {
|
||||
console.log(e.message, e);
|
||||
response.sendStatus(500).end();
|
||||
}
|
||||
});
|
||||
|
||||
app.post("/workspace/:slug/update-embeddings", async (request, response) => {
|
||||
try {
|
||||
const { slug = null } = request.params;
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ const { systemEndpoints } = require("./endpoints/system");
|
|||
const { workspaceEndpoints } = require("./endpoints/workspaces");
|
||||
const { chatEndpoints } = require("./endpoints/chat");
|
||||
const { getVectorDbClass } = require("./utils/helpers");
|
||||
const { validateTablePragmas } = require("./utils/database");
|
||||
const app = express();
|
||||
const apiRouter = express.Router();
|
||||
|
||||
|
|
@ -25,8 +26,9 @@ app.use(
|
|||
);
|
||||
|
||||
apiRouter.use("/system/*", validatedRequest);
|
||||
apiRouter.use("/workspace/*", validatedRequest);
|
||||
systemEndpoints(apiRouter);
|
||||
|
||||
apiRouter.use("/workspace/*", validatedRequest);
|
||||
workspaceEndpoints(apiRouter);
|
||||
chatEndpoints(apiRouter);
|
||||
|
||||
|
|
@ -75,7 +77,8 @@ app.all("*", function (_, response) {
|
|||
});
|
||||
|
||||
app
|
||||
.listen(process.env.SERVER_PORT || 3001, () => {
|
||||
.listen(process.env.SERVER_PORT || 3001, async () => {
|
||||
await validateTablePragmas();
|
||||
console.log(
|
||||
`Example app listening on port ${process.env.SERVER_PORT || 3001}`
|
||||
);
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
const { fileData } = require("../utils/files");
|
||||
const { v4: uuidv4 } = require("uuid");
|
||||
const { getVectorDbClass } = require("../utils/helpers");
|
||||
const { checkForMigrations } = require("../utils/database");
|
||||
|
||||
const Document = {
|
||||
tablename: "workspace_documents",
|
||||
|
|
@ -14,7 +15,15 @@ const Document = {
|
|||
createdAt TEXT DEFAULT CURRENT_TIMESTAMP,
|
||||
lastUpdatedAt TEXT DEFAULT CURRENT_TIMESTAMP
|
||||
`,
|
||||
db: async function () {
|
||||
migrateTable: async function () {
|
||||
console.log(`\x1b[34m[MIGRATING]\x1b[0m Checking for Document migrations`);
|
||||
const db = await this.db(false);
|
||||
await checkForMigrations(this, db);
|
||||
},
|
||||
migrations: function () {
|
||||
return [];
|
||||
},
|
||||
db: async function (tracing = true) {
|
||||
const sqlite3 = require("sqlite3").verbose();
|
||||
const { open } = require("sqlite");
|
||||
|
||||
|
|
@ -28,7 +37,8 @@ const Document = {
|
|||
await db.exec(
|
||||
`CREATE TABLE IF NOT EXISTS ${this.tablename} (${this.colsInit})`
|
||||
);
|
||||
db.on("trace", (sql) => console.log(sql));
|
||||
|
||||
if (tracing) db.on("trace", (sql) => console.log(sql));
|
||||
return db;
|
||||
},
|
||||
forWorkspace: async function (workspaceId = null) {
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
const { checkForMigrations } = require("../utils/database");
|
||||
const { Document } = require("./documents");
|
||||
|
||||
// TODO: Do we want to store entire vectorized chunks in here
|
||||
// so that we can easily spin up temp-namespace clones for threading
|
||||
//
|
||||
const DocumentVectors = {
|
||||
tablename: "document_vectors",
|
||||
colsInit: `
|
||||
|
|
@ -12,7 +12,17 @@ const DocumentVectors = {
|
|||
createdAt TEXT DEFAULT CURRENT_TIMESTAMP,
|
||||
lastUpdatedAt TEXT DEFAULT CURRENT_TIMESTAMP
|
||||
`,
|
||||
db: async function () {
|
||||
migrateTable: async function () {
|
||||
console.log(
|
||||
`\x1b[34m[MIGRATING]\x1b[0m Checking for DocumentVector migrations`
|
||||
);
|
||||
const db = await this.db(false);
|
||||
await checkForMigrations(this, db);
|
||||
},
|
||||
migrations: function () {
|
||||
return [];
|
||||
},
|
||||
db: async function (tracing = true) {
|
||||
const sqlite3 = require("sqlite3").verbose();
|
||||
const { open } = require("sqlite");
|
||||
|
||||
|
|
@ -26,7 +36,8 @@ const DocumentVectors = {
|
|||
await db.exec(
|
||||
`CREATE TABLE IF NOT EXISTS ${this.tablename} (${this.colsInit})`
|
||||
);
|
||||
db.on("trace", (sql) => console.log(sql));
|
||||
|
||||
if (tracing) db.on("trace", (sql) => console.log(sql));
|
||||
return db;
|
||||
},
|
||||
bulkInsert: async function (vectorRecords = []) {
|
||||
|
|
|
|||
|
|
@ -1,17 +1,50 @@
|
|||
const slugify = require("slugify");
|
||||
const { Document } = require("./documents");
|
||||
const { checkForMigrations } = require("../utils/database");
|
||||
|
||||
const Workspace = {
|
||||
tablename: "workspaces",
|
||||
writable: [
|
||||
// Used for generic updates so we can validate keys in request body
|
||||
"name",
|
||||
"slug",
|
||||
"vectorTag",
|
||||
"openAiTemp",
|
||||
"lastUpdatedAt",
|
||||
],
|
||||
colsInit: `
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
name TEXT NOT NULL,
|
||||
slug TEXT NOT NULL UNIQUE,
|
||||
vectorTag TEXT DEFAULT NULL,
|
||||
createdAt TEXT DEFAULT CURRENT_TIMESTAMP,
|
||||
openAiTemp REAL DEFAULT NULL,
|
||||
lastUpdatedAt TEXT DEFAULT CURRENT_TIMESTAMP
|
||||
`,
|
||||
db: async function () {
|
||||
migrateTable: async function () {
|
||||
console.log(`\x1b[34m[MIGRATING]\x1b[0m Checking for Workspace migrations`);
|
||||
const db = await this.db(false);
|
||||
await checkForMigrations(this, db);
|
||||
},
|
||||
migrations: function () {
|
||||
return [
|
||||
{
|
||||
colName: "openAiTemp",
|
||||
execCmd: `ALTER TABLE ${this.tablename} ADD COLUMN openAiTemp REAL DEFAULT NULL`,
|
||||
doif: false,
|
||||
},
|
||||
{
|
||||
colName: "id",
|
||||
execCmd: `CREATE TRIGGER IF NOT EXISTS Trg_LastUpdated AFTER UPDATE ON ${this.tablename}
|
||||
FOR EACH ROW
|
||||
BEGIN
|
||||
UPDATE ${this.tablename} SET lastUpdatedAt = CURRENT_TIMESTAMP WHERE id = old.id;
|
||||
END`,
|
||||
doif: true,
|
||||
},
|
||||
];
|
||||
},
|
||||
db: async function (tracing = true) {
|
||||
const sqlite3 = require("sqlite3").verbose();
|
||||
const { open } = require("sqlite");
|
||||
|
||||
|
|
@ -25,17 +58,25 @@ const Workspace = {
|
|||
await db.exec(
|
||||
`CREATE TABLE IF NOT EXISTS ${this.tablename} (${this.colsInit})`
|
||||
);
|
||||
db.on("trace", (sql) => console.log(sql));
|
||||
|
||||
if (tracing) db.on("trace", (sql) => console.log(sql));
|
||||
return db;
|
||||
},
|
||||
new: async function (name = null) {
|
||||
if (!name) return { result: null, message: "name cannot be null" };
|
||||
var slug = slugify(name, { lower: true });
|
||||
|
||||
const existingBySlug = await this.get(`slug = '${slug}'`);
|
||||
if (existingBySlug !== null) {
|
||||
const slugSeed = Math.floor(10000000 + Math.random() * 90000000);
|
||||
slug = slugify(`${name}-${slugSeed}`, { lower: true });
|
||||
}
|
||||
|
||||
const db = await this.db();
|
||||
const { id, success, message } = await db
|
||||
.run(`INSERT INTO ${this.tablename} (name, slug) VALUES (?, ?)`, [
|
||||
name,
|
||||
slugify(name, { lower: true }),
|
||||
slug,
|
||||
])
|
||||
.then((res) => {
|
||||
return { id: res.lastID, success: true, message: null };
|
||||
|
|
@ -43,19 +84,57 @@ const Workspace = {
|
|||
.catch((error) => {
|
||||
return { id: null, success: false, message: error.message };
|
||||
});
|
||||
if (!success) return { workspace: null, message };
|
||||
|
||||
if (!success) {
|
||||
db.close();
|
||||
return { workspace: null, message };
|
||||
}
|
||||
|
||||
const workspace = await db.get(
|
||||
`SELECT * FROM ${this.tablename} WHERE id = ${id}`
|
||||
);
|
||||
db.close();
|
||||
|
||||
return { workspace, message: null };
|
||||
},
|
||||
update: async function (id = null, data = {}) {
|
||||
if (!id) throw new Error("No workspace id provided for update");
|
||||
|
||||
const validKeys = Object.keys(data).filter((key) =>
|
||||
this.writable.includes(key)
|
||||
);
|
||||
const values = Object.values(data);
|
||||
if (validKeys.length === 0 || validKeys.length !== values.length)
|
||||
return { workspace: { id }, message: "No valid fields to update!" };
|
||||
|
||||
const template = `UPDATE ${this.tablename} SET ${validKeys.map((key) => {
|
||||
return `${key}=?`;
|
||||
})} WHERE id = ?`;
|
||||
const db = await this.db();
|
||||
const { success, message } = await db
|
||||
.run(template, [...values, id])
|
||||
.then(() => {
|
||||
return { success: true, message: null };
|
||||
})
|
||||
.catch((error) => {
|
||||
return { success: false, message: error.message };
|
||||
});
|
||||
|
||||
db.close();
|
||||
if (!success) {
|
||||
return { workspace: null, message };
|
||||
}
|
||||
|
||||
const updatedWorkspace = await this.get(`id = ${id}`);
|
||||
return { workspace: updatedWorkspace, message: null };
|
||||
},
|
||||
get: async function (clause = "") {
|
||||
const db = await this.db();
|
||||
const result = await db
|
||||
.get(`SELECT * FROM ${this.tablename} WHERE ${clause}`)
|
||||
.then((res) => res || null);
|
||||
if (!result) return null;
|
||||
db.close();
|
||||
|
||||
const documents = await Document.forWorkspace(result.id);
|
||||
return { ...result, documents };
|
||||
|
|
@ -63,6 +142,8 @@ const Workspace = {
|
|||
delete: async function (clause = "") {
|
||||
const db = await this.db();
|
||||
await db.get(`DELETE FROM ${this.tablename} WHERE ${clause}`);
|
||||
db.close();
|
||||
|
||||
return true;
|
||||
},
|
||||
where: async function (clause = "", limit = null) {
|
||||
|
|
@ -72,6 +153,8 @@ const Workspace = {
|
|||
!!limit ? `LIMIT ${limit}` : ""
|
||||
}`
|
||||
);
|
||||
db.close();
|
||||
|
||||
return results;
|
||||
},
|
||||
};
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
const { checkForMigrations } = require("../utils/database");
|
||||
|
||||
const WorkspaceChats = {
|
||||
tablename: "workspace_chats",
|
||||
colsInit: `
|
||||
|
|
@ -9,7 +11,17 @@ const WorkspaceChats = {
|
|||
createdAt TEXT DEFAULT CURRENT_TIMESTAMP,
|
||||
lastUpdatedAt TEXT DEFAULT CURRENT_TIMESTAMP
|
||||
`,
|
||||
db: async function () {
|
||||
migrateTable: async function () {
|
||||
console.log(
|
||||
`\x1b[34m[MIGRATING]\x1b[0m Checking for WorkspaceChats migrations`
|
||||
);
|
||||
const db = await this.db(false);
|
||||
await checkForMigrations(this, db);
|
||||
},
|
||||
migrations: function () {
|
||||
return [];
|
||||
},
|
||||
db: async function (tracing = true) {
|
||||
const sqlite3 = require("sqlite3").verbose();
|
||||
const { open } = require("sqlite");
|
||||
|
||||
|
|
@ -23,7 +35,8 @@ const WorkspaceChats = {
|
|||
await db.exec(
|
||||
`CREATE TABLE IF NOT EXISTS ${this.tablename} (${this.colsInit})`
|
||||
);
|
||||
db.on("trace", (sql) => console.log(sql));
|
||||
|
||||
if (tracing) db.on("trace", (sql) => console.log(sql));
|
||||
return db;
|
||||
},
|
||||
new: async function ({ workspaceId, prompt, response = {} }) {
|
||||
|
|
@ -39,11 +52,16 @@ const WorkspaceChats = {
|
|||
.catch((error) => {
|
||||
return { id: null, success: false, message: error.message };
|
||||
});
|
||||
if (!success) return { chat: null, message };
|
||||
if (!success) {
|
||||
db.close();
|
||||
return { chat: null, message };
|
||||
}
|
||||
|
||||
const chat = await db.get(
|
||||
`SELECT * FROM ${this.tablename} WHERE id = ${id}`
|
||||
);
|
||||
db.close();
|
||||
|
||||
return { chat, message: null };
|
||||
},
|
||||
forWorkspace: async function (workspaceId = null) {
|
||||
|
|
@ -61,6 +79,8 @@ const WorkspaceChats = {
|
|||
`UPDATE ${this.tablename} SET include = false WHERE workspaceId = ?`,
|
||||
[workspaceId]
|
||||
);
|
||||
db.close();
|
||||
|
||||
return;
|
||||
},
|
||||
get: async function (clause = "") {
|
||||
|
|
@ -68,12 +88,16 @@ const WorkspaceChats = {
|
|||
const result = await db
|
||||
.get(`SELECT * FROM ${this.tablename} WHERE ${clause}`)
|
||||
.then((res) => res || null);
|
||||
db.close();
|
||||
|
||||
if (!result) return null;
|
||||
return result;
|
||||
},
|
||||
delete: async function (clause = "") {
|
||||
const db = await this.db();
|
||||
await db.get(`DELETE FROM ${this.tablename} WHERE ${clause}`);
|
||||
db.close();
|
||||
|
||||
return true;
|
||||
},
|
||||
where: async function (clause = "", limit = null, order = null) {
|
||||
|
|
@ -83,6 +107,8 @@ const WorkspaceChats = {
|
|||
!!limit ? `LIMIT ${limit}` : ""
|
||||
} ${!!order ? order : ""}`
|
||||
);
|
||||
db.close();
|
||||
|
||||
return results;
|
||||
},
|
||||
};
|
||||
|
|
|
|||
|
|
@ -87,7 +87,7 @@ async function chatWithWorkspace(workspace, message, chatMode = "query") {
|
|||
if (!hasVectorizedSpace) {
|
||||
const rawHistory = await WorkspaceChats.forWorkspace(workspace.id);
|
||||
const chatHistory = convertToPromptHistory(rawHistory);
|
||||
const response = await openai.sendChat(chatHistory, message);
|
||||
const response = await openai.sendChat(chatHistory, message, workspace);
|
||||
const data = { text: response, sources: [], type: "chat" };
|
||||
|
||||
await WorkspaceChats.new({
|
||||
|
|
@ -108,7 +108,11 @@ async function chatWithWorkspace(workspace, message, chatMode = "query") {
|
|||
response,
|
||||
sources,
|
||||
message: error,
|
||||
} = await VectorDb[chatMode]({ namespace: workspace.slug, input: message });
|
||||
} = await VectorDb[chatMode]({
|
||||
namespace: workspace.slug,
|
||||
input: message,
|
||||
workspace,
|
||||
});
|
||||
if (!response) {
|
||||
return {
|
||||
id: uuid,
|
||||
|
|
|
|||
54
server/utils/database/index.js
Normal file
54
server/utils/database/index.js
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
function checkColumnTemplate(tablename = null, column = null) {
|
||||
if (!tablename || !column)
|
||||
throw new Error(`Migration Error`, { tablename, column });
|
||||
return `SELECT COUNT(*) AS _exists FROM pragma_table_info('${tablename}') WHERE name='${column}'`;
|
||||
}
|
||||
|
||||
// Note (tcarambat): Since there is no good way to track migrations in Node/SQLite we use this simple system
|
||||
// Each model has a `migrations` method that will return an array like...
|
||||
// { colName: 'stringColName', execCmd: `SQL Command to run when`, doif: boolean },
|
||||
// colName = name of column
|
||||
// execCmd = Command to run when doif matches the state of the DB
|
||||
// doif = condition to match that determines if execCmd will run.
|
||||
// eg: Table workspace has slug column.
|
||||
// execCmd: ALTER TABLE DROP COLUMN slug;
|
||||
// doif: true
|
||||
// => Will drop the slug column if the workspace table has a column named 'slug' otherwise nothing happens.
|
||||
// If you are adding a new table column if needs to exist in the Models `colsInit` and as a migration.
|
||||
// So both new and existing DBs will get the column when code is pulled in.
|
||||
|
||||
async function checkForMigrations(model, db) {
|
||||
if (model.migrations().length === 0) return;
|
||||
const toMigrate = [];
|
||||
for (const { colName, execCmd, doif } of model.migrations()) {
|
||||
const { _exists } = await db.get(
|
||||
checkColumnTemplate(model.tablename, colName)
|
||||
);
|
||||
const colExists = _exists !== 0;
|
||||
if (colExists !== doif) continue;
|
||||
|
||||
toMigrate.push(execCmd);
|
||||
}
|
||||
|
||||
if (toMigrate.length === 0) return;
|
||||
|
||||
console.log(`Running ${toMigrate.length} migrations`, toMigrate);
|
||||
await db.exec(toMigrate.join(";\n"));
|
||||
return;
|
||||
}
|
||||
|
||||
async function validateTablePragmas() {
|
||||
const { Workspace } = require("../../models/workspace");
|
||||
const { Document } = require("../../models/documents");
|
||||
const { DocumentVectors } = require("../../models/vectors");
|
||||
const { WorkspaceChats } = require("../../models/workspaceChats");
|
||||
await Workspace.migrateTable();
|
||||
await Document.migrateTable();
|
||||
await DocumentVectors.migrateTable();
|
||||
await WorkspaceChats.migrateTable();
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
checkForMigrations,
|
||||
validateTablePragmas,
|
||||
};
|
||||
|
|
@ -40,7 +40,7 @@ class OpenAi {
|
|||
return { safe: false, reasons };
|
||||
}
|
||||
|
||||
async sendChat(chatHistory = [], prompt) {
|
||||
async sendChat(chatHistory = [], prompt, workspace = {}) {
|
||||
const model = process.env.OPEN_MODEL_PREF;
|
||||
if (!this.isValidChatModel(model))
|
||||
throw new Error(
|
||||
|
|
@ -50,7 +50,7 @@ class OpenAi {
|
|||
const textResponse = await this.openai
|
||||
.createChatCompletion({
|
||||
model,
|
||||
temperature: 0.7,
|
||||
temperature: Number(workspace?.openAiTemp ?? 0.7),
|
||||
n: 1,
|
||||
messages: [
|
||||
{ role: "system", content: "" },
|
||||
|
|
|
|||
|
|
@ -56,12 +56,12 @@ const Chroma = {
|
|||
const openai = new OpenAIApi(config);
|
||||
return openai;
|
||||
},
|
||||
llm: function () {
|
||||
llm: function ({ temperature = 0.7 }) {
|
||||
const model = process.env.OPEN_MODEL_PREF || "gpt-3.5-turbo";
|
||||
return new OpenAI({
|
||||
openAIApiKey: process.env.OPEN_AI_KEY,
|
||||
temperature: 0.7,
|
||||
modelName: model,
|
||||
temperature,
|
||||
});
|
||||
},
|
||||
embedChunk: async function (openai, textChunk) {
|
||||
|
|
@ -253,7 +253,7 @@ const Chroma = {
|
|||
return true;
|
||||
},
|
||||
query: async function (reqBody = {}) {
|
||||
const { namespace = null, input } = reqBody;
|
||||
const { namespace = null, input, workspace = {} } = reqBody;
|
||||
if (!namespace || !input) throw new Error("Invalid request body");
|
||||
|
||||
const { client } = await this.connect();
|
||||
|
|
@ -269,7 +269,10 @@ const Chroma = {
|
|||
this.embedder(),
|
||||
{ collectionName: namespace, url: process.env.CHROMA_ENDPOINT }
|
||||
);
|
||||
const model = this.llm();
|
||||
const model = this.llm({
|
||||
temperature: workspace?.openAiTemp,
|
||||
});
|
||||
|
||||
const chain = VectorDBQAChain.fromLLM(model, vectorStore, {
|
||||
k: 5,
|
||||
returnSourceDocuments: true,
|
||||
|
|
|
|||
|
|
@ -69,11 +69,16 @@ const LanceDb = {
|
|||
? data[0].embedding
|
||||
: null;
|
||||
},
|
||||
getChatCompletion: async function (openai, messages = []) {
|
||||
getChatCompletion: async function (
|
||||
openai,
|
||||
messages = [],
|
||||
{ temperature = 0.7 }
|
||||
) {
|
||||
const model = process.env.OPEN_MODEL_PREF || "gpt-3.5-turbo";
|
||||
const { data } = await openai.createChatCompletion({
|
||||
model,
|
||||
messages,
|
||||
temperature,
|
||||
});
|
||||
|
||||
if (!data.hasOwnProperty("choices")) return null;
|
||||
|
|
@ -213,7 +218,7 @@ const LanceDb = {
|
|||
}
|
||||
},
|
||||
query: async function (reqBody = {}) {
|
||||
const { namespace = null, input } = reqBody;
|
||||
const { namespace = null, input, workspace = {} } = reqBody;
|
||||
if (!namespace || !input) throw new Error("Invalid request body");
|
||||
|
||||
const { client } = await this.connect();
|
||||
|
|
@ -242,7 +247,9 @@ const LanceDb = {
|
|||
},
|
||||
{ role: "user", content: input },
|
||||
];
|
||||
const responseText = await this.getChatCompletion(this.openai(), messages);
|
||||
const responseText = await this.getChatCompletion(this.openai(), messages, {
|
||||
temperature: workspace?.openAiTemp,
|
||||
});
|
||||
|
||||
return {
|
||||
response: responseText,
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
const { PineconeClient } = require("@pinecone-database/pinecone");
|
||||
const { PineconeStore } = require("langchain/vectorstores/pinecone");
|
||||
const { OpenAI } = require("langchain/llms/openai");
|
||||
const { ChatOpenAI } = require("langchain/chat_models/openai");
|
||||
const { VectorDBQAChain, LLMChain } = require("langchain/chains");
|
||||
const { OpenAIEmbeddings } = require("langchain/embeddings/openai");
|
||||
const { VectorStoreRetrieverMemory } = require("langchain/memory");
|
||||
|
|
@ -50,20 +49,12 @@ const Pinecone = {
|
|||
? data[0].embedding
|
||||
: null;
|
||||
},
|
||||
llm: function () {
|
||||
llm: function ({ temperature = 0.7 }) {
|
||||
const model = process.env.OPEN_MODEL_PREF || "gpt-3.5-turbo";
|
||||
return new OpenAI({
|
||||
openAIApiKey: process.env.OPEN_AI_KEY,
|
||||
temperature: 0.7,
|
||||
modelName: model,
|
||||
});
|
||||
},
|
||||
chatLLM: function () {
|
||||
const model = process.env.OPEN_MODEL_PREF || "gpt-3.5-turbo";
|
||||
return new ChatOpenAI({
|
||||
openAIApiKey: process.env.OPEN_AI_KEY,
|
||||
temperature: 0.7,
|
||||
modelName: model,
|
||||
temperature,
|
||||
});
|
||||
},
|
||||
totalIndicies: async function () {
|
||||
|
|
@ -233,7 +224,7 @@ const Pinecone = {
|
|||
};
|
||||
},
|
||||
query: async function (reqBody = {}) {
|
||||
const { namespace = null, input } = reqBody;
|
||||
const { namespace = null, input, workspace = {} } = reqBody;
|
||||
if (!namespace || !input) throw new Error("Invalid request body");
|
||||
|
||||
const { pineconeIndex } = await this.connect();
|
||||
|
|
@ -250,7 +241,9 @@ const Pinecone = {
|
|||
namespace,
|
||||
});
|
||||
|
||||
const model = this.llm();
|
||||
const model = this.llm({
|
||||
temperature: workspace?.openAiTemp,
|
||||
});
|
||||
const chain = VectorDBQAChain.fromLLM(model, vectorStore, {
|
||||
k: 5,
|
||||
returnSourceDocuments: true,
|
||||
|
|
@ -265,7 +258,7 @@ const Pinecone = {
|
|||
// This implementation of chat also expands the memory of the chat itself
|
||||
// and adds more tokens to the PineconeDB instance namespace
|
||||
chat: async function (reqBody = {}) {
|
||||
const { namespace = null, input } = reqBody;
|
||||
const { namespace = null, input, workspace = {} } = reqBody;
|
||||
if (!namespace || !input) throw new Error("Invalid request body");
|
||||
|
||||
const { pineconeIndex } = await this.connect();
|
||||
|
|
@ -284,7 +277,9 @@ const Pinecone = {
|
|||
memoryKey: "history",
|
||||
});
|
||||
|
||||
const model = this.llm();
|
||||
const model = this.llm({
|
||||
temperature: workspace?.openAiTemp,
|
||||
});
|
||||
const prompt =
|
||||
PromptTemplate.fromTemplate(`The following is a friendly conversation between a human and an AI. The AI is very casual and talkative and responds with a friendly tone. If the AI does not know the answer to a question, it truthfully says it does not know.
|
||||
Relevant pieces of previous conversation:
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user