diff --git a/collector/index.js b/collector/index.js index 9ebe5f1ce..a1142d756 100644 --- a/collector/index.js +++ b/collector/index.js @@ -25,7 +25,7 @@ app.use( ); app.post("/process", async function (request, response) { - const { filename } = reqBody(request); + const { filename, options = {} } = reqBody(request); try { const targetFilename = path .normalize(filename) @@ -34,7 +34,7 @@ app.post("/process", async function (request, response) { success, reason, documents = [], - } = await processSingleFile(targetFilename); + } = await processSingleFile(targetFilename, options); response .status(200) .json({ filename: targetFilename, success, reason, documents }); diff --git a/collector/package.json b/collector/package.json index d145ab865..8a0441d78 100644 --- a/collector/package.json +++ b/collector/package.json @@ -33,6 +33,7 @@ "moment": "^2.29.4", "multer": "^1.4.5-lts.1", "officeparser": "^4.0.5", + "openai": "^3.2.1", "pdf-parse": "^1.1.1", "puppeteer": "~21.5.2", "slugify": "^1.6.6", @@ -46,4 +47,4 @@ "nodemon": "^2.0.22", "prettier": "^2.4.1" } -} +} \ No newline at end of file diff --git a/collector/processSingleFile/convert/asAudio.js b/collector/processSingleFile/convert/asAudio.js index 15ae5cf00..170426e40 100644 --- a/collector/processSingleFile/convert/asAudio.js +++ b/collector/processSingleFile/convert/asAudio.js @@ -1,5 +1,3 @@ -const fs = require("fs"); -const path = require("path"); const { v4 } = require("uuid"); const { createdDate, @@ -9,39 +7,35 @@ const { const { tokenizeString } = require("../../utils/tokenizer"); const { default: slugify } = require("slugify"); const { LocalWhisper } = require("../../utils/WhisperProviders/localWhisper"); +const { OpenAiWhisper } = require("../../utils/WhisperProviders/OpenAiWhisper"); -async function asAudio({ fullFilePath = "", filename = "" }) { - const whisper = new LocalWhisper(); +const WHISPER_PROVIDERS = { + openai: OpenAiWhisper, + local: LocalWhisper, +}; + +async function asAudio({ fullFilePath = "", filename = "", options = {} }) { + const WhisperProvider = WHISPER_PROVIDERS.hasOwnProperty( + options?.whisperProvider + ) + ? WHISPER_PROVIDERS[options?.whisperProvider] + : WHISPER_PROVIDERS.local; console.log(`-- Working ${filename} --`); - const transcriberPromise = new Promise((resolve) => - whisper.client().then((client) => resolve(client)) - ); - const audioDataPromise = new Promise((resolve) => - convertToWavAudioData(fullFilePath).then((audioData) => resolve(audioData)) - ); - const [audioData, transcriber] = await Promise.all([ - audioDataPromise, - transcriberPromise, - ]); + const whisper = new WhisperProvider({ options }); + const { content, error } = await whisper.processFile(fullFilePath, filename); - if (!audioData) { - console.error(`Failed to parse content from ${filename}.`); + if (!!error) { + console.error(`Error encountered for parsing of ${filename}.`); trashFile(fullFilePath); return { success: false, - reason: `Failed to parse content from ${filename}.`, + reason: error, documents: [], }; } - console.log(`[Model Working]: Transcribing audio data to text`); - const { text: content } = await transcriber(audioData, { - chunk_length_s: 30, - stride_length_s: 5, - }); - - if (!content.length) { + if (!content?.length) { console.error(`Resulting text content was empty for ${filename}.`); trashFile(fullFilePath); return { @@ -76,79 +70,4 @@ async function asAudio({ fullFilePath = "", filename = "" }) { return { success: true, reason: null, documents: [document] }; } -async function convertToWavAudioData(sourcePath) { - try { - let buffer; - const wavefile = require("wavefile"); - const ffmpeg = require("fluent-ffmpeg"); - const outFolder = path.resolve(__dirname, `../../storage/tmp`); - if (!fs.existsSync(outFolder)) fs.mkdirSync(outFolder, { recursive: true }); - - const fileExtension = path.extname(sourcePath).toLowerCase(); - if (fileExtension !== ".wav") { - console.log( - `[Conversion Required] ${fileExtension} file detected - converting to .wav` - ); - const outputFile = path.resolve(outFolder, `${v4()}.wav`); - const convert = new Promise((resolve) => { - ffmpeg(sourcePath) - .toFormat("wav") - .on("error", (error) => { - console.error(`[Conversion Error] ${error.message}`); - resolve(false); - }) - .on("progress", (progress) => - console.log( - `[Conversion Processing]: ${progress.targetSize}KB converted` - ) - ) - .on("end", () => { - console.log("[Conversion Complete]: File converted to .wav!"); - resolve(true); - }) - .save(outputFile); - }); - const success = await convert; - if (!success) - throw new Error( - "[Conversion Failed]: Could not convert file to .wav format!" - ); - - const chunks = []; - const stream = fs.createReadStream(outputFile); - for await (let chunk of stream) chunks.push(chunk); - buffer = Buffer.concat(chunks); - fs.rmSync(outputFile); - } else { - const chunks = []; - const stream = fs.createReadStream(sourcePath); - for await (let chunk of stream) chunks.push(chunk); - buffer = Buffer.concat(chunks); - } - - const wavFile = new wavefile.WaveFile(buffer); - wavFile.toBitDepth("32f"); - wavFile.toSampleRate(16000); - - let audioData = wavFile.getSamples(); - if (Array.isArray(audioData)) { - if (audioData.length > 1) { - const SCALING_FACTOR = Math.sqrt(2); - - // Merge channels into first channel to save memory - for (let i = 0; i < audioData[0].length; ++i) { - audioData[0][i] = - (SCALING_FACTOR * (audioData[0][i] + audioData[1][i])) / 2; - } - } - audioData = audioData[0]; - } - - return audioData; - } catch (error) { - console.error(`convertToWavAudioData`, error); - return null; - } -} - module.exports = asAudio; diff --git a/collector/processSingleFile/index.js b/collector/processSingleFile/index.js index 569a2cde2..5d9e6a38a 100644 --- a/collector/processSingleFile/index.js +++ b/collector/processSingleFile/index.js @@ -7,7 +7,7 @@ const { const { trashFile, isTextType } = require("../utils/files"); const RESERVED_FILES = ["__HOTDIR__.md"]; -async function processSingleFile(targetFilename) { +async function processSingleFile(targetFilename, options = {}) { const fullFilePath = path.resolve(WATCH_DIRECTORY, targetFilename); if (RESERVED_FILES.includes(targetFilename)) return { @@ -54,6 +54,7 @@ async function processSingleFile(targetFilename) { return await FileTypeProcessor({ fullFilePath, filename: targetFilename, + options, }); } diff --git a/collector/utils/WhisperProviders/OpenAiWhisper.js b/collector/utils/WhisperProviders/OpenAiWhisper.js new file mode 100644 index 000000000..3b9d08e6a --- /dev/null +++ b/collector/utils/WhisperProviders/OpenAiWhisper.js @@ -0,0 +1,44 @@ +const fs = require("fs"); + +class OpenAiWhisper { + constructor({ options }) { + const { Configuration, OpenAIApi } = require("openai"); + if (!options.openAiKey) throw new Error("No OpenAI API key was set."); + + const config = new Configuration({ + apiKey: options.openAiKey, + }); + this.openai = new OpenAIApi(config); + this.model = "whisper-1"; + this.temperature = 0; + this.#log("Initialized."); + } + + #log(text, ...args) { + console.log(`\x1b[32m[OpenAiWhisper]\x1b[0m ${text}`, ...args); + } + + async processFile(fullFilePath) { + return await this.openai + .createTranscription( + fs.createReadStream(fullFilePath), + this.model, + undefined, + "text", + this.temperature + ) + .then((res) => { + if (res.hasOwnProperty("data")) + return { content: res.data, error: null }; + return { content: "", error: "No content was able to be transcribed." }; + }) + .catch((e) => { + this.#log(`Could not get any response from openai whisper`, e.message); + return { content: "", error: e.message }; + }); + } +} + +module.exports = { + OpenAiWhisper, +}; diff --git a/collector/utils/WhisperProviders/localWhisper.js b/collector/utils/WhisperProviders/localWhisper.js index 6503e2021..46dbe226b 100644 --- a/collector/utils/WhisperProviders/localWhisper.js +++ b/collector/utils/WhisperProviders/localWhisper.js @@ -1,5 +1,6 @@ -const path = require("path"); const fs = require("fs"); +const path = require("path"); +const { v4 } = require("uuid"); class LocalWhisper { constructor() { @@ -16,12 +17,94 @@ class LocalWhisper { // Make directory when it does not exist in existing installations if (!fs.existsSync(this.cacheDir)) fs.mkdirSync(this.cacheDir, { recursive: true }); + + this.#log("Initialized."); + } + + #log(text, ...args) { + console.log(`\x1b[32m[LocalWhisper]\x1b[0m ${text}`, ...args); + } + + async #convertToWavAudioData(sourcePath) { + try { + let buffer; + const wavefile = require("wavefile"); + const ffmpeg = require("fluent-ffmpeg"); + const outFolder = path.resolve(__dirname, `../../storage/tmp`); + if (!fs.existsSync(outFolder)) + fs.mkdirSync(outFolder, { recursive: true }); + + const fileExtension = path.extname(sourcePath).toLowerCase(); + if (fileExtension !== ".wav") { + this.#log( + `File conversion required! ${fileExtension} file detected - converting to .wav` + ); + const outputFile = path.resolve(outFolder, `${v4()}.wav`); + const convert = new Promise((resolve) => { + ffmpeg(sourcePath) + .toFormat("wav") + .on("error", (error) => { + this.#log(`Conversion Error! ${error.message}`); + resolve(false); + }) + .on("progress", (progress) => + this.#log( + `Conversion Processing! ${progress.targetSize}KB converted` + ) + ) + .on("end", () => { + this.#log(`Conversion Complete! File converted to .wav!`); + resolve(true); + }) + .save(outputFile); + }); + const success = await convert; + if (!success) + throw new Error( + "[Conversion Failed]: Could not convert file to .wav format!" + ); + + const chunks = []; + const stream = fs.createReadStream(outputFile); + for await (let chunk of stream) chunks.push(chunk); + buffer = Buffer.concat(chunks); + fs.rmSync(outputFile); + } else { + const chunks = []; + const stream = fs.createReadStream(sourcePath); + for await (let chunk of stream) chunks.push(chunk); + buffer = Buffer.concat(chunks); + } + + const wavFile = new wavefile.WaveFile(buffer); + wavFile.toBitDepth("32f"); + wavFile.toSampleRate(16000); + + let audioData = wavFile.getSamples(); + if (Array.isArray(audioData)) { + if (audioData.length > 1) { + const SCALING_FACTOR = Math.sqrt(2); + + // Merge channels into first channel to save memory + for (let i = 0; i < audioData[0].length; ++i) { + audioData[0][i] = + (SCALING_FACTOR * (audioData[0][i] + audioData[1][i])) / 2; + } + } + audioData = audioData[0]; + } + + return audioData; + } catch (error) { + console.error(`convertToWavAudioData`, error); + return null; + } } async client() { if (!fs.existsSync(this.modelPath)) { - console.log( - "\x1b[34m[INFO]\x1b[0m The native whisper model has never been run and will be downloaded right now. Subsequent runs will be faster. (~250MB)\n\n" + this.#log( + `The native whisper model has never been run and will be downloaded right now. Subsequent runs will be faster. (~250MB)` ); } @@ -48,10 +131,45 @@ class LocalWhisper { : {}), }); } catch (error) { - console.error("Failed to load the native whisper model:", error); + this.#log("Failed to load the native whisper model:", error); throw error; } } + + async processFile(fullFilePath, filename) { + try { + const transcriberPromise = new Promise((resolve) => + this.client().then((client) => resolve(client)) + ); + const audioDataPromise = new Promise((resolve) => + this.#convertToWavAudioData(fullFilePath).then((audioData) => + resolve(audioData) + ) + ); + const [audioData, transcriber] = await Promise.all([ + audioDataPromise, + transcriberPromise, + ]); + + if (!audioData) { + this.#log(`Failed to parse content from ${filename}.`); + return { + content: null, + error: `Failed to parse content from ${filename}.`, + }; + } + + this.#log(`Transcribing audio data to text...`); + const { text } = await transcriber(audioData, { + chunk_length_s: 30, + stride_length_s: 5, + }); + + return { content: text, error: null }; + } catch (error) { + return { content: null, error: error.message }; + } + } } module.exports = { diff --git a/collector/yarn.lock b/collector/yarn.lock index bf979c86c..3bb0f1ea7 100644 --- a/collector/yarn.lock +++ b/collector/yarn.lock @@ -372,6 +372,13 @@ asynckit@^0.4.0: resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" integrity sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q== +axios@^0.26.0: + version "0.26.1" + resolved "https://registry.yarnpkg.com/axios/-/axios-0.26.1.tgz#1ede41c51fcf51bbbd6fd43669caaa4f0495aaa9" + integrity sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA== + dependencies: + follow-redirects "^1.14.8" + b4a@^1.6.4: version "1.6.4" resolved "https://registry.yarnpkg.com/b4a/-/b4a-1.6.4.tgz#ef1c1422cae5ce6535ec191baeed7567443f36c9" @@ -1203,6 +1210,11 @@ fluent-ffmpeg@^2.1.2: async ">=0.2.9" which "^1.1.1" +follow-redirects@^1.14.8: + version "1.15.6" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.6.tgz#7f815c0cda4249c74ff09e95ef97c23b5fd0399b" + integrity sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA== + form-data-encoder@1.7.2: version "1.7.2" resolved "https://registry.yarnpkg.com/form-data-encoder/-/form-data-encoder-1.7.2.tgz#1f1ae3dccf58ed4690b86d87e4f57c654fbab040" @@ -2304,6 +2316,14 @@ onnxruntime-web@1.14.0: onnxruntime-common "~1.14.0" platform "^1.3.6" +openai@^3.2.1: + version "3.3.0" + resolved "https://registry.yarnpkg.com/openai/-/openai-3.3.0.tgz#a6408016ad0945738e1febf43f2fccca83a3f532" + integrity sha512-uqxI/Au+aPRnsaQRe8CojU0eCR7I0mBiKjD3sNMzY6DaC1ZVrc85u98mtJW6voDug8fgGN+DIZmTDxTthxb7dQ== + dependencies: + axios "^0.26.0" + form-data "^4.0.0" + openai@^4.19.0: version "4.20.1" resolved "https://registry.yarnpkg.com/openai/-/openai-4.20.1.tgz#afa0d496d125b5a0f6cebcb4b9aeabf71e00214e" diff --git a/docker/.env.example b/docker/.env.example index ae4913dc4..ed6fd3bce 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -131,6 +131,16 @@ GID='1000' # ASTRA_DB_APPLICATION_TOKEN= # ASTRA_DB_ENDPOINT= +########################################### +######## Audio Model Selection ############ +########################################### +# (default) use built-in whisper-small model. +# WHISPER_PROVIDER="local" + +# use openai hosted whisper model. +# WHISPER_PROVIDER="openai" +# OPEN_AI_KEY=sk-xxxxxxxx + # CLOUD DEPLOYMENT VARIRABLES ONLY # AUTH_TOKEN="hunter2" # This is the password to your application if remote hosting. # DISABLE_TELEMETRY="false" diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index 86f6eb08a..9ef160e72 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -29,6 +29,9 @@ const GeneralApiKeys = lazy(() => import("@/pages/GeneralSettings/ApiKeys")); const GeneralLLMPreference = lazy( () => import("@/pages/GeneralSettings/LLMPreference") ); +const GeneralTranscriptionPreference = lazy( + () => import("@/pages/GeneralSettings/TranscriptionPreference") +); const GeneralEmbeddingPreference = lazy( () => import("@/pages/GeneralSettings/EmbeddingPreference") ); @@ -47,6 +50,9 @@ const EmbedConfigSetup = lazy( () => import("@/pages/GeneralSettings/EmbedConfigs") ); const EmbedChats = lazy(() => import("@/pages/GeneralSettings/EmbedChats")); +const PrivacyAndData = lazy( + () => import("@/pages/GeneralSettings/PrivacyAndData") +); export default function App() { return ( @@ -76,6 +82,12 @@ export default function App() { path="/settings/llm-preference" element={} /> + + } + /> } @@ -101,6 +113,10 @@ export default function App() { path="/settings/security" element={} /> + } + /> } diff --git a/frontend/src/components/LLMSelection/AnthropicAiOptions/index.jsx b/frontend/src/components/LLMSelection/AnthropicAiOptions/index.jsx index 6bc18a5ac..e8c288d60 100644 --- a/frontend/src/components/LLMSelection/AnthropicAiOptions/index.jsx +++ b/frontend/src/components/LLMSelection/AnthropicAiOptions/index.jsx @@ -52,6 +52,7 @@ export default function AnthropicAiOptions({ settings, showAlert = false }) { "claude-instant-1.2", "claude-2.0", "claude-2.1", + "claude-3-haiku-20240307", "claude-3-opus-20240229", "claude-3-sonnet-20240229", ].map((model) => { diff --git a/frontend/src/components/Modals/MangeWorkspace/Documents/Directory/FileRow/index.jsx b/frontend/src/components/Modals/MangeWorkspace/Documents/Directory/FileRow/index.jsx index 7e2259b22..976c65988 100644 --- a/frontend/src/components/Modals/MangeWorkspace/Documents/Directory/FileRow/index.jsx +++ b/frontend/src/components/Modals/MangeWorkspace/Documents/Directory/FileRow/index.jsx @@ -13,7 +13,6 @@ export default function FileRow({ folderName, selected, toggleSelection, - expanded, fetchKeys, setLoading, setLoadingMessage, @@ -53,12 +52,13 @@ export default function FileRow({ const handleMouseEnter = debounce(handleShowTooltip, 500); const handleMouseLeave = debounce(handleHideTooltip, 500); + return ( -
toggleSelection(item)} - className={`transition-all duration-200 text-white/80 text-xs grid grid-cols-12 py-2 pl-3.5 pr-8 border-b border-white/20 hover:bg-sky-500/20 cursor-pointer ${`${ - selected ? "bg-sky-500/20" : "" - } ${expanded ? "bg-sky-500/10" : ""}`}`} + className={`transition-all duration-200 text-white/80 text-xs grid grid-cols-12 py-2 pl-3.5 pr-8 hover:bg-sky-500/20 cursor-pointer file-row ${ + selected ? "selected" : "" + }`} >
-
+ ); } diff --git a/frontend/src/components/Modals/MangeWorkspace/Documents/Directory/FolderRow/index.jsx b/frontend/src/components/Modals/MangeWorkspace/Documents/Directory/FolderRow/index.jsx index 5b7f1be39..48953ab1f 100644 --- a/frontend/src/components/Modals/MangeWorkspace/Documents/Directory/FolderRow/index.jsx +++ b/frontend/src/components/Modals/MangeWorkspace/Documents/Directory/FolderRow/index.jsx @@ -47,10 +47,10 @@ export default function FolderRow({ return ( <> -
@@ -88,7 +88,7 @@ export default function FolderRow({ /> )}
-
+ {expanded && (
{item.items.map((fileItem) => ( @@ -97,7 +97,6 @@ export default function FolderRow({ item={fileItem} folderName={item.name} selected={isSelected(fileItem.id)} - expanded={expanded} toggleSelection={toggleSelection} fetchKeys={fetchKeys} setLoading={setLoading} diff --git a/frontend/src/components/Modals/MangeWorkspace/Documents/WorkspaceDirectory/WorkspaceFileRow/index.jsx b/frontend/src/components/Modals/MangeWorkspace/Documents/WorkspaceDirectory/WorkspaceFileRow/index.jsx index 3367c7289..f73916290 100644 --- a/frontend/src/components/Modals/MangeWorkspace/Documents/WorkspaceDirectory/WorkspaceFileRow/index.jsx +++ b/frontend/src/components/Modals/MangeWorkspace/Documents/WorkspaceDirectory/WorkspaceFileRow/index.jsx @@ -53,8 +53,8 @@ export default function WorkspaceFileRow({ const handleMouseLeave = debounce(handleHideTooltip, 500); return (
-
+

Name

Date

Kind

@@ -148,7 +148,7 @@ const PinAlert = memo(() => {
-
+

diff --git a/frontend/src/components/SettingsSidebar/index.jsx b/frontend/src/components/SettingsSidebar/index.jsx index 84b78064a..66f881ff6 100644 --- a/frontend/src/components/SettingsSidebar/index.jsx +++ b/frontend/src/components/SettingsSidebar/index.jsx @@ -19,6 +19,8 @@ import { Notepad, CodeBlock, Barcode, + ClosedCaptioning, + EyeSlash, } from "@phosphor-icons/react"; import useUser from "@/hooks/useUser"; import { USER_BACKGROUND_COLOR } from "@/utils/constants"; @@ -278,9 +280,17 @@ const SidebarOptions = ({ user = null }) => ( flex={true} allowedRole={["admin"]} /> +

diff --git a/frontend/src/components/WorkspaceChat/ChatContainer/index.jsx b/frontend/src/components/WorkspaceChat/ChatContainer/index.jsx index 293da491f..209fed5d6 100644 --- a/frontend/src/components/WorkspaceChat/ChatContainer/index.jsx +++ b/frontend/src/components/WorkspaceChat/ChatContainer/index.jsx @@ -68,11 +68,7 @@ export default function ChatContainer({ workspace, knownHistory = [] }) { const remHistory = chatHistory.length > 0 ? chatHistory.slice(0, -1) : []; var _chatHistory = [...remHistory]; - if (!promptMessage || !promptMessage?.userMessage) { - setLoadingResponse(false); - return false; - } - + if (!promptMessage || !promptMessage?.userMessage) return false; if (!!threadSlug) { await Workspace.threads.streamChat( { workspaceSlug: workspace.slug, threadSlug }, diff --git a/frontend/src/hooks/useGetProvidersModels.js b/frontend/src/hooks/useGetProvidersModels.js index 57a95ea7a..f578c929f 100644 --- a/frontend/src/hooks/useGetProvidersModels.js +++ b/frontend/src/hooks/useGetProvidersModels.js @@ -19,6 +19,7 @@ const PROVIDER_DEFAULT_MODELS = { "claude-2.1", "claude-3-opus-20240229", "claude-3-sonnet-20240229", + "claude-3-haiku-20240307", ], azure: [], lmstudio: [], diff --git a/frontend/src/index.css b/frontend/src/index.css index e2141d8de..b355eb20a 100644 --- a/frontend/src/index.css +++ b/frontend/src/index.css @@ -597,3 +597,19 @@ dialog::backdrop { font-weight: 600; color: #fff; } + +.file-row:nth-child(odd) { + @apply bg-[#1C1E21]; +} + +.file-row:nth-child(even) { + @apply bg-[#2C2C2C]; +} + +.file-row.selected:nth-child(odd) { + @apply bg-sky-500/20; +} + +.file-row.selected:nth-child(even) { + @apply bg-sky-500/10; +} diff --git a/frontend/src/models/workspace.js b/frontend/src/models/workspace.js index 6786abffd..ae2cd5590 100644 --- a/frontend/src/models/workspace.js +++ b/frontend/src/models/workspace.js @@ -3,6 +3,7 @@ import { baseHeaders } from "@/utils/request"; import { fetchEventSource } from "@microsoft/fetch-event-source"; import WorkspaceThread from "@/models/workspaceThread"; import { v4 } from "uuid"; +import { ABORT_STREAM_EVENT } from "@/utils/chat"; const Workspace = { new: async function (data = {}) { @@ -75,6 +76,16 @@ const Workspace = { }, streamChat: async function ({ slug }, message, handleChat) { const ctrl = new AbortController(); + + // Listen for the ABORT_STREAM_EVENT key to be emitted by the client + // to early abort the streaming response. On abort we send a special `stopGeneration` + // event to be handled which resets the UI for us to be able to send another message. + // The backend response abort handling is done in each LLM's handleStreamResponse. + window.addEventListener(ABORT_STREAM_EVENT, () => { + ctrl.abort(); + handleChat({ id: v4(), type: "stopGeneration" }); + }); + await fetchEventSource(`${API_BASE}/workspace/${slug}/stream-chat`, { method: "POST", body: JSON.stringify({ message }), diff --git a/frontend/src/models/workspaceThread.js b/frontend/src/models/workspaceThread.js index f9fad3173..b1bcaf644 100644 --- a/frontend/src/models/workspaceThread.js +++ b/frontend/src/models/workspaceThread.js @@ -1,3 +1,4 @@ +import { ABORT_STREAM_EVENT } from "@/utils/chat"; import { API_BASE } from "@/utils/constants"; import { baseHeaders } from "@/utils/request"; import { fetchEventSource } from "@microsoft/fetch-event-source"; @@ -80,6 +81,16 @@ const WorkspaceThread = { handleChat ) { const ctrl = new AbortController(); + + // Listen for the ABORT_STREAM_EVENT key to be emitted by the client + // to early abort the streaming response. On abort we send a special `stopGeneration` + // event to be handled which resets the UI for us to be able to send another message. + // The backend response abort handling is done in each LLM's handleStreamResponse. + window.addEventListener(ABORT_STREAM_EVENT, () => { + ctrl.abort(); + handleChat({ id: v4(), type: "stopGeneration" }); + }); + await fetchEventSource( `${API_BASE}/workspace/${workspaceSlug}/thread/${threadSlug}/stream-chat`, { diff --git a/frontend/src/pages/GeneralSettings/PrivacyAndData/index.jsx b/frontend/src/pages/GeneralSettings/PrivacyAndData/index.jsx new file mode 100644 index 000000000..dfc4b29f9 --- /dev/null +++ b/frontend/src/pages/GeneralSettings/PrivacyAndData/index.jsx @@ -0,0 +1,206 @@ +import { useEffect, useState } from "react"; +import Sidebar from "@/components/SettingsSidebar"; +import { isMobile } from "react-device-detect"; +import showToast from "@/utils/toast"; +import System from "@/models/system"; +import PreLoader from "@/components/Preloader"; +import { + EMBEDDING_ENGINE_PRIVACY, + LLM_SELECTION_PRIVACY, + VECTOR_DB_PRIVACY, +} from "@/pages/OnboardingFlow/Steps/DataHandling"; + +export default function PrivacyAndDataHandling() { + const [settings, setSettings] = useState({}); + const [loading, setLoading] = useState(true); + + useEffect(() => { + async function fetchSettings() { + setLoading(true); + const settings = await System.keys(); + setSettings(settings); + setLoading(false); + } + fetchSettings(); + }, []); + + return ( +
+ +
+
+
+
+

+ Privacy & Data-Handling +

+
+

+ This is your configuration for how connected third party providers + and AnythingLLM handle your data. +

+
+ {loading ? ( +
+
+ +
+
+ ) : ( + <> + + + + )} +
+
+
+ ); +} + +function ThirdParty({ settings }) { + const llmChoice = settings?.LLMProvider || "openai"; + const embeddingEngine = settings?.EmbeddingEngine || "openai"; + const vectorDb = settings?.VectorDB || "pinecone"; + + return ( +
+
+
+
LLM Selection
+
+ LLM Logo +

+ {LLM_SELECTION_PRIVACY[llmChoice].name} +

+
+
    + {LLM_SELECTION_PRIVACY[llmChoice].description.map((desc) => ( +
  • {desc}
    • + ))} +
+
+
+
Embedding Engine
+
+ LLM Logo +

+ {EMBEDDING_ENGINE_PRIVACY[embeddingEngine].name} +

+
+
    + {EMBEDDING_ENGINE_PRIVACY[embeddingEngine].description.map( + (desc) => ( +
  • {desc}
    • + ) + )} +
+
+ +
+
Vector Database
+
+ LLM Logo +

+ {VECTOR_DB_PRIVACY[vectorDb].name} +

+
+
    + {VECTOR_DB_PRIVACY[vectorDb].description.map((desc) => ( +
  • {desc}
    • + ))} +
+
+
+
+ ); +} + +function TelemetryLogs({ settings }) { + const [telemetry, setTelemetry] = useState( + settings?.DisableTelemetry !== "true" + ); + async function toggleTelemetry() { + await System.updateSystem({ + DisableTelemetry: !telemetry ? "false" : "true", + }); + setTelemetry(!telemetry); + showToast( + `Anonymous Telemetry has been ${!telemetry ? "enabled" : "disabled"}.`, + "info", + { clear: true } + ); + } + + return ( +
+
+
+
+
+
+ + +
+
+
+
+

+ All events do not record IP-address and contain{" "} + no identifying content, settings, chats, or other non-usage + based information. To see the list of event tags collected you can + look on{" "} + + Github here + + . +

+

+ As an open-source project we respect your right to privacy. We are + dedicated to building the best solution for integrating AI and + documents privately and securely. If you do decide to turn off + telemetry all we ask is to consider sending us feedback and thoughts + so that we can continue to improve AnythingLLM for you.{" "} + + team@mintplexlabs.com + + . +

+
+
+
+ ); +} diff --git a/frontend/src/pages/GeneralSettings/TranscriptionPreference/index.jsx b/frontend/src/pages/GeneralSettings/TranscriptionPreference/index.jsx new file mode 100644 index 000000000..a56dc26e7 --- /dev/null +++ b/frontend/src/pages/GeneralSettings/TranscriptionPreference/index.jsx @@ -0,0 +1,180 @@ +import React, { useEffect, useState } from "react"; +import { isMobile } from "react-device-detect"; +import Sidebar from "@/components/SettingsSidebar"; +import System from "@/models/system"; +import showToast from "@/utils/toast"; +import PreLoader from "@/components/Preloader"; + +import OpenAiLogo from "@/media/llmprovider/openai.png"; +import AnythingLLMIcon from "@/media/logo/anything-llm-icon.png"; +import OpenAiWhisperOptions from "@/components/TranscriptionSelection/OpenAiOptions"; +import NativeTranscriptionOptions from "@/components/TranscriptionSelection/NativeTranscriptionOptions"; +import LLMItem from "@/components/LLMSelection/LLMItem"; +import { MagnifyingGlass } from "@phosphor-icons/react"; + +export default function TranscriptionModelPreference() { + const [saving, setSaving] = useState(false); + const [hasChanges, setHasChanges] = useState(false); + const [settings, setSettings] = useState(null); + const [loading, setLoading] = useState(true); + const [searchQuery, setSearchQuery] = useState(""); + const [filteredProviders, setFilteredProviders] = useState([]); + const [selectedProvider, setSelectedProvider] = useState(null); + + const handleSubmit = async (e) => { + e.preventDefault(); + const form = e.target; + const data = { WhisperProvider: selectedProvider }; + const formData = new FormData(form); + + for (var [key, value] of formData.entries()) data[key] = value; + const { error } = await System.updateSystem(data); + setSaving(true); + + if (error) { + showToast(`Failed to save preferences: ${error}`, "error"); + } else { + showToast("Transcription preferences saved successfully.", "success"); + } + setSaving(false); + setHasChanges(!!error); + }; + + const updateProviderChoice = (selection) => { + setSelectedProvider(selection); + setHasChanges(true); + }; + + useEffect(() => { + async function fetchKeys() { + const _settings = await System.keys(); + setSettings(_settings); + setSelectedProvider(_settings?.WhisperProvider || "local"); + setLoading(false); + } + fetchKeys(); + }, []); + + useEffect(() => { + const filtered = PROVIDERS.filter((provider) => + provider.name.toLowerCase().includes(searchQuery.toLowerCase()) + ); + setFilteredProviders(filtered); + }, [searchQuery, selectedProvider]); + + const PROVIDERS = [ + { + name: "OpenAI", + value: "openai", + logo: OpenAiLogo, + options: , + description: + "Leverage the OpenAI Whisper-large model using your API key.", + }, + { + name: "AnythingLLM Built-In", + value: "local", + logo: AnythingLLMIcon, + options: , + description: "Run a built-in whisper model on this instance privately.", + }, + ]; + + return ( +
+ + {loading ? ( +
+
+ +
+
+ ) : ( +
+
+
+
+
+

+ Transcription Model Preference +

+ {hasChanges && ( + + )} +
+

+ These are the credentials and settings for your preferred + transcription model provider. Its important these keys are + current and correct or else media files and audio will not + transcribe. +

+
+
+ Transcription Providers +
+
+
+
+
+ + setSearchQuery(e.target.value)} + autoComplete="off" + onKeyDown={(e) => { + if (e.key === "Enter") e.preventDefault(); + }} + /> +
+
+
+ {filteredProviders.map((provider) => { + return ( + updateProviderChoice(provider.value)} + /> + ); + })} +
+
+
setHasChanges(true)} + className="mt-4 flex flex-col gap-y-1" + > + {selectedProvider && + PROVIDERS.find( + (provider) => provider.value === selectedProvider + )?.options} +
+
+
+
+
+ )} +
+ ); +} diff --git a/frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx b/frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx index af3b3a9d0..bd8487842 100644 --- a/frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx +++ b/frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx @@ -29,7 +29,7 @@ import { useNavigate } from "react-router-dom"; const TITLE = "Data Handling & Privacy"; const DESCRIPTION = "We are committed to transparency and control when it comes to your personal data."; -const LLM_SELECTION_PRIVACY = { +export const LLM_SELECTION_PRIVACY = { openai: { name: "OpenAI", description: [ @@ -138,7 +138,7 @@ const LLM_SELECTION_PRIVACY = { }, }; -const VECTOR_DB_PRIVACY = { +export const VECTOR_DB_PRIVACY = { chroma: { name: "Chroma", description: [ @@ -199,7 +199,7 @@ const VECTOR_DB_PRIVACY = { }, }; -const EMBEDDING_ENGINE_PRIVACY = { +export const EMBEDDING_ENGINE_PRIVACY = { native: { name: "AnythingLLM Embedder", description: [ diff --git a/frontend/src/utils/chat/index.js b/frontend/src/utils/chat/index.js index f1df11fea..37237c9ec 100644 --- a/frontend/src/utils/chat/index.js +++ b/frontend/src/utils/chat/index.js @@ -1,3 +1,5 @@ +export const ABORT_STREAM_EVENT = "abort-chat-stream"; + // For handling of chat responses in the frontend by their various types. export default function handleChat( chatResult, @@ -108,6 +110,22 @@ export default function handleChat( _chatHistory[chatIdx] = updatedHistory; } setChatHistory([..._chatHistory]); + setLoadingResponse(false); + } else if (type === "stopGeneration") { + const chatIdx = _chatHistory.length - 1; + const existingHistory = { ..._chatHistory[chatIdx] }; + const updatedHistory = { + ...existingHistory, + sources: [], + closed: true, + error: null, + animate: false, + pending: false, + }; + _chatHistory[chatIdx] = updatedHistory; + + setChatHistory([..._chatHistory]); + setLoadingResponse(false); } } diff --git a/frontend/src/utils/paths.js b/frontend/src/utils/paths.js index da10aa23c..0f42e2237 100644 --- a/frontend/src/utils/paths.js +++ b/frontend/src/utils/paths.js @@ -92,6 +92,9 @@ export default { llmPreference: () => { return "/settings/llm-preference"; }, + transcriptionPreference: () => { + return "/settings/transcription-preference"; + }, embeddingPreference: () => { return "/settings/embedding-preference"; }, @@ -110,6 +113,9 @@ export default { logs: () => { return "/settings/event-logs"; }, + privacy: () => { + return "/settings/privacy"; + }, embedSetup: () => { return `/settings/embed-config`; }, diff --git a/server/.env.example b/server/.env.example index 88e60182c..c5681db4a 100644 --- a/server/.env.example +++ b/server/.env.example @@ -128,6 +128,16 @@ VECTOR_DB="lancedb" # ZILLIZ_ENDPOINT="https://sample.api.gcp-us-west1.zillizcloud.com" # ZILLIZ_API_TOKEN=api-token-here +########################################### +######## Audio Model Selection ############ +########################################### +# (default) use built-in whisper-small model. +WHISPER_PROVIDER="local" + +# use openai hosted whisper model. +# WHISPER_PROVIDER="openai" +# OPEN_AI_KEY=sk-xxxxxxxx + # CLOUD DEPLOYMENT VARIRABLES ONLY # AUTH_TOKEN="hunter2" # This is the password to your application if remote hosting. # STORAGE_DIR= # absolute filesystem path with no trailing slash diff --git a/server/models/systemSettings.js b/server/models/systemSettings.js index 137ce6c5e..55d5f641e 100644 --- a/server/models/systemSettings.js +++ b/server/models/systemSettings.js @@ -47,6 +47,7 @@ const SystemSettings = { EmbeddingModelMaxChunkLength: process.env.EMBEDDING_MODEL_MAX_CHUNK_LENGTH, LocalAiApiKey: !!process.env.LOCAL_AI_API_KEY, + DisableTelemetry: process.env.DISABLE_TELEMETRY || "false", ...(vectorDB === "pinecone" ? { PineConeKey: !!process.env.PINECONE_API_KEY, @@ -262,6 +263,7 @@ const SystemSettings = { AzureOpenAiEmbeddingModelPref: process.env.EMBEDDING_MODEL_PREF, } : {}), + WhisperProvider: process.env.WHISPER_PROVIDER || "local", }; }, diff --git a/server/storage/models/README.md b/server/storage/models/README.md index 965083dce..432f60572 100644 --- a/server/storage/models/README.md +++ b/server/storage/models/README.md @@ -14,6 +14,9 @@ AnythingLLM allows you to upload various audio and video formats as source docum Once transcribed you can embed these transcriptions into your workspace like you would any other file! +**Other external model/transcription providers are also live.** +- [OpenAI Whisper via API key.](https://openai.com/research/whisper) + ## Text generation (LLM selection) > [!IMPORTANT] > Use of a locally running LLM model is **experimental** and may behave unexpectedly, crash, or not function at all. diff --git a/server/utils/AiProviders/anthropic/index.js b/server/utils/AiProviders/anthropic/index.js index a48058e81..24a07f6e5 100644 --- a/server/utils/AiProviders/anthropic/index.js +++ b/server/utils/AiProviders/anthropic/index.js @@ -1,6 +1,9 @@ const { v4 } = require("uuid"); const { chatPrompt } = require("../../chats"); -const { writeResponseChunk } = require("../../helpers/chat/responses"); +const { + writeResponseChunk, + clientAbortedHandler, +} = require("../../helpers/chat/responses"); class AnthropicLLM { constructor(embedder = null, modelPreference = null) { if (!process.env.ANTHROPIC_API_KEY) @@ -45,6 +48,8 @@ class AnthropicLLM { return 200_000; case "claude-3-sonnet-20240229": return 200_000; + case "claude-3-haiku-20240307": + return 200_000; default: return 100_000; // assume a claude-instant-1.2 model } @@ -57,6 +62,7 @@ class AnthropicLLM { "claude-2.1", "claude-3-opus-20240229", "claude-3-sonnet-20240229", + "claude-3-haiku-20240307", ]; return validModels.includes(modelName); } @@ -150,6 +156,13 @@ class AnthropicLLM { let fullText = ""; const { uuid = v4(), sources = [] } = responseProps; + // Establish listener to early-abort a streaming response + // in case things go sideways or the user does not like the response. + // We preserve the generated text but continue as if chat was completed + // to preserve previously generated content. + const handleAbort = () => clientAbortedHandler(resolve, fullText); + response.on("close", handleAbort); + stream.on("streamEvent", (message) => { const data = message; if ( @@ -181,6 +194,7 @@ class AnthropicLLM { close: true, error: false, }); + response.removeListener("close", handleAbort); resolve(fullText); } }); diff --git a/server/utils/AiProviders/azureOpenAi/index.js b/server/utils/AiProviders/azureOpenAi/index.js index 2ac6de3a1..21fc5cd91 100644 --- a/server/utils/AiProviders/azureOpenAi/index.js +++ b/server/utils/AiProviders/azureOpenAi/index.js @@ -1,6 +1,9 @@ const { AzureOpenAiEmbedder } = require("../../EmbeddingEngines/azureOpenAi"); const { chatPrompt } = require("../../chats"); -const { writeResponseChunk } = require("../../helpers/chat/responses"); +const { + writeResponseChunk, + clientAbortedHandler, +} = require("../../helpers/chat/responses"); class AzureOpenAiLLM { constructor(embedder = null, _modelPreference = null) { @@ -174,6 +177,14 @@ class AzureOpenAiLLM { return new Promise(async (resolve) => { let fullText = ""; + + // Establish listener to early-abort a streaming response + // in case things go sideways or the user does not like the response. + // We preserve the generated text but continue as if chat was completed + // to preserve previously generated content. + const handleAbort = () => clientAbortedHandler(resolve, fullText); + response.on("close", handleAbort); + for await (const event of stream) { for (const choice of event.choices) { const delta = choice.delta?.content; @@ -198,6 +209,7 @@ class AzureOpenAiLLM { close: true, error: false, }); + response.removeListener("close", handleAbort); resolve(fullText); }); } diff --git a/server/utils/AiProviders/gemini/index.js b/server/utils/AiProviders/gemini/index.js index bd84a3856..3d334b291 100644 --- a/server/utils/AiProviders/gemini/index.js +++ b/server/utils/AiProviders/gemini/index.js @@ -1,5 +1,8 @@ const { chatPrompt } = require("../../chats"); -const { writeResponseChunk } = require("../../helpers/chat/responses"); +const { + writeResponseChunk, + clientAbortedHandler, +} = require("../../helpers/chat/responses"); class GeminiLLM { constructor(embedder = null, modelPreference = null) { @@ -198,6 +201,14 @@ class GeminiLLM { return new Promise(async (resolve) => { let fullText = ""; + + // Establish listener to early-abort a streaming response + // in case things go sideways or the user does not like the response. + // We preserve the generated text but continue as if chat was completed + // to preserve previously generated content. + const handleAbort = () => clientAbortedHandler(resolve, fullText); + response.on("close", handleAbort); + for await (const chunk of stream) { fullText += chunk.text(); writeResponseChunk(response, { @@ -218,6 +229,7 @@ class GeminiLLM { close: true, error: false, }); + response.removeListener("close", handleAbort); resolve(fullText); }); } diff --git a/server/utils/AiProviders/huggingface/index.js b/server/utils/AiProviders/huggingface/index.js index 416e622a3..751d3595c 100644 --- a/server/utils/AiProviders/huggingface/index.js +++ b/server/utils/AiProviders/huggingface/index.js @@ -1,7 +1,10 @@ const { NativeEmbedder } = require("../../EmbeddingEngines/native"); const { OpenAiEmbedder } = require("../../EmbeddingEngines/openAi"); const { chatPrompt } = require("../../chats"); -const { writeResponseChunk } = require("../../helpers/chat/responses"); +const { + writeResponseChunk, + clientAbortedHandler, +} = require("../../helpers/chat/responses"); class HuggingFaceLLM { constructor(embedder = null, _modelPreference = null) { @@ -172,6 +175,14 @@ class HuggingFaceLLM { return new Promise((resolve) => { let fullText = ""; let chunk = ""; + + // Establish listener to early-abort a streaming response + // in case things go sideways or the user does not like the response. + // We preserve the generated text but continue as if chat was completed + // to preserve previously generated content. + const handleAbort = () => clientAbortedHandler(resolve, fullText); + response.on("close", handleAbort); + stream.data.on("data", (data) => { const lines = data ?.toString() @@ -218,6 +229,7 @@ class HuggingFaceLLM { close: true, error: false, }); + response.removeListener("close", handleAbort); resolve(fullText); } else { let error = null; @@ -241,6 +253,7 @@ class HuggingFaceLLM { close: true, error, }); + response.removeListener("close", handleAbort); resolve(""); return; } @@ -266,6 +279,7 @@ class HuggingFaceLLM { close: true, error: false, }); + response.removeListener("close", handleAbort); resolve(fullText); } } diff --git a/server/utils/AiProviders/native/index.js b/server/utils/AiProviders/native/index.js index 157fb7520..5764d4ee2 100644 --- a/server/utils/AiProviders/native/index.js +++ b/server/utils/AiProviders/native/index.js @@ -2,7 +2,10 @@ const fs = require("fs"); const path = require("path"); const { NativeEmbedder } = require("../../EmbeddingEngines/native"); const { chatPrompt } = require("../../chats"); -const { writeResponseChunk } = require("../../helpers/chat/responses"); +const { + writeResponseChunk, + clientAbortedHandler, +} = require("../../helpers/chat/responses"); // Docs: https://api.js.langchain.com/classes/chat_models_llama_cpp.ChatLlamaCpp.html const ChatLlamaCpp = (...args) => @@ -176,6 +179,14 @@ class NativeLLM { return new Promise(async (resolve) => { let fullText = ""; + + // Establish listener to early-abort a streaming response + // in case things go sideways or the user does not like the response. + // We preserve the generated text but continue as if chat was completed + // to preserve previously generated content. + const handleAbort = () => clientAbortedHandler(resolve, fullText); + response.on("close", handleAbort); + for await (const chunk of stream) { if (chunk === undefined) throw new Error( @@ -202,6 +213,7 @@ class NativeLLM { close: true, error: false, }); + response.removeListener("close", handleAbort); resolve(fullText); }); } diff --git a/server/utils/AiProviders/ollama/index.js b/server/utils/AiProviders/ollama/index.js index 035d4a9d0..6bd857b4e 100644 --- a/server/utils/AiProviders/ollama/index.js +++ b/server/utils/AiProviders/ollama/index.js @@ -1,6 +1,9 @@ const { chatPrompt } = require("../../chats"); const { StringOutputParser } = require("langchain/schema/output_parser"); -const { writeResponseChunk } = require("../../helpers/chat/responses"); +const { + writeResponseChunk, + clientAbortedHandler, +} = require("../../helpers/chat/responses"); // Docs: https://github.com/jmorganca/ollama/blob/main/docs/api.md class OllamaAILLM { @@ -180,8 +183,16 @@ class OllamaAILLM { const { uuid = uuidv4(), sources = [] } = responseProps; return new Promise(async (resolve) => { + let fullText = ""; + + // Establish listener to early-abort a streaming response + // in case things go sideways or the user does not like the response. + // We preserve the generated text but continue as if chat was completed + // to preserve previously generated content. + const handleAbort = () => clientAbortedHandler(resolve, fullText); + response.on("close", handleAbort); + try { - let fullText = ""; for await (const chunk of stream) { if (chunk === undefined) throw new Error( @@ -210,6 +221,7 @@ class OllamaAILLM { close: true, error: false, }); + response.removeListener("close", handleAbort); resolve(fullText); } catch (error) { writeResponseChunk(response, { @@ -222,6 +234,7 @@ class OllamaAILLM { error?.cause ?? error.message }`, }); + response.removeListener("close", handleAbort); } }); } diff --git a/server/utils/AiProviders/openRouter/index.js b/server/utils/AiProviders/openRouter/index.js index 38a6f9f09..a1f606f60 100644 --- a/server/utils/AiProviders/openRouter/index.js +++ b/server/utils/AiProviders/openRouter/index.js @@ -1,7 +1,10 @@ const { NativeEmbedder } = require("../../EmbeddingEngines/native"); const { chatPrompt } = require("../../chats"); const { v4: uuidv4 } = require("uuid"); -const { writeResponseChunk } = require("../../helpers/chat/responses"); +const { + writeResponseChunk, + clientAbortedHandler, +} = require("../../helpers/chat/responses"); function openRouterModels() { const { MODELS } = require("./models.js"); @@ -195,6 +198,13 @@ class OpenRouterLLM { let chunk = ""; let lastChunkTime = null; // null when first token is still not received. + // Establish listener to early-abort a streaming response + // in case things go sideways or the user does not like the response. + // We preserve the generated text but continue as if chat was completed + // to preserve previously generated content. + const handleAbort = () => clientAbortedHandler(resolve, fullText); + response.on("close", handleAbort); + // NOTICE: Not all OpenRouter models will return a stop reason // which keeps the connection open and so the model never finalizes the stream // like the traditional OpenAI response schema does. So in the case the response stream @@ -220,6 +230,7 @@ class OpenRouterLLM { error: false, }); clearInterval(timeoutCheck); + response.removeListener("close", handleAbort); resolve(fullText); } }, 500); @@ -269,6 +280,7 @@ class OpenRouterLLM { error: false, }); clearInterval(timeoutCheck); + response.removeListener("close", handleAbort); resolve(fullText); } else { let finishReason = null; @@ -305,6 +317,7 @@ class OpenRouterLLM { error: false, }); clearInterval(timeoutCheck); + response.removeListener("close", handleAbort); resolve(fullText); } } diff --git a/server/utils/AiProviders/togetherAi/index.js b/server/utils/AiProviders/togetherAi/index.js index 15b254a15..def03df96 100644 --- a/server/utils/AiProviders/togetherAi/index.js +++ b/server/utils/AiProviders/togetherAi/index.js @@ -1,5 +1,8 @@ const { chatPrompt } = require("../../chats"); -const { writeResponseChunk } = require("../../helpers/chat/responses"); +const { + writeResponseChunk, + clientAbortedHandler, +} = require("../../helpers/chat/responses"); function togetherAiModels() { const { MODELS } = require("./models.js"); @@ -185,6 +188,14 @@ class TogetherAiLLM { return new Promise((resolve) => { let fullText = ""; let chunk = ""; + + // Establish listener to early-abort a streaming response + // in case things go sideways or the user does not like the response. + // We preserve the generated text but continue as if chat was completed + // to preserve previously generated content. + const handleAbort = () => clientAbortedHandler(resolve, fullText); + response.on("close", handleAbort); + stream.data.on("data", (data) => { const lines = data ?.toString() @@ -230,6 +241,7 @@ class TogetherAiLLM { close: true, error: false, }); + response.removeListener("close", handleAbort); resolve(fullText); } else { let finishReason = null; @@ -263,6 +275,7 @@ class TogetherAiLLM { close: true, error: false, }); + response.removeListener("close", handleAbort); resolve(fullText); } } diff --git a/server/utils/collectorApi/index.js b/server/utils/collectorApi/index.js index 7e8c11493..ed27a9289 100644 --- a/server/utils/collectorApi/index.js +++ b/server/utils/collectorApi/index.js @@ -5,13 +5,20 @@ class CollectorApi { constructor() { - this.endpoint = "http://0.0.0.0:8888"; + this.endpoint = `http://0.0.0.0:${process.env.COLLECTOR_PORT || 8888}`; } log(text, ...args) { console.log(`\x1b[36m[CollectorApi]\x1b[0m ${text}`, ...args); } + #attachOptions() { + return { + whisperProvider: process.env.WHISPER_PROVIDER || "local", + openAiKey: process.env.OPEN_AI_KEY || null, + }; + } + async online() { return await fetch(this.endpoint) .then((res) => res.ok) @@ -38,7 +45,10 @@ class CollectorApi { headers: { "Content-Type": "application/json", }, - body: JSON.stringify({ filename }), + body: JSON.stringify({ + filename, + options: this.#attachOptions(), + }), }) .then((res) => { if (!res.ok) throw new Error("Response could not be completed"); diff --git a/server/utils/helpers/chat/responses.js b/server/utils/helpers/chat/responses.js index c4371d818..e2ec7bd0d 100644 --- a/server/utils/helpers/chat/responses.js +++ b/server/utils/helpers/chat/responses.js @@ -1,6 +1,14 @@ const { v4: uuidv4 } = require("uuid"); const moment = require("moment"); +function clientAbortedHandler(resolve, fullText) { + console.log( + "\x1b[43m\x1b[34m[STREAM ABORTED]\x1b[0m Client requested to abort stream. Exiting LLM stream handler early." + ); + resolve(fullText); + return; +} + // The default way to handle a stream response. Functions best with OpenAI. // Currently used for LMStudio, LocalAI, Mistral API, and OpenAI function handleDefaultStreamResponse(response, stream, responseProps) { @@ -9,6 +17,14 @@ function handleDefaultStreamResponse(response, stream, responseProps) { return new Promise((resolve) => { let fullText = ""; let chunk = ""; + + // Establish listener to early-abort a streaming response + // in case things go sideways or the user does not like the response. + // We preserve the generated text but continue as if chat was completed + // to preserve previously generated content. + const handleAbort = () => clientAbortedHandler(resolve, fullText); + response.on("close", handleAbort); + stream.data.on("data", (data) => { const lines = data ?.toString() @@ -52,6 +68,7 @@ function handleDefaultStreamResponse(response, stream, responseProps) { close: true, error: false, }); + response.removeListener("close", handleAbort); resolve(fullText); } else { let finishReason = null; @@ -85,6 +102,7 @@ function handleDefaultStreamResponse(response, stream, responseProps) { close: true, error: false, }); + response.removeListener("close", handleAbort); resolve(fullText); } } @@ -141,4 +159,5 @@ module.exports = { convertToChatHistory, convertToPromptHistory, writeResponseChunk, + clientAbortedHandler, }; diff --git a/server/utils/helpers/updateENV.js b/server/utils/helpers/updateENV.js index 15d2efc80..01d437c2b 100644 --- a/server/utils/helpers/updateENV.js +++ b/server/utils/helpers/updateENV.js @@ -269,6 +269,13 @@ const KEY_MAPPING = { checks: [isNotEmpty], }, + // Whisper (transcription) providers + WhisperProvider: { + envKey: "WHISPER_PROVIDER", + checks: [isNotEmpty, supportedTranscriptionProvider], + postUpdate: [], + }, + // System Settings AuthToken: { envKey: "AUTH_TOKEN", @@ -278,6 +285,10 @@ const KEY_MAPPING = { envKey: "JWT_SECRET", checks: [requiresForceMode], }, + DisableTelemetry: { + envKey: "DISABLE_TELEMETRY", + checks: [], + }, }; function isNotEmpty(input = "") { @@ -351,6 +362,13 @@ function supportedLLM(input = "") { return validSelection ? null : `${input} is not a valid LLM provider.`; } +function supportedTranscriptionProvider(input = "") { + const validSelection = ["openai", "local"].includes(input); + return validSelection + ? null + : `${input} is not a valid transcription model provider.`; +} + function validGeminiModel(input = "") { const validModels = ["gemini-pro"]; return validModels.includes(input) @@ -365,6 +383,7 @@ function validAnthropicModel(input = "") { "claude-2.1", "claude-3-opus-20240229", "claude-3-sonnet-20240229", + "claude-3-haiku-20240307", ]; return validModels.includes(input) ? null