mirror of
https://github.com/Mintplex-Labs/anything-llm.git
synced 2024-10-05 18:30:09 +02:00
Implement streaming for workspace chats via API (#604)
This commit is contained in:
parent
bd158ce7b1
commit
f5bb064dee
@ -11,6 +11,11 @@ const {
|
|||||||
const { getVectorDbClass } = require("../../../utils/helpers");
|
const { getVectorDbClass } = require("../../../utils/helpers");
|
||||||
const { multiUserMode, reqBody } = require("../../../utils/http");
|
const { multiUserMode, reqBody } = require("../../../utils/http");
|
||||||
const { validApiKey } = require("../../../utils/middleware/validApiKey");
|
const { validApiKey } = require("../../../utils/middleware/validApiKey");
|
||||||
|
const {
|
||||||
|
streamChatWithWorkspace,
|
||||||
|
writeResponseChunk,
|
||||||
|
VALID_CHAT_MODE,
|
||||||
|
} = require("../../../utils/chats/stream");
|
||||||
|
|
||||||
function apiWorkspaceEndpoints(app) {
|
function apiWorkspaceEndpoints(app) {
|
||||||
if (!app) return;
|
if (!app) return;
|
||||||
@ -483,7 +488,28 @@ function apiWorkspaceEndpoints(app) {
|
|||||||
const workspace = await Workspace.get({ slug });
|
const workspace = await Workspace.get({ slug });
|
||||||
|
|
||||||
if (!workspace) {
|
if (!workspace) {
|
||||||
response.sendStatus(400).end();
|
response.status(400).json({
|
||||||
|
id: uuidv4(),
|
||||||
|
type: "abort",
|
||||||
|
textResponse: null,
|
||||||
|
sources: [],
|
||||||
|
close: true,
|
||||||
|
error: `Workspace ${slug} is not a valid workspace.`,
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!message?.length || !VALID_CHAT_MODE.includes(mode)) {
|
||||||
|
response.status(400).json({
|
||||||
|
id: uuidv4(),
|
||||||
|
type: "abort",
|
||||||
|
textResponse: null,
|
||||||
|
sources: [],
|
||||||
|
close: true,
|
||||||
|
error: !message?.length
|
||||||
|
? "message parameter cannot be empty."
|
||||||
|
: `${mode} is not a valid mode.`,
|
||||||
|
});
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -506,6 +532,126 @@ function apiWorkspaceEndpoints(app) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
|
||||||
|
app.post(
|
||||||
|
"/v1/workspace/:slug/stream-chat",
|
||||||
|
[validApiKey],
|
||||||
|
async (request, response) => {
|
||||||
|
/*
|
||||||
|
#swagger.tags = ['Workspaces']
|
||||||
|
#swagger.description = 'Execute a streamable chat with a workspace'
|
||||||
|
#swagger.requestBody = {
|
||||||
|
description: 'Send a prompt to the workspace and the type of conversation (query or chat).<br/><b>Query:</b> Will not use LLM unless there are relevant sources from vectorDB & does not recall chat history.<br/><b>Chat:</b> Uses LLM general knowledge w/custom embeddings to produce output, uses rolling chat history.',
|
||||||
|
required: true,
|
||||||
|
type: 'object',
|
||||||
|
content: {
|
||||||
|
"application/json": {
|
||||||
|
example: {
|
||||||
|
message: "What is AnythingLLM?",
|
||||||
|
mode: "query | chat"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#swagger.responses[200] = {
|
||||||
|
content: {
|
||||||
|
"text/event-stream": {
|
||||||
|
schema: {
|
||||||
|
type: 'array',
|
||||||
|
example: [
|
||||||
|
{
|
||||||
|
id: 'uuid-123',
|
||||||
|
type: "abort | textResponseChunk",
|
||||||
|
textResponse: "First chunk",
|
||||||
|
sources: [],
|
||||||
|
close: false,
|
||||||
|
error: "null | text string of the failure mode."
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 'uuid-123',
|
||||||
|
type: "abort | textResponseChunk",
|
||||||
|
textResponse: "chunk two",
|
||||||
|
sources: [],
|
||||||
|
close: false,
|
||||||
|
error: "null | text string of the failure mode."
|
||||||
|
},
|
||||||
|
{
|
||||||
|
id: 'uuid-123',
|
||||||
|
type: "abort | textResponseChunk",
|
||||||
|
textResponse: "final chunk of LLM output!",
|
||||||
|
sources: [{title: "anythingllm.txt", chunk: "This is a context chunk used in the answer of the prompt by the LLM. This will only return in the final chunk."}],
|
||||||
|
close: true,
|
||||||
|
error: "null | text string of the failure mode."
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#swagger.responses[403] = {
|
||||||
|
schema: {
|
||||||
|
"$ref": "#/definitions/InvalidAPIKey"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
*/
|
||||||
|
try {
|
||||||
|
const { slug } = request.params;
|
||||||
|
const { message, mode = "query" } = reqBody(request);
|
||||||
|
const workspace = await Workspace.get({ slug });
|
||||||
|
|
||||||
|
if (!workspace) {
|
||||||
|
response.status(400).json({
|
||||||
|
id: uuidv4(),
|
||||||
|
type: "abort",
|
||||||
|
textResponse: null,
|
||||||
|
sources: [],
|
||||||
|
close: true,
|
||||||
|
error: `Workspace ${slug} is not a valid workspace.`,
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!message?.length || !VALID_CHAT_MODE.includes(mode)) {
|
||||||
|
response.status(400).json({
|
||||||
|
id: uuidv4(),
|
||||||
|
type: "abort",
|
||||||
|
textResponse: null,
|
||||||
|
sources: [],
|
||||||
|
close: true,
|
||||||
|
error: !message?.length
|
||||||
|
? "Message is empty"
|
||||||
|
: `${mode} is not a valid mode.`,
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
response.setHeader("Cache-Control", "no-cache");
|
||||||
|
response.setHeader("Content-Type", "text/event-stream");
|
||||||
|
response.setHeader("Access-Control-Allow-Origin", "*");
|
||||||
|
response.setHeader("Connection", "keep-alive");
|
||||||
|
response.flushHeaders();
|
||||||
|
|
||||||
|
await streamChatWithWorkspace(response, workspace, message, mode);
|
||||||
|
await Telemetry.sendTelemetry("sent_chat", {
|
||||||
|
LLMSelection: process.env.LLM_PROVIDER || "openai",
|
||||||
|
Embedder: process.env.EMBEDDING_ENGINE || "inherit",
|
||||||
|
VectorDbSelection: process.env.VECTOR_DB || "pinecone",
|
||||||
|
});
|
||||||
|
response.end();
|
||||||
|
} catch (e) {
|
||||||
|
console.error(e);
|
||||||
|
writeResponseChunk(response, {
|
||||||
|
id: uuidv4(),
|
||||||
|
type: "abort",
|
||||||
|
textResponse: null,
|
||||||
|
sources: [],
|
||||||
|
close: true,
|
||||||
|
error: e.message,
|
||||||
|
});
|
||||||
|
response.end();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
module.exports = { apiWorkspaceEndpoints };
|
module.exports = { apiWorkspaceEndpoints };
|
||||||
|
@ -8,6 +8,7 @@ const { Telemetry } = require("../models/telemetry");
|
|||||||
const {
|
const {
|
||||||
streamChatWithWorkspace,
|
streamChatWithWorkspace,
|
||||||
writeResponseChunk,
|
writeResponseChunk,
|
||||||
|
VALID_CHAT_MODE,
|
||||||
} = require("../utils/chats/stream");
|
} = require("../utils/chats/stream");
|
||||||
|
|
||||||
function chatEndpoints(app) {
|
function chatEndpoints(app) {
|
||||||
@ -31,6 +32,20 @@ function chatEndpoints(app) {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (!message?.length || !VALID_CHAT_MODE.includes(mode)) {
|
||||||
|
response.status(400).json({
|
||||||
|
id: uuidv4(),
|
||||||
|
type: "abort",
|
||||||
|
textResponse: null,
|
||||||
|
sources: [],
|
||||||
|
close: true,
|
||||||
|
error: !message?.length
|
||||||
|
? "Message is empty."
|
||||||
|
: `${mode} is not a valid mode.`,
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
response.setHeader("Cache-Control", "no-cache");
|
response.setHeader("Cache-Control", "no-cache");
|
||||||
response.setHeader("Content-Type", "text/event-stream");
|
response.setHeader("Content-Type", "text/event-stream");
|
||||||
response.setHeader("Access-Control-Allow-Origin", "*");
|
response.setHeader("Access-Control-Allow-Origin", "*");
|
||||||
|
@ -1612,6 +1612,105 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"/v1/workspace/{slug}/stream-chat": {
|
||||||
|
"post": {
|
||||||
|
"tags": [
|
||||||
|
"Workspaces"
|
||||||
|
],
|
||||||
|
"description": "Execute a streamable chat with a workspace",
|
||||||
|
"parameters": [
|
||||||
|
{
|
||||||
|
"name": "slug",
|
||||||
|
"in": "path",
|
||||||
|
"required": true,
|
||||||
|
"schema": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "Authorization",
|
||||||
|
"in": "header",
|
||||||
|
"schema": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"responses": {
|
||||||
|
"200": {
|
||||||
|
"content": {
|
||||||
|
"text/event-stream": {
|
||||||
|
"schema": {
|
||||||
|
"type": "array",
|
||||||
|
"example": [
|
||||||
|
{
|
||||||
|
"id": "uuid-123",
|
||||||
|
"type": "abort | textResponseChunk",
|
||||||
|
"textResponse": "First chunk",
|
||||||
|
"sources": [],
|
||||||
|
"close": false,
|
||||||
|
"error": "null | text string of the failure mode."
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "uuid-123",
|
||||||
|
"type": "abort | textResponseChunk",
|
||||||
|
"textResponse": "chunk two",
|
||||||
|
"sources": [],
|
||||||
|
"close": false,
|
||||||
|
"error": "null | text string of the failure mode."
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "uuid-123",
|
||||||
|
"type": "abort | textResponseChunk",
|
||||||
|
"textResponse": "final chunk of LLM output!",
|
||||||
|
"sources": [
|
||||||
|
{
|
||||||
|
"title": "anythingllm.txt",
|
||||||
|
"chunk": "This is a context chunk used in the answer of the prompt by the LLM. This will only return in the final chunk."
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"close": true,
|
||||||
|
"error": "null | text string of the failure mode."
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"description": "OK"
|
||||||
|
},
|
||||||
|
"400": {
|
||||||
|
"description": "Bad Request"
|
||||||
|
},
|
||||||
|
"403": {
|
||||||
|
"description": "Forbidden",
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/components/schemas/InvalidAPIKey"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"application/xml": {
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/components/schemas/InvalidAPIKey"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"requestBody": {
|
||||||
|
"description": "Send a prompt to the workspace and the type of conversation (query or chat).<br/><b>Query:</b> Will not use LLM unless there are relevant sources from vectorDB & does not recall chat history.<br/><b>Chat:</b> Uses LLM general knowledge w/custom embeddings to produce output, uses rolling chat history.",
|
||||||
|
"required": true,
|
||||||
|
"type": "object",
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"example": {
|
||||||
|
"message": "What is AnythingLLM?",
|
||||||
|
"mode": "query | chat"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
"/v1/system/env-dump": {
|
"/v1/system/env-dump": {
|
||||||
"get": {
|
"get": {
|
||||||
"tags": [
|
"tags": [
|
||||||
|
@ -8,6 +8,7 @@ const {
|
|||||||
chatPrompt,
|
chatPrompt,
|
||||||
} = require(".");
|
} = require(".");
|
||||||
|
|
||||||
|
const VALID_CHAT_MODE = ["chat", "query"];
|
||||||
function writeResponseChunk(response, data) {
|
function writeResponseChunk(response, data) {
|
||||||
response.write(`data: ${JSON.stringify(data)}\n\n`);
|
response.write(`data: ${JSON.stringify(data)}\n\n`);
|
||||||
return;
|
return;
|
||||||
@ -503,6 +504,7 @@ function handleStreamResponses(response, stream, responseProps) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
|
VALID_CHAT_MODE,
|
||||||
streamChatWithWorkspace,
|
streamChatWithWorkspace,
|
||||||
writeResponseChunk,
|
writeResponseChunk,
|
||||||
};
|
};
|
||||||
|
Loading…
Reference in New Issue
Block a user