mirror of
https://github.com/Mintplex-Labs/anything-llm.git
synced 2024-07-04 16:20:12 +02:00
[FEAT] Improved CSV chat exports (#700)
* add more fields to csv export to make more useful * refactor from review comments * fix escapeCsv function * catch export errors properly --------- Co-authored-by: timothycarambat <rambat1010@gmail.com>
This commit is contained in:
parent
0e6bd030e9
commit
1b29882c71
|
@ -465,7 +465,10 @@ const System = {
|
||||||
method: "GET",
|
method: "GET",
|
||||||
headers: baseHeaders(),
|
headers: baseHeaders(),
|
||||||
})
|
})
|
||||||
.then((res) => res.text())
|
.then((res) => {
|
||||||
|
if (res.ok) return res.text();
|
||||||
|
throw new Error(res.statusText);
|
||||||
|
})
|
||||||
.catch((e) => {
|
.catch((e) => {
|
||||||
console.error(e);
|
console.error(e);
|
||||||
return null;
|
return null;
|
||||||
|
|
|
@ -21,7 +21,7 @@ export default function WorkspaceChats() {
|
||||||
};
|
};
|
||||||
const handleDumpChats = async () => {
|
const handleDumpChats = async () => {
|
||||||
const chats = await System.exportChats(exportType);
|
const chats = await System.exportChats(exportType);
|
||||||
if (chats) {
|
if (!!chats) {
|
||||||
const { mimeType, fileExtension } = exportOptions[exportType];
|
const { mimeType, fileExtension } = exportOptions[exportType];
|
||||||
const blob = new Blob([chats], { type: mimeType });
|
const blob = new Blob([chats], { type: mimeType });
|
||||||
const link = document.createElement("a");
|
const link = document.createElement("a");
|
||||||
|
|
|
@ -898,7 +898,7 @@ function systemEndpoints(app) {
|
||||||
async (request, response) => {
|
async (request, response) => {
|
||||||
try {
|
try {
|
||||||
const { type = "jsonl" } = request.query;
|
const { type = "jsonl" } = request.query;
|
||||||
const chats = await prepareWorkspaceChatsForExport();
|
const chats = await prepareWorkspaceChatsForExport(type);
|
||||||
const { contentType, data } = await exportChatsAsType(chats, type);
|
const { contentType, data } = await exportChatsAsType(chats, type);
|
||||||
await EventLogs.logEvent(
|
await EventLogs.logEvent(
|
||||||
"exported_chats",
|
"exported_chats",
|
||||||
|
|
|
@ -4,17 +4,19 @@
|
||||||
const { Workspace } = require("../../../models/workspace");
|
const { Workspace } = require("../../../models/workspace");
|
||||||
const { WorkspaceChats } = require("../../../models/workspaceChats");
|
const { WorkspaceChats } = require("../../../models/workspaceChats");
|
||||||
|
|
||||||
// Todo: make this more useful for export by adding other columns about workspace, user, time, etc for post-filtering.
|
// Todo: add RLHF feedbackScore field support
|
||||||
async function convertToCSV(workspaceChatsMap) {
|
async function convertToCSV(preparedData) {
|
||||||
const rows = ["role,content"];
|
const rows = ["id,username,workspace,prompt,response,sent_at"];
|
||||||
for (const workspaceChats of Object.values(workspaceChatsMap)) {
|
for (const item of preparedData) {
|
||||||
for (const message of workspaceChats.messages) {
|
const record = [
|
||||||
// Escape double quotes and wrap content in double quotes
|
item.id,
|
||||||
const escapedContent = `"${message.content
|
escapeCsv(item.username),
|
||||||
.replace(/"/g, '""')
|
escapeCsv(item.workspace),
|
||||||
.replace(/\n/g, " ")}"`;
|
escapeCsv(item.prompt),
|
||||||
rows.push(`${message.role},${escapedContent}`);
|
escapeCsv(item.response),
|
||||||
}
|
item.sent_at,
|
||||||
|
].join(",");
|
||||||
|
rows.push(record);
|
||||||
}
|
}
|
||||||
return rows.join("\n");
|
return rows.join("\n");
|
||||||
}
|
}
|
||||||
|
@ -33,10 +35,30 @@ async function convertToJSONL(workspaceChatsMap) {
|
||||||
.join("\n");
|
.join("\n");
|
||||||
}
|
}
|
||||||
|
|
||||||
async function prepareWorkspaceChatsForExport() {
|
async function prepareWorkspaceChatsForExport(format = "jsonl") {
|
||||||
|
if (!exportMap.hasOwnProperty(format))
|
||||||
|
throw new Error("Invalid export type.");
|
||||||
|
|
||||||
const chats = await WorkspaceChats.whereWithData({}, null, null, {
|
const chats = await WorkspaceChats.whereWithData({}, null, null, {
|
||||||
id: "asc",
|
id: "asc",
|
||||||
});
|
});
|
||||||
|
|
||||||
|
if (format === "csv") {
|
||||||
|
const preparedData = chats.map((chat) => {
|
||||||
|
const responseJson = JSON.parse(chat.response);
|
||||||
|
return {
|
||||||
|
id: chat.id,
|
||||||
|
username: chat.user ? chat.user.username : "unknown user",
|
||||||
|
workspace: chat.workspace ? chat.workspace.name : "unknown workspace",
|
||||||
|
prompt: chat.prompt,
|
||||||
|
response: responseJson.text,
|
||||||
|
sent_at: chat.createdAt,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
return preparedData;
|
||||||
|
}
|
||||||
|
|
||||||
const workspaceIds = [...new Set(chats.map((chat) => chat.workspaceId))];
|
const workspaceIds = [...new Set(chats.map((chat) => chat.workspaceId))];
|
||||||
|
|
||||||
const workspacesWithPrompts = await Promise.all(
|
const workspacesWithPrompts = await Promise.all(
|
||||||
|
@ -97,6 +119,10 @@ const exportMap = {
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
|
function escapeCsv(str) {
|
||||||
|
return `"${str.replace(/"/g, '""').replace(/\n/g, " ")}"`;
|
||||||
|
}
|
||||||
|
|
||||||
async function exportChatsAsType(workspaceChatsMap, format = "jsonl") {
|
async function exportChatsAsType(workspaceChatsMap, format = "jsonl") {
|
||||||
const { contentType, func } = exportMap.hasOwnProperty(format)
|
const { contentType, func } = exportMap.hasOwnProperty(format)
|
||||||
? exportMap[format]
|
? exportMap[format]
|
||||||
|
|
Loading…
Reference in New Issue
Block a user