add support for exporting to json and csv in workspace chats (#622)

* add support for exporting to json and csv in workspace chats * safety encode URL options * remove message about openai fine tuning on export success * all defaults to jsonl
2024-07-02 15:20:37 +02:00 · 2024-01-18 17:59:51 -08:00 · 2024-01-18 17:59:51 -08:00 · 7fb76cfef0
commit 7fb76cfef0
parent 08d33cfd8f
5 changed files with 172 additions and 23 deletions
--- a/frontend/src/index.css
+++ b/frontend/src/index.css
@ -407,3 +407,35 @@ dialog::backdrop {
 .Toastify__toast-body {
  white-space: pre-line;
 }
+
+@keyframes slideDown {
+  from {
+    max-height: 0;
+    opacity: 0;
+  }
+
+  to {
+    max-height: 400px;
+    opacity: 1;
+  }
+}
+
+.slide-down {
+  animation: slideDown 0.3s ease-out forwards;
+}
+
+@keyframes slideUp {
+  from {
+    max-height: 400px;
+    opacity: 1;
+  }
+
+  to {
+    max-height: 0;
+    opacity: 0;
+  }
+}
+
+.slide-up {
+  animation: slideUp 0.3s ease-out forwards;
+}
--- a/frontend/src/models/system.js
+++ b/frontend/src/models/system.js
@ -398,8 +398,10 @@ const System = {
        return { success: false, error: e.message };
      });
  },
-  exportChats: async () => {
-    return await fetch(`${API_BASE}/system/export-chats`, {
+  exportChats: async (type = "csv") => {
+    const url = new URL(`${fullApiUrl()}/system/export-chats`);
+    url.searchParams.append("type", encodeURIComponent(type));
+    return await fetch(url, {
      method: "GET",
      headers: baseHeaders(),
    })
--- a/frontend/src/pages/GeneralSettings/Chats/index.jsx
+++ b/frontend/src/pages/GeneralSettings/Chats/index.jsx
@ -1,4 +1,4 @@
-import { useEffect, useState } from "react";
+import { useEffect, useRef, useState } from "react";
 import Sidebar, { SidebarMobileHeader } from "@/components/SettingsSidebar";
 import { isMobile } from "react-device-detect";
 import * as Skeleton from "react-loading-skeleton";
@ -7,22 +7,32 @@ import useQuery from "@/hooks/useQuery";
 import ChatRow from "./ChatRow";
 import showToast from "@/utils/toast";
 import System from "@/models/system";
-
-const PAGE_SIZE = 20;
+import { CaretDown } from "@phosphor-icons/react";
 export default function WorkspaceChats() {
+  const [showMenu, setShowMenu] = useState(false);
+  const [exportType, setExportType] = useState("jsonl");
+  const menuRef = useRef();
+  const openMenuButton = useRef();
+
+  const exportOptions = {
+    csv: { mimeType: "text/csv", fileExtension: "csv" },
+    json: { mimeType: "application/json", fileExtension: "json" },
+    jsonl: { mimeType: "application/jsonl", fileExtension: "jsonl" },
+  };
  const handleDumpChats = async () => {
-    const chats = await System.exportChats();
+    const chats = await System.exportChats(exportType);
    if (chats) {
-      const blob = new Blob([chats], { type: "application/jsonl" });
+      const { mimeType, fileExtension } = exportOptions[exportType];
+      const blob = new Blob([chats], { type: mimeType });
      const link = document.createElement("a");
      link.href = window.URL.createObjectURL(blob);
-      link.download = "chats.jsonl";
+      link.download = `chats.${fileExtension}`;
      document.body.appendChild(link);
      link.click();
      window.URL.revokeObjectURL(link.href);
      document.body.removeChild(link);
      showToast(
-        "Chats exported successfully. Note: Must have at least 10 chats to be valid for OpenAI fine tuning.",
+        `Chats exported successfully as ${fileExtension.toUpperCase()}.`,
        "success"
      );
    } else {
@ -30,6 +40,27 @@ export default function WorkspaceChats() {
    }
  };

+  const toggleMenu = () => {
+    setShowMenu(!showMenu);
+  };
+
+  useEffect(() => {
+    function handleClickOutside(event) {
+      if (
+        menuRef.current &&
+        !menuRef.current.contains(event.target) &&
+        !openMenuButton.current.contains(event.target)
+      ) {
+        setShowMenu(false);
+      }
+    }
+
+    document.addEventListener("mousedown", handleClickOutside);
+    return () => {
+      document.removeEventListener("mousedown", handleClickOutside);
+    };
+  }, []);
+
  return (
    <div className="w-screen h-screen overflow-hidden bg-sidebar flex">
      {!isMobile && <Sidebar />}
@ -44,12 +75,46 @@ export default function WorkspaceChats() {
              <p className="text-2xl font-semibold text-white">
                Workspace Chats
              </p>
-              <button
-                onClick={handleDumpChats}
-                className="border border-slate-200 px-4 py-1 rounded-lg text-slate-200 text-sm items-center flex gap-x-2 hover:bg-slate-200 hover:text-slate-800"
-              >
-                Export Chats to JSONL
-              </button>
+              <div className="flex gap-x-1 relative">
+                <button
+                  onClick={handleDumpChats}
+                  className="border border-slate-200 px-4 py-1 rounded-lg text-slate-200 text-sm items-center flex gap-x-2 hover:bg-slate-200 hover:text-slate-800"
+                >
+                  Export Chats to {exportType.toUpperCase()}
+                </button>
+                <button
+                  ref={openMenuButton}
+                  onClick={toggleMenu}
+                  className={`transition-all duration-300 border border-slate-200 p-1 rounded-lg text-slate-200 text-sm items-center flex hover:bg-slate-200 hover:text-slate-800 ${
+                    showMenu ? "bg-slate-200 text-slate-800" : ""
+                  }`}
+                >
+                  <CaretDown weight="bold" className="h-4 w-4" />
+                </button>
+                <div
+                  ref={menuRef}
+                  className={`${
+                    showMenu ? "slide-down" : "slide-up hidden"
+                  } z-20 w-fit rounded-lg absolute top-full right-0 bg-sidebar p-4 flex items-center justify-center mt-2`}
+                >
+                  <div className="flex flex-col gap-y-2">
+                    {Object.keys(exportOptions)
+                      .filter((type) => type !== exportType)
+                      .map((type) => (
+                        <button
+                          key={type}
+                          onClick={() => {
+                            setExportType(type);
+                            setShowMenu(false);
+                          }}
+                          className="text-white hover:bg-slate-200/20 w-full text-left px-4 py-1.5 rounded-md"
+                        >
+                          {type.toUpperCase()}
+                        </button>
+                      ))}
+                  </div>
+                </div>
+              </div>
            </div>
            <p className="text-sm font-base text-white text-opacity-60">
              These are all the recorded chats and messages that have been sent
--- a/server/endpoints/system.js
+++ b/server/endpoints/system.js
@ -47,6 +47,7 @@ const { WorkspaceChats } = require("../models/workspaceChats");
 const { Workspace } = require("../models/workspace");
 const { flexUserRoleValid } = require("../utils/middleware/multiUserProtected");
 const { fetchPfp, determinePfpFilepath } = require("../utils/files/pfp");
+const { convertToCSV, convertToJSON, convertToJSONL } = require("./utils");

 function systemEndpoints(app) {
  if (!app) return;
@ -776,8 +777,9 @@ function systemEndpoints(app) {
  app.get(
    "/system/export-chats",
    [validatedRequest, flexUserRoleValid],
-    async (_request, response) => {
+    async (request, response) => {
      try {
+        const { type = "jsonl" } = request.query;
        const chats = await WorkspaceChats.whereWithData({}, null, null, {
          id: "asc",
        });
@ -828,13 +830,27 @@ function systemEndpoints(app) {
          return acc;
        }, {});

-        // Convert to JSONL
-        const jsonl = Object.values(workspaceChatsMap)
-          .map((workspaceChats) => JSON.stringify(workspaceChats))
-          .join("\n");
+        let output;
+        switch (type.toLowerCase()) {
+          case "json": {
+            response.setHeader("Content-Type", "application/json");
+            output = await convertToJSON(workspaceChatsMap);
+            break;
+          }
+          case "csv": {
+            response.setHeader("Content-Type", "text/csv");
+            output = await convertToCSV(workspaceChatsMap);
+            break;
+          }
+          // JSONL default
+          default: {
+            response.setHeader("Content-Type", "application/jsonl");
+            output = await convertToJSONL(workspaceChatsMap);
+            break;
+          }
+        }

-        response.setHeader("Content-Type", "application/jsonl");
-        response.status(200).send(jsonl);
+        response.status(200).send(output);
      } catch (e) {
        console.error(e);
        response.sendStatus(500).end();
--- a/server/endpoints/utils.js
+++ b/server/endpoints/utils.js
@ -32,6 +32,34 @@ async function getDiskStorage() {
  }
 }

+async function convertToCSV(workspaceChatsMap) {
+  const rows = ["role,content"];
+  for (const workspaceChats of Object.values(workspaceChatsMap)) {
+    for (const message of workspaceChats.messages) {
+      // Escape double quotes and wrap content in double quotes
+      const escapedContent = `"${message.content
+        .replace(/"/g, '""')
+        .replace(/\n/g, " ")}"`;
+      rows.push(`${message.role},${escapedContent}`);
+    }
+  }
+  return rows.join("\n");
+}
+
+async function convertToJSON(workspaceChatsMap) {
+  const allMessages = [].concat.apply(
+    [],
+    Object.values(workspaceChatsMap).map((workspace) => workspace.messages)
+  );
+  return JSON.stringify(allMessages);
+}
+
+async function convertToJSONL(workspaceChatsMap) {
+  return Object.values(workspaceChatsMap)
+    .map((workspaceChats) => JSON.stringify(workspaceChats))
+    .join("\n");
+}
+
 function utilEndpoints(app) {
  if (!app) return;

@ -54,4 +82,10 @@ function utilEndpoints(app) {
  });
 }

-module.exports = { utilEndpoints, getGitVersion };
+module.exports = {
+  utilEndpoints,
+  getGitVersion,
+  convertToCSV,
+  convertToJSON,
+  convertToJSONL,
+};