added JSONL export to workspace chats (#345)

* added JSONL export to workspace chats * change permissions for workspace chat settings * change permissions for workspace chat settings * Show error for correct limit on fine-tune Change sidebar position and permission Remove check for MUM --------- Co-authored-by: timothycarambat <rambat1010@gmail.com>
2024-07-02 15:20:37 +02:00 · 2023-11-08 17:36:54 -08:00 · 2023-11-08 17:36:54 -08:00 · 997482ef8f
commit 997482ef8f
parent 88d4808c52
10 changed files with 224 additions and 98 deletions
--- a/frontend/src/App.jsx
+++ b/frontend/src/App.jsx
@ -12,8 +12,8 @@ const WorkspaceChat = lazy(() => import("./pages/WorkspaceChat"));
 const AdminUsers = lazy(() => import("./pages/Admin/Users"));
 const AdminInvites = lazy(() => import("./pages/Admin/Invitations"));
 const AdminWorkspaces = lazy(() => import("./pages/Admin/Workspaces"));
-const AdminChats = lazy(() => import("./pages/Admin/Chats"));
 const AdminSystem = lazy(() => import("./pages/Admin/System"));
+const GeneralChats = lazy(() => import("./pages/GeneralSettings/Chats"));
 const GeneralAppearance = lazy(() =>
  import("./pages/GeneralSettings/Appearance")
 );
@ -77,6 +77,10 @@ export default function App() {
            path="/general/api-keys"
            element={<PrivateRoute Component={GeneralApiKeys} />}
          />
+          <Route
+            path="/general/workspace-chats"
+            element={<PrivateRoute Component={GeneralChats} />}
+          />

          {/* Admin Routes */}
          <Route
@ -95,11 +99,6 @@ export default function App() {
            path="/admin/workspaces"
            element={<AdminRoute Component={AdminWorkspaces} />}
          />
-          <Route
-            path="/admin/workspace-chats"
-            element={<AdminRoute Component={AdminChats} />}
-          />
-
          {/* Onboarding Flow */}
          <Route path="/onboarding" element={<OnboardingFlow />} />
        </Routes>
--- a/frontend/src/components/SettingsSidebar/index.jsx
+++ b/frontend/src/components/SettingsSidebar/index.jsx
@ -91,7 +91,7 @@ export default function SettingsSidebar() {
                      icon={<BookOpen className="h-5 w-5 flex-shrink-0" />}
                    />
                    <Option
-                      href={paths.admin.chats()}
+                      href={paths.general.chats()}
                      btnText="Workspace Chat"
                      icon={
                        <ChatCenteredText className="h-5 w-5 flex-shrink-0" />
@ -131,6 +131,15 @@ export default function SettingsSidebar() {
                  btnText="Export or Import"
                  icon={<DownloadSimple className="h-5 w-5 flex-shrink-0" />}
                />
+                {!user && (
+                  <Option
+                    href={paths.general.chats()}
+                    btnText="Chat History"
+                    icon={
+                      <ChatCenteredText className="h-5 w-5 flex-shrink-0" />
+                    }
+                  />
+                )}
                <Option
                  href={paths.general.security()}
                  btnText="Security"
@ -292,17 +301,17 @@ export function SidebarMobileHeader() {
                        btnText="Workspaces"
                        icon={<BookOpen className="h-5 w-5 flex-shrink-0" />}
                      />
-                      <Option
-                        href={paths.admin.chats()}
-                        btnText="Workspace Chat"
-                        icon={
-                          <ChatCenteredText className="h-5 w-5 flex-shrink-0" />
-                        }
-                      />
                    </>
                  )}

                  {/* General Settings */}
+                  <Option
+                    href={paths.general.chats()}
+                    btnText="Workspace Chat"
+                    icon={
+                      <ChatCenteredText className="h-5 w-5 flex-shrink-0" />
+                    }
+                  />
                  <Option
                    href={paths.general.appearance()}
                    btnText="Appearance"
--- a/frontend/src/models/admin.js
+++ b/frontend/src/models/admin.js
@ -139,31 +139,6 @@ const Admin = {
      });
  },

-  // Workspace Chats Mgmt
-  chats: async (offset = 0) => {
-    return await fetch(`${API_BASE}/admin/workspace-chats`, {
-      method: "POST",
-      headers: baseHeaders(),
-      body: JSON.stringify({ offset }),
-    })
-      .then((res) => res.json())
-      .catch((e) => {
-        console.error(e);
-        return [];
-      });
-  },
-  deleteChat: async (chatId) => {
-    return await fetch(`${API_BASE}/admin/workspace-chats/${chatId}`, {
-      method: "DELETE",
-      headers: baseHeaders(),
-    })
-      .then((res) => res.json())
-      .catch((e) => {
-        console.error(e);
-        return { success: false, error: e.message };
-      });
-  },
-
  // System Preferences
  systemPreferences: async () => {
    return await fetch(`${API_BASE}/admin/system-preferences`, {
--- a/frontend/src/models/system.js
+++ b/frontend/src/models/system.js
@ -339,6 +339,40 @@ const System = {
        return { models: [], error: e.message };
      });
  },
+  chats: async (offset = 0) => {
+    return await fetch(`${API_BASE}/system/workspace-chats`, {
+      method: "POST",
+      headers: baseHeaders(),
+      body: JSON.stringify({ offset }),
+    })
+      .then((res) => res.json())
+      .catch((e) => {
+        console.error(e);
+        return [];
+      });
+  },
+  deleteChat: async (chatId) => {
+    return await fetch(`${API_BASE}/system/workspace-chats/${chatId}`, {
+      method: "DELETE",
+      headers: baseHeaders(),
+    })
+      .then((res) => res.json())
+      .catch((e) => {
+        console.error(e);
+        return { success: false, error: e.message };
+      });
+  },
+  exportChats: async () => {
+    return await fetch(`${API_BASE}/system/export-chats`, {
+      method: "GET",
+      headers: baseHeaders(),
+    })
+      .then((res) => res.text())
+      .catch((e) => {
+        console.error(e);
+        return null;
+      });
+  },
 };

 export default System;
--- a/frontend/src/pages/GeneralSettings/Chats/ChatRow/index.jsx
+++ b/frontend/src/pages/GeneralSettings/Chats/ChatRow/index.jsx
@ -1,7 +1,7 @@
 import { useRef } from "react";
-import Admin from "../../../../models/admin";
 import truncate from "truncate";
 import { X, Trash } from "@phosphor-icons/react";
+import System from "../../../../models/system";

 export default function ChatRow({ chat }) {
  const rowRef = useRef(null);
@ -13,7 +13,7 @@ export default function ChatRow({ chat }) {
    )
      return false;
    rowRef?.current?.remove();
-    await Admin.deleteChat(chat.id);
+    await System.deleteChat(chat.id);
  };

  return (
--- a/frontend/src/pages/GeneralSettings/Chats/index.jsx
+++ b/frontend/src/pages/GeneralSettings/Chats/index.jsx
@ -5,12 +5,33 @@ import Sidebar, {
 import { isMobile } from "react-device-detect";
 import * as Skeleton from "react-loading-skeleton";
 import "react-loading-skeleton/dist/skeleton.css";
-import Admin from "../../../models/admin";
 import useQuery from "../../../hooks/useQuery";
 import ChatRow from "./ChatRow";
+import showToast from "../../../utils/toast";
+import System from "../../../models/system";

 const PAGE_SIZE = 20;
-export default function AdminChats() {
+export default function WorkspaceChats() {
+  const handleDumpChats = async () => {
+    const chats = await System.exportChats();
+    if (chats) {
+      const blob = new Blob([chats], { type: "application/jsonl" });
+      const link = document.createElement("a");
+      link.href = window.URL.createObjectURL(blob);
+      link.download = "chats.jsonl";
+      document.body.appendChild(link);
+      link.click();
+      window.URL.revokeObjectURL(link.href);
+      document.body.removeChild(link);
+      showToast(
+        "Chats exported successfully. Note: Must have at least 10 chats to be valid for OpenAI fine tuning.",
+        "success"
+      );
+    } else {
+      showToast("Failed to export chats.", "error");
+    }
+  };
+
  return (
    <div className="w-screen h-screen overflow-hidden bg-sidebar flex">
      {!isMobile && <Sidebar />}
@ -25,6 +46,12 @@ export default function AdminChats() {
              <p className="text-2xl font-semibold text-white">
                Workspace Chats
              </p>
+              <button
+                onClick={handleDumpChats}
+                className="border border-slate-200 px-4 py-1 rounded-lg text-slate-200 text-sm items-center flex gap-x-2 hover:bg-slate-200 hover:text-slate-800"
+              >
+                Export Chats to JSONL
+              </button>
            </div>
            <p className="text-sm font-base text-white text-opacity-60">
              These are all the recorded chats and messages that have been sent
@ -54,7 +81,7 @@ function ChatsContainer() {

  useEffect(() => {
    async function fetchChats() {
-      const { chats: _chats, hasPages = false } = await Admin.chats(offset);
+      const { chats: _chats, hasPages = false } = await System.chats(offset);
      setChats(_chats);
      setCanNext(hasPages);
      setLoading(false);
@ -105,9 +132,8 @@ function ChatsContainer() {
          </tr>
        </thead>
        <tbody>
-          {chats.map((chat) => (
-            <ChatRow key={chat.id} chat={chat} />
-          ))}
+          {!!chats &&
+            chats.map((chat) => <ChatRow key={chat.id} chat={chat} />)}
        </tbody>
      </table>
      <div className="flex w-full justify-between items-center">
--- a/frontend/src/utils/paths.js
+++ b/frontend/src/utils/paths.js
@ -61,6 +61,9 @@ export default {
    apiKeys: () => {
      return "/general/api-keys";
    },
+    chats: () => {
+      return "/general/workspace-chats";
+    },
  },
  admin: {
    system: () => {
--- a/server/endpoints/admin.js
+++ b/server/endpoints/admin.js
@ -251,56 +251,6 @@ function adminEndpoints(app) {
    }
  );

-  app.post(
-    "/admin/workspace-chats",
-    [validatedRequest],
-    async (request, response) => {
-      try {
-        const user = await userFromSession(request, response);
-        if (!user || user?.role !== "admin") {
-          response.sendStatus(401).end();
-          return;
-        }
-
-        const { offset = 0, limit = 20 } = reqBody(request);
-        const chats = await WorkspaceChats.whereWithData(
-          {},
-          limit,
-          offset * limit,
-          { id: "desc" }
-        );
-        const totalChats = await WorkspaceChats.count();
-        const hasPages = totalChats > (offset + 1) * limit;
-
-        response.status(200).json({ chats: chats, hasPages, totalChats });
-      } catch (e) {
-        console.error(e);
-        response.sendStatus(500).end();
-      }
-    }
-  );
-
-  app.delete(
-    "/admin/workspace-chats/:id",
-    [validatedRequest],
-    async (request, response) => {
-      try {
-        const user = await userFromSession(request, response);
-        if (!user || user?.role !== "admin") {
-          response.sendStatus(401).end();
-          return;
-        }
-
-        const { id } = request.params;
-        await WorkspaceChats.delete({ id: Number(id) });
-        response.status(200).json({ success, error });
-      } catch (e) {
-        console.error(e);
-        response.sendStatus(500).end();
-      }
-    }
-  );
-
  app.get(
    "/admin/system-preferences",
    [validatedRequest],
--- a/server/endpoints/system.js
+++ b/server/endpoints/system.js
@ -38,6 +38,8 @@ const { Telemetry } = require("../models/telemetry");
 const { WelcomeMessages } = require("../models/welcomeMessages");
 const { ApiKey } = require("../models/apiKeys");
 const { getCustomModels } = require("../utils/helpers/customModels");
+const { WorkspaceChats } = require("../models/workspaceChats");
+const { Workspace } = require("../models/workspace");

 function systemEndpoints(app) {
  if (!app) return;
@ -646,6 +648,134 @@ function systemEndpoints(app) {
      }
    }
  );
+
+  app.post(
+    "/system/workspace-chats",
+    [validatedRequest],
+    async (request, response) => {
+      try {
+        if (
+          response.locals.multiUserMode &&
+          response.locals.user?.role !== "admin"
+        ) {
+          return response.sendStatus(401).end();
+        }
+
+        const { offset = 0, limit = 20 } = reqBody(request);
+        const chats = await WorkspaceChats.whereWithData(
+          {},
+          limit,
+          offset * limit,
+          { id: "desc" }
+        );
+        const totalChats = await WorkspaceChats.count();
+        const hasPages = totalChats > (offset + 1) * limit;
+
+        response.status(200).json({ chats: chats, hasPages, totalChats });
+      } catch (e) {
+        console.error(e);
+        response.sendStatus(500).end();
+      }
+    }
+  );
+
+  app.delete(
+    "/system/workspace-chats/:id",
+    [validatedRequest],
+    async (request, response) => {
+      try {
+        if (
+          response.locals.multiUserMode &&
+          response.locals.user?.role !== "admin"
+        ) {
+          return response.sendStatus(401).end();
+        }
+
+        const { id } = request.params;
+        await WorkspaceChats.delete({ id: Number(id) });
+        response.status(200).json({ success, error });
+      } catch (e) {
+        console.error(e);
+        response.sendStatus(500).end();
+      }
+    }
+  );
+
+  app.get(
+    "/system/export-chats",
+    [validatedRequest],
+    async (request, response) => {
+      try {
+        if (
+          response.locals.multiUserMode &&
+          response.locals.user?.role !== "admin"
+        ) {
+          return response.sendStatus(401).end();
+        }
+
+        const chats = await WorkspaceChats.whereWithData({}, null, null, {
+          id: "asc",
+        });
+        const workspaceIds = [
+          ...new Set(chats.map((chat) => chat.workspaceId)),
+        ];
+
+        const workspacesWithPrompts = await Promise.all(
+          workspaceIds.map((id) => Workspace.get({ id: Number(id) }))
+        );
+
+        const workspacePromptsMap = workspacesWithPrompts.reduce(
+          (acc, workspace) => {
+            acc[workspace.id] = workspace.openAiPrompt;
+            return acc;
+          },
+          {}
+        );
+
+        const workspaceChatsMap = chats.reduce((acc, chat) => {
+          const { prompt, response, workspaceId } = chat;
+          const responseJson = JSON.parse(response);
+
+          if (!acc[workspaceId]) {
+            acc[workspaceId] = {
+              messages: [
+                {
+                  role: "system",
+                  content:
+                    workspacePromptsMap[workspaceId] ||
+                    "Given the following conversation, relevant context, and a follow up question, reply with an answer to the current question the user is asking. Return only your response to the question given the above information following the users instructions as needed.",
+                },
+              ],
+            };
+          }
+
+          acc[workspaceId].messages.push(
+            {
+              role: "user",
+              content: prompt,
+            },
+            {
+              role: "assistant",
+              content: responseJson.text,
+            }
+          );
+
+          return acc;
+        }, {});
+
+        // Convert to JSONL
+        const jsonl = Object.values(workspaceChatsMap)
+          .map((workspaceChats) => JSON.stringify(workspaceChats))
+          .join("\n");
+
+        response.setHeader("Content-Type", "application/jsonl");
+        response.status(200).send(jsonl);
+      } catch (e) {
+        console.error(e);
+        response.sendStatus(500).end();
+      }
+    }
+  );
 }

 module.exports = { systemEndpoints };
--- a/server/models/workspaceChats.js
+++ b/server/models/workspaceChats.js
@ -161,7 +161,7 @@ const WorkspaceChats = {
        const user = await User.get({ id: res.user_id });
        res.user = user
          ? { username: user.username }
-          : { username: "deleted user" };
+          : { username: "unknown user" };
      }

      return results;