From 11f6419c3c0fb53310743b747c1934b0492f6e8c Mon Sep 17 00:00:00 2001 From: Sean Hatfield Date: Thu, 25 Apr 2024 16:52:30 -0700 Subject: [PATCH 01/18] [FEAT] Implement new login screen UI & multi-user password reset (#1074) * WIP new login screen UI * update prisma schema/create new models for pw recovery * WIP password recovery backend * WIP reset password flow * WIP pw reset flow * password reset logic complete & functional UI * WIP login screen redesign for single and multi user * create placeholder modal to display recovery codes * implement UI for recovery code modals/download recovery codes * multiuser desktop password reset UI/functionality complete * support single user mode for pw reset * mobile styles for all password reset/login flows complete * lint * remove single user password recovery * create PasswordRecovery util file to make more readable * do not drop-replace users table in migration * review pr --------- Co-authored-by: timothycarambat --- .../Modals/DisplayRecoveryCodeModal/index.jsx | 86 +++++ .../Modals/Password/MultiUserAuth.jsx | 362 +++++++++++++++--- .../Modals/Password/SingleUserAuth.jsx | 127 ++++-- .../src/components/Modals/Password/index.jsx | 34 +- .../illustrations/login-illustration.svg | 174 +++++++++ .../src/media/illustrations/login-logo.svg | 37 ++ frontend/src/models/system.js | 37 ++ frontend/tailwind.config.js | 1 + server/endpoints/system.js | 73 ++++ server/models/passwordRecovery.js | 115 ++++++ .../20240425004220_init/migration.sql | 30 ++ server/prisma/schema.prisma | 24 ++ server/utils/PasswordRecovery/index.js | 98 +++++ server/utils/middleware/multiUserProtected.js | 16 + 14 files changed, 1109 insertions(+), 105 deletions(-) create mode 100644 frontend/src/components/Modals/DisplayRecoveryCodeModal/index.jsx create mode 100644 frontend/src/media/illustrations/login-illustration.svg create mode 100644 frontend/src/media/illustrations/login-logo.svg create mode 100644 server/models/passwordRecovery.js create mode 100644 server/prisma/migrations/20240425004220_init/migration.sql create mode 100644 server/utils/PasswordRecovery/index.js diff --git a/frontend/src/components/Modals/DisplayRecoveryCodeModal/index.jsx b/frontend/src/components/Modals/DisplayRecoveryCodeModal/index.jsx new file mode 100644 index 000000000..a75a35382 --- /dev/null +++ b/frontend/src/components/Modals/DisplayRecoveryCodeModal/index.jsx @@ -0,0 +1,86 @@ +import showToast from "@/utils/toast"; +import { DownloadSimple, Key } from "@phosphor-icons/react"; +import { saveAs } from "file-saver"; +import { useState } from "react"; + +export default function RecoveryCodeModal({ + recoveryCodes, + onDownloadComplete, + onClose, +}) { + const [downloadClicked, setDownloadClicked] = useState(false); + + const downloadRecoveryCodes = () => { + const blob = new Blob([recoveryCodes.join("\n")], { type: "text/plain" }); + saveAs(blob, "recovery_codes.txt"); + setDownloadClicked(true); + }; + + const handleClose = () => { + if (downloadClicked) { + onDownloadComplete(); + onClose(); + } + }; + + const handleCopyToClipboard = () => { + navigator.clipboard.writeText(recoveryCodes.join(",\n")).then(() => { + showToast("Recovery codes copied to clipboard", "success", { + clear: true, + }); + }); + }; + + return ( +
+
+
+ + +
+
+

+ In order to reset your password in the future, you will need these + recovery codes. Download or copy your recovery codes to save them.{" "} +
+ These recovery codes are only shown once! +

+
+
    + {recoveryCodes.map((code, index) => ( +
  • + {code} +
    • + ))} +
+
+
+
+
+ +
+
+ ); +} diff --git a/frontend/src/components/Modals/Password/MultiUserAuth.jsx b/frontend/src/components/Modals/Password/MultiUserAuth.jsx index de086fc08..a44e040c4 100644 --- a/frontend/src/components/Modals/Password/MultiUserAuth.jsx +++ b/frontend/src/components/Modals/Password/MultiUserAuth.jsx @@ -1,26 +1,203 @@ -import React, { useState } from "react"; +import React, { useEffect, useState } from "react"; import System from "../../../models/system"; import { AUTH_TOKEN, AUTH_USER } from "../../../utils/constants"; import useLogo from "../../../hooks/useLogo"; import paths from "../../../utils/paths"; +import showToast from "@/utils/toast"; +import ModalWrapper from "@/components/ModalWrapper"; +import { useModal } from "@/hooks/useModal"; +import RecoveryCodeModal from "@/components/Modals/DisplayRecoveryCodeModal"; + +const RecoveryForm = ({ onSubmit, setShowRecoveryForm }) => { + const [username, setUsername] = useState(""); + const [recoveryCodeInputs, setRecoveryCodeInputs] = useState( + Array(2).fill("") + ); + + const handleRecoveryCodeChange = (index, value) => { + const updatedCodes = [...recoveryCodeInputs]; + updatedCodes[index] = value; + setRecoveryCodeInputs(updatedCodes); + }; + + const handleSubmit = (e) => { + e.preventDefault(); + const recoveryCodes = recoveryCodeInputs.filter( + (code) => code.trim() !== "" + ); + onSubmit(username, recoveryCodes); + }; + + return ( +
+
+
+

+ Password Reset +

+

+ Provide the necessary information below to reset your password. +

+
+
+
+
+
+ + setUsername(e.target.value)} + className="bg-zinc-900 text-white placeholder-white/20 text-sm rounded-md p-2.5 w-full h-[48px] md:w-[300px] md:h-[34px]" + required + /> +
+
+ + {recoveryCodeInputs.map((code, index) => ( +
+ + handleRecoveryCodeChange(index, e.target.value) + } + className="bg-zinc-900 text-white placeholder-white/20 text-sm rounded-md p-2.5 w-full h-[48px] md:w-[300px] md:h-[34px]" + required + /> +
+ ))} +
+
+
+
+ + +
+
+ ); +}; + +const ResetPasswordForm = ({ onSubmit }) => { + const [newPassword, setNewPassword] = useState(""); + const [confirmPassword, setConfirmPassword] = useState(""); + + const handleSubmit = (e) => { + e.preventDefault(); + onSubmit(newPassword, confirmPassword); + }; + + return ( +
+
+
+

+ Reset Password +

+

+ Enter your new password. +

+
+
+
+
+
+ setNewPassword(e.target.value)} + className="bg-zinc-900 text-white placeholder-white/20 text-sm rounded-md p-2.5 w-full h-[48px] md:w-[300px] md:h-[34px]" + required + /> +
+
+ setConfirmPassword(e.target.value)} + className="bg-zinc-900 text-white placeholder-white/20 text-sm rounded-md p-2.5 w-full h-[48px] md:w-[300px] md:h-[34px]" + required + /> +
+
+
+
+ +
+
+ ); +}; export default function MultiUserAuth() { const [loading, setLoading] = useState(false); const [error, setError] = useState(null); const { logo: _initLogo } = useLogo(); + const [recoveryCodes, setRecoveryCodes] = useState([]); + const [downloadComplete, setDownloadComplete] = useState(false); + const [user, setUser] = useState(null); + const [token, setToken] = useState(null); + const [showRecoveryForm, setShowRecoveryForm] = useState(false); + const [showResetPasswordForm, setShowResetPasswordForm] = useState(false); + + const { + isOpen: isRecoveryCodeModalOpen, + openModal: openRecoveryCodeModal, + closeModal: closeRecoveryCodeModal, + } = useModal(); + const handleLogin = async (e) => { setError(null); setLoading(true); e.preventDefault(); const data = {}; - const form = new FormData(e.target); for (var [key, value] of form.entries()) data[key] = value; - const { valid, user, token, message } = await System.requestToken(data); + const { valid, user, token, message, recoveryCodes } = + await System.requestToken(data); if (valid && !!token && !!user) { - window.localStorage.setItem(AUTH_USER, JSON.stringify(user)); - window.localStorage.setItem(AUTH_TOKEN, token); - window.location = paths.home(); + setUser(user); + setToken(token); + + if (recoveryCodes) { + setRecoveryCodes(recoveryCodes); + openRecoveryCodeModal(); + } else { + window.localStorage.setItem(AUTH_USER, JSON.stringify(user)); + window.localStorage.setItem(AUTH_TOKEN, token); + window.location = paths.home(); + } } else { setError(message); setLoading(false); @@ -28,57 +205,134 @@ export default function MultiUserAuth() { setLoading(false); }; + const handleDownloadComplete = () => setDownloadComplete(true); + const handleResetPassword = () => setShowRecoveryForm(true); + const handleRecoverySubmit = async (username, recoveryCodes) => { + const { success, resetToken, error } = await System.recoverAccount( + username, + recoveryCodes + ); + + if (success && resetToken) { + window.localStorage.setItem("resetToken", resetToken); + setShowRecoveryForm(false); + setShowResetPasswordForm(true); + } else { + showToast(error, "error", { clear: true }); + } + }; + + const handleResetSubmit = async (newPassword, confirmPassword) => { + const resetToken = window.localStorage.getItem("resetToken"); + + if (resetToken) { + const { success, error } = await System.resetPassword( + resetToken, + newPassword, + confirmPassword + ); + + if (success) { + window.localStorage.removeItem("resetToken"); + setShowResetPasswordForm(false); + showToast("Password reset successful", "success", { clear: true }); + } else { + showToast(error, "error", { clear: true }); + } + } else { + showToast("Invalid reset token", "error", { clear: true }); + } + }; + + useEffect(() => { + if (downloadComplete && user && token) { + window.localStorage.setItem(AUTH_USER, JSON.stringify(user)); + window.localStorage.setItem(AUTH_TOKEN, token); + window.location = paths.home(); + } + }, [downloadComplete, user, token]); + + if (showRecoveryForm) { + return ( + + ); + } + + if (showResetPasswordForm) + return ; return ( -
-
-
-
-

- Sign In -

-
-
-
-
-
- -
- -
- -
- - {error && ( -

- Error: {error} + <> + +

+
+
+
+

+ Welcome to +

+

+ AnythingLLM +

+
+

+ Sign in to your AnythingLLM account.

- )} +
+
+
+
+
+ +
+
+ +
+ {error &&

Error: {error}

} +
+
+
+ +
-
- -
-
- + + + + + + ); } diff --git a/frontend/src/components/Modals/Password/SingleUserAuth.jsx b/frontend/src/components/Modals/Password/SingleUserAuth.jsx index 8135f8f94..c1f328ba2 100644 --- a/frontend/src/components/Modals/Password/SingleUserAuth.jsx +++ b/frontend/src/components/Modals/Password/SingleUserAuth.jsx @@ -1,25 +1,44 @@ -import React, { useState } from "react"; +import React, { useEffect, useState } from "react"; import System from "../../../models/system"; import { AUTH_TOKEN } from "../../../utils/constants"; import useLogo from "../../../hooks/useLogo"; import paths from "../../../utils/paths"; +import ModalWrapper from "@/components/ModalWrapper"; +import { useModal } from "@/hooks/useModal"; +import RecoveryCodeModal from "@/components/Modals/DisplayRecoveryCodeModal"; export default function SingleUserAuth() { const [loading, setLoading] = useState(false); const [error, setError] = useState(null); const { logo: _initLogo } = useLogo(); + const [recoveryCodes, setRecoveryCodes] = useState([]); + const [downloadComplete, setDownloadComplete] = useState(false); + const [token, setToken] = useState(null); + + const { + isOpen: isRecoveryCodeModalOpen, + openModal: openRecoveryCodeModal, + closeModal: closeRecoveryCodeModal, + } = useModal(); + const handleLogin = async (e) => { setError(null); setLoading(true); e.preventDefault(); const data = {}; - const form = new FormData(e.target); for (var [key, value] of form.entries()) data[key] = value; - const { valid, token, message } = await System.requestToken(data); + const { valid, token, message, recoveryCodes } = + await System.requestToken(data); if (valid && !!token) { - window.localStorage.setItem(AUTH_TOKEN, token); - window.location = paths.home(); + setToken(token); + if (recoveryCodes) { + setRecoveryCodes(recoveryCodes); + openRecoveryCodeModal(); + } else { + window.localStorage.setItem(AUTH_TOKEN, token); + window.location = paths.home(); + } } else { setError(message); setLoading(false); @@ -27,45 +46,71 @@ export default function SingleUserAuth() { setLoading(false); }; + const handleDownloadComplete = () => { + setDownloadComplete(true); + }; + + useEffect(() => { + if (downloadComplete && token) { + window.localStorage.setItem(AUTH_TOKEN, token); + window.location = paths.home(); + } + }, [downloadComplete, token]); + return ( -
-
-
-
-

- Sign In -

-
-
-
-
-
- -
- {error && ( -

- Error: {error} + <> + +

+
+
+
+

+ Welcome to +

+

+ AnythingLLM +

+
+

+ Sign in to your AnythingLLM instance.

- )} +
+
+
+
+
+ +
+ + {error &&

Error: {error}

} +
+
+
+
-
- -
-
- + + + + + + ); } diff --git a/frontend/src/components/Modals/Password/index.jsx b/frontend/src/components/Modals/Password/index.jsx index d31b80fdf..9305d032e 100644 --- a/frontend/src/components/Modals/Password/index.jsx +++ b/frontend/src/components/Modals/Password/index.jsx @@ -8,26 +8,40 @@ import { AUTH_TIMESTAMP, } from "../../../utils/constants"; import useLogo from "../../../hooks/useLogo"; +import illustration from "@/media/illustrations/login-illustration.svg"; +import loginLogo from "@/media/illustrations/login-logo.svg"; export default function PasswordModal({ mode = "single" }) { const { logo: _initLogo } = useLogo(); return ( -
+
- -
- logo +
+ login illustration +
+
+ logo {mode === "single" ? : }
diff --git a/frontend/src/media/illustrations/login-illustration.svg b/frontend/src/media/illustrations/login-illustration.svg new file mode 100644 index 000000000..8b8b92ceb --- /dev/null +++ b/frontend/src/media/illustrations/login-illustration.svg @@ -0,0 +1,174 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/frontend/src/media/illustrations/login-logo.svg b/frontend/src/media/illustrations/login-logo.svg new file mode 100644 index 000000000..729f847c0 --- /dev/null +++ b/frontend/src/media/illustrations/login-logo.svg @@ -0,0 +1,37 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/frontend/src/models/system.js b/frontend/src/models/system.js index 89deda75d..af532a047 100644 --- a/frontend/src/models/system.js +++ b/frontend/src/models/system.js @@ -77,6 +77,43 @@ const System = { return { valid: false, message: e.message }; }); }, + recoverAccount: async function (username, recoveryCodes) { + return await fetch(`${API_BASE}/system/recover-account`, { + method: "POST", + headers: baseHeaders(), + body: JSON.stringify({ username, recoveryCodes }), + }) + .then(async (res) => { + const data = await res.json(); + if (!res.ok) { + throw new Error(data.message || "Error recovering account."); + } + return data; + }) + .catch((e) => { + console.error(e); + return { success: false, error: e.message }; + }); + }, + resetPassword: async function (token, newPassword, confirmPassword) { + return await fetch(`${API_BASE}/system/reset-password`, { + method: "POST", + headers: baseHeaders(), + body: JSON.stringify({ token, newPassword, confirmPassword }), + }) + .then(async (res) => { + const data = await res.json(); + if (!res.ok) { + throw new Error(data.message || "Error resetting password."); + } + return data; + }) + .catch((e) => { + console.error(e); + return { success: false, error: e.message }; + }); + }, + checkDocumentProcessorOnline: async () => { return await fetch(`${API_BASE}/system/document-processing-status`, { headers: baseHeaders(), diff --git a/frontend/tailwind.config.js b/frontend/tailwind.config.js index 8add5a692..b0ac87c90 100644 --- a/frontend/tailwind.config.js +++ b/frontend/tailwind.config.js @@ -37,6 +37,7 @@ export default { "main-gradient": "linear-gradient(180deg, #3D4147 0%, #2C2F35 100%)", "modal-gradient": "linear-gradient(180deg, #3D4147 0%, #2C2F35 100%)", "sidebar-gradient": "linear-gradient(90deg, #5B616A 0%, #3F434B 100%)", + "login-gradient": "linear-gradient(180deg, #3D4147 0%, #2C2F35 100%)", "menu-item-gradient": "linear-gradient(90deg, #3D4147 0%, #2C2F35 100%)", "menu-item-selected-gradient": diff --git a/server/endpoints/system.js b/server/endpoints/system.js index 761f892fe..3ea7fb24c 100644 --- a/server/endpoints/system.js +++ b/server/endpoints/system.js @@ -36,6 +36,7 @@ const { WorkspaceChats } = require("../models/workspaceChats"); const { flexUserRoleValid, ROLES, + isMultiUserSetup, } = require("../utils/middleware/multiUserProtected"); const { fetchPfp, determinePfpFilepath } = require("../utils/files/pfp"); const { @@ -44,6 +45,11 @@ const { } = require("../utils/helpers/chat/convertTo"); const { EventLogs } = require("../models/eventLogs"); const { CollectorApi } = require("../utils/collectorApi"); +const { + recoverAccount, + resetPassword, + generateRecoveryCodes, +} = require("../utils/PasswordRecovery"); function systemEndpoints(app) { if (!app) return; @@ -174,6 +180,24 @@ function systemEndpoints(app) { existingUser?.id ); + // Check if the user has seen the recovery codes + if (!existingUser.seen_recovery_codes) { + const plainTextCodes = await generateRecoveryCodes(existingUser.id); + + // Return recovery codes to frontend + response.status(200).json({ + valid: true, + user: existingUser, + token: makeJWT( + { id: existingUser.id, username: existingUser.username }, + "30d" + ), + message: null, + recoveryCodes: plainTextCodes, + }); + return; + } + response.status(200).json({ valid: true, user: existingUser, @@ -221,6 +245,55 @@ function systemEndpoints(app) { } }); + app.post( + "/system/recover-account", + [isMultiUserSetup], + async (request, response) => { + try { + const { username, recoveryCodes } = reqBody(request); + const { success, resetToken, error } = await recoverAccount( + username, + recoveryCodes + ); + + if (success) { + response.status(200).json({ success, resetToken }); + } else { + response.status(400).json({ success, message: error }); + } + } catch (error) { + console.error("Error recovering account:", error); + response + .status(500) + .json({ success: false, message: "Internal server error" }); + } + } + ); + + app.post( + "/system/reset-password", + [isMultiUserSetup], + async (request, response) => { + try { + const { token, newPassword, confirmPassword } = reqBody(request); + const { success, message, error } = await resetPassword( + token, + newPassword, + confirmPassword + ); + + if (success) { + response.status(200).json({ success, message }); + } else { + response.status(400).json({ success, error }); + } + } catch (error) { + console.error("Error resetting password:", error); + response.status(500).json({ success: false, message: error.message }); + } + } + ); + app.get( "/system/system-vectors", [validatedRequest, flexUserRoleValid([ROLES.admin, ROLES.manager])], diff --git a/server/models/passwordRecovery.js b/server/models/passwordRecovery.js new file mode 100644 index 000000000..1d09d08b3 --- /dev/null +++ b/server/models/passwordRecovery.js @@ -0,0 +1,115 @@ +const { v4 } = require("uuid"); +const prisma = require("../utils/prisma"); +const bcrypt = require("bcrypt"); + +const RecoveryCode = { + tablename: "recovery_codes", + writable: [], + create: async function (userId, code) { + try { + const codeHash = await bcrypt.hash(code, 10); + const recoveryCode = await prisma.recovery_codes.create({ + data: { user_id: userId, code_hash: codeHash }, + }); + return { recoveryCode, error: null }; + } catch (error) { + console.error("FAILED TO CREATE RECOVERY CODE.", error.message); + return { recoveryCode: null, error: error.message }; + } + }, + createMany: async function (data) { + try { + const recoveryCodes = await prisma.$transaction( + data.map((recoveryCode) => + prisma.recovery_codes.create({ data: recoveryCode }) + ) + ); + return { recoveryCodes, error: null }; + } catch (error) { + console.error("FAILED TO CREATE RECOVERY CODES.", error.message); + return { recoveryCodes: null, error: error.message }; + } + }, + findFirst: async function (clause = {}) { + try { + const recoveryCode = await prisma.recovery_codes.findFirst({ + where: clause, + }); + return recoveryCode; + } catch (error) { + console.error("FAILED TO FIND RECOVERY CODE.", error.message); + return null; + } + }, + findMany: async function (clause = {}) { + try { + const recoveryCodes = await prisma.recovery_codes.findMany({ + where: clause, + }); + return recoveryCodes; + } catch (error) { + console.error("FAILED TO FIND RECOVERY CODES.", error.message); + return null; + } + }, + deleteMany: async function (clause = {}) { + try { + await prisma.recovery_codes.deleteMany({ where: clause }); + return true; + } catch (error) { + console.error("FAILED TO DELETE RECOVERY CODES.", error.message); + return false; + } + }, + hashesForUser: async function (userId = null) { + if (!userId) return []; + return (await this.findMany({ user_id: userId })).map( + (recovery) => recovery.code_hash + ); + }, +}; + +const PasswordResetToken = { + tablename: "password_reset_tokens", + resetExpiryMs: 600_000, // 10 minutes in ms; + writable: [], + calcExpiry: function () { + return new Date(Date.now() + this.resetExpiryMs); + }, + create: async function (userId) { + try { + const passwordResetToken = await prisma.password_reset_tokens.create({ + data: { user_id: userId, token: v4(), expiresAt: this.calcExpiry() }, + }); + return { passwordResetToken, error: null }; + } catch (error) { + console.error("FAILED TO CREATE PASSWORD RESET TOKEN.", error.message); + return { passwordResetToken: null, error: error.message }; + } + }, + findUnique: async function (clause = {}) { + try { + const passwordResetToken = await prisma.password_reset_tokens.findUnique({ + where: clause, + }); + return passwordResetToken; + } catch (error) { + console.error("FAILED TO FIND PASSWORD RESET TOKEN.", error.message); + return null; + } + }, + deleteMany: async function (clause = {}) { + try { + await prisma.password_reset_tokens.deleteMany({ where: clause }); + return true; + } catch (error) { + console.error("FAILED TO DELETE PASSWORD RESET TOKEN.", error.message); + return false; + } + }, +}; + +module.exports = { + RecoveryCode, + PasswordResetToken, +}; diff --git a/server/prisma/migrations/20240425004220_init/migration.sql b/server/prisma/migrations/20240425004220_init/migration.sql new file mode 100644 index 000000000..14ec7643f --- /dev/null +++ b/server/prisma/migrations/20240425004220_init/migration.sql @@ -0,0 +1,30 @@ +-- AlterTable +ALTER TABLE "users" ADD COLUMN "seen_recovery_codes" BOOLEAN DEFAULT false; + +-- CreateTable +CREATE TABLE "recovery_codes" ( + "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + "user_id" INTEGER NOT NULL, + "code_hash" TEXT NOT NULL, + "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT "recovery_codes_user_id_fkey" FOREIGN KEY ("user_id") REFERENCES "users" ("id") ON DELETE CASCADE ON UPDATE CASCADE +); + +-- CreateTable +CREATE TABLE "password_reset_tokens" ( + "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + "user_id" INTEGER NOT NULL, + "token" TEXT NOT NULL, + "expiresAt" DATETIME NOT NULL, + "createdAt" DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT "password_reset_tokens_user_id_fkey" FOREIGN KEY ("user_id") REFERENCES "users" ("id") ON DELETE CASCADE ON UPDATE CASCADE +); + +-- CreateIndex +CREATE INDEX "recovery_codes_user_id_idx" ON "recovery_codes"("user_id"); + +-- CreateIndex +CREATE UNIQUE INDEX "password_reset_tokens_token_key" ON "password_reset_tokens"("token"); + +-- CreateIndex +CREATE INDEX "password_reset_tokens_user_id_idx" ON "password_reset_tokens"("user_id"); diff --git a/server/prisma/schema.prisma b/server/prisma/schema.prisma index a07eb0058..6c8689b9e 100644 --- a/server/prisma/schema.prisma +++ b/server/prisma/schema.prisma @@ -62,6 +62,7 @@ model users { pfpFilename String? role String @default("default") suspended Int @default(0) + seen_recovery_codes Boolean? @default(false) createdAt DateTime @default(now()) lastUpdatedAt DateTime @default(now()) workspace_chats workspace_chats[] @@ -69,9 +70,32 @@ model users { embed_configs embed_configs[] embed_chats embed_chats[] threads workspace_threads[] + recovery_codes recovery_codes[] + password_reset_tokens password_reset_tokens[] workspace_agent_invocations workspace_agent_invocations[] } +model recovery_codes { + id Int @id @default(autoincrement()) + user_id Int + code_hash String + createdAt DateTime @default(now()) + user users @relation(fields: [user_id], references: [id], onDelete: Cascade) + + @@index([user_id]) +} + +model password_reset_tokens { + id Int @id @default(autoincrement()) + user_id Int + token String @unique + expiresAt DateTime + createdAt DateTime @default(now()) + user users @relation(fields: [user_id], references: [id], onDelete: Cascade) + + @@index([user_id]) +} + model document_vectors { id Int @id @default(autoincrement()) docId String diff --git a/server/utils/PasswordRecovery/index.js b/server/utils/PasswordRecovery/index.js new file mode 100644 index 000000000..fbcbe5799 --- /dev/null +++ b/server/utils/PasswordRecovery/index.js @@ -0,0 +1,98 @@ +const bcrypt = require("bcrypt"); +const { v4, validate } = require("uuid"); +const { User } = require("../../models/user"); +const { + RecoveryCode, + PasswordResetToken, +} = require("../../models/passwordRecovery"); + +async function generateRecoveryCodes(userId) { + const newRecoveryCodes = []; + const plainTextCodes = []; + for (let i = 0; i < 4; i++) { + const code = v4(); + const hashedCode = bcrypt.hashSync(code, 10); + newRecoveryCodes.push({ + user_id: userId, + code_hash: hashedCode, + }); + plainTextCodes.push(code); + } + + const { error } = await RecoveryCode.createMany(newRecoveryCodes); + if (!!error) throw new Error(error); + + const { success } = await User.update(userId, { + seen_recovery_codes: true, + }); + if (!success) throw new Error("Failed to generate user recovery codes!"); + + return plainTextCodes; +} + +async function recoverAccount(username = "", recoveryCodes = []) { + const user = await User.get({ username: String(username) }); + if (!user) return { success: false, error: "Invalid recovery codes." }; + + // If hashes do not exist for a user + // because this is a user who has not logged out and back in since upgrade. + const allUserHashes = await RecoveryCode.hashesForUser(user.id); + if (allUserHashes.length < 4) + return { success: false, error: "Invalid recovery codes" }; + + // If they tried to send more than two unique codes, we only take the first two + const uniqueRecoveryCodes = [...new Set(recoveryCodes)] + .map((code) => code.trim()) + .filter((code) => validate(code)) // we know that any provided code must be a uuid v4. + .slice(0, 2); + if (uniqueRecoveryCodes.length !== 2) + return { success: false, error: "Invalid recovery codes." }; + + const validCodes = uniqueRecoveryCodes.every((code) => { + let valid = false; + allUserHashes.forEach((hash) => { + if (bcrypt.compareSync(code, hash)) valid = true; + }); + return valid; + }); + if (!validCodes) return { success: false, error: "Invalid recovery codes" }; + + const { passwordResetToken, error } = await PasswordResetToken.create( + user.id + ); + if (!!error) return { success: false, error }; + return { success: true, resetToken: passwordResetToken.token }; +} + +async function resetPassword(token, _newPassword = "", confirmPassword = "") { + const newPassword = String(_newPassword).trim(); // No spaces in passwords + if (!newPassword) throw new Error("Invalid password."); + if (newPassword !== String(confirmPassword)) + throw new Error("Passwords do not match"); + + const resetToken = await PasswordResetToken.findUnique({ + token: String(token), + }); + if (!resetToken || resetToken.expiresAt < new Date()) { + return { success: false, message: "Invalid reset token" }; + } + + // JOI password rules will be enforced inside .update. + const { error } = await User.update(resetToken.user_id, { + password: newPassword, + seen_recovery_codes: false, + }); + + if (error) return { success: false, message: error }; + await PasswordResetToken.deleteMany({ user_id: resetToken.user_id }); + await RecoveryCode.deleteMany({ user_id: resetToken.user_id }); + + // New codes are provided on first new login. + return { success: true, message: "Password reset successful" }; +} + +module.exports = { + recoverAccount, + resetPassword, + generateRecoveryCodes, +}; diff --git a/server/utils/middleware/multiUserProtected.js b/server/utils/middleware/multiUserProtected.js index f8a28c962..4f128ace1 100644 --- a/server/utils/middleware/multiUserProtected.js +++ b/server/utils/middleware/multiUserProtected.js @@ -64,8 +64,24 @@ function flexUserRoleValid(allowedRoles = DEFAULT_ROLES) { }; } +// Middleware check on a public route if the instance is in a valid +// multi-user set up. +async function isMultiUserSetup(_request, response, next) { + const multiUserMode = await SystemSettings.isMultiUserMode(); + if (!multiUserMode) { + response.status(403).json({ + error: "Invalid request", + }); + return; + } + + next(); + return; +} + module.exports = { ROLES, strictMultiUserRoleValid, flexUserRoleValid, + isMultiUserSetup, }; From 348b36bf8593f3fcbed84fa4540bc0977af0b2cd Mon Sep 17 00:00:00 2001 From: Sean Hatfield Date: Thu, 25 Apr 2024 17:53:38 -0700 Subject: [PATCH 02/18] [FEAT] Confluence data connector (#1181) * WIP Confluence data connector backend * confluence data connector complete * confluence citations * fix citation for confluence * Patch confulence integration * fix Citation Icon for confluence --------- Co-authored-by: timothycarambat --- collector/extensions/index.js | 157 +++++++++++------ collector/package.json | 2 +- .../utils/extensions/Confluence/index.js | 110 ++++++++++++ .../DataConnectorOption/media/confluence.jpeg | Bin 0 -> 5659 bytes .../DataConnectorOption/media/index.js | 2 + .../Connectors/Confluence/index.jsx | 164 ++++++++++++++++++ .../MangeWorkspace/DataConnectors/index.jsx | 7 + .../ChatHistory/Citation/index.jsx | 39 +++-- .../src/media/dataConnectors/confluence.png | Bin 0 -> 9582 bytes frontend/src/models/dataConnector.js | 23 +++ server/endpoints/extensions/index.js | 22 +++ 11 files changed, 458 insertions(+), 68 deletions(-) create mode 100644 collector/utils/extensions/Confluence/index.js create mode 100644 frontend/src/components/DataConnectorOption/media/confluence.jpeg create mode 100644 frontend/src/components/Modals/MangeWorkspace/DataConnectors/Connectors/Confluence/index.jsx create mode 100644 frontend/src/media/dataConnectors/confluence.png diff --git a/collector/extensions/index.js b/collector/extensions/index.js index 0e91d1731..6a3f3393e 100644 --- a/collector/extensions/index.js +++ b/collector/extensions/index.js @@ -4,69 +4,112 @@ const { reqBody } = require("../utils/http"); function extensions(app) { if (!app) return; - app.post("/ext/github-repo", [verifyPayloadIntegrity], async function (request, response) { - try { - const loadGithubRepo = require("../utils/extensions/GithubRepo"); - const { success, reason, data } = await loadGithubRepo(reqBody(request)); - response.status(200).json({ - success, - reason, - data - }); - } catch (e) { - console.error(e); - response.status(200).json({ - success: false, - reason: e.message || "A processing error occurred.", - data: {}, - }); + app.post( + "/ext/github-repo", + [verifyPayloadIntegrity], + async function (request, response) { + try { + const loadGithubRepo = require("../utils/extensions/GithubRepo"); + const { success, reason, data } = await loadGithubRepo( + reqBody(request) + ); + response.status(200).json({ + success, + reason, + data, + }); + } catch (e) { + console.error(e); + response.status(200).json({ + success: false, + reason: e.message || "A processing error occurred.", + data: {}, + }); + } + return; } - return; - }); + ); // gets all branches for a specific repo - app.post("/ext/github-repo/branches", [verifyPayloadIntegrity], async function (request, response) { - try { - const GithubRepoLoader = require("../utils/extensions/GithubRepo/RepoLoader"); - const allBranches = await (new GithubRepoLoader(reqBody(request))).getRepoBranches() - response.status(200).json({ - success: true, - reason: null, - data: { - branches: allBranches - } - }); - } catch (e) { - console.error(e); - response.status(400).json({ - success: false, - reason: e.message, - data: { - branches: [] - } - }); + app.post( + "/ext/github-repo/branches", + [verifyPayloadIntegrity], + async function (request, response) { + try { + const GithubRepoLoader = require("../utils/extensions/GithubRepo/RepoLoader"); + const allBranches = await new GithubRepoLoader( + reqBody(request) + ).getRepoBranches(); + response.status(200).json({ + success: true, + reason: null, + data: { + branches: allBranches, + }, + }); + } catch (e) { + console.error(e); + response.status(400).json({ + success: false, + reason: e.message, + data: { + branches: [], + }, + }); + } + return; } - return; - }); + ); - app.post("/ext/youtube-transcript", [verifyPayloadIntegrity], async function (request, response) { - try { - const loadYouTubeTranscript = require("../utils/extensions/YoutubeTranscript"); - const { success, reason, data } = await loadYouTubeTranscript(reqBody(request)); - response.status(200).json({ success, reason, data }); - } catch (e) { - console.error(e); - response.status(400).json({ - success: false, - reason: e.message, - data: { - title: null, - author: null - } - }); + app.post( + "/ext/youtube-transcript", + [verifyPayloadIntegrity], + async function (request, response) { + try { + const loadYouTubeTranscript = require("../utils/extensions/YoutubeTranscript"); + const { success, reason, data } = await loadYouTubeTranscript( + reqBody(request) + ); + response.status(200).json({ success, reason, data }); + } catch (e) { + console.error(e); + response.status(400).json({ + success: false, + reason: e.message, + data: { + title: null, + author: null, + }, + }); + } + return; } - return; - }); + ); + + app.post( + "/ext/confluence", + [verifyPayloadIntegrity], + async function (request, response) { + try { + const loadConfluence = require("../utils/extensions/Confluence"); + const { success, reason, data } = await loadConfluence( + reqBody(request) + ); + response.status(200).json({ success, reason, data }); + } catch (e) { + console.error(e); + response.status(400).json({ + success: false, + reason: e.message, + data: { + title: null, + author: null, + }, + }); + } + return; + } + ); } module.exports = extensions; diff --git a/collector/package.json b/collector/package.json index 4a5a99fff..5d2e5f0f5 100644 --- a/collector/package.json +++ b/collector/package.json @@ -49,4 +49,4 @@ "nodemon": "^2.0.22", "prettier": "^2.4.1" } -} \ No newline at end of file +} diff --git a/collector/utils/extensions/Confluence/index.js b/collector/utils/extensions/Confluence/index.js new file mode 100644 index 000000000..1ea642e1a --- /dev/null +++ b/collector/utils/extensions/Confluence/index.js @@ -0,0 +1,110 @@ +const fs = require("fs"); +const path = require("path"); +const { default: slugify } = require("slugify"); +const { v4 } = require("uuid"); +const { writeToServerDocuments } = require("../../files"); +const { tokenizeString } = require("../../tokenizer"); +const { + ConfluencePagesLoader, +} = require("langchain/document_loaders/web/confluence"); + +function validSpaceUrl(spaceUrl = "") { + const UrlPattern = require("url-pattern"); + const pattern = new UrlPattern( + "https\\://(:subdomain).atlassian.net/wiki/spaces/(:spaceKey)*" + ); + const match = pattern.match(spaceUrl); + if (!match) return { valid: false, result: null }; + return { valid: true, result: match }; +} + +async function loadConfluence({ pageUrl, username, accessToken }) { + if (!pageUrl || !username || !accessToken) { + return { + success: false, + reason: + "You need either a username and access token, or a personal access token (PAT), to use the Confluence connector.", + }; + } + + const validSpace = validSpaceUrl(pageUrl); + if (!validSpace.result) { + return { + success: false, + reason: + "Confluence space URL is not in the expected format of https://domain.atlassian.net/wiki/space/~SPACEID/*", + }; + } + + const { subdomain, spaceKey } = validSpace.result; + console.log(`-- Working Confluence ${subdomain}.atlassian.net --`); + const loader = new ConfluencePagesLoader({ + baseUrl: `https://${subdomain}.atlassian.net/wiki`, + spaceKey, + username, + accessToken, + }); + + const { docs, error } = await loader + .load() + .then((docs) => { + return { docs, error: null }; + }) + .catch((e) => { + return { + docs: [], + error: e.message?.split("Error:")?.[1] || e.message, + }; + }); + + if (!docs.length || !!error) { + return { + success: false, + reason: error ?? "No pages found for that Confluence space.", + }; + } + const outFolder = slugify( + `${subdomain}-confluence-${v4().slice(0, 4)}` + ).toLowerCase(); + const outFolderPath = path.resolve( + __dirname, + `../../../../server/storage/documents/${outFolder}` + ); + fs.mkdirSync(outFolderPath); + + docs.forEach((doc) => { + const data = { + id: v4(), + url: doc.metadata.url + ".page", + title: doc.metadata.title || doc.metadata.source, + docAuthor: subdomain, + description: doc.metadata.title, + docSource: `${subdomain} Confluence`, + chunkSource: `confluence://${doc.metadata.url}`, + published: new Date().toLocaleString(), + wordCount: doc.pageContent.split(" ").length, + pageContent: doc.pageContent, + token_count_estimate: tokenizeString(doc.pageContent).length, + }; + + console.log( + `[Confluence Loader]: Saving ${doc.metadata.title} to ${outFolder}` + ); + writeToServerDocuments( + data, + `${slugify(doc.metadata.title)}-${data.id}`, + outFolderPath + ); + }); + + return { + success: true, + reason: null, + data: { + spaceKey, + destination: outFolder, + }, + }; +} + +module.exports = loadConfluence; diff --git a/frontend/src/components/DataConnectorOption/media/confluence.jpeg b/frontend/src/components/DataConnectorOption/media/confluence.jpeg new file mode 100644 index 0000000000000000000000000000000000000000..7559663a68ac1dfdc3f5a64f4de891d3d449272c GIT binary patch literal 5659 zcmc&&cT`i^w+LL6^<1^BKveb{mA+}qOXVG9axPyj#eI_^cQ%woJQ6_ zU7g#9)$8y4gC|5Tc^CzWxu{0l&HDnA_A&|k6V-2}_%&VETk z4#$>1PJha_NuCZ55nNK}v2(kyianRz^F0~;dQ-XgQ;meqqFdgA@0Dqmxby#@%VUS4 zX6f}6-n(BfX>L`5$Z}RSHGXIi9ewp`+44??+`>|pOw-bi{Y|OC-jj^G4F3)SQ9mqM z57-qKk~rHL0@Lc1fJrpgW)!4%$jhOhq0-&5a^oIN$Sk#VzDxx%TNiti#p4qKOK)1) z(Fa?MhS{ii`?*lftpJx)v*0)RQKI*ib);q{m*Sj> z1vXS&X;%w7n4*37*E+AJdx(s$Y;hJmZ^|%ZQWuj~Ta*pc#8>kr%+d9j-Oc$Hh-A}U z`4vpg2d%Jg_0M5OO8OtrL1VPB~bu9hnY9WF|MCSW3;9k5$YlEZgdrip_#2``e?F%`HM1kp>G> z8^Z$R)AdKuNo}>WYokY?z}8EVQ_! zboJXgZ?8W9Mz}}j>f#P~S+aEtflOL$@LV5k)`W=Ot$K`O;tJmy`wf=G*7%~z=-Xk} zzv@FCH7l9GlMRtH8g^(i6*S|O+3XR>*eKC_`fWFF?RXpogGw4ecbBFxv;j#K1DaESr;)6o*$=FBaLJgL5Yyf?u@d zQTU`aDPhuy-v(2-sJ8~6XA;CnqI>RQJAwK*6=9r_bOxIJIVw)*@v9Bz-q|&q%6>RU4CxlA zRyVS=xHR6FFJt6NO9(!{Z2h2_U;mPVL)7YhdxkNxh3)48uN?}*IP#hjULm^VE#IhA zN6S&uJKbtf6=&0Z{?^nK{v7A62oi7BGq0y9x>j3WX`4Ef-s&73*(KfO^b(4_+)yZA zl=A>oew3X*6IRva9H(8JXK(@ILZ7b)tttLu*73fQ_nctmofo{nJvhhvg17soL7{JR zK#aY-6V0qE?9sJbwdD@Ueu*y9)BT+lZHPR1-2$WEHC^pqw0&=#If03^<^FQ{83zr| zjadQnk{a#0`*(N^g>ddVV)7Yb~Nsne!LuVq4jR`*OF8wNm4peH==t<0PEp5sjY5WGC^Jd+ZkKt4sOMWFI#yss?| zb;QmnZ^ORr7&)FjU+9vN7e0c(CB-4mW%!vdBU$>oX=xoNDnvx^HCJNIen)bd9l3P$ zB1apOL{1bxGkGS>d5}N{;%>3qONA~+d`_a923A;|GeOY9PJ8$GYJ>u(gs9IU&c^NScKQTWnc9&kZU|4poV@w@sY_hTu& zUwtYb*Z71{^wdWxOh3|!+Q}88?In$C?|POTx5Fy^A|c^*i_>nN>}FA9l*V4mlJ$2| zeML$9&5oEBp~p>Dg#Tvd4tD8u1${W5-XU6Lr_})i+u{?fl`h&|$-QypqU|AD9!@-O zj|WF6S!oqt>aY%V(WQ*nRGb>Fgcqi|OdSumfklKOaZ{D+2(TiC44q*i@1&k;tdD6s(e`V=^UneM!&)>Y~L)c&8oM;-$LJRasFL{m+weT zsjK+r`sD0DI@sniKfi@qZ$qhkbtv}wPRxVxurwUBbt9CNeKb>Bp-RF=VC!z${~`IoAWY&%0Lazzm zXq%ZP?Yc!JA*$Z$#PS;dZ!{%s=D6JOn5<-Q4hz?FD{N#t@ zK9DXeWUeb+3*N~;)RV9ixHnA)c@>XALAR{jp^>;r_mLdcg2Yl^MLhCH$=e6%wB*c0vhfVLWg z0;91g*R1~Nw{nCiaOhsZEz-@Qg@W6}xSoL8ZNG%k1nYx?Q@p+o^%jIY;-QAA-F8H# z-`5Sr$D8Ay!L51vy_VY=h7{SRKlb7ey<_!_8DOoJSPrAjZuR{ozLT9)|KoxEwi>&m z>-_)ZQ@<2F$ba{VZ*2rPneoVVr?1_ADDn1fGLX=URry#GmLEnuy$^7Uj-B5I$7&Is z+1nn21u?Pj-Roa&xR;#)=Rjc%9aD?AQ<+8<5ldsnb)q^UI)NL7t>Mutkx!&70n%!+ zca>$=8z;|qp@|OR+4}$!+l0R1HZ-u46O(7Lp3GP#k)(G^!;(@d?QL+rMECJtez7rK zH0Y(XE9SZrxj0L6{&Z3^HRuMtu#`+$0(;_-eMhQ@igku#4-g7t_`c7jIcM|sS{wYG zp4!Lg200CV7$Tb!3b&h+Jx=t2f6dDttf#6)(v};mq^+;venkvfa+AHc z;>%fj{7L#~ls2E~kjm-&+$RZ)87d;6h|-V6vcnY049-qT4$oa}7+XA9jd~!8bW2)j zo3qCy;;-k6RcUoYL!d$DedOd7gWkVy2`(4@m!H5}0|~2myBAw|>-&Itvg^Hfoezl$eZJVeg|IgS zs3Wrjjk?O}e-;S30{}4Y9+Oo*9B}CR;Q-lVhdm&N(W}^Pkq79ezFZ>2*e=W&b;!x#VXwXG&=LWlOu`A0Tp^Wti`P zn(Y1j*T27(eTLUQP<`MvzWM;i3NKDYw#e`Uu|`N+%#hV`%n7$okwpIalS^Pn8DsiF z3adXbEHCt?(SdF>4*z# zLv2Xf=QIEM8}l)xO3O`KiljdO#!UL@@;&`jjLcygDVr1?xU#gmwG3O7zAC3z zfyf%^n5!KOF|}e; zcL%`L#O);9X#3a01LeXG54@JWpL-YhGeXR(6QPtc*{bTzbKuAz$olEnsiQ~l=UyIy zz=>`k@9Ntqs*1U^kIFv4II3jV@YJa|7y6t0dC9iFmEw~c@;UjU$cA8tSEO0xA#UCp zgsviB`v9A;vOpUstvG8F-G7r!kD%7RpwGv`B5Qj+l>=v(;r4VK~-jdA8toE>*xRRP zhte`)Hga&6+ntVz#>}Ylfih4?S@gsAiox+P-|dNp4Q1uUqAZo}8EN@AtLe5hCTzY% zSFNsJ;k_rF)R}C2V07R@65PLq~%c~&YBq?ThE9Frpn#(^mOpne#5K8*;^R07| zScsX{*?3nZW6`95-O!aZom#R*CIXi;;ZtwpJMfs@t?#5VsSB~+?U5JSBh$Tmi>#@o z(0AY5d;M{4(@!6^Z=;K+5CjG^v=_zv+ApDB;G-rWE;t}{^-`txAAgeJ##x{;cwCIYw=IKE;OlL-MM;;rt5g$q2gRY1gAtv5 zqfhlH8Xm{VKtfL^tI`*CK8dAay)$>s`)kpA%P@MAOI2UI!S-n-pKfj^3yy%l1zqSU z4{Gu#_E)km-RjjX^`5;-(;jS@k*zF~-q>mlN1my3lk82QZ8KM$72Kn`;wIFEVQV+FiBST03@byp3hb&+(|3=bIbU zEmC)0w~H(S>zMj-&rQ~$w|UKBs}|k6vPtdrywkLnygCz(-3?m5c{#=XCq995Nwt1+ zOS613IVj9Ma-^;MgUppbcdEC#>DvL$40h=}9M+Y<;_Tmgm1K{b{V$QmRLL>pFAITU zaJ&3bJ9nmC9WhV?F*#k@R^POERA>DO4O7T2p$Z2~u)45%uqtJZ{#<>eK0CeSQDQ2p$Y~FsFhigwv`WJq3 zFOHP*I<7UNH$zXIg>|&oXTxt*YeP~l^MkVv-Lyft;|ka`Ym(lG>38x-W-{tkz)+mM zso;{h#4iNugw0FAx+!1hYBez~b8uiouaxfF>ITi|md*xWGs0vooDz(B<*&~94{)ko zq7$cmMAP0rUvbzi4B0*$I*@4Y9tu`=(3n1Z&3|Fx5mh`&xy8Po2qXMU@c-{@fAGHm DdbL6T literal 0 HcmV?d00001 diff --git a/frontend/src/components/DataConnectorOption/media/index.js b/frontend/src/components/DataConnectorOption/media/index.js index 543bed5f7..ac8105975 100644 --- a/frontend/src/components/DataConnectorOption/media/index.js +++ b/frontend/src/components/DataConnectorOption/media/index.js @@ -1,9 +1,11 @@ import Github from "./github.svg"; import YouTube from "./youtube.svg"; +import Confluence from "./confluence.jpeg"; const ConnectorImages = { github: Github, youtube: YouTube, + confluence: Confluence, }; export default ConnectorImages; diff --git a/frontend/src/components/Modals/MangeWorkspace/DataConnectors/Connectors/Confluence/index.jsx b/frontend/src/components/Modals/MangeWorkspace/DataConnectors/Connectors/Confluence/index.jsx new file mode 100644 index 000000000..52ca7e63d --- /dev/null +++ b/frontend/src/components/Modals/MangeWorkspace/DataConnectors/Connectors/Confluence/index.jsx @@ -0,0 +1,164 @@ +import { useState } from "react"; +import System from "@/models/system"; +import showToast from "@/utils/toast"; +import { Warning } from "@phosphor-icons/react"; +import { Tooltip } from "react-tooltip"; + +export default function ConfluenceOptions() { + const [loading, setLoading] = useState(false); + + const handleSubmit = async (e) => { + e.preventDefault(); + const form = new FormData(e.target); + + try { + setLoading(true); + showToast( + "Fetching all pages for Confluence space - this may take a while.", + "info", + { + clear: true, + autoClose: false, + } + ); + const { data, error } = await System.dataConnectors.confluence.collect({ + pageUrl: form.get("pageUrl"), + username: form.get("username"), + accessToken: form.get("accessToken"), + }); + + if (!!error) { + showToast(error, "error", { clear: true }); + setLoading(false); + return; + } + + showToast( + `Pages collected from Confluence space ${data.spaceKey}. Output folder is ${data.destination}.`, + "success", + { clear: true } + ); + e.target.reset(); + setLoading(false); + } catch (e) { + console.error(e); + showToast(e.message, "error", { clear: true }); + setLoading(false); + } + }; + + return ( +
+
+
+
+
+
+
+ +

+ URL of a page in the Confluence space. +

+
+ +
+
+
+ +

+ Your Confluence username. +

+
+ +
+
+
+ +

+ Access token for authentication. +

+
+ +
+
+
+ +
+ + {loading && ( +

+ Once complete, all pages will be available for embedding into + workspaces. +

+ )} +
+
+
+
+ ); +} diff --git a/frontend/src/components/Modals/MangeWorkspace/DataConnectors/index.jsx b/frontend/src/components/Modals/MangeWorkspace/DataConnectors/index.jsx index 419fc1fc9..69d30e281 100644 --- a/frontend/src/components/Modals/MangeWorkspace/DataConnectors/index.jsx +++ b/frontend/src/components/Modals/MangeWorkspace/DataConnectors/index.jsx @@ -2,6 +2,7 @@ import ConnectorImages from "@/components/DataConnectorOption/media"; import { MagnifyingGlass } from "@phosphor-icons/react"; import GithubOptions from "./Connectors/Github"; import YoutubeOptions from "./Connectors/Youtube"; +import ConfluenceOptions from "./Connectors/Confluence"; import { useState } from "react"; import ConnectorOption from "./ConnectorOption"; @@ -20,6 +21,12 @@ export const DATA_CONNECTORS = { "Import the transcription of an entire YouTube video from a link.", options: , }, + confluence: { + name: "Confluence", + image: ConnectorImages.confluence, + description: "Import an entire Confluence page in a single click.", + options: , + }, }; export default function DataConnectors() { diff --git a/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/Citation/index.jsx b/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/Citation/index.jsx index 1dfeaaaf3..7105901d3 100644 --- a/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/Citation/index.jsx +++ b/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/Citation/index.jsx @@ -1,4 +1,4 @@ -import { memo, useState } from "react"; +import React, { memo, useState } from "react"; import { v4 } from "uuid"; import { decode as HTMLDecode } from "he"; import truncate from "truncate"; @@ -14,6 +14,7 @@ import { X, YoutubeLogo, } from "@phosphor-icons/react"; +import ConfluenceLogo from "@/media/dataConnectors/confluence.png"; import { Tooltip } from "react-tooltip"; import { toPercentString } from "@/utils/numbers"; @@ -202,13 +203,6 @@ function CitationDetailModal({ source, onClose }) { ); } -const ICONS = { - file: FileText, - link: Link, - youtube: YoutubeLogo, - github: GithubLogo, -}; - // Show the correct title and/or display text for citations // which contain valid outbound links that can be clicked by the // user when viewing a citation. Optionally allows various icons @@ -221,10 +215,17 @@ function parseChunkSource({ title = "", chunks = [] }) { icon: "file", }; - if (!chunks.length || !chunks[0].chunkSource.startsWith("link://")) + if ( + !chunks.length || + (!chunks[0].chunkSource.startsWith("link://") && + !chunks[0].chunkSource.startsWith("confluence://")) + ) return nullResponse; try { - const url = new URL(chunks[0].chunkSource.split("link://")[1]); + const url = new URL( + chunks[0].chunkSource.split("link://")[1] || + chunks[0].chunkSource.split("confluence://")[1] + ); let text = url.host + url.pathname; let icon = "link"; @@ -238,6 +239,11 @@ function parseChunkSource({ title = "", chunks = [] }) { icon = "github"; } + if (url.host.includes("atlassian.net")) { + text = title; + icon = "confluence"; + } + return { isUrl: true, href: url.toString(), @@ -247,3 +253,16 @@ function parseChunkSource({ title = "", chunks = [] }) { } catch {} return nullResponse; } + +// Patch to render Confluence icon as a element like we do with Phosphor +const ConfluenceIcon = ({ ...props }) => ( + +); + +const ICONS = { + file: FileText, + link: Link, + youtube: YoutubeLogo, + github: GithubLogo, + confluence: ConfluenceIcon, +}; diff --git a/frontend/src/media/dataConnectors/confluence.png b/frontend/src/media/dataConnectors/confluence.png new file mode 100644 index 0000000000000000000000000000000000000000..27a5da07bd1402f48dc9cc39d26225505dd85242 GIT binary patch literal 9582 zcmXXs2UrtL(-Z?3Iud$US`ZO5s3_8VO-MjW=tZg0JE04arUD{}bVx1~L81auL%!^>j56R8-WLDYpm%EroK+ zq2&%070r>UAyP}{p&_^1#0AeqF5h(y&m~UJMNXepcJ~E{&njQQ2DjfPr{5l@?>2|$ z$_0=43mywxf%_B|r_b62uVv1MTb!N?Tz;Dr-3y)z5U(YOF9qL#crVeTN6%Rf(U|u$ zyG&m2rpRw_dM)wz0Bpa!tK9BkN!zxGex0u`7Beqe8D?h57+3dhj{`vx&7980{-v@ zY|>Z^G1!iBKV0Q{xWfH#g~xB5H}s4*aGN3qvYVwr^;1}s2{1ZMP~OZRe##s8m%`%m zUE}dxqu~4zWRT?`t>qAd-58JG8ehmEuirXfz$Ts55WUR^z1DbHQ_2#+cB9NLGaMd^0>OVp zV#pU`PtV(pF}ch!IsFz4`3trmV{%*O4c-?H-xG}`3r8MZj5%SnA7^wJ=kQn%4BZwA z-4Vf@QnZ*KtO znO&zvA`h6ICYhZlSzKpD!uLcXC~cp@WpVz^>N?GSe^w-N|6qz zKsgL&0SIF=s&hsqi{#BpbMO%PziZqvkW6mAMkXITVo~%fxU@^5!n-J-jhX`R|77gw z$mC4|nWFGbbBZ%EmuEj#=pTJHoxDo^bD2!u``06ri5mZV{y!xB@3~AS?~^ylBW9HD zVIi3yP9|>!lgaO&kjd&~a{K=aIgrU-6tsylkL3T6$V4)^lKgKv$CSJKKVtSjI724) z!2au!$z$Yz@1rG5^5i3E3c@GOxos+Z6t1GI5~v{u242)vg^NnnXn+zR}Fh+Avmi^m&(83uto3onqfT&hgDGm(|ysGw{Vfw>;S(2cB5fb=WuPl3+ zXjPIB;7By}5s7HPG>?}d3R5In4q4-}eV$uZD#@0rEe|>JZ$Gh@`BYOQ1Kj4+Em{kr z|Giv1yZXZXd+tg?_lj#+(wDzo`;V~23Pn*$5m6_{3L;~=%eZJ691IBhDOI=GX9&fKJ7MFgGQb2OA;ug!UxYUWQNV{bgd`+hei%l~z5Q zUM+0`$x-LYyQ!4iXtrOfi5g0qx_GY9<=kv+jgYi$ z5`M4N+B#a~ryUX!k~1C_=6h3jR2>9R`3MW!>8N0y1g(646^J7B0Wg&r0(Ire6>4hX zdYZVKM}n(I>|7{sKNLe>ASxxl(P>>859|*ezsdMr(0m#B6?v7-k)KcTd7c_jRSaxx zG4#3cN`I)vEgo1iR{>v-HOfn?i(~8`pk&~pSDu-PCM3p?->ZWc4@2e2PhJoT%AbGla^ za?C6)(#s(xpP9u}0aOk6_JM(c_By{^osogt(^&O+BRsSLYG;~&!{JhD{L*J4hb=H6 za%Kqz6Ny650ZwxicxY&7xV+=@5dK!(n$d!=K{?|T9u&E&0~z=qQH2EPt!WXt`IfclS5HkrQjCyjLs3*JEp42} zCumxA@MK3R^lI59KyX*oKS0WDemIR4OCe;et5LZb%< z2TNpHtI*_V4a;p}PZA(zmavIM)0fnjC8GEx8+}k2V#~kol(FzUfD`{z|CQF9P zbI~*8bXeg!=@JtEMgvrQW+<9?rjS-$7HoKv8Db6!AUka`l@voqlw07tRMCaf!wEkW z=5|~@{GB%p8peWM(AD=n0!2a4fsxhm5-pQcn^m|c1H+x)rqxZ>yEvLie)7IYY51UPrVrvC- z%naOPLI`5^&qEE_&=GxX6(5q`{8Lx z45^RWkcAy(t+ujik_>F-5TZE;4?j7g&%T0|2TzWLc~MD24UC0HybjNXkNfB*PBK&9 zpd|KG>d!b4w0<}F&skx@IMadmB`Y$rX8C5Oq!E*RK5oTBlD(Wk^0HjnG)}#A33rsT zY=K~cWVDy_^?iBFEtlL7pmwyj%9K?L2l2&&IeB^6h3?9BFoug88FtU5Ld~N$%1XdR zjZBqM2Wp;|%+6k+aD<{_aE%I^*e+4s{WW0)f?TsAr!=T&n=HNz18Jb$gL+ex6a@Hal(}?wKSU2FE32&Pe{)b^17wWkjYX-s;-57w+lG3$kK4Bt}wlqompKKt>HVGSDKTKZo zlbx2@m-iFy*F(JIi+JA#07$AoL9m=5QwucBtuc_%#F5kliH8$~#cwh94Tb*sqhdVX zAQECeFr_Rdh4n+_<*U z-fCDSVLBL+q3%|e{?L$=ng-R{FkX+XPiiIHQVoW!92bD0Gu3PuBt97S`(-;$3SqdX zWaK2STpQbq+&Q~}O2==3Og|YDBT}G($-Bp5_6deWEqO>9P; zVn;1msiC&as z^e2dGs8TE*SwQSBZ|7G1Qw6;PsMBvbBl9atzy{tT5~VJ}{CA>@&{`j&w;ZVahaTbD zpEhcoTaOJ;nIJ{qZ2HfCh2X&ub`Y%nu@2<4u_zh)Fes{$MI<+(T+` zDOJ`O^=ErCv;*E)Pa=&ePDn<_v*7 zOZz1aeLLlM5e6oK_EJ@_bpAFsn_AaRv;JdY!4>G!aNO+H?D4Mg?hocvfS=Tudpom~ zMjp9Tt<+DNqRC7HCK7cZ&+ZxI!<65BTB?@VD#2#_GH6ylbpG@e&79s^<-Ilf@tn_7{su}3PIYE>yi~T_X zQPiRzomvD>=#>J0+YfW0?-7RZJ3%vIR@3j6*oOtuH&OwPz>(;&FdF06+~%V9MG*D- zP=Qo(i4>nSU!;sGg>|TimL{b0i2plYNV>sBWcSwt zba~^GPUPJ?9@~-R61^vW?XP2oHjUHSP)|pDLBxMFnOUrk2Gk-tG3+rxcwj& zpHtex>2S$X`0?>@n>FgBTw$sDRZX2iLzrz$!btdy=xHg{8~0`tYtwqEU;zJMxc;$F)hM zvpsub=!~obdl#O|G8$TWR{4HKt&Sjims;bFjQA!Pep>Lvi>2sz*yQ}<60}wTOk}W; zZx*hW;KCl_%ak(7DV7f>6jtJy*c7{c5 zo<>Q#4}S!8=Ds<77y-q0wMs#>>5b+VLUhoi=W(u&EplB^1$ZrR^Vcnx^J&H}!{x-j zz@kHAlAxGC*iIGfcfiPxp@XvErE8?wj^Ct`0wOOI_evfIsgH{CCw;@c2Q%!7vk5ecb43);4r~Kz;UGk&cw8u2c17OzyDyX(JNXPA`U!o3)aYwQ*B_hV-4#Cx7C}3p! z#TGO83|>n%a#51WD3dJ`x&5pG+}Z|fBAkyfuQYB}2`BK%J?x~GG3|cwyv~Aet1>kO z@NEZYa%>0f>1W{-W-ap^U>c9?v!8Nb{$|X7H~LWOA&bcUhL`iP$@+~)$RNdj(!my5 z^uj&8!`KGwI>^R4vXROu3R(NY&?DKtF}8Z*#p#%NDsG0d`s|lPqgWNr%<&iK56d?! z)|uCEj!!)R5C)ImIh~of8)flT6UowWO>3(J+%)-nZyzmj$K#snl?F-8t(vhg7O(Jv zqM2rt3lxab>)}4BSy`~7;tn*&wt6I2gt>m37`b#V-YOS!%e-f}K$!k&0BQ1tp1rOoS- ztGt94Ghv7z!bbERTkX&wnb)^1c=+`<*6W+rPo;n!Sq%?<#{_w?$6pYUrCrPQ$DpiF z=#iI_DdLv#rBAEPE&X!R^(%2)9fekWjp(|itK2AyqO01Fp<(#N7aiRjL0Md}_|9te z+_aoe-Qw@|>P3-G+0PBKwXbaoKD8nBi>0t8a0Ug8yHt2&c3iC^yt$#rqf>`md_Js{ zdLyTMy;5mFB1NTaPH4H>zlqu4Dt*QZ;eGa{6cfu>t1uaN0q=L4EN=|zkR*RuOPftIHo{%qUQ6ztS#3^Ngq&HLmI7{%H{}^v#jhHO$~G`V zvWOOKlE(H64^1-#uC!e;s6wIV8$!9Jk21yaN;g^?>YHjv;@|4{U+WbwAueH`o=b85 z2^sUy&#O_KPhcR|6qbH?`)e}sqB1%^3kHkCy1PSfH-#zv4bQWBszE2gH~SM7N-`2Z zoR6mBxf&Q0xGNytyZK$V$3SR?BO#uYo7t1;Oe_}etWf`(8+N-9dTb?{LC^Kv3^5nb zsJyE0hsIYeS4z?Men#Xi9%K{et#m9*`e)bZncHFiY&BQW$N&9m2ei?)MeK?|W%|<5|oa=#wr?#ocr3Z#-M!6QNcg9trM>-HOJ`aG*F9v7*Sp@=5hwj~V zT_Yhx_0Gh;I0vV(hg)usyF9{}GQU1Lr0@F@1^4v)%6Q<{j0id$L1c2wiyl)B&T+N(BCB-Yw~~stkl+Q|bOSamh$2LNH@_35x^gM}74!RQ(D^e9v#e1A zwyM_004^$!M1`Mt#Pabu2lgE*`ydh=Ni_aTEB!B!?=+26EQ~B@e!GshZRN7fzUXby zmJ1HhRU>{Y8Nu?`9QkPH42j=9A(HbYP4k^`oQQCG)Ad-^7H96(<~EZ+cm1+8ISaG| zyl8jP7ve+=T%{NOV0RJ}inVV62*1v*K$dXr)nnkwGde5FlYtj^2J=2zxwRgyvVQge z&`mtjs-UpVW^Rj?y^(~4Ahm{d1g1@gvK$Rin7_-OUBQi7MkxNmF)g0ZZ-r?}ETS>@ z;o|iCKIzU^ZTk&kk7m{;KFAj-I7K+mR_ljH|=*bDTd1 zHkerW=qnHQTYKvbL5`)IX7b2e9_BSaYDYINpXXhzq{=dZ@3PNB6WzbN_stRpJ&P=y zzPq_ZA@PB*!lJVFXg29Y@I7Ug0=M4Rvb@B)v)nm_g3LeVj-{%Wmg%1LvdO$zb$8$W zfaR7Z!Q~Z6dIG4G-3qXD5}<~wqk`NIEnnY*ORX}Fe~H4F?D7C=t3Ja1q$@;(n%b9p zRfjO(Dlp$I9C39bh}>>Tw;xADV4?$H(by4`N8>l}vl*{9MHR zny?DItWbZq(29=Skoo>*y8iFH55z+1~I$Ocdw`VF2krh^cT1^;hmp(SL$kN})bk_5`^d!SC zx)XFqBcBiS?hfHe>yGx?kQlHQii;Q3g9^TCzICJd#>91}jm+Ks&<6J5n?^PmH=S0b zhz$s*>c8;$d$Y^T_sPrcS4Mv^UZrGI)-G&fAY6SDpBTkVw+g>KbNf*cKDc?nSmQU^ zT3%ez6=Fs$Z4cY-z_UB}_Fo8*WZr`-M<1v|Nbo9J1|zeolKobCcIm#t()My1L?NX@ zf;W?V)O+)zS;gBS>}&;rS{irXLMygBc4hU`CzB=H6wxg2?Iykgx~pOOfrj8FY>{K> zik!`RQ(>WCTAlfgUfh^-Je~(%7Ome(DXRefi$5ZT+3%wJN;oP$WuCG=8%wQ$vXw(k zb@A`IRQ27EM&|MqkH#J>dOA4IlNxDC38Pqq;JrVAFoQrH5^@g8N{FLxWezv)EL%5z zR}260o3u8w>5jvR>3&Z^29nh5cfIB$znws`Esuq*F!A^RZBp|I9L4}yI#`}!ciSkA z9IA$z!imo@XR$_K;1~qFEJG>kWOa}%(Wmd8sVz%d#!-^@7MxfUr9#05)U@7hwk;%G z%nPyj_@3Gj@P&UdjbTZdtMoALT*_OKf}nmPeK^p}nP3sl6$6+7sBi1&LcFd#QWYF9 z)fO{|;gk~JPnOUkiDI-!sbPko(w3u@DU~I4g8IgdD4^2<`a!3pxQVYr-^Zdv2qmHl z8v6t76uTY@?8o5I%asu!DGAD!L9?!6oG)yf#T-n6 zZ&3Fr!Tx1OI+_+2^a5e-a9G>Qph@?nNE-5)2YR-W@COR;;7wFc>epDElU-(TJb|!y zys24SF?IwQfOIg6WPpl+5e7wbQ&%Cc8x1ufP{0()Uew}!6MuPzrbi6D&HV%Ch*KT! zjxzbRwVKVe>b`_irxde>4-}(ce7YU|ah0`btYtd+(#mk|Ep#fG5(^^bk=AcBCp2D0 z5}Q&YWYqJ(6QqlKt@G~XR+=eTUnIIufkZ6^L~|C7)saU>hd8_A-#wM6@A_3frt%9} zNOZw)P>OuFKBxwo@1sXqm#CRA7N73PBSq{Q`O*^$t(qd^g4FmB7kN)>S!Op^cz=`Z zExgFgq^35mNLJym{OFNswCD_jR7{)rmor zY$%5hnm(I35vu`V%gBlUmqc2u4n0Jje(ZgLw72|Kri8C#{dlp zim7r1lqnSwM`+)w2c?)XXUsyGbh15PZ^Q^Yme$y8c>W*UeD#t=RZJYsD** zIUzNF(9D7;Ftjhx8e~LzA3({Ax$Ti9Q2HYMcP{ZA_0lh2$GXyYYMPAlsQBR1aE((M z$l9X|dElMD-Grpa*HC*!1yjutqZQm+0Z8RcSZ@%f9c>Zq^)! zM6QoOC@1AwZ{_>jntjL$+(n>ed)`SfEhrf(GAz(V7}JI~358?~o@4$ASrT%f~B-p5+&OjZ~GrBP{{Rgtq1!RfLORdR3ToS6Rw2K|=J=RP*J-l*n9l^|oLPRh3iXB9@UG(m#Z^COt$x4&P#wZeAUE6{U}} z)N6{=(xOi)lMkF~FxU)xfQrJpS6RcEt@K+m5X}J-%}0u@=2|`OB->YXGtgwEZIV>)n1vkgD$Dp9jr z-F*4{WAxAWYuk%Lp{ZwwI$RLP+IpUtcX9N_1szq`MceKV;0wARgJvG<*v{G^xt#Ty z&d0v??;Na|T<8b3&c&niX*g<8Eqh-*xP1vq#{IxUr0!#mSh4Vk?q$iid&BZ>ws&=P zT6{JyZrU1$WYjP$u`>*8nU-s()29RQ-m1MWlvf02-pMZ3tq zoG3Uuw|$r(Ttq(0ceV^gO@wMbzW39UExIN2H22sVa%w9Vcu6;Pc1OdvL5Daj3ssPg zuX0so+YR&>_F)aN)Cu)UIIms=m!Gl;OsPXX&_PP8iuBm&48I)PI3^_ik|{M#V0)6q z)G@z!_~@B>c72!XReIKdK#-yg;@J*_wy0fR{EhjoblE><$e#lsky4zXhk2#{<1Vin zabJs?cW;H%le_QM4mEVUnRQ;L%W5N@ZgFi(teiiv*$rTTGPb+Od(ia)PzV%tCp?J+>y1RT9$1kQDq_1xH(D`_0AX#qLK+Kg`8Ss>u(YHx|q zMf%%vj{;vU*?&!FR6D7Kcgy=|!BW1{`3{Y|*j_u!4$SN%efsI3;O*(qtb@8|iUx~0 zAY>L~H+LFhgUh{@Qp9aaYN@}>O9Seq`qVSGaxjN=y_Qd|4^4FPyeJNn_7KIoXOt7! zKXXE^)dZ#O*D6lP@`U&Oxz^nXA-ygJKq6O0Ke6KBNFg~wMQ72*T`qa?TbEabAYJ+f zEg9IYf%&h5KOt4EOo98g&~dBinRZk{lI6X7T_>T<&dBNIMqQpyo>{q2*)#wLt>7#z z^riT7mci?O`GZ_yHW%BLe#dRoSs J)#~=K{|7a%wvPY+ literal 0 HcmV?d00001 diff --git a/frontend/src/models/dataConnector.js b/frontend/src/models/dataConnector.js index e0b3c0c3e..19fa5f912 100644 --- a/frontend/src/models/dataConnector.js +++ b/frontend/src/models/dataConnector.js @@ -60,6 +60,29 @@ const DataConnector = { }); }, }, + + confluence: { + collect: async function ({ pageUrl, username, accessToken }) { + return await fetch(`${API_BASE}/ext/confluence`, { + method: "POST", + headers: baseHeaders(), + body: JSON.stringify({ + pageUrl, + username, + accessToken, + }), + }) + .then((res) => res.json()) + .then((res) => { + if (!res.success) throw new Error(res.reason); + return { data: res.data, error: null }; + }) + .catch((e) => { + console.error(e); + return { data: null, error: e.message }; + }); + }, + }, }; export default DataConnector; diff --git a/server/endpoints/extensions/index.js b/server/endpoints/extensions/index.js index bf07ec56c..07eb7130d 100644 --- a/server/endpoints/extensions/index.js +++ b/server/endpoints/extensions/index.js @@ -71,6 +71,28 @@ function extensionEndpoints(app) { } } ); + + app.post( + "/ext/confluence", + [validatedRequest, flexUserRoleValid([ROLES.admin, ROLES.manager])], + async (request, response) => { + try { + const responseFromProcessor = + await new CollectorApi().forwardExtensionRequest({ + endpoint: "/ext/confluence", + method: "POST", + body: request.body, + }); + await Telemetry.sendTelemetry("extension_invoked", { + type: "confluence", + }); + response.status(200).json(responseFromProcessor); + } catch (e) { + console.error(e); + response.sendStatus(500).end(); + } + } + ); } module.exports = { extensionEndpoints }; From 331423f86409b373e2766d319c1e71f365e6e1ca Mon Sep 17 00:00:00 2001 From: timothycarambat Date: Thu, 25 Apr 2024 17:54:14 -0700 Subject: [PATCH 03/18] patch Citation --- .../WorkspaceChat/ChatContainer/ChatHistory/Citation/index.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/Citation/index.jsx b/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/Citation/index.jsx index 7105901d3..a3a579c95 100644 --- a/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/Citation/index.jsx +++ b/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/Citation/index.jsx @@ -1,4 +1,4 @@ -import React, { memo, useState } from "react"; +import { memo, useState } from "react"; import { v4 } from "uuid"; import { decode as HTMLDecode } from "he"; import truncate from "truncate"; From 2e813846dca07f363af9914950526f7507292609 Mon Sep 17 00:00:00 2001 From: Timothy Carambat Date: Fri, 26 Apr 2024 11:18:55 -0700 Subject: [PATCH 04/18] Agent skill: chart generation (#1103) * WIP agent support * move agent folder * wip frontend socket * checkpoint * fix schema * Checkpoint for plugins and AgentHandler * refactor plugins and agent arch * agent error reporting and handling * add frontend elements for agents in prompt input * WIP integrations for agents * enable web-search agent config from frontend * persist chat history * update alert * update migration remove console logs update close state for agent invocations * add examples to dockerignore Extract statusResponse to its own component * update close method * wrap scraping rejections * add RAG search as funciton * Add telem and link highlight * chat support * patch memory * Add rechart as a plugin option * Toggles for abilites of default agent (system wide) Validate values for agent skills Enable dynamic loading of skills UI for toggle of skills * add UI for toggle of configs for agent * toggle WS or WSS protocol * update NGNIX proxy pass * move components around and capture failed websocket creation * fix name * tmp docker image * reset workflow * safety mark functions * telem on tool calls * remove hardcode short circuit * separate web-browser from scrape * extract summarizer to util add abort handlers and controller for langchain stuff so socket close kills process * langchain summarize verbose when in dev * chart styling improvements + add title to chart * fix legend from being cutoff in chart downloads * remove cursor blink --------- Co-authored-by: shatfield4 --- .vscode/settings.json | 3 + frontend/package.json | 3 + .../ChatHistory/Chartable/CustomCell.jsx | 50 ++ .../ChatHistory/Chartable/CustomTooltip.jsx | 89 ++++ .../ChatHistory/Chartable/chart-utils.js | 98 ++++ .../ChatHistory/Chartable/index.jsx | 467 ++++++++++++++++++ .../ChatHistory/PromptReply/index.jsx | 2 +- .../ChatContainer/ChatHistory/index.jsx | 7 + .../PromptInput/AgentMenu/index.jsx | 1 + frontend/src/index.css | 13 + .../WorkspaceSettings/AgentConfig/index.jsx | 10 +- frontend/src/utils/chat/agent.js | 20 + frontend/tailwind.config.js | 34 +- frontend/yarn.lock | 390 ++++++++++++++- .../agents/aibitat/plugins/chat-history.js | 35 ++ server/utils/agents/aibitat/plugins/index.js | 3 + .../utils/agents/aibitat/plugins/rechart.js | 109 ++++ server/utils/agents/aibitat/utils/dedupe.js | 33 ++ server/utils/helpers/chat/responses.js | 1 + 19 files changed, 1361 insertions(+), 7 deletions(-) create mode 100644 frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/Chartable/CustomCell.jsx create mode 100644 frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/Chartable/CustomTooltip.jsx create mode 100644 frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/Chartable/chart-utils.js create mode 100644 frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/Chartable/index.jsx create mode 100644 server/utils/agents/aibitat/plugins/rechart.js diff --git a/.vscode/settings.json b/.vscode/settings.json index 7d17c99ee..e9344f9f5 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -5,7 +5,10 @@ "aibitat", "anythingllm", "Astra", + "Chartable", "comkey", + "cooldown", + "cooldowns", "Deduplicator", "Dockerized", "Embeddable", diff --git a/frontend/package.json b/frontend/package.json index 4eaa89dc5..ded06aa9c 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -14,6 +14,7 @@ "@metamask/jazzicon": "^2.0.0", "@microsoft/fetch-event-source": "^2.0.1", "@phosphor-icons/react": "^2.0.13", + "@tremor/react": "^3.15.1", "dompurify": "^3.0.8", "file-saver": "^2.0.5", "he": "^1.2.0", @@ -30,6 +31,8 @@ "react-tag-input-component": "^2.0.2", "react-toastify": "^9.1.3", "react-tooltip": "^5.25.2", + "recharts": "^2.12.5", + "recharts-to-png": "^2.3.1", "text-case": "^1.0.9", "truncate": "^3.0.0", "uuid": "^9.0.0" diff --git a/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/Chartable/CustomCell.jsx b/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/Chartable/CustomCell.jsx new file mode 100644 index 000000000..b0e94390a --- /dev/null +++ b/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/Chartable/CustomCell.jsx @@ -0,0 +1,50 @@ +export default function CustomCell({ ...props }) { + const { + root, + depth, + x, + y, + width, + height, + index, + payload, + colors, + rank, + name, + } = props; + return ( + + + {depth === 1 ? ( + + {name} + + ) : null} + {depth === 1 ? ( + + {index + 1} + + ) : null} + + ); +} diff --git a/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/Chartable/CustomTooltip.jsx b/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/Chartable/CustomTooltip.jsx new file mode 100644 index 000000000..b2002e23f --- /dev/null +++ b/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/Chartable/CustomTooltip.jsx @@ -0,0 +1,89 @@ +import { Tooltip as RechartsTooltip } from "recharts"; + +// Given a hex, convert to the opposite highest-contrast color +// and if `bw` is enabled, force it to be black/white to normalize +// interface. +function invertColor(hex, bw) { + if (hex.indexOf("#") === 0) { + hex = hex.slice(1); + } + // convert 3-digit hex to 6-digits. + if (hex.length === 3) { + hex = hex[0] + hex[0] + hex[1] + hex[1] + hex[2] + hex[2]; + } + if (hex.length !== 6) { + throw new Error("Invalid HEX color."); + } + var r = parseInt(hex.slice(0, 2), 16), + g = parseInt(hex.slice(2, 4), 16), + b = parseInt(hex.slice(4, 6), 16); + if (bw) { + // https://stackoverflow.com/a/3943023/112731 + return r * 0.299 + g * 0.587 + b * 0.114 > 186 ? "#FFFFFF" : "#000000"; + // : '#FFFFFF'; + } + // invert color components + r = (255 - r).toString(16); + g = (255 - g).toString(16); + b = (255 - b).toString(16); + // pad each with zeros and return + return "#" + padZero(r) + padZero(g) + padZero(b); +} + +function padZero(str, len) { + len = len || 2; + var zeros = new Array(len).join("0"); + return (zeros + str).slice(-len); +} + +export default function Tooltip({ legendColor, ...props }) { + return ( + { + return active && payload ? ( +
+
+

{label}

+
+
+ {payload.map(({ value, name }, idx) => ( +
+
+ +

+ {value} +

+
+

+ {name} +

+
+ ))} +
+
+ ) : null; + }} + /> + ); +} diff --git a/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/Chartable/chart-utils.js b/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/Chartable/chart-utils.js new file mode 100644 index 000000000..53e501f25 --- /dev/null +++ b/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/Chartable/chart-utils.js @@ -0,0 +1,98 @@ +export const Colors = { + blue: "#3b82f6", + sky: "#0ea5e9", + cyan: "#06b6d4", + teal: "#14b8a6", + emerald: "#10b981", + green: "#22c55e", + lime: "#84cc16", + yellow: "#eab308", + amber: "#f59e0b", + orange: "#f97316", + red: "#ef4444", + rose: "#f43f5e", + pink: "#ec4899", + fuchsia: "#d946ef", + purple: "#a855f7", + violet: "#8b5cf6", + indigo: "#6366f1", + neutral: "#737373", + stone: "#78716c", + gray: "#6b7280", + slate: "#64748b", + zinc: "#71717a", +}; + +export function getTremorColor(color) { + switch (color) { + case "blue": + return Colors.blue; + case "sky": + return Colors.sky; + case "cyan": + return Colors.cyan; + case "teal": + return Colors.teal; + case "emerald": + return Colors.emerald; + case "green": + return Colors.green; + case "lime": + return Colors.lime; + case "yellow": + return Colors.yellow; + case "amber": + return Colors.amber; + case "orange": + return Colors.orange; + case "red": + return Colors.red; + case "rose": + return Colors.rose; + case "pink": + return Colors.pink; + case "fuchsia": + return Colors.fuchsia; + case "purple": + return Colors.purple; + case "violet": + return Colors.violet; + case "indigo": + return Colors.indigo; + case "neutral": + return Colors.neutral; + case "stone": + return Colors.stone; + case "gray": + return Colors.gray; + case "slate": + return Colors.slate; + case "zinc": + return Colors.zinc; + } +} + +export const themeColorRange = [ + "slate", + "gray", + "zinc", + "neutral", + "stone", + "red", + "orange", + "amber", + "yellow", + "lime", + "green", + "emerald", + "teal", + "cyan", + "sky", + "blue", + "indigo", + "violet", + "purple", + "fuchsia", + "pink", + "rose", +]; diff --git a/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/Chartable/index.jsx b/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/Chartable/index.jsx new file mode 100644 index 000000000..8217fe95b --- /dev/null +++ b/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/Chartable/index.jsx @@ -0,0 +1,467 @@ +import { v4 } from "uuid"; +import { + AreaChart, + BarChart, + DonutChart, + Legend, + LineChart, +} from "@tremor/react"; +import { + Bar, + CartesianGrid, + ComposedChart, + Funnel, + FunnelChart, + Line, + PolarAngleAxis, + PolarGrid, + PolarRadiusAxis, + Radar, + RadarChart, + RadialBar, + RadialBarChart, + Scatter, + ScatterChart, + Treemap, + XAxis, + YAxis, +} from "recharts"; +import { Colors, getTremorColor } from "./chart-utils.js"; +import CustomCell from "./CustomCell.jsx"; +import Tooltip from "./CustomTooltip.jsx"; +import { safeJsonParse } from "@/utils/request.js"; +import renderMarkdown from "@/utils/chat/markdown.js"; +import { WorkspaceProfileImage } from "../PromptReply/index.jsx"; +import { memo, useCallback, useState } from "react"; +import { saveAs } from "file-saver"; +import { useGenerateImage } from "recharts-to-png"; +import { CircleNotch, DownloadSimple } from "@phosphor-icons/react"; + +const dataFormatter = (number) => { + return Intl.NumberFormat("us").format(number).toString(); +}; + +export function Chartable({ props, workspace }) { + const [getDivJpeg, { ref }] = useGenerateImage({ + quality: 1, + type: "image/jpeg", + options: { + backgroundColor: "#393d43", + padding: 20, + }, + }); + const handleDownload = useCallback(async () => { + const jpeg = await getDivJpeg(); + if (jpeg) saveAs(jpeg, `chart-${v4().split("-")[0]}.jpg`); + }, []); + + const color = null; + const showLegend = true; + const content = + typeof props.content === "string" + ? safeJsonParse(props.content, null) + : props.content; + if (content === null) return null; + + const chartType = content?.type?.toLowerCase(); + const data = + typeof content.dataset === "string" + ? safeJsonParse(content.dataset, null) + : content.dataset; + const value = data.length > 0 ? Object.keys(data[0])[1] : "value"; + const title = content?.title; + + const renderChart = () => { + switch (chartType) { + case "area": + return ( +
+

{title}

+ +
+ ); + case "bar": + return ( +
+

{title}

+ +
+ ); + case "line": + return ( +
+

{title}

+ +
+ ); + case "composed": + return ( +
+

{title}

+ {showLegend && ( + + )} + + + + + + + + +
+ ); + case "scatter": + return ( +
+

{title}

+ {showLegend && ( +
+ +
+ )} + + + + + + + +
+ ); + case "pie": + return ( +
+

{title}

+ +
+ ); + case "radar": + return ( +
+

{title}

+ {showLegend && ( +
+ +
+ )} + + + + + + + +
+ ); + case "radialbar": + return ( +
+

{title}

+ {showLegend && ( +
+ +
+ )} + + + + +
+ ); + case "treemap": + return ( +
+

{title}

+ {showLegend && ( +
+ +
+ )} + } + > + + +
+ ); + case "funnel": + return ( +
+

{title}

+ {showLegend && ( +
+ +
+ )} + + + + +
+ ); + default: + return

Unsupported chart type.

; + } + }; + + if (!!props.chatId) { + return ( +
+
+
+ +
+ +
{renderChart()}
+ +
+
+
+
+ ); + } + + return ( +
+
+
+ +
{renderChart()}
+
+
+ +
+
+
+ ); +} + +const customTooltip = (props) => { + const { payload, active } = props; + if (!active || !payload) return null; + const categoryPayload = payload?.[0]; + if (!categoryPayload) return null; + return ( +
+
+
+
+
+

+ {categoryPayload.name} +

+

+ {categoryPayload.value} +

+
+
+
+
+ ); +}; + +function DownloadGraph({ onClick }) { + const [loading, setLoading] = useState(false); + const handleClick = async () => { + setLoading(true); + await onClick?.(); + setLoading(false); + }; + + return ( +
+
+
+ {loading ? ( + + ) : ( + + )} +
+
+
+ ); +} + +export default memo(Chartable); diff --git a/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/PromptReply/index.jsx b/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/PromptReply/index.jsx index 98d41a11c..858c773ea 100644 --- a/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/PromptReply/index.jsx +++ b/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/PromptReply/index.jsx @@ -71,7 +71,7 @@ const PromptReply = ({ ); }; -function WorkspaceProfileImage({ workspace }) { +export function WorkspaceProfileImage({ workspace }) { if (!!workspace.pfpUrl) { return (
diff --git a/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/index.jsx b/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/index.jsx index 902409e49..c0eb5bf4c 100644 --- a/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/index.jsx +++ b/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/index.jsx @@ -6,6 +6,7 @@ import ManageWorkspace from "../../../Modals/MangeWorkspace"; import { ArrowDown } from "@phosphor-icons/react"; import debounce from "lodash.debounce"; import useUser from "@/hooks/useUser"; +import Chartable from "./Chartable"; export default function ChatHistory({ history = [], workspace, sendCommand }) { const { user } = useUser(); @@ -133,6 +134,12 @@ export default function ChatHistory({ history = [], workspace, sendCommand }) { return ; } + if (props.type === "rechartVisualize" && !!props.content) { + return ( + + ); + } + if (isLastBotReply && props.animate) { return ( +
diff --git a/frontend/src/index.css b/frontend/src/index.css index 9241d6184..e5066f679 100644 --- a/frontend/src/index.css +++ b/frontend/src/index.css @@ -679,3 +679,16 @@ does not extend the close button beyond the viewport. */ .white-scrollbar::-webkit-scrollbar-thumb:hover { background-color: #cccccc; } + +/* Recharts rendering styles */ +.recharts-text > * { + fill: #fff; +} + +.recharts-legend-wrapper { + margin-bottom: 10px; +} + +.text-tremor-content { + padding-bottom: 10px; +} diff --git a/frontend/src/pages/WorkspaceSettings/AgentConfig/index.jsx b/frontend/src/pages/WorkspaceSettings/AgentConfig/index.jsx index c5ba61f27..c96cc1246 100644 --- a/frontend/src/pages/WorkspaceSettings/AgentConfig/index.jsx +++ b/frontend/src/pages/WorkspaceSettings/AgentConfig/index.jsx @@ -170,7 +170,7 @@ function AvailableAgentSkills({ skills, settings, toggleAgentSkill }) { disabled={true} /> + { + return [ + ...prev.filter((msg) => !!msg.content), + { + type: "rechartVisualize", + uuid: v4(), + content: data.content, + role: "assistant", + sources: [], + closed: true, + error: null, + animate: false, + pending: false, + }, + ]; + }); + } + if (data.type === "wssFailure") { return setChatHistory((prev) => { return [ diff --git a/frontend/tailwind.config.js b/frontend/tailwind.config.js index b0ac87c90..cd9e1ed1f 100644 --- a/frontend/tailwind.config.js +++ b/frontend/tailwind.config.js @@ -1,5 +1,6 @@ /** @type {import('tailwindcss').Config} */ export default { + darkMode: 'false', content: { relative: true, files: [ @@ -9,7 +10,8 @@ export default { "./src/pages/**/*.{js,jsx}", "./src/utils/**/*.js", "./src/*.jsx", - "./index.html" + "./index.html", + './node_modules/@tremor/**/*.{js,ts,jsx,tsx}' ] }, theme: { @@ -86,5 +88,35 @@ export default { } } }, + // Required for rechart styles to show since they can be rendered dynamically and will be tree-shaken if not safe-listed. + safelist: [ + { + pattern: + /^(bg-(?:slate|gray|zinc|neutral|stone|red|orange|amber|yellow|lime|green|emerald|teal|cyan|sky|blue|indigo|violet|purple|fuchsia|pink|rose)-(?:50|100|200|300|400|500|600|700|800|900|950))$/, + variants: ['hover', 'ui-selected'], + }, + { + pattern: + /^(text-(?:slate|gray|zinc|neutral|stone|red|orange|amber|yellow|lime|green|emerald|teal|cyan|sky|blue|indigo|violet|purple|fuchsia|pink|rose)-(?:50|100|200|300|400|500|600|700|800|900|950))$/, + variants: ['hover', 'ui-selected'], + }, + { + pattern: + /^(border-(?:slate|gray|zinc|neutral|stone|red|orange|amber|yellow|lime|green|emerald|teal|cyan|sky|blue|indigo|violet|purple|fuchsia|pink|rose)-(?:50|100|200|300|400|500|600|700|800|900|950))$/, + variants: ['hover', 'ui-selected'], + }, + { + pattern: + /^(ring-(?:slate|gray|zinc|neutral|stone|red|orange|amber|yellow|lime|green|emerald|teal|cyan|sky|blue|indigo|violet|purple|fuchsia|pink|rose)-(?:50|100|200|300|400|500|600|700|800|900|950))$/, + }, + { + pattern: + /^(stroke-(?:slate|gray|zinc|neutral|stone|red|orange|amber|yellow|lime|green|emerald|teal|cyan|sky|blue|indigo|violet|purple|fuchsia|pink|rose)-(?:50|100|200|300|400|500|600|700|800|900|950))$/, + }, + { + pattern: + /^(fill-(?:slate|gray|zinc|neutral|stone|red|orange|amber|yellow|lime|green|emerald|teal|cyan|sky|blue|indigo|violet|purple|fuchsia|pink|rose)-(?:50|100|200|300|400|500|600|700|800|900|950))$/, + }, + ], plugins: [] } diff --git a/frontend/yarn.lock b/frontend/yarn.lock index 4cd24e725..bd12e9fa3 100644 --- a/frontend/yarn.lock +++ b/frontend/yarn.lock @@ -184,6 +184,13 @@ dependencies: "@babel/helper-plugin-utils" "^7.22.5" +"@babel/runtime@^7.21.0", "@babel/runtime@^7.5.5", "@babel/runtime@^7.8.7": + version "7.24.4" + resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.24.4.tgz#de795accd698007a66ba44add6cc86542aff1edd" + integrity sha512-dkxf7+hn8mFBwKjs9bvBlArzLVxVbS8usaPUDd5p2a9JCL9tB8OaOVN1isD4+Xyk4ns89/xeOmbQvgdK7IIVdA== + dependencies: + regenerator-runtime "^0.14.0" + "@babel/template@^7.22.15": version "7.22.15" resolved "https://registry.yarnpkg.com/@babel/template/-/template-7.22.15.tgz#09576efc3830f0430f4548ef971dde1350ef2f38" @@ -365,6 +372,13 @@ resolved "https://registry.yarnpkg.com/@eslint/js/-/js-8.53.0.tgz#bea56f2ed2b5baea164348ff4d5a879f6f81f20d" integrity sha512-Kn7K8dx/5U6+cT1yEhpX1w4PCSg0M+XyRILPgvwcEBjerFWCwQj5sbr3/VmxqV0JGHCBCzyd6LxypEuehypY1w== +"@floating-ui/core@^1.0.0": + version "1.6.0" + resolved "https://registry.yarnpkg.com/@floating-ui/core/-/core-1.6.0.tgz#fa41b87812a16bf123122bf945946bae3fdf7fc1" + integrity sha512-PcF++MykgmTj3CIyOQbKA/hDzOAiqI3mhuoN44WRCopIs1sgoDoU4oty4Jtqaj/y3oDU6fnVSm4QG0a3t5i0+g== + dependencies: + "@floating-ui/utils" "^0.2.1" + "@floating-ui/core@^1.5.3": version "1.5.3" resolved "https://registry.yarnpkg.com/@floating-ui/core/-/core-1.5.3.tgz#b6aa0827708d70971c8679a16cf680a515b8a52a" @@ -380,11 +394,48 @@ "@floating-ui/core" "^1.5.3" "@floating-ui/utils" "^0.2.0" -"@floating-ui/utils@^0.2.0": +"@floating-ui/dom@^1.2.1": + version "1.6.3" + resolved "https://registry.yarnpkg.com/@floating-ui/dom/-/dom-1.6.3.tgz#954e46c1dd3ad48e49db9ada7218b0985cee75ef" + integrity sha512-RnDthu3mzPlQ31Ss/BTwQ1zjzIhr3lk1gZB1OC56h/1vEtaXkESrOqL5fQVMfXpwGtRwX+YsZBdyHtJMQnkArw== + dependencies: + "@floating-ui/core" "^1.0.0" + "@floating-ui/utils" "^0.2.0" + +"@floating-ui/react-dom@^1.3.0": + version "1.3.0" + resolved "https://registry.yarnpkg.com/@floating-ui/react-dom/-/react-dom-1.3.0.tgz#4d35d416eb19811c2b0e9271100a6aa18c1579b3" + integrity sha512-htwHm67Ji5E/pROEAr7f8IKFShuiCKHwUC/UY4vC3I5jiSvGFAYnSYiZO5MlGmads+QqvUkR9ANHEguGrDv72g== + dependencies: + "@floating-ui/dom" "^1.2.1" + +"@floating-ui/react@^0.19.2": + version "0.19.2" + resolved "https://registry.yarnpkg.com/@floating-ui/react/-/react-0.19.2.tgz#c6e4d2097ed0dca665a7c042ddf9cdecc95e9412" + integrity sha512-JyNk4A0Ezirq8FlXECvRtQOX/iBe5Ize0W/pLkrZjfHW9GUV7Xnq6zm6fyZuQzaHHqEnVizmvlA96e1/CkZv+w== + dependencies: + "@floating-ui/react-dom" "^1.3.0" + aria-hidden "^1.1.3" + tabbable "^6.0.1" + +"@floating-ui/utils@^0.2.0", "@floating-ui/utils@^0.2.1": version "0.2.1" resolved "https://registry.yarnpkg.com/@floating-ui/utils/-/utils-0.2.1.tgz#16308cea045f0fc777b6ff20a9f25474dd8293d2" integrity sha512-9TANp6GPoMtYzQdt54kfAyMmz1+osLlXdg2ENroU7zzrtflTLrrC/lgrIfaSe+Wu0b89GKccT7vxXA0MoAIO+Q== +"@headlessui/react@^1.7.18": + version "1.7.18" + resolved "https://registry.yarnpkg.com/@headlessui/react/-/react-1.7.18.tgz#30af4634d2215b2ca1aa29d07f33d02bea82d9d7" + integrity sha512-4i5DOrzwN4qSgNsL4Si61VMkUcWbcSKueUV7sFhpHzQcSShdlHENE5+QBntMSRvHt8NyoFO2AGG8si9lq+w4zQ== + dependencies: + "@tanstack/react-virtual" "^3.0.0-beta.60" + client-only "^0.0.1" + +"@headlessui/tailwindcss@^0.2.0": + version "0.2.0" + resolved "https://registry.yarnpkg.com/@headlessui/tailwindcss/-/tailwindcss-0.2.0.tgz#2c55c98fd8eee4b4f21ec6eb35a014b840059eec" + integrity sha512-fpL830Fln1SykOCboExsWr3JIVeQKieLJ3XytLe/tt1A0XzqUthOftDmjcCYLW62w7mQI7wXcoPXr3tZ9QfGxw== + "@humanwhocodes/config-array@^0.11.13": version "0.11.13" resolved "https://registry.yarnpkg.com/@humanwhocodes/config-array/-/config-array-0.11.13.tgz#075dc9684f40a531d9b26b0822153c1e832ee297" @@ -492,6 +543,32 @@ resolved "https://registry.yarnpkg.com/@remix-run/router/-/router-1.14.1.tgz#6d2dd03d52e604279c38911afc1079d58c50a755" integrity sha512-Qg4DMQsfPNAs88rb2xkdk03N3bjK4jgX5fR24eHCTR9q6PrhZQZ4UJBPzCHJkIpTRN1UKxx2DzjZmnC+7Lj0Ow== +"@tanstack/react-virtual@^3.0.0-beta.60": + version "3.2.1" + resolved "https://registry.yarnpkg.com/@tanstack/react-virtual/-/react-virtual-3.2.1.tgz#58ac9af23ff08b5f05a6dfe6a59deac2f9451508" + integrity sha512-i9Nt0ssIh2bSjomJZlr6Iq5usT/9+ewo2/fKHRNk6kjVKS8jrhXbnO8NEawarCuBx/efv0xpoUUKKGxa0cQb4Q== + dependencies: + "@tanstack/virtual-core" "3.2.1" + +"@tanstack/virtual-core@3.2.1": + version "3.2.1" + resolved "https://registry.yarnpkg.com/@tanstack/virtual-core/-/virtual-core-3.2.1.tgz#b3e4214b8f462054501d80e8777068faa139bd06" + integrity sha512-nO0d4vRzsmpBQCJYyClNHPPoUMI4nXNfrm6IcCRL33ncWMoNVpURh9YebEHPw8KrtsP2VSJIHE4gf4XFGk1OGg== + +"@tremor/react@^3.15.1": + version "3.15.1" + resolved "https://registry.yarnpkg.com/@tremor/react/-/react-3.15.1.tgz#a9c10887bd067ffe0e18ca763e425db057f3722f" + integrity sha512-vCUqgYo993VePn6yOs4102ibY2XYcDDp7I1ZV/+i5hdfp+XgsHyQvYeixQcETBMpcajwM8E8NOOO7k9ANLkrrw== + dependencies: + "@floating-ui/react" "^0.19.2" + "@headlessui/react" "^1.7.18" + "@headlessui/tailwindcss" "^0.2.0" + date-fns "^2.30.0" + react-day-picker "^8.9.1" + react-transition-state "^2.1.1" + recharts "^2.10.3" + tailwind-merge "^1.14.0" + "@types/babel__core@^7.20.3": version "7.20.3" resolved "https://registry.yarnpkg.com/@types/babel__core/-/babel__core-7.20.3.tgz#d5625a50b6f18244425a1359a858c73d70340778" @@ -525,6 +602,57 @@ dependencies: "@babel/types" "^7.20.7" +"@types/d3-array@^3.0.3": + version "3.2.1" + resolved "https://registry.yarnpkg.com/@types/d3-array/-/d3-array-3.2.1.tgz#1f6658e3d2006c4fceac53fde464166859f8b8c5" + integrity sha512-Y2Jn2idRrLzUfAKV2LyRImR+y4oa2AntrgID95SHJxuMUrkNXmanDSed71sRNZysveJVt1hLLemQZIady0FpEg== + +"@types/d3-color@*": + version "3.1.3" + resolved "https://registry.yarnpkg.com/@types/d3-color/-/d3-color-3.1.3.tgz#368c961a18de721da8200e80bf3943fb53136af2" + integrity sha512-iO90scth9WAbmgv7ogoq57O9YpKmFBbmoEoCHDB2xMBY0+/KVrqAaCDyCE16dUspeOvIxFFRI+0sEtqDqy2b4A== + +"@types/d3-ease@^3.0.0": + version "3.0.2" + resolved "https://registry.yarnpkg.com/@types/d3-ease/-/d3-ease-3.0.2.tgz#e28db1bfbfa617076f7770dd1d9a48eaa3b6c51b" + integrity sha512-NcV1JjO5oDzoK26oMzbILE6HW7uVXOHLQvHshBUW4UMdZGfiY6v5BeQwh9a9tCzv+CeefZQHJt5SRgK154RtiA== + +"@types/d3-interpolate@^3.0.1": + version "3.0.4" + resolved "https://registry.yarnpkg.com/@types/d3-interpolate/-/d3-interpolate-3.0.4.tgz#412b90e84870285f2ff8a846c6eb60344f12a41c" + integrity sha512-mgLPETlrpVV1YRJIglr4Ez47g7Yxjl1lj7YKsiMCb27VJH9W8NVM6Bb9d8kkpG/uAQS5AmbA48q2IAolKKo1MA== + dependencies: + "@types/d3-color" "*" + +"@types/d3-path@*": + version "3.1.0" + resolved "https://registry.yarnpkg.com/@types/d3-path/-/d3-path-3.1.0.tgz#2b907adce762a78e98828f0b438eaca339ae410a" + integrity sha512-P2dlU/q51fkOc/Gfl3Ul9kicV7l+ra934qBFXCFhrZMOL6du1TM0pm1ThYvENukyOn5h9v+yMJ9Fn5JK4QozrQ== + +"@types/d3-scale@^4.0.2": + version "4.0.8" + resolved "https://registry.yarnpkg.com/@types/d3-scale/-/d3-scale-4.0.8.tgz#d409b5f9dcf63074464bf8ddfb8ee5a1f95945bb" + integrity sha512-gkK1VVTr5iNiYJ7vWDI+yUFFlszhNMtVeneJ6lUTKPjprsvLLI9/tgEGiXJOnlINJA8FyA88gfnQsHbybVZrYQ== + dependencies: + "@types/d3-time" "*" + +"@types/d3-shape@^3.1.0": + version "3.1.6" + resolved "https://registry.yarnpkg.com/@types/d3-shape/-/d3-shape-3.1.6.tgz#65d40d5a548f0a023821773e39012805e6e31a72" + integrity sha512-5KKk5aKGu2I+O6SONMYSNflgiP0WfZIQvVUMan50wHsLG1G94JlxEVnCpQARfTtzytuY0p/9PXXZb3I7giofIA== + dependencies: + "@types/d3-path" "*" + +"@types/d3-time@*", "@types/d3-time@^3.0.0": + version "3.0.3" + resolved "https://registry.yarnpkg.com/@types/d3-time/-/d3-time-3.0.3.tgz#3c186bbd9d12b9d84253b6be6487ca56b54f88be" + integrity sha512-2p6olUZ4w3s+07q3Tm2dbiMZy5pCDfYwtLXXHUnVzXgQlZ/OyPtUz6OL382BkOuGlLXqfT+wqv8Fw2v8/0geBw== + +"@types/d3-timer@^3.0.0": + version "3.0.2" + resolved "https://registry.yarnpkg.com/@types/d3-timer/-/d3-timer-3.0.2.tgz#70bbda77dc23aa727413e22e214afa3f0e852f70" + integrity sha512-Ps3T8E8dZDam6fUyNiMkekK3XUsaUEik+idO9/YjPtfj2qruF8tFBXS7XhtE4iIXBLxhmLjP3SXpLhVf21I9Lw== + "@types/history@^4.7.11": version "4.7.11" resolved "https://registry.yarnpkg.com/@types/history/-/history-4.7.11.tgz#56588b17ae8f50c53983a524fc3cc47437969d64" @@ -651,6 +779,13 @@ argparse@^2.0.1: resolved "https://registry.yarnpkg.com/argparse/-/argparse-2.0.1.tgz#246f50f3ca78a3240f6c997e8a9bd1eac49e4b38" integrity sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q== +aria-hidden@^1.1.3: + version "1.2.4" + resolved "https://registry.yarnpkg.com/aria-hidden/-/aria-hidden-1.2.4.tgz#b78e383fdbc04d05762c78b4a25a501e736c4522" + integrity sha512-y+CcFFwelSXpLZk/7fMB2mUbGtX9lKycf1MWJ7CaTIERyitVlyQx6C+sxcROU2BAJ24OiZyK+8wj2i8AlBoS3A== + dependencies: + tslib "^2.0.0" + array-buffer-byte-length@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/array-buffer-byte-length/-/array-buffer-byte-length-1.0.0.tgz#fabe8bc193fea865f317fe7807085ee0dee5aead" @@ -748,6 +883,11 @@ balanced-match@^1.0.0: resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee" integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw== +base64-arraybuffer@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/base64-arraybuffer/-/base64-arraybuffer-1.0.2.tgz#1c37589a7c4b0746e34bd1feb951da2df01c1bdc" + integrity sha512-I3yl4r9QB5ZRY3XuJVEPfc2XhZO6YweFPI+UovAzn+8/hb3oJ6lnysaFcjVpkCPfVWFUDvoZ8kmVDP7WyRtYtQ== + base64-js@^1.3.1: version "1.5.1" resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" @@ -871,6 +1011,11 @@ classnames@^2.3.0: resolved "https://registry.yarnpkg.com/classnames/-/classnames-2.5.1.tgz#ba774c614be0f016da105c858e7159eae8e7687b" integrity sha512-saHYOzhIQs6wy2sVxTM6bUDsQO4F50V9RQ22qBpEdCW+I+/Wmke2HOl6lS6dTpdxVhb88/I6+Hs+438c3lfUow== +client-only@^0.0.1: + version "0.0.1" + resolved "https://registry.yarnpkg.com/client-only/-/client-only-0.0.1.tgz#38bba5d403c41ab150bff64a95c85013cf73bca1" + integrity sha512-IV3Ou0jSMzZrd3pZ48nLkT9DA7Ag1pnPzaiQhpW7c3RbcqqzvzzVu+L8gfqMp/8IM2MQtSiqaCxrrcfu8I8rMA== + cliui@^8.0.1: version "8.0.1" resolved "https://registry.yarnpkg.com/cliui/-/cliui-8.0.1.tgz#0c04b075db02cbfe60dc8e6cf2f5486b1a3608aa" @@ -890,6 +1035,11 @@ clsx@^1.1.1: resolved "https://registry.yarnpkg.com/clsx/-/clsx-1.2.1.tgz#0ddc4a20a549b59c93a4116bb26f5294ca17dc12" integrity sha512-EcR6r5a8bj6pu3ycsa/E/cKVGuTgZJZdsyUYHOksG/UHIiKfjxzRxYJpyVBwYaQeOvghal9fcc4PidlgzugAQg== +clsx@^2.0.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/clsx/-/clsx-2.1.0.tgz#e851283bcb5c80ee7608db18487433f7b23f77cb" + integrity sha512-m3iNNWpd9rl3jvvcBnu70ylMdrXt8Vlq4HYadnU5fwcOtvkSQWPmj7amUcDT2qYI7risszBjI5AUIUox9D16pg== + color-convert@^1.3.0, color-convert@^1.9.0: version "1.9.3" resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8" @@ -954,6 +1104,13 @@ cross-spawn@^7.0.2, cross-spawn@^7.0.3: shebang-command "^2.0.0" which "^2.0.1" +css-line-break@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/css-line-break/-/css-line-break-2.1.0.tgz#bfef660dfa6f5397ea54116bb3cb4873edbc4fa0" + integrity sha512-FHcKFCZcAha3LwfVBhCQbW2nCNbkZXn7KVUJcsT5/P8YmfsVja0FMPJr0B903j/E69HUphKiV9iQArX8SDYA4w== + dependencies: + utrie "^1.0.2" + cssesc@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/cssesc/-/cssesc-3.0.0.tgz#37741919903b868565e1c09ea747445cd18983ee" @@ -964,6 +1121,84 @@ csstype@^3.0.2: resolved "https://registry.yarnpkg.com/csstype/-/csstype-3.1.2.tgz#1d4bf9d572f11c14031f0436e1c10bc1f571f50b" integrity sha512-I7K1Uu0MBPzaFKg4nI5Q7Vs2t+3gWWW648spaF+Rg7pI9ds18Ugn+lvg4SHczUdKlHI5LWBXyqfS8+DufyBsgQ== +"d3-array@2 - 3", "d3-array@2.10.0 - 3", d3-array@^3.1.6: + version "3.2.4" + resolved "https://registry.yarnpkg.com/d3-array/-/d3-array-3.2.4.tgz#15fec33b237f97ac5d7c986dc77da273a8ed0bb5" + integrity sha512-tdQAmyA18i4J7wprpYq8ClcxZy3SC31QMeByyCFyRt7BVHdREQZ5lpzoe5mFEYZUWe+oq8HBvk9JjpibyEV4Jg== + dependencies: + internmap "1 - 2" + +"d3-color@1 - 3": + version "3.1.0" + resolved "https://registry.yarnpkg.com/d3-color/-/d3-color-3.1.0.tgz#395b2833dfac71507f12ac2f7af23bf819de24e2" + integrity sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA== + +d3-ease@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/d3-ease/-/d3-ease-3.0.1.tgz#9658ac38a2140d59d346160f1f6c30fda0bd12f4" + integrity sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w== + +"d3-format@1 - 3": + version "3.1.0" + resolved "https://registry.yarnpkg.com/d3-format/-/d3-format-3.1.0.tgz#9260e23a28ea5cb109e93b21a06e24e2ebd55641" + integrity sha512-YyUI6AEuY/Wpt8KWLgZHsIU86atmikuoOmCfommt0LYHiQSPjvX2AcFc38PX0CBpr2RCyZhjex+NS/LPOv6YqA== + +"d3-interpolate@1.2.0 - 3", d3-interpolate@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/d3-interpolate/-/d3-interpolate-3.0.1.tgz#3c47aa5b32c5b3dfb56ef3fd4342078a632b400d" + integrity sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g== + dependencies: + d3-color "1 - 3" + +d3-path@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/d3-path/-/d3-path-3.1.0.tgz#22df939032fb5a71ae8b1800d61ddb7851c42526" + integrity sha512-p3KP5HCf/bvjBSSKuXid6Zqijx7wIfNW+J/maPs+iwR35at5JCbLUT0LzF1cnjbCHWhqzQTIN2Jpe8pRebIEFQ== + +d3-scale@^4.0.2: + version "4.0.2" + resolved "https://registry.yarnpkg.com/d3-scale/-/d3-scale-4.0.2.tgz#82b38e8e8ff7080764f8dcec77bd4be393689396" + integrity sha512-GZW464g1SH7ag3Y7hXjf8RoUuAFIqklOAq3MRl4OaWabTFJY9PN/E1YklhXLh+OQ3fM9yS2nOkCoS+WLZ6kvxQ== + dependencies: + d3-array "2.10.0 - 3" + d3-format "1 - 3" + d3-interpolate "1.2.0 - 3" + d3-time "2.1.1 - 3" + d3-time-format "2 - 4" + +d3-shape@^3.1.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/d3-shape/-/d3-shape-3.2.0.tgz#a1a839cbd9ba45f28674c69d7f855bcf91dfc6a5" + integrity sha512-SaLBuwGm3MOViRq2ABk3eLoxwZELpH6zhl3FbAoJ7Vm1gofKx6El1Ib5z23NUEhF9AsGl7y+dzLe5Cw2AArGTA== + dependencies: + d3-path "^3.1.0" + +"d3-time-format@2 - 4": + version "4.1.0" + resolved "https://registry.yarnpkg.com/d3-time-format/-/d3-time-format-4.1.0.tgz#7ab5257a5041d11ecb4fe70a5c7d16a195bb408a" + integrity sha512-dJxPBlzC7NugB2PDLwo9Q8JiTR3M3e4/XANkreKSUxF8vvXKqm1Yfq4Q5dl8budlunRVlUUaDUgFt7eA8D6NLg== + dependencies: + d3-time "1 - 3" + +"d3-time@1 - 3", "d3-time@2.1.1 - 3", d3-time@^3.0.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/d3-time/-/d3-time-3.1.0.tgz#9310db56e992e3c0175e1ef385e545e48a9bb5c7" + integrity sha512-VqKjzBLejbSMT4IgbmVgDjpkYrNWUYJnbCGo874u7MMKIWsILRX+OpX/gTk8MqjpT1A/c6HY2dCA77ZN0lkQ2Q== + dependencies: + d3-array "2 - 3" + +d3-timer@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/d3-timer/-/d3-timer-3.0.1.tgz#6284d2a2708285b1abb7e201eda4380af35e63b0" + integrity sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA== + +date-fns@^2.30.0: + version "2.30.0" + resolved "https://registry.yarnpkg.com/date-fns/-/date-fns-2.30.0.tgz#f367e644839ff57894ec6ac480de40cae4b0f4d0" + integrity sha512-fnULvOpxnC5/Vg3NCiWelDsLiUc9bRwAPs/+LfTLNvetFCtCTN+yQz15C/fs4AwX1R9K5GLtLfn8QW+dWisaAw== + dependencies: + "@babel/runtime" "^7.21.0" + debug@^4.1.0, debug@^4.1.1, debug@^4.3.2: version "4.3.4" resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#1319f6579357f2338d3337d2cdd4914bb5dcc865" @@ -971,6 +1206,11 @@ debug@^4.1.0, debug@^4.1.1, debug@^4.3.2: dependencies: ms "2.1.2" +decimal.js-light@^2.4.1: + version "2.5.1" + resolved "https://registry.yarnpkg.com/decimal.js-light/-/decimal.js-light-2.5.1.tgz#134fd32508f19e208f4fb2f8dac0d2626a867934" + integrity sha512-qIMFpTMZmny+MMIitAB6D7iVPEorVw6YQRWkvarTkT4tBeSLLiHzcwj6q0MmYSFCiVpiqPJTJEYIrpcPzVEIvg== + deep-is@^0.1.3: version "0.1.4" resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.4.tgz#a6f2dce612fadd2ef1f519b73551f17e85199831" @@ -1046,6 +1286,14 @@ doctrine@^3.0.0: dependencies: esutils "^2.0.2" +dom-helpers@^5.0.1: + version "5.2.1" + resolved "https://registry.yarnpkg.com/dom-helpers/-/dom-helpers-5.2.1.tgz#d9400536b2bf8225ad98fe052e029451ac40e902" + integrity sha512-nRCa7CK3VTrM2NmGkIy4cbK7IZlgBE/PYMn55rrXefr5xXDP0LdtfPnblFDoVdcAfslJ7or6iqAUnx0CCGIWQA== + dependencies: + "@babel/runtime" "^7.8.7" + csstype "^3.0.2" + dompurify@^3.0.8: version "3.0.8" resolved "https://registry.yarnpkg.com/dompurify/-/dompurify-3.0.8.tgz#e0021ab1b09184bc8af7e35c7dd9063f43a8a437" @@ -1342,6 +1590,11 @@ esutils@^2.0.2: resolved "https://registry.yarnpkg.com/esutils/-/esutils-2.0.3.tgz#74d2eb4de0b8da1293711910d50775b9b710ef64" integrity sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g== +eventemitter3@^4.0.1: + version "4.0.7" + resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-4.0.7.tgz#2de9b68f6528d5644ef5c59526a1b4a07306169f" + integrity sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw== + execa@^5.0.0: version "5.1.1" resolved "https://registry.yarnpkg.com/execa/-/execa-5.1.1.tgz#f80ad9cbf4298f7bd1d4c9555c21e93741c411dd" @@ -1382,6 +1635,11 @@ fast-diff@^1.1.2: resolved "https://registry.yarnpkg.com/fast-diff/-/fast-diff-1.3.0.tgz#ece407fa550a64d638536cd727e129c61616e0f0" integrity sha512-VxPP4NqbUjj6MaAOafWeUn2cXWLcCtljklUtZf0Ind4XQ+QPtmA0b18zZy0jIQx+ExRVCR/ZQpBmik5lXshNsw== +fast-equals@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/fast-equals/-/fast-equals-5.0.1.tgz#a4eefe3c5d1c0d021aeed0bc10ba5e0c12ee405d" + integrity sha512-WF1Wi8PwwSY7/6Kx0vKXtw8RwuSGoM1bvDaJbu7MxDlR1vovZjIAKrnzyrThgAjm6JDTu0fVgWXDlMGspodfoQ== + fast-glob@^3.3.0: version "3.3.1" resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.3.1.tgz#784b4e897340f3dbbef17413b3f11acf03c874c4" @@ -1698,6 +1956,14 @@ highlight.js@^11.9.0: resolved "https://registry.yarnpkg.com/highlight.js/-/highlight.js-11.9.0.tgz#04ab9ee43b52a41a047432c8103e2158a1b8b5b0" integrity sha512-fJ7cW7fQGCYAkgv4CPfwFHrfd/cLS4Hau96JuJ+ZTOWhjnhoeN1ub1tFmALm/+lW5z4WCAuAV9bm05AP0mS6Gw== +html2canvas@^1.2.0: + version "1.4.1" + resolved "https://registry.yarnpkg.com/html2canvas/-/html2canvas-1.4.1.tgz#7cef1888311b5011d507794a066041b14669a543" + integrity sha512-fPU6BHNpsyIhr8yyMpTLLxAbkaK8ArIBcmZIRiBLiDhjeqvXolaEmDGmELFuX9I4xDcaKKcJl+TKZLqruBbmWA== + dependencies: + css-line-break "^2.1.0" + text-segmentation "^1.0.3" + human-signals@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-2.1.0.tgz#dc91fcba42e4d06e4abaed33b3e7a3c02f514ea0" @@ -1753,6 +2019,11 @@ internal-slot@^1.0.5: hasown "^2.0.0" side-channel "^1.0.4" +"internmap@1 - 2": + version "2.0.3" + resolved "https://registry.yarnpkg.com/internmap/-/internmap-2.0.3.tgz#6685f23755e43c524e251d29cbc97248e3061009" + integrity sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg== + is-array-buffer@^3.0.1, is-array-buffer@^3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/is-array-buffer/-/is-array-buffer-3.0.2.tgz#f2653ced8412081638ecb0ebbd0c41c6e0aecbbe" @@ -2483,7 +2754,7 @@ prettier@^3.0.3: resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.0.3.tgz#432a51f7ba422d1469096c0fdc28e235db8f9643" integrity sha512-L/4pUDMxcNa8R/EthV08Zt42WBO4h1rarVtK0K+QJG0X187OLo7l699jWw0GKuwzkPQ//jMFA/8Xm6Fh3J/DAg== -prop-types@^15.8.1: +prop-types@^15.6.2, prop-types@^15.8.1: version "15.8.1" resolved "https://registry.yarnpkg.com/prop-types/-/prop-types-15.8.1.tgz#67d87bf1a694f48435cf332c24af10214a3140b5" integrity sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg== @@ -2502,6 +2773,11 @@ queue-microtask@^1.2.2: resolved "https://registry.yarnpkg.com/queue-microtask/-/queue-microtask-1.2.3.tgz#4929228bbc724dfac43e0efb058caf7b6cfb6243" integrity sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A== +react-day-picker@^8.9.1: + version "8.10.0" + resolved "https://registry.yarnpkg.com/react-day-picker/-/react-day-picker-8.10.0.tgz#729c5b9564967a924213978fb9c0751884a60595" + integrity sha512-mz+qeyrOM7++1NCb1ARXmkjMkzWVh2GL9YiPbRjKe0zHccvekk4HE+0MPOZOrosn8r8zTHIIeOUXTmXRqmkRmg== + react-device-detect@^2.2.2: version "2.2.3" resolved "https://registry.yarnpkg.com/react-device-detect/-/react-device-detect-2.2.3.tgz#97a7ae767cdd004e7c3578260f48cf70c036e7ca" @@ -2526,7 +2802,7 @@ react-dropzone@^14.2.3: file-selector "^0.6.0" prop-types "^15.8.1" -react-is@^16.13.1: +react-is@^16.10.2, react-is@^16.13.1: version "16.13.1" resolved "https://registry.yarnpkg.com/react-is/-/react-is-16.13.1.tgz#789729a4dc36de2999dc156dd6c1d9c18cea56a4" integrity sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ== @@ -2556,6 +2832,15 @@ react-router@6.21.1: dependencies: "@remix-run/router" "1.14.1" +react-smooth@^4.0.0: + version "4.0.1" + resolved "https://registry.yarnpkg.com/react-smooth/-/react-smooth-4.0.1.tgz#6200d8699bfe051ae40ba187988323b1449eab1a" + integrity sha512-OE4hm7XqR0jNOq3Qmk9mFLyd6p2+j6bvbPJ7qlB7+oo0eNcL2l7WQzG6MBnT3EXY6xzkLMUBec3AfewJdA0J8w== + dependencies: + fast-equals "^5.0.1" + prop-types "^15.8.1" + react-transition-group "^4.4.5" + react-tag-input-component@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/react-tag-input-component/-/react-tag-input-component-2.0.2.tgz#f62f013c6a535141dd1c6c3a88858223170150f1" @@ -2576,6 +2861,21 @@ react-tooltip@^5.25.2: "@floating-ui/dom" "^1.0.0" classnames "^2.3.0" +react-transition-group@^4.4.5: + version "4.4.5" + resolved "https://registry.yarnpkg.com/react-transition-group/-/react-transition-group-4.4.5.tgz#e53d4e3f3344da8521489fbef8f2581d42becdd1" + integrity sha512-pZcd1MCJoiKiBR2NRxeCRg13uCXbydPnmB4EOeRrY7480qNWO8IIgQG6zlDkm6uRMsURXPuKq0GWtiM59a5Q6g== + dependencies: + "@babel/runtime" "^7.5.5" + dom-helpers "^5.0.1" + loose-envify "^1.4.0" + prop-types "^15.6.2" + +react-transition-state@^2.1.1: + version "2.1.1" + resolved "https://registry.yarnpkg.com/react-transition-state/-/react-transition-state-2.1.1.tgz#1601a6177926b647041b7d598bf124321ab8d25b" + integrity sha512-kQx5g1FVu9knoz1T1WkapjUgFz08qQ/g1OmuWGi3/AoEFfS0kStxrPlZx81urjCXdz2d+1DqLpU6TyLW/Ro04Q== + react@^18.2.0: version "18.2.0" resolved "https://registry.yarnpkg.com/react/-/react-18.2.0.tgz#555bd98592883255fa00de14f1151a917b5d77d5" @@ -2597,6 +2897,34 @@ readdirp@~3.6.0: dependencies: picomatch "^2.2.1" +recharts-scale@^0.4.4: + version "0.4.5" + resolved "https://registry.yarnpkg.com/recharts-scale/-/recharts-scale-0.4.5.tgz#0969271f14e732e642fcc5bd4ab270d6e87dd1d9" + integrity sha512-kivNFO+0OcUNu7jQquLXAxz1FIwZj8nrj+YkOKc5694NbjCvcT6aSZiIzNzd2Kul4o4rTto8QVR9lMNtxD4G1w== + dependencies: + decimal.js-light "^2.4.1" + +recharts-to-png@^2.3.1: + version "2.3.1" + resolved "https://registry.yarnpkg.com/recharts-to-png/-/recharts-to-png-2.3.1.tgz#94d4edb8461ba4b16318edea77a34c421c16d7c1" + integrity sha512-a+OaAi03oFJMa+Burf3vyH060iFTrb35W8bBYUatNjZVrrMKUcFM3VOI1ym078WIH7XfgYQb17K9p2spVA2FzQ== + dependencies: + html2canvas "^1.2.0" + +recharts@^2.10.3, recharts@^2.12.5: + version "2.12.5" + resolved "https://registry.yarnpkg.com/recharts/-/recharts-2.12.5.tgz#b335eb66173317dccb3e126fce1d7ac5b3cee1e9" + integrity sha512-Cy+BkqrFIYTHJCyKHJEPvbHE2kVQEP6PKbOHJ8ztRGTAhvHuUnCwDaKVb13OwRFZ0QNUk1QvGTDdgWSMbuMtKw== + dependencies: + clsx "^2.0.0" + eventemitter3 "^4.0.1" + lodash "^4.17.21" + react-is "^16.10.2" + react-smooth "^4.0.0" + recharts-scale "^0.4.4" + tiny-invariant "^1.3.1" + victory-vendor "^36.6.8" + reflect.getprototypeof@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/reflect.getprototypeof/-/reflect.getprototypeof-1.0.4.tgz#aaccbf41aca3821b87bb71d9dcbc7ad0ba50a3f3" @@ -2609,6 +2937,11 @@ reflect.getprototypeof@^1.0.4: globalthis "^1.0.3" which-builtin-type "^1.1.3" +regenerator-runtime@^0.14.0: + version "0.14.1" + resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz#356ade10263f685dda125100cd862c1db895327f" + integrity sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw== + regexp.prototype.flags@^1.5.0, regexp.prototype.flags@^1.5.1: version "1.5.1" resolved "https://registry.yarnpkg.com/regexp.prototype.flags/-/regexp.prototype.flags-1.5.1.tgz#90ce989138db209f81492edd734183ce99f9677e" @@ -2893,6 +3226,16 @@ synckit@^0.8.5: "@pkgr/utils" "^2.3.1" tslib "^2.5.0" +tabbable@^6.0.1: + version "6.2.0" + resolved "https://registry.yarnpkg.com/tabbable/-/tabbable-6.2.0.tgz#732fb62bc0175cfcec257330be187dcfba1f3b97" + integrity sha512-Cat63mxsVJlzYvN51JmVXIgNoUokrIaT2zLclCXjRd8boZ0004U4KCs/sToJ75C6sdlByWxpYnb5Boif1VSFew== + +tailwind-merge@^1.14.0: + version "1.14.0" + resolved "https://registry.yarnpkg.com/tailwind-merge/-/tailwind-merge-1.14.0.tgz#e677f55d864edc6794562c63f5001f45093cdb8b" + integrity sha512-3mFKyCo/MBcgyOTlrY8T7odzZFx+w+qKSMAmdFzRvqBfLlSigU6TZnlFHK0lkMwj9Bj8OYU+9yW9lmGuS0QEnQ== + tailwindcss@^3.3.1: version "3.3.5" resolved "https://registry.yarnpkg.com/tailwindcss/-/tailwindcss-3.3.5.tgz#22a59e2fbe0ecb6660809d9cc5f3976b077be3b8" @@ -3031,6 +3374,13 @@ text-path-case@^1.0.2: dependencies: text-dot-case "^1.0.2" +text-segmentation@^1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/text-segmentation/-/text-segmentation-1.0.3.tgz#52a388159efffe746b24a63ba311b6ac9f2d7943" + integrity sha512-iOiPUo/BGnZ6+54OsWxZidGCsdU8YbE4PSpdPinp7DeMtUJNJBoJ/ouUSTJjHkh1KntHaltHl/gDs2FC4i5+Nw== + dependencies: + utrie "^1.0.2" + text-sentence-case@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/text-sentence-case/-/text-sentence-case-1.0.2.tgz#e692a9aea3c8dcb1fb12242838e0ca3e9a22a90f" @@ -3085,6 +3435,11 @@ thenify-all@^1.0.0: dependencies: any-promise "^1.0.0" +tiny-invariant@^1.3.1: + version "1.3.3" + resolved "https://registry.yarnpkg.com/tiny-invariant/-/tiny-invariant-1.3.3.tgz#46680b7a873a0d5d10005995eb90a70d74d60127" + integrity sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg== + titleize@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/titleize/-/titleize-3.0.0.tgz#71c12eb7fdd2558aa8a44b0be83b8a76694acd53" @@ -3112,7 +3467,7 @@ ts-interface-checker@^0.1.9: resolved "https://registry.yarnpkg.com/ts-interface-checker/-/ts-interface-checker-0.1.13.tgz#784fd3d679722bc103b1b4b8030bcddb5db2a699" integrity sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA== -tslib@^2.4.0, tslib@^2.5.0, tslib@^2.6.0: +tslib@^2.0.0, tslib@^2.4.0, tslib@^2.5.0, tslib@^2.6.0: version "2.6.2" resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.2.tgz#703ac29425e7b37cd6fd456e92404d46d1f3e4ae" integrity sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q== @@ -3213,11 +3568,38 @@ util-deprecate@^1.0.2: resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" integrity sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw== +utrie@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/utrie/-/utrie-1.0.2.tgz#d42fe44de9bc0119c25de7f564a6ed1b2c87a645" + integrity sha512-1MLa5ouZiOmQzUbjbu9VmjLzn1QLXBhwpUa7kdLUQK+KQ5KA9I1vk5U4YHe/X2Ch7PYnJfWuWT+VbuxbGwljhw== + dependencies: + base64-arraybuffer "^1.0.2" + uuid@^9.0.0: version "9.0.1" resolved "https://registry.yarnpkg.com/uuid/-/uuid-9.0.1.tgz#e188d4c8853cc722220392c424cd637f32293f30" integrity sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA== +victory-vendor@^36.6.8: + version "36.9.2" + resolved "https://registry.yarnpkg.com/victory-vendor/-/victory-vendor-36.9.2.tgz#668b02a448fa4ea0f788dbf4228b7e64669ff801" + integrity sha512-PnpQQMuxlwYdocC8fIJqVXvkeViHYzotI+NJrCuav0ZYFoq912ZHBk3mCeuj+5/VpodOjPe1z0Fk2ihgzlXqjQ== + dependencies: + "@types/d3-array" "^3.0.3" + "@types/d3-ease" "^3.0.0" + "@types/d3-interpolate" "^3.0.1" + "@types/d3-scale" "^4.0.2" + "@types/d3-shape" "^3.1.0" + "@types/d3-time" "^3.0.0" + "@types/d3-timer" "^3.0.0" + d3-array "^3.1.6" + d3-ease "^3.0.1" + d3-interpolate "^3.0.1" + d3-scale "^4.0.2" + d3-shape "^3.1.0" + d3-time "^3.0.0" + d3-timer "^3.0.1" + vite@^4.3.0: version "4.5.0" resolved "https://registry.yarnpkg.com/vite/-/vite-4.5.0.tgz#ec406295b4167ac3bc23e26f9c8ff559287cff26" diff --git a/server/utils/agents/aibitat/plugins/chat-history.js b/server/utils/agents/aibitat/plugins/chat-history.js index b4b51d0e2..e3123a83b 100644 --- a/server/utils/agents/aibitat/plugins/chat-history.js +++ b/server/utils/agents/aibitat/plugins/chat-history.js @@ -21,6 +21,19 @@ const chatHistory = { // We need a full conversation reply with prev being from // the USER and the last being from anyone other than the user. if (prev.from !== "USER" || last.from === "USER") return; + + // If we have a post-reply flow we should save the chat using this special flow + // so that post save cleanup and other unique properties can be run as opposed to regular chat. + if (aibitat.hasOwnProperty("_replySpecialAttributes")) { + await this._storeSpecial(aibitat, { + prompt: prev.content, + response: last.content, + options: aibitat._replySpecialAttributes, + }); + delete aibitat._replySpecialAttributes; + return; + } + await this._store(aibitat, { prompt: prev.content, response: last.content, @@ -42,6 +55,28 @@ const chatHistory = { threadId: invocation?.thread_id || null, }); }, + _storeSpecial: async function ( + aibitat, + { prompt, response, options = {} } = {} + ) { + const invocation = aibitat.handlerProps.invocation; + await WorkspaceChats.new({ + workspaceId: Number(invocation.workspace_id), + prompt, + response: { + sources: options?.sources ?? [], + // when we have a _storeSpecial called the options param can include a storedResponse() function + // that will override the text property to store extra information in, depending on the special type of chat. + text: options.hasOwnProperty("storedResponse") + ? options.storedResponse(response) + : response, + type: options?.saveAsType ?? "chat", + }, + user: { id: invocation?.user_id || null }, + threadId: invocation?.thread_id || null, + }); + options?.postSave(); + }, }; }, }; diff --git a/server/utils/agents/aibitat/plugins/index.js b/server/utils/agents/aibitat/plugins/index.js index 5892df4a4..d9ff544e2 100644 --- a/server/utils/agents/aibitat/plugins/index.js +++ b/server/utils/agents/aibitat/plugins/index.js @@ -5,6 +5,7 @@ const { docSummarizer } = require("./summarize.js"); const { saveFileInBrowser } = require("./save-file-browser.js"); const { chatHistory } = require("./chat-history.js"); const { memory } = require("./memory.js"); +const { rechart } = require("./rechart.js"); module.exports = { webScraping, @@ -14,6 +15,7 @@ module.exports = { saveFileInBrowser, chatHistory, memory, + rechart, // Plugin name aliases so they can be pulled by slug as well. [webScraping.name]: webScraping, @@ -23,4 +25,5 @@ module.exports = { [saveFileInBrowser.name]: saveFileInBrowser, [chatHistory.name]: chatHistory, [memory.name]: memory, + [rechart.name]: rechart, }; diff --git a/server/utils/agents/aibitat/plugins/rechart.js b/server/utils/agents/aibitat/plugins/rechart.js new file mode 100644 index 000000000..a41ddd658 --- /dev/null +++ b/server/utils/agents/aibitat/plugins/rechart.js @@ -0,0 +1,109 @@ +const { safeJsonParse } = require("../../../http"); +const { Deduplicator } = require("../utils/dedupe"); + +const rechart = { + name: "create-chart", + startupConfig: { + params: {}, + }, + plugin: function () { + return { + name: this.name, + setup(aibitat) { + // Scrape a website and summarize the content based on objective if the content is too large.', + aibitat.function({ + super: aibitat, + name: this.name, + tracker: new Deduplicator(), + description: + "Generates the JSON data required to generate a RechartJS chart to the user based on their prompt and available data.", + parameters: { + $schema: "http://json-schema.org/draft-07/schema#", + type: "object", + properties: { + type: { + type: "string", + enum: [ + "area", + "bar", + "line", + "composed", + "scatter", + "pie", + "radar", + "radialBar", + "treemap", + "funnel", + ], + description: "The type of chart to be generated.", + }, + title: { + type: "string", + description: + "Title of the chart. There MUST always be a title. Do not leave it blank.", + }, + dataset: { + type: "string", + description: `Valid JSON in which each element is an object for Recharts API for the 'type' of chart defined WITHOUT new line characters. Strictly using this FORMAT and naming: +{ "name": "a", "value": 12 }]. +Make sure field "name" always stays named "name". Instead of naming value field value in JSON, name it based on user metric and make it the same across every item. +Make sure the format use double quotes and property names are string literals. Provide JSON data only.`, + }, + }, + additionalProperties: false, + }, + required: ["type", "title", "dataset"], + handler: async function ({ type, dataset, title }) { + try { + if (!this.tracker.isUnique(this.name)) { + this.super.handlerProps.log( + `${this.name} has been run for this chat response already. It can only be called once per chat.` + ); + return "The chart was generated and returned to the user. This function completed successfully. Do not call this function again."; + } + + const data = safeJsonParse(dataset, null); + if (data === null) { + this.super.introspect( + `${this.caller}: ${this.name} provided invalid JSON data - so we cant make a ${type} chart.` + ); + return "Invalid JSON provided. Please only provide valid RechartJS JSON to generate a chart."; + } + + this.super.introspect(`${this.caller}: Rendering ${type} chart.`); + this.super.socket.send("rechartVisualize", { + type, + dataset, + title, + }); + + this.super._replySpecialAttributes = { + saveAsType: "rechartVisualize", + storedResponse: (additionalText = "") => + JSON.stringify({ + type, + dataset, + title, + caption: additionalText, + }), + postSave: () => this.tracker.removeUniqueConstraint(this.name), + }; + + this.tracker.markUnique(this.name); + return "The chart was generated and returned to the user. This function completed successfully. Do not make another chart."; + } catch (error) { + this.super.handlerProps.log( + `create-chart raised an error. ${error.message}` + ); + return `Let the user know this action was not successful. An error was raised while generating the chart. ${error.message}`; + } + }, + }); + }, + }; + }, +}; + +module.exports = { + rechart, +}; diff --git a/server/utils/agents/aibitat/utils/dedupe.js b/server/utils/agents/aibitat/utils/dedupe.js index b59efec6f..8c4d582d0 100644 --- a/server/utils/agents/aibitat/utils/dedupe.js +++ b/server/utils/agents/aibitat/utils/dedupe.js @@ -9,10 +9,17 @@ // ... do random # of times. // We want to block all the reruns of a plugin, so we can add this to prevent that behavior from // spamming the user (or other costly function) that have the exact same signatures. + +// Track Run/isDuplicate prevents _exact_ data re-runs based on the SHA of their inputs +// StartCooldown/isOnCooldown does prevention of _near-duplicate_ runs based on only the function name that is running. +// isUnique/markUnique/removeUniqueConstraint prevents one-time functions from re-running. EG: charting. const crypto = require("crypto"); +const DEFAULT_COOLDOWN_MS = 5 * 1000; class Deduplicator { #hashes = {}; + #cooldowns = {}; + #uniques = {}; constructor() {} trackRun(key, params = {}) { @@ -30,6 +37,32 @@ class Deduplicator { .digest("hex"); return this.#hashes.hasOwnProperty(newSig); } + + startCooldown( + key, + parameters = { + cooldownInMs: DEFAULT_COOLDOWN_MS, + } + ) { + this.#cooldowns[key] = Number(new Date()) + Number(parameters.cooldownInMs); + } + + isOnCooldown(key) { + if (!this.#cooldowns.hasOwnProperty(key)) return false; + return Number(new Date()) <= this.#cooldowns[key]; + } + + isUnique(key) { + return !this.#uniques.hasOwnProperty(key); + } + + removeUniqueConstraint(key) { + delete this.#uniques[key]; + } + + markUnique(key) { + this.#uniques[key] = Number(new Date()); + } } module.exports.Deduplicator = Deduplicator; diff --git a/server/utils/helpers/chat/responses.js b/server/utils/helpers/chat/responses.js index e2ec7bd0d..d7e20e549 100644 --- a/server/utils/helpers/chat/responses.js +++ b/server/utils/helpers/chat/responses.js @@ -123,6 +123,7 @@ function convertToChatHistory(history = []) { sentAt: moment(createdAt).unix(), }, { + type: data?.type || "chart", role: "assistant", content: data.text, sources: data.sources || [], From df2c01b176a5db8c41a2e7fa4ca5f41cb610aecc Mon Sep 17 00:00:00 2001 From: timothycarambat Date: Fri, 26 Apr 2024 15:58:30 -0700 Subject: [PATCH 05/18] patch OpenRouter model fetcher when key is not present --- server/utils/AiProviders/openRouter/index.js | 100 +++++++++++-------- server/utils/helpers/customModels.js | 8 +- 2 files changed, 62 insertions(+), 46 deletions(-) diff --git a/server/utils/AiProviders/openRouter/index.js b/server/utils/AiProviders/openRouter/index.js index dbed87fa3..ac72b1679 100644 --- a/server/utils/AiProviders/openRouter/index.js +++ b/server/utils/AiProviders/openRouter/index.js @@ -8,6 +8,11 @@ const { const fs = require("fs"); const path = require("path"); const { safeJsonParse } = require("../../http"); +const cacheFolder = path.resolve( + process.env.STORAGE_DIR + ? path.resolve(process.env.STORAGE_DIR, "models", "openrouter") + : path.resolve(__dirname, `../../../storage/models/openrouter`) +); class OpenRouterLLM { constructor(embedder = null, modelPreference = null) { @@ -38,12 +43,8 @@ class OpenRouterLLM { this.embedder = !embedder ? new NativeEmbedder() : embedder; this.defaultTemp = 0.7; - const cacheFolder = path.resolve( - process.env.STORAGE_DIR - ? path.resolve(process.env.STORAGE_DIR, "models", "openrouter") - : path.resolve(__dirname, `../../../storage/models/openrouter`) - ); - fs.mkdirSync(cacheFolder, { recursive: true }); + if (!fs.existsSync(cacheFolder)) + fs.mkdirSync(cacheFolder, { recursive: true }); this.cacheModelPath = path.resolve(cacheFolder, "models.json"); this.cacheAtPath = path.resolve(cacheFolder, ".cached_at"); } @@ -52,11 +53,6 @@ class OpenRouterLLM { console.log(`\x1b[36m[${this.constructor.name}]\x1b[0m ${text}`, ...args); } - async init() { - await this.#syncModels(); - return this; - } - // This checks if the .cached_at file has a timestamp that is more than 1Week (in millis) // from the current date. If it is, then we will refetch the API so that all the models are up // to date. @@ -80,37 +76,7 @@ class OpenRouterLLM { this.log( "Model cache is not present or stale. Fetching from OpenRouter API." ); - await fetch(`${this.basePath}/models`, { - method: "GET", - headers: { - "Content-Type": "application/json", - }, - }) - .then((res) => res.json()) - .then(({ data = [] }) => { - const models = {}; - data.forEach((model) => { - models[model.id] = { - id: model.id, - name: model.name, - organization: - model.id.split("/")[0].charAt(0).toUpperCase() + - model.id.split("/")[0].slice(1), - maxLength: model.context_length, - }; - }); - fs.writeFileSync(this.cacheModelPath, JSON.stringify(models), { - encoding: "utf-8", - }); - fs.writeFileSync(this.cacheAtPath, String(Number(new Date())), { - encoding: "utf-8", - }); - return models; - }) - .catch((e) => { - console.error(e); - return {}; - }); + await fetchOpenRouterModels(); return; } @@ -330,7 +296,7 @@ class OpenRouterLLM { try { JSON.parse(message); validJSON = true; - } catch {} + } catch { } if (!validJSON) { // It can be possible that the chunk decoding is running away @@ -420,6 +386,54 @@ class OpenRouterLLM { } } +async function fetchOpenRouterModels() { + return await fetch(`https://openrouter.ai/api/v1/models`, { + method: "GET", + headers: { + "Content-Type": "application/json", + }, + }) + .then((res) => res.json()) + .then(({ data = [] }) => { + const models = {}; + data.forEach((model) => { + models[model.id] = { + id: model.id, + name: model.name, + organization: + model.id.split("/")[0].charAt(0).toUpperCase() + + model.id.split("/")[0].slice(1), + maxLength: model.context_length, + }; + }); + + // Cache all response information + if (!fs.existsSync(cacheFolder)) + fs.mkdirSync(cacheFolder, { recursive: true }); + fs.writeFileSync( + path.resolve(cacheFolder, "models.json"), + JSON.stringify(models), + { + encoding: "utf-8", + } + ); + fs.writeFileSync( + path.resolve(cacheFolder, ".cached_at"), + String(Number(new Date())), + { + encoding: "utf-8", + } + ); + + return models; + }) + .catch((e) => { + console.error(e); + return {}; + }); +} + module.exports = { OpenRouterLLM, + fetchOpenRouterModels, }; diff --git a/server/utils/helpers/customModels.js b/server/utils/helpers/customModels.js index 3c6b5350b..edd8bac1e 100644 --- a/server/utils/helpers/customModels.js +++ b/server/utils/helpers/customModels.js @@ -1,4 +1,7 @@ -const { OpenRouterLLM } = require("../AiProviders/openRouter"); +const { + OpenRouterLLM, + fetchOpenRouterModels, +} = require("../AiProviders/openRouter"); const { perplexityModels } = require("../AiProviders/perplexity"); const { togetherAiModels } = require("../AiProviders/togetherAi"); const SUPPORT_CUSTOM_MODELS = [ @@ -232,8 +235,7 @@ async function getPerplexityModels() { } async function getOpenRouterModels() { - const openrouter = await new OpenRouterLLM().init(); - const knownModels = openrouter.models(); + const knownModels = await fetchOpenRouterModels(); if (!Object.keys(knownModels).length === 0) return { models: [], error: null }; From 1b35bcbeab10b77e6dbd263cceecf1b965a40789 Mon Sep 17 00:00:00 2001 From: Timothy Carambat Date: Fri, 26 Apr 2024 16:46:04 -0700 Subject: [PATCH 06/18] Strengthen field validations on user Updates (#1201) * Strengthen field validations on user Updates * update writables --- server/models/user.js | 54 ++++++++++++++++++-- server/utils/AiProviders/openRouter/index.js | 2 +- server/utils/PasswordRecovery/index.js | 7 ++- 3 files changed, 58 insertions(+), 5 deletions(-) diff --git a/server/models/user.js b/server/models/user.js index c447950ca..ecb620ee4 100644 --- a/server/models/user.js +++ b/server/models/user.js @@ -2,6 +2,23 @@ const prisma = require("../utils/prisma"); const { EventLogs } = require("./eventLogs"); const User = { + writable: [ + // Used for generic updates so we can validate keys in request body + "username", + "password", + "pfpFilename", + "role", + "suspended", + ], + // validations for the above writable fields. + castColumnValue: function (key, value) { + switch (key) { + case "suspended": + return Number(Boolean(value)); + default: + return String(value); + } + }, create: async function ({ username, password, role = "default" }) { const passwordCheck = this.checkPasswordComplexity(password); if (!passwordCheck.checkedOK) { @@ -42,13 +59,26 @@ const User = { update: async function (userId, updates = {}) { try { + if (!userId) throw new Error("No user id provided for update"); const currentUser = await prisma.users.findUnique({ where: { id: parseInt(userId) }, }); - if (!currentUser) { - return { success: false, error: "User not found" }; - } + if (!currentUser) return { success: false, error: "User not found" }; + // Removes non-writable fields for generic updates + // and force-casts to the proper type; + Object.entries(updates).forEach(([key, value]) => { + if (this.writable.includes(key)) { + updates[key] = this.castColumnValue(key, value); + return; + } + delete updates[key]; + }); + + if (Object.keys(updates).length === 0) + return { success: false, error: "No valid updates applied." }; + + // Handle password specific updates if (updates.hasOwnProperty("password")) { const passwordCheck = this.checkPasswordComplexity(updates.password); if (!passwordCheck.checkedOK) { @@ -78,6 +108,24 @@ const User = { } }, + // Explicit direct update of user object. + // Only use this method when directly setting a key value + // that takes no user input for the keys being modified. + _update: async function (id = null, data = {}) { + if (!id) throw new Error("No user id provided for update"); + + try { + const user = await prisma.users.update({ + where: { id }, + data, + }); + return { user, message: null }; + } catch (error) { + console.error(error.message); + return { user: null, message: error.message }; + } + }, + get: async function (clause = {}) { try { const user = await prisma.users.findFirst({ where: clause }); diff --git a/server/utils/AiProviders/openRouter/index.js b/server/utils/AiProviders/openRouter/index.js index ac72b1679..655839dc3 100644 --- a/server/utils/AiProviders/openRouter/index.js +++ b/server/utils/AiProviders/openRouter/index.js @@ -296,7 +296,7 @@ class OpenRouterLLM { try { JSON.parse(message); validJSON = true; - } catch { } + } catch {} if (!validJSON) { // It can be possible that the chunk decoding is running away diff --git a/server/utils/PasswordRecovery/index.js b/server/utils/PasswordRecovery/index.js index fbcbe5799..2383dd2c3 100644 --- a/server/utils/PasswordRecovery/index.js +++ b/server/utils/PasswordRecovery/index.js @@ -22,7 +22,7 @@ async function generateRecoveryCodes(userId) { const { error } = await RecoveryCode.createMany(newRecoveryCodes); if (!!error) throw new Error(error); - const { success } = await User.update(userId, { + const { user: success } = await User._update(userId, { seen_recovery_codes: true, }); if (!success) throw new Error("Failed to generate user recovery codes!"); @@ -80,6 +80,11 @@ async function resetPassword(token, _newPassword = "", confirmPassword = "") { // JOI password rules will be enforced inside .update. const { error } = await User.update(resetToken.user_id, { password: newPassword, + }); + + // seen_recovery_codes is not publicly writable + // so we have to do direct update here + await User._update(resetToken.user_id, { seen_recovery_codes: false, }); From 8eda75d624148a18054931c6d41e3bc3dd7aa3c2 Mon Sep 17 00:00:00 2001 From: Sean Hatfield Date: Fri, 26 Apr 2024 17:08:10 -0700 Subject: [PATCH 07/18] [FIX] Loading message in document picker bug (#1202) * fix loading message in document picker bug * linting --------- Co-authored-by: timothycarambat --- .../MangeWorkspace/Documents/Directory/index.jsx | 1 + .../Documents/UploadFile/FileUploadProgress/index.jsx | 6 ++++++ .../MangeWorkspace/Documents/UploadFile/index.jsx | 10 +++++++++- 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/frontend/src/components/Modals/MangeWorkspace/Documents/Directory/index.jsx b/frontend/src/components/Modals/MangeWorkspace/Documents/Directory/index.jsx index 83544f72d..2a455b357 100644 --- a/frontend/src/components/Modals/MangeWorkspace/Documents/Directory/index.jsx +++ b/frontend/src/components/Modals/MangeWorkspace/Documents/Directory/index.jsx @@ -306,6 +306,7 @@ function Directory({ workspace={workspace} fetchKeys={fetchKeys} setLoading={setLoading} + setLoadingMessage={setLoadingMessage} />
diff --git a/frontend/src/components/Modals/MangeWorkspace/Documents/UploadFile/FileUploadProgress/index.jsx b/frontend/src/components/Modals/MangeWorkspace/Documents/UploadFile/FileUploadProgress/index.jsx index c375aa2e3..e1f4651eb 100644 --- a/frontend/src/components/Modals/MangeWorkspace/Documents/UploadFile/FileUploadProgress/index.jsx +++ b/frontend/src/components/Modals/MangeWorkspace/Documents/UploadFile/FileUploadProgress/index.jsx @@ -12,6 +12,8 @@ function FileUploadProgressComponent({ reason = null, onUploadSuccess, onUploadError, + setLoading, + setLoadingMessage, }) { const [timerMs, setTimerMs] = useState(10); const [status, setStatus] = useState("pending"); @@ -19,6 +21,8 @@ function FileUploadProgressComponent({ useEffect(() => { async function uploadFile() { + setLoading(true); + setLoadingMessage("Uploading file..."); const start = Number(new Date()); const formData = new FormData(); formData.append("file", file, file.name); @@ -34,6 +38,8 @@ function FileUploadProgressComponent({ onUploadError(data.error); setError(data.error); } else { + setLoading(false); + setLoadingMessage(""); setStatus("complete"); clearInterval(timer); onUploadSuccess(); diff --git a/frontend/src/components/Modals/MangeWorkspace/Documents/UploadFile/index.jsx b/frontend/src/components/Modals/MangeWorkspace/Documents/UploadFile/index.jsx index acf319d92..e57997a83 100644 --- a/frontend/src/components/Modals/MangeWorkspace/Documents/UploadFile/index.jsx +++ b/frontend/src/components/Modals/MangeWorkspace/Documents/UploadFile/index.jsx @@ -7,7 +7,12 @@ import { v4 } from "uuid"; import FileUploadProgress from "./FileUploadProgress"; import Workspace from "../../../../../models/workspace"; -export default function UploadFile({ workspace, fetchKeys, setLoading }) { +export default function UploadFile({ + workspace, + fetchKeys, + setLoading, + setLoadingMessage, +}) { const [ready, setReady] = useState(false); const [files, setFiles] = useState([]); const [fetchingUrl, setFetchingUrl] = useState(false); @@ -15,6 +20,7 @@ export default function UploadFile({ workspace, fetchKeys, setLoading }) { const handleSendLink = async (e) => { e.preventDefault(); setLoading(true); + setLoadingMessage("Scraping link..."); setFetchingUrl(true); const formEl = e.target; const form = new FormData(formEl); @@ -114,6 +120,8 @@ export default function UploadFile({ workspace, fetchKeys, setLoading }) { reason={file?.reason} onUploadSuccess={handleUploadSuccess} onUploadError={handleUploadError} + setLoading={setLoading} + setLoadingMessage={setLoadingMessage} /> ))}
From 360f17cd58d1c41c4a71cc2065f44878e2dd97cc Mon Sep 17 00:00:00 2001 From: Sean Hatfield Date: Fri, 26 Apr 2024 17:38:41 -0700 Subject: [PATCH 08/18] [FIX] Move to Workspace popup UI bug fix (#1204) fix for popup menu transparent container --- .../Modals/MangeWorkspace/Documents/Directory/index.jsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/components/Modals/MangeWorkspace/Documents/Directory/index.jsx b/frontend/src/components/Modals/MangeWorkspace/Documents/Directory/index.jsx index 2a455b357..d479a6cce 100644 --- a/frontend/src/components/Modals/MangeWorkspace/Documents/Directory/index.jsx +++ b/frontend/src/components/Modals/MangeWorkspace/Documents/Directory/index.jsx @@ -261,8 +261,8 @@ function Directory({ )}
{amountSelected !== 0 && ( -
-
+
+