diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 00bdb5428..254ae0244 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -78,8 +78,8 @@ }, "updateContentCommand": "cd server && yarn && cd ../collector && PUPPETEER_DOWNLOAD_BASE_URL=https://storage.googleapis.com/chrome-for-testing-public yarn && cd ../frontend && yarn && cd .. && yarn setup:envs && yarn prisma:setup && echo \"Please run yarn dev:server, yarn dev:collector, and yarn dev:frontend in separate terminal tabs.\"", // Use 'postCreateCommand' to run commands after the container is created. - // This configures VITE for github codespaces - "postCreateCommand": "if [ \"${CODESPACES}\" = \"true\" ]; then echo 'VITE_API_BASE=\"https://$CODESPACE_NAME-3001.$GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN/api\"' > ./frontend/.env; fi", + // This configures VITE for github codespaces and installs gh cli + "postCreateCommand": "if [ \"${CODESPACES}\" = \"true\" ]; then echo 'VITE_API_BASE=\"https://$CODESPACE_NAME-3001.$GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN/api\"' > ./frontend/.env && (type -p wget >/dev/null || (sudo apt update && sudo apt-get install wget -y)) && sudo mkdir -p -m 755 /etc/apt/keyrings && wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null && sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg && echo \"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main\" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null && sudo apt update && sudo apt install gh -y; fi", "portsAttributes": { "3001": { "label": "Backend", @@ -208,4 +208,4 @@ } // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root. // "remoteUser": "root" -} \ No newline at end of file +} diff --git a/.github/workflows/build-and-push-image-semver.yaml b/.github/workflows/build-and-push-image-semver.yaml new file mode 100644 index 000000000..8fb6d35c2 --- /dev/null +++ b/.github/workflows/build-and-push-image-semver.yaml @@ -0,0 +1,115 @@ +name: Publish AnythingLLM Docker image on Release (amd64 & arm64) + +concurrency: + group: build-${{ github.ref }} + cancel-in-progress: true + +on: + release: + types: [published] + +jobs: + push_multi_platform_to_registries: + name: Push Docker multi-platform image to multiple registries + runs-on: ubuntu-latest + permissions: + packages: write + contents: read + steps: + - name: Check out the repo + uses: actions/checkout@v4 + + - name: Check if DockerHub build needed + shell: bash + run: | + # Check if the secret for USERNAME is set (don't even check for the password) + if [[ -z "${{ secrets.DOCKER_USERNAME }}" ]]; then + echo "DockerHub build not needed" + echo "enabled=false" >> $GITHUB_OUTPUT + else + echo "DockerHub build needed" + echo "enabled=true" >> $GITHUB_OUTPUT + fi + id: dockerhub + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Docker Hub + uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a + # Only login to the Docker Hub if the repo is mintplex/anythingllm, to allow for forks to build on GHCR + if: steps.dockerhub.outputs.enabled == 'true' + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 + with: + images: | + ${{ steps.dockerhub.outputs.enabled == 'true' && 'mintplexlabs/anythingllm' || '' }} + ghcr.io/${{ github.repository }} + tags: | + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + + - name: Build and push multi-platform Docker image + uses: docker/build-push-action@v6 + with: + context: . + file: ./docker/Dockerfile + push: true + sbom: true + provenance: mode=max + platforms: linux/amd64,linux/arm64 + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + # For Docker scout there are some intermediary reported CVEs which exists outside + # of execution content or are unreachable by an attacker but exist in image. + # We create VEX files for these so they don't show in scout summary. + - name: Collect known and verified CVE exceptions + id: cve-list + run: | + # Collect CVEs from filenames in vex folder + CVE_NAMES="" + for file in ./docker/vex/*.vex.json; do + [ -e "$file" ] || continue + filename=$(basename "$file") + stripped_filename=${filename%.vex.json} + CVE_NAMES+=" $stripped_filename" + done + echo "CVE_EXCEPTIONS=$CVE_NAMES" >> $GITHUB_OUTPUT + shell: bash + + # About VEX attestations https://docs.docker.com/scout/explore/exceptions/ + # Justifications https://github.com/openvex/spec/blob/main/OPENVEX-SPEC.md#status-justifications + - name: Add VEX attestations + env: + CVE_EXCEPTIONS: ${{ steps.cve-list.outputs.CVE_EXCEPTIONS }} + run: | + echo $CVE_EXCEPTIONS + curl -sSfL https://raw.githubusercontent.com/docker/scout-cli/main/install.sh | sh -s -- + for cve in $CVE_EXCEPTIONS; do + for tag in "${{ join(fromJSON(steps.meta.outputs.json).tags, ' ') }}"; do + echo "Attaching VEX exception $cve to $tag" + docker scout attestation add \ + --file "./docker/vex/$cve.vex.json" \ + --predicate-type https://openvex.dev/ns/v0.2.0 \ + $tag + done + done + shell: bash diff --git a/.github/workflows/dev-build.yaml b/.github/workflows/dev-build.yaml index a7632dfd0..e3bb1d556 100644 --- a/.github/workflows/dev-build.yaml +++ b/.github/workflows/dev-build.yaml @@ -6,7 +6,7 @@ concurrency: on: push: - branches: ['558-multi-modal-support'] # put your current branch to create a build. Core team only. + branches: ['pipertts-support'] # put your current branch to create a build. Core team only. paths-ignore: - '**.md' - 'cloud-deployments/*' diff --git a/.vscode/tasks.json b/.vscode/tasks.json index fd0eb3bd2..6783e17e9 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -46,7 +46,7 @@ } } }, - "command": "cd ${workspaceFolder}/server/ && yarn dev", + "command": "if [ \"${CODESPACES}\" = \"true\" ]; then while ! gh codespace ports -c $CODESPACE_NAME | grep 3001; do sleep 1; done; gh codespace ports visibility 3001:public -c $CODESPACE_NAME; fi & cd ${workspaceFolder}/server/ && yarn dev", "runOptions": { "instanceLimit": 1, "reevaluateOnRerun": true diff --git a/README.md b/README.md index d7812265d..178fef08e 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,7 @@ AnythingLLM divides your documents into objects called `workspaces`. A Workspace **TTS (text-to-speech) support:** - Native Browser Built-in (default) +- [PiperTTSLocal - runs in browser](https://github.com/rhasspy/piper) - [OpenAI TTS](https://platform.openai.com/docs/guides/text-to-speech/voice-options) - [ElevenLabs](https://elevenlabs.io/) diff --git a/cloud-deployments/digitalocean/terraform/user_data.tp1 b/cloud-deployments/digitalocean/terraform/user_data.tp1 index cd239c6b4..1853f691c 100644 --- a/cloud-deployments/digitalocean/terraform/user_data.tp1 +++ b/cloud-deployments/digitalocean/terraform/user_data.tp1 @@ -9,8 +9,10 @@ sudo systemctl enable docker sudo systemctl start docker mkdir -p /home/anythingllm -touch /home/anythingllm/.env - +cat </home/anythingllm/.env +${env_content} +EOF + sudo docker pull mintplexlabs/anythingllm sudo docker run -d -p 3001:3001 --cap-add SYS_ADMIN -v /home/anythingllm:/app/server/storage -v /home/anythingllm/.env:/app/server/.env -e STORAGE_DIR="/app/server/storage" mintplexlabs/anythingllm echo "Container ID: $(sudo docker ps --latest --quiet)" diff --git a/collector/utils/extensions/RepoLoader/GitlabRepo/RepoLoader/index.js b/collector/utils/extensions/RepoLoader/GitlabRepo/RepoLoader/index.js index c90932986..7d5c8438c 100644 --- a/collector/utils/extensions/RepoLoader/GitlabRepo/RepoLoader/index.js +++ b/collector/utils/extensions/RepoLoader/GitlabRepo/RepoLoader/index.js @@ -223,10 +223,6 @@ class GitLabRepoLoader { const objects = Array.isArray(data) ? data.filter((item) => item.type === "blob") : []; // only get files, not paths or submodules - if (objects.length === 0) { - fetching = false; - break; - } // Apply ignore path rules to found objects. If any rules match it is an invalid file path. console.log( diff --git a/collector/utils/extensions/WebsiteDepth/index.js b/collector/utils/extensions/WebsiteDepth/index.js index d03e661b4..2a9994aa5 100644 --- a/collector/utils/extensions/WebsiteDepth/index.js +++ b/collector/utils/extensions/WebsiteDepth/index.js @@ -9,34 +9,36 @@ const { tokenizeString } = require("../../tokenizer"); const path = require("path"); const fs = require("fs"); -async function discoverLinks(startUrl, depth = 1, maxLinks = 20) { +async function discoverLinks(startUrl, maxDepth = 1, maxLinks = 20) { const baseUrl = new URL(startUrl); - const discoveredLinks = new Set(); - const pendingLinks = [startUrl]; - let currentLevel = 0; - depth = depth < 1 ? 1 : depth; - maxLinks = maxLinks < 1 ? 1 : maxLinks; + const discoveredLinks = new Set([startUrl]); + let queue = [[startUrl, 0]]; // [url, currentDepth] + const scrapedUrls = new Set(); - // Check depth and if there are any links left to scrape - while (currentLevel < depth && pendingLinks.length > 0) { - const newLinks = await getPageLinks(pendingLinks[0], baseUrl); - pendingLinks.shift(); + for (let currentDepth = 0; currentDepth < maxDepth; currentDepth++) { + const levelSize = queue.length; + const nextQueue = []; - for (const link of newLinks) { - if (!discoveredLinks.has(link)) { - discoveredLinks.add(link); - pendingLinks.push(link); - } + for (let i = 0; i < levelSize && discoveredLinks.size < maxLinks; i++) { + const [currentUrl, urlDepth] = queue[i]; - // Exit out if we reach maxLinks - if (discoveredLinks.size >= maxLinks) { - return Array.from(discoveredLinks).slice(0, maxLinks); + if (!scrapedUrls.has(currentUrl)) { + scrapedUrls.add(currentUrl); + const newLinks = await getPageLinks(currentUrl, baseUrl); + + for (const link of newLinks) { + if (!discoveredLinks.has(link) && discoveredLinks.size < maxLinks) { + discoveredLinks.add(link); + if (urlDepth + 1 < maxDepth) { + nextQueue.push([link, urlDepth + 1]); + } + } + } } } - if (pendingLinks.length === 0) { - currentLevel++; - } + queue = nextQueue; + if (queue.length === 0 || discoveredLinks.size >= maxLinks) break; } return Array.from(discoveredLinks); diff --git a/collector/utils/extensions/YoutubeTranscript/YoutubeLoader/youtube-transcript.js b/collector/utils/extensions/YoutubeTranscript/YoutubeLoader/youtube-transcript.js index c81c0ec56..f868875b2 100644 --- a/collector/utils/extensions/YoutubeTranscript/YoutubeLoader/youtube-transcript.js +++ b/collector/utils/extensions/YoutubeTranscript/YoutubeLoader/youtube-transcript.js @@ -47,10 +47,12 @@ class YoutubeTranscript { let transcript = ""; const chunks = transcriptXML.getElementsByTagName("text"); for (const chunk of chunks) { - transcript += chunk.textContent; + // Add space after each text chunk + transcript += chunk.textContent + " "; } - return transcript; + // Trim extra whitespace + return transcript.trim().replace(/\s+/g, " "); } catch (e) { throw new YoutubeTranscriptError(e); } diff --git a/collector/utils/files/mime.js b/collector/utils/files/mime.js index 6cd88f82e..b747d5975 100644 --- a/collector/utils/files/mime.js +++ b/collector/utils/files/mime.js @@ -37,6 +37,7 @@ class MimeDetector { "lua", "pas", "r", + "go", ], }, true diff --git a/frontend/package.json b/frontend/package.json index fa40e7b33..8a60c1109 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -13,6 +13,7 @@ "dependencies": { "@metamask/jazzicon": "^2.0.0", "@microsoft/fetch-event-source": "^2.0.1", + "@mintplex-labs/piper-tts-web": "^1.0.4", "@phosphor-icons/react": "^2.1.7", "@tremor/react": "^3.15.1", "dompurify": "^3.0.8", @@ -24,7 +25,9 @@ "js-levenshtein": "^1.1.6", "lodash.debounce": "^4.0.8", "markdown-it": "^13.0.1", + "markdown-it-katex": "^2.0.3", "moment": "^2.30.1", + "onnxruntime-web": "^1.18.0", "pluralize": "^8.0.0", "react": "^18.2.0", "react-device-detect": "^2.2.2", diff --git a/frontend/src/components/TextToSpeech/PiperTTSOptions/index.jsx b/frontend/src/components/TextToSpeech/PiperTTSOptions/index.jsx new file mode 100644 index 000000000..48b87efe7 --- /dev/null +++ b/frontend/src/components/TextToSpeech/PiperTTSOptions/index.jsx @@ -0,0 +1,220 @@ +import { useState, useEffect, useRef } from "react"; +import PiperTTSClient from "@/utils/piperTTS"; +import { titleCase } from "text-case"; +import { humanFileSize } from "@/utils/numbers"; +import showToast from "@/utils/toast"; +import { CircleNotch, PauseCircle, PlayCircle } from "@phosphor-icons/react"; + +export default function PiperTTSOptions({ settings }) { + return ( + <> +

+ All PiperTTS models will run in your browser locally. This can be + resource intensive on lower-end devices. +

+
+ +
+ + ); +} + +function voicesByLanguage(voices = []) { + const voicesByLanguage = voices.reduce((acc, voice) => { + const langName = voice?.language?.name_english ?? "Unlisted"; + acc[langName] = acc[langName] || []; + acc[langName].push(voice); + return acc; + }, {}); + return Object.entries(voicesByLanguage); +} + +function voiceDisplayName(voice) { + const { is_stored, name, quality, files } = voice; + const onnxFileKey = Object.keys(files).find((key) => key.endsWith(".onnx")); + const fileSize = files?.[onnxFileKey]?.size_bytes || 0; + return `${is_stored ? "✔ " : ""}${titleCase(name)}-${quality === "low" ? "Low" : "HQ"} (${humanFileSize(fileSize)})`; +} + +function PiperTTSModelSelection({ settings }) { + const [loading, setLoading] = useState(true); + const [voices, setVoices] = useState([]); + const [selectedVoice, setSelectedVoice] = useState( + settings?.TTSPiperTTSVoiceModel + ); + + function flushVoices() { + PiperTTSClient.flush() + .then(() => + showToast("All voices flushed from browser storage", "info", { + clear: true, + }) + ) + .catch((e) => console.error(e)); + } + + useEffect(() => { + PiperTTSClient.voices() + .then((voices) => { + if (voices?.length !== 0) return setVoices(voices); + throw new Error("Could not fetch voices from web worker."); + }) + .catch((e) => { + console.error(e); + }) + .finally(() => setLoading(false)); + }, []); + + if (loading) { + return ( +
+ + +
+ ); + } + + return ( +
+
+ +
+ + +
+

+ The "✔" indicates this model is already stored locally and does not + need to be downloaded when run. +

+
+ {!!voices.find((voice) => voice.is_stored) && ( + + )} +
+ ); +} + +function DemoVoiceSample({ voiceId }) { + const playerRef = useRef(null); + const [speaking, setSpeaking] = useState(false); + const [loading, setLoading] = useState(false); + const [audioSrc, setAudioSrc] = useState(null); + + async function speakMessage(e) { + e.preventDefault(); + if (speaking) { + playerRef?.current?.pause(); + return; + } + + try { + if (!audioSrc) { + setLoading(true); + const client = new PiperTTSClient({ voiceId }); + const blobUrl = await client.getAudioBlobForText( + "Hello, welcome to AnythingLLM!" + ); + setAudioSrc(blobUrl); + setLoading(false); + client.worker?.terminate(); + PiperTTSClient._instance = null; + } else { + playerRef.current.play(); + } + } catch (e) { + console.error(e); + setLoading(false); + setSpeaking(false); + } + } + + useEffect(() => { + function setupPlayer() { + if (!playerRef?.current) return; + playerRef.current.addEventListener("play", () => { + setSpeaking(true); + }); + + playerRef.current.addEventListener("pause", () => { + playerRef.current.currentTime = 0; + setSpeaking(false); + setAudioSrc(null); + }); + } + setupPlayer(); + }, []); + + return ( + + ); +} diff --git a/frontend/src/components/UserMenu/AccountModal/index.jsx b/frontend/src/components/UserMenu/AccountModal/index.jsx index cb39fdd59..9fac4aaeb 100644 --- a/frontend/src/components/UserMenu/AccountModal/index.jsx +++ b/frontend/src/components/UserMenu/AccountModal/index.jsx @@ -7,6 +7,7 @@ import { Plus, X } from "@phosphor-icons/react"; export default function AccountModal({ user, hideModal }) { const { pfp, setPfp } = usePfp(); + const handleFileUpload = async (event) => { const file = event.target.files[0]; if (!file) return false; @@ -133,6 +134,10 @@ export default function AccountModal({ user, hideModal }) { required autoComplete="off" /> +

+ Username must be only contain lowercase letters, numbers, + underscores, and hyphens with no spaces +

diff --git a/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/Actions/TTSButton/index.jsx b/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/Actions/TTSButton/index.jsx index 56d32e847..88d063387 100644 --- a/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/Actions/TTSButton/index.jsx +++ b/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/Actions/TTSButton/index.jsx @@ -1,9 +1,11 @@ import { useEffect, useState } from "react"; import NativeTTSMessage from "./native"; import AsyncTTSMessage from "./asyncTts"; +import PiperTTSMessage from "./piperTTS"; import System from "@/models/system"; export default function TTSMessage({ slug, chatId, message }) { + const [settings, setSettings] = useState({}); const [provider, setProvider] = useState("native"); const [loading, setLoading] = useState(true); @@ -11,13 +13,26 @@ export default function TTSMessage({ slug, chatId, message }) { async function getSettings() { const _settings = await System.keys(); setProvider(_settings?.TextToSpeechProvider ?? "native"); + setSettings(_settings); setLoading(false); } getSettings(); }, []); if (!chatId || loading) return null; - if (provider !== "native") - return ; - return ; + + switch (provider) { + case "openai": + case "elevenlabs": + return ; + case "piper_local": + return ( + + ); + default: + return ; + } } diff --git a/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/Actions/TTSButton/piperTTS.jsx b/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/Actions/TTSButton/piperTTS.jsx new file mode 100644 index 000000000..f8431a3bd --- /dev/null +++ b/frontend/src/components/WorkspaceChat/ChatContainer/ChatHistory/HistoricalMessage/Actions/TTSButton/piperTTS.jsx @@ -0,0 +1,91 @@ +import { useEffect, useState, useRef } from "react"; +import { SpeakerHigh, PauseCircle, CircleNotch } from "@phosphor-icons/react"; +import { Tooltip } from "react-tooltip"; +import PiperTTSClient from "@/utils/piperTTS"; + +export default function PiperTTS({ voiceId = null, message }) { + const playerRef = useRef(null); + const [speaking, setSpeaking] = useState(false); + const [loading, setLoading] = useState(false); + const [audioSrc, setAudioSrc] = useState(null); + + async function speakMessage(e) { + e.preventDefault(); + if (speaking) { + playerRef?.current?.pause(); + return; + } + + try { + if (!audioSrc) { + setLoading(true); + const client = new PiperTTSClient({ voiceId }); + const blobUrl = await client.getAudioBlobForText(message); + setAudioSrc(blobUrl); + setLoading(false); + } else { + playerRef.current.play(); + } + } catch (e) { + console.error(e); + setLoading(false); + setSpeaking(false); + } + } + + useEffect(() => { + function setupPlayer() { + if (!playerRef?.current) return; + playerRef.current.addEventListener("play", () => { + setSpeaking(true); + }); + + playerRef.current.addEventListener("pause", () => { + playerRef.current.currentTime = 0; + setSpeaking(false); + }); + } + setupPlayer(); + }, []); + + return ( +
+ + +
+ ); +} diff --git a/frontend/src/components/WorkspaceChat/ChatContainer/PromptInput/AttachItem/index.jsx b/frontend/src/components/WorkspaceChat/ChatContainer/PromptInput/AttachItem/index.jsx index 74f22f90c..bebbfe570 100644 --- a/frontend/src/components/WorkspaceChat/ChatContainer/PromptInput/AttachItem/index.jsx +++ b/frontend/src/components/WorkspaceChat/ChatContainer/PromptInput/AttachItem/index.jsx @@ -1,3 +1,4 @@ +import useUser from "@/hooks/useUser"; import { PaperclipHorizontal } from "@phosphor-icons/react"; import { Tooltip } from "react-tooltip"; @@ -6,6 +7,9 @@ import { Tooltip } from "react-tooltip"; * @returns */ export default function AttachItem() { + const { user } = useUser(); + if (!!user && user.role === "default") return null; + return ( <>