kamp/anything-llm
1
0
Fork 0
mirror of https://github.com/Mintplex-Labs/anything-llm.git synced 2024-11-05 06:20:10 +01:00
Code Issues Packages Projects Releases Wiki Activity

feat: enable HTML uploads from UI (#422)

Browse Source 
resolves #418
This commit is contained in:
Timothy Carambat 2023-12-11 14:40:33 -08:00 committed by GitHub
parent cba66150d7
commit ce9233c258
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 45 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
collector/scripts/watch/convert/as_html.py Normal file
View File

@ -0,0 +1,42 @@
import os, re
from slugify import slugify
from langchain.document_loaders import BSHTMLLoader
from ..utils import guid, file_creation_time, write_to_server_documents, move_source
from ...utils import tokenize
# Process all html-related documents.
def as_html(**kwargs):
parent_dir = kwargs.get('directory', 'hotdir')
filename = kwargs.get('filename')
ext = kwargs.get('ext', '.html')
remove = kwargs.get('remove_on_complete', False)
fullpath = f"{parent_dir}/{filename}{ext}"
loader = BSHTMLLoader(fullpath)
document = loader.load()[0]
content = re.sub(r"\n+", "\n", document.page_content)
if len(content) == 0:
print(f"Resulting text content was empty for {filename}{ext}.")
return(False, f"No text content found in {filename}{ext}")
print(f"-- Working {fullpath} --")
data = {
'id': guid(),
'url': "file://"+os.path.abspath(f"{parent_dir}/processed/{filename}{ext}"),
'title': document.metadata.get('title', f"{filename}{ext}"),
'docAuthor': 'Unknown', # TODO: Find a better author
'description': 'Unknown', # TODO: Find a better description
'docSource': 'an HTML file uploaded by the user.',
'chunkSource': f"{filename}{ext}",
'published': file_creation_time(fullpath),
'wordCount': len(content),
'pageContent': content,
'token_count_estimate': len(tokenize(content))
}
write_to_server_documents(data, f"{slugify(filename)}-{data.get('id')}")
move_source(parent_dir, f"{filename}{ext}", remove=remove)
print(f"[SUCCESS]: {filename}{ext} converted & ready for embedding.\n")
return(True, None)

3
collector/scripts/watch/filetypes.py
View File

@ -3,6 +3,7 @@ from .convert.as_markdown import as_markdown
from .convert.as_pdf import as_pdf
from .convert.as_docx import as_docx, as_odt
from .convert.as_mbox import as_mbox
from .convert.as_html import as_html
FILETYPES = {
'.txt': as_text,
@ -11,10 +12,12 @@ FILETYPES = {
'.docx': as_docx,
'.odt': as_odt,
'.mbox': as_mbox,
'.html': as_html,
}
ACCEPTED_MIMES = {
'text/plain': ['.txt', '.md'],
'text/html': ['.html'],
'application/vnd.openxmlformats-officedocument.wordprocessingml.document': ['.docx'],
'application/vnd.oasis.opendocument.text': ['.odt'],
'application/pdf': ['.pdf'],