kamp/anything-llm
1
0
Fork 0
mirror of https://github.com/Mintplex-Labs/anything-llm.git synced 2024-10-02 08:50:11 +02:00
Code Issues Packages Projects Releases Wiki Activity

normalize parser struct for all file types (#321)

Browse Source
This commit is contained in:
Timothy Carambat 2023-11-01 16:44:02 -07:00 committed by GitHub
parent 1c5d55c425
commit 5441717294
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 3 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
collector/scripts/watch/convert/as_docx.py
View File

@ -61,7 +61,7 @@ def as_odt(**kwargs):
'id': guid(), 'id': guid(),
'url': "file://"+os.path.abspath(f"{parent_dir}/processed/{filename}{ext}"), 'url': "file://"+os.path.abspath(f"{parent_dir}/processed/{filename}{ext}"),
'title': f"{filename}{ext}", 'title': f"{filename}{ext}",
'author': 'Unknown', # TODO: Find a better author 'docAuthor': 'Unknown', # TODO: Find a better author
'description': 'Unknown', # TODO: Find a better bescription 'description': 'Unknown', # TODO: Find a better bescription
'docSource': 'ODT Text file uploaded by the user.', 'docSource': 'ODT Text file uploaded by the user.',
'chunkSource': f"{filename}{ext}", 'chunkSource': f"{filename}{ext}",

5
collector/scripts/watch/convert/as_mbox.py
View File

@ -110,11 +110,8 @@ def as_mbox(**kwargs):
"docAuthor": message["From"], "docAuthor": message["From"],
"description": f"email from {message['From']} to {message['To']}", "description": f"email from {message['From']} to {message['To']}",
"docSource": "mbox file uploaded by the user.", "docSource": "mbox file uploaded by the user.",
"chunkSource": subject,
"published": file_creation_time(fullpath), "published": file_creation_time(fullpath),
"sender": message["From"],
"recipient": message["To"],
"subject": subject,
"date_sent": date_sent,
"wordCount": len(content), "wordCount": len(content),
"pageContent": content, "pageContent": content,
"token_count_estimate": len(tokenize(content)), "token_count_estimate": len(tokenize(content)),

1
collector/scripts/watch/convert/as_text.py
View File

@ -23,6 +23,7 @@ def as_text(**kwargs):
'title': f"{filename}{ext}", 'title': f"{filename}{ext}",
'docAuthor': 'Unknown', # TODO: Find a better author 'docAuthor': 'Unknown', # TODO: Find a better author
'description': 'Unknown', # TODO: Find a better description 'description': 'Unknown', # TODO: Find a better description
'docSource': 'a text file uploaded by the user.',
'chunkSource': f"{filename}{ext}", 'chunkSource': f"{filename}{ext}",
'published': file_creation_time(fullpath), 'published': file_creation_time(fullpath),
'wordCount': len(content), 'wordCount': len(content),