mirror of
https://github.com/Mintplex-Labs/anything-llm.git
synced 2024-10-04 01:40:12 +02:00
Add id to all metadata to prevent errors in frontend document picker (#378)
add id to all metadata to prevent errors in frontend docuemnt picker Co-authored-by: timothycarambat <rambat1010@gmail.com>
This commit is contained in:
parent
73f342eb19
commit
f40309cfdb
@ -1,6 +1,7 @@
|
||||
import os, json, requests, tempfile
|
||||
from requests_html import HTMLSession
|
||||
from langchain.document_loaders import UnstructuredHTMLLoader
|
||||
from .watch.utils import guid
|
||||
|
||||
def fetch_all_publications(subdomain):
|
||||
file_path = f"./outputs/substack-logs/substack-{subdomain}.json"
|
||||
@ -75,6 +76,7 @@ def get_content(article_link):
|
||||
|
||||
def append_meta(publication, text):
|
||||
meta = {
|
||||
'id': guid(),
|
||||
'url': publication.get('canonical_url'),
|
||||
'thumbnail': publication.get('cover_image'),
|
||||
'title': publication.get('title'),
|
||||
|
@ -7,6 +7,7 @@ import os, time
|
||||
import pandas as pd
|
||||
import json
|
||||
from .utils import tokenize, ada_v2_cost
|
||||
from .watch.utils import guid
|
||||
|
||||
def twitter():
|
||||
#get user and number of tweets to read
|
||||
@ -92,6 +93,7 @@ def twitter_meta(row, metadata_only = False):
|
||||
url = f"http://twitter.com/anyuser/status/{row['id']}"
|
||||
title = f"Tweet {row['id']}"
|
||||
meta = {
|
||||
'id': guid(),
|
||||
'url': url,
|
||||
'title': title,
|
||||
'description': 'Tweet from ' + row["Screen Name"],
|
||||
|
@ -1,6 +1,7 @@
|
||||
import json, requests, os, re
|
||||
from slugify import slugify
|
||||
from dotenv import load_dotenv
|
||||
from .watch.utils import guid
|
||||
load_dotenv()
|
||||
|
||||
def is_yt_short(videoId):
|
||||
@ -34,6 +35,7 @@ def clean_text(text):
|
||||
|
||||
def append_meta(video, duration, text):
|
||||
meta = {
|
||||
'id': guid(),
|
||||
'youtubeURL': f"https://youtube.com/watch?v={video.get('id')}",
|
||||
'thumbnail': video.get('thumbnail'),
|
||||
'description': video.get('description'),
|
||||
|
Loading…
Reference in New Issue
Block a user