From f3a6147ffd5cd773fb6af97cc417b965b1678f01 Mon Sep 17 00:00:00 2001 From: Timothy Carambat Date: Tue, 8 Aug 2023 18:02:30 -0700 Subject: [PATCH] Add support for Weaviate VectorDB (#181) --- .vscode/settings.json | 3 +- docker/.env.example | 5 + .../Modals/Settings/ExportImport/index.jsx | 2 +- .../Modals/Settings/LLMSelection/index.jsx | 4 +- .../Modals/Settings/MultiUserMode/index.jsx | 2 +- .../Settings/PasswordProtection/index.jsx | 2 +- .../Modals/Settings/VectorDbs/index.jsx | 49 +- .../src/components/Modals/Settings/index.jsx | 2 +- frontend/src/media/vectordbs/weaviate.png | Bin 0 -> 32173 bytes server/.env.example | 6 + server/endpoints/system.js | 6 + server/package.json | 4 +- server/utils/helpers/camelcase.js | 143 +++++ server/utils/helpers/index.js | 3 + server/utils/helpers/updateENV.js | 11 +- .../weaviate/WEAVIATE_SETUP.md | 17 + .../utils/vectorDbProviders/weaviate/index.js | 503 ++++++++++++++++++ server/yarn.lock | 43 ++ 18 files changed, 794 insertions(+), 11 deletions(-) create mode 100644 frontend/src/media/vectordbs/weaviate.png create mode 100644 server/utils/helpers/camelcase.js create mode 100644 server/utils/vectorDbProviders/weaviate/WEAVIATE_SETUP.md create mode 100644 server/utils/vectorDbProviders/weaviate/index.js diff --git a/.vscode/settings.json b/.vscode/settings.json index 450dd779..c8c7ea99 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,6 @@ { "cSpell.words": [ - "openai" + "openai", + "Weaviate" ] } \ No newline at end of file diff --git a/docker/.env.example b/docker/.env.example index 6b9791eb..77550b6f 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -32,6 +32,11 @@ PINECONE_INDEX= # Enable all below if you are using vector database: LanceDB. # VECTOR_DB="lancedb" +# Enable all below if you are using vector database: Weaviate. +# VECTOR_DB="weaviate" +# WEAVIATE_ENDPOINT="http://localhost:8080" +# WEAVIATE_API_KEY= + # CLOUD DEPLOYMENT VARIRABLES ONLY # AUTH_TOKEN="hunter2" # This is the password to your application if remote hosting. # NO_DEBUG="true" diff --git a/frontend/src/components/Modals/Settings/ExportImport/index.jsx b/frontend/src/components/Modals/Settings/ExportImport/index.jsx index 4099e8c0..e2245d53 100644 --- a/frontend/src/components/Modals/Settings/ExportImport/index.jsx +++ b/frontend/src/components/Modals/Settings/ExportImport/index.jsx @@ -7,7 +7,7 @@ import paths from "../../../../utils/paths"; const noop = () => false; export default function ExportOrImportData({ hideModal = noop }) { return ( -
+

diff --git a/frontend/src/components/Modals/Settings/LLMSelection/index.jsx b/frontend/src/components/Modals/Settings/LLMSelection/index.jsx index 94b75ace..2cf01435 100644 --- a/frontend/src/components/Modals/Settings/LLMSelection/index.jsx +++ b/frontend/src/components/Modals/Settings/LLMSelection/index.jsx @@ -37,7 +37,7 @@ export default function LLMSelection({ setHasChanges(!!error ? true : false); }; return ( -

+

@@ -59,7 +59,7 @@ export default function LLMSelection({

LLM providers

-
+
+

diff --git a/frontend/src/components/Modals/Settings/PasswordProtection/index.jsx b/frontend/src/components/Modals/Settings/PasswordProtection/index.jsx index 387c44bc..5e626912 100644 --- a/frontend/src/components/Modals/Settings/PasswordProtection/index.jsx +++ b/frontend/src/components/Modals/Settings/PasswordProtection/index.jsx @@ -41,7 +41,7 @@ export default function PasswordProtection({ }; return ( -

+

diff --git a/frontend/src/components/Modals/Settings/VectorDbs/index.jsx b/frontend/src/components/Modals/Settings/VectorDbs/index.jsx index c4ad0aec..b1a5a97b 100644 --- a/frontend/src/components/Modals/Settings/VectorDbs/index.jsx +++ b/frontend/src/components/Modals/Settings/VectorDbs/index.jsx @@ -3,6 +3,7 @@ import System from "../../../../models/system"; import ChromaLogo from "../../../../media/vectordbs/chroma.png"; import PineconeLogo from "../../../../media/vectordbs/pinecone.png"; import LanceDbLogo from "../../../../media/vectordbs/lancedb.png"; +import WeaviateLogo from "../../../../media/vectordbs/weaviate.png"; const noop = () => false; export default function VectorDBSelection({ @@ -37,7 +38,7 @@ export default function VectorDBSelection({ setHasChanges(!!error ? true : false); }; return ( -

+

@@ -59,7 +60,7 @@ export default function VectorDBSelection({

Vector database providers

-
+
+
)} + {vectorDB === "weaviate" && ( + <> +
+ + +
+
+ + +
+ + )}
diff --git a/frontend/src/components/Modals/Settings/index.jsx b/frontend/src/components/Modals/Settings/index.jsx index bdf8e6e5..f644c5e1 100644 --- a/frontend/src/components/Modals/Settings/index.jsx +++ b/frontend/src/components/Modals/Settings/index.jsx @@ -46,7 +46,7 @@ export default function SystemSettingsModal({ hideModal = noop }) { className="flex fixed top-0 left-0 right-0 w-full h-full" onClick={hideModal} /> -
+
diff --git a/frontend/src/media/vectordbs/weaviate.png b/frontend/src/media/vectordbs/weaviate.png new file mode 100644 index 0000000000000000000000000000000000000000..d7980bf6a6ba14a9b8e1bbc02188d006f7290eea GIT binary patch literal 32173 zcmeFXWmKHqvM!ngg1fuZI5f~m1Hs)jxN8&KNr1-P9fCWA-~@MfO>lPzPH;JV-`;2K zwbnl8{J8i2xnrC;Mvs2qnl)?As(PlpH7iV6Q3@4_2fhw;;L`nyp{dyhXD6V z`NDwr{`C*hL0a4S%^S4dzkY9%n9zt{iTo}SS}tPtwzlSWE^j=X_{{jkMI;?g4Krj#(ZxaH3^9FVUqORqlr6A8| zYH!PGY-Vp_&gx<7@GAe!8vzf#*GF4(7h?(!TN^uPK93J{e{=A?KL15#qZ34;_|GEX zWM;vqDlYk-tgmk$=&W2^9QfGS+}+(--8ot9oh;ed!C){Okb{kbgXNWj#o5!&#n^+z z&YAjE(!YulH+MF5f;hN9?CmK25^Zc^@9Oe_j_w~^|1l&p(|>g1;Ob=acPD11Z00uR zf7!*Ejh&VKUq)$W%I5-cu`&O*fs5F<{2Rpv@|PxjHpX_AALu+-%*-u}U2RU&Gt{4{y#K=h>MF8#N;1_uybZL z{+r_Zm-Sd*9m{5H_LnJu|6+>&XvHUD=KN0s|4S1!bH{(9yc*-L3I}5+XLEH=2lEef zs^-r2u1==r|BQhD6e#9oZtP-iCddKg;A8>vvH-yXZ2wmE4<`!zXN81^i;S5d57?Lk zWWmYJ0tT9Muz)x@IarK=W?&W&FT1%puZ0PRG4FrL`j006O_HR^s~|28E)H%kFgu8y z2h7C_{73(PSN^vq>h`XtR)47_$ng)(|DydD`~Swx|7x!Psp`L)_5UMBJN(nE`pe6I zPuIWv`&#kpU(SE=?<||_fV(cRD7xVwD_&4SL-$U_r!T1*( z{$qq5#t#1tfv?Pbf1}0TN!{MwMo`Sy&dpeW?cd4&r$PQ>`T5(zuj|i0A(!nR>zAPA z`-6)&Z#l;T6bQC*(YD;RfPv4ql%A3oJL6I$E&4N z_TTMMi#P1Mg#F8tAb$v%X>)eI_9J%W9Trs*y^vgLKTFHlm#eR9-AzJ;?3#7@VwQ@) z-M5m8n9jiv!59}qkwwEdVAQpWn~U0aDnC=e3XBF!I2sYP4K^R4{nxuU)mt|h&SNI* zHCj50?|pj~SQOvSw2Qf42~E4buuxf6KER@pl|>uh1b!V^Gg^LL3-uCSBAsF-n_?lG zc9p6-1`mP{b*g;%QBPn_P+iY833V2H};? zst3G=6hrfdJHGBf2tEik)glK1-#k9en)m}d{ns1RVN5ri{*?icg4~x!A=C61w7A35 z`>Wim7RzOUmz&6*4#}&d61_=oA-NgRhB_Bg(Dz9YOOdue_}w^0Q|H0g{YK#d%TUGC z6S>JJMiU>Tf?TZKBauHU?H^(N)ki=+z4yDHY$~AWuEg@SovH8puTqPOAiUm{so-#ox(KDz@A;qKnmn|h}!7eRUs@0C)EgkYbUWYNcS6k;6u z*OkR_+3_MJ#(W3i4#@J4m-elyXWU~C_@!@SHgKqOkmhfyt{bdIUvQe$ zj@7f_)Jeboyh#hTfA>OO_3o&%EoK;Xu{G+D!Om$PL|No_l3H{g7^VZnmX1l)H9O~C z^m@x9IPeD=O1Rd;3?+p?tIrHSdJ(-MKqrT{#dyDC?C|2-)snq0k^I9Oc05z=&;xdh zZh{bqFi{r)24*jzI-j_rFY>zWI_BXs(_uzR`pMeHGTT%t^eTPgk8%#m6K>1#jj|8A z2a;kJ=hyg>F-W-_47_&RSVD0&=H;)DGYk)))o;}04M zOqq8T8f@_}=(q4($(M*9-k}ghWll;y|45_H;&yS@n2W6u#CBC%?(8Rs7Izb3d@7?( zC4qNk%3&93etCygGn~J8I!1PPh1jE%*mnsaDiH}HiAW=&uF#%F+{210 zy8zjUVHlztZWA9Ok2j0?Zc)zI1R+*1Clkt-DWP7pBvQxHmG8<76r#7dhUfgaa`tV0 zL~Y`rY+_GtIg^a98ZG2rgNwjDpuN=HcMwGwV>BbUK$fx1ceWjY#P-!KFTyj=h%PTU z5FfI`f&k6nS3r@3*YeoR8SY(Q@DSeHn~DRE!Bi9$5pk5I(v>t_1laP&nE4GQZM{vz z>MW_3lM8qtlDBX_-+r3QNpM#$^VQqBB1RC3nQS{jKF3Cqa5dRyQ@MG!!=_AI;prl7bfG zN-eWA606UxF2qnU`<9SpKpRO9-AWdzBs_Z$c&x6VZ%W1Q%GQ?$c`KB&ay}!{Jj+c` zYZD0vbYB7K;b@mRT}!TcNgeE>sx?8!x~bRfdEBu(zJquWRtKrm&7!yc9m4&mCU_m; zbuSpB67x{nQKfgUcysg!G9I8SxAO`4`R6q68jHO+br;3#f`P)KcaR{lz(k%xhWfe5 zBVN5h)D?y=n0qa+21RC21RR>jx9;-B)gX5r19k@y*#3*@A|V1OVzjuvjMhklZ6rJ@ zE{rh+_LO~`NJuN|NZEM5kM`}ucm?n}zQ~h!#()A)HbN@-%jyBa)(4N&zWV9Zs7Oev z8R}nW_!c}2uX@fSNjt2BQa#gZ)N+sgaqfjp>?OtE19!~1+x7U@X$ur_2#n0=+Fih-Tq27t$2!V;1~c4opRk5t&(2b@o#(rzYtk z&tzZ8j3XfAqL*-Xhm!|ifbAg%K8UrquCY>F+E4bOvWZY?aq@yZJ0lVyBk!l|`Q>N9 zas2==99W|Fbpm6J**FqxVQejxqwn)Z{>E7GUCL>eaD7t*eCjs>mehqZV`v){$VC}w zsF11vlq05Zwa^S8j59~AT0(cqdzJNNJlXXW`9|7OFiYqen{7#(Yd@=UOY$~tT~RkD zeCc#w#C+fDb4KxJwB;QLh8(PlT%oX2cfToTKc=p`gShBy^;hO!i^<k*FBRec21H;wE0B$)LEU9gu5KkubI`j~=>TqxektA@#{#}Ub{r#YA8xmlTPp)7 z84b>nDE$BdgbKMhroOtp-z3hni*1MZ6+sP!p_e7uHDyC+=#^Yf<$X;|b%q=Lmaiv~ z)N6&_4AB{wPvX&5c$QXQ{D?oh+$xM-{lPf9^YrRkmrWExd&RBnS=8(z>bI{#o4Xfl zoG5DrkYwpg3nQV^fGZ>?@yJV`;dq)Ts=#h|xA)EGGfOVL3EI|Ao_@Y&MO=Oq{Nl}2 zn&V0j$~>^xOLW|HYGYt)zYbcR4I8IIIx2AL9%4VX2sH+>wH53vw} zB#|O>BnQRig<$ne#hx>`>r05X!TPeUd=IU~19Xzv2lL>W7?_Q|%d0-! z8Y7$dkb_o?z4WJ{(dS&_{+JvFM&Ut{qGD)Ie0X2O>`ZKxZV$>)&!}nEt|VIT`eQYELgMEI&? zRS#Q(5=jVWg&BX!(a5Fpr~M#NFHwEZ_p6|VO9R0}5zoMq6eKui5N#$SPik-9r5EG_ z!`T+2{b*Y71W$op_Hse{dZx(7K(Hjqa&o>~S~h^ozS$<}Gnw~~r0}DN66@u$ zzz)c^5T(J|#+zcv`ylAq3JjA7XMJCoAOxU&pm#?G<+8)8DbQt{I9NaBCM3z?C2(m6j)|+Vq~i$|{zCPj!LPq^&O4AIPZ1=Yc30$@2C$e42su{Wmy&w6V9~ z8xOz$@xj6m@M+U7(g)*H*9yOZoA+z|kA1-gQWFQ9F& zWAB{`oCKjfXB5B_HzE5S=q~6~|McP)7S-0KOwEwpK%H%z6XCP;MmEYvDdn#Wh4eNF zfmR44r8g|m~KDpf3TQx&NnV+@UTZ;3rc2L@rVtSA( z3?AWlQVb;)OhmgIv1AQ{IJ84i^S@YahG07UjdFShcwUe959nllLq)eCwoTT7(aZa2I1wMqJB+@6E^`h?%T5 zCVCnDx-|mzt22Zv1*udD`v)@ul{}ue4XX`a>{6ec{LtgCJfPcwp?`2ciwBUa3W%g5 z8t*yN9bn=kI*k^&9pY1__4>vna(r56a0I6=yWzk=b*S;YH~i>U z)o3@G`BfJ|wcvYtMG?-x8#wWMjH#0@)J_CGDn88QdE`K?MDbXTOd7_J48BbU&h#}@ zS!__~(25Fac=q0RztH@KQ3zfYZYA2n80Ox>1Ai(LJmw&1_sJ}#nvwUdTI}*D3qfKb z&I2Y_iAOxw=8#%Ok%+2^|L0}HWprP9#fHXn?~InB1HqVsRsVkM!u>#C&Pc-)B;ptv zAn7iB^9lYtGy2V_rE<}E`X2dHIveWgO)METFpo8U3zo<` zE~r7K#xAn30ZXPi_ESjU_I9fK-q>+si5_45?8agsb6L(YwoVoUojvf(OS)9Gktl3Z5HVte2L)3|H=dl!Q5o*oE#okzk z4B~~tf+@$}`RG=K<y8^#>I3cCsr%?R?&j!QJU>0*#Tgfnf3dZNOqwe?2v@CiJ;|9-|uSb;|y$OIT zKMFdpCJuZAC*&G9w_8G9M?e%ZjRwG1m~1CMh9RD!$uO}uCA~6s+7EO~6CqhyYm3`A zaXJ3XhFEmdy7jS@C{cvuW%1=S+fSy5* z>&kCe+0k#+fj*u0S@qV0q#I|@@qG>At_RBKyg)(m3Ff{{GJcwd_@b&mJ1drjNo@S6 z9C=}pZ}bcQ%P3S%?0i+bcy^hcNuk{&&?wqE_n@{>E^N1#0<~a)U&Q5|r*P)!JTab5 zyx|2zE%nF;Jxs5M>@?n7z0`+O;pB~YYcq*pFUWq9B>T>Ub%n6GrEh_5{yZl3gh~2F zt#X~p=Z9mb_<`;!=sEW1G4yTLW=3Ad+X3OdeBW~6UnRG76n#=Ha&#XsNKQO6;1p0B zZHh{%t#Vl((L&XhO)#YL7Zl2M2T2z%1!YCU>ZKD|OClHnLM**GI#&4AcqRRBBV6n! zFQo?KFS0jfR3RgC`zmBvN1k)l83B3!b=44@K6ay6AAkY{Qzv~fHSeu|S;t--B?%Yw~x?-I{8R!wns zM;HwQdXff!S^L^X{0N6wD9WZ&O=Z$_J~mr!3d&{wxTRu>OQOXxJ0iD!Op_;Gc93&^ z#9J4Q8o4y`hx#^?nncqtohk95Wze&<95*Tmv)X`MDO|l({RD1iH>0L0uX9aZZRPcge4p-V!Le}D5c1->X+rPG&P-=vJNPd zDN=flyHMJIqO6IXJYZoD;hv^sR{W0MsZCp)y|`gCphXIilD&?(p*(zSv1M4zPsG-3 z&0dU5?og={8|J1ysJBnSTFe!p-Ce%F;)tqkse60edFSLUyY;hx;Dr#?tyyvaALHX) zFG;?&;|bQ!v2X1K*(bvTvx7UUGXQD%6=9z|82=P>@mBRlmb zveE`j5+@)p20`jSS)N5%F9w4Bk4NWiHKrP^qE^a(_9JOVzQmGJ7pB~e8>N!D8+@|) zy15jUY^&iMtBgby4Z}}~P!b13m5nHyC<*24#Omswez10&QS*9>C(^6*RilBHRmn-3 z*J@^|i7P?>{pR#bIZs^~A$kI|c*N*m09=?b!%aE7} zP&%xFdr>0Sxea?FhOA%QurQOA-;ovjDQ;eG1Q2>LD&?n=5bCJ8K`O=Nw z%kFyW$}PDvMw+?E&W1tJ@Sxb1Mgi%Qf}2N@OFav)6UzBfi(tl_6(SRG|yE;l)_fV3+yBr8Y%j z%Rp%#o34{q{;3GlFixpF07)jQP$zohgJC1ZkG%&*7O9pd>_jvq+(EzW#z0ZE59M-!Ivds{+?*eqx$@QR5+ zm->6Cgic$bv)Fq|#pq(k%L0ZstfVrmZo+Y83Ewl!qWcu%XZuF&51>7;)Z^~5FBq;q zWhNrA&tAjp%KCc0HK@m;k@CMK=wb9we;Y8ewfxo#%?JUr5J^s%fnKyyZS!+&`!=Sw zPXnW7QVnjYMq2fwEE;63^Q*7EPM~IX-{kA&>Xn~3z^(NFWG|SLd8hkt_PD45vgx)N ziX>A7Bck~CI+LQZo%|kP)8cd=^>g>>rUb$V@Z(dXBx@p@zW6cExb+FCjAunfq1g-F19x2Zm^I7Eu3al`0dKC?@6N2w5>=yNBI}qc&bx z<$oBw+5UrV&AQCbQ|pXd43@c{YnRtS!|6SW>jB;z$^?%M6NZ+~rhm?EprQJSb0+gZ zzoc8|CYvB|IAdf>yT-J$c$S^{@egF9Gwgzu@IFrH&L-5_t^$p!#+62bT5glVMyEBU zBMr(K54pk*isCxy0JVvt5AWV2YDC_5MxXL-w94up*XIz3gKj^%nh@}4HRNYkq%Ff0 zV}FD%s$wiemLm=Y5kMeg@sfDVqYM$dkw3t>pA<#yvKeaB4X}-^0bigoj}!z^vnB61 zMf#=URN@xJ??_CJ?p1ms8AGSG4I@$aR&7{amldk%$L_RpXm4TS>k}Eb;a|QDA6d&) z;=?N>?l6bf@CMlvguQQ)b6Y42`?P{{in)|upop!lITD4HNBy)4zk|1+PAv~ri%^rx z=f+UBnh%o6CeP;f^N4q|u67&XwVhX(vk9@hx(?24vq{|-hP<}|2RI<^t>{hpdhZLIpPFr9`bNJ=A( zI53&#w&F>(=pAK{AwQnM)OKa0XM&2!IH`cS9K!QZj(+`P>3&N_=GMj?t=#JdFpd+0 zb69Z;;$CP}AX_v4ftN@psp?tZs*y_oI@vNus;z>q`xvz)=9Cubs3))rT;lTUsq#Ah zlI&?4zX_w(lhf#jj5I^Li=f?hpU1~5=Bo=+vZhe36ej4gv8g`-^aeHmTpr7r>P!dS za3u;DyPdDqy=D)9Ny!`l=-^MQ>*Ws&(Mt`$B4jzw?Y6dI=a`$G_oC<4s?(RsyseF@ zJC{vAzM@Y$*Mi$ZcN?q??sMn!trahET?`z023sBR;ge)5RnS^5dw6>x)gcOr)%Pye zuxX=@M*;xkz(YaL@~HNLMiw)4bCSMGz2!5X_zZw8X_D~uTFS6!kM$QFk`x@w2kGP- z0t{8VLLtew2FKmXW#P`$PcCC)yAR9P8-5KEn}KapZ0?_481vIk< zcIqqAly^@M_H9acU5>;0*3!E+9LP{oM~}_jx~J}Vn9}^Ty)n^OYCa>h^ZAmjq2*>_ zKb#bXc%j-@jE%p%x`ZB?Jwtk=)P*F%m?2m(kQ)o>xq0`G|KOY)&1^b8^W-N3@;; zGA>d%DJgS4KOgFR*mm1O)jYCynt{e7wU~@VC;gtUGLI#ri&jKIFWAu2QG@~i$Q?~# z`$1v6>I@K{Uh`NG<-rR>Hmd@%>>(u6wq;d)9Em#CZpkumFV}Y5`Y1r0bzybS`j@gE z=d{Vk)UmtRv~sj)AGKBPlM-4PCz&aShlo7W(7KLPXNGiQ-bOR-F?`gKT@4x{sSJ0LQK1_dZfn$s-%SfXF_zzi%#X7i zN0N29%NN*z6q4Y=9ns_r;vP&m(@4#569+1h6?2o>FR+AC`RmBh>BWO|SI{6+YLc0= zdFF*Or=8Cgzk`|F`&b&5n{nBK_XLPLM)vRwcTLWnnHM9@j+K0YT#UVX*r4({91HOQ zP!7kEh-#rTRG(ITT&V>`BuU=TaR}}-+Ea#6yG}?JLi7~p-k|U>EIai;cRAaCRj54S zl(p`rW#;+nSmQko76tavS~7!VdV}5LL?77~NuroA1+NMUBsIa`cF+8LUv392_T3KS z!?I93X$uhU`j0n9V~s4Dl+5()J79{Ve_q3|&O!UfZoTca9K|MBh}|M7y2T)#%q*55 zX895wYB>B6_n5oaD3u6aK513G_|@bj|Vn&EVYQVJy=&Y zVyR+eIPHz7l&;!Vkug;A6cN$N(S1E|2c3$%?sn6rSxtC9O;WS9xEWP8vGUzrl$KR) zqqpHmvH=VmYtGdRNx73wx__;GI?QO~xb3yS#jM9gx=QYk-^u6{Lr0c*Ugs-8rAPHM zADi3SdhjQ^`BzP&F=6zi2dE)nxguA35+*>JLxGdNAN`C?QTKMVCRh5d48<7!;P8;1 zCw~Vq^eJWWtmz~%;yM+7CU|d)Me`&*YSrl*TjR3#;L?`6=cha540oXb_K}XH=12z7 zb04kNBkEuCX}JCLC-ngsG?Cny>A}qs~Rz^mt~-4Cpd@ z7(J@ONC%`VYcB=54h5uN6%gVS;vqThl1F$PC@10%Gj`+_Ldqy#rlAfr!R{>~0#=G1 z9D0si`qi#;<(|WfA2zJ-Sx;R$dcR6_^g2Fp2j)y0${6F_2+-bzSVqT_IA3^z<>(wu zYm`Cy(HW{AX`pI#19|#8@(~&9S(O?j5l@?qkV%aM_+1w~=exu2jKn^pcpa)?7oBlW zT%6Y}4I0{+Cl!Y@Y12akI7H9Pq@`NhWV-=%uYrZn&cq?Zu#4-dKI4|zm31=3an}* z6ge-0VPHLziFz4Aq&j7M`s-GHn0>;G@uwN+9a761WyLZ7Z1CrVkQqd!5W4c;v~7Yr zl7Tpp>X#YROT0UMa2jB@UZ$O?*-zE*%0XsUs0jjN~sYu%E6Igp{3K0RI>}Q zg;oAWbZ^(b9p2O&=7VcL22bj}Ue9>rdy6>2WAW+EhfGa&e$Xz{5eJ}9D-KLLun<|+ zO3xyJyRAvJzTKd|NEx6y*qLxX##$Z;twUZ0^v*Nzm#exqt-w>xtDA4}r>A;LtSDqy z4AkN+s3Z|Fm!zYE$W#ZP>qS=w;)+^nbyB~bvz7uXLvl9kGvG;MI?o!tJ0Z@!3Aq~! zw~4x>S}n!P&BPVI(LKIA3TE>^bB*H}=CDg`7knjG%3#%V#vkuHda+6A{Sxn$Xbj|p z)IfjG)##ysO5C4=KNT$h(d>5rvAs?HQ(de{5xo&)iGQoZisfRnx3G3p5pW37KJuCJlZ+ zEDaj)p4FKCO;Y@B8t(zRpmEkwODeRx_4D+?@r6!cj@+Ry9W@!Tr!?atSa2J&*v0S7 zcaR?evMR{Q8BTAe6uD9WPzD#clzk+A;zb86K*(`k zBZc`*>G1k%Uuv^un;MYD$IIRwq7{Eq z*Q7^f{J<#qbLjm~V=>@BY1d$Uibz1-e(|2b%HvydyBBJul?T0A^QhPC(|A*#^Z|is zXFiAskqyW9+B8x_sY`x1n`sblm^GsT{MardSPLlB?$DOxLTag=bjWl+5p^pQjg=Ka zkbXvrt#DbTNsWr0p1s+d;k@A0JFtcP(QN?Ry$88%r|0;_v-XAW)WvWYnXaSX^JWGl zM755+6pOlBp;@!-OowMWzSMuW%8)9iy;Ggar6%1!uV`7wIJ2&TA~1BtiUw?Q&V#BB zWogtb$~$ocPjLL{_)SKN^ct&zQN617K#pGyThX#0UtQtlnZDCmOc#pCEIbn#b5HVC zzf;SmkvDb=L*s3b`16ZN@rQ3E{=O)k3sOSQ#7lBtX3rne@kP{$2I%Mwx8csv6+UJB z_um+Nc&%ga?T@IV!Elh&cqf~aSW`9e2Jb90#a7mD#C)JjubRPtV@Phg%}&`>}nhh7K?(rB%NYHpPO`eg$7Y4EL#IPI{1yl=tn z#cUVwv-w>8jxq;1nG?>}*hl`o$TPF@$+KY)W?YK8iWDw-V7ig+^|z zfRnx4ETHVqdny~PAKk&*KHriN4Zlp|o$~8ZO{*FLh)}jIQxl9Ij~XDMy011WER0M3 zNThRMRd~J>iUVk-a_Sf5TQwl51u5Nmv~F4)E^t(vkA3$^!C$iUnuKPuWj?A2LNAz@ z(?g{5R!rQ~Gt#9Wkr(zSZkCknN56T5-*4jGr0Yy2s}iZx)YRRPB5FtPOMI9CZNqcw ziHanVkj>e!$_UzpZ$WWqgkD(JJHt5A`*JfOpiebXJah4m`Q3UG*=FWxyMg$O^}r6w zRBed~$AjAW-0!OoiblrdQ`f|zT zlQW^cwST^QInC@5!nwMMNq7II&^%E-mcb!_PMU6A+|h)S5{{!z3y?|RYBC)rA|)v) z$-~a!*{x38p5Tm;4}{XPpPlYw@%h-{UQ8_U z1QMxNcQcSzb2d`LuE|8K&y*I0qZ5iH0DIX>!l41tSA5f-$PFx=T&nHzSARyUEZ}P@ zp3r~ywO@R{mEW7Au&k%QvmaekTcku9afP> zyuD{g!ZmHYz8oGyJfCTHL;n*Urb(k>oXzFutEuogqt7kEHLGoEaLypEK948o)P0Ph zi>lYK%n0P9#PfqqTZg_C2txE{r9ESGhyPSOn!?@H%fjsg>vtwwWb-;bV{vxh|zzSn#YS;OC<&_9I98eb4* z2C1UrRWWo(8A{cPX-YqQF?`OyjozxOqwr#u&}b zO-K$p;7z8b+vWWbJ%*Xd+$s}&Ucdl1MN4I03mRyhYFlq?+O*bj1|MrJ#SwGX?1t|> zot}3k*q=8-ed|B)^RRF7avMbrT`pj@a$%%!p6%Dl)2d|8N8% z+RL6Li2{!-0iKjI8X`7T@~scObd8o!`h&SG;=Gn=(#_ad(^3Nb;flhX2q*#4rhJjR6=Dz*RW$jo6e^HGh$FfcgOh&qa5Yfk5F^4m zLH>;1@HH}fx?}K~4_Jk{m@G?CFKr=$TEJ_h?4{#XS=U#;-x-F7ldYEOxnv!$kdO}bKc7{+9`<#avE&{Zx(1*B}NB#=d4Sqvpj_3j5vBF{sUL9ctY5pNhf6^?NO5_I zIta7UF6_N&V2A*Uiet?LXPouHTvSTK6?Rada}sC)w_BdAx0f1GVFKHG!JxjeBP1|@ zZkS*P4dOkTH@+ij%pba2)9aUslY;VH2=%OsJTX2FIaxm0`yMe0J^^|$ev@#bZb)cV zhwbJ+_J(|=^M;L-c#ro3j~*Iv7GaOs>UuSJXPKEdE;$%4golk3o#5&%zi=O$3!BQ~ zn|#IJ%bk)9<(YItBE=ZrvnIewE5$TC?j3T4ty3u_7fqY9cTU@ENvZBn{Zr+MSIt)= zg-CzOP)rt@{XVs9%r0N#fVY-29Dik0p&W9+c@cS~$8E67fwxqggX3ZxUq2|-CiY>q zNowuBxT#I7@dq4zGOx#DRZNV)=CvNO3C?yU#icb?jxGXi%FK$_3W)c9pzKFzDw@$u zinUPJQ>HAWx|=J-%7c1&OEd? z>t+3lt(wOwmsj>KKdC5Jv~TOycI}ye8M7b5*3O%#ylQL0uuuUccIkYyTc0oY#K75& z)2@mQiphz{og&c=;n5!UkGQ7(@wrEbrz9;^zDI|IXCy44L3l2DkIA?`K&sf`$5r40 zA+!R31r8)E$11`NCXtUTU^TV9-j56`@*M49zT_N9-zLgt@X=v5ACt}QH5V0Wl#i4+ z_Er9{zbYXJe2wN>T>6WuB-grl@yL0lsT+7e1z%X>OqWL@F~Up3!-#5K^&bsRF#1kH zeMYQMY`@&DWBGU>kEnUdw64sl3CoM5n9Z6CfR~U&$i`-ns8)0SsEs-N_ zZ>+Z3i5WOz5t_O&T~-rEr<&wo!DoXZ@K zD%gn46x>pG7044or*RMD)f-Mv*bT*Y?p4JTB*1z8F-?V#&sz0agc>_KF3zw1Ui|x% z3qtT#L{>ia$Z`D!yrx5YqIfDYbsd%k3TD69dtD3zj;?bW#=KIrN*v3qX+Kfju=jqK zL~9$rmYG<#iM?s$?0tBec7h3w9pAQikZ;wrLX(;)g&26P@*-1r?pNydTr-$EjClUS7(VY}M3-DR_kH^2i01t?H!tN6`oRylkX=n#Ez_2t#u!_4jLairT1W&&XO)-FIRh zUhn9>JG27JX_Hkir@P9OP#PFZE)m5)l9pkMZWW&)hNfdMBM{Nv-43fIda<17=&S0n zu}pk`e)zWKPsWvWju0R$60f4t9!PIlK;Y%}Jh>~&q4S=71+GSh$oV~KHG+=uIFd13 z+>QA)$LGRhBAuD(h3KFIE2^ZD;sP1ek55^)gJ0{$6P%Xl>58M>j2FJ6l2_s_+v$>J z6Oxg1)dYP^1i;#(#KX^c=!rO5H`)X4nQ3WqL)_AxS$aB)jnmD_+VCn3=4%m=qcM{T z#d-@LcDtL;Yd5}J-~V;xs~xX)c(lFTNAkiXPdpWWdSUf-aZ10#EVsl0KzH{vP1n1O zYvi|5BG87HRW6)-*1zO1tsPCwYmGAU!?@T#SByaT{WpKLtuU;i^XVRR?4Dcy$59I0FQ1y%t8~ObG*Syiact#q^ix}d z$o|8pXvi?t9d#-XeUO>I6lW35td(^?SJU_VJiH0bReXrW0#QTzKx^vKUa z>)Q3r@~w?9hiL|Vicw%!JTE(!!K{mq@uXO1TGJw4*P8AHpWx=By;z1IqI4>HK{%1U zMT?2AvgH#DetBt6M!(0|D|U3RH7v*@rAwT(?8vjm5{=`0Fr&ug1Dy>?W1YDeWo(ilV#d+DNhK z>k1Ii5y7L7@@Gb!Mg)jUIGyBb%3&c)e&awCXB1%|lFu!9+QiKbrrll^e~%dwR(S0{ zEgQBaqY^w=8(R66XdNd5+sj+gew-&)F^s@mkN5s(ba4aq(XoF-E%(CoqB zS45LIFH>04pL0U&j&EjhM%h&hkR%Ej&uN;mX|QTlo6+yQ({wUAu3!3U^oB-1&MS1I8+0b%GAjiQ*n^p2G`%Ob zNunV8vrJqlb+*W3)taUd$w=UPlMGS#ERZ7P(jqg$nq#pl$UwfQJCY<~1hCT*6wfR@ z4XMBZ9RvDDFdI;D{fR{QL?&RD=s!N0W{_;HJN#%YLIp|g6Nwdzng&_mj1DX9BnDBS zqc~X$7#DGk((EZoItyUrBgQXa%POZdo-M~6VU8g$^dp;C%!v$X#OK86?Q+BY*}Px% zM^=zy4{v}Khp+`cdzjSM*cj9;ld&}M@?@|WFWmNK0^+=E($h^qhXM@g>wF+dN0B~B zCICc)7WU*ZgnfTY!JdAk!joqRmhU}P(!<+!n6B5`JvPS{Ov{O08bcxGJpx6Q@3bfR zk9Nz&IN2d-IB}91wHMBZ$nDV|J=*=r4aCp|Jjy}(6UV&3`Jq$&8u;;2u_!8A_+5gZ z3J$GXxU{Hc-6gB2J3+d@2_%!m&9%%CVC`VxBZZt`Woa+1#&5)F59~v_}Vj@AFX4F^yb%4)n}Kziw<>XwAZ@0A)Lpf~_~KtXd4li%M7Np@`K!Dv{xa z?ek{1$M(eq!f|T&@skP!wDvJ{c!Sn>t&Sb24p#cNJ*Uqn?urSHPG%t@zt*^t15xx3qZX%CzlEC)ScOh)FXkK?C+Lt>s_$(GSW5 z<@u*9Pez2FLo$~~n^KX_cGrf%zVQ_+pwi0DQbE2Yx9%EtX1I<0AB8J{N3Ms!Vz}nt z6Ep@=G}}sWoO(}|lC-oo5urEO;kk%=i#YgHIyj1(@46Wl!er9wv5WmZPW)0G*M$%Z zzDb|S{jda*f`b(%A|RW)%{$_JU21_d^R7Zowo)0!5LETf5PoWJJSC`<+V&ev_F4M zM=EXO(p3+^WY{e>3sT*0dYEHuBUZ4nmsN5zX+9>*Ms1wRv-Bqk$U^qEU(i#$(v5JI zNu&mGdgk@L;hOl#TyRI4^bKhXmO301{SlxTxnadoNAAZ=vfE8z30DgoijF2UtbG$J z*-;>O_$gl4p#N4l(HAbn(A59WdZ380!vZ?5Xmj$E>oHC*;c2#fF?OU&*e4D~lN@b? z2sFSaLdlVx&)XLg12S{xXQK{2uu_@R;XHrx8Ke6GqWr5@c@{j_VEO~Q25*1a%2$Pp zv-f|46#HXBl@z2go%lBNVtVk3ZZ;OylIUmF@{A-@)v)eM0dzjk(MoTJ4poMIl+dQd zRWZv0d;()C*8`XwkUg#d%={zAleK9@vrS%|W_FBwE(9V!a~3z$9jvP>%IC5DdCGY4 zqd7}xvpKWIZ0XorMYvhQ(tkHTreuTmKTqQosdVJkS#@S7HFTL4FUM~R>p0fjc^FG1(Ub>e($_9 z-MF2fQu;9QgE|4Z$jCb2CaFMp@CI9)CKpqqJ{C#eO)KR_`=be{w(j`{Gl)D{Q%~IgqrI={imMB@{USiH5G-gQxNFd0 z!QI{6-5r88?$EefaEFG*rGduX-9m7e2KsRBzqn(Zr~SO=uG*`rR?RsH_iAgYDdqk& zvB3|x2$DvrodgiYPI9ghw5_jo=SgV*)X-gz!R{d`}3Wgjj z^ZpFsp2VpXNCO9rcz2hL`7jQcG+j~+{xAfQQ^$!}v0v(4NbAg}q?J$4spVN!laWy; zm7-Rqt7QCkxr?c&4YWRu&_`d2_t87wBIdakaBFO^tC55V+MF%#luIX?8JcOR{cz>$ z$c>eBCYB0-c=5>;#tY5oPt>o|iyrg%FV(HlA4G~fe7r`Kx-u*krB^Efn2u4m#vW3< zLbY+^F6?(3(K@1N_G%{fYXQgQY`yF{6lB^QXwH{=$ydOsSJr?`o7EmKj(T0~+*&@3 zECh^6^g|k_2Bn|7{*}t+kSzl}$7uxFBIN?*%ke+Rv6K?~^8nc9*36S!fjygcT#f!;Dg#J10 z(tsZSHPbvrtIA;_L|`8WZ%Z0Bm?%##mro@lo3TVqrZ`P-bRnOc|2#pnIR1g(kj!Ai zd0rR6L&-s6|1VA)siUEV z)F{C(dmsXozE^uk-BoOxH}7I+?RppA)?t!T8glpUW=H^h99gC*t8omBM(PW}HlG$r z3s&aUHc0ll87i;H{VzD>>*c#_9_*3iy$Dwdw(PTWd0m5-ke*(wl8P&{&h5@)Tp|Cx z5AVAYy~5kG#w(C*Dy_iMyjT)Cu_J?ZIVokD7t@?E=-Gn7gVO^J|CVo6KhKFZOy$## z=tSSpOKzCn==M8qyrjwPiva`d&l%M^ZEnShDtgQK~7&9{1h63gvI+AjUAz=0a@eCVp`kwV1E-?b`ELzq}xxh!26!Sbcg3rF0V! zS#Fu8_ooV`_a6XjxRER20nq*=&MtCdpbKN$jPuyUE3!Ot@>6@7TBdBnpt z*{C01GB<1ws7abOp z4A7d6MK3ifS!Ta`Ixx9INc)HX%XVp50ay9(HA47cc^jk5_T-H;i6@9C7RwOlSq#6@Z**_T?YS1fv(Qm_iKO`!Aw|De~u`N3ENLUIx zoXwZwc5dt5C&~W`RJz|qDiQEF9ka<^*@*#>^TMZFTm`fn@ zbl^0L%%=g`)R}bt@p1w>2ju?9>a%+vGM8liJoD9oC`QO%?Sl+B(X}^8J)$x_f9I~~ zM_ih0`l6OHe`Leu-l=oET$vo?VoO+5r+;E<&;w0LyEU_b{q@X#U3$4TvI?DXp)Wi> zJ4^m^vcD*hQ%{XSW;b|e#+AEY4S)4(q9-2LYaFRw_iGplWog#fZoTnCrQ)ZLm5Vax zwod-)w&MuVy&v%=!OJZ*YWg4_=Q{W@V?QGi@qLZ4A9>o|>5no_;I%Eq&`Yv# z-{apvjn}1SWKyXzfJ>c;zlJNE=WvJI_>EIM#V%!CEo+IxQ>jI?*=9<10In#E72Sw z#xoy(p(=}W==k4mHR7tL0E~?WZVyYzl_emV>l_AToiO8_Gu%P5^=QO55!_6U!Xw+{ z0$^JZ+{!fDu@@}YvOW#O47wAkKWV_LF(aTcQD5K3jdc4_J-_XCbT+{wMpW4(K|K`rcKz z737d(&*Vh8s5Sl^0l~iS?uaykkEyH5pj1VK|Cvh1V9Vw$Au)O?q6@wLX93Ra6k%Q1 zqNF$vw1O{Bo-__5Go-xFd|%BG4rciJn9fHdQe>pE1(3m)RTZ{)r_4;J)LVoK@-@*W z)I4kCX6M*>YKYZT-500t&}P8eC_LsUpdl2{Y$Vc~5~x9c2-hlP>(unLU61Ha&^tB6 zZ=cp?pw?piZy|oVg5gA4+SNB!z}kqm?(k!NyQLL#&)q=@{_LRI`ZWXGf}%%e{ei(>yo*Whh|nsR^P^EHQ@mLXeEEcIw3>$ zZupZF0u~E7AtNVZtBxr(N=P5=yoUiow4j|Z*xvwOpPyeV+JEThr_{WEA@@?j-w5aC z5LwFZnhtQo**O!U~A7 zpg(J7d5^imG`P_#ZOhvZBx2RBwLc?|7|SM1{d{&GIPm{e`db#X_Ln&MVz( zsY}iFr9G@d&2ssp=PJjNdSqL7-v;wKZsV>@K*bL$1*VqeHK+nCxa2Q_vK13l;~6IX z$H@Fn{_iV5B%KWW$R^BZaB{gjAo;qWj$lW*2yV@X4{uF?iw@}DL1mOxi|j>Rx`mn0 zh_%d06W`MSeS-4e%j&rrUb4N=J4bau zCrU7ZLDQ-z_u671)8C+89MlT)!AI>0m(+wqdi+sJa!M0AhHW>5k^dYF)b(=r#-fPr z(bGfSlYir!KHe2y?4eL874uPgNT}2B>(wsh^qqmWCg`#av=*q+3FUV6jS(Kde7+|e zp$Cp9MMKeRmS|tj(q3WJGv&)OEacE0>V zyR7fKcrJzHR@I+Z*JBXT{uU5dMIH|~V(?kr*o4%P&S)o+*EoT3C4_x6yF!OnH@Y6y zmm}Dg;!y~1NLIUayz+1McPTdQ?b`g*SWDn45v?A$5StjAKkBi9wbFQ;?A%W2-?5|3 z&PD))6vpG8nq%KgQVPCub{Gqdl3MW53$ZqwHlHJuh!Pz`$gaQI1fp$7#-){vN3`aK z3en*tpwrKENMxjpp=mL5M3+p> zLrXjhZb(Sq|FhTYqDh)T-D)Mtez?NgrEbRum)7GZc+>~Lxg>ly3!U+Jk*< zPry_y9K&LDI@A1EV@b>$M&MIN44tg7&z1N-L2UlpZj-)x7Tk9w=sb76;X)hQ7Q3}s zHmi;EtGmmsi~W4J`j_L|q&eT)Ci|$^s#KQ5z|xoMv0b(a;$g(rW*EgzlRxIt2T;GY zR2JD)OC146txTbBpJKzO2{NOOvgBm7R^%J}H_5ccbM2>Q$IG$<3Ek7B3*4TGipv`R ztQhT(E?>IYB(t9b8&F;-rGj`wm7;gB-bi|J!M}U-_^G-9eqwo}y&s7y^IHt}1l@2e z=epaB39*`~gicvqyn?fw63s>PN6TAbP->w)Ki5*}0*gSH1%~jyE*jN`l7dnHbmr@7 z@_{soSk^vMv!W2HDs#*Jvxua1Ca`TcM~?v=?Q)%(>5TK>T+ME&k1u5j}Q7g1xOQ;u- z(;%A51pblc-q7`H&bCZ;DNm`JI?sVTv|OtK9A&-C4w0+v^k&}P9mH~-`t){V>A&8m zD_GTm8wa=x`sv#@%PDUwaCdAICJ*HTw;i?ze+{k9+&rUQ)Hy7q*Rw&NQB;kqg|{1S z5UmE7kf*;dHS}%h_VZCOqT>W?hnOOyrU@LibmCGOW;6H7q4(kIonOpPG%d)m5JC4|clF@VBjx-N!^+25jC}-8&?A%pXyW5gGdTs&Rq(4Cbm_nnTJV zGV=QK!!S-4v)p&!kk|1({6E%F5lE&!-|?o=_zNyM=Il)Ye|z}w!STK&SZ zm!>K{#~cb1K2E#A#>FICa77fW^01DqIpKs%zlyfSa<=lTIO4LPW#y`tt`a51o4i#d z;Z^bFN7RyQU*_@d9)4c{#_{g*cY$syq3{cIvmKL2+Da^3gcs zW>!~>ensJdCvv_j+fmy+6Fq{=Ho>Snt#m+}UGAp%<8M(Dh9`_O^1hBCXKh{ie0G2- zSXmmO_~R+%Rr*n^|I{c$N|<-mqKX??dT}&G)LK zuqpR{p}08cEuD{U=Kr>}KJK69@plrrFa5ro7Ut;xbTiR%GNIzRL64b2x#!cDIxUo+IXzEMSnneO{1OG+`Kse^)(M_z!pj4hdc((eYt~#cH z27O^HnKihKDV5(egg@GI3s$NVSVw2*zkCmUN37*M>2D__$}4AxVh>^qHKo{Cxnf&# za*Gk-nnB#_kmL7==tEr{3Y_K}=!9w58heBUmEcalUUL?F>J~~Or|pKvx>p(|KLk;Z z62cl$eD%n=(bzxE{`l0C%3UFWV zx;W?(o}59)K;brn6VOl7PE*?SC^(17&iYe65}5Punk^^%iE;Mm`2)HtR!4lN2vUmr zq_fZOdKB+{zPEWjaTQ*bh1z=7{BP*^sVmFAx!C>md2&5loU7LCXeeoobd?aCOVCUP zeaZVdc*HnaTHrtrZ6BzZQxs`PY1UOK^fs$IuiXv;lmv@W#ESShOI?m{C|vNRHM?ud zJ5{H06Q=5_6j++K+o#--7{eCrEIEQ-Kpy;#(#>2ItrZC#)tOz)AMd9_uZULTM{cV| zmAsf{U>n`4Uldt>_ClMsG>7fa~Be86+jygtHq*c{uQ@?JRshAw|j zPwpOzbT^l+giTX_1&c)Ir;0G_FaEi7p2`)stTN0R;*o8y->!tO4t11OQk=9jq8G@} zu%G_ILU!42S$H^XQZ0TmtiCxBgG+PYE{D}L8DqINnm-2aBR>ug`6QBZ&N*onw9+{an6l0idmy zMvP^K&uwghIYrC1ie?a^LM8G~GUj@lj}#=?@NUL)7O$nXk``OSi7Ghpk5f{Lixoik;@>taGKMHfuZpcXUezFVOGVl0rd+y6BJ7 zEB37@i&mO4*fyN2nNo0nqM>7d7TPqy%`YQEsB28?bsMp=V3@*pVX2e^die@$P2+rQ ztdblf0NvS^2=aowhNGP5EQeZYX^kztuIfBzbKdiGi(3o8c5&iwz#9`;;a3Swc7-P% zGE2dyYu<5)Jj>Eh@Vk<_@MmaK+WtKc|<4f&dGHP9z&S^hKosqY0_ zy(9QBNhC}AT%ooL0pkU`e)Chgl3W9u=Fnm~`e(XPWKV_oHc#g<{>{Uun^C>Z7f}^~ zjR(0r%3QC4kB0$rS7mMP7wEuBq$O7`t<91Gk9CF_F9zBHcb~~qB)b&@hgIagQsw|P zd{~X=F(9m$s8jMjw@7+fM2}317hqqXfEn6iVkz(aq*BCc;GkM8$CG7ADV%!9y@8kvx{DK7 z{~5pyN>fW|FD)$Opt4m}&B|N>F`t}80J4HN0Sjqq9!ie7z61`RI;TVvIA|DFSlWau zTcOd~@?q)@Mjz#4=Q10AZog)jw*X8&(j#-Zvk1%*1aa**hS@EO1uFj5{rj2i> zbPoM5n;wCUbyaAi4FwH|(qSCfnKH&iVRR*~$*}h=$_pQ|>>~v4B0vvjQhDKMqMN>G zCPXcs{XQqDH4)%+Y)tl4rDA07Ol3+s>!c(ax{MhE%9047uL{m{nf(_R7A3$j}LYdF<+H3c{fma5z~!((sF1^t~r6$YKk0CZFtBczws?kuLk*ltJ- z%^HJ5emRZ(&hZD|r#P%2uqbd|&R5Z3F_M3V+}TXYyyq}kBBb_Pv+7|ejB z5d3GI-O#$KO(Y*+$?q%9sBOz5l0zADoJhP;BloDyJh6dAl3)(>8LLHPqZf22Jyr=> zrS=?fKZ8|En606>9Ap0z(RP*rEm4BQPFLt^(u0|zrF={LdB;;1Q@62k`??z}^(fez z%My5+Z{Cv=ZOYyMYm$fdGl(M~NEfzglx=vXF2LU=R&YXAh;JsjwgR)nS1ojxaLK)P z@(*7Da&X}5np!{GZ$8wDa&7_+12KPcuJxBN5w=Wy*0N`;vK_q?(VA1j z0R&YjgGkoYtZ~}GJJ|$cG*$wQG6NV5I6C+3Ea~xst88XHXLH6GNN za@HuR36(J-D(n=+U)if^XJU+jiAO1Fe4{g3#|zChwm|cd26}A`+*E5;$n63R1q>Td&neo$%hNPUDVO9$nJz�li{q5(uvc3wpfvw7! zo6&xu>9xR6V|J4yc>*fzh1d%tZ-DUtPflaEZ-_|y#cTb9Nfr|WNL zgaj%ShT|MX?D&m^3?fx!qT1LH(#P2MU^t(8j{kRhw6PdB@%tqDPgC&ngb8jt>Kfoy zi_x4yuWH-PzWftj#^4H}U96N0J+Gk5a_kaU=lvBqU4$W}ELjb1QE)~nAz-QqnFv@`qf(6dcI_2sLWPHtdPRk_t#on9WTFv#gF1UT_|qF zypNYZ*5B;V<)>^ph(cNB!ffeoMXZSuAl*UE6l?z=0+Nv+Dk2nF?cSoWeLwVMGBlJa z=n#4TDDU_4B*=^Gs*zI#5(?Vrx*WQokl%n?@XsXt^S(ULJ`Toe}3OE6V9EpyLV zM?FC7FXXLv-NK2dlClkwZ*U5Wu|aDu$6keK6g`HM&e=MH+vA5K5A&myD-HE>19;g|lPK8o?~nrwbNUR#3wEYE$GsVSPo zQl<@jr`_Hknw_|Wb~fZM9N|>@nofjwR%|ef4g*U z)Ffy(o6fY4wZO`nlS{BA{thyBRx(Q@(u!AviE$t+MOdU9$RQzw&(bY=c(k!?ZfCrF z$o*7-)og70>TK6Nvb@zh4mfyXX?_O|A(y@Cmegizthe3L& zgoU&5k_y#b0j%iGJ5XDCX20=S8D+YK$mj~IuniEJU0&dkc!DK^@dy~qp674~YN~tR ziWc{P=3{dUX6t*Cifs2b1*1*?7{gfU)9gF5^4h!;WTSlf#APL0X`(}%%h8CYA*BMb z<;M5-t7a zo%X|kDm@)vGVk?e%wY}x;OKehAp9k{dpq_G;R$U8C~uj^yKzFh4=n|E!W*n)f}a9} z48yLdvl{Y07OmYkyU2oUPWe0;9imj~Xo+h7#JYVoU*tI}-P<+ECnG6;eho$FhORVJ zg$@z|+E__+q6WOt3RLMh_C8i`rrWdZ^rvq7$``7j2oz5t=%`dcOh%;UQ^<2zWroHY z^gj$bH|X)4w9X zcG(v1@4e29@@m)oqAY9lEVa;Jl6Prv4jE2AcRm)*i!@>*m?985X{8eF+PQ-#3Rpr#Vk*Z#Pop@m_2|J1xz!d$I{M0;Aa^ zS1j#ty`m;%4uCYgEx2+n)EZgB7K*!_o~VQuXsk04GglB^D-jnKR4AlOUixk}q9{{V z-8d6uaH4Q*u@-JKAsm9DgrPG?D{7TqJk1`XBoTMr-1==_1Tc?0HeRsB#0T1v%dHwH z$rPI=QTkcsGcXW|CrbJKy?RE|;vZ(J%i0ctH--Q!CiYI z;h4sa)ZY%_$L-7c@~m3OCIn_`BHQmGwNUKWQS#CG+ntVAa556!q=;4@o zi&VI>=P0iN%>9EAR5;COx}p4hW|^lA+ebKcWTxz zv`DZplMepsNsgWrn$sIG>g(!oc7C~1eeCy`VVMy_ou`!{9Rd4%^{%%20tKm&H}Dbe z7w-ta`PO8GCC$r;S5=X;!WF?b2XPDG6oFD|C8Jl1N4gY7HK;A97)!fnT{~~1?lQ?5 zE=tdm%-{WosO+Ip@~h6egebGX3Ay~^l$KNho3XT+-l*k2M7%IG?_n$3@3Y7{mib2J znp&Y^B{9kV2g{rm%go6q2@H*kZR~-u!e6V04FR+%-ITpY)AWZURvvL%Yh$YSn93#hIBvmDthVWB~*kaUn7j(qL>q< z*#GHtU;gTy<#b*b;hf{@#FgsQJ0mAu5Od7;#6gIK{cJn|uZvls+Tb9}ic z9v7PQ415*3_KfeE>_2#iH{Mp+jN^$Asrzmv@429^BXH#A8f7K`;pb=)>z_it?Hu1j zCK)!$3TN_VAW|6K`A))&L6jgVL+3-W+qa*=3H0JM*Z^&X8f>gkH2hNJv^x*4gDXM9 zF85T;yg*>i$W0p53QmvdOmM_M`Ut&dakci04}2l*8f|^Ovp@Tb!$fx&ji7!yz+PZ{ zI@H38s;pDQH`fFx?}}|sd6jH!X*D7|YS@VV-8cEeUHdM6TBiNcsxu*?1pMxOQ>*H% zPTH0rOw1fDnXTU~I3M(8dIOfIGR`KcQ`IxC1r5YL9=6rPeqL|83z3WmtSyMWlDq+e z`(?*0lEld$uI@3U=zMx-cmX->-g=qmmNf2$ipiG{)7R?O!GYOyM{Slsowrpqsq=P zR9eZLOY^L*h-$y211c#JM9oy&56!9bEE2)XAfHN0bqD1`X3hZk{D3d7=-lmRkbT#s zQ~AkAOSr_6fT9vTndXP6;_u2)D^*(DWgH7hBkAiqKf^;RiQ{UVUGFRPG6EwR3GPHl z-|DJ@|HWcQ!D<}Dv%{bW_aq)S*otk1Qw84t1+P_}z47Q^xb z5nJ|9IU_+rmzNomJh1cdq9B~HwMI|8d_$^5@tv{m!7d$q&$A0Uy}=CsSKB?g+2M~D zdvLR+4V2DIN~mkXHJ=5EDe7_)s?675{+EBVE<`S3K9Gv=XcniX`R{t#{?afJz4IS{ zzxCGi)pKR2Ous|*jn8(}v*bEa$r)Qu$!-QDsn^uoaq@~hZe+QbJ3Af&EPg2oEaI;E zfA#!6r~#dcZt>2*Jn#x_QRtZ=;9(=r)%Vi%EbU%9ldAeL=#n?Y2x8Y{5}I~WLy=cL z2wnyfGN30EQc_`BPMc|mVTB+s6rL)Qoi|fG2|Bjcnbn!Bhi0f*Ba%^4vo(E@hp%+5YuIaW2uPGxT{yxU>B6x}&?fTO0Vyd@kF!JQ!TzFzhB# zLH~AaL))Dh99j^~KPB_nn6d?Q6|))HL|4D4V3O~(5k0K6UpO#uReXJGI$zNZ!t$S+ zJPNK9LLR(xcu{DNXJ|qJX!9vKJM6F!|Igu?-ahdAHvYgcHH--Vm2RCvr2z8K|2j>S zPOjRB;4wuxc#qS*@F?zcxTJ$&$B?$aguQJ-I};WLu!KSD4C2&0lB-0qZ1B-xS*IRX z3IL1|vrcNFW}r_hF3_6=zDHo0c|u+24)tWvKaSV}no5bQQquKvnCD}oP{l;Ygsc;2 zVpw!M_2CBG?;kGZjIX@BaO_GnSv%-;^##Hd%TCUpBLZJeySLZDQM1>$5S=99C=us+ zt%eGsOx6nYb0%HQ3DYXs9iHP!(9hgbVXH6ZBUgx+qC}VI?SDlHq`He+Dt}zex30Gf z)3WCz9urd`$4-Aj5EZaYpU5yX^|=1_HtyP5cs;M@3m~lCp?Z=26911zs;(|Yh9|po z;2gci$rR4;ytN)UGD-T9j&zK~I36Y8*9~eVe~w~uAdKmj0V6L38~LFYUB*@TPG1tfGhSa$cY zg6D9#c5A!0>}Xkk<;HQ){k&qU$G385=ywf%{%c9<g!m_~Kqx+8xI7cU{xOWd#1@M!}*OSSBS)xmAnmu;x)W#nj6RD5U zXl@F#jI_t8h9*6?M=h8P=MMLhvQOPN25AQN=D|s5R6q!G`R`~+iV0G=p3ts0ZgtjF z-(S-#nYDJ#4?;vP=_P2Y4G!r~4Rn1wc@6CS=sDK<2{8)f}aU0z^7<`D#?FGa!)=T=lYN)nr= zHD4j4F7#vZ_zN4tU`=!&W24H{v&8ZF4MZcON#zZ>`K`=lNt5To%KtV2M(uDrQsm9- zeAeotzLNeDcC{!K0mog=9=)D)=4gv#=d8SCYcBVXgV2cCIYGGaa9zi_x>(!ru2l-Z zj7%l{XIn*2R~eIhPc!E(14jJJy!#DEhdXUmGU5hh9VIy6En;+jS?SMyt0i;0x`r0J z+;W7Tp|W}_0uQMzL8em(ea@@4xIMsC11*)+j-Fe{7Nntqn~tBk1QhKY?%N z#DRR8Mz34%9)=7%2P;_whukgHOk3rA zQBh}3vg_fZ)g|+gF2th@0X!;?%pWQ!J-)A|Z?aK>wrCVb$X)yq-Llj*e6KP%NLDeMd-q2P^v@4 zY?COd6jc$uWfExt89Or@pSf$#um3TGKTjV|Uhc$}1=v?dbR&zc)q?`8^DG^SFD`CU zFwQoHxH8FI2?DEgIIZkLto7c@*Hccvy;+o-4d`-)AY|QCOv6gO{i{ZOfIqP;pN(Ss zy*edjQZn?n8v4k5Oo3GjR@BEbP)Pgy0}vGgxH&0*X}bWFiu z^V^8LO*og3AAHjL@IhVyiY?Pv*nrzyZDn`LtVX!lY8=ZTu2OT)t(+;RC_RruTR!H` zyl^o~D$oyeJZs>YPiGe`e=ihGT+QBOmBJy16ouy~ILW}GzkfSvxP3jjX7kU*m@e-=Y923S|9>xe@+~);1 z2s%J3HmfjO8hb|*!Ous#IXYO5(x0Wwr4q-6laWK*dKk0nG>M&T?>TGEyCAHawzl*X z-2rb0OH`d!OnF=2E49$;#pcBm;jVs3Mj1}4`eL2r zBg{yWqCu4mRfP$h^KSQyaLx{B=uNe_0nG>vfTL7Yk6HndvN9_m zt&P|+mmr;>a+x2acENN~l9U9sC{5gQHaP*i9OB8IYmGwQ;ljpwosc(4b;DB83lTNS ze%WVDoUWQluk&fo$B`VztBsMY?5a5JlrqNgq?NgVagxQNm zk5Xd|eOx?JHh*>l4{5!D+0su-U_9?iMty6o_dluY_}oXrN&VLkNi_V)Hr0dafYtbo zoj}+GyeIHs#8Zp$M+LoQ+gC#Ksa=G@UeT)&UKu7OMofC{486w{d1oq3lvu1Vi6Tnj zBSUrfW@}vd-a+VzeIlI-J{&K$?aJY9H=4_5Hgot{pM)#;&|B@ zzPY{e=$p+*>Y|*Z$~8gdXm^VE^C760%z7*7(?V@^(Ys#!0vq;FpTbR*s{t$v@mwawhj^hEhOWotz~0kcSoR{`l1UgLvyxBHmI;s? z!en_o$cEYK5Q}OLI@;AKDYp5Q7>93R3OI_e<}!DDvl&=sSX=T>b#~rE9on_LuqJ!M zI-T`yQ>VXZkgwzVFSz67Z}=LoZ`SNBg~ohRgSINCEnYX8E1ExJRcv@ z<=`l*KIZF;foXCQ91qluC2sb;wX$B z0?biDA*}XeRI#*5O)L|QGW(TiCO3mDSwy6l6&T0RIQfHg;}@NL_SN;#+yB)OxyHTb zpwO?U^E&V9?2>3r0M1KO>C0G&psxC8*LEb~5t*+pvIAFi7pYt zEhAo-v(T4GN#GoXYC8`!fyT_7!Z-*DD5=J>VxC{>7Q%Q4Rb5)zCoiu=NtiZw-Q45V z{S}UwAF`)_*Z=?k literal 0 HcmV?d00001 diff --git a/server/.env.example b/server/.env.example index e06c0f7e..606dd898 100644 --- a/server/.env.example +++ b/server/.env.example @@ -31,6 +31,12 @@ PINECONE_INDEX= # Enable all below if you are using vector database: LanceDB. # VECTOR_DB="lancedb" +# Enable all below if you are using vector database: Weaviate. +# VECTOR_DB="weaviate" +# WEAVIATE_ENDPOINT="http://localhost:8080" +# WEAVIATE_API_KEY= + + # CLOUD DEPLOYMENT VARIRABLES ONLY # AUTH_TOKEN="hunter2" # This is the password to your application if remote hosting. # STORAGE_DIR= # absolute filesystem path with no trailing slash diff --git a/server/endpoints/system.js b/server/endpoints/system.js index 98354f4a..01a367c7 100644 --- a/server/endpoints/system.js +++ b/server/endpoints/system.js @@ -60,6 +60,12 @@ function systemEndpoints(app) { ChromaEndpoint: process.env.CHROMA_ENDPOINT, } : {}), + ...(vectorDB === "weaviate" + ? { + WeaviateEndpoint: process.env.WEAVIATE_ENDPOINT, + WeaviateApiKey: process.env.WEAVIATE_API_KEY, + } + : {}), LLMProvider: llmProvider, ...(llmProvider === "openai" ? { diff --git a/server/package.json b/server/package.json index 3d8ec2b8..e35eb69d 100644 --- a/server/package.json +++ b/server/package.json @@ -26,6 +26,7 @@ "dotenv": "^16.0.3", "express": "^4.18.2", "extract-zip": "^2.0.1", + "graphql": "^16.7.1", "jsonwebtoken": "^8.5.1", "langchain": "^0.0.90", "moment": "^2.29.4", @@ -38,7 +39,8 @@ "sqlite3": "^5.1.6", "uuid": "^9.0.0", "uuid-apikey": "^1.5.3", - "vectordb": "0.1.12" + "vectordb": "0.1.12", + "weaviate-ts-client": "^1.4.0" }, "devDependencies": { "nodemon": "^2.0.22", diff --git a/server/utils/helpers/camelcase.js b/server/utils/helpers/camelcase.js new file mode 100644 index 00000000..4a8e1b28 --- /dev/null +++ b/server/utils/helpers/camelcase.js @@ -0,0 +1,143 @@ +const UPPERCASE = /[\p{Lu}]/u; +const LOWERCASE = /[\p{Ll}]/u; +const LEADING_CAPITAL = /^[\p{Lu}](?![\p{Lu}])/gu; +const IDENTIFIER = /([\p{Alpha}\p{N}_]|$)/u; +const SEPARATORS = /[_.\- ]+/; + +const LEADING_SEPARATORS = new RegExp("^" + SEPARATORS.source); +const SEPARATORS_AND_IDENTIFIER = new RegExp( + SEPARATORS.source + IDENTIFIER.source, + "gu" +); +const NUMBERS_AND_IDENTIFIER = new RegExp("\\d+" + IDENTIFIER.source, "gu"); + +const preserveCamelCase = ( + string, + toLowerCase, + toUpperCase, + preserveConsecutiveUppercase +) => { + let isLastCharLower = false; + let isLastCharUpper = false; + let isLastLastCharUpper = false; + let isLastLastCharPreserved = false; + + for (let index = 0; index < string.length; index++) { + const character = string[index]; + isLastLastCharPreserved = index > 2 ? string[index - 3] === "-" : true; + + if (isLastCharLower && UPPERCASE.test(character)) { + string = string.slice(0, index) + "-" + string.slice(index); + isLastCharLower = false; + isLastLastCharUpper = isLastCharUpper; + isLastCharUpper = true; + index++; + } else if ( + isLastCharUpper && + isLastLastCharUpper && + LOWERCASE.test(character) && + (!isLastLastCharPreserved || preserveConsecutiveUppercase) + ) { + string = string.slice(0, index - 1) + "-" + string.slice(index - 1); + isLastLastCharUpper = isLastCharUpper; + isLastCharUpper = false; + isLastCharLower = true; + } else { + isLastCharLower = + toLowerCase(character) === character && + toUpperCase(character) !== character; + isLastLastCharUpper = isLastCharUpper; + isLastCharUpper = + toUpperCase(character) === character && + toLowerCase(character) !== character; + } + } + + return string; +}; + +const preserveConsecutiveUppercase = (input, toLowerCase) => { + LEADING_CAPITAL.lastIndex = 0; + + return input.replace(LEADING_CAPITAL, (m1) => toLowerCase(m1)); +}; + +const postProcess = (input, toUpperCase) => { + SEPARATORS_AND_IDENTIFIER.lastIndex = 0; + NUMBERS_AND_IDENTIFIER.lastIndex = 0; + + return input + .replace(SEPARATORS_AND_IDENTIFIER, (_, identifier) => + toUpperCase(identifier) + ) + .replace(NUMBERS_AND_IDENTIFIER, (m) => toUpperCase(m)); +}; + +function camelCase(input, options) { + if (!(typeof input === "string" || Array.isArray(input))) { + throw new TypeError("Expected the input to be `string | string[]`"); + } + + options = { + pascalCase: true, + preserveConsecutiveUppercase: false, + ...options, + }; + + if (Array.isArray(input)) { + input = input + .map((x) => x.trim()) + .filter((x) => x.length) + .join("-"); + } else { + input = input.trim(); + } + + if (input.length === 0) { + return ""; + } + + const toLowerCase = + options.locale === false + ? (string) => string.toLowerCase() + : (string) => string.toLocaleLowerCase(options.locale); + + const toUpperCase = + options.locale === false + ? (string) => string.toUpperCase() + : (string) => string.toLocaleUpperCase(options.locale); + + if (input.length === 1) { + if (SEPARATORS.test(input)) { + return ""; + } + + return options.pascalCase ? toUpperCase(input) : toLowerCase(input); + } + + const hasUpperCase = input !== toLowerCase(input); + + if (hasUpperCase) { + input = preserveCamelCase( + input, + toLowerCase, + toUpperCase, + options.preserveConsecutiveUppercase + ); + } + + input = input.replace(LEADING_SEPARATORS, ""); + input = options.preserveConsecutiveUppercase + ? preserveConsecutiveUppercase(input, toLowerCase) + : toLowerCase(input); + + if (options.pascalCase) { + input = toUpperCase(input.charAt(0)) + input.slice(1); + } + + return postProcess(input, toUpperCase); +} + +module.exports = { + camelCase, +}; diff --git a/server/utils/helpers/index.js b/server/utils/helpers/index.js index 5be56507..b7fb5ae0 100644 --- a/server/utils/helpers/index.js +++ b/server/utils/helpers/index.js @@ -10,6 +10,9 @@ function getVectorDbClass() { case "lancedb": const { LanceDb } = require("../vectorDbProviders/lance"); return LanceDb; + case "weaviate": + const { Weaviate } = require("../vectorDbProviders/weaviate"); + return Weaviate; default: throw new Error("ENV: No VECTOR_DB value found in environment!"); } diff --git a/server/utils/helpers/updateENV.js b/server/utils/helpers/updateENV.js index 64c91988..9f00ec42 100644 --- a/server/utils/helpers/updateENV.js +++ b/server/utils/helpers/updateENV.js @@ -39,6 +39,15 @@ const KEY_MAPPING = { envKey: "CHROMA_ENDPOINT", checks: [isValidURL, validChromaURL], }, + WeaviateEndpoint: { + envKey: "WEAVIATE_ENDPOINT", + checks: [isValidURL], + }, + WeaviateApiKey: { + envKey: "WEAVIATE_API_KEY", + checks: [], + }, + PineConeEnvironment: { envKey: "PINECONE_ENVIRONMENT", checks: [], @@ -103,7 +112,7 @@ function validOpenAIModel(input = "") { } function supportedVectorDB(input = "") { - const supported = ["chroma", "pinecone", "lancedb"]; + const supported = ["chroma", "pinecone", "lancedb", "weaviate"]; return supported.includes(input) ? null : `Invalid VectorDB type. Must be one of ${supported.join(", ")}.`; diff --git a/server/utils/vectorDbProviders/weaviate/WEAVIATE_SETUP.md b/server/utils/vectorDbProviders/weaviate/WEAVIATE_SETUP.md new file mode 100644 index 00000000..fc0acaec --- /dev/null +++ b/server/utils/vectorDbProviders/weaviate/WEAVIATE_SETUP.md @@ -0,0 +1,17 @@ +# How to setup a local (or cloud) Weaviate Vector Database + +[Get a Weaviate Cloud instance](https://weaviate.io/developers/weaviate/quickstart#create-an-instance). +[Set up Weaviate locally on Docker](https://weaviate.io/developers/weaviate/installation/docker-compose). + +Fill out the variables in the "Vector Database" tab of settings. Select Weaviate as your provider and fill out the appropriate fields +with the information from either of the above steps. + +### How to get started _Development mode only_ + +After setting up either the Weaviate cloud or local dockerized instance you just need to set these variable in `.env.development` or defined them at runtime via the UI. + +``` +VECTOR_DB="weaviate" +WEAVIATE_ENDPOINT='http://localhost:8080' +WEAVIATE_API_KEY= # Optional +``` diff --git a/server/utils/vectorDbProviders/weaviate/index.js b/server/utils/vectorDbProviders/weaviate/index.js new file mode 100644 index 00000000..884c08e0 --- /dev/null +++ b/server/utils/vectorDbProviders/weaviate/index.js @@ -0,0 +1,503 @@ +const { default: weaviate } = require("weaviate-ts-client"); +const { RecursiveCharacterTextSplitter } = require("langchain/text_splitter"); +const { storeVectorResult, cachedVectorInformation } = require("../../files"); +const { v4: uuidv4 } = require("uuid"); +const { toChunks, getLLMProvider } = require("../../helpers"); +const { chatPrompt } = require("../../chats"); +const { camelCase } = require("../../helpers/camelcase"); + +const Weaviate = { + name: "Weaviate", + connect: async function () { + if (process.env.VECTOR_DB !== "weaviate") + throw new Error("Weaviate::Invalid ENV settings"); + + const weaviateUrl = new URL(process.env.WEAVIATE_ENDPOINT); + const options = { + scheme: weaviateUrl.protocol?.replace(":", "") || "http", + host: weaviateUrl?.host, + ...(process.env?.WEAVIATE_API_KEY?.length > 0 + ? { apiKey: new weaviate.ApiKey(process.env?.WEAVIATE_API_KEY) } + : {}), + }; + const client = weaviate.client(options); + const isAlive = await await client.misc.liveChecker().do(); + if (!isAlive) + throw new Error( + "Weaviate::Invalid Alive signal received - is the service online?" + ); + return { client }; + }, + heartbeat: async function () { + await this.connect(); + return { heartbeat: Number(new Date()) }; + }, + totalIndicies: async function () { + const { client } = await this.connect(); + const collectionNames = await this.allNamespaces(client); + var totalVectors = 0; + for (const name of collectionNames) { + totalVectors += await this.namespaceCountWithClient(client, name); + } + return totalVectors; + }, + namespaceCountWithClient: async function (client, namespace) { + try { + const response = await client.graphql + .aggregate() + .withClassName(camelCase(namespace)) + .withFields("meta { count }") + .do(); + return ( + response?.data?.Aggregate?.[camelCase(namespace)]?.[0]?.meta?.count || 0 + ); + } catch (e) { + console.error(`Weaviate:namespaceCountWithClient`, e.message); + return 0; + } + }, + namespaceCount: async function (namespace = null) { + try { + const { client } = await this.connect(); + const response = await client.graphql + .aggregate() + .withClassName(camelCase(namespace)) + .withFields("meta { count }") + .do(); + + return ( + response?.data?.Aggregate?.[camelCase(namespace)]?.[0]?.meta?.count || 0 + ); + } catch (e) { + console.error(`Weaviate:namespaceCountWithClient`, e.message); + return 0; + } + }, + similarityResponse: async function (client, namespace, queryVector) { + const result = { + contextTexts: [], + sourceDocuments: [], + }; + + const weaviateClass = await this.namespace(client, namespace); + const fields = weaviateClass.properties.map((prop) => prop.name).join(" "); + const queryResponse = await client.graphql + .get() + .withClassName(camelCase(namespace)) + .withFields(`${fields} _additional { id }`) + .withNearVector({ vector: queryVector }) + .withLimit(4) + .do(); + + const responses = queryResponse?.data?.Get?.[camelCase(namespace)]; + responses.forEach((response) => { + // In Weaviate we have to pluck id from _additional and spread it into the rest + // of the properties. + const { + _additional: { id }, + ...rest + } = response; + result.contextTexts.push(rest.text); + result.sourceDocuments.push({ ...rest, id }); + }); + + return result; + }, + allNamespaces: async function (client) { + try { + const { classes = [] } = await client.schema.getter().do(); + return classes.map((classObj) => classObj.class); + } catch (e) { + console.error("Weaviate::AllNamespace", e); + return []; + } + }, + namespace: async function (client, namespace = null) { + if (!namespace) throw new Error("No namespace value provided."); + if (!(await this.namespaceExists(client, namespace))) return null; + + const weaviateClass = await client.schema + .classGetter() + .withClassName(camelCase(namespace)) + .do(); + + return { + ...weaviateClass, + vectorCount: await this.namespaceCount(namespace), + }; + }, + addVectors: async function (client, vectors = []) { + const response = { success: true, errors: new Set([]) }; + const results = await client.batch + .objectsBatcher() + .withObjects(...vectors) + .do(); + + results.forEach((res) => { + const { status, errors = [] } = res.result; + if (status === "SUCCESS" || errors.length === 0) return; + response.success = false; + response.errors.add(errors.error?.[0]?.message || null); + }); + + response.errors = [...response.errors]; + return response; + }, + hasNamespace: async function (namespace = null) { + if (!namespace) return false; + const { client } = await this.connect(); + const weaviateClasses = await this.allNamespaces(client); + return weaviateClasses.includes(camelCase(namespace)); + }, + namespaceExists: async function (client, namespace = null) { + if (!namespace) throw new Error("No namespace value provided."); + const weaviateClasses = await this.allNamespaces(client); + return weaviateClasses.includes(camelCase(namespace)); + }, + deleteVectorsInNamespace: async function (client, namespace = null) { + await client.schema.classDeleter().withClassName(camelCase(namespace)).do(); + return true; + }, + addDocumentToNamespace: async function ( + namespace, + documentData = {}, + fullFilePath = null + ) { + const { DocumentVectors } = require("../../../models/vectors"); + try { + const { + pageContent, + docId, + id: _id, // Weaviate will abort if `id` is present in properties + ...metadata + } = documentData; + if (!pageContent || pageContent.length == 0) return false; + + console.log("Adding new vectorized document into namespace", namespace); + const cacheResult = await cachedVectorInformation(fullFilePath); + if (cacheResult.exists) { + const { client } = await this.connect(); + const weaviateClassExits = await this.hasNamespace(namespace); + if (!weaviateClassExits) { + await client.schema + .classCreator() + .withClass({ + class: camelCase(namespace), + description: `Class created by AnythingLLM named ${camelCase( + namespace + )}`, + vectorizer: "none", + }) + .do(); + } + + const { chunks } = cacheResult; + const documentVectors = []; + const vectors = []; + + for (const chunk of chunks) { + // Before sending to Weaviate and saving the records to our db + // we need to assign the id of each chunk that is stored in the cached file. + chunk.forEach((chunk) => { + const id = uuidv4(); + const flattenedMetadata = this.flattenObjectForWeaviate( + chunk.properties + ); + documentVectors.push({ docId, vectorId: id }); + const vectorRecord = { + id, + class: camelCase(namespace), + vector: chunk.vector || chunk.values || [], + properties: { ...flattenedMetadata }, + }; + vectors.push(vectorRecord); + }); + + const { success: additionResult, errors = [] } = + await this.addVectors(client, vectors); + if (!additionResult) { + console.error("Weaviate::addVectors failed to insert", errors); + throw new Error("Error embedding into Weaviate"); + } + } + + await DocumentVectors.bulkInsert(documentVectors); + return true; + } + + // If we are here then we are going to embed and store a novel document. + // We have to do this manually as opposed to using LangChains `Chroma.fromDocuments` + // because we then cannot atomically control our namespace to granularly find/remove documents + // from vectordb. + const textSplitter = new RecursiveCharacterTextSplitter({ + chunkSize: 1000, + chunkOverlap: 20, + }); + const textChunks = await textSplitter.splitText(pageContent); + + console.log("Chunks created from document:", textChunks.length); + const LLMConnector = getLLMProvider(); + const documentVectors = []; + const vectors = []; + const vectorValues = await LLMConnector.embedChunks(textChunks); + const submission = { + ids: [], + vectors: [], + properties: [], + }; + + if (!!vectorValues && vectorValues.length > 0) { + for (const [i, vector] of vectorValues.entries()) { + const flattenedMetadata = this.flattenObjectForWeaviate(metadata); + const vectorRecord = { + class: camelCase(namespace), + id: uuidv4(), + vector: vector, + // [DO NOT REMOVE] + // LangChain will be unable to find your text if you embed manually and dont include the `text` key. + // https://github.com/hwchase17/langchainjs/blob/5485c4af50c063e257ad54f4393fa79e0aff6462/langchain/src/vectorstores/weaviate.ts#L133 + properties: { ...flattenedMetadata, text: textChunks[i] }, + }; + + submission.ids.push(vectorRecord.id); + submission.vectors.push(vectorRecord.values); + submission.properties.push(metadata); + + vectors.push(vectorRecord); + documentVectors.push({ docId, vectorId: vectorRecord.id }); + } + } else { + console.error( + "Could not use OpenAI to embed document chunks! This document will not be recorded." + ); + } + + const { client } = await this.connect(); + const weaviateClassExits = await this.hasNamespace(namespace); + if (!weaviateClassExits) { + await client.schema + .classCreator() + .withClass({ + class: camelCase(namespace), + description: `Class created by AnythingLLM named ${camelCase( + namespace + )}`, + vectorizer: "none", + }) + .do(); + } + + if (vectors.length > 0) { + const chunks = []; + for (const chunk of toChunks(vectors, 500)) chunks.push(chunk); + + console.log("Inserting vectorized chunks into Weaviate collection."); + const { success: additionResult, errors = [] } = await this.addVectors( + client, + vectors + ); + if (!additionResult) { + console.error("Weaviate::addVectors failed to insert", errors); + throw new Error("Error embedding into Weaviate"); + } + await storeVectorResult(chunks, fullFilePath); + } + + await DocumentVectors.bulkInsert(documentVectors); + return true; + } catch (e) { + console.error(e); + console.error("addDocumentToNamespace", e.message); + return false; + } + }, + deleteDocumentFromNamespace: async function (namespace, docId) { + const { DocumentVectors } = require("../../../models/vectors"); + const { client } = await this.connect(); + if (!(await this.namespaceExists(client, namespace))) return; + + const knownDocuments = await DocumentVectors.where(`docId = '${docId}'`); + if (knownDocuments.length === 0) return; + + for (const doc of knownDocuments) { + await client.data + .deleter() + .withClassName(camelCase(namespace)) + .withId(doc.vectorId) + .do(); + } + + const indexes = knownDocuments.map((doc) => doc.id); + await DocumentVectors.deleteIds(indexes); + return true; + }, + query: async function (reqBody = {}) { + const { namespace = null, input, workspace = {} } = reqBody; + if (!namespace || !input) throw new Error("Invalid request body"); + + const { client } = await this.connect(); + if (!(await this.namespaceExists(client, namespace))) { + return { + response: null, + sources: [], + message: "Invalid query - no documents found for workspace!", + }; + } + + const LLMConnector = getLLMProvider(); + const queryVector = await LLMConnector.embedTextInput(input); + const { contextTexts, sourceDocuments } = await this.similarityResponse( + client, + namespace, + queryVector + ); + + const prompt = { + role: "system", + content: `${chatPrompt(workspace)} + Context: + ${contextTexts + .map((text, i) => { + return `[CONTEXT ${i}]:\n${text}\n[END CONTEXT ${i}]\n\n`; + }) + .join("")}`, + }; + const memory = [prompt, { role: "user", content: input }]; + const responseText = await LLMConnector.getChatCompletion(memory, { + temperature: workspace?.openAiTemp ?? 0.7, + }); + + return { + response: responseText, + sources: this.curateSources(sourceDocuments), + message: false, + }; + }, + // This implementation of chat uses the chat history and modifies the system prompt at execution + // this is improved over the regular langchain implementation so that chats do not directly modify embeddings + // because then multi-user support will have all conversations mutating the base vector collection to which then + // the only solution is replicating entire vector databases per user - which will very quickly consume space on VectorDbs + chat: async function (reqBody = {}) { + const { + namespace = null, + input, + workspace = {}, + chatHistory = [], + } = reqBody; + if (!namespace || !input) throw new Error("Invalid request body"); + + const { client } = await this.connect(); + if (!(await this.namespaceExists(client, namespace))) { + return { + response: null, + sources: [], + message: "Invalid query - no documents found for workspace!", + }; + } + + const LLMConnector = getLLMProvider(); + const queryVector = await LLMConnector.embedTextInput(input); + const { contextTexts, sourceDocuments } = await this.similarityResponse( + client, + namespace, + queryVector + ); + const prompt = { + role: "system", + content: `${chatPrompt(workspace)} + Context: + ${contextTexts + .map((text, i) => { + return `[CONTEXT ${i}]:\n${text}\n[END CONTEXT ${i}]\n\n`; + }) + .join("")}`, + }; + const memory = [prompt, ...chatHistory, { role: "user", content: input }]; + const responseText = await LLMConnector.getChatCompletion(memory, { + temperature: workspace?.openAiTemp ?? 0.7, + }); + + return { + response: responseText, + sources: this.curateSources(sourceDocuments), + message: false, + }; + }, + "namespace-stats": async function (reqBody = {}) { + const { namespace = null } = reqBody; + if (!namespace) throw new Error("namespace required"); + const { client } = await this.connect(); + const stats = await this.namespace(client, namespace); + return stats + ? stats + : { message: "No stats were able to be fetched from DB for namespace" }; + }, + "delete-namespace": async function (reqBody = {}) { + const { namespace = null } = reqBody; + const { client } = await this.connect(); + const details = await this.namespace(client, namespace); + await this.deleteVectorsInNamespace(client, namespace); + return { + message: `Namespace ${camelCase(namespace)} was deleted along with ${details?.vectorCount + } vectors.`, + }; + }, + reset: async function () { + const { client } = await this.connect(); + const weaviateClasses = await this.allNamespaces(client); + for (const weaviateClass of weaviateClasses) { + await client.schema.classDeleter().withClassName(weaviateClass).do(); + } + return { reset: true }; + }, + curateSources: function (sources = []) { + const documents = []; + for (const source of sources) { + if (Object.keys(source).length > 0) { + documents.push(source); + } + } + + return documents; + }, + flattenObjectForWeaviate: function (obj = {}) { + // Note this function is not generic, it is designed specifically for Weaviate + // https://weaviate.io/developers/weaviate/config-refs/datatypes#introduction + // Credit to LangchainJS + // https://github.com/hwchase17/langchainjs/blob/5485c4af50c063e257ad54f4393fa79e0aff6462/langchain/src/vectorstores/weaviate.ts#L11C1-L50C3 + const flattenedObject = {}; + + for (const key in obj) { + if (!Object.hasOwn(obj, key)) { + continue; + } + const value = obj[key]; + if (typeof obj[key] === "object" && !Array.isArray(value)) { + const recursiveResult = this.flattenObjectForWeaviate(value); + + for (const deepKey in recursiveResult) { + if (Object.hasOwn(obj, key)) { + flattenedObject[`${key}_${deepKey}`] = recursiveResult[deepKey]; + } + } + } else if (Array.isArray(value)) { + if ( + value.length > 0 && + typeof value[0] !== "object" && + // eslint-disable-next-line @typescript-eslint/no-explicit-any + value.every((el) => typeof el === typeof value[0]) + ) { + // Weaviate only supports arrays of primitive types, + // where all elements are of the same type + flattenedObject[key] = value; + } + } else { + flattenedObject[key] = value; + } + } + + return flattenedObject; + }, +}; + +module.exports.Weaviate = Weaviate; diff --git a/server/yarn.lock b/server/yarn.lock index cd1514e7..3b1caaa8 100644 --- a/server/yarn.lock +++ b/server/yarn.lock @@ -130,6 +130,11 @@ dependencies: googleapis-common "^6.0.3" +"@graphql-typed-document-node/core@^3.1.1": + version "3.2.0" + resolved "https://registry.yarnpkg.com/@graphql-typed-document-node/core/-/core-3.2.0.tgz#5f3d96ec6b2354ad6d8a28bf216a1d97b5426861" + integrity sha512-mB9oAsNCm9aM3/SOv4YtBMqZbYj10R7dkq8byBqxGY/ncFwhf2oQzMV+LCRlWoDSEBJ3COiR1yeDvMtsoOsuFQ== + "@mapbox/node-pre-gyp@^1.0.0", "@mapbox/node-pre-gyp@^1.0.10": version "1.0.11" resolved "https://registry.yarnpkg.com/@mapbox/node-pre-gyp/-/node-pre-gyp-1.0.11.tgz#417db42b7f5323d79e93b34a6d7a2a12c0df43fa" @@ -916,6 +921,11 @@ extend@^3.0.2: resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.2.tgz#f8b1136b4071fbd8eb140aff858b1019ec2915fa" integrity sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g== +extract-files@^9.0.0: + version "9.0.0" + resolved "https://registry.yarnpkg.com/extract-files/-/extract-files-9.0.0.tgz#8a7744f2437f81f5ed3250ed9f1550de902fe54a" + integrity sha512-CvdFfHkC95B4bBBk36hcEmvdR2awOdhhVUYH6S/zrVj3477zven/fJMYg7121h4T1xHZC+tetUpubpAhxwI7hQ== + extract-zip@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/extract-zip/-/extract-zip-2.0.1.tgz#663dca56fe46df890d5f131ef4a06d22bb8ba13a" @@ -981,6 +991,15 @@ follow-redirects@^1.14.8: resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.2.tgz#b460864144ba63f2681096f274c4e57026da2c13" integrity sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA== +form-data@^3.0.0: + version "3.0.1" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-3.0.1.tgz#ebd53791b78356a99af9a300d4282c4d5eb9755f" + integrity sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg== + dependencies: + asynckit "^0.4.0" + combined-stream "^1.0.8" + mime-types "^2.1.12" + form-data@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452" @@ -1149,6 +1168,21 @@ graceful-fs@^4.2.0, graceful-fs@^4.2.6: resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.11.tgz#4183e4e8bf08bb6e05bbb2f7d2e0c8f712ca40e3" integrity sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ== +graphql-request@^5.1.0: + version "5.2.0" + resolved "https://registry.yarnpkg.com/graphql-request/-/graphql-request-5.2.0.tgz#a05fb54a517d91bb2d7aefa17ade4523dc5ebdca" + integrity sha512-pLhKIvnMyBERL0dtFI3medKqWOz/RhHdcgbZ+hMMIb32mEPa5MJSzS4AuXxfI4sRAu6JVVk5tvXuGfCWl9JYWQ== + dependencies: + "@graphql-typed-document-node/core" "^3.1.1" + cross-fetch "^3.1.5" + extract-files "^9.0.0" + form-data "^3.0.0" + +graphql@^16.7.1: + version "16.7.1" + resolved "https://registry.yarnpkg.com/graphql/-/graphql-16.7.1.tgz#11475b74a7bff2aefd4691df52a0eca0abd9b642" + integrity sha512-DRYR9tf+UGU0KOsMcKAlXeFfX89UiiIZ0dRU3mR0yJfu6OjZqUcp68NnFLnqQU5RexygFoDy1EW+ccOYcPfmHg== + gtoken@^6.1.0: version "6.1.2" resolved "https://registry.yarnpkg.com/gtoken/-/gtoken-6.1.2.tgz#aeb7bdb019ff4c3ba3ac100bbe7b6e74dce0e8bc" @@ -2507,6 +2541,15 @@ vectordb@0.1.12: "@apache-arrow/ts" "^12.0.0" apache-arrow "^12.0.0" +weaviate-ts-client@^1.4.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/weaviate-ts-client/-/weaviate-ts-client-1.4.0.tgz#e1adb670f2c1930a82601efb915b0131f6988b7e" + integrity sha512-G2V/IWMHXDjoJeATUYKkZXzAs7iRj4GE8B3AX59XDqMRW12X7VUkRgo4xWcHH1bjpLIHUYTzD5qZXcB8P9Hdmw== + dependencies: + graphql-request "^5.1.0" + isomorphic-fetch "^3.0.0" + uuid "^9.0.0" + webidl-conversions@^3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871"