From e0a0a8976db2bb4cbf060f54ab2f2598292c2ace Mon Sep 17 00:00:00 2001 From: Timothy Carambat Date: Wed, 27 Dec 2023 17:21:47 -0800 Subject: [PATCH] Add Ollama as LLM provider option (#494) * Add support for Ollama as LLM provider resolves #493 --- .vscode/settings.json | 1 + README.md | 1 + docker/.env.example | 5 + .../LLMSelection/OllamaLLMOptions/index.jsx | 120 ++++++++++ frontend/src/media/llmprovider/ollama.png | Bin 0 -> 23630 bytes .../GeneralSettings/LLMPreference/index.jsx | 14 ++ .../Steps/DataHandling/index.jsx | 8 + .../Steps/LLMSelection/index.jsx | 24 +- server/.env.example | 5 + server/models/systemSettings.js | 14 ++ server/utils/AiProviders/ollama/index.js | 208 ++++++++++++++++++ server/utils/chats/stream.js | 29 +++ server/utils/helpers/customModels.js | 35 ++- server/utils/helpers/index.js | 3 + server/utils/helpers/updateENV.js | 25 +++ 15 files changed, 486 insertions(+), 6 deletions(-) create mode 100644 frontend/src/components/LLMSelection/OllamaLLMOptions/index.jsx create mode 100644 frontend/src/media/llmprovider/ollama.png create mode 100644 server/utils/AiProviders/ollama/index.js diff --git a/.vscode/settings.json b/.vscode/settings.json index dde2d134..459f57fc 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,6 @@ { "cSpell.words": [ + "Ollama", "openai", "Qdrant", "Weaviate" diff --git a/README.md b/README.md index 44e0557f..36127cb3 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,7 @@ Some cool features of AnythingLLM - [Azure OpenAI](https://azure.microsoft.com/en-us/products/ai-services/openai-service) - [Anthropic ClaudeV2](https://www.anthropic.com/) - [Google Gemini Pro](https://ai.google.dev/) +- [Ollama (chat models)](https://ollama.ai/) - [LM Studio (all models)](https://lmstudio.ai) - [LocalAi (all models)](https://localai.io/) diff --git a/docker/.env.example b/docker/.env.example index cc9fa06f..0db90aa2 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -35,6 +35,11 @@ GID='1000' # LOCAL_AI_MODEL_TOKEN_LIMIT=4096 # LOCAL_AI_API_KEY="sk-123abc" +# LLM_PROVIDER='ollama' +# OLLAMA_BASE_PATH='http://host.docker.internal:11434' +# OLLAMA_MODEL_PREF='llama2' +# OLLAMA_MODEL_TOKEN_LIMIT=4096 + ########################################### ######## Embedding API SElECTION ########## ########################################### diff --git a/frontend/src/components/LLMSelection/OllamaLLMOptions/index.jsx b/frontend/src/components/LLMSelection/OllamaLLMOptions/index.jsx new file mode 100644 index 00000000..a2034bf7 --- /dev/null +++ b/frontend/src/components/LLMSelection/OllamaLLMOptions/index.jsx @@ -0,0 +1,120 @@ +import { useEffect, useState } from "react"; +import System from "@/models/system"; + +export default function OllamaLLMOptions({ settings }) { + const [basePathValue, setBasePathValue] = useState( + settings?.OllamaLLMBasePath + ); + const [basePath, setBasePath] = useState(settings?.OllamaLLMBasePath); + + return ( +
+
+
+ + setBasePathValue(e.target.value)} + onBlur={() => setBasePath(basePathValue)} + /> +
+ +
+ + e.target.blur()} + defaultValue={settings?.OllamaLLMTokenLimit} + required={true} + autoComplete="off" + /> +
+
+
+ ); +} + +function OllamaLLMModelSelection({ settings, basePath = null }) { + const [customModels, setCustomModels] = useState([]); + const [loading, setLoading] = useState(true); + + useEffect(() => { + async function findCustomModels() { + if (!basePath) { + setCustomModels([]); + setLoading(false); + return; + } + setLoading(true); + const { models } = await System.customModels("ollama", null, basePath); + setCustomModels(models || []); + setLoading(false); + } + findCustomModels(); + }, [basePath]); + + if (loading || customModels.length == 0) { + return ( +
+ + +
+ ); + } + + return ( +
+ + +
+ ); +} diff --git a/frontend/src/media/llmprovider/ollama.png b/frontend/src/media/llmprovider/ollama.png new file mode 100644 index 0000000000000000000000000000000000000000..2a898a6ebbd85026715453aed13885742cbb7712 GIT binary patch literal 23630 zcmeFZcRbhq|38Xs%E&0$DUy&?Nj8-t$tIiZz4w+R4dY75C?r{_P)21XNs**%Ns?Kz z`Q2Y#*Z1@Lem>uG&h6aJIe(sRt}DIAYdpu}aev(J_s1(v7xnDx!fP`c~LU}w1 z3Hh{xuA#S~=1Ey=Hy2S$8#gOkQGXYAoB*frm&LCxw%(S!{w~h0Ub6m21pj8bwq6eBTzU8Lx;Z;rx>(AI5m)K#@ZS#e=aqjyve^H@`Tu{Wezg;+SCueLu z{^J#l2=RuyrKgvzZh*V(5kVbWFE<}gYuo?64E{Nwil?omx2=u5gt&y{zWoyWBo4@l z{l{DXnn}5T-ca-RK5ip#WoK=D=+J)geFqL4ve_rK|KNdr2gRkW_lfVfJz!&PV$d7**~kr%~RLS%~@W>($yEc z=|5lozq-i(ib4PJ8APN1wuHt070u)u#2dm%Nc3xtt19XG-~5pg;I3=?WTP&BVONdq z_>UDW=3``}0*WL?6sjz`-rnAvTout?odfDxaSEuC1y_x4i@2wtPzWFonROWO*|G8YlR9hup zM(Su9rq{p7sL9DJAFD)PRpe!?R{a0}`oBK}yy@cnI`_uLm)h4M=e6H38KzRyi@VEalo}U(`~F?o%S&>3 zWo5|ssnTf{Req+N+qbV=zaIbbqoEKB?J)snOwXLV*4Ew-k1EqrLy>P&Q|i{%JBe@1 z`YWDhX|UQ)d?PhAwzBe|P>P19`PXf#LSrQK_0NXfvubL%;uv_EW=EReefU6G*hQPx z>XV_i=gP4xUEwRYZe5ag`e0EXyZM-wR;;dYjH064zKH0jVr^}0Li_eL#%~pT+tERG zqs_N-B}b>=a(em&;`2YhwjDcmjCaBIl44nJpTKL`gHE|P&&tAd^YbXGckkXsT)Dz@ zkn_^dl+g936#ye|(JKI_%4P=+Je!WYWky2dM)GMxI4%IvBhtZfs&=Da9dJ zBgUepwEG&LhzNa4OUuV9FTts*G46(@CgusxRF!B?HtX^p_qi{}tJjw&4tjm1r>Cs< zsTDJ4j8{E>;lhhoDSKyM?o6_1OP9W^k=yxjcqyWONYTMzPk+D|*Udv-lu=PpGyZa+ z86%%o=Du7zW!3Y*_Hv6&=R>O(x=gjdI!=hS{ogp019HjiszKE#8yq0053^o?m%zU@`$fItv z`o;wIwckgV$?CTnWvZ|(cB(B}F1dD4&o4{`cU zQP0CM91V?)e0%o9_of+Ey`5(TA-((z)^Xm_w3JD2O6>-C%u(w{AxYfk8ris;$duy#o7wPY6*|CN5V0d0$UY2>?)2BOl zLKpeIPfP^lR`uTVm*dtoFt8q|4f!@b&9FRQSXk&WGpOY4<3q9bDS8K=@}VgAh)CiL zbEBXW-WFBaV{D%yPX z5~q@%-@?VBh?yg*LgyHhnxk<3oPvV*3dQ3B%mTZ2^UKTM@|)tS7ytI-M*}g%Wo2an zE_$9Hz7YTD?>Floh|v{RQ&uJ$9vR_dO9;q_D*w&EBY(1&Uy`2b)%HlsZS-5V48Odx zvpU2=r+_6UCdPlZk$rZ;e1n66f};AwR?Xd8i3xf5`0*R3!j4Jb+XV$Q10f4sO-)Vy z87jN4W#7F^Emph8ANupg0I!%>QiY+fkI&F&x4K5`r|Qs%HmN(&96N>3Ec*59ldIni>RvqYnLvEBvbLVxSRdG^PuXj*%62%`vgvk_gM-7*2Ny5d8eP0} zX}DFaF8aB}plu`W`pekZNF%#>1GeapXB8s{$AyjMk`3iEXRZyLcFymN$8O|Q2<7za z^I?xx&K_{$Kgnfv`VEOlD^}qUmhDi=7S+=%IXO8qZ&etw@7y7Ge8eG~LOb!HsQz}6 z{Ds9uf3tm0PvWkZzV5)-ylE}s#_�H@~}mG{EQ@Sp*3}%VNu^&`1PY#cXZG!Y-@H zPmla(o>MV#bHACW{q^hDAkI@&+`Nmg5(|&iK3yFP4;$a2%y3oSOf>B_#_Xx@WTbu} zh1%&XAzD5@zK=CQ2lPwKHm7KeE(G+s9{BY6v&U3#RhB?ayUcz4y+rJK^G3^TimFII z!Dr%w<-$}SLgUE7E;%{Qr+(AwmX>^C-7mj=6B6FL_g#O#yqLK~IVpd#WO(c7ids{w#Csx_84^P&u2iHFaP8J-tFq)%S%ZZ16Xk3Q4uKwE^?mfpXlxe*YdG zZcM12Gnx-PqM@NtYSVIcNA%?!A!OG3W;GlE0Rh#4T3Z+xMt+TV+N>N=Va%qNkkY)3 zXoyf&{kS5qhTEjVHAcU%rl9fVk>ta^KNK}IXvHRM-Q5}4#msIRe0ED{jHF{1wea`< zI6b*f?exnxZ<K3<_%b9Yo2iyxqoBlwH)k z_VwOA6(JUWK0Z>X?&A4$Nh4j|Xte}3J^_J-Y3WbSGRbLaKR-pj(R*!2B%{qw`=%Dk zI1(wbkzTg9Xa4f?tm4EM)lQzgIPlJ?_=uUU?P#^uK>D8Z+LYU|T2G{j&NB05XvK}T zrm2*4pK{!z!$Og(6Myw8vAvf=xDeIwnddeVZFCbcmfMhbvT|}F4Z7zZK9ss|SVl%0 zW7W}4Lr$iure=wJsa@C=YToQDGt!ZJqVv=DM0YQoi0O?&!00Ph!y^5Sjepm!#_6mrNJrbYA>Y(re4k0R1G#Q_Ind1EG*-Kjk&)pO?}F}j&okRj zXlVE^B{erSY3u2!ttLy7VCIKs{onTWCDzu;*AB-Ntf{H_Ha>pc zAmVWkar_s0Ml(IWPEs6SzI@@8l6pTzl4vNRuB1eYh_+UHt94XaPp_+*>D9e^!e%C) z-oMxVGCHaxFjw?alb4YfyMvBRWN=i0d&drykdPxip;X)Xla(4qJ+q68FLnFM-~)(r zp;V-7`;4Qbqsi+^nv3T37tVODAFnNU*Ew^h%;Gs&-6f`YRUr|PFM7LQRnZ(%6vVcs z=f|aLYHC(TO8^aFdHp)vCnzt^4P2xxoD#~_tY-A|$&*M4GRr*ogPh27!znRY0BBdQ zMq-5OE^VcaF??vxR1$s(3lCtiOehKgnfEz`=ciAfR@t~QlIPD;-^t5c4mo@MWGrSo zGsM1o*!StXk01R{jGZwSVe4-Tu(r0oa`Wa7KcgINA=SV@Sz;;o^Bv^e^01=f+B+_# zSZdqLSq>ca_4R-SzdtkaFf&IY!rkq1G}C15=ZOO4?M*zjqt5>gH zPEP)^-~ps}`yDUVy=PT#iS-d~hS=Cx|6NU67gavpr=Uq|Z4BB`*f|cs`RarKDL~_P zpvq95F|3A;ju(4rR^k9L3cD@t zB1(v(Mfxf@p_CL0M`F{MR6z@%x4nbI&@S28+1cvVx@QYY85b|og($} zcH+cNm+nX+{s5Z4>+Gy|O33dV_cs#d+`oVS`S|z6PQvWE{_miIh+aNj&f zwNpy{=uu1YZQ=(Ho{xwiMF0Xg-$blB1o@d6am0SkEmR_Gbevn7D=m)a>L3!m9QC}b zD{TMB_}hG7Lqh{ECK^|D{^CVdRaNp|fk$4{*Yl-ltQqfYeLz$UKYsi&5!%H}Gdw)Z zxPAM0w+APS8lSXuib0wn25zN7b&%?xYySMXwUwsbrE?Jz$jHUD8R7l;bEV8A>ur<( zf8y6qW;uLvqP_TeGPcD=+1#85@#)?B_t9AF@87@ADl28Jm^7&Xsa3)w(*dH^QM+ydV2a>T&POi7A4WNoq~c?d-d-tdU)(779AH? z;@h`xX(onUWOleckH2(D9iuos@5ME~)}~HUTPH~1L!l^YVvJ_}F;CwXl1z9Ru|jsr z$pve4CtbKOJa6H(lRs1_nLCL}{WcS3uQGT^LgT~JK2T zt*srNxZ^-hW#zVlf`S8&Bqn_i1A_S%Ni!;M-@ZNm@?}O;aCMK@0e4Y@+>sSLj}6Yu zofK!#U?Q5PlHWPYt=0H^X?8WniKCSwyVv!Ay1M$?4$Qxq{$;jPLuDi!v&rI~_T%q~om< zQ$iy(9bJ^J@cdqjrYd@bY+@QBr;`o~2t+U3PjRB5qZ{cu9|5#RPyfud!rI=R8U-K$ zCFM6(y-=vt54pd7wU1Y`6T5wKa?*dgX2Jc-OC}a}_7T)SO%H5av$`jS+zi-vVgl8h1T}VSekaY1wy%EPdIh?3%#}R&Tvq3};^|~_HkOL{ z+Q4~XM@Pq*7QxJ{+*}Ho8!q~^Ul@jJR{%5#D1NT}mKvsbs%|6nrHzCc+b*@!Z^yl> zg`GmF1(UJv(_$a;m z+5V*FP31j5&d88ie!uK?klLML50Cj@y27I=%5y4}dlCV(LTPrjJ{VzDt5DuWT`ykR zVevOq~po&y4!*@rUq+q0kAdJPthlchA1Op>F=i5vXu)#uE;Lv-XgG3nO6 zrHksfhf+5DC2b_OmlR3Xi{I%CU_@cjN^?isQ@S&{AX`^Bz$oH?ifG!;-}{-b&~pRz zA7YLduo&K**1Cx~{I6*tmJ8v=OZ~eYneDilUtb@kCXWpFIgj;I{yqDS5G{Z#6BE;j zE9v!Tnwz0KE zfMSi0kH!I&MoPjdff?#AZ8zh3tbxz4&xzsJ z(_;nFx|o{EX10}OiJv+C7|+W9N+u2EXIENBQARv-spyN1j3l$Mv*SmiIeq$cb^7E5 zRsQJMSbm%c*cGcDsqovkZ;D`Q?xAu+wQ5@F&L^~V%cf9ngBN3*?I3HBB2E_ZY}%+} zJxjlk2$I+y*5e(yhWI2iGc#F7{Hk%ZGT%evefXK1n=7?z<9OfIm7SAAhPwXwcxQh8 zqs`Z~h3G#neDi(kR>j$6!o;6k&Y9NAT6$etr@-4d*a9oJ`O=hGkZb$$?T@6c=)MQ#7t*W+wIani7C8H#c|j>kT(| z1Z99l@*l<#EE=_aX+jtHMZrn zR#tv-Kfqm$pNW>1_Hs%}4BiS}9?w4*_)Eyx*qF?!#^YxQ8R>xNNfxywsi{cDZ3tJS z+1k{^t|=LHKPZl^IV$~DDzT|a{e5@$sc|{f0H~QhRtHFO`wx<`u(7?l*ZbNDb8A@P z$_19<@{Jn_AuGR$akeWD&=@N!E>=gWc=g(~e9kqQ8*SCQy=Y^Uk${;)7?_b5!WDVL zO-09cC2Qqx)i3mUemdryX(PHPA- z+e>i}(@z9QZvI@I@de&Jgk6r-7gp7|JX<)!kNkz3^N{So|ReB!S z$zrpb3*WyxnoSrK=o&ut-50txVSc-`R6L6`+d}y?i-d&4%#ZSRe&F7c+GXLArWz9M z)2AZ{W~liR*QVA7;Oa)K4+GKwUT5nU@?C4;O=|_VY2Fy&u+(OW$6YA!l9rLe^{_F^68 z)A?g5DM21!uaVu_y4@M&CLj}0m;(?FhOLizD>rT&9#gxjf_Nq&!MxDx#URJc zu)f%=kb@IUPq%+=jl1DzcIJ%atvhchey56_y}-IxKdZR-%{29sV*<*c;Cg(i5FwP5 z1(^Bdr4UV!DX0@9_6nZAY@l1pZc0>;AM_+l79wRnK2vmR+mRGPTo(55Yc?KM@P%CBT&BhVB0 z_tlE9?Zh6w+ZVJ5#sEms1B4I3lnAGMPh~m21*b$gvid7uK`k(F`DSSck_C8$uMNg- ztcV3PDCa;D7>OmL_66#DwuwP598o-Ai0Ya~F3PBe_R!zuSzUKeQ55K20EdnN`tl)+ z){X`Q6yikrC*_gHz2Hj)P=1ZIMIAfM0%#!P+I#-+ROL>KsQSE35sD;_j1y518f|pc zq!2TVm4g{Z7*ueg3U-jOmZx4_p74iF!xC z>QLfqx1W~jpv~hiQyuDe}Z+lcbYoSyC3b}E>tqFwD*CMyMIQQ+e3QH}gfI^FJObM|iWWTe=?~q2a zwS_>MH$9a#D$+-VNxymXa|>|`{P`5zanq2Xu>cB)ucG9ZnD(RSByH5{-YR#>LpI}X zQBfmtfk+6{AYn;ak6*~i(iTG98=aiY@;9x@jS6x^xj?^=3BaI0S5`EcErG(Wd^9}5SW64>Z76BnqqI>a^9u-&)5eftHKRC@xv?2vM2UiO4r4|P zHcBQaJ+U7?=!0LR?47$UL<&8OZ2K(jN&CeECk9c@^ z5Y&Y0(@7bEfkURUDSgk`FfvO>5Wx6^=tLw993HTk@}#()Fe`V^kNtA94O_kPd_zPP zqa{eNy7TgO|LA+l?pC{Grx+W7La>E|iKS6c>vU0@KH*equ;|`N3L= z>QVRaix3+Stg5oUJ{!|ZlVqqt$*HMU;Pd}hqBfPxK2i`s9B)6ez{wrDXsLSi=(T}% zmVYTrUnMY0Ji#-h{r)4HLE@eqHwFC#u=(+!!`542>jAxC8*7NJ{0eg>+Yq^a{BUMr zWkq#s?eD)0=<|%|>4$%6e^r0~gB=|mU(Zg4W&zz(fc~mo9Ht|#`EFt1jgBe-W*WRd zvpAA8(sM3TO-bqbKFPoc@k6EyIoM(X0s_yRWvB#-UfN4p;n1M;3DIZ$)gH*6f>N3U zwMX2$*3E_}JF=t6W8 zPe*q*6$wF{JiYekc&MC-gArw9UH-ST3Ch{e2%YZpz&ab|DcrbdcY*v*mSRb5z}UZa zs<*wpa=B8Hv6gQ?f2PCPfjjT&-XkgYU&r-PVG&N(u88(ytY#0$@UYNQd?-GfFtGl+ z0*I9Y|71E&hCGCbj_8hh=d`(b5^9wGF|UK1;v9+83=Buj%(&wte}>DYguZwnLV$S` zvJW3Untc;?4UnptWpM!dUcq+4yki&ZNL0a9#f4q{^RpZN4-_zM&ou@!X z&B&)gIRosU#WZQQ=t-ILDUVap*AKogICC&`^S&QEbOUSC9nCzH)xJ-hpxqJ5+ z=<0x62jD^EZ@J$et&!h(YshG-!EG29vF$kE+JGG%kaUEbi%ad?xpRt+ug2(9g+iW< zdJYwq)GD7k#USRSbNihIWEUY((cRJ&(R=t5;JFgd0YNPq@q445huF*BdX z_d%0kXODCF{P+sV7E5CQI-&puu=HONPE_A1YFbH>(5U=)!0ewdVrh+~FyGM?F0+5f z-%%7JUIe<*x#s}|c!1ESzF^`ohM=m{?`wzwD3G|4;JR2#vQHu9OmvqBKI)BjP~*Qs zfCWfy)lHr0ZFai4x|eVM{qf>@Z$H0^fuAP;uz(%QJV!OGG@0{QkuY@%N*%j|&AMa} z`>;T7&Rx7^!o{DwbRx3U$|cAg z4_Fir1k0ZsorRj96huIO%NSTQRUyF$>QQoVY!=?$@26Lf17xC%!+`$zC<`qi2AC<2 zgGNjH_J$p+MA5(?ZQz5zOo+6Wr>6uHu7Qb~9ML*bBX@44V$8GZZD*%SaPl?9XiqYN zajdbF=GY9XA@}%Jnd&Tat!P1j zF}s=uP|I27C!&lFO^unt=E8JNE&F2D9im4Zb~i+u&74NZip=uxCNetv#pr8YFELZ#pr7N)f-^EZ_{H|Iv-o}qAK z>#RkIAhIpggXs}SX&2RZqDIKG)brxO8o{0lpvhIv9!yzmW!Jh4G5PJMPqMW!9rjX0 zO^2g5Sb4nVHq?@;e3gd@9fal%)TR~|rU2U6S!M?qdMs@m_YXx?HMR1!K?iyQYIR5o z_yKDxnwI1y>E-FEZD@EN@{(!V(QU8m(puG=-Q1c{3;2Ik(a?D1A5>#;KQ2coIwoc| zz2oI076F2K+4}6<4&~GD7wb9cD^1%`DiWn9(A$oZQf$b#j*nipw$?v(`CRHft^;Az z;VgsTB|)v5iXd~ofB#Mh3xw>^W>*(!-7Ax^^%U$m_2=&B?17ae;grG}3-%sJA|fF| z@dWLEWU^=y0|fBnu{dM-{=)}MR{f+esCn>xvl1Ur_-sOD)Y)Ho9hVv#JIAb%o)_RV zlZh=&&kLX_oZ|RtuP2-yEjBOCWroDAtt{Mr@`Ph;ZS4RxliKOK_wKzw0w@bv@%_oBV>q%s z!Wj9lCnBvmd3ls5km|e5!xB?dQ*ZdvrfK9NIS^FLRLw&F?7PjK4rgDDdYY|KY>OSz zi;*!$siug0U9v@V9ZW>z!ax|0Tg^NhlDwd^J*-)v)!m0)Q2bh0sJD?AdUupZO-oBl z@q?wz4G+A`dB~F-@$M+(vXcIci145Q0DmbdsjPeVXo-A?MN$^<(+>6B%su0(?Syk> zZN8gG-e3gWUT`=-aQ*$;_t*S4_(*8PK?dS<2g6YFZjQ!?ZSdyLrew>m?qAOK^gK)ch%Lo`vLm zf;zje)N=Y zAmHHAH?Go4FSfF%7d`l$S%C+C014f@b5J`Y_CU|KyO+G;&?(?KNE>nJWMg=*M zcmIA5lU`~VYaZv;6JcWzm+OaR6tlM&jA1Z7Z~oN&M32Pb_Yz^$o}84UPPf>^DhiOJ`iEl)?x#N)PbDW5&d1uW*#eb&!%cDOOLN`sP;(*4)B z(*&|JhGFUtfhR0XC=$6kEfaFrEXxf2``9BC8QsmNilFh)z%&+GXIaGpHx}~@wIoJm zeEfNSW+?6LcP~F*7|EON>h4|(AX%ae@ay%U zf(TdB?2rH(Am|UKX#bzv;&Gs`%xO%KVW>2vDt9eK`bGzM_?1LY-W0_-_9fRio zEgjM(!4INt-ILim>e(kHHfxO2qadBdUbCF;uR$sQg|+RgC++Q=oTiNvkV6RX(C{$S zqwFq6up`Cg!-pn%q068+v9KUP!HKV+l%j$bNHBcj+x&gTd}T8#{b%{Hn?VL&IQ!H$ zVDgyTBjfzl8~{pSA41;&y9`H&Z0^ssnO;-TA_Z+0it8t3iry_&ZNXeaTK$qzDu-`_ zG6^p^2j2h^H0dFBgS|(XZXR2QLFq+*P0-pFQ<8xsK~T5nlah|w+3ijhF(PORn{($> ziu6~Ui#Ei+DR)Z2oZ;x?gid@ANGC>WR10 zr97l}Oo@f~;@aUW#{@8V0#P%&2`mFLWA0UjBuApXi%Ul8OF!QG+oAfgvc{mm-Laks zL2>iuA(%YY>p$Mke}$8D?)Y^WVOMvc&K_Ragrj@s$__wUax^_cTLZAhfGZewj41J%%!xTAT8u{$Avp^Sru9o=Yky(7KL(MQqgokMyPxD@}C z#8);F0=sr;2Dz-7#!5|Pkf8qm^2MgENF&x#9Kf{DpoCJ2$)vE0|M20USD|-6M~Il$ zuEu^XMm-F}%Ock61@IP`W?N9JkyrQIq(^?5@)gG$v02FK@hK!|GHALGVJ$P`{;Cm{N z87h9%9|Y(H!-h;B>X$%Sd+KB@P_AN#Hq34yoKaE??o-C^jg5^cziQv;SZ$*OK=y~n z5@#8?7M8RU8VY7_#IJ6J3tlP8T%ALEAG`gb{h9G4|MA?k8CZ1rrKP*_(=LLjvbEjS zA2_iOs$l85%6l%l)vd9?IQ@HE z>Dn;?At9k>Vqr@sh*}h(fI&l&G3<&)E}-~}GpCNkjNJ;_0w znE+Svts_C&Iy%D(Cz4Mb4jPQ0x{S*bFZ_%`b#>`yHOJLsxjGHa&G2SjJM`KHs^3yz1% zL=s?Bss8!qu)UjG-J8v*l*@vbJc+dfp>eiHx<393vH@HMAHqjP9{jPw0=AhQdJMqw zvaKyfm7niX|25uV%$iyHxY}#=N}e{*8h|z9e$w>y;z&z`p6J6Wcn0FFB5XKdVZmIU zJ$n{>?$!%skR(71p`<66k=vD(ih=P;rl>Hn0VY`RYtH{h}v)^-STGSQaeq+>WO*61$isr>-BI~ zFNbt>)OThf^LpOCzRNXHaEUajFdQLD!UyxWh!vxF`hDee@q-5)PPXoiw8>088)I&XQRahALC8xZJRAPKS640LIqJpX+-TxipP|CK z4GeinyxVHN&J080d}~g|W_PRYJUk50Gxi!3`)lTUJnaKTg&~Jn^2H~t1O+X^KLR@6 zm1A!Q1{m@?1Nx3!J|;j6=OahfKN~25iksR#-|KH*_fQ;9&clcAr;o|0odyS253G?; zT3UL@Ma9ys8gTwPQ^H7ks{`^@Zd>{@Xih97@Vbyg{ zk7!JQrwx(-yiL5|L*Om@MY@L;%&Hu3KJA}UmKcj=4sB;ya7rg|CjdJ z!~XP(3&A&?+5;v_rBZF1JHzwlUJO01`&{966iWa4>X^bQj>;Sd4lqEld>_@(>DYXH z87m44dOvDd_puB=!V5iO;se7Xh;u@ohcOCimUKF_U7UV-d3k0u%^aC&Iiy-6SJ~Bd zYO@&Wi$tQ%#h79yVLZBBUfww^I{?!Y93=L^b{f1P0SGh3MZUp%&2)98`iB!y1*N>2 zHLZ!~j+~luMQRLOdkP9s#^qBqXc9hQVMh&y!w0A*VGsrRO(0rGCJ0)wNGr2mCPn&? z+y+6KqB8?)ta9@4XD~$J#dX&SwL_IuQCe-!S-ibz;K&Mf3vi-G_3O6HGArB<|jC^P`?PK#FT;2U_}2% zj~*R#A0)XI^1H{u!Mbsm$q9k0G_A~T01B!Qxk2cNkEh;%@T00a_?(Id9C=gnsv&)S z7RW9l>d*H14Oqe}L0?b!pCl!>5*!r4sbVaC4Ak)ugxEHEY*Ro==mF2Ila9n{-jLCB zSP^DRxRRC@w=!MFgQ_X>JqN()*qaQMbLYmQ_o(+C0oOWKvpWo{8sMOjpaXr<*s00k zn07R)6xElag%FrTUbWL3n;hNYTE<1L!C6tA( z1w!7}s@j~^35+G7i6O#m@)<-2@IewxAc5&P$md2O9p$wGb$O6__D`soWn4Fv(s}HX!a0G$} zCdl*tYoa5=!vweZVDdLKU$a4Bb}$67FJB%r8^g*YxM=yMFW2@Q$k@yRlbOA}{a?Q% z5eNw)pwvQ1G5tx0RANL##PKx;*NUJHU6Hg_=~~-NVW?%uT<|R4dM`WL)Hz4<7db~- zYYZyEYSeQFzW6U|0Ccg5nx4nF@n4k(RZ4{t`$9qj;czjFe6A)qfTTkZ-;`07>ibTl z=b^7RvbebTxPsu7W2M1MbK$v{&zUI(0 zZ34FeRPnGO(BIWZbpS1`I`cG9)qvQI2B(Z1KrCY9K~-mGkrFSMS4?XHS;IC~24?YM zR$X0K6Sr(!eFQpoN=7f>-eJ8ZrBmLMq5*NX>?dHiJ)PqK)}jYPpG!ybnmC z?SRnwP$&Sfn$P?e(`1FaP5OL&qC04&Erwg}BC8=%5M?w1+jFr(?OYu~=_cGGNCsr7 zNq3kcrW#%InUcG6u^%#$mW0FlizvBdZF;~V_#E&rR7TS zio}$ovUo81dn-l`*jQiqSm`nH@U;)@Wuz_-IPD3>$jB%jDZM{cz#zY!bu;5? z6m%6x1q7`G&-ev~nKjj|ps7RP3HyJ+%0osdfOKNj z;V42Jwn|*cX*#O^S{(!!inRPMotJ>h05!@2=f9H_JO5FY2pa(!s@(o!{KjBf-le#i z{wKRzj$)@2~k74M(X@fWD+;| zxq|Ax(F~f3b}?X=5WxTSLcje-`{PEQt~n%?b{Q zQj+vh5UMhxbwt?Ok4%qm@5oFFmpK8Dn$ngH{U#<&-Uq+fU3%QSsmVYXl{tc(|Opro_BrF^{(nVtwj(6_Gok(8k^Pw><&Rdo4>8yryJRUQ|Fl!i_j zf+s7fknCod1wK+YHSN7fK~0pPDJjz9=PFlg&>VK^1jklDrsKyW)Nfaf*U`l&qr!A> zGMB;ZpgkfRBc-!HPly&3^%)JC8?>-n@88b^%1(Uxs{p8Nm_}cKwzAW-POXCKqOhHt zTOBN=8=*lB!YfdJb=L{IhV(B)n_`YyLbbd%xGsd4qtLFN@ra}C<>}NDIzlJZFGNEX z*L2aIv{s@86p&l<5!XyM4bI@#T1?_H6 z@Is#-DpvYM=M-S4)hWNa`mxNgOUS-xBSmL0udFNwB%Nkp7x~=X90#Iojga^_j|8${ z#%}$hct+s?(#7s|4!QVx#*a^r=m`HIfm051ASk1R9R_*$5lKs`=+_4XbRGn7d^@>9 zx&R^;gnuG;kt1w_iY8E$`{K;daj#uy8Zrx8^Rcn9v6%nfMOq$R^=6Rpe!@rSFU-8K z;UjzY+0z$ZBWGJ%TW3F*xCL-=An<_exCkP7`!u;Ikn5Ez5l4c4XFYn9V(_^FsEQaE zxIk{r)}0eodfYD$_5careg3;%N#}}dESQwgPF@dnC{&$62_AfP!HpAlkDsqn|-i zd-#xr@U?-DmbV^F3x;EnfE|Qmjo9S!Z4TY`QXk9DF(5eE3GB1y#~h*dC6r9Fab<#4 z_WJtz=94hVwjWh1<=b2*deF*IsG)`tf#Vb@?z%KANyy6}6QKnRcB-5MF%J%R74DG( z1czKFc;*c`bR>T))!8_ORZ?49Tlp$2qIL&b2bH3nt3zR#Qd{qHVE}q{$)g@;8CX=n z_e9*GU80`-x=b?Gy*#9=XigNuV8fT^C(cQ6+4JL0!N~q;*()yI0&U-{q6@m?$Oj(~ z3Pk5JDj%bTg`P>@n+jpv1VjalM+3r_PHhR-tllUF`SXj5i$=DZp0FL>H?QLclKUB? zN&ylXT^~z7$3Q+vCKguvF#cDgvE5;ahCvHch`A{P^L-#oh^&R_Mo>E^$LlVM^o%I= zT$~G{+TngH z@cSMZSA6uz`&}u!g>FMq0Z0hl5Oqk8g#!lI#I8^DoQ$%FhM0Tg+m9c=T?=1jM7tOu zxk|S7>sM4;7Uk19&;$vRHM_8o4y9p#bfR$R%V*C>YS)*YK>ZJ)E+V))>=+PfknE_+ z4CN-n4s;6_;R`5Fi0TJc0{7WrwYsC^6W+4DkQ#{QZt$G^4mi3D5V)Mv$w?r9GYcP}&UEN>=4T?>b zFxw?8qUe$VIXMWXHmo+)hE;jw-o=214(tV;1!Q*Ls6196GbkeXuo?3G-GYK4X>Q&8 zPD`W@ui@+wf`f&7lCZ_XP&$OBU81SWBpk~N^(cUz2jcJSh8QC?w0XF8;YP*-B=Arl z%CuIBcvZ3(<^5CVyiq7>%UeeN)WftgYmT(3iIN9Y*EQ!sAqpT!Ik@UHLo!qx9g+P(38BEyvWh z)QgMXXu}WDPyNhssb15+w>IQ3R(yfcY2)u=y$PCyT{-#r%_Z#c91y+1kaoX#F5RDR zena{7!&XikU{;VDP(!{~sO?J8VB+DKUReF!6A5pvudhrf>WZB*GHjW_b2LOXNb%5a zC&7221|}>#c+$Y@?|Uv@d#?8#4=4cX|E}cQ0bJ7clK?l0_V(Xy24z!ioohP@e~3DA zcS-gF_Ba59*K)HUOc?M8AiHv-_WG~(cS!Un9q5}*^5atQ?(neHDk5tWdYJQEKG}#5 za-2u&;Sj1%Vp=BrU0Is2yvKgI`8?qpcrD=0wky+-6L=Bcz1K_j{C_z0j}L6=h|a?# zh4pQONCr9wJtC)Qddo@SFab;>OsJ^Y2?UJ-!Ij1XVFl$PX|hJ{fU-lyXBY|9SCh^D zTB*D0KsfZu%dZ(|-Ur!YXZQ6cJNmUGYf+ZYpp>>oQX^DD^hmwK6CLPvos`kr~i3;gi?X7GExfsiyzSv0B>sy z9*gnP=7GtJ3)O7J^eKh>b7N2a4suo>wrUe}EiykzGIZ|f7%RP-%4@0&wdd(TV_a)9Z;Z4iwOXD z0y)nK+{BsLvNNr9%tpJ7y^_7$tRzmAe|DmA%Ciaqgc^?vNwsxr8F1ny$aGWHx2_c{ zYvJ$@V>$KDDadMxt0*f&u8de+4VIgGxpSmV0s<$|9fMA7qyp3e;`EZX6KcGS7fmX_ ziXA(766_8op=M-cB+ksZ<53`Bwz877-$-F#&}ED!1&Qmg#j?M6>TB-qyqUO4wTU1) z(Xptm?mxOknB{`eJEp^i?xZ}p69hq+{Q$d&ZiFFsSgT;ufb>bYX^|SML#)7~VM*gh zHe-v#b?53=N6~2sX9O?f>l@Rz@YhAU+!CK<;wcYcRp{$`b&7Kom80i2D*VpXX_|)Z zc$Rs6_T#0=gO?_?l?^i#Kfb8FyGNHl`JubgF4wyanIFt_hwm{bYh*pvCfD4{M{au} zT_)lV&GDBS!wvC%?Kc<5w$~`^dGq?Fzi_%-=1SS6`?DkEp{FRo23uMxeW#jSSb6vM z?Vxg@eDLUyyAaEACT*`WHMFyBv1Rux{i_!4ID85VG&%KX&xJ^I92HFP!QR8h#&+;j z6&a;1MV|UQSM#)lGc9EMl((e zR%Xlv*1&-2m93}RiqAru)d^EA4CwBr!@-oP$HNTIb}M}3uX!kg^JxnSJeqfqo?Z@o zo^?OVv`F3Vl8&C5yN?kDTajRL&CxE4Cz z;_+tbZQXI|+4U*Dd@0SKurQ{5NlH~@J%=)b<&&0{0@#gJAIkBOe;pgk{T`J%cA!84 z_c}Us{$%5IlGqAI+ zNlNqNsZ%FduZz}A(7$@M*rp3}wN=?)?;S06Uwhur5KULaZ+wf9Y-D6ar#toAlS}qe zud+-@>#oBErzXH0(LUxScj~%UzL9kLx?Zg1VB3uYm!ck+GTltoJ@#v;H>=``NZ7?|tjTg%%}G(#xkCK?|57?$ zEJ?4jydF?Dkc6jtNC#=EKN1%*046DQ`7DDGYHk-56-B#7QpM*h6Iea#>nmhSa%H$ZY#4!huTBImY$+(fnI80&DxjTlric4kRoQun0;|Ye=vEVbcOcNNhoV(*}UCUrzr)uCXt%K*WoEsrENj}T38 zVAWn`J&ad!xYj)KoZUP)0&M&7BXy;v;dq=3X%hF5tM|4aBd;#>Ssr}8skb`d%yM44 zDt`nj&bi=xTWNi5g^8XL=M_%*M?$u8cfU+`_>18Z%Nw9MbpHtW4DaRK zLOcovE5SEA#)&iUIcvbX8#iOI+Scp3IA8Nix6#mwMf$0>IFJ(HVATUS3g}nt& z3Ay&^08RCTrmwg6u}M0!_HD^qGL8hXxwyFK9Oa1cqdN?4c;8fi&E6y4IrD-k8be^& z;Lf=B-j{cA$e|#iJypJ%p-?A+OY3k{;|c?Hp-N@)&8_ z{QC84-724lA1)>(jlclqzx=Dfd)GZ|Ji;Y~-pc~di4RnixzV5o(AG(cUWqIHW>?zB zJkXp1+edQwYEBTO9+BRF?XkPs=RZX`EkCF{bSPDlcH(h0VLPA0rvaOoI| zM+qC7nl>yhR{pBK(>>8hn5*GW8&~kL($?5JT_}f>!4G^#LGVVK=QmB(6pdUxF71za z)(9aLa2|9IuW8Xkcz~%B#XorSj!ekz^)JYEuQZQbF_R(>*A|Ml5Px~*U}1yXsjG}+ zD5)!h7DUnRkSnj!J-?8<7@xW|UbPe!fpLY-jg3nWn!@EMbMKgn#zDJ8A$a0@G9D6u zE~bu!*|*DRZ-*>Ne7YdpOEj$>G?V+n)YGKvD0i;TT_7@fh#e1^;kN zJ^#JCt8sz&*Z5Nx+_d|2{}DtugDRg1s<3Q0Nty9{nkcYB$M-d$7(!7%RNiHVuVdIc zj5~mWevY-nxuEv_RF^otT<{_z#AJ~s{&=N5*-*BUcodotK5}oB6cjv114<0~@cP4# zrh1=kh9FNguZ|blcXoFA|N6!R9r0f8llcfN9_$&!t!wwzN9F^GwgBQeQN7*jkJWCU zM!_{a_&iD{tT@(^XjEa~mUGYcapH;U$nNPek_-HGEYMYgbIS-6OAN~o&8sd?imkhg z&*HfO4}SSRMExokFh|~E^WasH$qKd!HQ&tYk{2-lF&DDmD{G60JRVh^p^>v%5>o(e|@WTdZa zi)wb3TGmsdmB_&H1l2pw)&JGZ`F}N4#&JAM2xNKiD8zvggAnK3Y=!&)1+u}UP0%q0 z0}9*#7b6nAi*K?6DIg$ffkR*@Br5u4_{d9Mqd!PG!KF{m#C+ts&K zqUFexR)t`3+}S}_%q7aVWCBnReS#H!YWwf`2*P?wnrWaFc2K)|vZ#ja=#?fY6pQ(S zJ(q`+1R*20*iH`|yJ_#U6)}78F7t(uwGCZHR#qvsyr5x{!J?Qai(cB61a7KIWw)Vg zGG}|b>)oQp24-fij(3VSVaVU@8x{rS0(k7i%;iK_{oT9V603uQv5b(qRf&K=x==LW zo-v;wq6^k}JN0EkJFQmBka(^;eAzSV+txK;zrE&zM;T{ zYp$<`eE)Y(w2dAlW$nu3q3a|^(L0nA+I)1$D}eDVI*P(KUnf2w(xfqrtChYzfJl_$ zj*m`5b%e*m@&GG9NTW5yKY9#D)QKB@6o_BRX9xDB8QR8ATw1FT{N?LYDnF_pbowcv z`vZXntl+%X zlk_ohuA$8%W%2DHz%E>y{pXL159$qu)V=!%)ZEGPg6vi6Uy)u(^cq%wfi)|{i1!=GBaUal`N*f5mj1y} z8+8fze3*Ak7);EgXz@V-qD@MGfjOJ(CKevU%~?VuH>s+O1Lvovl3+3T1M~$E^=EoV z?h>*ZqUI24?#*f<_t*XK2a1GfZi?ihzaJaR@o(K4*Ljm*e;Pt*czF%TJX`!UIWr?d zE-rX;6h&F*z-g8d%4Tcn>T2i{)3yQogYmtzYgvP?-k9w(P z5j^F)7vzEP%&YltEy+5e6+_3%WrZcwKGA25X@|Sg4Z}@?guQPEU-qS=&u@y^x&rUM ul$T4ab`HCDRNLpCs;J=d*_HpppGV#f2f~wB=3C# + {!window.location.hostname.includes("useanything.com") && ( )} + {llmChoice === "ollama" && ( + + )} {llmChoice === "native" && ( )} diff --git a/frontend/src/pages/OnboardingFlow/OnboardingModal/Steps/DataHandling/index.jsx b/frontend/src/pages/OnboardingFlow/OnboardingModal/Steps/DataHandling/index.jsx index cd63d74d..81b93c5d 100644 --- a/frontend/src/pages/OnboardingFlow/OnboardingModal/Steps/DataHandling/index.jsx +++ b/frontend/src/pages/OnboardingFlow/OnboardingModal/Steps/DataHandling/index.jsx @@ -5,6 +5,7 @@ import OpenAiLogo from "@/media/llmprovider/openai.png"; import AzureOpenAiLogo from "@/media/llmprovider/azure.png"; import AnthropicLogo from "@/media/llmprovider/anthropic.png"; import GeminiLogo from "@/media/llmprovider/gemini.png"; +import OllamaLogo from "@/media/llmprovider/ollama.png"; import LMStudioLogo from "@/media/llmprovider/lmstudio.png"; import LocalAiLogo from "@/media/llmprovider/localai.png"; import ChromaLogo from "@/media/vectordbs/chroma.png"; @@ -61,6 +62,13 @@ const LLM_SELECTION_PRIVACY = { ], logo: LocalAiLogo, }, + ollama: { + name: "Ollama", + description: [ + "Your model and chats are only accessible on the machine running Ollama models", + ], + logo: OllamaLogo, + }, native: { name: "Custom Llama Model", description: [ diff --git a/frontend/src/pages/OnboardingFlow/OnboardingModal/Steps/LLMSelection/index.jsx b/frontend/src/pages/OnboardingFlow/OnboardingModal/Steps/LLMSelection/index.jsx index f877e31d..850dea3c 100644 --- a/frontend/src/pages/OnboardingFlow/OnboardingModal/Steps/LLMSelection/index.jsx +++ b/frontend/src/pages/OnboardingFlow/OnboardingModal/Steps/LLMSelection/index.jsx @@ -4,6 +4,7 @@ import OpenAiLogo from "@/media/llmprovider/openai.png"; import AzureOpenAiLogo from "@/media/llmprovider/azure.png"; import AnthropicLogo from "@/media/llmprovider/anthropic.png"; import GeminiLogo from "@/media/llmprovider/gemini.png"; +import OllamaLogo from "@/media/llmprovider/ollama.png"; import LMStudioLogo from "@/media/llmprovider/lmstudio.png"; import LocalAiLogo from "@/media/llmprovider/localai.png"; import System from "@/models/system"; @@ -16,6 +17,7 @@ import LMStudioOptions from "@/components/LLMSelection/LMStudioOptions"; import LocalAiOptions from "@/components/LLMSelection/LocalAiOptions"; import NativeLLMOptions from "@/components/LLMSelection/NativeLLMOptions"; import GeminiLLMOptions from "@/components/LLMSelection/GeminiLLMOptions"; +import OllamaLLMOptions from "@/components/LLMSelection/OllamaLLMOptions"; function LLMSelection({ nextStep, prevStep, currentStep }) { const [llmChoice, setLLMChoice] = useState("openai"); @@ -124,13 +126,24 @@ function LLMSelection({ nextStep, prevStep, currentStep }) { onClick={updateLLMChoice} /> + {!window.location.hostname.includes("useanything.com") && ( + + )}
{llmChoice === "openai" && } @@ -143,6 +156,7 @@ function LLMSelection({ nextStep, prevStep, currentStep }) { )} {llmChoice === "localai" && } + {llmChoice === "ollama" && } {llmChoice === "native" && }
diff --git a/server/.env.example b/server/.env.example index f73e0e08..07abed62 100644 --- a/server/.env.example +++ b/server/.env.example @@ -32,6 +32,11 @@ JWT_SECRET="my-random-string-for-seeding" # Please generate random string at lea # LOCAL_AI_MODEL_TOKEN_LIMIT=4096 # LOCAL_AI_API_KEY="sk-123abc" +# LLM_PROVIDER='ollama' +# OLLAMA_BASE_PATH='http://host.docker.internal:11434' +# OLLAMA_MODEL_PREF='llama2' +# OLLAMA_MODEL_TOKEN_LIMIT=4096 + ########################################### ######## Embedding API SElECTION ########## ########################################### diff --git a/server/models/systemSettings.js b/server/models/systemSettings.js index b5dfeb70..a66f93e1 100644 --- a/server/models/systemSettings.js +++ b/server/models/systemSettings.js @@ -126,6 +126,20 @@ const SystemSettings = { AzureOpenAiEmbeddingModelPref: process.env.EMBEDDING_MODEL_PREF, } : {}), + + ...(llmProvider === "ollama" + ? { + OllamaLLMBasePath: process.env.OLLAMA_BASE_PATH, + OllamaLLMModelPref: process.env.OLLAMA_MODEL_PREF, + OllamaLLMTokenLimit: process.env.OLLAMA_MODEL_TOKEN_LIMIT, + + // For embedding credentials when ollama is selected. + OpenAiKey: !!process.env.OPEN_AI_KEY, + AzureOpenAiEndpoint: process.env.AZURE_OPENAI_ENDPOINT, + AzureOpenAiKey: !!process.env.AZURE_OPENAI_KEY, + AzureOpenAiEmbeddingModelPref: process.env.EMBEDDING_MODEL_PREF, + } + : {}), ...(llmProvider === "native" ? { NativeLLMModelPref: process.env.NATIVE_LLM_MODEL_PREF, diff --git a/server/utils/AiProviders/ollama/index.js b/server/utils/AiProviders/ollama/index.js new file mode 100644 index 00000000..3aa58f76 --- /dev/null +++ b/server/utils/AiProviders/ollama/index.js @@ -0,0 +1,208 @@ +const { chatPrompt } = require("../../chats"); + +// Docs: https://github.com/jmorganca/ollama/blob/main/docs/api.md +class OllamaAILLM { + constructor(embedder = null) { + if (!process.env.OLLAMA_BASE_PATH) + throw new Error("No Ollama Base Path was set."); + + this.basePath = process.env.OLLAMA_BASE_PATH; + this.model = process.env.OLLAMA_MODEL_PREF; + this.limits = { + history: this.promptWindowLimit() * 0.15, + system: this.promptWindowLimit() * 0.15, + user: this.promptWindowLimit() * 0.7, + }; + + if (!embedder) + throw new Error( + "INVALID OLLAMA SETUP. No embedding engine has been set. Go to instance settings and set up an embedding interface to use Ollama as your LLM." + ); + this.embedder = embedder; + } + + streamingEnabled() { + return "streamChat" in this && "streamGetChatCompletion" in this; + } + + // Ensure the user set a value for the token limit + // and if undefined - assume 4096 window. + promptWindowLimit() { + const limit = process.env.OLLAMA_MODEL_TOKEN_LIMIT || 4096; + if (!limit || isNaN(Number(limit))) + throw new Error("No Ollama token context limit was set."); + return Number(limit); + } + + async isValidChatCompletionModel(_ = "") { + return true; + } + + constructPrompt({ + systemPrompt = "", + contextTexts = [], + chatHistory = [], + userPrompt = "", + }) { + const prompt = { + role: "system", + content: `${systemPrompt} +Context: + ${contextTexts + .map((text, i) => { + return `[CONTEXT ${i}]:\n${text}\n[END CONTEXT ${i}]\n\n`; + }) + .join("")}`, + }; + return [prompt, ...chatHistory, { role: "user", content: userPrompt }]; + } + + async isSafe(_input = "") { + // Not implemented so must be stubbed + return { safe: true, reasons: [] }; + } + + async sendChat(chatHistory = [], prompt, workspace = {}, rawHistory = []) { + const textResponse = await fetch(`${this.basePath}/api/chat`, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + model: this.model, + stream: false, + options: { + temperature: Number(workspace?.openAiTemp ?? 0.7), + }, + messages: await this.compressMessages( + { + systemPrompt: chatPrompt(workspace), + userPrompt: prompt, + chatHistory, + }, + rawHistory + ), + }), + }) + .then((res) => { + if (!res.ok) + throw new Error(`Ollama:sendChat ${res.status} ${res.statusText}`); + return res.json(); + }) + .then((data) => data?.message?.content) + .catch((e) => { + console.error(e); + throw new Error(`Ollama::sendChat failed with: ${error.message}`); + }); + + if (!textResponse.length) + throw new Error(`Ollama::sendChat text response was empty.`); + + return textResponse; + } + + async streamChat(chatHistory = [], prompt, workspace = {}, rawHistory = []) { + const response = await fetch(`${this.basePath}/api/chat`, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + model: this.model, + stream: true, + options: { + temperature: Number(workspace?.openAiTemp ?? 0.7), + }, + messages: await this.compressMessages( + { + systemPrompt: chatPrompt(workspace), + userPrompt: prompt, + chatHistory, + }, + rawHistory + ), + }), + }).catch((e) => { + console.error(e); + throw new Error(`Ollama:streamChat ${error.message}`); + }); + + return { type: "ollamaStream", response }; + } + + async getChatCompletion(messages = null, { temperature = 0.7 }) { + const textResponse = await fetch(`${this.basePath}/api/chat`, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + model: this.model, + messages, + stream: false, + options: { + temperature, + }, + }), + }) + .then((res) => { + if (!res.ok) + throw new Error( + `Ollama:getChatCompletion ${res.status} ${res.statusText}` + ); + return res.json(); + }) + .then((data) => data?.message?.content) + .catch((e) => { + console.error(e); + throw new Error( + `Ollama::getChatCompletion failed with: ${error.message}` + ); + }); + + if (!textResponse.length) + throw new Error(`Ollama::getChatCompletion text response was empty.`); + + return textResponse; + } + + async streamGetChatCompletion(messages = null, { temperature = 0.7 }) { + const response = await fetch(`${this.basePath}/api/chat`, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + model: this.model, + stream: true, + messages, + options: { + temperature, + }, + }), + }).catch((e) => { + console.error(e); + throw new Error(`Ollama:streamGetChatCompletion ${error.message}`); + }); + + return { type: "ollamaStream", response }; + } + + // Simple wrapper for dynamic embedder & normalize interface for all LLM implementations + async embedTextInput(textInput) { + return await this.embedder.embedTextInput(textInput); + } + async embedChunks(textChunks = []) { + return await this.embedder.embedChunks(textChunks); + } + + async compressMessages(promptArgs = {}, rawHistory = []) { + const { messageArrayCompressor } = require("../../helpers/chat"); + const messageArray = this.constructPrompt(promptArgs); + return await messageArrayCompressor(this, messageArray, rawHistory); + } +} + +module.exports = { + OllamaAILLM, +}; diff --git a/server/utils/chats/stream.js b/server/utils/chats/stream.js index 5bdb7a1f..b0dc9186 100644 --- a/server/utils/chats/stream.js +++ b/server/utils/chats/stream.js @@ -199,6 +199,7 @@ async function streamEmptyEmbeddingChat({ return; } +// TODO: Refactor this implementation function handleStreamResponses(response, stream, responseProps) { const { uuid = uuidv4(), sources = [] } = responseProps; @@ -231,6 +232,34 @@ function handleStreamResponses(response, stream, responseProps) { }); } + if (stream?.type === "ollamaStream") { + return new Promise(async (resolve) => { + let fullText = ""; + for await (const dataChunk of stream.response.body) { + const chunk = JSON.parse(Buffer.from(dataChunk).toString()); + fullText += chunk.message.content; + writeResponseChunk(response, { + uuid, + sources: [], + type: "textResponseChunk", + textResponse: chunk.message.content, + close: false, + error: false, + }); + } + + writeResponseChunk(response, { + uuid, + sources, + type: "textResponseChunk", + textResponse: "", + close: true, + error: false, + }); + resolve(fullText); + }); + } + // If stream is not a regular OpenAI Stream (like if using native model) // we can just iterate the stream content instead. if (!stream.hasOwnProperty("data")) { diff --git a/server/utils/helpers/customModels.js b/server/utils/helpers/customModels.js index 3b4397c3..5bd7b299 100644 --- a/server/utils/helpers/customModels.js +++ b/server/utils/helpers/customModels.js @@ -1,4 +1,4 @@ -const SUPPORT_CUSTOM_MODELS = ["openai", "localai", "native-llm"]; +const SUPPORT_CUSTOM_MODELS = ["openai", "localai", "ollama", "native-llm"]; async function getCustomModels(provider = "", apiKey = null, basePath = null) { if (!SUPPORT_CUSTOM_MODELS.includes(provider)) @@ -9,6 +9,8 @@ async function getCustomModels(provider = "", apiKey = null, basePath = null) { return await openAiModels(apiKey); case "localai": return await localAIModels(basePath, apiKey); + case "ollama": + return await ollamaAIModels(basePath, apiKey); case "native-llm": return nativeLLMModels(); default: @@ -59,6 +61,37 @@ async function localAIModels(basePath = null, apiKey = null) { return { models, error: null }; } +async function ollamaAIModels(basePath = null, _apiKey = null) { + let url; + try { + new URL(basePath); + if (basePath.split("").slice(-1)?.[0] === "/") + throw new Error("BasePath Cannot end in /!"); + url = basePath; + } catch { + return { models: [], error: "Not a valid URL." }; + } + + const models = await fetch(`${url}/api/tags`) + .then((res) => { + if (!res.ok) + throw new Error(`Could not reach Ollama server! ${res.status}`); + return res.json(); + }) + .then((data) => data?.models || []) + .then((models) => + models.map((model) => { + return { id: model.name }; + }) + ) + .catch((e) => { + console.error(e); + return []; + }); + + return { models, error: null }; +} + function nativeLLMModels() { const fs = require("fs"); const path = require("path"); diff --git a/server/utils/helpers/index.js b/server/utils/helpers/index.js index 115df400..bde5e8a0 100644 --- a/server/utils/helpers/index.js +++ b/server/utils/helpers/index.js @@ -43,6 +43,9 @@ function getLLMProvider() { case "localai": const { LocalAiLLM } = require("../AiProviders/localAi"); return new LocalAiLLM(embedder); + case "ollama": + const { OllamaAILLM } = require("../AiProviders/ollama"); + return new OllamaAILLM(embedder); case "native": const { NativeLLM } = require("../AiProviders/native"); return new NativeLLM(embedder); diff --git a/server/utils/helpers/updateENV.js b/server/utils/helpers/updateENV.js index fe4f4f5c..11278f97 100644 --- a/server/utils/helpers/updateENV.js +++ b/server/utils/helpers/updateENV.js @@ -81,6 +81,19 @@ const KEY_MAPPING = { checks: [], }, + OllamaLLMBasePath: { + envKey: "OLLAMA_BASE_PATH", + checks: [isNotEmpty, validOllamaLLMBasePath], + }, + OllamaLLMModelPref: { + envKey: "OLLAMA_MODEL_PREF", + checks: [], + }, + OllamaLLMTokenLimit: { + envKey: "OLLAMA_MODEL_TOKEN_LIMIT", + checks: [nonZero], + }, + // Native LLM Settings NativeLLMModelPref: { envKey: "NATIVE_LLM_MODEL_PREF", @@ -208,6 +221,17 @@ function validLLMExternalBasePath(input = "") { } } +function validOllamaLLMBasePath(input = "") { + try { + new URL(input); + if (input.split("").slice(-1)?.[0] === "/") + return "URL cannot end with a slash"; + return null; + } catch { + return "Not a valid URL"; + } +} + function supportedLLM(input = "") { return [ "openai", @@ -216,6 +240,7 @@ function supportedLLM(input = "") { "gemini", "lmstudio", "localai", + "ollama", "native", ].includes(input); }