Implement AzureOpenAI model chat streaming (#518)

resolves #492

server/package.json

@@ -21,7 +21,7 @@
   },
   "dependencies": {
     "@anthropic-ai/sdk": "^0.8.1",
-    "@azure/openai": "^1.0.0-beta.3",
+    "@azure/openai": "1.0.0-beta.10",
     "@google/generative-ai": "^0.1.3",
     "@googleapis/youtube": "^9.0.0",
     "@pinecone-database/pinecone": "^0.1.6",

server/utils/AiProviders/azureOpenAi/index.js

@@ -98,9 +98,9 @@ class AzureOpenAiLLM {
       })
       .then((res) => {
         if (!res.hasOwnProperty("choices"))
-          throw new Error("OpenAI chat: No results!");
+          throw new Error("AzureOpenAI chat: No results!");
         if (res.choices.length === 0)
-          throw new Error("OpenAI chat: No results length!");
+          throw new Error("AzureOpenAI chat: No results length!");
         return res.choices[0].message.content;
       })
       .catch((error) => {
@@ -112,6 +112,31 @@ class AzureOpenAiLLM {
     return textResponse;
   }
 
+  async streamChat(chatHistory = [], prompt, workspace = {}, rawHistory = []) {
+    if (!this.model)
+      throw new Error(
+        "No OPEN_MODEL_PREF ENV defined. This must the name of a deployment on your Azure account for an LLM chat model like GPT-3.5."
+      );
+
+    const messages = await this.compressMessages(
+      {
+        systemPrompt: chatPrompt(workspace),
+        userPrompt: prompt,
+        chatHistory,
+      },
+      rawHistory
+    );
+    const stream = await this.openai.streamChatCompletions(
+      this.model,
+      messages,
+      {
+        temperature: Number(workspace?.openAiTemp ?? 0.7),
+        n: 1,
+      }
+    );
+    return { type: "azureStream", stream };
+  }
+
   async getChatCompletion(messages = [], { temperature = 0.7 }) {
     if (!this.model)
       throw new Error(
@@ -125,6 +150,23 @@ class AzureOpenAiLLM {
     return data.choices[0].message.content;
   }
 
+  async streamGetChatCompletion(messages = [], { temperature = 0.7 }) {
+    if (!this.model)
+      throw new Error(
+        "No OPEN_MODEL_PREF ENV defined. This must the name of a deployment on your Azure account for an LLM chat model like GPT-3.5."
+      );
+
+    const stream = await this.openai.streamChatCompletions(
+      this.model,
+      messages,
+      {
+        temperature,
+        n: 1,
+      }
+    );
+    return { type: "azureStream", stream };
+  }
+
   // Simple wrapper for dynamic embedder & normalize interface for all LLM implementations
   async embedTextInput(textInput) {
     return await this.embedder.embedTextInput(textInput);

server/utils/chats/stream.js

@@ -232,6 +232,36 @@ function handleStreamResponses(response, stream, responseProps) {
     });
   }
 
+  if (stream?.type === "azureStream") {
+    return new Promise(async (resolve) => {
+      let fullText = "";
+      for await (const event of stream.stream) {
+        for (const choice of event.choices) {
+          const delta = choice.delta?.content;
+          if (!delta) continue;
+          writeResponseChunk(response, {
+            uuid,
+            sources: [],
+            type: "textResponseChunk",
+            textResponse: delta,
+            close: false,
+            error: false,
+          });
+        }
+      }
+
+      writeResponseChunk(response, {
+        uuid,
+        sources,
+        type: "textResponseChunk",
+        textResponse: "",
+        close: true,
+        error: false,
+      });
+      resolve(fullText);
+    });
+  }
+
   // If stream is not a regular OpenAI Stream (like if using native model, Ollama, or most LangChain interfaces)
   // we can just iterate the stream content instead.
   if (!stream.hasOwnProperty("data")) {

server/yarn.lock

@@ -48,10 +48,10 @@
     pad-left "^2.1.0"
     tslib "^2.5.0"
 
-"@azure-rest/core-client@^1.1.3":
+"@azure-rest/core-client@^1.1.7":
-  version "1.1.4"
+  version "1.1.7"
-  resolved "https://registry.yarnpkg.com/@azure-rest/core-client/-/core-client-1.1.4.tgz#628381c3653f6dbae584ca6f2ae5f74a5c015526"
+  resolved "https://registry.yarnpkg.com/@azure-rest/core-client/-/core-client-1.1.7.tgz#83d20a6f1bf6222bd5b4e22e3e3cf65b98a61448"
-  integrity sha512-RUIQOA8T0WcbNlddr8hjl2MuC5GVRqmMwPXqBVsgvdKesLy+eg3y/6nf3qe2fvcJMI1gF6VtgU5U4hRaR4w4ag==
+  integrity sha512-eQdtieYrOfRwsHFuz6vNANpOT567456m8/CpE3cqdVQgLMrn1uua5O8nLS/XN727MsbTgiZP6C0rAkupt3ky7Q==
   dependencies:
     "@azure/abort-controller" "^1.1.0"
     "@azure/core-auth" "^1.3.0"
@@ -76,17 +76,21 @@
     "@azure/core-util" "^1.1.0"
     tslib "^2.2.0"
 
-"@azure/core-lro@^2.5.3":
+"@azure/core-rest-pipeline@^1.13.0":
-  version "2.5.4"
+  version "1.13.0"
-  resolved "https://registry.yarnpkg.com/@azure/core-lro/-/core-lro-2.5.4.tgz#b21e2bcb8bd9a8a652ff85b61adeea51a8055f90"
+  resolved "https://registry.yarnpkg.com/@azure/core-rest-pipeline/-/core-rest-pipeline-1.13.0.tgz#770b003c351b4869e3f1c85800bacb947c98cd33"
-  integrity sha512-3GJiMVH7/10bulzOKGrrLeG/uCBH/9VtxqaMcB9lIqAeamI/xYQSHJL/KcsLDuH+yTjYpro/u6D/MuRe4dN70Q==
+  integrity sha512-a62aP/wppgmnfIkJLfcB4ssPBcH94WzrzPVJ3tlJt050zX4lfmtnvy95D3igDo3f31StO+9BgPrzvkj4aOxnoA==
   dependencies:
-    "@azure/abort-controller" "^1.0.0"
+    "@azure/abort-controller" "^1.1.0"
-    "@azure/core-util" "^1.2.0"
+    "@azure/core-auth" "^1.4.0"
+    "@azure/core-tracing" "^1.0.1"
+    "@azure/core-util" "^1.3.0"
     "@azure/logger" "^1.0.0"
+    http-proxy-agent "^5.0.0"
+    https-proxy-agent "^5.0.0"
     tslib "^2.2.0"
 
-"@azure/core-rest-pipeline@^1.10.2", "@azure/core-rest-pipeline@^1.5.0":
+"@azure/core-rest-pipeline@^1.5.0":
   version "1.12.0"
   resolved "https://registry.yarnpkg.com/@azure/core-rest-pipeline/-/core-rest-pipeline-1.12.0.tgz#a36dd361807494845522824532c076daa27c2786"
   integrity sha512-+MnSB0vGZjszSzr5AW8z93/9fkDu2RLtWmAN8gskURq7EW2sSwqy8jZa0V26rjuBVkwhdA3Hw8z3VWoeBUOw+A==
@@ -101,6 +105,13 @@
     https-proxy-agent "^5.0.0"
     tslib "^2.2.0"
 
+"@azure/core-sse@^2.0.0":
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/@azure/core-sse/-/core-sse-2.0.0.tgz#e94d29d0d8ade9f11c664a1695e60ef15720c087"
+  integrity sha512-PFmmaUwDmcmtt+q9NLzfhwC5qA2ACDn/5fuy8GVxI+YRv2qRvy1C0rZrwZLvOHe//G4cSRMz1X+CekY/Nlem2w==
+  dependencies:
+    tslib "^2.4.0"
+
 "@azure/core-tracing@^1.0.1":
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/@azure/core-tracing/-/core-tracing-1.0.1.tgz#352a38cbea438c4a83c86b314f48017d70ba9503"
@@ -108,7 +119,7 @@
   dependencies:
     tslib "^2.2.0"
 
-"@azure/core-util@^1.0.0", "@azure/core-util@^1.1.0", "@azure/core-util@^1.2.0", "@azure/core-util@^1.3.0":
+"@azure/core-util@^1.0.0", "@azure/core-util@^1.1.0", "@azure/core-util@^1.3.0":
   version "1.4.0"
   resolved "https://registry.yarnpkg.com/@azure/core-util/-/core-util-1.4.0.tgz#c120a56b3e48a9e4d20619a0b00268ae9de891c7"
   integrity sha512-eGAyJpm3skVQoLiRqm/xPa+SXi/NPDdSHMxbRAz2lSprd+Zs+qrpQGQQ2VQ3Nttu+nSZR4XoYQC71LbEI7jsig==
@@ -116,6 +127,14 @@
     "@azure/abort-controller" "^1.0.0"
     tslib "^2.2.0"
 
+"@azure/core-util@^1.4.0":
+  version "1.6.1"
+  resolved "https://registry.yarnpkg.com/@azure/core-util/-/core-util-1.6.1.tgz#fea221c4fa43c26543bccf799beb30c1c7878f5a"
+  integrity sha512-h5taHeySlsV9qxuK64KZxy4iln1BtMYlNt5jbuEFN3UFSAd1EwKg/Gjl5a6tZ/W8t6li3xPnutOx7zbDyXnPmQ==
+  dependencies:
+    "@azure/abort-controller" "^1.0.0"
+    tslib "^2.2.0"
+
 "@azure/logger@^1.0.0", "@azure/logger@^1.0.3":
   version "1.0.4"
   resolved "https://registry.yarnpkg.com/@azure/logger/-/logger-1.0.4.tgz#28bc6d0e5b3c38ef29296b32d35da4e483593fa1"
@@ -123,15 +142,16 @@
   dependencies:
     tslib "^2.2.0"
 
-"@azure/openai@^1.0.0-beta.3":
+"@azure/openai@1.0.0-beta.10":
-  version "1.0.0-beta.3"
+  version "1.0.0-beta.10"
-  resolved "https://registry.yarnpkg.com/@azure/openai/-/openai-1.0.0-beta.3.tgz#bf4f5ec0a5644b3a9ce4372620856a65e7721e24"
+  resolved "https://registry.yarnpkg.com/@azure/openai/-/openai-1.0.0-beta.10.tgz#13bcf5c5bc34dd27e33dc6aab5db3dc97dd4545b"
-  integrity sha512-gW4odbuy/X/W34SdvXomj/JzR09MyMHCY5Kd2ZxJkQo3IUGqJXz1rEv6QER7IAGgBFgNawE97K6UuJfMmoT0rw==
+  integrity sha512-6kixZSMOI5jk9TBwgXrVo5fKUPUudOXxjwCJvAGaQN6NT1Tp3IMrjGou+2iP9iX7GwND9lptxfvafHtK7RX/VA==
   dependencies:
-    "@azure-rest/core-client" "^1.1.3"
+    "@azure-rest/core-client" "^1.1.7"
     "@azure/core-auth" "^1.4.0"
-    "@azure/core-lro" "^2.5.3"
+    "@azure/core-rest-pipeline" "^1.13.0"
-    "@azure/core-rest-pipeline" "^1.10.2"
+    "@azure/core-sse" "^2.0.0"
+    "@azure/core-util" "^1.4.0"
     "@azure/logger" "^1.0.3"
     tslib "^2.4.0"