Add Anthropic agent support with new API and tool_calling (#1116)

* Add Anthropic agent support with new API and tool_calling

* patch useProviderHook to unset default models on provider change

frontend/src/hooks/useGetProvidersModels.js

@@ -47,8 +47,12 @@ export default function useGetProviderModels(provider = null) {
       if (
         PROVIDER_DEFAULT_MODELS.hasOwnProperty(provider) &&
         !groupedProviders.includes(provider)
-      )
+      ) {
         setDefaultModels(PROVIDER_DEFAULT_MODELS[provider]);
+      } else {
+        setDefaultModels([]);
+      }
+
       groupedProviders.includes(provider)
         ? setCustomModels(groupModels(models))
         : setCustomModels(models);

frontend/src/pages/WorkspaceSettings/AgentConfig/AgentLLMSelection/index.jsx

@@ -5,10 +5,7 @@ import { AVAILABLE_LLM_PROVIDERS } from "@/pages/GeneralSettings/LLMPreference";
 import { CaretUpDown, MagnifyingGlass, X } from "@phosphor-icons/react";
 import AgentModelSelection from "../AgentModelSelection";
 
-const ENABLED_PROVIDERS = [
+const ENABLED_PROVIDERS = ["openai", "anthropic"];
-  "openai",
-  // "anthropic"
-];
 
 const LLM_DEFAULT = {
   name: "Please make a selection",

server/utils/agents/aibitat/providers/anthropic.js

@@ -25,6 +25,90 @@ class AnthropicProvider extends Provider {
     this.model = model;
   }
 
+  // For Anthropic we will always need to ensure the message sequence is role,content
+  // as we can attach any data to message nodes and this keeps the message property
+  // sent to the API always in spec.
+  #sanitize(chats) {
+    const sanitized = [...chats];
+
+    // If the first message is not a USER, Anthropic will abort so keep shifting the
+    // message array until that is the case.
+    while (sanitized.length > 0 && sanitized[0].role !== "user")
+      sanitized.shift();
+
+    return sanitized.map((msg) => {
+      const { role, content } = msg;
+      return { role, content };
+    });
+  }
+
+  #normalizeChats(messages = []) {
+    if (!messages.length) return messages;
+    const normalized = [];
+
+    [...messages].forEach((msg, i) => {
+      if (msg.role !== "function") return normalized.push(msg);
+
+      // If the last message is a role "function" this is our special aibitat message node.
+      // and we need to remove it from the array of messages.
+      // Since Anthropic needs to have the tool call resolved, we look at the previous chat to "function"
+      // and go through its content "thought" from ~ln:143 and get the tool_call id so we can resolve
+      // this tool call properly.
+      const functionCompletion = msg;
+      const toolCallId = messages[i - 1]?.content?.find(
+        (msg) => msg.type === "tool_use"
+      )?.id;
+
+      // Append the Anthropic acceptable node to the message chain so function can resolve.
+      normalized.push({
+        role: "user",
+        content: [
+          {
+            type: "tool_result",
+            tool_use_id: toolCallId,
+            content: functionCompletion.content,
+          },
+        ],
+      });
+    });
+    return normalized;
+  }
+
+  // Anthropic handles system message as a property, so here we split the system message prompt
+  // from all the chats and then normalize them so they will be useable in case of tool_calls or general chat.
+  #parseSystemPrompt(messages = []) {
+    const chats = [];
+    let systemPrompt =
+      "You are a helpful ai assistant who can assist the user and use tools available to help answer the users prompts and questions.";
+    for (const msg of messages) {
+      if (msg.role === "system") {
+        systemPrompt = msg.content;
+        continue;
+      }
+      chats.push(msg);
+    }
+
+    return [systemPrompt, this.#normalizeChats(chats)];
+  }
+
+  // Anthropic does not use the regular schema for functions so here we need to ensure it is in there specific format
+  // so that the call can run correctly.
+  #formatFunctions(functions = []) {
+    return functions.map((func) => {
+      const { name, description, parameters, required } = func;
+      const { type, properties } = parameters;
+      return {
+        name,
+        description,
+        input_schema: {
+          type,
+          properties,
+          required,
+        },
+      };
+    });
+  }
+
   /**
    * Create a completion based on the received messages.
    *
@@ -32,89 +116,78 @@ class AnthropicProvider extends Provider {
    * @param functions
    * @returns The completion.
    */
-  async complete(messages, functions) {
+  async complete(messages, functions = null) {
-    // clone messages to avoid mutating the original array
-    const promptMessages = [...messages];
-
-    if (functions) {
-      const functionPrompt = this.getFunctionPrompt(functions);
-
-      // add function prompt after the first message
-      promptMessages.splice(1, 0, {
-        content: functionPrompt,
-        role: "system",
-      });
-    }
-
-    const prompt = promptMessages
-      .map((message) => {
-        const { content, role } = message;
-
-        switch (role) {
-          case "system":
-            return content
-              ? `${Anthropic.HUMAN_PROMPT} <admin>${content}</admin>`
-              : "";
-
-          case "function":
-          case "user":
-            return `${Anthropic.HUMAN_PROMPT} ${content}`;
-
-          case "assistant":
-            return `${Anthropic.AI_PROMPT} ${content}`;
-
-          default:
-            return content;
-        }
-      })
-      .filter(Boolean)
-      .join("\n")
-      .concat(` ${Anthropic.AI_PROMPT}`);
-
     try {
-      const response = await this.client.completions.create({
+      const [systemPrompt, chats] = this.#parseSystemPrompt(messages);
-        model: this.model,
+      const response = await this.client.messages.create(
-        max_tokens_to_sample: 3000,
+        {
-        stream: false,
+          model: this.model,
-        prompt,
+          max_tokens: 4096,
-      });
+          system: systemPrompt,
+          messages: this.#sanitize(chats),
+          stream: false,
+          ...(Array.isArray(functions) && functions?.length > 0
+            ? { tools: this.#formatFunctions(functions) }
+            : {}),
+        },
+        { headers: { "anthropic-beta": "tools-2024-04-04" } } // Required to we can use tools.
+      );
 
-      const result = response.completion.trim();
+      // We know that we need to call a tool. So we are about to recurse through completions/handleExecution
-      // TODO: get cost from response
+      // https://docs.anthropic.com/claude/docs/tool-use#how-tool-use-works
-      const cost = 0;
+      if (response.stop_reason === "tool_use") {
+        // Get the tool call explicitly.
+        const toolCall = response.content.find(
+          (res) => res.type === "tool_use"
+        );
 
-      // Handle function calls if the model returns a function call
+        // Here we need the chain of thought the model may or may not have generated alongside the call.
-      if (result.includes("function_name") && functions) {
+        // this needs to be in a very specific format so we always ensure there is a 2-item content array
-        let functionCall;
+        // so that we can ensure the tool_call content is correct. For anthropic all text items must not
-        try {
+        // be empty, but the api will still return empty text so we need to make 100% sure text is not empty
-          functionCall = JSON.parse(result);
+        // or the tool call will fail.
-        } catch (error) {
+        // wtf.
-          // call the complete function again in case it gets a json error
+        let thought = response.content.find((res) => res.type === "text");
-          return await this.complete(
+        thought =
-            [
+          thought?.content?.length > 0
-              ...messages,
+            ? {
-              {
+                role: thought.role,
-                role: "function",
+                content: [
-                content: `You gave me this function call: ${result} but I couldn't parse it.
+                  { type: "text", text: thought.content },
-                ${error?.message}
+                  { ...toolCall },
-                
+                ],
-                Please try again.`,
+              }
-              },
+            : {
-            ],
+                role: "assistant",
-            functions
+                content: [
-          );
+                  {
-        }
+                    type: "text",
+                    text: `Okay, im going to use ${toolCall.name} to help me.`,
+                  },
+                  { ...toolCall },
+                ],
+              };
 
+        // Modify messages forcefully by adding system thought so that tool_use/tool_result
+        // messaging works with Anthropic's disastrous tool calling API.
+        messages.push(thought);
+
+        const functionArgs = toolCall.input;
         return {
           result: null,
-          functionCall,
+          functionCall: {
-          cost,
+            name: toolCall.name,
+            arguments: functionArgs,
+          },
+          cost: 0,
         };
       }
 
+      const completion = response.content.find((msg) => msg.type === "text");
       return {
-        result,
+        result:
-        cost,
+          completion?.text ?? "I could not generate a response from this.",
+        cost: 0,
       };
     } catch (error) {
       // If invalid Auth error we need to abort because no amount of waiting
@@ -132,24 +205,6 @@ class AnthropicProvider extends Provider {
       throw error;
     }
   }
-
-  getFunctionPrompt(functions = []) {
-    const functionPrompt = `<functions>You have been trained to directly call a Javascript function passing a JSON Schema parameter as a response to this chat. This function will return a string that you can use to keep chatting.
-  
-  Here is a list of functions available to you:
-  ${JSON.stringify(functions, null, 2)}
-  
-  When calling any of those function in order to complete your task, respond only this JSON format. Do not include any other information or any other stuff.
-  
-  Function call format:
-  {
-     function_name: "givenfunctionname",
-     parameters: {}
-  }
-  </functions>`;
-
-    return functionPrompt;
-  }
 }
 
 module.exports = AnthropicProvider;

server/utils/agents/index.js

@@ -50,6 +50,7 @@ class AgentHandler {
             from: USER_AGENT.name,
             to: WORKSPACE_AGENT.name,
             content: chatLog.prompt,
+            state: "success",
           },
           {
             from: WORKSPACE_AGENT.name,