Merge conflicts

.github/workflows/dev-build.yaml

@@ -6,7 +6,7 @@ concurrency:
 
 on:
   push:
-    branches: ['1915-docker-perms'] # master branch only. Do not modify.
+    branches: ['-dev'] # put your current branch to create a build. Core team only.
     paths-ignore:
       - '**.md'
       - 'cloud-deployments/*'

.vscode/settings.json

@@ -31,6 +31,7 @@
     "Mintplex",
     "moderations",
     "numpages",
+    "odbc",
     "Ollama",
     "Oobabooga",
     "openai",

collector/extensions/index.js

@@ -1,5 +1,6 @@
 const { setDataSigner } = require("../middleware/setDataSigner");
 const { verifyPayloadIntegrity } = require("../middleware/verifyIntegrity");
+const { resolveRepoLoader, resolveRepoLoaderFunction } = require("../utils/extensions/RepoLoader");
 const { reqBody } = require("../utils/http");
 const { validURL } = require("../utils/url");
 const RESYNC_METHODS = require("./resync");
@@ -28,15 +29,16 @@ function extensions(app) {
   )
 
   app.post(
-    "/ext/github-repo",
+    "/ext/:repo_platform-repo",
     [verifyPayloadIntegrity, setDataSigner],
     async function (request, response) {
       try {
-        const { loadGithubRepo } = require("../utils/extensions/GithubRepo");
+        const loadRepo = resolveRepoLoaderFunction(request.params.repo_platform);
-        const { success, reason, data } = await loadGithubRepo(
+        const { success, reason, data } = await loadRepo(
           reqBody(request),
           response,
         );
+        console.log({ success, reason, data })
         response.status(200).json({
           success,
           reason,
@@ -56,12 +58,12 @@ function extensions(app) {
 
   // gets all branches for a specific repo
   app.post(
-    "/ext/github-repo/branches",
+    "/ext/:repo_platform-repo/branches",
     [verifyPayloadIntegrity],
     async function (request, response) {
       try {
-        const GithubRepoLoader = require("../utils/extensions/GithubRepo/RepoLoader");
+        const RepoLoader = resolveRepoLoader(request.params.repo_platform);
-        const allBranches = await new GithubRepoLoader(
+        const allBranches = await new RepoLoader(
           reqBody(request)
         ).getRepoBranches();
         response.status(200).json({

collector/extensions/resync/index.js

@@ -86,7 +86,7 @@ async function resyncGithub({ chunkSource }, response) {
     // Github file data is `payload` encrypted (might contain PAT). So we need to expand its
     // encrypted payload back into query params so we can reFetch the page with same access token/params.
     const source = response.locals.encryptionWorker.expandPayload(chunkSource);
-    const { fetchGithubFile } = require("../../utils/extensions/GithubRepo");
+    const { fetchGithubFile } = require("../../utils/extensions/RepoLoader/GithubRepo");
     const { success, reason, content } = await fetchGithubFile({
       repoUrl: `https:${source.pathname}`, // need to add back the real protocol
       branch: source.searchParams.get('branch'),

collector/package.json

@@ -32,6 +32,7 @@
     "mammoth": "^1.6.0",
     "mbox-parser": "^1.0.1",
     "mime": "^3.0.0",
+    "minimatch": "5.1.0",
     "moment": "^2.29.4",
     "multer": "^1.4.5-lts.1",
     "node-html-parser": "^6.1.13",

collector/utils/extensions/RepoLoader/GithubRepo/RepoLoader/index.js

@@ -1,4 +1,21 @@
-class RepoLoader {
+/**
+ * @typedef {Object} RepoLoaderArgs
+ * @property {string} repo - The GitHub repository URL.
+ * @property {string} [branch] - The branch to load from (optional).
+ * @property {string} [accessToken] - GitHub access token for authentication (optional).
+ * @property {string[]} [ignorePaths] - Array of paths to ignore when loading (optional).
+ */
+
+/**
+ * @class
+ * @classdesc Loads and manages GitHub repository content.
+ */
+class GitHubRepoLoader {
+  /**
+   * Creates an instance of RepoLoader.
+   * @param {RepoLoaderArgs} [args] - The configuration options.
+   * @returns {GitHubRepoLoader}
+   */
   constructor(args = {}) {
     this.ready = false;
     this.repo = args?.repo;
@@ -67,6 +84,10 @@ class RepoLoader {
     return;
   }
 
+  /**
+   * Initializes the RepoLoader instance.
+   * @returns {Promise<RepoLoader>} The initialized RepoLoader instance.
+   */
   async init() {
     if (!this.#validGithubUrl()) return;
     await this.#validBranch();
@@ -75,6 +96,11 @@ class RepoLoader {
     return this;
   }
 
+  /**
+   * Recursively loads the repository content.
+   * @returns {Promise<Array<Object>>} An array of loaded documents.
+   * @throws {Error} If the RepoLoader is not in a ready state.
+   */
   async recursiveLoader() {
     if (!this.ready) throw new Error("[Github Loader]: not in ready state!");
     const {
@@ -109,7 +135,10 @@ class RepoLoader {
     }, []);
   }
 
-  // Get all branches for a given repo.
+  /**
+   * Retrieves all branches for the repository.
+   * @returns {Promise<string[]>} An array of branch names.
+   */
   async getRepoBranches() {
     if (!this.#validGithubUrl() || !this.author || !this.project) return [];
     await this.#validateAccessToken(); // Ensure API access token is valid for pre-flight
@@ -151,6 +180,11 @@ class RepoLoader {
     return this.#branchPrefSort(this.branches);
   }
 
+  /**
+   * Fetches the content of a single file from the repository.
+   * @param {string} sourceFilePath - The path to the file in the repository.
+   * @returns {Promise<string|null>} The content of the file, or null if fetching fails.
+   */
   async fetchSingleFile(sourceFilePath) {
     try {
       return fetch(
@@ -182,4 +216,4 @@ class RepoLoader {
   }
 }
 
-module.exports = RepoLoader;
+module.exports = GitHubRepoLoader;

collector/utils/extensions/RepoLoader/GithubRepo/index.js

@@ -3,8 +3,8 @@ const fs = require("fs");
 const path = require("path");
 const { default: slugify } = require("slugify");
 const { v4 } = require("uuid");
-const { writeToServerDocuments, documentsFolder } = require("../../files");
+const { writeToServerDocuments, documentsFolder } = require("../../../files");
-const { tokenizeString } = require("../../tokenizer");
+const { tokenizeString } = require("../../../tokenizer");
 
 /**
  * Load in a Github Repo recursively or just the top level if no PAT is provided
@@ -37,6 +37,7 @@ async function loadGithubRepo(args, response) {
   const outFolder = slugify(
     `${repo.author}-${repo.project}-${repo.branch}-${v4().slice(0, 4)}`
   ).toLowerCase();
+
   const outFolderPath = path.resolve(documentsFolder, outFolder);
   if (!fs.existsSync(outFolderPath))
     fs.mkdirSync(outFolderPath, { recursive: true });

collector/utils/extensions/RepoLoader/GitlabRepo/RepoLoader/index.js

@@ -0,0 +1,289 @@
+const minimatch = require("minimatch");
+
+/**
+ * @typedef {Object} RepoLoaderArgs
+ * @property {string} repo - The GitLab repository URL.
+ * @property {string} [branch] - The branch to load from (optional).
+ * @property {string} [accessToken] - GitLab access token for authentication (optional).
+ * @property {string[]} [ignorePaths] - Array of paths to ignore when loading (optional).
+ */
+
+/**
+ * @typedef {Object} FileTreeObject
+ * @property {string} id - The file object ID.
+ * @property {string} name - name of file.
+ * @property {('blob'|'tree')} type - type of file object.
+ * @property {string} path - path + name of file.
+ * @property {string} mode - Linux permission code.
+ */
+
+/**
+ * @class
+ * @classdesc Loads and manages GitLab repository content.
+ */
+class GitLabRepoLoader {
+  /**
+   * Creates an instance of RepoLoader.
+   * @param {RepoLoaderArgs} [args] - The configuration options.
+   * @returns {GitLabRepoLoader}
+   */
+  constructor(args = {}) {
+    this.ready = false;
+    this.repo = args?.repo;
+    this.branch = args?.branch;
+    this.accessToken = args?.accessToken || null;
+    this.ignorePaths = args?.ignorePaths || [];
+
+    this.projectId = null;
+    this.apiBase = "https://gitlab.com";
+    this.author = null;
+    this.project = null;
+    this.branches = [];
+  }
+
+  #validGitlabUrl() {
+    const UrlPattern = require("url-pattern");
+    const validPatterns = [
+      new UrlPattern("https\\://gitlab.com/(:projectId(*))", {
+        segmentValueCharset: "a-zA-Z0-9-._~%/+",
+      }),
+      // This should even match the regular hosted URL, but we may want to know
+      // if this was a hosted GitLab (above) or a self-hosted (below) instance
+      // since the API interface could be different.
+      new UrlPattern(
+        "(:protocol(http|https))\\://(:hostname*)/(:projectId(*))",
+        {
+          segmentValueCharset: "a-zA-Z0-9-._~%/+",
+        }
+      ),
+    ];
+
+    let match = null;
+    for (const pattern of validPatterns) {
+      if (match !== null) continue;
+      match = pattern.match(this.repo);
+    }
+    if (!match) return false;
+    const [author, project] = match.projectId.split("/");
+
+    this.projectId = encodeURIComponent(match.projectId);
+    this.apiBase = new URL(this.repo).origin;
+    this.author = author;
+    this.project = project;
+    return true;
+  }
+
+  async #validBranch() {
+    await this.getRepoBranches();
+    if (!!this.branch && this.branches.includes(this.branch)) return;
+
+    console.log(
+      "[Gitlab Loader]: Branch not set! Auto-assigning to a default branch."
+    );
+    this.branch = this.branches.includes("main") ? "main" : "master";
+    console.log(`[Gitlab Loader]: Branch auto-assigned to ${this.branch}.`);
+    return;
+  }
+
+  async #validateAccessToken() {
+    if (!this.accessToken) return;
+    try {
+      await fetch(`${this.apiBase}/api/v4/user`, {
+        method: "GET",
+        headers: this.accessToken ? { "PRIVATE-TOKEN": this.accessToken } : {},
+      }).then((res) => res.ok);
+    } catch (e) {
+      console.error(
+        "Invalid Gitlab Access Token provided! Access token will not be used",
+        e.message
+      );
+      this.accessToken = null;
+    }
+  }
+
+  /**
+   * Initializes the RepoLoader instance.
+   * @returns {Promise<RepoLoader>} The initialized RepoLoader instance.
+   */
+  async init() {
+    if (!this.#validGitlabUrl()) return;
+    await this.#validBranch();
+    await this.#validateAccessToken();
+    this.ready = true;
+    return this;
+  }
+
+  /**
+   * Recursively loads the repository content.
+   * @returns {Promise<Array<Object>>} An array of loaded documents.
+   * @throws {Error} If the RepoLoader is not in a ready state.
+   */
+  async recursiveLoader() {
+    if (!this.ready) throw new Error("[Gitlab Loader]: not in ready state!");
+
+    if (this.accessToken)
+      console.log(
+        `[Gitlab Loader]: Access token set! Recursive loading enabled!`
+      );
+
+    const files = await this.fetchFilesRecursive();
+    const docs = [];
+
+    for (const file of files) {
+      if (this.ignorePaths.some((path) => file.path.includes(path))) continue;
+
+      const content = await this.fetchSingleFileContents(file.path);
+      if (content) {
+        docs.push({
+          pageContent: content,
+          metadata: { source: file.path },
+        });
+      }
+    }
+
+    return docs;
+  }
+
+  #branchPrefSort(branches = []) {
+    const preferredSort = ["main", "master"];
+    return branches.reduce((acc, branch) => {
+      if (preferredSort.includes(branch)) return [branch, ...acc];
+      return [...acc, branch];
+    }, []);
+  }
+
+  /**
+   * Retrieves all branches for the repository.
+   * @returns {Promise<string[]>} An array of branch names.
+   */
+  async getRepoBranches() {
+    if (!this.#validGitlabUrl() || !this.projectId) return [];
+    await this.#validateAccessToken();
+
+    try {
+      this.branches = await fetch(
+        `${this.apiBase}/api/v4/projects/${this.projectId}/repository/branches`,
+        {
+          method: "GET",
+          headers: {
+            Accepts: "application/json",
+            ...(this.accessToken ? { "PRIVATE-TOKEN": this.accessToken } : {}),
+          },
+        }
+      )
+        .then((res) => res.json())
+        .then((branches) => {
+          return branches.map((b) => b.name);
+        })
+        .catch((e) => {
+          console.error(e);
+          return [];
+        });
+
+      return this.#branchPrefSort(this.branches);
+    } catch (err) {
+      console.log(`RepoLoader.getRepoBranches`, err);
+      this.branches = [];
+      return [];
+    }
+  }
+
+  /**
+   * Returns list of all file objects from tree API for GitLab
+   * @returns {Promise<FileTreeObject[]>}
+   */
+  async fetchFilesRecursive() {
+    const files = [];
+    let perPage = 100;
+    let fetching = true;
+    let page = 1;
+
+    while (fetching) {
+      try {
+        const params = new URLSearchParams({
+          ref: this.branch,
+          recursive: true,
+          per_page: perPage,
+          page,
+        });
+        const queryUrl = `${this.apiBase}/api/v4/projects/${
+          this.projectId
+        }/repository/tree?${params.toString()}`;
+        const response = await fetch(queryUrl, {
+          method: "GET",
+          headers: this.accessToken
+            ? { "PRIVATE-TOKEN": this.accessToken }
+            : {},
+        });
+        const totalPages = Number(response.headers.get("x-total-pages"));
+        const nextPage = Number(response.headers.get("x-next-page"));
+        const data = await response.json();
+
+        /** @type {FileTreeObject[]} */
+        const objects = Array.isArray(data)
+          ? data.filter((item) => item.type === "blob")
+          : []; // only get files, not paths or submodules
+        if (objects.length === 0) {
+          fetching = false;
+          break;
+        }
+
+        // Apply ignore path rules to found objects. If any rules match it is an invalid file path.
+        console.log(
+          `Found ${objects.length} blobs from repo from pg ${page}/${totalPages}`
+        );
+        for (const file of objects) {
+          const isIgnored = this.ignorePaths.some((ignorePattern) =>
+            minimatch(file.path, ignorePattern, { matchBase: true })
+          );
+          if (!isIgnored) files.push(file);
+        }
+
+        if (page === totalPages) {
+          fetching = false;
+          break;
+        }
+
+        page = Number(nextPage);
+      } catch (e) {
+        console.error(`RepoLoader.getRepositoryTree`, e);
+        fetching = false;
+        break;
+      }
+    }
+    return files;
+  }
+
+  /**
+   * Fetches the content of a single file from the repository.
+   * @param {string} sourceFilePath - The path to the file in the repository.
+   * @returns {Promise<string|null>} The content of the file, or null if fetching fails.
+   */
+  async fetchSingleFileContents(sourceFilePath) {
+    try {
+      const data = await fetch(
+        `${this.apiBase}/api/v4/projects/${
+          this.projectId
+        }/repository/files/${encodeURIComponent(sourceFilePath)}/raw?ref=${
+          this.branch
+        }`,
+        {
+          method: "GET",
+          headers: this.accessToken
+            ? { "PRIVATE-TOKEN": this.accessToken }
+            : {},
+        }
+      ).then((res) => {
+        if (res.ok) return res.text();
+        throw new Error(`Failed to fetch single file ${sourceFilePath}`);
+      });
+
+      return data;
+    } catch (e) {
+      console.error(`RepoLoader.fetchSingleFileContents`, e);
+      return null;
+    }
+  }
+}
+
+module.exports = GitLabRepoLoader;

collector/utils/extensions/RepoLoader/GitlabRepo/index.js

@@ -0,0 +1,138 @@
+const RepoLoader = require("./RepoLoader");
+const fs = require("fs");
+const path = require("path");
+const { default: slugify } = require("slugify");
+const { v4 } = require("uuid");
+const { writeToServerDocuments, documentsFolder } = require("../../../files");
+const { tokenizeString } = require("../../../tokenizer");
+
+/**
+ * Load in a Gitlab Repo recursively or just the top level if no PAT is provided
+ * @param {object} args - forwarded request body params
+ * @param {import("../../../middleware/setDataSigner").ResponseWithSigner} response - Express response object with encryptionWorker
+ * @returns
+ */
+async function loadGitlabRepo(args, response) {
+  const repo = new RepoLoader(args);
+  await repo.init();
+
+  if (!repo.ready)
+    return {
+      success: false,
+      reason: "Could not prepare Gitlab repo for loading! Check URL",
+    };
+
+  console.log(
+    `-- Working GitLab ${repo.author}/${repo.project}:${repo.branch} --`
+  );
+  const docs = await repo.recursiveLoader();
+  if (!docs.length) {
+    return {
+      success: false,
+      reason: "No files were found for those settings.",
+    };
+  }
+
+  console.log(`[GitLab Loader]: Found ${docs.length} source files. Saving...`);
+  const outFolder = slugify(
+    `${repo.author}-${repo.project}-${repo.branch}-${v4().slice(0, 4)}`
+  ).toLowerCase();
+
+  const outFolderPath = path.resolve(documentsFolder, outFolder);
+  if (!fs.existsSync(outFolderPath))
+    fs.mkdirSync(outFolderPath, { recursive: true });
+
+  for (const doc of docs) {
+    if (!doc.pageContent) continue;
+    const data = {
+      id: v4(),
+      url: "gitlab://" + doc.metadata.source,
+      title: doc.metadata.source,
+      docAuthor: repo.author,
+      description: "No description found.",
+      docSource: doc.metadata.source,
+      chunkSource: generateChunkSource(
+        repo,
+        doc,
+        response.locals.encryptionWorker
+      ),
+      published: new Date().toLocaleString(),
+      wordCount: doc.pageContent.split(" ").length,
+      pageContent: doc.pageContent,
+      token_count_estimate: tokenizeString(doc.pageContent).length,
+    };
+    console.log(
+      `[GitLab Loader]: Saving ${doc.metadata.source} to ${outFolder}`
+    );
+    writeToServerDocuments(
+      data,
+      `${slugify(doc.metadata.source)}-${data.id}`,
+      outFolderPath
+    );
+  }
+
+  return {
+    success: true,
+    reason: null,
+    data: {
+      author: repo.author,
+      repo: repo.project,
+      projectId: repo.projectId,
+      branch: repo.branch,
+      files: docs.length,
+      destination: outFolder,
+    },
+  };
+}
+
+async function fetchGitlabFile({
+  repoUrl,
+  branch,
+  accessToken = null,
+  sourceFilePath,
+}) {
+  const repo = new RepoLoader({
+    repo: repoUrl,
+    branch,
+    accessToken,
+  });
+  await repo.init();
+
+  if (!repo.ready)
+    return {
+      success: false,
+      content: null,
+      reason: "Could not prepare GitLab repo for loading! Check URL or PAT.",
+    };
+  console.log(
+    `-- Working GitLab ${repo.author}/${repo.project}:${repo.branch} file:${sourceFilePath} --`
+  );
+  const fileContent = await repo.fetchSingleFile(sourceFilePath);
+  if (!fileContent) {
+    return {
+      success: false,
+      reason: "Target file returned a null content response.",
+      content: null,
+    };
+  }
+
+  return {
+    success: true,
+    reason: null,
+    content: fileContent,
+  };
+}
+
+function generateChunkSource(repo, doc, encryptionWorker) {
+  const payload = {
+    projectId: decodeURIComponent(repo.projectId),
+    branch: repo.branch,
+    path: doc.metadata.source,
+    pat: !!repo.accessToken ? repo.accessToken : null,
+  };
+  return `gitlab://${repo.repo}?payload=${encryptionWorker.encrypt(
+    JSON.stringify(payload)
+  )}`;
+}
+
+module.exports = { loadGitlabRepo, fetchGitlabFile };

collector/utils/extensions/RepoLoader/index.js

@@ -0,0 +1,41 @@
+/**
+ * Dynamically load the correct repository loader from a specific platform
+ * by default will return Github.
+ * @param {('github'|'gitlab')} platform
+ * @returns {import("./GithubRepo/RepoLoader")|import("./GitlabRepo/RepoLoader")} the repo loader class for provider
+ */
+function resolveRepoLoader(platform = "github") {
+  switch (platform) {
+    case "github":
+      console.log(`Loading GitHub RepoLoader...`);
+      return require("./GithubRepo/RepoLoader");
+    case "gitlab":
+      console.log(`Loading GitLab RepoLoader...`);
+      return require("./GitlabRepo/RepoLoader");
+    default:
+      console.log(`Loading GitHub RepoLoader...`);
+      return require("./GithubRepo/RepoLoader");
+  }
+}
+
+/**
+ * Dynamically load the correct repository loader function from a specific platform
+ * by default will return Github.
+ * @param {('github'|'gitlab')} platform
+ * @returns {import("./GithubRepo")['fetchGithubFile'] | import("./GitlabRepo")['fetchGitlabFile']} the repo loader class for provider
+ */
+function resolveRepoLoaderFunction(platform = "github") {
+  switch (platform) {
+    case "github":
+      console.log(`Loading GitHub loader function...`);
+      return require("./GithubRepo").loadGithubRepo;
+    case "gitlab":
+      console.log(`Loading GitLab loader function...`);
+      return require("./GitlabRepo").loadGitlabRepo;
+    default:
+      console.log(`Loading GitHub loader function...`);
+      return require("./GithubRepo").loadGithubRepo;
+  }
+}
+
+module.exports = { resolveRepoLoader, resolveRepoLoaderFunction };

collector/yarn.lock

@@ -581,6 +581,13 @@ brace-expansion@^1.1.7:
     balanced-match "^1.0.0"
     concat-map "0.0.1"
 
+brace-expansion@^2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-2.0.1.tgz#1edc459e0f0c548486ecf9fc99f2221364b9a0ae"
+  integrity sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==
+  dependencies:
+    balanced-match "^1.0.0"
+
 braces@~3.0.2:
   version "3.0.2"
   resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.2.tgz#3454e1a462ee8d599e236df336cd9ea4f8afe107"
@@ -2226,6 +2233,13 @@ mimic-response@^3.1.0:
   resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-3.1.0.tgz#2d1d59af9c1b129815accc2c46a022a5ce1fa3c9"
   integrity sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==
 
+minimatch@5.1.0:
+  version "5.1.0"
+  resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-5.1.0.tgz#1717b464f4971b144f6aabe8f2d0b8e4511e09c7"
+  integrity sha512-9TPBGGak4nHfGZsPBohm9AWg6NoT7QTCehS3BIJABslyZbzxfV78QM2Y6+i741OPZIafFAaiiEMh5OyIrJPgtg==
+  dependencies:
+    brace-expansion "^2.0.1"
+
 minimatch@^3.1.1, minimatch@^3.1.2:
   version "3.1.2"
   resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b"

frontend/src/components/DataConnectorOption/media/gitlab.svg

@@ -0,0 +1,7 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 380 380">
+  <rect width="380" height="380" fill="#FFFFFF"/>
+  <path fill="#e24329" d="M282.83,170.73l-.27-.69-26.14-68.22a6.81,6.81,0,0,0-2.69-3.24,7,7,0,0,0-8,.43,7,7,0,0,0-2.32,3.52l-17.65,54H154.29l-17.65-54A6.86,6.86,0,0,0,134.32,99a7,7,0,0,0-8-.43,6.87,6.87,0,0,0-2.69,3.24L97.44,170l-.26.69a48.54,48.54,0,0,0,16.1,56.1l.09.07.24.17,39.82,29.82,19.7,14.91,12,9.06a8.07,8.07,0,0,0,9.76,0l12-9.06,19.7-14.91,40.06-30,.1-.08A48.56,48.56,0,0,0,282.83,170.73Z"/>
+  <path fill="#fc6d26" d="M282.83,170.73l-.27-.69a88.3,88.3,0,0,0-35.15,15.8L190,229.25c19.55,14.79,36.57,27.64,36.57,27.64l40.06-30,.1-.08A48.56,48.56,0,0,0,282.83,170.73Z"/>
+  <path fill="#fca326" d="M153.43,256.89l19.7,14.91,12,9.06a8.07,8.07,0,0,0,9.76,0l12-9.06,19.7-14.91S209.55,244,190,229.25C170.45,244,153.43,256.89,153.43,256.89Z"/>
+  <path fill="#fc6d26" d="M132.58,185.84A88.19,88.19,0,0,0,97.44,170l-.26.69a48.54,48.54,0,0,0,16.1,56.1l.09.07.24.17,39.82,29.82s17-12.85,36.57-27.64Z"/>
+</svg>

frontend/src/components/DataConnectorOption/media/index.js

@@ -1,10 +1,12 @@
 import Github from "./github.svg";
+import Gitlab from "./gitlab.svg";
 import YouTube from "./youtube.svg";
 import Link from "./link.svg";
 import Confluence from "./confluence.jpeg";
 
 const ConnectorImages = {
   github: Github,
+  gitlab: Gitlab,
   youtube: YouTube,
   websiteDepth: Link,
   confluence: Confluence,

frontend/src/components/Modals/ManageWorkspace/DataConnectors/Connectors/Gitlab/index.jsx

@@ -0,0 +1,310 @@
+import React, { useEffect, useState } from "react";
+import System from "@/models/system";
+import showToast from "@/utils/toast";
+import pluralize from "pluralize";
+import { TagsInput } from "react-tag-input-component";
+import { Info, Warning } from "@phosphor-icons/react";
+import { Tooltip } from "react-tooltip";
+
+const DEFAULT_BRANCHES = ["main", "master"];
+export default function GitlabOptions() {
+  const [loading, setLoading] = useState(false);
+  const [repo, setRepo] = useState(null);
+  const [accessToken, setAccessToken] = useState(null);
+  const [ignores, setIgnores] = useState([]);
+  const [settings, setSettings] = useState({
+    repo: null,
+    accessToken: null,
+  });
+
+  const handleSubmit = async (e) => {
+    e.preventDefault();
+    const form = new FormData(e.target);
+
+    try {
+      setLoading(true);
+      showToast(
+        `Fetching all files for repo ${repo} - this may take a while.`,
+        "info",
+        { clear: true, autoClose: false }
+      );
+
+      const { data, error } = await System.dataConnectors.gitlab.collect({
+        repo: form.get("repo"),
+        accessToken: form.get("accessToken"),
+        branch: form.get("branch"),
+        ignorePaths: ignores,
+      });
+
+      if (!!error) {
+        showToast(error, "error", { clear: true });
+        setLoading(false);
+        return;
+      }
+
+      showToast(
+        `${data.files} ${pluralize("file", data.files)} collected from ${
+          data.author
+        }/${data.repo}:${data.branch}. Output folder is ${data.destination}.`,
+        "success",
+        { clear: true }
+      );
+      e.target.reset();
+      setLoading(false);
+      return;
+    } catch (e) {
+      console.error(e);
+      showToast(e.message, "error", { clear: true });
+      setLoading(false);
+    }
+  };
+
+  return (
+    <div className="flex w-full">
+      <div className="flex flex-col w-full px-1 md:pb-6 pb-16">
+        <form className="w-full" onSubmit={handleSubmit}>
+          <div className="w-full flex flex-col py-2">
+            <div className="w-full flex flex-col gap-4">
+              <div className="flex flex-col pr-10">
+                <div className="flex flex-col gap-y-1 mb-4">
+                  <label className="text-white text-sm font-bold">
+                    GitLab Repo URL
+                  </label>
+                  <p className="text-xs font-normal text-white/50">
+                    URL of the GitLab repo you wish to collect.
+                  </p>
+                </div>
+                <input
+                  type="url"
+                  name="repo"
+                  className="border-none bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
+                  placeholder="https://gitlab.com/gitlab-org/gitlab"
+                  required={true}
+                  autoComplete="off"
+                  onChange={(e) => setRepo(e.target.value)}
+                  onBlur={() => setSettings({ ...settings, repo })}
+                  spellCheck={false}
+                  rows={2}
+                />
+              </div>
+              <div className="flex flex-col pr-10">
+                <div className="flex flex-col gap-y-1 mb-4">
+                  <label className="text-white font-bold text-sm flex gap-x-2 items-center">
+                    <p className="font-bold text-white">GitLab Access Token</p>{" "}
+                    <p className="text-xs text-white/50 font-light flex items-center">
+                      optional
+                      <PATTooltip accessToken={accessToken} />
+                    </p>
+                  </label>
+                  <p className="text-xs font-normal text-white/50">
+                    Access Token to prevent rate limiting.
+                  </p>
+                </div>
+                <input
+                  type="text"
+                  name="accessToken"
+                  className="border-none bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:border-white block w-full p-2.5"
+                  placeholder="glpat-XXXXXXXXXXXXXXXXXXXX"
+                  required={false}
+                  autoComplete="off"
+                  spellCheck={false}
+                  onChange={(e) => setAccessToken(e.target.value)}
+                  onBlur={() => setSettings({ ...settings, accessToken })}
+                />
+              </div>
+              <GitLabBranchSelection
+                repo={settings.repo}
+                accessToken={settings.accessToken}
+              />
+            </div>
+
+            <div className="flex flex-col w-full py-4 pr-10">
+              <div className="flex flex-col gap-y-1 mb-4">
+                <label className="text-white text-sm flex gap-x-2 items-center">
+                  <p className="text-white text-sm font-bold">File Ignores</p>
+                </label>
+                <p className="text-xs font-normal text-white/50">
+                  List in .gitignore format to ignore specific files during
+                  collection. Press enter after each entry you want to save.
+                </p>
+              </div>
+              <TagsInput
+                value={ignores}
+                onChange={setIgnores}
+                name="ignores"
+                placeholder="!*.js, images/*, .DS_Store, bin/*"
+                classNames={{
+                  tag: "bg-blue-300/10 text-zinc-800",
+                  input:
+                    "flex bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:border-white",
+                }}
+              />
+            </div>
+          </div>
+
+          <div className="flex flex-col gap-y-2 w-full pr-10">
+            <PATAlert accessToken={accessToken} />
+            <button
+              type="submit"
+              disabled={loading}
+              className="mt-2 w-full justify-center border border-slate-200 px-4 py-2 rounded-lg text-dark-text text-sm font-bold items-center flex gap-x-2 bg-slate-200 hover:bg-slate-300 hover:text-slate-800 disabled:bg-slate-300 disabled:cursor-not-allowed"
+            >
+              {loading ? "Collecting files..." : "Submit"}
+            </button>
+            {loading && (
+              <p className="text-xs text-white/50">
+                Once complete, all files will be available for embedding into
+                workspaces in the document picker.
+              </p>
+            )}
+          </div>
+        </form>
+      </div>
+    </div>
+  );
+}
+
+function GitLabBranchSelection({ repo, accessToken }) {
+  const [allBranches, setAllBranches] = useState(DEFAULT_BRANCHES);
+  const [loading, setLoading] = useState(true);
+
+  useEffect(() => {
+    async function fetchAllBranches() {
+      if (!repo) {
+        setAllBranches(DEFAULT_BRANCHES);
+        setLoading(false);
+        return;
+      }
+
+      setLoading(true);
+      const { branches } = await System.dataConnectors.gitlab.branches({
+        repo,
+        accessToken,
+      });
+      setAllBranches(branches.length > 0 ? branches : DEFAULT_BRANCHES);
+      setLoading(false);
+    }
+    fetchAllBranches();
+  }, [repo, accessToken]);
+
+  if (loading) {
+    return (
+      <div className="flex flex-col w-60">
+        <div className="flex flex-col gap-y-1 mb-4">
+          <label className="text-white text-sm font-bold">Branch</label>
+          <p className="text-xs font-normal text-white/50">
+            Branch you wish to collect files from.
+          </p>
+        </div>
+        <select
+          name="branch"
+          required={true}
+          className="bg-zinc-900 border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
+        >
+          <option disabled={true} selected={true}>
+            -- loading available branches --
+          </option>
+        </select>
+      </div>
+    );
+  }
+
+  return (
+    <div className="flex flex-col w-60">
+      <div className="flex flex-col gap-y-1 mb-4">
+        <label className="text-white text-sm font-bold">Branch</label>
+        <p className="text-xs font-normal text-white/50">
+          Branch you wish to collect files from.
+        </p>
+      </div>
+      <select
+        name="branch"
+        required={true}
+        className="bg-zinc-900 border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
+      >
+        {allBranches.map((branch) => {
+          return (
+            <option key={branch} value={branch}>
+              {branch}
+            </option>
+          );
+        })}
+      </select>
+    </div>
+  );
+}
+
+function PATAlert({ accessToken }) {
+  if (!!accessToken) return null;
+  return (
+    <div className="flex flex-col md:flex-row md:items-center gap-x-2 text-white mb-4 bg-blue-800/30 w-fit rounded-lg px-4 py-2">
+      <div className="gap-x-2 flex items-center">
+        <Info className="shrink-0" size={25} />
+        <p className="text-sm">
+          Without filling out the <b>GitLab Access Token</b> this data connector
+          will only be able to collect the <b>top-level</b> files of the repo
+          due to GitLab's public API rate-limits.
+          <br />
+          <br />
+          <a
+            href="https://gitlab.com/-/profile/personal_access_tokens"
+            rel="noreferrer"
+            target="_blank"
+            className="underline"
+            onClick={(e) => e.stopPropagation()}
+          >
+            {" "}
+            Get a free Personal Access Token with a GitLab account here.
+          </a>
+        </p>
+      </div>
+    </div>
+  );
+}
+
+function PATTooltip({ accessToken }) {
+  if (!!accessToken) return null;
+  return (
+    <>
+      {!accessToken && (
+        <Warning
+          size={14}
+          className="ml-1 text-orange-500 cursor-pointer"
+          data-tooltip-id="access-token-tooltip"
+          data-tooltip-place="right"
+        />
+      )}
+      <Tooltip
+        delayHide={300}
+        id="access-token-tooltip"
+        className="max-w-xs"
+        clickable={true}
+      >
+        <p className="text-sm">
+          Without a{" "}
+          <a
+            href="https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html"
+            rel="noreferrer"
+            target="_blank"
+            className="underline"
+            onClick={(e) => e.stopPropagation()}
+          >
+            Personal Access Token
+          </a>
+          , the GitLab API may limit the number of files that can be collected
+          due to rate limits. You can{" "}
+          <a
+            href="https://gitlab.com/-/profile/personal_access_tokens"
+            rel="noreferrer"
+            target="_blank"
+            className="underline"
+            onClick={(e) => e.stopPropagation()}
+          >
+            create a temporary Access Token
+          </a>{" "}
+          to avoid this issue.
+        </p>
+      </Tooltip>
+    </>
+  );
+}

frontend/src/components/Modals/ManageWorkspace/DataConnectors/index.jsx

@@ -1,6 +1,7 @@
 import ConnectorImages from "@/components/DataConnectorOption/media";
 import { MagnifyingGlass } from "@phosphor-icons/react";
 import GithubOptions from "./Connectors/Github";
+import GitlabOptions from "./Connectors/Gitlab";
 import YoutubeOptions from "./Connectors/Youtube";
 import ConfluenceOptions from "./Connectors/Confluence";
 import { useState } from "react";
@@ -15,6 +16,13 @@ export const DATA_CONNECTORS = {
       "Import an entire public or private Github repository in a single click.",
     options: <GithubOptions />,
   },
+  gitlab: {
+    name: "GitLab Repo",
+    image: ConnectorImages.gitlab,
+    description:
+      "Import an entire public or private GitLab repository in a single click.",
+    options: <GitlabOptions />,
+  },
   "youtube-transcript": {
     name: "YouTube Transcript",
     image: ConnectorImages.youtube,

frontend/src/models/dataConnector.js

@@ -42,6 +42,45 @@ const DataConnector = {
         });
     },
   },
+  gitlab: {
+    branches: async ({ repo, accessToken }) => {
+      return await fetch(`${API_BASE}/ext/gitlab/branches`, {
+        method: "POST",
+        headers: baseHeaders(),
+        cache: "force-cache",
+        body: JSON.stringify({ repo, accessToken }),
+      })
+        .then((res) => res.json())
+        .then((res) => {
+          if (!res.success) throw new Error(res.reason);
+          return res.data;
+        })
+        .then((data) => {
+          return { branches: data?.branches || [], error: null };
+        })
+        .catch((e) => {
+          console.error(e);
+          showToast(e.message, "error");
+          return { branches: [], error: e.message };
+        });
+    },
+    collect: async function ({ repo, accessToken, branch, ignorePaths = [] }) {
+      return await fetch(`${API_BASE}/ext/gitlab/repo`, {
+        method: "POST",
+        headers: baseHeaders(),
+        body: JSON.stringify({ repo, accessToken, branch, ignorePaths }),
+      })
+        .then((res) => res.json())
+        .then((res) => {
+          if (!res.success) throw new Error(res.reason);
+          return { data: res.data, error: null };
+        })
+        .catch((e) => {
+          console.error(e);
+          return { data: null, error: e.message };
+        });
+    },
+  },
   youtube: {
     transcribe: async ({ url }) => {
       return await fetch(`${API_BASE}/ext/youtube/transcript`, {

frontend/src/pages/Admin/Agents/SQLConnectorSelection/DBConnection.jsx

@@ -1,12 +1,14 @@
 import PostgreSQLLogo from "./icons/postgresql.png";
 import MySQLLogo from "./icons/mysql.png";
 import MSSQLLogo from "./icons/mssql.png";
+import ODBCLogo from "./icons/odbc.png";
 import { X } from "@phosphor-icons/react";
 
 export const DB_LOGOS = {
   postgresql: PostgreSQLLogo,
   mysql: MySQLLogo,
   "sql-server": MSSQLLogo,
+  odbc: ODBCLogo,
 };
 
 export default function DBConnection({ connection, onRemove, setHasChanges }) {

frontend/src/pages/Admin/Agents/SQLConnectorSelection/NewConnectionModal.jsx

@@ -11,6 +11,7 @@ function assembleConnectionString({
   host = "",
   port = "",
   database = "",
+  driver = "",
 }) {
   if ([username, password, host, database].every((i) => !!i) === false)
     return `Please fill out all the fields above.`;
@@ -21,6 +22,9 @@ function assembleConnectionString({
       return `mysql://${username}:${password}@${host}:${port}/${database}`;
     case "sql-server":
       return `mssql://${username}:${password}@${host}:${port}/${database}`;
+    case "odbc":
+      if (!driver) return `Please fill out the driver field.`;
+      return `Driver={${driver}};Server=${host};Port=${port};Database=${database};UID=${username};PWD=${password}`;
     default:
       return null;
   }
@@ -33,6 +37,7 @@ const DEFAULT_CONFIG = {
   host: null,
   port: null,
   database: null,
+  driver: null,
 };
 
 export default function NewSQLConnection({ isOpen, closeModal, onSubmit }) {
@@ -48,12 +53,14 @@ export default function NewSQLConnection({ isOpen, closeModal, onSubmit }) {
 
   function onFormChange() {
     const form = new FormData(document.getElementById("sql-connection-form"));
+
     setConfig({
       username: form.get("username").trim(),
       password: form.get("password"),
       host: form.get("host").trim(),
       port: form.get("port").trim(),
       database: form.get("database").trim(),
+      driver: form.get("driver")?.trim(),
     });
   }
 
@@ -74,7 +81,7 @@ export default function NewSQLConnection({ isOpen, closeModal, onSubmit }) {
   // to the parent container form so we don't have nested forms.
   return createPortal(
     <ModalWrapper isOpen={isOpen}>
-      <div className="relative w-full md:w-1/3 max-w-2xl max-h-full md:mt-8">
+      <div className="relative w-full md:w-fit max-w-2xl max-h-full md:mt-8">
         <div className="relative bg-main-gradient rounded-xl shadow-[0_4px_14px_rgba(0,0,0,0.25)] max-h-[85vh] overflow-y-scroll no-scroll">
           <div className="flex items-start justify-between p-4 border-b rounded-t border-gray-500/50">
             <h3 className="text-xl font-semibold text-white">
@@ -114,7 +121,7 @@ export default function NewSQLConnection({ isOpen, closeModal, onSubmit }) {
                 <label className="text-white text-sm font-semibold block my-4">
                   Select your SQL engine
                 </label>
-                <div className="grid md:grid-cols-4 gap-4 grid-cols-2">
+                <div className="flex flex-wrap gap-x-4 gap-y-4">
                   <DBEngine
                     provider="postgresql"
                     active={engine === "postgresql"}
@@ -130,6 +137,11 @@ export default function NewSQLConnection({ isOpen, closeModal, onSubmit }) {
                     active={engine === "sql-server"}
                     onClick={() => setEngine("sql-server")}
                   />
+                  <DBEngine
+                    provider="odbc"
+                    active={engine === "odbc"}
+                    onClick={() => setEngine("odbc")}
+                  />
                 </div>
               </div>
 
@@ -224,6 +236,23 @@ export default function NewSQLConnection({ isOpen, closeModal, onSubmit }) {
                   spellCheck={false}
                 />
               </div>
+
+              {engine === "odbc" && (
+                <div className="flex flex-col">
+                  <label className="text-white text-sm font-semibold block mb-3">
+                    Driver
+                  </label>
+                  <input
+                    type="text"
+                    name="driver"
+                    className="border-none bg-zinc-900 text-white placeholder:text-white/20 text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
+                    placeholder="the driver to use eg: MongoDB ODBC 1.2.0 ANSI Driver"
+                    required={true}
+                    autoComplete="off"
+                    spellCheck={false}
+                  />
+                </div>
+              )}
               <p className="text-white/40 text-sm">
                 {assembleConnectionString({ engine, ...config })}
               </p>

server/endpoints/extensions/index.js

@@ -5,18 +5,26 @@ const {
   ROLES,
 } = require("../../utils/middleware/multiUserProtected");
 const { validatedRequest } = require("../../utils/middleware/validatedRequest");
+const {
+  isSupportedRepoProvider,
+} = require("../../utils/middleware/isSupportedRepoProviders");
 
 function extensionEndpoints(app) {
   if (!app) return;
 
   app.post(
-    "/ext/github/branches",
+    "/ext/:repo_platform/branches",
-    [validatedRequest, flexUserRoleValid([ROLES.admin, ROLES.manager])],
+    [
+      validatedRequest,
+      flexUserRoleValid([ROLES.admin, ROLES.manager]),
+      isSupportedRepoProvider,
+    ],
     async (request, response) => {
       try {
+        const { repo_platform } = request.params;
         const responseFromProcessor =
           await new CollectorApi().forwardExtensionRequest({
-            endpoint: "/ext/github-repo/branches",
+            endpoint: `/ext/${repo_platform}-repo/branches`,
             method: "POST",
             body: request.body,
           });
@@ -29,18 +37,23 @@ function extensionEndpoints(app) {
   );
 
   app.post(
-    "/ext/github/repo",
+    "/ext/:repo_platform/repo",
-    [validatedRequest, flexUserRoleValid([ROLES.admin, ROLES.manager])],
+    [
+      validatedRequest,
+      flexUserRoleValid([ROLES.admin, ROLES.manager]),
+      isSupportedRepoProvider,
+    ],
     async (request, response) => {
       try {
+        const { repo_platform } = request.params;
         const responseFromProcessor =
           await new CollectorApi().forwardExtensionRequest({
-            endpoint: "/ext/github-repo",
+            endpoint: `/ext/${repo_platform}-repo`,
             method: "POST",
             body: request.body,
           });
         await Telemetry.sendTelemetry("extension_invoked", {
-          type: "github_repo",
+          type: `${repo_platform}_repo`,
         });
         response.status(200).json(responseFromProcessor);
       } catch (e) {

server/package.json

@@ -66,6 +66,7 @@
     "mysql2": "^3.9.8",
     "node-html-markdown": "^1.3.0",
     "node-llama-cpp": "^2.8.0",
+    "odbc": "^2.4.8",
     "ollama": "^0.5.0",
     "openai": "4.38.5",
     "pg": "^8.11.5",

server/utils/agents/aibitat/plugins/sql-agent/SQLConnectors/ODBC.js

@@ -0,0 +1,60 @@
+const odbc = require("odbc");
+const UrlPattern = require("url-pattern");
+
+class ODBCConnector {
+  #connected = false;
+  database_id = "";
+  constructor(
+    config = {
+      connectionString: null,
+    }
+  ) {
+    this.connectionString = config.connectionString;
+    this._client = null;
+    this.database_id = this.#parseDatabase();
+  }
+
+  #parseDatabase() {
+    const regex = /Database=([^;]+)/;
+    const match = this.connectionString.match(regex);
+    return match ? match[1] : null;
+  }
+
+  async connect() {
+    this._client = await odbc.connect(this.connectionString);
+    this.#connected = true;
+    return this._client;
+  }
+
+  /**
+   *
+   * @param {string} queryString the SQL query to be run
+   * @returns {import(".").QueryResult}
+   */
+  async runQuery(queryString = "") {
+    const result = { rows: [], count: 0, error: null };
+    try {
+      if (!this.#connected) await this.connect();
+      const query = await this._client.query(queryString);
+      result.rows = query;
+      result.count = query.length;
+    } catch (err) {
+      console.log(this.constructor.name, err);
+      result.error = err.message;
+    } finally {
+      await this._client.close();
+      this.#connected = false;
+    }
+    return result;
+  }
+
+  getTablesSql() {
+    return `SELECT table_name FROM information_schema.tables WHERE table_schema = '${this.database_id}'`;
+  }
+
+  getTableSchemaSql(table_name) {
+    return `SHOW COLUMNS FROM ${this.database_id}.${table_name};`;
+  }
+}
+
+module.exports.ODBCConnector = ODBCConnector;

server/utils/agents/aibitat/plugins/sql-agent/SQLConnectors/index.js

@@ -2,7 +2,7 @@ const { SystemSettings } = require("../../../../../../models/systemSettings");
 const { safeJsonParse } = require("../../../../../http");
 
 /**
- * @typedef {('postgresql'|'mysql'|'sql-server')} SQLEngine
+ * @typedef {('postgresql'|'mysql'|'sql-server'|'odbc')} SQLEngine
  */
 
 /**
@@ -36,6 +36,9 @@ function getDBClient(identifier = "", connectionConfig = {}) {
     case "sql-server":
       const { MSSQLConnector } = require("./MSSQL");
       return new MSSQLConnector(connectionConfig);
+    case "odbc":
+      const { ODBCConnector } = require("./ODBC");
+      return new ODBCConnector(connectionConfig);
     default:
       throw new Error(
         `There is no supported database connector for ${identifier}`

server/utils/middleware/isSupportedRepoProviders.js

@@ -0,0 +1,12 @@
+// Middleware to validate that a repo provider URL is supported.
+const REPO_PLATFORMS = ["github", "gitlab"];
+
+function isSupportedRepoProvider(request, response, next) {
+  const { repo_platform = null } = request.params;
+  if (!repo_platform || !REPO_PLATFORMS.includes(repo_platform))
+    return response
+      .status(500)
+      .text(`Unsupported repo platform ${repo_platform}`);
+  next();
+}
+module.exports = { isSupportedRepoProvider };

server/yarn.lock

@@ -673,7 +673,7 @@
     "@langchain/core" "~0.1"
     js-tiktoken "^1.0.11"
 
-"@mapbox/node-pre-gyp@^1.0.11":
+"@mapbox/node-pre-gyp@^1.0.11", "@mapbox/node-pre-gyp@^1.0.5":
   version "1.0.11"
   resolved "https://registry.yarnpkg.com/@mapbox/node-pre-gyp/-/node-pre-gyp-1.0.11.tgz#417db42b7f5323d79e93b34a6d7a2a12c0df43fa"
   integrity sha512-Yhlar6v9WQgUp/He7BdgzOz8lqMQ8sU+jkCq7Wx8Myc5YFJLbEe7lgui/V7G1qB1DJykHSGwreceSaD60Y0PUQ==
@@ -1588,7 +1588,7 @@ arrify@^2.0.0:
   resolved "https://registry.yarnpkg.com/arrify/-/arrify-2.0.1.tgz#c9655e9331e0abcd588d2a7cad7e9956f66701fa"
   integrity sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==
 
-async@^3.2.3, async@^3.2.4:
+async@^3.0.1, async@^3.2.3, async@^3.2.4:
   version "3.2.5"
   resolved "https://registry.yarnpkg.com/async/-/async-3.2.5.tgz#ebd52a8fdaf7a2289a24df399f8d8485c8a46b66"
   integrity sha512-baNZyqaaLhyLVKm/DlvdW051MSgO6b8eVfIezl9E5PqWxFgzLm/wQntEW4zOytVburDEr0JlALEpdOFwvErLsg==
@@ -4813,6 +4813,11 @@ node-abort-controller@^3.1.1:
   resolved "https://registry.yarnpkg.com/node-abort-controller/-/node-abort-controller-3.1.1.tgz#a94377e964a9a37ac3976d848cb5c765833b8548"
   integrity sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==
 
+node-addon-api@^3.0.2:
+  version "3.2.1"
+  resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-3.2.1.tgz#81325e0a2117789c0128dab65e7e38f07ceba161"
+  integrity sha512-mmcei9JghVNDYydghQmeDX8KoAm0FAiYyIcUt/N4nhyAipB17pllZQDOJD2fotxABnt4Mdz+dKTO7eftLg4d0A==
+
 node-addon-api@^5.0.0:
   version "5.1.0"
   resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-5.1.0.tgz#49da1ca055e109a23d537e9de43c09cca21eb762"
@@ -5065,6 +5070,15 @@ octokit@^3.1.0:
     "@octokit/request-error" "^5.0.0"
     "@octokit/types" "^12.0.0"
 
+odbc@^2.4.8:
+  version "2.4.8"
+  resolved "https://registry.yarnpkg.com/odbc/-/odbc-2.4.8.tgz#56e34a1cafbaf1c2c53eec229b3a7604f890e3bf"
+  integrity sha512-W4VkBcr8iSe8hqpp2GoFPybCAJefC7eK837XThJkYCW4tBzyQisqkciwt1UYidU1OpKy1589y9dMN0tStiVB1Q==
+  dependencies:
+    "@mapbox/node-pre-gyp" "^1.0.5"
+    async "^3.0.1"
+    node-addon-api "^3.0.2"
+
 ollama@^0.5.0:
   version "0.5.0"
   resolved "https://registry.yarnpkg.com/ollama/-/ollama-0.5.0.tgz#cb9bc709d4d3278c9f484f751b0d9b98b06f4859"