From 21857530684f91b80df33041f19c8e2198bc3a59 Mon Sep 17 00:00:00 2001
From: timothycarambat <rambat1010@gmail.com>
Date: Mon, 22 Jul 2024 12:53:11 -0700
Subject: [PATCH] patch text.substring bug from compressor

---
 server/utils/helpers/tiktoken.js | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/server/utils/helpers/tiktoken.js b/server/utils/helpers/tiktoken.js
index c5852892..a3fa3b63 100644
--- a/server/utils/helpers/tiktoken.js
+++ b/server/utils/helpers/tiktoken.js
@@ -19,8 +19,13 @@ class TokenManager {
   // https://github.com/openai/tiktoken/blob/9e79899bc248d5313c7dd73562b5e211d728723d/tiktoken/core.py#L91C20-L91C38
   // Returns number[]
   tokensFromString(input = "") {
-    const tokens = this.encoder.encode(input, undefined, []);
-    return tokens;
+    try {
+      const tokens = this.encoder.encode(String(input), undefined, []);
+      return tokens;
+    } catch (e) {
+      console.error(e);
+      return [];
+    }
   }
 
   bytesFromTokens(tokens = []) {