kamp/anything-llm
1
0
Fork 0
mirror of https://github.com/Mintplex-Labs/anything-llm.git synced 2024-07-02 15:20:37 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix: Patch tiktoken method missing

Browse Source 
resolves #541
This commit is contained in:
timothycarambat 2024-01-05 09:39:19 -08:00
parent d95d1a9dfd
commit 3e088f22b1
2 changed files with 11 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
server/utils/helpers/chat/index.js
View File

@ -300,7 +300,7 @@ function cannonball({
// if the delta is the token difference between where our prompt is in size
// and where we ideally need to land.
const delta = initialInputSize - targetTokenSize;
const tokenChunks = tokenManager.countFromString(input);
const tokenChunks = tokenManager.tokensFromString(input);
const middleIdx = Math.floor(tokenChunks.length / 2);
// middle truncate the text going left and right of midpoint

13
server/utils/helpers/tiktoken.js
View File

@ -15,15 +15,22 @@ class TokenManager {
}
}
// Pass in an empty array of disallowedSpecials to handle all tokens as text and to be tokenized.
// https://github.com/openai/tiktoken/blob/9e79899bc248d5313c7dd73562b5e211d728723d/tiktoken/core.py#L91C20-L91C38
// Returns number[]
tokensFromString(input = "") {
const tokens = this.encoder.encode(input, undefined, []);
return tokens;
}
bytesFromTokens(tokens = []) {
const bytes = this.encoder.decode(tokens);
return bytes;
}
// Pass in an empty array of disallowedSpecials to handle all tokens as text and to be tokenized.
// https://github.com/openai/tiktoken/blob/9e79899bc248d5313c7dd73562b5e211d728723d/tiktoken/core.py#L91C20-L91C38
// Returns number
countFromString(input = "") {
const tokens = this.encoder.encode(input, undefined, []);
const tokens = this.tokensFromString(input);
return tokens.length;
}