kamp/anything-llm
1
0
Fork 0
mirror of https://github.com/Mintplex-Labs/anything-llm.git synced 2024-07-02 15:20:37 +02:00
Code Issues Packages Projects Releases Wiki Activity

Handle special token in TikToken (#528)

Browse Source 
* Handle special token in TikToken
resolves #525

* remove duplicate method
add clarification comment on implementation
This commit is contained in:
Timothy Carambat 2024-01-04 15:47:00 -08:00 committed by GitHub
parent a2a903741d
commit 92da23e963
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 6 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
server/utils/helpers/chat/index.js
View File

@ -300,7 +300,7 @@ function cannonball({
// if the delta is the token difference between where our prompt is in size
// and where we ideally need to land.
const delta = initialInputSize - targetTokenSize;
const tokenChunks = tokenManager.tokensFromString(input);
const tokenChunks = tokenManager.countFromString(input);
const middleIdx = Math.floor(tokenChunks.length / 2);
// middle truncate the text going left and right of midpoint

14
server/utils/helpers/tiktoken.js
View File

@ -3,12 +3,11 @@ const { getEncodingNameForModel, getEncoding } = require("js-tiktoken");
class TokenManager {
constructor(model = "gpt-3.5-turbo") {
this.model = model;
this.encoderName = this.getEncodingFromModel(model);
this.encoderName = this.#getEncodingFromModel(model);
this.encoder = getEncoding(this.encoderName);
this.buffer = 50;
}
getEncodingFromModel(model) {
#getEncodingFromModel(model) {
try {
return getEncodingNameForModel(model);
} catch {
@ -16,18 +15,15 @@ class TokenManager {
}
}
tokensFromString(input = "") {
const tokens = this.encoder.encode(input);
return tokens;
}
bytesFromTokens(tokens = []) {
const bytes = this.encoder.decode(tokens);
return bytes;
}
// Pass in an empty array of disallowedSpecials to handle all tokens as text and to be tokenized.
// https://github.com/openai/tiktoken/blob/9e79899bc248d5313c7dd73562b5e211d728723d/tiktoken/core.py#L91C20-L91C38
countFromString(input = "") {
const tokens = this.encoder.encode(input);
const tokens = this.encoder.encode(input, undefined, []);
return tokens.length;
}