mirror of
https://github.com/Mintplex-Labs/anything-llm.git
synced 2024-07-04 16:20:12 +02:00
parent
d95d1a9dfd
commit
3e088f22b1
|
@ -300,7 +300,7 @@ function cannonball({
|
||||||
// if the delta is the token difference between where our prompt is in size
|
// if the delta is the token difference between where our prompt is in size
|
||||||
// and where we ideally need to land.
|
// and where we ideally need to land.
|
||||||
const delta = initialInputSize - targetTokenSize;
|
const delta = initialInputSize - targetTokenSize;
|
||||||
const tokenChunks = tokenManager.countFromString(input);
|
const tokenChunks = tokenManager.tokensFromString(input);
|
||||||
const middleIdx = Math.floor(tokenChunks.length / 2);
|
const middleIdx = Math.floor(tokenChunks.length / 2);
|
||||||
|
|
||||||
// middle truncate the text going left and right of midpoint
|
// middle truncate the text going left and right of midpoint
|
||||||
|
|
|
@ -15,15 +15,22 @@ class TokenManager {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Pass in an empty array of disallowedSpecials to handle all tokens as text and to be tokenized.
|
||||||
|
// https://github.com/openai/tiktoken/blob/9e79899bc248d5313c7dd73562b5e211d728723d/tiktoken/core.py#L91C20-L91C38
|
||||||
|
// Returns number[]
|
||||||
|
tokensFromString(input = "") {
|
||||||
|
const tokens = this.encoder.encode(input, undefined, []);
|
||||||
|
return tokens;
|
||||||
|
}
|
||||||
|
|
||||||
bytesFromTokens(tokens = []) {
|
bytesFromTokens(tokens = []) {
|
||||||
const bytes = this.encoder.decode(tokens);
|
const bytes = this.encoder.decode(tokens);
|
||||||
return bytes;
|
return bytes;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Pass in an empty array of disallowedSpecials to handle all tokens as text and to be tokenized.
|
// Returns number
|
||||||
// https://github.com/openai/tiktoken/blob/9e79899bc248d5313c7dd73562b5e211d728723d/tiktoken/core.py#L91C20-L91C38
|
|
||||||
countFromString(input = "") {
|
countFromString(input = "") {
|
||||||
const tokens = this.encoder.encode(input, undefined, []);
|
const tokens = this.tokensFromString(input);
|
||||||
return tokens.length;
|
return tokens.length;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue
Block a user