mirror of
https://github.com/Mintplex-Labs/anything-llm.git
synced 2024-09-21 20:00:10 +02:00
16 lines
304 B
JavaScript
16 lines
304 B
JavaScript
|
const { getEncoding } = require("js-tiktoken");
|
||
|
|
|
||
|
|
function tokenizeString(input = "") {
|
||
|
|
try {
|
||
|
|
const encoder = getEncoding("cl100k_base");
|
||
|
|
return encoder.encode(input);
|
||
|
|
} catch (e) {
|
||
|
|
console.error("Could not tokenize string!");
|
||
|
|
return [];
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
module.exports = {
|
||
|
|
tokenizeString,
|
||
|
|
};
|