Bump script bytecode version after token enum change

This commit is contained in:
nikitalita
2025-05-18 02:33:56 -07:00
parent 34f005d810
commit 6909309ca0
3 changed files with 9 additions and 10 deletions

View File

@@ -39,6 +39,7 @@ public:
COMPRESS_ZSTD,
};
static constexpr uint32_t TOKENIZER_VERSION = 101;
static constexpr uint32_t TOKEN_BYTE_MASK = 0x80;
static constexpr uint32_t TOKEN_BITS = 8;
static constexpr uint32_t TOKEN_MASK = (1 << (TOKEN_BITS - 1)) - 1;