1 Commits

Author SHA1 Message Date
dependabot[bot]
2a00534a4f chore(deps): bump tokenizers from 0.21.4 to 0.22.1
Bumps [tokenizers](https://github.com/huggingface/tokenizers) from 0.21.4 to 0.22.1.
- [Release notes](https://github.com/huggingface/tokenizers/releases)
- [Changelog](https://github.com/huggingface/tokenizers/blob/main/RELEASE.md)
- [Commits](https://github.com/huggingface/tokenizers/compare/v0.21.4...v0.22.1)

---
updated-dependencies:
- dependency-name: tokenizers
  dependency-version: 0.22.1
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
2025-11-17 20:37:09 +00:00
2 changed files with 3 additions and 3 deletions

4
Cargo.lock generated
View File

@@ -2747,9 +2747,9 @@ dependencies = [
[[package]]
name = "tokenizers"
version = "0.21.4"
version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a620b996116a59e184c2fa2dfd8251ea34a36d0a514758c6f966386bd2e03476"
checksum = "6475a27088c98ea96d00b39a9ddfb63780d1ad4cceb6f48374349a96ab2b7842"
dependencies = [
"ahash",
"aho-corasick",

View File

@@ -14,7 +14,7 @@ hf-hub = "0.4.2"
hora = "0.1.1"
kdl = "6.3.4"
serde_json = "1.0.140"
tokenizers = "0.21.4"
tokenizers = "0.22.1"
tracing = "0.1.41"
tracing-subscriber = "0.3.20"
tree-sitter = "0.25.8"