From bc8c663670e3ec391f27daf90dcf1c4823fbfaf6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 May 2026 01:05:44 +0000 Subject: [PATCH 1/2] Update tokenizers requirement from ^0.21 to ^0.23 Updates the requirements on [tokenizers](https://github.com/huggingface/tokenizers) to permit the latest version. - [Release notes](https://github.com/huggingface/tokenizers/releases) - [Changelog](https://github.com/huggingface/tokenizers/blob/main/RELEASE.md) - [Commits](https://github.com/huggingface/tokenizers/compare/v0.21.0...v0.23.1) --- updated-dependencies: - dependency-name: tokenizers dependency-version: 0.23.1 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index c00e7ad..78db2dd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,7 +30,7 @@ regex = "1.10.3" serde = "1.0.197" serde_json = "1.0.114" serde_with = "3.8.0" -tokenizers = { version = "^0.21"} +tokenizers = { version = "^0.23"} [dev-dependencies] tempfile = "3.10.1" From e92c1b835db590ce2351ad19493bef1579b1f0b7 Mon Sep 17 00:00:00 2001 From: Alexius Wadell Date: Sun, 10 May 2026 19:11:24 -0600 Subject: [PATCH 2/2] fix: token addition for tokenizer v0.23 --- src/tokenizer.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/tokenizer.rs b/src/tokenizer.rs index a6c53c5..67bed1d 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -349,7 +349,9 @@ impl SmirkTokenizer { }) }) .collect::, PyErr>>()?; - Ok(self.tokenizer.add_tokens(&tokens)) + self.tokenizer + .add_tokens(tokens) + .map_err(|e| PyValueError::new_err(e.to_string())) } #[pyo3(signature = (files, **kwargs))]