fix(tokenizer): expose errors

2024-01-25 16:17:34 +00:00 · 2024-01-25 16:17:34 +00:00 · 21ee10d32c
commit 21ee10d32c
parent 810b45c00e
1 changed files with 2 additions and 0 deletions
--- a/tokenization_arcade100k.py
+++ b/tokenization_arcade100k.py
@ -111,6 +111,8 @@ class Arcade100kTokenizer(PreTrainedTokenizer):
        **kwargs,
    ):
        super().__init__(errors=errors, **kwargs)
+        self.errors = errors
+
        self._tiktoken_config = _arcade100k(vocab_file)
        self.tokenizer = tiktoken.Encoding(**self._tiktoken_config)