groupuser/stablelm-2-1_6b
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(tokenizer): expose errors

Browse Source
This commit is contained in:
Jonathan Tow 2024-01-25 16:17:34 +00:00 committed by system
parent 810b45c00e
commit 21ee10d32c
No known key found for this signature in database
GPG Key ID: 6A528E38E0733467
1 changed files with 2 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
tokenization_arcade100k.py
View File

@ -111,6 +111,8 @@ class Arcade100kTokenizer(PreTrainedTokenizer):
**kwargs, **kwargs,
): ):
super().__init__(errors=errors, **kwargs) super().__init__(errors=errors, **kwargs)
self.errors = errors
self._tiktoken_config = _arcade100k(vocab_file) self._tiktoken_config = _arcade100k(vocab_file)
self.tokenizer = tiktoken.Encoding(**self._tiktoken_config) self.tokenizer = tiktoken.Encoding(**self._tiktoken_config)