feat(tokenizer): expose merge ranks and special tokens for GGUF

2024-01-19 18:22:13 +00:00 · 2024-01-19 18:22:13 +00:00 · a8f2f2862b
commit a8f2f2862b
parent 3aeae29673
1 changed files with 4 additions and 1 deletions
--- a/tokenization_arcade100k.py
+++ b/tokenization_arcade100k.py
@ -126,6 +126,9 @@ class Arcade100kTokenizer(PreTrainedTokenizer):
        self.decoder.update({i: n for n, i in self.tokenizer._special_tokens.items()})
        self.eos_token = self.decoder[self.tokenizer.eot_token]
        self.pad_token = self.decoder[self.tokenizer.eot_token]
+        # Expose for convenience
+        self.mergeable_ranks = self.tokenizer._mergeable_ranks
+        self.special_tokens = self.tokenizer._special_tokens

    def __len__(self):
        return self.tokenizer.n_vocab