From 8879812cccd176fbbe9ceb747b815bcc7d6499f8 Mon Sep 17 00:00:00 2001
From: Jonathan Tow <jon-tow@users.noreply.huggingface.co>
Date: Wed, 5 Jun 2024 19:45:00 +0000
Subject: [PATCH] tmpfix(tokenizer_config): force `GPT2TokenizerFast`

---
 tokenizer_config.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tokenizer_config.json b/tokenizer_config.json
index a6000a7..0431558 100644
--- a/tokenizer_config.json
+++ b/tokenizer_config.json
@@ -38,6 +38,6 @@
   "bos_token": "<|endoftext|>",
   "clean_up_tokenization_spaces": true,
   "eos_token": "<|endoftext|>",
-  "tokenizer_class": "GPT2Tokenizer",
+  "tokenizer_class": "GPT2TokenizerFast",
   "unk_token": "<|endoftext|>"
 }
\ No newline at end of file