diff --git a/tokenizer_config.json b/tokenizer_config.json index a6000a7..0431558 100644 --- a/tokenizer_config.json +++ b/tokenizer_config.json @@ -38,6 +38,6 @@ "bos_token": "<|endoftext|>", "clean_up_tokenization_spaces": true, "eos_token": "<|endoftext|>", - "tokenizer_class": "GPT2Tokenizer", + "tokenizer_class": "GPT2TokenizerFast", "unk_token": "<|endoftext|>" } \ No newline at end of file