From ca32832c44abf0e1b6c5247b6a07a69d9b1037bc Mon Sep 17 00:00:00 2001
From: Jonathan Tow <jon-tow@users.noreply.huggingface.co>
Date: Fri, 19 Jan 2024 22:52:04 +0000
Subject: [PATCH] fix: add default  special tokens

---
 tokenizer_config.json | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tokenizer_config.json b/tokenizer_config.json
index e0422ac..9f65345 100644
--- a/tokenizer_config.json
+++ b/tokenizer_config.json
@@ -5,5 +5,8 @@
       "tokenization_arcade100k.Arcade100kTokenizer",
       null
     ]
-  }
+  },
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "pad_token": "<|endoftext|>"
 }
\ No newline at end of file