diff --git a/tokenizer_config.json b/tokenizer_config.json index 9f65345..d656805 100644 --- a/tokenizer_config.json +++ b/tokenizer_config.json @@ -6,7 +6,6 @@ null ] }, - "bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "pad_token": "<|endoftext|>" } \ No newline at end of file