diff --git a/tokenizer_config.json b/tokenizer_config.json index 4019fcb..dc69ac5 100644 --- a/tokenizer_config.json +++ b/tokenizer_config.json @@ -1,46 +1,4 @@ { - "added_tokens_decoder": { - "0": { - "content": "", - "lstrip": false, - "normalized": false, - "rstrip": false, - "single_word": false, - "special": true - }, - "1": { - "content": "", - "lstrip": false, - "normalized": false, - "rstrip": false, - "single_word": false, - "special": true - }, - "2": { - "content": "", - "lstrip": false, - "normalized": false, - "rstrip": false, - "single_word": false, - "special": true - }, - "250001": { - "content": "", - "lstrip": true, - "normalized": false, - "rstrip": false, - "single_word": false, - "special": true - }, - "3": { - "content": "", - "lstrip": false, - "normalized": false, - "rstrip": false, - "single_word": false, - "special": true - } - }, "bos_token": "", "clean_up_tokenization_spaces": true, "cls_token": "",