From 50f9396f75618b3389c1fd1068a1ff58dc7b5b26 Mon Sep 17 00:00:00 2001
From: Xiao <Shitao@users.noreply.huggingface.co>
Date: Tue, 26 Mar 2024 02:54:48 +0000
Subject: [PATCH] Update tokenizer_config.json

---
 tokenizer_config.json | 42 ------------------------------------------
 1 file changed, 42 deletions(-)

diff --git a/tokenizer_config.json b/tokenizer_config.json
index 4019fcb..dc69ac5 100644
--- a/tokenizer_config.json
+++ b/tokenizer_config.json
@@ -1,46 +1,4 @@
 {
-  "added_tokens_decoder": {
-    "0": {
-      "content": "<s>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "1": {
-      "content": "<pad>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "2": {
-      "content": "</s>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "250001": {
-      "content": "<mask>",
-      "lstrip": true,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "3": {
-      "content": "<unk>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    }
-  },
   "bos_token": "<s>",
   "clean_up_tokenization_spaces": true,
   "cls_token": "<s>",