Upload folder using huggingface_hub (#8)

- Upload folder using huggingface_hub (270281d0d0cc91dc6fa781a4a968e5c34c468345)


Co-authored-by: Vaibhav Srivastav <reach-vb@users.noreply.huggingface.co>

special_tokens_map.json

@@ -1,4 +1,16 @@
 {
-  "bos_token": "<|begin_of_text|>",
+  "bos_token": {
-  "eos_token": "<|eot_id|>"
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|eot_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
 }

tokenizer.json

@@ -2329,11 +2329,70 @@
     ]
   },
   "post_processor": {
+    "type": "Sequence",
+    "processors": [
+      {
         "type": "ByteLevel",
         "add_prefix_space": true,
         "trim_offsets": false,
         "use_regex": true
       },
+      {
+        "type": "TemplateProcessing",
+        "single": [
+          {
+            "SpecialToken": {
+              "id": "<|begin_of_text|>",
+              "type_id": 0
+            }
+          },
+          {
+            "Sequence": {
+              "id": "A",
+              "type_id": 0
+            }
+          }
+        ],
+        "pair": [
+          {
+            "SpecialToken": {
+              "id": "<|begin_of_text|>",
+              "type_id": 0
+            }
+          },
+          {
+            "Sequence": {
+              "id": "A",
+              "type_id": 0
+            }
+          },
+          {
+            "SpecialToken": {
+              "id": "<|begin_of_text|>",
+              "type_id": 1
+            }
+          },
+          {
+            "Sequence": {
+              "id": "B",
+              "type_id": 1
+            }
+          }
+        ],
+        "special_tokens": {
+          "<|begin_of_text|>": {
+            "id": "<|begin_of_text|>",
+            "ids": [
+              128000
+            ],
+            "tokens": [
+              "<|begin_of_text|>"
+            ]
+          }
+        }
+      }
+    ]
+  },
   "decoder": {
     "type": "ByteLevel",
     "add_prefix_space": true,