groupuser/Llama-3.2-1B-Instruct
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Upload folder using huggingface_hub (#8)

Browse Source 
- Upload folder using huggingface_hub (270281d0d0cc91dc6fa781a4a968e5c34c468345)


Co-authored-by: Vaibhav Srivastav <reach-vb@users.noreply.huggingface.co>
This commit is contained in:
Omar Sanseviero 2024-09-25 08:00:24 +00:00 committed by system
parent efe927efa7
commit b0445fe70d
No known key found for this signature in database
GPG Key ID: 6A528E38E0733467
2 changed files with 77 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
special_tokens_map.json
View File

@ -1,4 +1,16 @@
{
"bos_token": "<|begin_of_text|>",
"eos_token": "<|eot_id|>"
"bos_token": {
"content": "<|begin_of_text|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
},
"eos_token": {
"content": "<|eot_id|>",
"lstrip": false,
"normalized": false,
"rstrip": false,
"single_word": false
}
}

59
tokenizer.json
View File

@ -2329,11 +2329,70 @@
]
},
"post_processor": {
"type": "Sequence",
"processors": [
{
"type": "ByteLevel",
"add_prefix_space": true,
"trim_offsets": false,
"use_regex": true
},
{
"type": "TemplateProcessing",
"single": [
{
"SpecialToken": {
"id": "<|begin_of_text|>",
"type_id": 0
}
},
{
"Sequence": {
"id": "A",
"type_id": 0
}
}
],
"pair": [
{
"SpecialToken": {
"id": "<|begin_of_text|>",
"type_id": 0
}
},
{
"Sequence": {
"id": "A",
"type_id": 0
}
},
{
"SpecialToken": {
"id": "<|begin_of_text|>",
"type_id": 1
}
},
{
"Sequence": {
"id": "B",
"type_id": 1
}
}
],
"special_tokens": {
"<|begin_of_text|>": {
"id": "<|begin_of_text|>",
"ids": [
128000
],
"tokens": [
"<|begin_of_text|>"
]
}
}
}
]
},
"decoder": {
"type": "ByteLevel",
"add_prefix_space": true,