groupuser/Meta-Llama-3-8B-Instruct
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update post-processor to add bos (#42)

Browse Source 
- Update post-processor to add bos (4d3ac242e1d717fbebaa94154be38077f4e1623b)
This commit is contained in:
Pedro Cuenca 2024-04-23 11:33:49 +00:00 committed by system
parent 74fedae5f7
commit 339ce92d05
No known key found for this signature in database
GPG Key ID: 6A528E38E0733467
1 changed files with 63 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
tokenizer.json
View File

@ -2329,10 +2329,69 @@
]
},
"post_processor": {
"type": "ByteLevel",
"add_prefix_space": true,
"trim_offsets": false,
"use_regex": true
"type": "Sequence",
"processors": [
{
"type": "ByteLevel",
"add_prefix_space": true,
"trim_offsets": false,
"use_regex": true
},
{
"type": "TemplateProcessing",
"single": [
{
"SpecialToken": {
"id": "<|begin_of_text|>",
"type_id": 0
}
},
{
"Sequence": {
"id": "A",
"type_id": 0
}
}
],
"pair": [
{
"SpecialToken": {
"id": "<|begin_of_text|>",
"type_id": 0
}
},
{
"Sequence": {
"id": "A",
"type_id": 0
}
},
{
"SpecialToken": {
"id": "<|begin_of_text|>",
"type_id": 1
}
},
{
"Sequence": {
"id": "B",
"type_id": 1
}
}
],
"special_tokens": {
"<|begin_of_text|>": {
"id": "<|begin_of_text|>",
"ids": [
128000
],
"tokens": [
"<|begin_of_text|>"
]
}
}
}
]
},
"decoder": {
"type": "ByteLevel",