groupuser/Mistral-7B-Instruct-v0.3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Upload tokenizer (#6)

Browse Source 
- Upload tokenizer (37dbd2be561e67c67d4a11dd4ebdd8bb18052cfb)


Co-authored-by: Arthur Zucker <ArthurZ@users.noreply.huggingface.co>
This commit is contained in:
Patrick von Platen 2024-05-22 15:54:33 +00:00 committed by system
parent 405950b956
commit bb73aaeea2
No known key found for this signature in database
GPG Key ID: 6A528E38E0733467
1 changed files with 1 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
tokenizer_config.json
View File

@ -6173,6 +6173,7 @@
} }
}, },
"bos_token": "<s>", "bos_token": "<s>",
"chat_template": "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}",
"clean_up_tokenization_spaces": false, "clean_up_tokenization_spaces": false,
"eos_token": "</s>", "eos_token": "</s>",
"legacy": false, "legacy": false,