Updates the tokenizer configuration file (#62)

- Adds tokenizer_config.json file (87565a3098077d493d402e1b109e67baf52c5281)
This commit is contained in:
Lysandre 2024-02-19 11:06:12 +00:00 committed by system
parent 1dbc166cf8
commit 86b5e09344
No known key found for this signature in database
GPG Key ID: 6A528E38E0733467

@ -1,3 +1 @@
{
"do_lower_case": true
}
{"do_lower_case": true, "model_max_length": 512}