segmind/SSD-1B
segmind/SSD-1B
1
0
Overview Code Discussions Pull Requests Wiki Activity

Update tokenizer/special_tokens_map.json

Browse Source
This commit is contained in:
Yatharth Gupta 2023-10-21 15:29:07 +00:00 committed by huggingface-web
parent 2be6cf02d3
commit 00c0aebe18
1 changed files with 21 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
tokenizer/special_tokens_map.json
View File

@ -1,6 +1,24 @@
{
"bos_token": "<|startoftext|>",
"eos_token": "<|endoftext|>",
"bos_token": {
"content": "<|startoftext|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"eos_token": {
"content": "<|endoftext|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
},
"pad_token": "<|endoftext|>",
"unk_token": "<|endoftext|>"
"unk_token": {
"content": "<|endoftext|>",
"lstrip": false,
"normalized": true,
"rstrip": false,
"single_word": false
}
}