diff --git a/tokenizer/tokenizer_config.json b/tokenizer/tokenizer_config.json index ea4f9b6..b966c7c 100644 --- a/tokenizer/tokenizer_config.json +++ b/tokenizer/tokenizer_config.json @@ -1,32 +1,33 @@ { "add_prefix_space": false, - "added_tokens_decoder": { - "49406": { - "content": "<|startoftext|>", - "lstrip": false, - "normalized": true, - "rstrip": false, - "single_word": false, - "special": true - }, - "49407": { - "content": "<|endoftext|>", - "lstrip": false, - "normalized": true, - "rstrip": false, - "single_word": false, - "special": true - } + "bos_token": { + "__type": "AddedToken", + "content": "<|startoftext|>", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false }, - "additional_special_tokens": [], - "bos_token": "<|startoftext|>", "clean_up_tokenization_spaces": true, "do_lower_case": true, - "eos_token": "<|endoftext|>", + "eos_token": { + "__type": "AddedToken", + "content": "<|endoftext|>", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + }, "errors": "replace", "model_max_length": 77, "pad_token": "<|endoftext|>", "tokenizer_class": "CLIPTokenizer", - "tokenizer_file": null, - "unk_token": "<|endoftext|>" -} + "unk_token": { + "__type": "AddedToken", + "content": "<|endoftext|>", + "lstrip": false, + "normalized": true, + "rstrip": false, + "single_word": false + } +} \ No newline at end of file