diff --git a/special_tokens_map.json b/special_tokens_map.json index ab4001a..3c18e97 100644 --- a/special_tokens_map.json +++ b/special_tokens_map.json @@ -122,6 +122,7 @@ "rstrip": false, "single_word": false }, + "pad_token": "<|endoftext|>", "unk_token": { "content": "<|endoftext|>", "lstrip": false, diff --git a/tokenizer_config.json b/tokenizer_config.json index 7c3aa9d..06ffdc8 100644 --- a/tokenizer_config.json +++ b/tokenizer_config.json @@ -12988,8 +12988,9 @@ "eos_token": "<|endoftext|>", "errors": "replace", "model_max_length": 1000000000000000019884624838656, - "pad_token": null, + "pad_token": "<|endoftext|>", "processor_class": "WhisperProcessor", "tokenizer_class": "WhisperTokenizer", + "trust_remote_code": false, "unk_token": "<|endoftext|>" }