From d8411bd4e55c0bca39e60653a0fe26ae8591859a Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Tue, 7 Nov 2023 18:53:52 +0000 Subject: [PATCH] Upload processor (#2) - Upload processor (b634ee66a93a2f270ba8bfab680050a66dd096e4) Co-authored-by: Sanchit Gandhi --- special_tokens_map.json | 1 + tokenizer_config.json | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/special_tokens_map.json b/special_tokens_map.json index ab4001a..3c18e97 100644 --- a/special_tokens_map.json +++ b/special_tokens_map.json @@ -122,6 +122,7 @@ "rstrip": false, "single_word": false }, + "pad_token": "<|endoftext|>", "unk_token": { "content": "<|endoftext|>", "lstrip": false, diff --git a/tokenizer_config.json b/tokenizer_config.json index 7c3aa9d..06ffdc8 100644 --- a/tokenizer_config.json +++ b/tokenizer_config.json @@ -12988,8 +12988,9 @@ "eos_token": "<|endoftext|>", "errors": "replace", "model_max_length": 1000000000000000019884624838656, - "pad_token": null, + "pad_token": "<|endoftext|>", "processor_class": "WhisperProcessor", "tokenizer_class": "WhisperTokenizer", + "trust_remote_code": false, "unk_token": "<|endoftext|>" }