Upload processor (#2)

- Upload processor (b634ee66a93a2f270ba8bfab680050a66dd096e4) Co-authored-by: Sanchit Gandhi <sanchit-gandhi@users.noreply.huggingface.co>
2023-11-07 18:53:52 +00:00 · 2023-11-07 18:53:52 +00:00 · d8411bd4e5
commit d8411bd4e5
parent f0e19c5763
2 changed files with 3 additions and 1 deletions
--- a/special_tokens_map.json
+++ b/special_tokens_map.json
@ -122,6 +122,7 @@
    "rstrip": false,
    "single_word": false
  },
+  "pad_token": "<|endoftext|>",
  "unk_token": {
    "content": "<|endoftext|>",
    "lstrip": false,
--- a/tokenizer_config.json
+++ b/tokenizer_config.json
@ -12988,8 +12988,9 @@
  "eos_token": "<|endoftext|>",
  "errors": "replace",
  "model_max_length": 1000000000000000019884624838656,
-  "pad_token": null,
+  "pad_token": "<|endoftext|>",
  "processor_class": "WhisperProcessor",
  "tokenizer_class": "WhisperTokenizer",
+  "trust_remote_code": false,
  "unk_token": "<|endoftext|>"
 }