diff --git a/tokenizer_2/added_tokens.json b/tokenizer_2/added_tokens.json deleted file mode 100644 index 5ff4ac5..0000000 --- a/tokenizer_2/added_tokens.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "!": 0, - "<|endoftext|>": 49407, - "<|startoftext|>": 49406 -}