From e762ad4cc9ae3e6e8e7db5df20c085712b2b3cac Mon Sep 17 00:00:00 2001
From: Yatharth Gupta <Warlord-K@users.noreply.huggingface.co>
Date: Sat, 21 Oct 2023 15:31:49 +0000
Subject: [PATCH] Update tokenizer_2/special_tokens_map.json

---
 tokenizer_2/special_tokens_map.json | 24 +++++++++++++++++++++---
 1 file changed, 21 insertions(+), 3 deletions(-)

diff --git a/tokenizer_2/special_tokens_map.json b/tokenizer_2/special_tokens_map.json
index 39808ac..ae0c5be 100644
--- a/tokenizer_2/special_tokens_map.json
+++ b/tokenizer_2/special_tokens_map.json
@@ -1,6 +1,24 @@
 {
-  "bos_token": "<|startoftext|>",
-  "eos_token": "<|endoftext|>",
+  "bos_token": {
+    "content": "<|startoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
   "pad_token": "!",
-  "unk_token": "<|endoftext|>"
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
 }