Update nava_src/models/nava/modules/tokenizers.py

nava_src/models/nava/modules/tokenizers.py

@@ -45,7 +45,17 @@ class HuggingfaceTokenizer:
         # init tokenizer
         self.tokenizer = AutoTokenizer.from_pretrained(name, **kwargs)
         if self.tokenizer.pad_token is None:
-            self.tokenizer.pad_token = self.tokenizer.eos_token
+            try:
+                self.tokenizer.add_special_tokens({'pad_token': '<pad>'})
+            except Exception:
+                self.tokenizer.pad_token = "<pad>"
+                self.tokenizer.pad_token_id = 0
+        if self.tokenizer.eos_token is None:
+            self.tokenizer.eos_token = "</s>"
+            self.tokenizer.eos_token_id = 1
+        if self.tokenizer.unk_token is None:
+            self.tokenizer.unk_token = "<unk>"
+            self.tokenizer.unk_token_id = 2
         self.vocab_size = self.tokenizer.vocab_size
 
     def __call__(self, sequence, **kwargs):