add tokenizer

special_tokens_map.json

@@ -0,0 +1 @@
+{"unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]"}

tokenizer_config.json

@@ -0,0 +1 @@
+{"do_lower_case": true, "do_basic_tokenize": true, "never_split": null, "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]", "tokenize_chinese_chars": true, "strip_accents": null, "special_tokens_map_file": "../models/special_tokens_map.json", "tokenizer_file": null, "name_or_path": "/home/idsedykh/thesis/models/bert-7-big/checkpoint-88000", "tokenizer_class": "CharBertTokenizer"}

vocab.txt

@@ -0,0 +1,38 @@
+[PAD]
+[UNK]
+[CLS]
+[SEP]
+[MASK]
+э
+т
+о
+н
+а
+и
+с
+к
+з
+ь
+р
+ы
+й
+ч
+е
+л
+в
+д
+б
+я
+у
+ж
+г
+м
+ш
+п
+х
+ц
+щ
+ю
+ф
+ъ
+-