Upload folder using huggingface_hub

README.md

@@ -0,0 +1,54 @@
+---
+license: mit
+language:
+- jpn
+
+tags:
+- tokenizer
+- bpe
+- flexitok
+- fineweb2
+---
+
+# Byte-Level BPE Tokenizer: ['jpn_Jpan'] (0K)
+
+A **Byte-Level BPE** tokenizer trained on **['jpn_Jpan']** data from Fineweb-2-HQ.
+
+## Training Details
+
+| Parameter | Value |
+|-----------|-------|
+| Algorithm | Byte-Level BPE |
+| Language | `['jpn_Jpan']` |
+| Target Vocab Size | 360 |
+| Final Vocab Size | 360 |
+| Pre-tokenizer | custom:addition |
+| Number handling | ltr_3digit |
+| Contraction handling | False |
+| Normalizer | NFC |
+| Special Tokens | `<s>`, `</s>`, `<pad>`, `<unk>` |
+| Training Shards | 2 |
+
+## Usage
+
+```python
+from transformers import AutoTokenizer
+
+tokenizer = AutoTokenizer.from_pretrained("flexitok/maddition_jpn_Jpan_360")
+tokens = tokenizer.encode("Hello, world!")
+```
+
+## Files
+
+- `tokenizer.json` — Full HuggingFace tokenizer
+- `vocab.json` — Vocabulary mapping
+- `merges.txt` — BPE merge rules
+
+## Sample Encoding
+| Text | Tokens | Token IDs |
+|------|--------|-----------|
+| `yirmi iki+dokuz=otuz bir\ntwenty two+nine=thirty one` | `y, i, r, m, i, Ġ, i, k, i, +, d, o, k, u, z, =, o, t, u, z` | `91, 75, 84, 79, 75, 223, 75, 77, 75, 13, 70, 81, 77, 87, 92, 31, 81, 86, 87, 92` |
+
+Command used to create this tokenizer:
+```bash
+['/home/gsa/tokenizers2/flexitok/tokenizer_training/train_tokenizers.py', 'algorithm=bpe', 'vocab_size=360', 'langs=[jpn_Jpan]', 'data_dir=/scratch/gsa/data/multilingual-addition/', 'output_dir=/scratch/gsa/trained_tokenizers/multilingual_addition', 'pretokenizer=custom:addition', 'number_handling=ltr_3digit', 'add_numbers=false', 'handle_contractions=false', 'unicode_normalization=nfc', 'use_byte_level_regex=false', 'byte_fallback=false', 'strip_zero_width=false', 'cjk_char_split=false', 'add_cjk_chars=false', 'max_lines=-1', 'test_string=yirmi iki+dokuz=otuz bir\\ntwenty two+nine=thirty one', 'hf.publish_to_hf=true', 'hf_repo_prefix=flexitok/', 'hf.hf_repo_id=flexitok/maddition_jpn_Jpan_360', 'hf.collections=[flexitok/multilingual_addition_tokenizers]']

merges.txt

@@ -0,0 +1,102 @@
+#version: 0.2
+['å', 'į']
+['ç', 'Ļ']
+['åį', 'ģ']
+['çĻ', '¾']
+['ä', '¸']
+['ä', 'º']
+['å', 'ħ']
+['¹', 'Ŀ']
+['ä', '¹Ŀ']
+['å', 'Ľ']
+['ä¸', 'ĥ']
+['ä¸', 'ī']
+['äº', 'Į']
+['äº', 'Ķ']
+['åħ', '«']
+['åħ', 'Ń']
+['åĽ', 'Ľ']
+['åį', 'ĥ']
+['ä¸', 'Ģ']
+['ä¹Ŀ', 'åįģ']
+['ä¹Ŀ', 'çĻ¾']
+['ä¸ĥ', 'åįģ']
+['ä¸ĥ', 'çĻ¾']
+['ä¸ī', 'åįģ']
+['ä¸ī', 'çĻ¾']
+['äºĮ', 'åįģ']
+['äºĮ', 'çĻ¾']
+['äºĶ', 'åįģ']
+['äºĶ', 'çĻ¾']
+['åħ«', 'åįģ']
+['åħ«', 'çĻ¾']
+['åħŃ', 'åįģ']
+['åħŃ', 'çĻ¾']
+['åĽĽ', 'åįģ']
+['åĽĽ', 'çĻ¾']
+['åįĥ', 'çĻ¾']
+['åįĥ', 'äºĮçĻ¾']
+['åįĥ', 'ä¸īçĻ¾']
+['åįĥ', 'åĽĽçĻ¾']
+['åįĥ', 'äºĶçĻ¾']
+['åįĥ', 'åħŃçĻ¾']
+['åįģ', 'ä¹Ŀ']
+['åįģ', 'ä¸ĥ']
+['åįģ', 'ä¸ī']
+['åįģ', 'äºĮ']
+['åįģ', 'äºĶ']
+['åįģ', 'åħ«']
+['åįģ', 'åħŃ']
+['åįģ', 'åĽĽ']
+['åįģ', 'ä¸Ģ']
+['ä¹Ŀåįģ', 'ä¹Ŀ']
+['ä¹Ŀåįģ', 'ä¸ĥ']
+['ä¹Ŀåįģ', 'ä¸ī']
+['ä¹Ŀåįģ', 'äºĮ']
+['ä¹Ŀåįģ', 'äºĶ']
+['ä¹Ŀåįģ', 'åħ«']
+['ä¹Ŀåįģ', 'åħŃ']
+['ä¹Ŀåįģ', 'åĽĽ']
+['ä¹Ŀåįģ', 'ä¸Ģ']
+['ä¹ĿçĻ¾', 'ä¸ĥåįģ']
+['ä¹ĿçĻ¾', 'ä¸īåįģ']
+['ä¹ĿçĻ¾', 'äºĮåįģ']
+['ä¹ĿçĻ¾', 'äºĶåįģ']
+['ä¹ĿçĻ¾', 'åħ«åįģ']
+['ä¹ĿçĻ¾', 'åħŃåįģ']
+['ä¹ĿçĻ¾', 'åĽĽåįģ']
+['ä¸ĥçĻ¾', 'ä¸ĥåįģ']
+['ä¸ĥçĻ¾', 'ä¸īåįģ']
+['ä¸ĥçĻ¾', 'äºĮåįģ']
+['ä¸ĥçĻ¾', 'äºĶåįģ']
+['ä¸ĥçĻ¾', 'åħ«åįģ']
+['ä¸ĥçĻ¾', 'åħŃåįģ']
+['ä¸ĥçĻ¾', 'åĽĽåįģ']
+['åħ«çĻ¾', 'ä¸ĥåįģ']
+['åħ«çĻ¾', 'ä¸īåįģ']
+['åħ«çĻ¾', 'äºĮåįģ']
+['åħ«çĻ¾', 'äºĶåįģ']
+['åħ«çĻ¾', 'åħ«åįģ']
+['åħ«çĻ¾', 'åħŃåįģ']
+['åħ«çĻ¾', 'åĽĽåįģ']
+['åħŃçĻ¾', 'åħ«åįģ']
+['åħŃçĻ¾', 'ä¸ĥåįģ']
+['åħŃçĻ¾', 'åħŃåįģ']
+['åħŃçĻ¾', 'äºĶåįģ']
+['åħŃçĻ¾', 'åĽĽåįģ']
+['åħŃçĻ¾', 'ä¸īåįģ']
+['åħŃçĻ¾', 'äºĮåįģ']
+['äºĶçĻ¾', 'åħ«åįģ']
+['äºĶçĻ¾', 'ä¸ĥåįģ']
+['äºĶçĻ¾', 'åħŃåįģ']
+['äºĶçĻ¾', 'äºĶåįģ']
+['äºĶçĻ¾', 'åĽĽåįģ']
+['äºĶçĻ¾', 'ä¸īåįģ']
+['äºĶçĻ¾', 'äºĮåįģ']
+['åĽĽçĻ¾', 'åħ«åįģ']
+['åĽĽçĻ¾', 'ä¸ĥåįģ']
+['åĽĽçĻ¾', 'åħŃåįģ']
+['åĽĽçĻ¾', 'äºĶåįģ']
+['åĽĽçĻ¾', 'åĽĽåįģ']
+['åĽĽçĻ¾', 'ä¸īåįģ']
+['åĽĽçĻ¾', 'äºĮåįģ']

special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "pad_token": "<pad>"
+}

tokenizer.json

@@ -0,0 +1,849 @@
+{
+  "version": "1.0",
+  "truncation": null,
+  "padding": null,
+  "added_tokens": [
+    {
+      "id": 0,
+      "content": "<s>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 1,
+      "content": "</s>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    },
+    {
+      "id": 2,
+      "content": "<pad>",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": false,
+      "special": true
+    }
+  ],
+  "normalizer": {
+    "type": "NFC"
+  },
+  "pre_tokenizer": {
+    "type": "Sequence",
+    "pretokenizers": [
+      {
+        "type": "Split",
+        "pattern": {
+          "Regex": "[+=]|[^\\S\\r\\n]*[\\n\\r]+|[^\\S\\r\\n]+"
+        },
+        "behavior": "Isolated",
+        "invert": false
+      },
+      {
+        "type": "Split",
+        "pattern": {
+          "Regex": "\\p{N}{1,3}"
+        },
+        "behavior": "Isolated",
+        "invert": false
+      },
+      {
+        "type": "ByteLevel",
+        "add_prefix_space": false,
+        "trim_offsets": true,
+        "use_regex": false
+      }
+    ]
+  },
+  "post_processor": null,
+  "decoder": {
+    "type": "ByteLevel",
+    "add_prefix_space": true,
+    "trim_offsets": true,
+    "use_regex": true
+  },
+  "model": {
+    "type": "BPE",
+    "dropout": null,
+    "unk_token": null,
+    "continuing_subword_prefix": null,
+    "end_of_word_suffix": null,
+    "fuse_unk": false,
+    "byte_fallback": false,
+    "ignore_merges": false,
+    "vocab": {
+      "<s>": 0,
+      "</s>": 1,
+      "<pad>": 2,
+      "!": 3,
+      "\"": 4,
+      "#": 5,
+      "$": 6,
+      "%": 7,
+      "&": 8,
+      "'": 9,
+      "(": 10,
+      ")": 11,
+      "*": 12,
+      "+": 13,
+      ",": 14,
+      "-": 15,
+      ".": 16,
+      "/": 17,
+      "0": 18,
+      "1": 19,
+      "2": 20,
+      "3": 21,
+      "4": 22,
+      "5": 23,
+      "6": 24,
+      "7": 25,
+      "8": 26,
+      "9": 27,
+      ":": 28,
+      ";": 29,
+      "<": 30,
+      "=": 31,
+      ">": 32,
+      "?": 33,
+      "@": 34,
+      "A": 35,
+      "B": 36,
+      "C": 37,
+      "D": 38,
+      "E": 39,
+      "F": 40,
+      "G": 41,
+      "H": 42,
+      "I": 43,
+      "J": 44,
+      "K": 45,
+      "L": 46,
+      "M": 47,
+      "N": 48,
+      "O": 49,
+      "P": 50,
+      "Q": 51,
+      "R": 52,
+      "S": 53,
+      "T": 54,
+      "U": 55,
+      "V": 56,
+      "W": 57,
+      "X": 58,
+      "Y": 59,
+      "Z": 60,
+      "[": 61,
+      "\\": 62,
+      "]": 63,
+      "^": 64,
+      "_": 65,
+      "`": 66,
+      "a": 67,
+      "b": 68,
+      "c": 69,
+      "d": 70,
+      "e": 71,
+      "f": 72,
+      "g": 73,
+      "h": 74,
+      "i": 75,
+      "j": 76,
+      "k": 77,
+      "l": 78,
+      "m": 79,
+      "n": 80,
+      "o": 81,
+      "p": 82,
+      "q": 83,
+      "r": 84,
+      "s": 85,
+      "t": 86,
+      "u": 87,
+      "v": 88,
+      "w": 89,
+      "x": 90,
+      "y": 91,
+      "z": 92,
+      "{": 93,
+      "|": 94,
+      "}": 95,
+      "~": 96,
+      "¡": 97,
+      "¢": 98,
+      "£": 99,
+      "¤": 100,
+      "¥": 101,
+      "¦": 102,
+      "§": 103,
+      "¨": 104,
+      "©": 105,
+      "ª": 106,
+      "«": 107,
+      "¬": 108,
+      "®": 109,
+      "¯": 110,
+      "°": 111,
+      "±": 112,
+      "²": 113,
+      "³": 114,
+      "´": 115,
+      "µ": 116,
+      "¶": 117,
+      "·": 118,
+      "¸": 119,
+      "¹": 120,
+      "º": 121,
+      "»": 122,
+      "¼": 123,
+      "½": 124,
+      "¾": 125,
+      "¿": 126,
+      "À": 127,
+      "Á": 128,
+      "Â": 129,
+      "Ã": 130,
+      "Ä": 131,
+      "Å": 132,
+      "Æ": 133,
+      "Ç": 134,
+      "È": 135,
+      "É": 136,
+      "Ê": 137,
+      "Ë": 138,
+      "Ì": 139,
+      "Í": 140,
+      "Î": 141,
+      "Ï": 142,
+      "Ð": 143,
+      "Ñ": 144,
+      "Ò": 145,
+      "Ó": 146,
+      "Ô": 147,
+      "Õ": 148,
+      "Ö": 149,
+      "×": 150,
+      "Ø": 151,
+      "Ù": 152,
+      "Ú": 153,
+      "Û": 154,
+      "Ü": 155,
+      "Ý": 156,
+      "Þ": 157,
+      "ß": 158,
+      "à": 159,
+      "á": 160,
+      "â": 161,
+      "ã": 162,
+      "ä": 163,
+      "å": 164,
+      "æ": 165,
+      "ç": 166,
+      "è": 167,
+      "é": 168,
+      "ê": 169,
+      "ë": 170,
+      "ì": 171,
+      "í": 172,
+      "î": 173,
+      "ï": 174,
+      "ð": 175,
+      "ñ": 176,
+      "ò": 177,
+      "ó": 178,
+      "ô": 179,
+      "õ": 180,
+      "ö": 181,
+      "÷": 182,
+      "ø": 183,
+      "ù": 184,
+      "ú": 185,
+      "û": 186,
+      "ü": 187,
+      "ý": 188,
+      "þ": 189,
+      "ÿ": 190,
+      "Ā": 191,
+      "ā": 192,
+      "Ă": 193,
+      "ă": 194,
+      "Ą": 195,
+      "ą": 196,
+      "Ć": 197,
+      "ć": 198,
+      "Ĉ": 199,
+      "ĉ": 200,
+      "Ċ": 201,
+      "ċ": 202,
+      "Č": 203,
+      "č": 204,
+      "Ď": 205,
+      "ď": 206,
+      "Đ": 207,
+      "đ": 208,
+      "Ē": 209,
+      "ē": 210,
+      "Ĕ": 211,
+      "ĕ": 212,
+      "Ė": 213,
+      "ė": 214,
+      "Ę": 215,
+      "ę": 216,
+      "Ě": 217,
+      "ě": 218,
+      "Ĝ": 219,
+      "ĝ": 220,
+      "Ğ": 221,
+      "ğ": 222,
+      "Ġ": 223,
+      "ġ": 224,
+      "Ģ": 225,
+      "ģ": 226,
+      "Ĥ": 227,
+      "ĥ": 228,
+      "Ħ": 229,
+      "ħ": 230,
+      "Ĩ": 231,
+      "ĩ": 232,
+      "Ī": 233,
+      "ī": 234,
+      "Ĭ": 235,
+      "ĭ": 236,
+      "Į": 237,
+      "į": 238,
+      "İ": 239,
+      "ı": 240,
+      "IJ": 241,
+      "ij": 242,
+      "Ĵ": 243,
+      "ĵ": 244,
+      "Ķ": 245,
+      "ķ": 246,
+      "ĸ": 247,
+      "Ĺ": 248,
+      "ĺ": 249,
+      "Ļ": 250,
+      "ļ": 251,
+      "Ľ": 252,
+      "ľ": 253,
+      "Ŀ": 254,
+      "ŀ": 255,
+      "Ł": 256,
+      "ł": 257,
+      "Ń": 258,
+      "åį": 259,
+      "çĻ": 260,
+      "åįģ": 261,
+      "çĻ¾": 262,
+      "ä¸": 263,
+      "äº": 264,
+      "åħ": 265,
+      "¹Ŀ": 266,
+      "ä¹Ŀ": 267,
+      "åĽ": 268,
+      "ä¸ĥ": 269,
+      "ä¸ī": 270,
+      "äºĮ": 271,
+      "äºĶ": 272,
+      "åħ«": 273,
+      "åħŃ": 274,
+      "åĽĽ": 275,
+      "åįĥ": 276,
+      "ä¸Ģ": 277,
+      "ä¹Ŀåįģ": 278,
+      "ä¹ĿçĻ¾": 279,
+      "ä¸ĥåįģ": 280,
+      "ä¸ĥçĻ¾": 281,
+      "ä¸īåįģ": 282,
+      "ä¸īçĻ¾": 283,
+      "äºĮåįģ": 284,
+      "äºĮçĻ¾": 285,
+      "äºĶåįģ": 286,
+      "äºĶçĻ¾": 287,
+      "åħ«åįģ": 288,
+      "åħ«çĻ¾": 289,
+      "åħŃåįģ": 290,
+      "åħŃçĻ¾": 291,
+      "åĽĽåįģ": 292,
+      "åĽĽçĻ¾": 293,
+      "åįĥçĻ¾": 294,
+      "åįĥäºĮçĻ¾": 295,
+      "åįĥä¸īçĻ¾": 296,
+      "åįĥåĽĽçĻ¾": 297,
+      "åįĥäºĶçĻ¾": 298,
+      "åįĥåħŃçĻ¾": 299,
+      "åįģä¹Ŀ": 300,
+      "åįģä¸ĥ": 301,
+      "åįģä¸ī": 302,
+      "åįģäºĮ": 303,
+      "åįģäºĶ": 304,
+      "åįģåħ«": 305,
+      "åįģåħŃ": 306,
+      "åįģåĽĽ": 307,
+      "åįģä¸Ģ": 308,
+      "ä¹Ŀåįģä¹Ŀ": 309,
+      "ä¹Ŀåįģä¸ĥ": 310,
+      "ä¹Ŀåįģä¸ī": 311,
+      "ä¹ĿåįģäºĮ": 312,
+      "ä¹ĿåįģäºĶ": 313,
+      "ä¹Ŀåįģåħ«": 314,
+      "ä¹ĿåįģåħŃ": 315,
+      "ä¹ĿåįģåĽĽ": 316,
+      "ä¹Ŀåįģä¸Ģ": 317,
+      "ä¹ĿçĻ¾ä¸ĥåįģ": 318,
+      "ä¹ĿçĻ¾ä¸īåįģ": 319,
+      "ä¹ĿçĻ¾äºĮåįģ": 320,
+      "ä¹ĿçĻ¾äºĶåįģ": 321,
+      "ä¹ĿçĻ¾åħ«åįģ": 322,
+      "ä¹ĿçĻ¾åħŃåįģ": 323,
+      "ä¹ĿçĻ¾åĽĽåįģ": 324,
+      "ä¸ĥçĻ¾ä¸ĥåįģ": 325,
+      "ä¸ĥçĻ¾ä¸īåįģ": 326,
+      "ä¸ĥçĻ¾äºĮåįģ": 327,
+      "ä¸ĥçĻ¾äºĶåįģ": 328,
+      "ä¸ĥçĻ¾åħ«åįģ": 329,
+      "ä¸ĥçĻ¾åħŃåįģ": 330,
+      "ä¸ĥçĻ¾åĽĽåįģ": 331,
+      "åħ«çĻ¾ä¸ĥåįģ": 332,
+      "åħ«çĻ¾ä¸īåįģ": 333,
+      "åħ«çĻ¾äºĮåįģ": 334,
+      "åħ«çĻ¾äºĶåįģ": 335,
+      "åħ«çĻ¾åħ«åįģ": 336,
+      "åħ«çĻ¾åħŃåįģ": 337,
+      "åħ«çĻ¾åĽĽåįģ": 338,
+      "åħŃçĻ¾åħ«åįģ": 339,
+      "åħŃçĻ¾ä¸ĥåįģ": 340,
+      "åħŃçĻ¾åħŃåįģ": 341,
+      "åħŃçĻ¾äºĶåįģ": 342,
+      "åħŃçĻ¾åĽĽåįģ": 343,
+      "åħŃçĻ¾ä¸īåįģ": 344,
+      "åħŃçĻ¾äºĮåįģ": 345,
+      "äºĶçĻ¾åħ«åįģ": 346,
+      "äºĶçĻ¾ä¸ĥåįģ": 347,
+      "äºĶçĻ¾åħŃåįģ": 348,
+      "äºĶçĻ¾äºĶåįģ": 349,
+      "äºĶçĻ¾åĽĽåįģ": 350,
+      "äºĶçĻ¾ä¸īåįģ": 351,
+      "äºĶçĻ¾äºĮåįģ": 352,
+      "åĽĽçĻ¾åħ«åįģ": 353,
+      "åĽĽçĻ¾ä¸ĥåįģ": 354,
+      "åĽĽçĻ¾åħŃåįģ": 355,
+      "åĽĽçĻ¾äºĶåįģ": 356,
+      "åĽĽçĻ¾åĽĽåįģ": 357,
+      "åĽĽçĻ¾ä¸īåįģ": 358,
+      "åĽĽçĻ¾äºĮåįģ": 359
+    },
+    "merges": [
+      [
+        "å",
+        "į"
+      ],
+      [
+        "ç",
+        "Ļ"
+      ],
+      [
+        "åį",
+        "ģ"
+      ],
+      [
+        "çĻ",
+        "¾"
+      ],
+      [
+        "ä",
+        "¸"
+      ],
+      [
+        "ä",
+        "º"
+      ],
+      [
+        "å",
+        "ħ"
+      ],
+      [
+        "¹",
+        "Ŀ"
+      ],
+      [
+        "ä",
+        "¹Ŀ"
+      ],
+      [
+        "å",
+        "Ľ"
+      ],
+      [
+        "ä¸",
+        "ĥ"
+      ],
+      [
+        "ä¸",
+        "ī"
+      ],
+      [
+        "äº",
+        "Į"
+      ],
+      [
+        "äº",
+        "Ķ"
+      ],
+      [
+        "åħ",
+        "«"
+      ],
+      [
+        "åħ",
+        "Ń"
+      ],
+      [
+        "åĽ",
+        "Ľ"
+      ],
+      [
+        "åį",
+        "ĥ"
+      ],
+      [
+        "ä¸",
+        "Ģ"
+      ],
+      [
+        "ä¹Ŀ",
+        "åįģ"
+      ],
+      [
+        "ä¹Ŀ",
+        "çĻ¾"
+      ],
+      [
+        "ä¸ĥ",
+        "åįģ"
+      ],
+      [
+        "ä¸ĥ",
+        "çĻ¾"
+      ],
+      [
+        "ä¸ī",
+        "åįģ"
+      ],
+      [
+        "ä¸ī",
+        "çĻ¾"
+      ],
+      [
+        "äºĮ",
+        "åįģ"
+      ],
+      [
+        "äºĮ",
+        "çĻ¾"
+      ],
+      [
+        "äºĶ",
+        "åįģ"
+      ],
+      [
+        "äºĶ",
+        "çĻ¾"
+      ],
+      [
+        "åħ«",
+        "åįģ"
+      ],
+      [
+        "åħ«",
+        "çĻ¾"
+      ],
+      [
+        "åħŃ",
+        "åįģ"
+      ],
+      [
+        "åħŃ",
+        "çĻ¾"
+      ],
+      [
+        "åĽĽ",
+        "åįģ"
+      ],
+      [
+        "åĽĽ",
+        "çĻ¾"
+      ],
+      [
+        "åįĥ",
+        "çĻ¾"
+      ],
+      [
+        "åįĥ",
+        "äºĮçĻ¾"
+      ],
+      [
+        "åįĥ",
+        "ä¸īçĻ¾"
+      ],
+      [
+        "åįĥ",
+        "åĽĽçĻ¾"
+      ],
+      [
+        "åįĥ",
+        "äºĶçĻ¾"
+      ],
+      [
+        "åįĥ",
+        "åħŃçĻ¾"
+      ],
+      [
+        "åįģ",
+        "ä¹Ŀ"
+      ],
+      [
+        "åįģ",
+        "ä¸ĥ"
+      ],
+      [
+        "åįģ",
+        "ä¸ī"
+      ],
+      [
+        "åįģ",
+        "äºĮ"
+      ],
+      [
+        "åįģ",
+        "äºĶ"
+      ],
+      [
+        "åįģ",
+        "åħ«"
+      ],
+      [
+        "åįģ",
+        "åħŃ"
+      ],
+      [
+        "åįģ",
+        "åĽĽ"
+      ],
+      [
+        "åįģ",
+        "ä¸Ģ"
+      ],
+      [
+        "ä¹Ŀåįģ",
+        "ä¹Ŀ"
+      ],
+      [
+        "ä¹Ŀåįģ",
+        "ä¸ĥ"
+      ],
+      [
+        "ä¹Ŀåįģ",
+        "ä¸ī"
+      ],
+      [
+        "ä¹Ŀåįģ",
+        "äºĮ"
+      ],
+      [
+        "ä¹Ŀåįģ",
+        "äºĶ"
+      ],
+      [
+        "ä¹Ŀåįģ",
+        "åħ«"
+      ],
+      [
+        "ä¹Ŀåįģ",
+        "åħŃ"
+      ],
+      [
+        "ä¹Ŀåįģ",
+        "åĽĽ"
+      ],
+      [
+        "ä¹Ŀåįģ",
+        "ä¸Ģ"
+      ],
+      [
+        "ä¹ĿçĻ¾",
+        "ä¸ĥåįģ"
+      ],
+      [
+        "ä¹ĿçĻ¾",
+        "ä¸īåįģ"
+      ],
+      [
+        "ä¹ĿçĻ¾",
+        "äºĮåįģ"
+      ],
+      [
+        "ä¹ĿçĻ¾",
+        "äºĶåįģ"
+      ],
+      [
+        "ä¹ĿçĻ¾",
+        "åħ«åįģ"
+      ],
+      [
+        "ä¹ĿçĻ¾",
+        "åħŃåįģ"
+      ],
+      [
+        "ä¹ĿçĻ¾",
+        "åĽĽåįģ"
+      ],
+      [
+        "ä¸ĥçĻ¾",
+        "ä¸ĥåįģ"
+      ],
+      [
+        "ä¸ĥçĻ¾",
+        "ä¸īåįģ"
+      ],
+      [
+        "ä¸ĥçĻ¾",
+        "äºĮåįģ"
+      ],
+      [
+        "ä¸ĥçĻ¾",
+        "äºĶåįģ"
+      ],
+      [
+        "ä¸ĥçĻ¾",
+        "åħ«åįģ"
+      ],
+      [
+        "ä¸ĥçĻ¾",
+        "åħŃåįģ"
+      ],
+      [
+        "ä¸ĥçĻ¾",
+        "åĽĽåįģ"
+      ],
+      [
+        "åħ«çĻ¾",
+        "ä¸ĥåįģ"
+      ],
+      [
+        "åħ«çĻ¾",
+        "ä¸īåįģ"
+      ],
+      [
+        "åħ«çĻ¾",
+        "äºĮåįģ"
+      ],
+      [
+        "åħ«çĻ¾",
+        "äºĶåįģ"
+      ],
+      [
+        "åħ«çĻ¾",
+        "åħ«åįģ"
+      ],
+      [
+        "åħ«çĻ¾",
+        "åħŃåįģ"
+      ],
+      [
+        "åħ«çĻ¾",
+        "åĽĽåįģ"
+      ],
+      [
+        "åħŃçĻ¾",
+        "åħ«åįģ"
+      ],
+      [
+        "åħŃçĻ¾",
+        "ä¸ĥåįģ"
+      ],
+      [
+        "åħŃçĻ¾",
+        "åħŃåįģ"
+      ],
+      [
+        "åħŃçĻ¾",
+        "äºĶåįģ"
+      ],
+      [
+        "åħŃçĻ¾",
+        "åĽĽåįģ"
+      ],
+      [
+        "åħŃçĻ¾",
+        "ä¸īåįģ"
+      ],
+      [
+        "åħŃçĻ¾",
+        "äºĮåįģ"
+      ],
+      [
+        "äºĶçĻ¾",
+        "åħ«åįģ"
+      ],
+      [
+        "äºĶçĻ¾",
+        "ä¸ĥåįģ"
+      ],
+      [
+        "äºĶçĻ¾",
+        "åħŃåįģ"
+      ],
+      [
+        "äºĶçĻ¾",
+        "äºĶåįģ"
+      ],
+      [
+        "äºĶçĻ¾",
+        "åĽĽåįģ"
+      ],
+      [
+        "äºĶçĻ¾",
+        "ä¸īåįģ"
+      ],
+      [
+        "äºĶçĻ¾",
+        "äºĮåįģ"
+      ],
+      [
+        "åĽĽçĻ¾",
+        "åħ«åįģ"
+      ],
+      [
+        "åĽĽçĻ¾",
+        "ä¸ĥåįģ"
+      ],
+      [
+        "åĽĽçĻ¾",
+        "åħŃåįģ"
+      ],
+      [
+        "å��ĽçĻ¾",
+        "äºĶåįģ"
+      ],
+      [
+        "åĽĽçĻ¾",
+        "åĽĽåįģ"
+      ],
+      [
+        "åĽĽçĻ¾",
+        "ä¸īåįģ"
+      ],
+      [
+        "åĽĽçĻ¾",
+        "äºĮåįģ"
+      ]
+    ]
+  }
+}

tokenizer_config.json

@@ -0,0 +1,37 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "unk_token": null,
+  "number_handling": "ltr_3digit"
+}

vocab.json

@@ -0,0 +1,362 @@
+{
+  "¸": 119,
+  "s": 85,
+  "ä¸ĥ": 269,
+  "]": 63,
+  "åħŃåįģ": 290,
+  "ì": 171,
+  "ñ": 176,
+  "åįģä¹Ŀ": 300,
+  "¹Ŀ": 266,
+  "I": 43,
+  "ä¸ĥåįģ": 280,
+  "¼": 123,
+  "ä¸ĥçĻ¾ä¸īåįģ": 326,
+  "åħ«çĻ¾åħŃåįģ": 337,
+  "º": 121,
+  "4": 22,
+  "#": 5,
+  "S": 53,
+  "n": 80,
+  "F": 40,
+  "åįģä¸ĥ": 301,
+  "%": 7,
+  "İ": 239,
+  "Ü": 155,
+  "þ": 189,
+  "\\": 62,
+  "ä¹Ŀåįģåħ«": 314,
+  "+": 13,
+  "åįģä¸Ģ": 308,
+  "ê": 169,
+  "ä¸ĥçĻ¾äºĮåįģ": 327,
+  "§": 103,
+  "a": 67,
+  "ø": 183,
+  "åħŃ": 274,
+  "E": 39,
+  "ī": 234,
+  "X": 58,
+  "çĻ": 260,
+  "åį": 259,
+  "±": 112,
+  "³": 114,
+  "µ": 116,
+  "ĥ": 228,
+  "ä¹ĿçĻ¾": 279,
+  "ć": 198,
+  "¥": 101,
+  "«": 107,
+  "Ě": 217,
+  "åĽĽçĻ¾äºĶåįģ": 356,
+  "Q": 51,
+  "äºĮåįģ": 284,
+  "åįĥ": 276,
+  "<": 30,
+  "Ð": 143,
+  "Ñ": 144,
+  "å": 164,
+  "åįģ": 261,
+  "J": 44,
+  "¦": 102,
+  "Į": 237,
+  "5": 23,
+  "Č": 203,
+  ".": 16,
+  "ù": 184,
+  "åįĥçĻ¾": 294,
+  "ú": 185,
+  "Ĩ": 231,
+  "ó": 178,
+  "åħ«çĻ¾åħ«åįģ": 336,
+  "Û": 154,
+  "Ĉ": 199,
+  "ä¹Ŀåįģä¸ī": 311,
+  "äºĶçĻ¾ä¸īåįģ": 351,
+  "ä¸īåįģ": 282,
+  "~": 96,
+  "ä¸ĥçĻ¾åħŃåįģ": 330,
+  "Ā": 191,
+  "äºĶçĻ¾äºĮåįģ": 352,
+  "åįģåħŃ": 306,
+  "c": 69,
+  "L": 46,
+  "åħ«çĻ¾ä¸īåįģ": 333,
+  "A": 35,
+  "Ã": 130,
+  "p": 82,
+  "!": 3,
+  "T": 54,
+  "ä¸ĥçĻ¾äºĶåįģ": 328,
+  "»": 122,
+  "&": 8,
+  "¹": 120,
+  "ĺ": 249,
+  "ç": 166,
+  "åĽĽåįģ": 292,
+  "åħ": 265,
+  "Ä": 131,
+  "Ê": 137,
+  "e": 71,
+  "Ķ": 245,
+  "$": 6,
+  "3": 21,
+  "À": 127,
+  "ä¹Ŀåįģä¹Ŀ": 309,
+  "ķ": 246,
+  "ę": 216,
+  "åħ«åįģ": 288,
+  "¿": 126,
+  "äºĶçĻ¾åħ«åįģ": 346,
+  "ÿ": 190,
+  "ü": 187,
+  "i": 75,
+  "ª": 106,
+  "ë": 170,
+  "?": 33,
+  ">": 32,
+  "åħ«": 273,
+  "ä¹ĿåįģäºĮ": 312,
+  "ħ": 230,
+  "W": 57,
+  "ā": 192,
+  "Ģ": 225,
+  "ý": 188,
+  "Ė": 213,
+  "Ý": 156,
+  "ĕ": 212,
+  "åĽĽçĻ¾åĽĽåįģ": 357,
+  "Ĵ": 243,
+  "q": 83,
+  "}": 95,
+  "ä¸ĥçĻ¾ä¸ĥåįģ": 325,
+  "åħ«çĻ¾äºĶåįģ": 335,
+  "ĸ": 247,
+  "1": 19,
+  "ð": 175,
+  "í": 172,
+  "Æ": 133,
+  "ä¹Ŀåįģ": 278,
+  "/": 17,
+  "0": 18,
+  "åħŃçĻ¾": 291,
+  "ä¹ĿåįģäºĶ": 313,
+  "đ": 208,
+  "Ë": 138,
+  "äºĶçĻ¾äºĶåįģ": 349,
+  "b": 68,
+  "ä¸īçĻ¾": 283,
+  "h": 74,
+  "(": 10,
+  "Ô": 147,
+  "u": 87,
+  "ä¸ĥçĻ¾åħ«åįģ": 329,
+  "ă": 194,
+  "<pad>": 2,
+  "6": 24,
+  ";": 29,
+  "äº": 264,
+  "Á": 128,
+  "Õ": 148,
+  "Í": 140,
+  "åĽĽçĻ¾åħ«åįģ": 353,
+  "^": 64,
+  "_": 65,
+  "ä¹ĿçĻ¾ä¸ĥåįģ": 318,
+  "äºĶçĻ¾": 287,
+  "ä¹ĿçĻ¾äºĶåįģ": 321,
+  "äºĶçĻ¾åħŃåįģ": 348,
+  "£": 99,
+  "·": 118,
+  "ß": 158,
+  "ł": 257,
+  "¾": 125,
+  "k": 77,
+  "o": 81,
+  "åħŃçĻ¾åĽĽåįģ": 343,
+  "Ē": 209,
+  "î": 173,
+  "Ĺ": 248,
+  "à": 159,
+  "8": 26,
+  "ä¹ĿçĻ¾åĽĽåįģ": 324,
+  "¶": 117,
+  "åįĥåĽĽçĻ¾": 297,
+  "æ": 165,
+  "åĽĽçĻ¾": 293,
+  "B": 36,
+  "ä¹Ŀåįģä¸ĥ": 310,
+  "ô": 179,
+  "-": 15,
+  "¨": 104,
+  "Đ": 207,
+  "åįĥä¸īçĻ¾": 296,
+  "Ĭ": 235,
+  "O": 49,
+  "Â": 129,
+  "ē": 210,
+  "2": 20,
+  "ļ": 251,
+  "åħ«çĻ¾ä¸ĥåįģ": 332,
+  "x": 90,
+  "Ń": 258,
+  "9": 27,
+  "¤": 100,
+  ":": 28,
+  "äºĶçĻ¾ä¸ĥåįģ": 347,
+  "R": 52,
+  "</s>": 1,
+  "ä¹ĿçĻ¾äºĮåįģ": 320,
+  "m": 79,
+  "t": 86,
+  "ä¹Ŀ": 267,
+  "Ì": 139,
+  "´": 115,
+  "©": 105,
+  "N": 48,
+  "`": 66,
+  "Ġ": 223,
+  "ı": 240,
+  "ä¸ĥçĻ¾": 281,
+  "z": 92,
+  "D": 38,
+  "U": 55,
+  "¡": 97,
+  "Ó": 146,
+  "Z": 60,
+  "ò": 177,
+  "ä¹Ŀåįģä¸Ģ": 317,
+  "Y": 59,
+  "åħŃçĻ¾åħ«åįģ": 339,
+  "ą": 196,
+  "åįģäºĮ": 303,
+  "åħŃçĻ¾äºĮåįģ": 345,
+  "åĽĽçĻ¾ä¸ĥåįģ": 354,
+  "²": 113,
+  "¬": 108,
+  "Ę": 215,
+  "Ċ": 201,
+  "=": 31,
+  "åħ«çĻ¾äºĮåįģ": 334,
+  "Ă": 193,
+  "åįģä¸ī": 302,
+  "åħ«çĻ¾": 289,
+  "ä¸ĥçĻ¾åĽĽåįģ": 331,
+  "×": 150,
+  "Ļ": 250,
+  "Ĥ": 227,
+  "C": 37,
+  "á": 160,
+  "ä": 163,
+  "Ł": 256,
+  "Ù": 152,
+  "j": 76,
+  "ö": 181,
+  "@": 34,
+  "é": 168,
+  "[": 61,
+  "¢": 98,
+  "ä¹ĿçĻ¾åħ«åįģ": 322,
+  "â": 161,
+  "Å": 132,
+  "ä¸Ģ": 277,
+  "åįĥäºĶçĻ¾": 298,
+  "|": 94,
+  "\"": 4,
+  "Ú": 153,
+  "½": 124,
+  "äºĮ": 271,
+  "äºĶçĻ¾åĽĽåįģ": 350,
+  "d": 70,
+  "åħŃçĻ¾ä¸ĥåįģ": 340,
+  "P": 50,
+  "åĽĽçĻ¾äºĮåįģ": 359,
+  "ä¸": 263,
+  "*": 12,
+  "åĽ": 268,
+  "Ø": 151,
+  "w": 89,
+  "ä¹ĿåįģåĽĽ": 316,
+  "÷": 182,
+  "¯": 110,
+  "ċ": 202,
+  "Ĕ": 211,
+  ")": 11,
+  "ĩ": 232,
+  "ï": 174,
+  "åĽĽçĻ¾ä¸īåįģ": 358,
+  "Ğ": 221,
+  "Ī": 233,
+  "G": 41,
+  "ď": 206,
+  "ġ": 224,
+  "õ": 180,
+  "r": 84,
+  "ŀ": 255,
+  "y": 91,
+  "V": 56,
+  "ĉ": 200,
+  "åįģåĽĽ": 307,
+  "åĽĽ": 275,
+  "v": 88,
+  "Ĝ": 219,
+  "ä¸ī": 270,
+  "Þ": 157,
+  "Ď": 205,
+  "Ï": 142,
+  "äºĶ": 272,
+  "Ŀ": 254,
+  "è": 167,
+  "Ľ": 252,
+  "Ħ": 229,
+  "ė": 214,
+  "åħ«çĻ¾åĽĽåįģ": 338,
+  "ě": 218,
+  "{": 93,
+  "ã": 162,
+  "ij": 242,
+  "åħŃçĻ¾åħŃåįģ": 341,
+  "K": 45,
+  "åįĥäºĮçĻ¾": 295,
+  "Ą": 195,
+  ",": 14,
+  "ä¹ĿçĻ¾åħŃåįģ": 323,
+  "äºĮçĻ¾": 285,
+  "åħŃçĻ¾ä¸īåįģ": 344,
+  "äºĶåįģ": 286,
+  "g": 73,
+  "IJ": 241,
+  "7": 25,
+  "f": 72,
+  "č": 204,
+  "ğ": 222,
+  "H": 42,
+  "ľ": 253,
+  "Ć": 197,
+  "ĝ": 220,
+  "Ö": 149,
+  "åįģäºĶ": 304,
+  "®": 109,
+  "ģ": 226,
+  "ĵ": 244,
+  "'": 9,
+  "åĽĽçĻ¾åħŃåįģ": 355,
+  "åįĥåħŃçĻ¾": 299,
+  "È": 135,
+  "Ç": 134,
+  "ĭ": 236,
+  "åįģåħ«": 305,
+  "l": 78,
+  "ä¹ĿçĻ¾ä¸īåįģ": 319,
+  "û": 186,
+  "į": 238,
+  "°": 111,
+  "åħŃçĻ¾äºĶåįģ": 342,
+  "çĻ¾": 262,
+  "Ò": 145,
+  "ä¹ĿåįģåħŃ": 315,
+  "<s>": 0,
+  "M": 47,
+  "É": 136,
+  "Î": 141
+}