Update tokenization_qwen.py
#3
by ShushengYang - opened
- tokenization_qwen.py +5 -0
tokenization_qwen.py
CHANGED
|
@@ -25,6 +25,11 @@ from matplotlib.font_manager import FontProperties
|
|
| 25 |
|
| 26 |
logger = logging.getLogger(__name__)
|
| 27 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 28 |
|
| 29 |
VOCAB_FILES_NAMES = {"vocab_file": "qwen.tiktoken", "ttf": "SimSun.ttf"}
|
| 30 |
|
|
|
|
| 25 |
|
| 26 |
logger = logging.getLogger(__name__)
|
| 27 |
|
| 28 |
+
if not os.path.exists("SimSun.ttf"):
|
| 29 |
+
# download if not exists
|
| 30 |
+
logger.warning("SimSun font is required for Chinese display. Start downloading...")
|
| 31 |
+
print("wget https://ofasys-wlcb.oss-cn-wulanchabu.aliyuncs.com/Qwen-VL/SimSun.ttf")
|
| 32 |
+
os.system("wget https://ofasys-wlcb.oss-cn-wulanchabu.aliyuncs.com/Qwen-VL/SimSun.ttf")
|
| 33 |
|
| 34 |
VOCAB_FILES_NAMES = {"vocab_file": "qwen.tiktoken", "ttf": "SimSun.ttf"}
|
| 35 |
|