Update README.md
Browse files
README.md
CHANGED
|
@@ -71,23 +71,29 @@ Translates informal Sri Lankan code-mixed text written in Roman script into Sinh
|
|
| 71 |
|
| 72 |
### CTranslate2 (recommended)
|
| 73 |
``` python
|
|
|
|
| 74 |
import ctranslate2
|
| 75 |
from transformers import AutoTokenizer
|
| 76 |
|
|
|
|
|
|
|
|
|
|
| 77 |
translator = ctranslate2.Translator(
|
| 78 |
-
|
| 79 |
device="cpu",
|
| 80 |
compute_type="int8",
|
| 81 |
intra_threads=4,
|
| 82 |
)
|
|
|
|
| 83 |
tokenizer = AutoTokenizer.from_pretrained(
|
| 84 |
"savinugunarathna/small-100-Singlish-Sinhala-CodeMix2-CT2"
|
| 85 |
)
|
| 86 |
|
| 87 |
def translate(text: str) -> str:
|
| 88 |
tokenizer.src_lang = "en"
|
| 89 |
-
|
| 90 |
-
|
|
|
|
| 91 |
|
| 92 |
result = translator.translate_batch(
|
| 93 |
[tokens],
|
|
@@ -97,17 +103,14 @@ def translate(text: str) -> str:
|
|
| 97 |
repetition_penalty=1.2,
|
| 98 |
)
|
| 99 |
|
| 100 |
-
|
| 101 |
-
if hyp and hyp[0] == "__si__":
|
| 102 |
-
hyp = hyp[1:]
|
| 103 |
-
if hyp and hyp[-1] == "</s>":
|
| 104 |
-
hyp = hyp[:-1]
|
| 105 |
-
|
| 106 |
-
hyp_ids = tokenizer.convert_tokens_to_ids(hyp)
|
| 107 |
return tokenizer.decode(hyp_ids, skip_special_tokens=True).strip()
|
| 108 |
|
| 109 |
-
|
| 110 |
-
|
|
|
|
|
|
|
|
|
|
| 111 |
```
|
| 112 |
|
| 113 |
### FastAPI REST API
|
|
|
|
| 71 |
|
| 72 |
### CTranslate2 (recommended)
|
| 73 |
``` python
|
| 74 |
+
from huggingface_hub import snapshot_download
|
| 75 |
import ctranslate2
|
| 76 |
from transformers import AutoTokenizer
|
| 77 |
|
| 78 |
+
model_path = snapshot_download("savinugunarathna/small-100-Singlish-Sinhala-CodeMix2-CT2")
|
| 79 |
+
print(f"✓ Downloaded to: {model_path}")
|
| 80 |
+
|
| 81 |
translator = ctranslate2.Translator(
|
| 82 |
+
model_path,
|
| 83 |
device="cpu",
|
| 84 |
compute_type="int8",
|
| 85 |
intra_threads=4,
|
| 86 |
)
|
| 87 |
+
|
| 88 |
tokenizer = AutoTokenizer.from_pretrained(
|
| 89 |
"savinugunarathna/small-100-Singlish-Sinhala-CodeMix2-CT2"
|
| 90 |
)
|
| 91 |
|
| 92 |
def translate(text: str) -> str:
|
| 93 |
tokenizer.src_lang = "en"
|
| 94 |
+
tokens = tokenizer.convert_ids_to_tokens(
|
| 95 |
+
tokenizer.encode(text, add_special_tokens=True)
|
| 96 |
+
)
|
| 97 |
|
| 98 |
result = translator.translate_batch(
|
| 99 |
[tokens],
|
|
|
|
| 103 |
repetition_penalty=1.2,
|
| 104 |
)
|
| 105 |
|
| 106 |
+
hyp_ids = tokenizer.convert_tokens_to_ids(result[0].hypotheses[0])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 107 |
return tokenizer.decode(hyp_ids, skip_special_tokens=True).strip()
|
| 108 |
|
| 109 |
+
while True:
|
| 110 |
+
text = input("\nSinglish: ").strip()
|
| 111 |
+
if text.lower() == "q":
|
| 112 |
+
break
|
| 113 |
+
print(f"සිංහල: {translate(text)}")
|
| 114 |
```
|
| 115 |
|
| 116 |
### FastAPI REST API
|