savi2006 commited on
Commit
93b5e09
·
verified ·
1 Parent(s): 66d3f6a

Update README.md

Browse files
Files changed (1) hide show
  1. README.md +15 -12
README.md CHANGED
@@ -71,23 +71,29 @@ Translates informal Sri Lankan code-mixed text written in Roman script into Sinh
71
 
72
  ### CTranslate2 (recommended)
73
  ``` python
 
74
  import ctranslate2
75
  from transformers import AutoTokenizer
76
 
 
 
 
77
  translator = ctranslate2.Translator(
78
- "savinugunarathna/small-100-Singlish-Sinhala-CodeMix2-CT2",
79
  device="cpu",
80
  compute_type="int8",
81
  intra_threads=4,
82
  )
 
83
  tokenizer = AutoTokenizer.from_pretrained(
84
  "savinugunarathna/small-100-Singlish-Sinhala-CodeMix2-CT2"
85
  )
86
 
87
  def translate(text: str) -> str:
88
  tokenizer.src_lang = "en"
89
- ids = tokenizer.encode(text, add_special_tokens=True)
90
- tokens = tokenizer.convert_ids_to_tokens(ids)
 
91
 
92
  result = translator.translate_batch(
93
  [tokens],
@@ -97,17 +103,14 @@ def translate(text: str) -> str:
97
  repetition_penalty=1.2,
98
  )
99
 
100
- hyp = result[0].hypotheses[0]
101
- if hyp and hyp[0] == "__si__":
102
- hyp = hyp[1:]
103
- if hyp and hyp[-1] == "</s>":
104
- hyp = hyp[:-1]
105
-
106
- hyp_ids = tokenizer.convert_tokens_to_ids(hyp)
107
  return tokenizer.decode(hyp_ids, skip_special_tokens=True).strip()
108
 
109
- print(translate("mama heta school yanne nehe"))
110
- # → මම හෙට පාසලට යන්නේ නෑ
 
 
 
111
  ```
112
 
113
  ### FastAPI REST API
 
71
 
72
  ### CTranslate2 (recommended)
73
  ``` python
74
+ from huggingface_hub import snapshot_download
75
  import ctranslate2
76
  from transformers import AutoTokenizer
77
 
78
+ model_path = snapshot_download("savinugunarathna/small-100-Singlish-Sinhala-CodeMix2-CT2")
79
+ print(f"✓ Downloaded to: {model_path}")
80
+
81
  translator = ctranslate2.Translator(
82
+ model_path,
83
  device="cpu",
84
  compute_type="int8",
85
  intra_threads=4,
86
  )
87
+
88
  tokenizer = AutoTokenizer.from_pretrained(
89
  "savinugunarathna/small-100-Singlish-Sinhala-CodeMix2-CT2"
90
  )
91
 
92
  def translate(text: str) -> str:
93
  tokenizer.src_lang = "en"
94
+ tokens = tokenizer.convert_ids_to_tokens(
95
+ tokenizer.encode(text, add_special_tokens=True)
96
+ )
97
 
98
  result = translator.translate_batch(
99
  [tokens],
 
103
  repetition_penalty=1.2,
104
  )
105
 
106
+ hyp_ids = tokenizer.convert_tokens_to_ids(result[0].hypotheses[0])
 
 
 
 
 
 
107
  return tokenizer.decode(hyp_ids, skip_special_tokens=True).strip()
108
 
109
+ while True:
110
+ text = input("\nSinglish: ").strip()
111
+ if text.lower() == "q":
112
+ break
113
+ print(f"සිංහල: {translate(text)}")
114
  ```
115
 
116
  ### FastAPI REST API