Instructions to use zai-org/codegeex2-6b with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use zai-org/codegeex2-6b with Transformers:
# Load model directly from transformers import AutoModel model = AutoModel.from_pretrained("zai-org/codegeex2-6b", trust_remote_code=True, dtype="auto") - Notebooks
- Google Colab
- Kaggle
zR commited on
Update tokenization_chatglm.py
Browse files- tokenization_chatglm.py +1 -2
tokenization_chatglm.py
CHANGED
|
@@ -66,7 +66,6 @@ class ChatGLMTokenizer(PreTrainedTokenizer):
|
|
| 66 |
model_input_names = ["input_ids", "attention_mask", "position_ids"]
|
| 67 |
|
| 68 |
def __init__(self, vocab_file, padding_side="left", clean_up_tokenization_spaces=False, **kwargs):
|
| 69 |
-
super().__init__(padding_side=padding_side, clean_up_tokenization_spaces=clean_up_tokenization_spaces, **kwargs)
|
| 70 |
self.name = "GLMTokenizer"
|
| 71 |
|
| 72 |
self.vocab_file = vocab_file
|
|
@@ -76,7 +75,7 @@ class ChatGLMTokenizer(PreTrainedTokenizer):
|
|
| 76 |
"<eos>": self.tokenizer.eos_id,
|
| 77 |
"<pad>": self.tokenizer.pad_id
|
| 78 |
}
|
| 79 |
-
|
| 80 |
def get_command(self, token):
|
| 81 |
if token in self.special_tokens:
|
| 82 |
return self.special_tokens[token]
|
|
|
|
| 66 |
model_input_names = ["input_ids", "attention_mask", "position_ids"]
|
| 67 |
|
| 68 |
def __init__(self, vocab_file, padding_side="left", clean_up_tokenization_spaces=False, **kwargs):
|
|
|
|
| 69 |
self.name = "GLMTokenizer"
|
| 70 |
|
| 71 |
self.vocab_file = vocab_file
|
|
|
|
| 75 |
"<eos>": self.tokenizer.eos_id,
|
| 76 |
"<pad>": self.tokenizer.pad_id
|
| 77 |
}
|
| 78 |
+
super().__init__(padding_side=padding_side, clean_up_tokenization_spaces=clean_up_tokenization_spaces, **kwargs)
|
| 79 |
def get_command(self, token):
|
| 80 |
if token in self.special_tokens:
|
| 81 |
return self.special_tokens[token]
|