Hugging Face's logo

silver
/
chatglm-6b-int4-qe-slim

Feature Extraction
Transformers
PyTorch
Chinese
English
chatglm
glm
thudm
custom_code
Model card Files Community
DrSong commited on
Commit
437fc94
•
1 Parent(s): 2d2c7f4

Fix tokenizer config

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +13 -11
tokenizer_config.json CHANGED
@@ -1,17 +1,19 @@
1
  {
2
- "auto_map": {
3
- "AutoTokenizer": [
4
- "tokenization_chatglm.ChatGLMTokenizer",
5
- null
6
- ]
7
- },
8
- "do_lower_case": false,
9
- "model_max_length": 2048,
10
  "name_or_path": "THUDM/chatglm-6b",
 
 
 
 
 
11
  "pad_token": "<pad>",
12
- "padding_side": "left",
13
  "remove_space": false,
14
- "special_tokens_map_file": null,
15
  "tokenizer_class": "ChatGLMTokenizer",
16
- "unk_token": "<unk>"
 
 
 
 
 
17
  }
 
1
  {
 
 
 
 
 
 
 
 
2
  "name_or_path": "THUDM/chatglm-6b",
3
+ "bos_token": "<sop>",
4
+ "eop_token": "<eop>",
5
+ "eos_token": "</s>",
6
+ "gmask_token": "[gMASK]",
7
+ "mask_token": "[MASK]",
8
  "pad_token": "<pad>",
9
+ "unk_token": "<unk>",
10
  "remove_space": false,
11
+ "do_lower_case": false,
12
  "tokenizer_class": "ChatGLMTokenizer",
13
+ "auto_map": {
14
+ "AutoTokenizer": [
15
+ "tokenization_chatglm.ChatGLMTokenizer",
16
+ null
17
+ ]
18
+ }
19
  }