zxdu20 commited on
Commit
c4575e7
1 Parent(s): fc93714

Update tokenizer

Browse files
Files changed (1) hide show
  1. tokenization_chatglm.py +1 -3
tokenization_chatglm.py CHANGED
@@ -15,8 +15,6 @@ from transformers.utils import logging
15
 
16
  logger = logging.get_logger(__name__)
17
 
18
- VOCAB_FILES_NAMES = {"vocab_file": "ice_text.model"}
19
-
20
  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
21
  "THUDM/chatglm-6b": 2048,
22
  }
@@ -179,7 +177,7 @@ class ChatGLMTokenizer(PreTrainedTokenizer):
179
  Path to the vocabulary file.
180
  """
181
 
182
- vocab_files_names = VOCAB_FILES_NAMES
183
  max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
184
  model_input_names = ["input_ids"]
185
 
 
15
 
16
  logger = logging.get_logger(__name__)
17
 
 
 
18
  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
19
  "THUDM/chatglm-6b": 2048,
20
  }
 
177
  Path to the vocabulary file.
178
  """
179
 
180
+ vocab_files_names = {"vocab_file": "ice_text.model"}
181
  max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
182
  model_input_names = ["input_ids"]
183