puffy310 commited on
Commit
bfd7b19
1 Parent(s): 8b7d335

Update tokenization_chatglm.py

Browse files
Files changed (1) hide show
  1. tokenization_chatglm.py +1 -2
tokenization_chatglm.py CHANGED
@@ -193,6 +193,7 @@ class ChatGLMTokenizer(PreTrainedTokenizer):
193
  num_image_tokens=20000,
194
  **kwargs
195
  ) -> None:
 
196
  super().__init__(
197
  do_lower_case=do_lower_case,
198
  remove_space=remove_space,
@@ -218,8 +219,6 @@ class ChatGLMTokenizer(PreTrainedTokenizer):
218
  self.mask_token = mask_token
219
  self.gmask_token = gmask_token
220
 
221
- self.sp_tokenizer = SPTokenizer(vocab_file, num_image_tokens=num_image_tokens)
222
-
223
  """ Initialisation """
224
 
225
  @property
 
193
  num_image_tokens=20000,
194
  **kwargs
195
  ) -> None:
196
+ self.sp_tokenizer = SPTokenizer(vocab_file, num_image_tokens=num_image_tokens)
197
  super().__init__(
198
  do_lower_case=do_lower_case,
199
  remove_space=remove_space,
 
219
  self.mask_token = mask_token
220
  self.gmask_token = gmask_token
221
 
 
 
222
  """ Initialisation """
223
 
224
  @property