Upload tokenizer

README.md

@@ -2,7 +2,7 @@
 language:
 - en
 - ko
-pipeline_tag: text-generation
+license: other
 tags:
 - facebook
 - meta
@@ -10,7 +10,7 @@ tags:
 - llama
 - llama-3
 - llama-3-ko
-license: other
+pipeline_tag: text-generation
 license_name: llama3
 license_link: LICENSE
 ---

tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0ac333c83e2d107910928928b5912d8ade91594d08c7c73c4606d05c032d7632
-size 9084463
+oid sha256:e134af98b985517b4f068e3755ae90d4e9cd2d45d328325dc503f1c6b2d06cc7
+size 9085698

tokenizer_config.json

@@ -2050,6 +2050,7 @@
     }
   },
   "bos_token": "<|begin_of_text|>",
+  "chat_template": "{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}",
   "clean_up_tokenization_spaces": true,
   "eos_token": "<|end_of_text|>",
   "model_input_names": [