|
from datasets import load_dataset |
|
from tokenizers import ByteLevelBPETokenizer |
|
|
|
|
|
kopi = load_dataset("/data/final_train.py", "full",split='train',cache_dir="/data/cache") |
|
|
|
datasetv2 = kopi.shuffle(seed=42) |
|
dataset = datasetv2[0:8000000] |
|
|
|
|
|
tokenizer = ByteLevelBPETokenizer() |
|
def batch_iterator(batch_size=100_000): |
|
for i in range(0, len(dataset), batch_size): |
|
yield dataset["text"][i: i + batch_size] |
|
|
|
|
|
tokenizer.train_from_iterator(batch_iterator(), vocab_size=50265, min_frequency=2, special_tokens=[ |
|
"<s>", |
|
"<pad>", |
|
"</s>", |
|
"<unk>", |
|
"<mask>", |
|
]) |
|
|
|
tokenizer.save("./tokenizer.json") |
