tiktoken不支持多线程tokenize? about qwen HOT 8 CLOSED

from transformers import AutoTokenizer
from datasets import load_dataset

tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen-7B", trust_remote_code=True)
def process(example):
    ids = tokenizer.encode(example['text'])
    out = {'ids': ids, 'len': len(ids)}
    return out

dataset = load_dataset("stas/openwebtext-10k") # just an example
tokenized = dataset.map(
    process,
    remove_columns=['text'],
    desc="tokenizing the OWT splits",
    num_proc=3,
)

import os
import threading
from transformers import AutoTokenizer

tokenizer_dict = {}

def process(example):
    k = str(os.getpid()) + str(threading.get_ident())
    if k not in tokenizer_dict:
        for _ in range(100):  # try multiple times when the network is unreliable
            try:
                tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen-7B", trust_remote_code=True)
                break
            except Exception:
                pass
        tokenizer_dict[k] = tokenizer
    else:
        tokenizer = tokenizer_dict[k]
    ids = tokenizer.encode(example["text"])
    out = {"ids": ids, "len": len(ids)}
    return out