Question regarding multiprocessing about someweta HOT 2 CLOSED

#!/usr/bin/env python3

import multiprocessing
import threading

from someweta import ASPTagger
import someweta.cli


def parallel_tagging(corpus, asptagger, parallel, xml=False):
    """Tag file object `corpus` using `parallel` worker processes."""
    def output_result(data, xml):
        if xml:
            i, result, lines, word_indexes = data
            tags = {idx: (t[1],) for idx, t in zip(word_indexes, result)}
            return [(t,) + tags.get(idx, ()) for idx, t in enumerate(lines)]
        else:
            i, result = data
            return result

    sentinel = someweta.cli.Sentinel()
    processes = min(parallel, multiprocessing.cpu_count())
    input_queue = multiprocessing.Queue(maxsize=processes * 100)
    output_queue = multiprocessing.Queue(maxsize=processes * 100)
    producer = threading.Thread(target=someweta.cli.fill_input_queue, args=(input_queue, corpus, processes, sentinel, xml))
    with multiprocessing.Pool(processes=processes, initializer=someweta.cli.process_input_queue, initargs=(asptagger.tag_sentence, input_queue, output_queue, sentinel, xml)):
        producer.start()
        observed_sentinels = 0
        current = 0
        cached_results = {}
        while True:
            data = output_queue.get()
            if isinstance(data, someweta.cli.Sentinel):
                observed_sentinels += 1
                if observed_sentinels == processes:
                    break
                else:
                    continue
            i = data[0]
            cached_results[i] = data
            while current in cached_results:
                yield output_result(cached_results[current], xml)
                del cached_results[current]
                current += 1
        corpus_size = input_queue.get()
        producer.join()


tagger = ASPTagger()
tagger.load("german_web_social_media_2020-05-28.model")

with open("input.txt", encoding="utf-8") as f:
    for sentence in parallel_tagging(f, tagger, parallel=4, xml=False):
        print("\n".join(["\t".join(t) for t in sentence]))
        print()