The text-cluster from shayneobrien

text-cluster's People

Contributors

Stargazers

Watchers

text-cluster's Issues

OnlineCluster StopIteration error

streamer = StreamData(filename='../data/talk_radio.csv', chunk=500, use_column='sentences')
init_text = streamer._init_data(10)
online = OnlineCluster(text=list(init_text.processed), method='kmeans', n_clusters=10)
num_batches = 25
for _ in tqdm(range(num_batches)):
    new_text = streamer()
    labels = online.predict(list(new_text.processed))

I run this code but I got the below error

StopIteration                             Traceback (most recent call last)
<ipython-input-91-064fb5ae967b> in <module>
     22 
     23     # Get new batch
---> 24     new_text = streamer()
     25 
     26     # Predict

~/Desktop/text-cluster-master/src/modules.py in __call__(self)
     25     def __call__(self):
     26         """ Get a batch from the generator """
---> 27         return self._process(self.stream())
     28 
     29     def stream(self):

~/Desktop/text-cluster-master/src/modules.py in stream(self)
     29     def stream(self):
     30         """ Iterate generator """
---> 31         return next(self.generator)
     32 
     33     def _init_data(self, num_chunks):

~/.local/lib/python3.8/site-packages/pandas/io/parsers.py in __next__(self)
   1027     def __next__(self):
   1028         try:
-> 1029             return self.get_chunk()
   1030         except StopIteration:
   1031             self.close()

~/.local/lib/python3.8/site-packages/pandas/io/parsers.py in get_chunk(self, size)
   1077                 raise StopIteration
   1078             size = min(size, self.nrows - self._currow)
-> 1079         return self.read(nrows=size)
   1080 
   1081     def __enter__(self):

~/.local/lib/python3.8/site-packages/pandas/io/parsers.py in read(self, nrows)
   1050     def read(self, nrows=None):
   1051         nrows = validate_integer("nrows", nrows)
-> 1052         index, columns, col_dict = self._engine.read(nrows)
   1053 
   1054         if index is None:

~/.local/lib/python3.8/site-packages/pandas/io/parsers.py in read(self, nrows)
   2054     def read(self, nrows=None):
   2055         try:
-> 2056             data = self._reader.read(nrows)
   2057         except StopIteration:
   2058             if self._first_chunk:

pandas/_libs/parsers.pyx in pandas._libs.parsers.TextReader.read()

pandas/_libs/parsers.pyx in pandas._libs.parsers.TextReader._read_low_memory()

StopIteration: