Process Process-1:
Traceback (most recent call last):
File "/usr/lib/python3.5/multiprocessing/process.py", line 249, in _bootstrap
self.run()
File "/usr/lib/python3.5/multiprocessing/process.py", line 93, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.5/dist-packages/paddle/reader/decorator.py", line 549, in _read_into_queue
for sample in reader():
File "/usr/local/lib/python3.5/dist-packages/six.py", line 703, in reraise
raise value
File "/usr/local/lib/python3.5/dist-packages/paddle/reader/decorator.py", line 549, in _read_into_queue
for sample in reader():
File "/home/pd_source/cla/ppcls/data/reader.py", line 191, in reader
for line in full_lines:
File "/home/pd_source/cla/ppcls/data/reader.py", line 191, in reader
for line in full_lines:
File "/usr/lib/python3.5/bdb.py", line 48, in trace_dispatch
return self.dispatch_line(frame)
File "/usr/lib/python3.5/bdb.py", line 67, in dispatch_line
if self.quitting: raise BdbQuit
bdb.BdbQuit
/home/pd_source/cla/ppcls/data/reader.py(191)reader()
-> for line in full_lines:
(Pdb)
/home/pd_source/cla/ppcls/data/reader.py(191)reader()
-> for line in full_lines:
Process Process-2:
(Pdb)
Traceback (most recent call last):
File "/usr/lib/python3.5/multiprocessing/process.py", line 249, in _bootstrap
self.run()
File "/usr/lib/python3.5/multiprocessing/process.py", line 93, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.5/dist-packages/paddle/reader/decorator.py", line 549, in _read_into_queue
for sample in reader():
File "/usr/local/lib/python3.5/dist-packages/six.py", line 703, in reraise
raise value
File "/usr/local/lib/python3.5/dist-packages/paddle/reader/decorator.py", line 549, in _read_into_queue
for sample in reader():
File "/home/pd_source/cla/ppcls/data/reader.py", line 191, in reader
for line in full_lines:
File "/home/pd_source/cla/ppcls/data/reader.py", line 191, in reader
for line in full_lines:
File "/usr/lib/python3.5/bdb.py", line 48, in trace_dispatch
return self.dispatch_line(frame)
File "/usr/lib/python3.5/bdb.py", line 67, in dispatch_line
if self.quitting: raise BdbQuit
bdb.BdbQuit
Process Process-1:
Traceback (most recent call last):
File "/usr/lib/python3.5/multiprocessing/process.py", line 249, in _bootstrap
self.run()
File "/usr/lib/python3.5/multiprocessing/process.py", line 93, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.5/dist-packages/paddle/reader/decorator.py", line 549, in _read_into_queue
for sample in reader():
File "/usr/local/lib/python3.5/dist-packages/six.py", line 703, in reraise
raise value
File "/usr/local/lib/python3.5/dist-packages/paddle/reader/decorator.py", line 549, in _read_into_queue
for sample in reader():
File "/home/pd_source/cla/ppcls/data/reader.py", line 191, in reader
for line in full_lines:
File "/home/pd_source/cla/ppcls/data/reader.py", line 191, in reader
for line in full_lines:
File "/usr/lib/python3.5/bdb.py", line 48, in trace_dispatch
return self.dispatch_line(frame)
File "/usr/lib/python3.5/bdb.py", line 67, in dispatch_line
if self.quitting: raise BdbQuit
bdb.BdbQuit
/home/pd_source/cla/ppcls/data/reader.py(191)reader()
-> for line in full_lines:
(Pdb)
Process Process-3:
Traceback (most recent call last):
File "/usr/lib/python3.5/multiprocessing/process.py", line 249, in _bootstrap
self.run()
File "/usr/lib/python3.5/multiprocessing/process.py", line 93, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.5/dist-packages/paddle/reader/decorator.py", line 549, in _read_into_queue
for sample in reader():
File "/usr/local/lib/python3.5/dist-packages/six.py", line 703, in reraise
raise value
File "/usr/local/lib/python3.5/dist-packages/paddle/reader/decorator.py", line 549, in _read_into_queue
for sample in reader():
File "/home/pd_source/cla/ppcls/data/reader.py", line 191, in reader
for line in full_lines:
File "/home/pd_source/cla/ppcls/data/reader.py", line 191, in reader
for line in full_lines:
File "/usr/lib/python3.5/bdb.py", line 48, in trace_dispatch
return self.dispatch_line(frame)
File "/usr/lib/python3.5/bdb.py", line 67, in dispatch_line
if self.quitting: raise BdbQuit
bdb.BdbQuit
2020-05-27 14:43:10 WARNING: Your reader has raised an exception!
Exception in thread Thread-1:
Traceback (most recent call last):
File "/usr/lib/python3.5/threading.py", line 914, in _bootstrap_inner
self.run()
File "/usr/lib/python3.5/threading.py", line 862, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.5/dist-packages/paddle/fluid/reader.py", line 1156, in thread_main
six.reraise(*sys.exc_info())
File "/usr/local/lib/python3.5/dist-packages/six.py", line 703, in reraise
raise value
File "/usr/local/lib/python3.5/dist-packages/paddle/fluid/reader.py", line 1136, in thread_main
for tensors in self._tensor_reader():
File "/usr/local/lib/python3.5/dist-packages/paddle/fluid/reader.py", line 1206, in tensor_reader_impl
for slots in paddle_reader():
File "/usr/local/lib/python3.5/dist-packages/paddle/fluid/data_feeder.py", line 506, in reader_creator
for item in reader():
File "/home/pd_source/cla/ppcls/data/reader.py", line 267, in wrapper
for idx, sample in enumerate(reader()):
File "/usr/local/lib/python3.5/dist-packages/paddle/reader/decorator.py", line 572, in queue_reader
raise ValueError("multiprocess reader raises an exception")
ValueError: multiprocess reader raises an exception
/home/pd_source/cla/ppcls/data/reader.py(191)reader()
-> for line in full_lines:
(Pdb)
Process Process-4:
Traceback (most recent call last):
File "/usr/lib/python3.5/multiprocessing/process.py", line 249, in _bootstrap
self.run()
File "/usr/lib/python3.5/multiprocessing/process.py", line 93, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.5/dist-packages/paddle/reader/decorator.py", line 549, in _read_into_queue
for sample in reader():
File "/usr/local/lib/python3.5/dist-packages/six.py", line 703, in reraise
raise value
File "/usr/local/lib/python3.5/dist-packages/paddle/reader/decorator.py", line 549, in _read_into_queue
for sample in reader():
File "/home/pd_source/cla/ppcls/data/reader.py", line 191, in reader
for line in full_lines:
File "/home/pd_source/cla/ppcls/data/reader.py", line 191, in reader
for line in full_lines:
File "/usr/lib/python3.5/bdb.py", line 48, in trace_dispatch
return self.dispatch_line(frame)
File "/usr/lib/python3.5/bdb.py", line 67, in dispatch_line
if self.quitting: raise BdbQuit
bdb.BdbQuit
/home/pd_source/cla/ppcls/data/reader.py(191)reader()
-> for line in full_lines:
(Pdb)
Process Process-2:
Traceback (most recent call last):
File "/usr/lib/python3.5/multiprocessing/process.py", line 249, in _bootstrap
self.run()
File "/usr/lib/python3.5/multiprocessing/process.py", line 93, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.5/dist-packages/paddle/reader/decorator.py", line 549, in _read_into_queue
for sample in reader():
File "/usr/local/lib/python3.5/dist-packages/six.py", line 703, in reraise
raise value
File "/usr/local/lib/python3.5/dist-packages/paddle/reader/decorator.py", line 549, in _read_into_queue
for sample in reader():
File "/home/pd_source/cla/ppcls/data/reader.py", line 191, in reader
for line in full_lines:
File "/home/pd_source/cla/ppcls/data/reader.py", line 191, in reader
for line in full_lines:
File "/usr/lib/python3.5/bdb.py", line 48, in trace_dispatch
return self.dispatch_line(frame)
File "/usr/lib/python3.5/bdb.py", line 67, in dispatch_line
if self.quitting: raise BdbQuit
bdb.BdbQuit
Traceback (most recent call last):
File "./jaits_utils/task_tools.py", line 494, in inner
func(jif,*args, **kwargs)
File "cla/jaits_train.py", line 215, in main
epoch_id, 'train')
File "/home/pd_source/cla/program.py", line 413, in run
for idx, batch in enumerate(dataloader()):
File "/usr/local/lib/python3.5/dist-packages/paddle/fluid/reader.py", line 1102, in next
return self._reader.read_next()
paddle.fluid.core_avx.EnforceNotMet:
C++ Call Stacks (More useful to developers):
0 std::string paddle::platform::GetTraceBackString<std::string const&>(std::string const&, char const*, int)
1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int)
2 paddle::operators::reader::BlockingQueue<std::vector<paddle::framework::LoDTensor, std::allocatorpaddle::framework::LoDTensor > >::Receive(std::vector<paddle::framework::LoDTensor, std::allocatorpaddle::framework::LoDTensor >)
3 paddle::operators::reader::PyReader::ReadNext(std::vector<paddle::framework::LoDTensor, std::allocatorpaddle::framework::LoDTensor >)
4 std::_Function_handler<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> (), std::__future_base::_Task_setter<std::unique_ptr<std::__future_base::_Result, std::__future_base::_Result_base::_Deleter>, unsigned long> >::_M_invoke(std::_Any_data const&)
5 std::__future_base::_State_base::_M_do_set(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>&, bool&)
6 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const
Error Message Summary:
Error: Blocking queue is killed because the data reader raises an exception
[Hint: Expected killed_ != true, but received killed_:1 == true:1.] at (/paddle/paddle/fluid/operators/reader/blocking_queue.h:141)
2020-05-27 14:43:10 INFO: SO:exception-Traceback (most recent call last):
File "./jaits_utils/task_tools.py", line 494, in inner
func(jif,*args, **kwargs)
File "cla/jaits_train.py", line 215, in main
epoch_id, 'train')
File "/home/pd_source/cla/program.py", line 413, in run
for idx, batch in enumerate(dataloader()):
File "/usr/local/lib/python3.5/dist-packages/paddle/fluid/reader.py", line 1102, in next
return self._reader.read_next()
paddle.fluid.core_avx.EnforceNotMet:
C++ Call Stacks (More useful to developers):
0 std::string paddle::platform::GetTraceBackString<std::string const&>(std::string const&, char const*, int)
1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int)
2 paddle::operators::reader::BlockingQueue<std::vector<paddle::framework::LoDTensor, std::allocatorpaddle::framework::LoDTensor > >::Receive(std::vector<paddle::framework::LoDTensor, std::allocatorpaddle::framework::LoDTensor >)
3 paddle::operators::reader::PyReader::ReadNext(std::vector<paddle::framework::LoDTensor, std::allocatorpaddle::framework::LoDTensor >)
4 std::_Function_handler<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> (), std::__future_base::_Task_setter<std::unique_ptr<std::__future_base::_Result, std::__future_base::_Result_base::_Deleter>, unsigned long> >::_M_invoke(std::_Any_data const&)
5 std::__future_base::_State_base::_M_do_set(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>&, bool&)
6 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const
Error Message Summary:
Error: Blocking queue is killed because the data reader raises an exception
[Hint: Expected killed_ != true, but received killed_:1 == true:1.] at (/paddle/paddle/fluid/operators/reader/blocking_queue.h:141)
/home/pd_source/cla/ppcls/data/reader.py(191)reader()
-> for line in full_lines:
(Pdb)
Process Process-3:
Traceback (most recent call last):
File "/usr/lib/python3.5/multiprocessing/process.py", line 249, in _bootstrap
self.run()
File "/usr/lib/python3.5/multiprocessing/process.py", line 93, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.5/dist-packages/paddle/reader/decorator.py", line 549, in _read_into_queue
for sample in reader():
File "/usr/local/lib/python3.5/dist-packages/six.py", line 703, in reraise
raise value
File "/usr/local/lib/python3.5/dist-packages/paddle/reader/decorator.py", line 549, in _read_into_queue
for sample in reader():
File "/home/pd_source/cla/ppcls/data/reader.py", line 191, in reader
for line in full_lines:
File "/home/pd_source/cla/ppcls/data/reader.py", line 191, in reader
for line in full_lines:
File "/usr/lib/python3.5/bdb.py", line 48, in trace_dispatch
return self.dispatch_line(frame)
File "/usr/lib/python3.5/bdb.py", line 67, in dispatch_line
if self.quitting: raise BdbQuit
bdb.BdbQuit
2020-05-27 14:43:10 WARNING: Your reader has raised an exception!
Exception in thread Thread-1:
Traceback (most recent call last):
File "/usr/lib/python3.5/threading.py", line 914, in _bootstrap_inner
self.run()
File "/usr/lib/python3.5/threading.py", line 862, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.5/dist-packages/paddle/fluid/reader.py", line 1156, in thread_main
six.reraise(*sys.exc_info())
File "/usr/local/lib/python3.5/dist-packages/six.py", line 703, in reraise
raise value
File "/usr/local/lib/python3.5/dist-packages/paddle/fluid/reader.py", line 1136, in thread_main
for tensors in self._tensor_reader():
File "/usr/local/lib/python3.5/dist-packages/paddle/fluid/reader.py", line 1206, in tensor_reader_impl
for slots in paddle_reader():
File "/usr/local/lib/python3.5/dist-packages/paddle/fluid/data_feeder.py", line 506, in reader_creator
for item in reader():
File "/home/pd_source/cla/ppcls/data/reader.py", line 267, in wrapper
for idx, sample in enumerate(reader()):
File "/usr/local/lib/python3.5/dist-packages/paddle/reader/decorator.py", line 572, in queue_reader
raise ValueError("multiprocess reader raises an exception")
ValueError: multiprocess reader raises an exception
/home/pd_source/cla/ppcls/data/reader.py(191)reader()
-> for line in full_lines:
(Pdb)
Process Process-4:
Traceback (most recent call last):
File "/usr/lib/python3.5/multiprocessing/process.py", line 249, in _bootstrap
self.run()
File "/usr/lib/python3.5/multiprocessing/process.py", line 93, in run
self._target(*self._args, **self._kwargs)
File "/usr/local/lib/python3.5/dist-packages/paddle/reader/decorator.py", line 549, in _read_into_queue
for sample in reader():
File "/usr/local/lib/python3.5/dist-packages/six.py", line 703, in reraise
raise value
File "/usr/local/lib/python3.5/dist-packages/paddle/reader/decorator.py", line 549, in _read_into_queue
for sample in reader():
File "/home/pd_source/cla/ppcls/data/reader.py", line 191, in reader
for line in full_lines:
File "/home/pd_source/cla/ppcls/data/reader.py", line 191, in reader
for line in full_lines:
File "/usr/lib/python3.5/bdb.py", line 48, in trace_dispatch
return self.dispatch_line(frame)
File "/usr/lib/python3.5/bdb.py", line 67, in dispatch_line
if self.quitting: raise BdbQuit
bdb.BdbQuit
Traceback (most recent call last):
File "./jaits_utils/task_tools.py", line 494, in inner
func(jif,*args, **kwargs)
File "cla/jaits_train.py", line 215, in main
epoch_id, 'train')
File "/home/pd_source/cla/program.py", line 413, in run
for idx, batch in enumerate(dataloader()):
File "/usr/local/lib/python3.5/dist-packages/paddle/fluid/reader.py", line 1102, in next
return self._reader.read_next()
paddle.fluid.core_avx.EnforceNotMet:
C++ Call Stacks (More useful to developers):
0 std::string paddle::platform::GetTraceBackString<std::string const&>(std::string const&, char const*, int)
1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int)
2 paddle::operators::reader::BlockingQueue<std::vector<paddle::framework::LoDTensor, std::allocatorpaddle::framework::LoDTensor > >::Receive(std::vector<paddle::framework::LoDTensor, std::allocatorpaddle::framework::LoDTensor >)
3 paddle::operators::reader::PyReader::ReadNext(std::vector<paddle::framework::LoDTensor, std::allocatorpaddle::framework::LoDTensor >)
4 std::_Function_handler<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> (), std::__future_base::_Task_setter<std::unique_ptr<std::__future_base::_Result, std::__future_base::_Result_base::_Deleter>, unsigned long> >::_M_invoke(std::_Any_data const&)
5 std::__future_base::_State_base::_M_do_set(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>&, bool&)
6 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const
Error Message Summary:
Error: Blocking queue is killed because the data reader raises an exception
[Hint: Expected killed_ != true, but received killed_:1 == true:1.] at (/paddle/paddle/fluid/operators/reader/blocking_queue.h:141)
2020-05-27 14:43:10 INFO: SO:exception-Traceback (most recent call last):
File "./jaits_utils/task_tools.py", line 494, in inner
func(jif,*args, **kwargs)
File "cla/jaits_train.py", line 215, in main
epoch_id, 'train')
File "/home/pd_source/cla/program.py", line 413, in run
for idx, batch in enumerate(dataloader()):
File "/usr/local/lib/python3.5/dist-packages/paddle/fluid/reader.py", line 1102, in next
return self._reader.read_next()
paddle.fluid.core_avx.EnforceNotMet:
C++ Call Stacks (More useful to developers):
0 std::string paddle::platform::GetTraceBackString<std::string const&>(std::string const&, char const*, int)
1 paddle::platform::EnforceNotMet::EnforceNotMet(std::string const&, char const*, int)
2 paddle::operators::reader::BlockingQueue<std::vector<paddle::framework::LoDTensor, std::allocatorpaddle::framework::LoDTensor > >::Receive(std::vector<paddle::framework::LoDTensor, std::allocatorpaddle::framework::LoDTensor >)
3 paddle::operators::reader::PyReader::ReadNext(std::vector<paddle::framework::LoDTensor, std::allocatorpaddle::framework::LoDTensor >)
4 std::_Function_handler<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> (), std::__future_base::_Task_setter<std::unique_ptr<std::__future_base::_Result, std::__future_base::_Result_base::_Deleter>, unsigned long> >::_M_invoke(std::_Any_data const&)
5 std::__future_base::_State_base::_M_do_set(std::function<std::unique_ptr<std::__future_base::_Result_base, std::__future_base::_Result_base::_Deleter> ()>&, bool&)
6 ThreadPool::ThreadPool(unsigned long)::{lambda()#1}::operator()() const
Error Message Summary:
Error: Blocking queue is killed because the data reader raises an exception
[Hint: Expected killed_ != true, but received killed_:1 == true:1.] at (/paddle/paddle/fluid/operators/reader/blocking_queue.h:141)