Same with the .py file for preprocessing used by that shell script.

python2 train_ner.py --train-file train.json
Traceback (most recent call last):
File "train_ner.py", line 1, in
from torch_scope import wrapper
File "/home/zakaria/anaconda3/lib/python2.7/site-packages/torch_scope/init.py", line 7, in
from torch_scope.wrapper import wrapper, basic_wrapper
File "/home/zakaria/anaconda3/lib/python2.7/site-packages/torch_scope/wrapper.py", line 325
path: str,
^
SyntaxError: invalid syntax

it this normal and in ./data i have my train.txt fil in arabic
bash train.sh ./data
=== Removing Pre-trained Models ===
=== Cleaning Data ===
Traceback (most recent call last):
File "data_clean/data_clean.py", line 68, in
with codecs.open(args.train_file_output, 'w', 'utf-8') as fout:
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/codecs.py", line 904, in open
file = builtins.open(filename, mode, buffering)
FileNotFoundError: [Errno 2] No such file or directory: '/wdata/train_cleaned.txt'
=== Building Dictionary Based Model ===
Traceback (most recent call last):
File "post_process/save_dictionary.py", line 45, in
with open(args.dict_output, 'w') as fout:
FileNotFoundError: [Errno 2] No such file or directory: '/wdata/train_dict.json'
=== Data Pre-processing ===
Traceback (most recent call last):
File "pre_process_train.py", line 21, in
encoder = strRealTimeEncoderWrapper(args)
File "/home/zakaria/Downloads/ArabicNER/abnlp/encoder/str_encoder.py", line 35, in init
self.build_pipelines(arg)
File "/home/zakaria/Downloads/ArabicNER/abnlp/encoder/str_encoder.py", line 281, in build_pipelines
self.pipeline_dict[key] = strWERealTimePipeline(arg['strEncoder'][key])
File "/home/zakaria/Downloads/ArabicNER/abnlp/encoder/str_encoder.py", line 66, in init
self.build_pipeline(arg)
File "/home/zakaria/Downloads/ArabicNER/abnlp/encoder/str_encoder.py", line 159, in build_pipeline
for line in f.readlines():
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/codecs.py", line 709, in readlines
return self.reader.readlines(sizehint)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/codecs.py", line 618, in readlines
data = self.read()
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/codecs.py", line 496, in read
newdata = self.stream.read()
MemoryError
=== Training Model 0 ===
Traceback (most recent call last):
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/record_writer.py", line 40, in directory_check
factory = REGISTERED_FACTORIES[prefix]
KeyError: '/wdata/sner0/log/'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "train_ner.py", line 32, in
pw = wrapper(os.path.join(conf.cp_root, args["checkpoint_name"]), args["checkpoint_name"])
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/torch_scope/wrapper.py", line 361, in init
self.writer = SummaryWriter(log_dir=os.path.join(path, 'log/'))
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/writer.py", line 257, in init
self._get_file_writer()
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/writer.py", line 321, in _get_file_writer
**self.kwargs)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/writer.py", line 93, in init
logdir, max_queue, flush_secs, filename_suffix)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/event_file_writer.py", line 104, in init
directory_check(self._logdir)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/record_writer.py", line 44, in directory_check
os.makedirs(path)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/os.py", line 211, in makedirs
makedirs(head, exist_ok=exist_ok)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/os.py", line 211, in makedirs
makedirs(head, exist_ok=exist_ok)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/os.py", line 221, in makedirs
mkdir(name, mode)
PermissionError: [Errno 13] Permission denied: '/wdata'

If you suspect this is an IPython bug, please report it at:
https://github.com/ipython/ipython/issues
or send an email to the mailing list at [email protected]

You can print a more detailed traceback right now with "%tb", or use "%debug"
to interactively debug it.

Extra-detailed tracebacks for bug-reporting purposes can be enabled via:
%config Application.verbose_crash=True

=== Training Model 1 ===
Traceback (most recent call last):
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/record_writer.py", line 40, in directory_check
factory = REGISTERED_FACTORIES[prefix]
KeyError: '/wdata/sner1/log/'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "train_ner.py", line 32, in
pw = wrapper(os.path.join(conf.cp_root, args["checkpoint_name"]), args["checkpoint_name"])
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/torch_scope/wrapper.py", line 361, in init
self.writer = SummaryWriter(log_dir=os.path.join(path, 'log/'))
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/writer.py", line 257, in init
self._get_file_writer()
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/writer.py", line 321, in _get_file_writer
**self.kwargs)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/writer.py", line 93, in init
logdir, max_queue, flush_secs, filename_suffix)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/event_file_writer.py", line 104, in init
directory_check(self._logdir)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/record_writer.py", line 44, in directory_check
os.makedirs(path)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/os.py", line 211, in makedirs
makedirs(head, exist_ok=exist_ok)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/os.py", line 211, in makedirs
makedirs(head, exist_ok=exist_ok)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/os.py", line 221, in makedirs
mkdir(name, mode)
PermissionError: [Errno 13] Permission denied: '/wdata'

If you suspect this is an IPython bug, please report it at:
https://github.com/ipython/ipython/issues
or send an email to the mailing list at [email protected]

You can print a more detailed traceback right now with "%tb", or use "%debug"
to interactively debug it.

Extra-detailed tracebacks for bug-reporting purposes can be enabled via:
%config Application.verbose_crash=True

=== Training Model 2 ===
Traceback (most recent call last):
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/record_writer.py", line 40, in directory_check
factory = REGISTERED_FACTORIES[prefix]
KeyError: '/wdata/sner2/log/'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "train_ner.py", line 32, in
pw = wrapper(os.path.join(conf.cp_root, args["checkpoint_name"]), args["checkpoint_name"])
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/torch_scope/wrapper.py", line 361, in init
self.writer = SummaryWriter(log_dir=os.path.join(path, 'log/'))
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/writer.py", line 257, in init
self._get_file_writer()
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/writer.py", line 321, in _get_file_writer
**self.kwargs)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/writer.py", line 93, in init
logdir, max_queue, flush_secs, filename_suffix)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/event_file_writer.py", line 104, in init
directory_check(self._logdir)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/record_writer.py", line 44, in directory_check
os.makedirs(path)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/os.py", line 211, in makedirs
makedirs(head, exist_ok=exist_ok)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/os.py", line 211, in makedirs
makedirs(head, exist_ok=exist_ok)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/os.py", line 221, in makedirs
mkdir(name, mode)
PermissionError: [Errno 13] Permission denied: '/wdata'

If you suspect this is an IPython bug, please report it at:
https://github.com/ipython/ipython/issues
or send an email to the mailing list at [email protected]

You can print a more detailed traceback right now with "%tb", or use "%debug"
to interactively debug it.

Extra-detailed tracebacks for bug-reporting purposes can be enabled via:
%config Application.verbose_crash=True

=== Training Model 3 ===
Traceback (most recent call last):
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/record_writer.py", line 40, in directory_check
factory = REGISTERED_FACTORIES[prefix]
KeyError: '/wdata/sner3/log/'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "train_ner.py", line 32, in
pw = wrapper(os.path.join(conf.cp_root, args["checkpoint_name"]), args["checkpoint_name"])
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/torch_scope/wrapper.py", line 361, in init
self.writer = SummaryWriter(log_dir=os.path.join(path, 'log/'))
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/writer.py", line 257, in init
self._get_file_writer()
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/writer.py", line 321, in _get_file_writer
**self.kwargs)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/writer.py", line 93, in init
logdir, max_queue, flush_secs, filename_suffix)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/event_file_writer.py", line 104, in init
directory_check(self._logdir)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/record_writer.py", line 44, in directory_check
os.makedirs(path)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/os.py", line 211, in makedirs
makedirs(head, exist_ok=exist_ok)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/os.py", line 211, in makedirs
makedirs(head, exist_ok=exist_ok)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/os.py", line 221, in makedirs
mkdir(name, mode)
PermissionError: [Errno 13] Permission denied: '/wdata'

If you suspect this is an IPython bug, please report it at:
https://github.com/ipython/ipython/issues
or send an email to the mailing list at [email protected]

You can print a more detailed traceback right now with "%tb", or use "%debug"
to interactively debug it.

Extra-detailed tracebacks for bug-reporting purposes can be enabled via:
%config Application.verbose_crash=True

=== Training Model 4 ===
Traceback (most recent call last):
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/record_writer.py", line 40, in directory_check
factory = REGISTERED_FACTORIES[prefix]
KeyError: '/wdata/sner4/log/'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "train_ner.py", line 32, in
pw = wrapper(os.path.join(conf.cp_root, args["checkpoint_name"]), args["checkpoint_name"])
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/torch_scope/wrapper.py", line 361, in init
self.writer = SummaryWriter(log_dir=os.path.join(path, 'log/'))
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/writer.py", line 257, in init
self._get_file_writer()
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/writer.py", line 321, in _get_file_writer
**self.kwargs)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/writer.py", line 93, in init
logdir, max_queue, flush_secs, filename_suffix)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/event_file_writer.py", line 104, in init
directory_check(self._logdir)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/record_writer.py", line 44, in directory_check
os.makedirs(path)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/os.py", line 211, in makedirs
makedirs(head, exist_ok=exist_ok)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/os.py", line 211, in makedirs
makedirs(head, exist_ok=exist_ok)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/os.py", line 221, in makedirs
mkdir(name, mode)
PermissionError: [Errno 13] Permission denied: '/wdata'

If you suspect this is an IPython bug, please report it at:
https://github.com/ipython/ipython/issues
or send an email to the mailing list at [email protected]

You can print a more detailed traceback right now with "%tb", or use "%debug"
to interactively debug it.

Extra-detailed tracebacks for bug-reporting purposes can be enabled via:
%config Application.verbose_crash=True

=== Training Model 5 ===
Traceback (most recent call last):
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/record_writer.py", line 40, in directory_check
factory = REGISTERED_FACTORIES[prefix]
KeyError: '/wdata/sner5/log/'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "train_ner.py", line 32, in
pw = wrapper(os.path.join(conf.cp_root, args["checkpoint_name"]), args["checkpoint_name"])
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/torch_scope/wrapper.py", line 361, in init
self.writer = SummaryWriter(log_dir=os.path.join(path, 'log/'))
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/writer.py", line 257, in init
self._get_file_writer()
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/writer.py", line 321, in _get_file_writer
**self.kwargs)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/writer.py", line 93, in init
logdir, max_queue, flush_secs, filename_suffix)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/event_file_writer.py", line 104, in init
directory_check(self._logdir)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/record_writer.py", line 44, in directory_check
os.makedirs(path)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/os.py", line 211, in makedirs
makedirs(head, exist_ok=exist_ok)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/os.py", line 211, in makedirs
makedirs(head, exist_ok=exist_ok)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/os.py", line 221, in makedirs
mkdir(name, mode)
PermissionError: [Errno 13] Permission denied: '/wdata'

If you suspect this is an IPython bug, please report it at:
https://github.com/ipython/ipython/issues
or send an email to the mailing list at [email protected]

You can print a more detailed traceback right now with "%tb", or use "%debug"
to interactively debug it.

Extra-detailed tracebacks for bug-reporting purposes can be enabled via:
%config Application.verbose_crash=True

=== Training Model 6 ===
Traceback (most recent call last):
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/record_writer.py", line 40, in directory_check
factory = REGISTERED_FACTORIES[prefix]
KeyError: '/wdata/sner6/log/'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "train_ner.py", line 32, in
pw = wrapper(os.path.join(conf.cp_root, args["checkpoint_name"]), args["checkpoint_name"])
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/torch_scope/wrapper.py", line 361, in init
self.writer = SummaryWriter(log_dir=os.path.join(path, 'log/'))
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/writer.py", line 257, in init
self._get_file_writer()
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/writer.py", line 321, in _get_file_writer
**self.kwargs)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/writer.py", line 93, in init
logdir, max_queue, flush_secs, filename_suffix)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/event_file_writer.py", line 104, in init
directory_check(self._logdir)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/record_writer.py", line 44, in directory_check
os.makedirs(path)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/os.py", line 211, in makedirs
makedirs(head, exist_ok=exist_ok)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/os.py", line 211, in makedirs
makedirs(head, exist_ok=exist_ok)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/os.py", line 221, in makedirs
mkdir(name, mode)
PermissionError: [Errno 13] Permission denied: '/wdata'

If you suspect this is an IPython bug, please report it at:
https://github.com/ipython/ipython/issues
or send an email to the mailing list at [email protected]

You can print a more detailed traceback right now with "%tb", or use "%debug"
to interactively debug it.

Extra-detailed tracebacks for bug-reporting purposes can be enabled via:
%config Application.verbose_crash=True

=== Training Model 7 ===
Traceback (most recent call last):
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/record_writer.py", line 40, in directory_check
factory = REGISTERED_FACTORIES[prefix]
KeyError: '/wdata/sner7/log/'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "train_ner.py", line 32, in
pw = wrapper(os.path.join(conf.cp_root, args["checkpoint_name"]), args["checkpoint_name"])
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/torch_scope/wrapper.py", line 361, in init
self.writer = SummaryWriter(log_dir=os.path.join(path, 'log/'))
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/writer.py", line 257, in init
self._get_file_writer()
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/writer.py", line 321, in _get_file_writer
**self.kwargs)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/writer.py", line 93, in init
logdir, max_queue, flush_secs, filename_suffix)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/event_file_writer.py", line 104, in init
directory_check(self._logdir)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/record_writer.py", line 44, in directory_check
os.makedirs(path)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/os.py", line 211, in makedirs
makedirs(head, exist_ok=exist_ok)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/os.py", line 211, in makedirs
makedirs(head, exist_ok=exist_ok)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/os.py", line 221, in makedirs
mkdir(name, mode)
PermissionError: [Errno 13] Permission denied: '/wdata'

If you suspect this is an IPython bug, please report it at:
https://github.com/ipython/ipython/issues
or send an email to the mailing list at [email protected]

You can print a more detailed traceback right now with "%tb", or use "%debug"
to interactively debug it.

Extra-detailed tracebacks for bug-reporting purposes can be enabled via:
%config Application.verbose_crash=True

=== Training Model 8 ===
Traceback (most recent call last):
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/record_writer.py", line 40, in directory_check
factory = REGISTERED_FACTORIES[prefix]
KeyError: '/wdata/sner8/log/'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "train_ner.py", line 32, in
pw = wrapper(os.path.join(conf.cp_root, args["checkpoint_name"]), args["checkpoint_name"])
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/torch_scope/wrapper.py", line 361, in init
self.writer = SummaryWriter(log_dir=os.path.join(path, 'log/'))
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/writer.py", line 257, in init
self._get_file_writer()
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/writer.py", line 321, in _get_file_writer
**self.kwargs)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/writer.py", line 93, in init
logdir, max_queue, flush_secs, filename_suffix)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/event_file_writer.py", line 104, in init
directory_check(self._logdir)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/site-packages/tensorboardX/record_writer.py", line 44, in directory_check
os.makedirs(path)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/os.py", line 211, in makedirs
makedirs(head, exist_ok=exist_ok)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/os.py", line 211, in makedirs
makedirs(head, exist_ok=exist_ok)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/os.py", line 221, in makedirs
mkdir(name, mode)
PermissionError: [Errno 13] Permission denied: '/wdata'

If you suspect this is an IPython bug, please report it at:
https://github.com/ipython/ipython/issues
or send an email to the mailing list at [email protected]

You can print a more detailed traceback right now with "%tb", or use "%debug"
to interactively debug it.

Extra-detailed tracebacks for bug-reporting purposes can be enabled via:
%config Application.verbose_crash=True

why you considered 3 word shape and whats happen if we just consider only 2 shapes !
another question; what is the recommended machine configuration to have a best result
thank you

i don't know why it keep given me this error:
python3 train_ner.py --train_file train.txt
[2019-09-02 15:46:33,390] Checkpoint Folder Already Exists: ./checkpoint/sner0
[2019-09-02 15:46:33,390] Input 'yes' to confirm deleting this folder; or 'no' to exit.
yes for delete or no for exit: yes
[2019-09-02 15:46:35,951] Saving system environemnt and python packages
DEPRECATION: Python 2.7 will reach the end of its life on January 1st, 2020. Please upgrade your Python as Python 2.7 won't be maintained after that date. A future version of pip will drop support for Python 2.7. More details about Python 2 support in pip, can be found at https://pip.pypa.io/en/latest/development/release-process/#python-2-support
[2019-09-02 15:46:36,744] Loading the data...
[2019-09-02 15:46:36,744] Building lm pipeline...
Traceback (most recent call last):
File "train_ner.py", line 35, in
train_data = strFromFileEncoderWrapper(args, processed_file = conf.train_file)
File "/home/zakaria/Downloads/ArabicNER/abnlp/encoder/str_encoder.py", line 301, in init
super(strFromFileEncoderWrapper, self).init(arg)
File "/home/zakaria/Downloads/ArabicNER/abnlp/encoder/str_encoder.py", line 35, in init
self.build_pipelines(arg)
File "/home/zakaria/Downloads/ArabicNER/abnlp/encoder/str_encoder.py", line 309, in build_pipelines
self.pipeline_dict[key] = strFromFilePipeline(arg['strEncoder'][key], key)
File "/home/zakaria/Downloads/ArabicNER/abnlp/encoder/str_encoder.py", line 252, in init
super(strFromFilePipeline, self).init(arg)
File "/home/zakaria/Downloads/ArabicNER/abnlp/encoder/str_encoder.py", line 66, in init
self.build_pipeline(arg)
File "/home/zakaria/Downloads/ArabicNER/abnlp/encoder/str_encoder.py", line 258, in build_pipeline
self.instances = json.load(fin)[self.name]
File "/usr/lib/python3.6/json/init.py", line 299, in load
parse_constant=parse_constant, object_pairs_hook=object_pairs_hook, **kw)
File "/usr/lib/python3.6/json/init.py", line 354, in loads
return _default_decoder.decode(s)
File "/usr/lib/python3.6/json/decoder.py", line 339, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "/usr/lib/python3.6/json/decoder.py", line 357, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)

If you suspect this is an IPython bug, please report it at:
https://github.com/ipython/ipython/issues
or send an email to the mailing list at [email protected]

You can print a more detailed traceback right now with "%tb", or use "%debug"
to interactively debug it.

Extra-detailed tracebacks for bug-reporting purposes can be enabled via:
%config Application.verbose_crash=True

Changing this line to simply for line in f helped me to avoid having my Docker environment kill pre_process_test.py

after training i'm getting memory error
bash test.sh ./data ./wdata
=== Data Pre-processing ===
Traceback (most recent call last):
File "pre_process_test.py", line 23, in
encoder = strRealTimeEncoderWrapper(args)
File "/home/zakaria/Downloads/ArabicNER/abnlp/encoder/str_encoder.py", line 35, in init
self.build_pipelines(arg)
File "/home/zakaria/Downloads/ArabicNER/abnlp/encoder/str_encoder.py", line 281, in build_pipelines
self.pipeline_dict[key] = strWERealTimePipeline(arg['strEncoder'][key])
File "/home/zakaria/Downloads/ArabicNER/abnlp/encoder/str_encoder.py", line 66, in init
self.build_pipeline(arg)
File "/home/zakaria/Downloads/ArabicNER/abnlp/encoder/str_encoder.py", line 159, in build_pipeline
for line in f.readlines():
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/codecs.py", line 709, in readlines
return self.reader.readlines(sizehint)
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/codecs.py", line 618, in readlines
data = self.read()
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/codecs.py", line 504, in read
newchars, decodedbytes = self.decode(data, self.errors)
MemoryError
=== Model Ensembling and Inferencing ===
[2019-09-04 13:45:26,511] CPU would be used.
[2019-09-04 13:45:26,520] Loading the data...
[2019-09-04 13:45:26,520] Building lm pipeline...
[2019-09-04 13:45:26,520] File not exist: /home/zakaria/Downloads/ArabicNER/wdata/dev.json
Traceback (most recent call last):
File "ensemble_ner.py", line 48, in
dev_data = strFromFileEncoderWrapper(args, processed_file = conf.input[0])
File "/home/zakaria/Downloads/ArabicNER/abnlp/encoder/str_encoder.py", line 301, in init
super(strFromFileEncoderWrapper, self).init(arg)
File "/home/zakaria/Downloads/ArabicNER/abnlp/encoder/str_encoder.py", line 35, in init
self.build_pipelines(arg)
File "/home/zakaria/Downloads/ArabicNER/abnlp/encoder/str_encoder.py", line 309, in build_pipelines
self.pipeline_dict[key] = strFromFilePipeline(arg['strEncoder'][key], key)
File "/home/zakaria/Downloads/ArabicNER/abnlp/encoder/str_encoder.py", line 252, in init
super(strFromFilePipeline, self).init(arg)
File "/home/zakaria/Downloads/ArabicNER/abnlp/encoder/str_encoder.py", line 66, in init
self.build_pipeline(arg)
File "/home/zakaria/Downloads/ArabicNER/abnlp/encoder/str_encoder.py", line 257, in build_pipeline
with open(arg['processed_file'], 'r') as fin:
FileNotFoundError: [Errno 2] No such file or directory: '/home/zakaria/Downloads/ArabicNER/wdata/dev.json'

If you suspect this is an IPython bug, please report it at:
https://github.com/ipython/ipython/issues
or send an email to the mailing list at [email protected]

You can print a more detailed traceback right now with "%tb", or use "%debug"
to interactively debug it.

Extra-detailed tracebacks for bug-reporting purposes can be enabled via:
%config Application.verbose_crash=True

=== Dictionary Based Model Inferencing ===
=== Results Merging ===
Traceback (most recent call last):
File "post_process/merge_result.py", line 13, in
with codecs.open(args.model_csv, 'r', 'utf-8') as fin:
File "/home/zakaria/anaconda3/envs/py37/lib/python3.7/codecs.py", line 904, in open
file = builtins.open(filename, mode, buffering)
FileNotFoundError: [Errno 2] No such file or directory: '/home/zakaria/Downloads/ArabicNER/wdata/tmp0.csv'

~/ArabicNER$ nvidia-docker run -v ``pwd''/data:/data:ro -v :/wdata -it arabic_ner

-bash: Output: No such file or directory