training network heads

Starting at epoch 0. LR=0.001

Checkpoint Path: /root/Downloads/datacode/crowdai-mapping-challenge-mask-rcnn/logs/crowdai-mapping-challenge20190103T1545/mask_rcnn_crowdai-mapping-challenge_{epoch:04d}.h5
Selecting layers to train
fpn_c5p5 (Conv2D)
fpn_c4p4 (Conv2D)
fpn_c3p3 (Conv2D)
fpn_c2p2 (Conv2D)
fpn_p5 (Conv2D)
fpn_p2 (Conv2D)
fpn_p3 (Conv2D)
fpn_p4 (Conv2D)
In model: rpn_model
rpn_conv_shared (Conv2D)
rpn_class_raw (Conv2D)
rpn_bbox_pred (Conv2D)
mrcnn_mask_conv1 (TimeDistributed)
mrcnn_mask_bn1 (TimeDistributed)
mrcnn_mask_conv2 (TimeDistributed)
mrcnn_mask_bn2 (TimeDistributed)
mrcnn_class_conv1 (TimeDistributed)
mrcnn_class_bn1 (TimeDistributed)
mrcnn_mask_conv3 (TimeDistributed)
mrcnn_mask_bn3 (TimeDistributed)
mrcnn_class_conv2 (TimeDistributed)
mrcnn_class_bn2 (TimeDistributed)
mrcnn_mask_conv4 (TimeDistributed)
mrcnn_mask_bn4 (TimeDistributed)
mrcnn_bbox_fc (TimeDistributed)
mrcnn_mask_deconv (TimeDistributed)
mrcnn_class_logits (TimeDistributed)
mrcnn_mask (TimeDistributed)

/root/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/gradients_impl.py:100: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.
"Converting sparse IndexedSlices to a dense Tensor of unknown shape. "
/root/anaconda3/lib/python3.6/site-packages/keras/engine/training.py:2087: UserWarning: Using a generator with use_multiprocessing=True and multiple workers may duplicate your data. Please consider using thekeras.utils.Sequence class. UserWarning('Using a generator with use_multiprocessing=True`'

OSError Traceback (most recent call last)
in ()
6 learning_rate=config.LEARNING_RATE,
7 epochs=10,
----> 8 layers='heads')
9
10 # Training - Stage 2

~/Downloads/datacode/crowdai-mapping-challenge-mask-rcnn/mrcnn/model.py in train(self, train_dataset, val_dataset, learning_rate, epochs, layers, augmentation)
2314 max_queue_size=100,
2315 workers=workers,
-> 2316 use_multiprocessing=False,
2317 )
2318 self.epoch = max(self.epoch, epochs)

~/anaconda3/lib/python3.6/site-packages/keras/legacy/interfaces.py in wrapper(*args, **kwargs)
89 warnings.warn('Update your ' + object_name + 90 ' call to the Keras 2 API: ' + signature, stacklevel=2)
---> 91 return func(*args, **kwargs)
92 wrapper._original_function = func
93 return wrapper

~/anaconda3/lib/python3.6/site-packages/keras/engine/training.py in fit_generator(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)
2174 use_multiprocessing=use_multiprocessing,
2175 wait_time=wait_time)
-> 2176 enqueuer.start(workers=workers, max_queue_size=max_queue_size)
2177 output_generator = enqueuer.get()
2178 else:

~/anaconda3/lib/python3.6/site-packages/keras/utils/data_utils.py in start(self, workers, max_queue_size)
724 thread = threading.Thread(target=self._data_generator_task)
725 self._threads.append(thread)
--> 726 thread.start()
727 except:
728 self.stop()

~/anaconda3/lib/python3.6/multiprocessing/process.py in start(self)
103 'daemonic processes are not allowed to have children'
104 _cleanup()
--> 105 self._popen = self._Popen(self)
106 self._sentinel = self._popen.sentinel
107 # Avoid a refcycle if the target function holds an indirect

~/anaconda3/lib/python3.6/multiprocessing/context.py in _Popen(process_obj)
221 @staticmethod
222 def _Popen(process_obj):
--> 223 return _default_context.get_context().Process._Popen(process_obj)
224
225 class DefaultContext(BaseContext):

~/anaconda3/lib/python3.6/multiprocessing/context.py in _Popen(process_obj)
275 def _Popen(process_obj):
276 from .popen_fork import Popen
--> 277 return Popen(process_obj)
278
279 class SpawnProcess(process.BaseProcess):

~/anaconda3/lib/python3.6/multiprocessing/popen_fork.py in init(self, process_obj)
17 util._flush_std_streams()
18 self.returncode = None
---> 19 self._launch(process_obj)
20
21 def duplicate_for_child(self, fd):

~/anaconda3/lib/python3.6/multiprocessing/popen_fork.py in _launch(self, process_obj)
64 code = 1
65 parent_r, child_w = os.pipe()
---> 66 self.pid = os.fork()
67 if self.pid == 0:
68 try:

OSError: [Errno 12] Cannot allocate memory

What is the specification of the training data (e.g. spatial resolution or the satellite that took the images)?
Also, the training data seems like it is blue shifted compared to other dataset that I have and the pretrained model has poor performance in that dataset. Can you provide the data specification so that I can modify my dataset to be closer to this one. Thanks! :)

Hello,
When I submitted my results, the errors occurred. The server is down? Can anyone help solve the problem? Many thanks.

ssh: Could not resolve hostname gitlab.crowdai.org: Name or service not known
fatal: Could not read from remote repository.