somebody who can help me, cudnn 5.1 or cudnn 6 both not work after recompiling ... about fcis HOT 3 OPEN

Stack trace returned 10 entries:
[bt] (0) /home/shel/mxnet/python/mxnet/../../lib/libmxnet.so(_ZN5mxnet7storage23GPUPooledStorageManager5AllocEm+0x3e4) [0x7f6e47f1fde4]
[bt] (1) /home/shel/mxnet/python/mxnet/../../lib/libmxnet.so(_ZN5mxnet11StorageImpl5AllocEmNS_7ContextE+0x57) [0x7f6e47f23817]
[bt] (2) /home/sheldon606/mxnet/python/mxnet/../../lib/libmxnet.so(+0x1433443) [0x7f6e47f72443]
[bt] (3) /home/shel/mxnet/python/mxnet/../../lib/libmxnet.so(_ZN5mxnet4exec13GraphExecutor19InitDataEntryMemoryEPSt6vectorINS_7NDArrayESaIS3_EE+0x2284) [0x7f6e47f7b1c4]
[bt] (4) /home/shel/mxnet/python/mxnet/../../lib/libmxnet.so(_ZN5mxnet4exec13GraphExecutor4InitEN4nnvm6SymbolERKNS_7ContextERKSt3mapISsS4_St4lessISsESaISt4pairIKSsS4_EEERKSt6vectorINS_7NDArrayESaISI_EESM_RKSH_INS_9OpReqTypeESaISN_EESM_PNS_8ExecutorERKSt13unordered_mapINS2_9NodeEntryESI_NS2_13NodeEntryHashENS2_14NodeEntryEqualESaISA_IKSV_SI_EEE+0x5bb) [0x7f6e47f8154b]
[bt] (5) /home/shel/mxnet/python/mxnet/../../lib/libmxnet.so(ZN5mxnet8Executor4BindEN4nnvm6SymbolERKNS_7ContextERKSt3mapISsS3_St4lessISsESaISt4pairIKSsS3_EEERKSt6vectorINS_7NDArrayESaISH_EESL_RKSG_INS_9OpReqTypeESaISM_EESL_PS0+0x684) [0x7f6e47f81c74]
[bt] (6) /home/shel/mxnet/python/mxnet/../../lib/libmxnet.so(MXExecutorBindEX+0x2425) [0x7f6e47f506f5]
[bt] (7) /usr/lib/x86_64-linux-gnu/libffi.so.6(ffi_call_unix64+0x4c) [0x7f6e7cfcfe40]
[bt] (8) /usr/lib/x86_64-linux-gnu/libffi.so.6(ffi_call+0x2eb) [0x7f6e7cfcf8ab]
[bt] (9) /usr/lib/python2.7/lib-dynload/_ctypes.x86_64-linux-gnu.so(_ctypes_callproc+0x48f) [0x7f6e5fdd43df]

Traceback (most recent call last):
File "experiments/fcis/fcis_end2end_train_test.py", line 13, in
train_end2end.main()
File "experiments/fcis/../../fcis/train_end2end.py", line 181, in main
config.TRAIN.lr, config.TRAIN.lr_step)
File "experiments/fcis/../../fcis/train_end2end.py", line 173, in train_net
arg_params=arg_params, aux_params=aux_params, begin_epoch=begin_epoch, num_epoch=end_epoch)
File "experiments/fcis/../../fcis/core/module.py", line 932, in fit
for_training=True, force_rebind=force_rebind)
File "experiments/fcis/../../fcis/core/module.py", line 840, in bind
for_training, inputs_need_grad, force_rebind=False, shared_module=None)
File "experiments/fcis/../../fcis/core/module.py", line 397, in bind
state_names=self._state_names)
File "experiments/fcis/../../fcis/core/DataParallelExecutorGroup.py", line 178, in init
self.bind_exec(data_shapes, label_shapes, shared_group)
File "experiments/fcis/../../fcis/core/DataParallelExecutorGroup.py", line 278, in bind_exec
shared_group))
File "experiments/fcis/../../fcis/core/DataParallelExecutorGroup.py", line 613, in _bind_ith_exec
grad_req=self.grad_req, shared_exec=shared_exec)
File "/home/shel/mxnet/python/mxnet/symbol.py", line 1413, in bind
ctypes.byref(handle)))
File "/home/shel/mxnet/python/mxnet/base.py", line 85, in check_call
raise MXNetError(py_str(_LIB.MXGetLastError()))
mxnet.base.MXNetError: [10:32:26] src/storage/./pooled_storage_manager.h:84: cudaMalloc failed: out of memory

Stack trace returned 10 entries:
[bt] (0) /home/shel/mxnet/python/mxnet/../../lib/libmxnet.so(_ZN5mxnet7storage23GPUPooledStorageManager5AllocEm+0x3e4) [0x7f6e47f1fde4]
[bt] (1) /home/shel/mxnet/python/mxnet/../../lib/libmxnet.so(_ZN5mxnet11StorageImpl5AllocEmNS_7ContextE+0x57) [0x7f6e47f23817]
[bt] (2) /home/shel/mxnet/python/mxnet/../../lib/libmxnet.so(+0x1433443) [0x7f6e47f72443]
[bt] (3) /home/sheld/mxnet/python/mxnet/../../lib/libmxnet.so(_ZN5mxnet4exec13GraphExecutor19InitDataEntryMemoryEPSt6vectorINS_7NDArrayESaIS3_EE+0x2284) [0x7f6e47f7b1c4]
[bt] (4) /home/shel/mxnet/python/mxnet/../../lib/libmxnet.so(_ZN5mxnet4exec13GraphExecutor4InitEN4nnvm6SymbolERKNS_7ContextERKSt3mapISsS4_St4lessISsESaISt4pairIKSsS4_EEERKSt6vectorINS_7NDArrayESaISI_EESM_RKSH_INS_9OpReqTypeESaISN_EESM_PNS_8ExecutorERKSt13unordered_mapINS2_9NodeEntryESI_NS2_13NodeEntryHashENS2_14NodeEntryEqualESaISA_IKSV_SI_EEE+0x5bb) [0x7f6e47f8154b]
[bt] (5) /home/shel/mxnet/python/mxnet/../../lib/libmxnet.so(ZN5mxnet8Executor4BindEN4nnvm6SymbolERKNS_7ContextERKSt3mapISsS3_St4lessISsESaISt4pairIKSsS3_EEERKSt6vectorINS_7NDArrayESaISH_EESL_RKSG_INS_9OpReqTypeESaISM_EESL_PS0+0x684) [0x7f6e47f81c74]
[bt] (6) /home/sheldon606/mxnet/python/mxnet/../../lib/libmxnet.so(MXExecutorBindEX+0x2425) [0x7f6e47f506f5]
[bt] (7) /usr/lib/x86_64-linux-gnu/libffi.so.6(ffi_call_unix64+0x4c) [0x7f6e7cfcfe40]
[bt] (8) /usr/lib/x86_64-linux-gnu/libffi.so.6(ffi_call+0x2eb) [0x7f6e7cfcf8ab]
[bt] (9) /usr/lib/python2.7/lib-dynload/_ctypes.x86_64-linux-gnu.so(_ctypes_callproc+0x48f) [0x7f6e5fdd43df]

from fcis.

Floor 2 shows the error after compiling the mxnet not using cudnn. Demo works ok for me, but the train process doesn't work any more.

from fcis.

@sheldon606 have you solved this problem??

from fcis.