Hello,
i am experiencing memory leak when running the evaluation (run_eval.sh script) .
I have this issue both with and without using the provided docker image and for both CPU and GPU evaluation.
My GPU goes OOM even when the file duration is a couple of seconds. When using the CPU the script will end up using all my RAM after processing a few hundreds of wavs of 4-5 seconds each.
Below i am providing my traceback for GPU=true and when running the run_eval script with the docker image provided. Does anyone have any clue about why this is happening ? Is the script not suitable to process many short wav files ?
UPDATE: I experience the same issue even with long wav files and regardless of the truncates_minutes value.
`About to throw exception 'CUDA failure 2: out of memory ; GPU=0 ; hostname=fe609e0f3385 ; expr=cudaMalloc((void**) &deviceBufferPtr, sizeof(AllocatedElemType) * AsMultipleOf(numElements, 2))'
CUDA failure 2: out of memory ; GPU=0 ; hostname=fe609e0f3385 ; expr=cudaMalloc((void**) &deviceBufferPtr, sizeof(AllocatedElemType) * AsMultipleOf(numElements, 2))
Traceback (most recent call last):
File "main_denoising.py", line 121, in
args.gpu_id, args.truncate_minutes)
File "main_denoising.py", line 89, in main_denoising
decode_model(use_gpu=use_gpu, gpu_id=gpu_id)
File "/dihard18/decode_model.py", line 39, in decode_model
out_noisy_fea = output_nodes.eval(real_noisy_fea)
File "/root/anaconda3/envs/dihard18/lib/python3.5/site-packages/cntk/ops/functions.py", line 733, in eval
_, output_map = self.forward(arguments, outputs, device=device, as_numpy=as_numpy)
File "/root/anaconda3/envs/dihard18/lib/python3.5/site-packages/cntk/internal/swig_helper.py", line 69, in wrapper
result = f(*args, *kwds)
File "/root/anaconda3/envs/dihard18/lib/python3.5/site-packages/cntk/ops/functions.py", line 867, in forward
keep_for_backward)
File "/root/anaconda3/envs/dihard18/lib/python3.5/site-packages/cntk/cntk_py.py", line 1980, in _forward
return _cntk_py.Function__forward(self, args)
RuntimeError: CUDA failure 2: out of memory ; GPU=0 ; hostname=fe609e0f3385 ; expr=cudaMalloc((void) &deviceBufferPtr, sizeof(AllocatedElemType) * AsMultipleOf(numElements, 2))
[CALL STACK]
[0x7f2ad32d7e89] + 0x732e89
[0x7f2acb69ef4f] + 0xec4f4f
[0x7f2acb6f2347] float* Microsoft::MSR::CNTK::TracingGPUMemoryAllocator:: Allocate (int, unsigned long, unsigned long) + 0x57
[0x7f2acb6f2676] Microsoft::MSR::CNTK::GPUMatrix:: Resize (unsigned long, unsigned long, bool) + 0xf6
[0x7f2acb5f5509] Microsoft::MSR::CNTK::Matrix:: Resize (unsigned long, unsigned long, unsigned long, bool, bool) + 0xc9
[0x7f2ad3778b09] Microsoft::MSR::CNTK::LearnableParameter:: InitShape (Microsoft::MSR::CNTK::TensorShape const&) + 0x309
[0x7f2ad38c7d72] std::shared_ptr<Microsoft::MSR::CNTK::ComputationNode> Microsoft::MSR::CNTK::ComputationNetworkBuilder:: TypedCreateLearnableParameter (std::__cxx11::basic_string<wchar_t,std::char_traits<wchar_t>,std::allocator<wchar_t>> const&, Microsoft::MSR::CNTK::TensorShape const&) + 0x1b2
[0x7f2ad3544f35] std::shared_ptrMicrosoft::MSR::CNTK::ComputationNodeBase CNTK::CompositeFunction:: CreateLearnableParameterFromVariable (CNTK::Variable const&, Microsoft::MSR::CNTK::ComputationNetworkBuilder&, CNTK::NDShape const&, std::__cxx11::basic_string<wchar_t,std::char_traits<wchar_t>,std::allocator<wchar_t>> const&) + 0x65
[0x7f2ad35ee328] std::shared_ptrMicrosoft::MSR::CNTK::ComputationNodeBase CNTK::CompositeFunction:: GetNode (CNTK::Variable const&, std::shared_ptrMicrosoft::MSR::CNTK::ComputationNetwork&, Microsoft::MSR::CNTK::ComputationNetworkBuilder&, std::unordered_map<CNTK::Variable,CNTK::Variable,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,CNTK::Variable>>> const&, std::unordered_map<CNTK::Variable,std::shared_ptrMicrosoft::MSR::CNTK::ComputationNodeBase,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,std::shared_ptrMicrosoft::MSR::CNTK::ComputationNodeBase>>>&, std::unordered_map<CNTK::Variable,bool,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,bool>>>&, std::unordered_set<CNTK::Variable,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocatorCNTK::Variable> const&, bool) + 0x2e8
[0x7f2ad35f2e08] std::shared_ptrMicrosoft::MSR::CNTK::ComputationNodeBase CNTK::CompositeFunction:: GetOutputVariableNode (CNTK::Variable const&, std::shared_ptrMicrosoft::MSR::CNTK::ComputationNetwork&, Microsoft::MSR::CNTK::ComputationNetworkBuilder&, std::unordered_map<CNTK::Variable,CNTK::Variable,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,CNTK::Variable>>> const&, std::unordered_map<CNTK::Variable,std::shared_ptrMicrosoft::MSR::CNTK::ComputationNodeBase,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,std::shared_ptrMicrosoft::MSR::CNTK::ComputationNodeBase>>>&, std::unordered_map<CNTK::Variable,bool,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,bool>>>&, std::unordered_set<CNTK::Variable,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocatorCNTK::Variable> const&, bool) + 0x238
[0x7f2ad35ee65e] std::shared_ptrMicrosoft::MSR::CNTK::ComputationNodeBase CNTK::CompositeFunction:: GetNode (CNTK::Variable const&, std::shared_ptrMicrosoft::MSR::CNTK::ComputationNetwork&, Microsoft::MSR::CNTK::ComputationNetworkBuilder&, std::unordered_map<CNTK::Variable,CNTK::Variable,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,CNTK::Variable>>> const&, std::unordered_map<CNTK::Variable,std::shared_ptrMicrosoft::MSR::CNTK::ComputationNodeBase,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,std::shared_ptrMicrosoft::MSR::CNTK::ComputationNodeBase>>>&, std::unordered_map<CNTK::Variable,bool,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,bool>>>&, std::unordered_set<CNTK::Variable,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocatorCNTK::Variable> const&, bool) + 0x61e
[0x7f2ad35f2e08] std::shared_ptrMicrosoft::MSR::CNTK::ComputationNodeBase CNTK::CompositeFunction:: GetOutputVariableNode (CNTK::Variable const&, std::shared_ptrMicrosoft::MSR::CNTK::ComputationNetwork&, Microsoft::MSR::CNTK::ComputationNetworkBuilder&, std::unordered_map<CNTK::Variable,CNTK::Variable,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,CNTK::Variable>>> const&, std::unordered_map<CNTK::Variable,std::shared_ptrMicrosoft::MSR::CNTK::ComputationNodeBase,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,std::shared_ptrMicrosoft::MSR::CNTK::ComputationNodeBase>>>&, std::unordered_map<CNTK::Variable,bool,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,bool>>>&, std::unordered_set<CNTK::Variable,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocatorCNTK::Variable> const&, bool) + 0x238
[0x7f2ad35ee65e] std::shared_ptrMicrosoft::MSR::CNTK::ComputationNodeBase CNTK::CompositeFunction:: GetNode (CNTK::Variable const&, std::shared_ptrMicrosoft::MSR::CNTK::ComputationNetwork&, Microsoft::MSR::CNTK::ComputationNetworkBuilder&, std::unordered_map<CNTK::Variable,CNTK::Variable,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,CNTK::Variable>>> const&, std::unordered_map<CNTK::Variable,std::shared_ptrMicrosoft::MSR::CNTK::ComputationNodeBase,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,std::shared_ptrMicrosoft::MSR::CNTK::ComputationNodeBase>>>&, std::unordered_map<CNTK::Variable,bool,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,bool>>>&, std::unordered_set<CNTK::Variable,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocatorCNTK::Variable> const&, bool) + 0x61e
[0x7f2ad35f2e08] std::shared_ptrMicrosoft::MSR::CNTK::ComputationNodeBase CNTK::CompositeFunction:: GetOutputVariableNode (CNTK::Variable const&, std::shared_ptrMicrosoft::MSR::CNTK::ComputationNetwork&, Microsoft::MSR::CNTK::ComputationNetworkBuilder&, std::unordered_map<CNTK::Variable,CNTK::Variable,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,CNTK::Variable>>> const&, std::unordered_map<CNTK::Variable,std::shared_ptrMicrosoft::MSR::CNTK::ComputationNodeBase,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,std::shared_ptrMicrosoft::MSR::CNTK::ComputationNodeBase>>>&, std::unordered_map<CNTK::Variable,bool,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,bool>>>&, std::unordered_set<CNTK::Variable,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocatorCNTK::Variable> const&, bool) + 0x238
[0x7f2ad35ee65e] std::shared_ptrMicrosoft::MSR::CNTK::ComputationNodeBase CNTK::CompositeFunction:: GetNode (CNTK::Variable const&, std::shared_ptrMicrosoft::MSR::CNTK::ComputationNetwork&, Microsoft::MSR::CNTK::ComputationNetworkBuilder&, std::unordered_map<CNTK::Variable,CNTK::Variable,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,CNTK::Variable>>> const&, std::unordered_map<CNTK::Variable,std::shared_ptrMicrosoft::MSR::CNTK::ComputationNodeBase,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,std::shared_ptrMicrosoft::MSR::CNTK::ComputationNodeBase>>>&, std::unordered_map<CNTK::Variable,bool,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,bool>>>&, std::unordered_set<CNTK::Variable,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocatorCNTK::Variable> const&, bool) + 0x61e
[0x7f2ad35f2e08] std::shared_ptrMicrosoft::MSR::CNTK::ComputationNodeBase CNTK::CompositeFunction:: GetOutputVariableNode (CNTK::Variable const&, std::shared_ptrMicrosoft::MSR::CNTK::ComputationNetwork&, Microsoft::MSR::CNTK::ComputationNetworkBuilder&, std::unordered_map<CNTK::Variable,CNTK::Variable,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,CNTK::Variable>>> const&, std::unordered_map<CNTK::Variable,std::shared_ptrMicrosoft::MSR::CNTK::ComputationNodeBase,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,std::shared_ptrMicrosoft::MSR::CNTK::ComputationNodeBase>>>&, std::unordered_map<CNTK::Variable,bool,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,bool>>>&, std::unordered_set<CNTK::Variable,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocatorCNTK::Variable> const&, bool) + 0x238
[0x7f2ad35ee65e] std::shared_ptrMicrosoft::MSR::CNTK::ComputationNodeBase CNTK::CompositeFunction:: GetNode (CNTK::Variable const&, std::shared_ptrMicrosoft::MSR::CNTK::ComputationNetwork&, Microsoft::MSR::CNTK::ComputationNetworkBuilder&, std::unordered_map<CNTK::Variable,CNTK::Variable,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,CNTK::Variable>>> const&, std::unordered_map<CNTK::Variable,std::shared_ptrMicrosoft::MSR::CNTK::ComputationNodeBase,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,std::shared_ptrMicrosoft::MSR::CNTK::ComputationNodeBase>>>&, std::unordered_map<CNTK::Variable,bool,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,bool>>>&, std::unordered_set<CNTK::Variable,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocatorCNTK::Variable> const&, bool) + 0x61e
[0x7f2ad35f2e08] std::shared_ptrMicrosoft::MSR::CNTK::ComputationNodeBase CNTK::CompositeFunction:: GetOutputVariableNode (CNTK::Variable const&, std::shared_ptrMicrosoft::MSR::CNTK::ComputationNetwork&, Microsoft::MSR::CNTK::ComputationNetworkBuilder&, std::unordered_map<CNTK::Variable,CNTK::Variable,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,CNTK::Variable>>> const&, std::unordered_map<CNTK::Variable,std::shared_ptrMicrosoft::MSR::CNTK::ComputationNodeBase,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,std::shared_ptrMicrosoft::MSR::CNTK::ComputationNodeBase>>>&, std::unordered_map<CNTK::Variable,bool,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,bool>>>&, std::unordered_set<CNTK::Variable,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocatorCNTK::Variable> const&, bool) + 0x238
[0x7f2ad35ee65e] std::shared_ptrMicrosoft::MSR::CNTK::ComputationNodeBase CNTK::CompositeFunction:: GetNode (CNTK::Variable const&, std::shared_ptrMicrosoft::MSR::CNTK::ComputationNetwork&, Microsoft::MSR::CNTK::ComputationNetworkBuilder&, std::unordered_map<CNTK::Variable,CNTK::Variable,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,CNTK::Variable>>> const&, std::unordered_map<CNTK::Variable,std::shared_ptrMicrosoft::MSR::CNTK::ComputationNodeBase,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,std::shared_ptrMicrosoft::MSR::CNTK::ComputationNodeBase>>>&, std::unordered_map<CNTK::Variable,bool,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,bool>>>&, std::unordered_set<CNTK::Variable,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocatorCNTK::Variable> const&, bool) + 0x61e
[0x7f2ad35f2e08] std::shared_ptrMicrosoft::MSR::CNTK::ComputationNodeBase CNTK::CompositeFunction:: GetOutputVariableNode (CNTK::Variable const&, std::shared_ptrMicrosoft::MSR::CNTK::ComputationNetwork&, Microsoft::MSR::CNTK::ComputationNetworkBuilder&, std::unordered_map<CNTK::Variable,CNTK::Variable,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,CNTK::Variable>>> const&, std::unordered_map<CNTK::Variable,std::shared_ptrMicrosoft::MSR::CNTK::ComputationNodeBase,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,std::shared_ptrMicrosoft::MSR::CNTK::ComputationNodeBase>>>&, std::unordered_map<CNTK::Variable,bool,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,bool>>>&, std::unordered_set<CNTK::Variable,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocatorCNTK::Variable> const&, bool) + 0x238
[0x7f2ad35ee65e] std::shared_ptrMicrosoft::MSR::CNTK::ComputationNodeBase CNTK::CompositeFunction:: GetNode (CNTK::Variable const&, std::shared_ptrMicrosoft::MSR::CNTK::ComputationNetwork&, Microsoft::MSR::CNTK::ComputationNetworkBuilder&, std::unordered_map<CNTK::Variable,CNTK::Variable,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,CNTK::Variable>>> const&, std::unordered_map<CNTK::Variable,std::shared_ptrMicrosoft::MSR::CNTK::ComputationNodeBase,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,std::shared_ptrMicrosoft::MSR::CNTK::ComputationNodeBase>>>&, std::unordered_map<CNTK::Variable,bool,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,bool>>>&, std::unordered_set<CNTK::Variable,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocatorCNTK::Variable> const&, bool) + 0x61e
[0x7f2ad35f2e08] std::shared_ptrMicrosoft::MSR::CNTK::ComputationNodeBase CNTK::CompositeFunction:: GetOutputVariableNode (CNTK::Variable const&, std::shared_ptrMicrosoft::MSR::CNTK::ComputationNetwork&, Microsoft::MSR::CNTK::ComputationNetworkBuilder&, std::unordered_map<CNTK::Variable,CNTK::Variable,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,CNTK::Variable>>> const&, std::unordered_map<CNTK::Variable,std::shared_ptrMicrosoft::MSR::CNTK::ComputationNodeBase,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,std::shared_ptrMicrosoft::MSR::CNTK::ComputationNodeBase>>>&, std::unordered_map<CNTK::Variable,bool,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,bool>>>&, std::unordered_set<CNTK::Variable,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocatorCNTK::Variable> const&, bool) + 0x238
[0x7f2ad35ee65e] std::shared_ptrMicrosoft::MSR::CNTK::ComputationNodeBase CNTK::CompositeFunction:: GetNode (CNTK::Variable const&, std::shared_ptrMicrosoft::MSR::CNTK::ComputationNetwork&, Microsoft::MSR::CNTK::ComputationNetworkBuilder&, std::unordered_map<CNTK::Variable,CNTK::Variable,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,CNTK::Variable>>> const&, std::unordered_map<CNTK::Variable,std::shared_ptrMicrosoft::MSR::CNTK::ComputationNodeBase,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,std::shared_ptrMicrosoft::MSR::CNTK::ComputationNodeBase>>>&, std::unordered_map<CNTK::Variable,bool,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,bool>>>&, std::unordered_set<CNTK::Variable,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocatorCNTK::Variable> const&, bool) + 0x61e
[0x7f2ad35efa51] std::pair<std::shared_ptrMicrosoft::MSR::CNTK::ComputationNetwork,std::unordered_map<CNTK::Variable,std::shared_ptrMicrosoft::MSR::CNTK::ComputationNodeBase,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,std::shared_ptrMicrosoft::MSR::CNTK::ComputationNodeBase>>>> CNTK::CompositeFunction:: CreateComputationNetwork (std::shared_ptrCNTK::Function const&, CNTK::DeviceDescriptor const&, std::unordered_set<CNTK::Variable,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocatorCNTK::Variable> const&, std::unordered_map<CNTK::Variable,CNTK::Variable,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,CNTK::Variable>>> const&, std::unordered_set<CNTK::Variable,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocatorCNTK::Variable> const&, bool) + 0x251
[0x7f2ad35f18fd] std::shared_ptrMicrosoft::MSR::CNTK::ComputationNetwork CNTK::CompositeFunction:: GetComputationNetwork (CNTK::DeviceDescriptor const&, std::unordered_set<CNTK::Variable,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocatorCNTK::Variable> const&, std::unordered_set<CNTK::Variable,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocatorCNTK::Variable> const&, std::unordered_set<CNTK::Variable,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocatorCNTK::Variable> const&, bool) + 0x69d
[0x7f2ad3500e9f] CNTK::CompositeFunction:: Forward (std::unordered_map<CNTK::Variable,std::shared_ptrCNTK::Value,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,std::shared_ptrCNTK::Value>>> const&, std::unordered_map<CNTK::Variable,std::shared_ptrCNTK::Value,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,std::shared_ptrCNTK::Value>>>&, CNTK::DeviceDescriptor const&, std::unordered_set<CNTK::Variable,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocatorCNTK::Variable> const&, std::unordered_set<CNTK::Variable,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocatorCNTK::Variable> const&) + 0xf9f
[0x7f2ad3498603] CNTK::Function:: Forward (std::unordered_map<CNTK::Variable,std::shared_ptrCNTK::Value,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,std::shared_ptrCNTK::Value>>> const&, std::unordered_map<CNTK::Variable,std::shared_ptrCNTK::Value,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocator<std::pair<CNTK::Variable const,std::shared_ptrCNTK::Value>>>&, CNTK::DeviceDescriptor const&, std::unordered_set<CNTK::Variable,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocatorCNTK::Variable> const&, std::unordered_set<CNTK::Variable,std::hashCNTK::Variable,std::equal_toCNTK::Variable,std::allocatorCNTK::Variable> const&) + 0x93
[0x7f2ad41c92bd] + 0x6a2bd
[0x7f2b052425e9] PyCFunction_Call + 0xf9
[0x7f2b052c77c0] PyEval_EvalFrameEx + 0x6ba0
[0x7f2b052cab49] + 0x144b49
[0x7f2b052c9df5] PyEval_EvalFrameEx + 0x91d5
[0x7f2b052cab49] + 0x144b49
[0x7f2b052cacd8] PyEval_EvalCodeEx + 0x48
[0x7f2b05220661] + 0x9a661
[0x7f2b051ed236] PyObject_Call + 0x56
[0x7f2b052c7234] PyEval_EvalFrameEx + 0x6614
[0x7f2b052cab49] + 0x144b49
[0x7f2b052c9df5] PyEval_EvalFrameEx + 0x91d5
[0x7f2b052cab49] + 0x144b49
[0x7f2b052c9df5] PyEval_EvalFrameEx + 0x91d5
[0x7f2b052cab49] + 0x144b49
[0x7f2b052c9df5] PyEval_EvalFrameEx + 0x91d5
[0x7f2b052cab49] + 0x144b49
[0x7f2b052c9df5] PyEval_EvalFrameEx + 0x91d5
[0x7f2b052cab49] + 0x144b49
[0x7f2b052cacd8] PyEval_EvalCodeEx + 0x48
[0x7f2b052cad1b] PyEval_EvalCode + 0x3b
[0x7f2b052f0020] PyRun_FileExFlags + 0x130
[0x7f2b052f1623] PyRun_SimpleFileExFlags + 0x173
[0x7f2b0530c8c7] Py_Main + 0xca7
[0x400add] main + 0x15d
[0x7f2b042a7830] __libc_start_main + 0xf0
[0x4008b9] `