Extraction not working with RTX card about deepfacelab HOT 23 CLOSED

Here's the MT error:

Exception while initialization: Traceback (most recent call last):
File "L:\Archives\deepfake\JF_internal\bin\lib\site-packages\tensorflow\python\client\session.py", line 1322, in _do_call
return fn(*args)
File "L:\Archives\deepfake\JF_internal\bin\lib\site-packages\tensorflow\python\client\session.py", line 1307, in _run_fn
options, feed_dict, fetch_list, target_list, run_metadata)
File "L:\Archives\deepfake\JF_internal\bin\lib\site-packages\tensorflow\python\client\session.py", line 1409, in _call_tf_sessionrun
run_metadata)
tensorflow.python.framework.errors_impl.InternalError: Blas SGEMM launch failed : m=19044, n=4, k=32
[[Node: pnet2/conv4-2/Conv2D = Conv2D[T=DT_FLOAT, data_format="NHWC", dilations=[1, 1, 1, 1], padding="SAME", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true, _device="/job:localhost/replica:0/task:0/device:GPU:0"](pnet2/PReLU3/add, pnet2/conv4-2/weights/read)]]
[[Node: pnet2/conv4-2/BiasAdd/_203 = _Recvclient_terminated=false, recv_device="/job:localhost/replica:0/task:0/device:CPU:0", send_device="/job:localhost/replica:0/task:0/device:GPU:0", send_device_incarnation=1, tensor_name="edge_87_pnet2/conv4-2/BiasAdd", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/device:CPU:0"]]

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "L:\Archives\deepfake\JF_internal\bin\DeepFaceLab\utils\SubprocessorBase.py", line 215, in subprocess
fail_message = self.onClientInitialize(client_dict)
File "L:\Archives\deepfake\JF_internal\bin\DeepFaceLab\mainscripts\Extractor.py", line 251, in onClientInitialize
self.e.enter()
File "L:\Archives\deepfake\JF_internal\bin\DeepFaceLab\facelib\MTCExtractor.py", line 48, in enter
faces, pnts = detect_face ( np.zeros ( (self.scale_to, self.scale_to, 3)), self.min_face_size, self.pnet_fun, self.rnet_fun, self.onet_fun, [ self.thresh1, self.thresh2, self.thresh3 ], self.scale_factor )
File "L:\Archives\deepfake\JF_internal\bin\DeepFaceLab\facelib\mtcnn.py", line 309, in detect_face
out = pnet([img_y])
File "L:\Archives\deepfake\JF_internal\bin\lib\site-packages\keras\backend\tensorflow_backend.py", line 2482, in call
**self.session_kwargs)
File "L:\Archives\deepfake\JF_internal\bin\lib\site-packages\tensorflow\python\client\session.py", line 900, in run
run_metadata_ptr)
File "L:\Archives\deepfake\JF_internal\bin\lib\site-packages\tensorflow\python\client\session.py", line 1135, in _run
feed_dict_tensor, options, run_metadata)
File "L:\Archives\deepfake\JF_internal\bin\lib\site-packages\tensorflow\python\client\session.py", line 1316, in _do_run
run_metadata)
File "L:\Archives\deepfake\JF_internal\bin\lib\site-packages\tensorflow\python\client\session.py", line 1335, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.InternalError: Blas SGEMM launch failed : m=19044, n=4, k=32
[[Node: pnet2/conv4-2/Conv2D = Conv2D[T=DT_FLOAT, data_format="NHWC", dilations=[1, 1, 1, 1], padding="SAME", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true, _device="/job:localhost/replica:0/task:0/device:GPU:0"](pnet2/PReLU3/add, pnet2/conv4-2/weights/read)]]
[[Node: pnet2/conv4-2/BiasAdd/_203 = _Recvclient_terminated=false, recv_device="/job:localhost/replica:0/task:0/device:CPU:0", send_device="/job:localhost/replica:0/task:0/device:GPU:0", send_device_incarnation=1, tensor_name="edge_87_pnet2/conv4-2/BiasAdd", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/device:CPU:0"]]

Caused by op 'pnet2/conv4-2/Conv2D', defined at:
File "", line 1, in
File "multiprocessing\spawn.py", line 105, in spawn_main
File "multiprocessing\spawn.py", line 118, in _main
File "multiprocessing\process.py", line 258, in _bootstrap
File "multiprocessing\process.py", line 93, in run
File "L:\Archives\deepfake\JF_internal\bin\DeepFaceLab\utils\SubprocessorBase.py", line 215, in subprocess
fail_message = self.onClientInitialize(client_dict)
File "L:\Archives\deepfake\JF_internal\bin\DeepFaceLab\mainscripts\Extractor.py", line 251, in onClientInitialize
self.e.enter()
File "L:\Archives\deepfake\JF_internal\bin\DeepFaceLab\facelib\MTCExtractor.py", line 33, in enter
pnet2 = PNet(self.tf, {'data':data})
File "L:\Archives\deepfake\JF_internal\bin\DeepFaceLab\facelib\mtcnn.py", line 75, in init
self.setup()
File "L:\Archives\deepfake\JF_internal\bin\DeepFaceLab\facelib\mtcnn.py", line 232, in setup
.conv(1, 1, 4, 1, 1, relu=False, name='conv4-2'))
File "L:\Archives\deepfake\JF_internal\bin\DeepFaceLab\facelib\mtcnn.py", line 52, in layer_decorated
layer_output = op(self, layer_input, *args, **kwargs)
File "L:\Archives\deepfake\JF_internal\bin\DeepFaceLab\facelib\mtcnn.py", line 158, in conv
output = convolve(inp, kernel)
File "L:\Archives\deepfake\JF_internal\bin\DeepFaceLab\facelib\mtcnn.py", line 154, in
convolve = lambda i, k: self.tf.nn.conv2d(i, k, [1, s_h, s_w, 1], padding=padding)
File "L:\Archives\deepfake\JF_internal\bin\lib\site-packages\tensorflow\python\ops\gen_nn_ops.py", line 1042, in conv2d
data_format=data_format, dilations=dilations, name=name)
File "L:\Archives\deepfake\JF_internal\bin\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 787, in _apply_op_helper
op_def=op_def)
File "L:\Archives\deepfake\JF_internal\bin\lib\site-packages\tensorflow\python\framework\ops.py", line 3392, in create_op
op_def=op_def)
File "L:\Archives\deepfake\JF_internal\bin\lib\site-packages\tensorflow\python\framework\ops.py", line 1718, in init
self._traceback = self._graph._extract_stack() # pylint: disable=protected-access

InternalError (see above for traceback): Blas SGEMM launch failed : m=19044, n=4, k=32
[[Node: pnet2/conv4-2/Conv2D = Conv2D[T=DT_FLOAT, data_format="NHWC", dilations=[1, 1, 1, 1], padding="SAME", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true, _device="/job:localhost/replica:0/task:0/device:GPU:0"](pnet2/PReLU3/add, pnet2/conv4-2/weights/read)]]
[[Node: pnet2/conv4-2/BiasAdd/_203 = _Recvclient_terminated=false, recv_device="/job:localhost/replica:0/task:0/device:CPU:0", send_device="/job:localhost/replica:0/task:0/device:GPU:0", send_device_incarnation=1, tensor_name="edge_87_pnet2/conv4-2/BiasAdd", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/device:CPU:0"]]

You have no capable GPUs. Try to close programs which can consume VRAM, and run again.
None
Performing 2nd pass...
Running on GeForce RTX 2080 Ti.
0it [00:00, ?it/s]

Images found: 1330
Faces detected: 0

Press any key to continue . . .

Thanks for looking and no worries if it's not fixable!

from deepfacelab.

WE ARE THE SAME

from deepfacelab.

Running on GeForce RTX 2080 Ti.
Exception while initialization: Traceback (most recent call last):
File "G:\DeepFaceLab_build_07_08_2018\DeepFaceLabTorrent_internal\bin\DeepFaceLab\utils\SubprocessorBase.py", line 215, in subprocess
fail_message = self.onClientInitialize(client_dict)
File "G:\DeepFaceLab_build_07_08_2018\DeepFaceLabTorrent_internal\bin\DeepFaceLab\mainscripts\Extractor.py", line 251, in onClientInitialize
self.e.enter()
File "G:\DeepFaceLab_build_07_08_2018\DeepFaceLabTorrent_internal\bin\DeepFaceLab\facelib\DLIBExtractor.py", line 17, in enter
self.dlib_cnn_face_detector ( np.zeros ( (self.scale_to, self.scale_to, 3), dtype=np.uint8), 0 )
RuntimeError: Error while calling cudnnConvolutionForward( context(), &alpha, descriptor(data), data.device(), (const cudnnFilterDescriptor_t)filter_handle, filters.device(), (const cudnnConvolutionDescriptor_t)conv_handle, (cudnnConvolutionFwdAlgo_t)forward_algo, forward_workspace, forward_workspace_size_in_bytes, &beta, descriptor(output), output.device()) in file D:\Python365_internal\bin\dlib\dlib\dnn\cudnn_dlibapi.cpp:1056. code: 8, reason: CUDNN_STATUS_EXECUTION_FAILED

You have no capable GPUs. Try to close programs which can consume VRAM, and run again.

from deepfacelab.

Very interested in the 2080ti's performance due to inclusion of tensor cores. Any info on that?

from deepfacelab.

由于包含张量核心，对2080ti的性能非常感兴趣。关于那的任何信息？
The 2080TI training is 50% faster than 1080TI, but it is currently unable to extract faces.

from deepfacelab.

Interesting, where did the 50% figure come from?

from deepfacelab.

有意思的是，50％的数字来自哪里？

I used 1080TI before, after changing the same material after 2080TI, I continued to train, the speed is 50% faster, and LOSS drops very fast, I think the actual is probably more than 50% improvement. Unfortunately, after I finished training, I went to break down the new material and found that I couldn't extract the face. I hope the author can solve this problem as soon as possible, even if it is using the CPU, at least

from deepfacelab.

I have no 2080ti card to test and fix it.

from deepfacelab.

有意思的是，50％的数字来自哪里？

I used 1080TI before, after changing the same material after 2080TI, I continued to train, the speed is 50% faster, and LOSS drops very fast, I think the actual is probably more than 50% improvement. Unfortunately, after I finished training, I went to break down the new material and found that I couldn't extract the face. I hope the author can solve this problem as soon as possible, even if it is using the CPU, at least

That's awesome, not using a pre-trained model or anything? I thought it would need to be updated before utilizing the tensor cores.

from deepfacelab.

iperov, it's probably an issue with the current dlib code or CUDA 9 not being completely compatible with the RTX series hardware. But the repro works fine with Pascal 10 series cards.

from deepfacelab.

I am one of the RTX 2070 user. As mentioned above, using 2070 to train is far more faster than my old 970. But I am getting the same problem as mentioned, I can only train but not extract. It would be nice if we can just use the cpu for extraction and use gpu for trainning.

from deepfacelab.

There are hundreds of people in our group who use the author's software, and many people can't extract their faces because they bought an RTX card. I sincerely hope that the author can think of ways to help us. Thank you

from deepfacelab.

as I said
I have no 2080ti card to test and fix it.

from deepfacelab.

Is there any way to implement CPU extraction? That way you won't need an RTX to test and fix!

from deepfacelab.

cpu extaction will be 40x times slower

from deepfacelab.

really thanks for your effort, I do understand thats not a easy thing to do.

Anyways, I do understand using cpu extracting is much slower, but its still better for no working. Someone like me which sold his/ her old gpu can only use the old face extracted before. So it would be really thankful if adding an option to use cpu to extract

from deepfacelab.

I'm not a coder, but to use your CPU-only to extract the faces, you would need to compile dlib without CUDA support, e.g. python setup.py install --no DLIB_USE_CUDA. But that's outside of this repro, which was built on CUDA 9 and for the GTX series and earlier cards. Extraction will be painfully slow. You really can't expect the developers to fix issues with new hardware if they don't have access to it. When the RTX series cards become more affordable and widely used, support will improve.

from deepfacelab.

@iDavros post error with MT extractor

from deepfacelab.

Here come my mt error,

Running extractor.

Performing 1st pass...
Running on GeForce RTX 2070 #0.
Running on GeForce RTX 2070 #1.
Running on GeForce RTX 2070 #2.
Running on GeForce RTX 2070 #3.
2018-11-14 07:13:40.397058: E T:\src\github\tensorflow\tensorflow\stream_executor\cuda\cuda_blas.cc:654] failed to run cuBLAS routine cublasSgemm_v2: CUBLAS_STATUS_EXECUTION_2018-11-14 07:13:40.421997: E T:\src\github\tensorflow\tensorflow\stream_executor\cuda\cuda_blas.cc:654] failed to run cuBLAS routine cublasSgemm_v2: CUBLAS_STATUS_EXECUTION_FAILED
2018-11-14 07:12018-11-14 07:13:40.467869: E T:\src\github\tensorflow\tensorflow\stream_executor\cuda\cuda_blas.cc:654] failed to run cuBLAS routine cublasSgemm_v2: CUBLAS_STATUS_EXECUTION_FAILED
FAILED
3:40.469065: E T:\src\github\tensorflow\tensorflow\stream_executor\cuda\cuda_blas.cc:654] failed to run cuBLAS routine cublasSgemm_v2: CUBLAS_STATUS_EXECUTION_FAILED
Exception while initialization: Traceback (most recent call last):
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\lib\site-packages\tensorflow\python\client\session.py", line 1322, in _do_call
return fn(*args)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\lib\site-packages\tensorflow\python\client\session.py", line 1307, in _run_fn
options, feed_dict, fetch_list, target_list, run_metadata)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\lib\site-packages\tensorflow\python\client\session.py", line 1409, in _call_tf_sessionrun
run_metadata)
tensorflow.python.framework.errors_impl.InternalError: Blas SGEMM launch failed : m=19044, n=4, k=32
[[Node: pnet2/conv4-2/Conv2D = Conv2D[T=DT_FLOAT, data_format="NHWC", dilations=[1, 1, 1, 1], padding="SAME", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true, _device="/job:localhost/replica:0/task:0/device:GPU:0"](pnet2/PReLU3/add, pnet2/conv4-2/weights/read)]]
[[Node: pnet2/conv4-2/BiasAdd/_203 = _Recvclient_terminated=false, recv_device="/job:localhost/replica:0/task:0/device:CPU:0", send_device="/job:localhost/replica:0/task:0/device:GPU:0", send_device_incarnation=1, tensor_name="edge_87_pnet2/conv4-2/BiasAdd", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/device:CPU:0"]]

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\DeepFaceLab\utils\SubprocessorBase.py", line 215, in subprocess
fail_message = self.onClientInitialize(client_dict)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\DeepFaceLab\mainscripts\Extractor.py", line 251, in onClientInitialize
self.e.enter()
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\DeepFaceLab\facelib\MTCExtractor.py", line 48, in enter
faces, pnts = detect_face ( np.zeros ( (self.scale_to, self.scale_to, 3)), self.min_face_size, self.pnet_fun, self.rnet_fun, self.onet_fun, [ self.thresh1, self.thresh2, self.thresh3 ], self.scale_factor )
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\DeepFaceLab\facelib\mtcnn.py", line 309, in detect_face
out = pnet([img_y])
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\lib\site-packages\keras\backend\tensorflow_backend.py", line 2482, in call
**self.session_kwargs)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\lib\site-packages\tensorflow\python\client\session.py", line 900, in run
run_metadata_ptr)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\lib\site-packages\tensorflow\python\client\session.py", line 1135, in _run
feed_dict_tensor, options, run_metadata)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\lib\site-packages\tensorflow\python\client\session.py", line 1316, in _do_run
run_metadata)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\lib\site-packages\tensorflow\python\client\session.py", line 1335, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.InternalError: Blas SGEMM launch failed : m=19044, n=4, k=32
[[Node: pnet2/conv4-2/Conv2D = Conv2D[T=DT_FLOAT, data_format="NHWC", dilations=[1, 1, 1, 1], padding="SAME", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true, _device="/job:localhost/replica:0/task:0/device:GPU:0"](pnet2/PReLU3/add, pnet2/conv4-2/weights/read)]]
[[Node: pnet2/conv4-2/BiasAdd/_203 = _Recvclient_terminated=false, recv_device="/job:localhost/replica:0/task:0/device:CPU:0", send_device="/job:localhost/replica:0/task:0/device:GPU:0", send_device_incarnation=1, tensor_name="edge_87_pnet2/conv4-2/BiasAdd", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/device:CPU:0"]]

Caused by op 'pnet2/conv4-2/Conv2D', defined at:
File "", line 1, in
File "multiprocessing\spawn.py", line 105, in spawn_main
File "multiprocessing\spawn.py", line 118, in _main
File "multiprocessing\process.py", line 258, in _bootstrap
File "multiprocessing\process.py", line 93, in run
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\DeepFaceLab\utils\SubprocessorBase.py", line 215, in subprocess
fail_message = self.onClientInitialize(client_dict)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\DeepFaceLab\mainscripts\Extractor.py", line 251, in onClientInitialize
self.e.enter()
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\DeepFaceLab\facelib\MTCExtractor.py", line 33, in enter
pnet2 = PNet(self.tf, {'data':data})
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\DeepFaceLab\facelib\mtcnn.py", line 75, in init
self.setup()
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\DeepFaceLab\facelib\mtcnn.py", line 232, in setup
.conv(1, 1, 4, 1, 1, relu=False, name='conv4-2'))
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\DeepFaceLab\facelib\mtcnn.py", line 52, in layer_decorated
layer_output = op(self, layer_input, *args, **kwargs)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\DeepFaceLab\facelib\mtcnn.py", line 158, in conv
output = convolve(inp, kernel)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\DeepFaceLab\facelib\mtcnn.py", line 154, in
convolve = lambda i, k: self.tf.nn.conv2d(i, k, [1, s_h, s_w, 1], padding=padding)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\lib\site-packages\tensorflow\python\ops\gen_nn_ops.py", line 1042, in conv2d
data_format=data_format, dilations=dilations, name=name)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 787, in _apply_op_helper
op_def=op_def)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\lib\site-packages\tensorflow\python\framework\ops.py", line 3392, in create_op
op_def=op_def)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\lib\site-packages\tensorflow\python\framework\ops.py", line 1718, in init
self._traceback = self._graph._extract_stack() # pylint: disable=protected-access

InternalError (see above for traceback): Blas SGEMM launch failed : m=19044, n=4, k=32
[[Node: pnet2/conv4-2/Conv2D = Conv2D[T=DT_FLOAT, data_format="NHWC", dilations=[1, 1, 1, 1], padding="SAME", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true, _device="/job:localhost/replica:0/task:0/device:GPU:0"](pnet2/PReLU3/add, pnet2/conv4-2/weights/read)]]
[[Node: pnet2/conv4-2/BiasAdd/_203 = _Recvclient_terminated=false, recv_device="/job:localhost/replica:0/task:0/device:CPU:0", send_device="/job:localhost/replica:0/task:0/device:GPU:0", send_device_incarnation=1, tensor_name="edge_87_pnet2/conv4-2/BiasAdd", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/device:CPU:0"]]

Exception while initialization: Traceback (most recent call last):
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\lib\site-packages\tensorflow\python\client\session.py", line 1322, in _do_call
return fn(*args)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\lib\site-packages\tensorflow\python\client\session.py", line 1307, in _run_fn
options, feed_dict, fetch_list, target_list, run_metadata)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\lib\site-packages\tensorflow\python\client\session.py", line 1409, in _call_tf_sessionrun
run_metadata)
tensorflow.python.framework.errors_impl.InternalError: Blas SGEMM launch failed : m=19044, n=4, k=32
[[Node: pnet2/conv4-2/Conv2D = Conv2D[T=DT_FLOAT, data_format="NHWC", dilations=[1, 1, 1, 1], padding="SAME", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true, _device="/job:localhost/replica:0/task:0/device:GPU:0"](pnet2/PReLU3/add, pnet2/conv4-2/weights/read)]]

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\DeepFaceLab\utils\SubprocessorBase.py", line 215, in subprocess
fail_message = self.onClientInitialize(client_dict)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\DeepFaceLab\mainscripts\Extractor.py", line 251, in onClientInitialize
self.e.enter()
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\DeepFaceLab\facelib\MTCExtractor.py", line 48, in enter
faces, pnts = detect_face ( np.zeros ( (self.scale_to, self.scale_to, 3)), self.min_face_size, self.pnet_fun, self.rnet_fun, self.onet_fun, [ self.thresh1, self.thresh2, self.thresh3 ], self.scale_factor )
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\DeepFaceLab\facelib\mtcnn.py", line 309, in detect_face
out = pnet([img_y])
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\lib\site-packages\keras\backend\tensorflow_backend.py", line 2482, in call
**self.session_kwargs)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\lib\site-packages\tensorflow\python\client\session.py", line 900, in run
run_metadata_ptr)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\lib\site-packages\tensorflow\python\client\session.py", line 1135, in _run
feed_dict_tensor, options, run_metadata)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\lib\site-packages\tensorflow\python\client\session.py", line 1316, in _do_run
run_metadata)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\lib\site-packages\tensorflow\python\client\session.py", line 1335, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.InternalError: Blas SGEMM launch failed : m=19044, n=4, k=32
[[Node: pnet2/conv4-2/Conv2D = Conv2D[T=DT_FLOAT, data_format="NHWC", dilations=[1, 1, 1, 1], padding="SAME", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true, _device="/job:localhost/replica:0/task:0/device:GPU:0"](pnet2/PReLU3/add, pnet2/conv4-2/weights/read)]]

Caused by op 'pnet2/conv4-2/Conv2D', defined at:
File "", line 1, in
File "multiprocessing\spawn.py", line 105, in spawn_main
File "multiprocessing\spawn.py", line 118, in _main
File "multiprocessing\process.py", line 258, in _bootstrap
File "multiprocessing\process.py", line 93, in run
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\DeepFaceLab\utils\SubprocessorBase.py", line 215, in subprocess
fail_message = self.onClientInitialize(client_dict)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\DeepFaceLab\mainscripts\Extractor.py", line 251, in onClientInitialize
self.e.enter()
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\DeepFaceLab\facelib\MTCExtractor.py", line 33, in enter
pnet2 = PNet(self.tf, {'data':data})
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\DeepFaceLab\facelib\mtcnn.py", line 75, in init
self.setup()
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\DeepFaceLab\facelib\mtcnn.py", line 232, in setup
.conv(1, 1, 4, 1, 1, relu=False, name='conv4-2'))
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\DeepFaceLab\facelib\mtcnn.py", line 52, in layer_decorated
layer_output = op(self, layer_input, *args, **kwargs)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\DeepFaceLab\facelib\mtcnn.py", line 158, in conv
output = convolve(inp, kernel)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\DeepFaceLab\facelib\mtcnn.py", line 154, in
convolve = lambda i, k: self.tf.nn.conv2d(i, k, [1, s_h, s_w, 1], padding=padding)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\lib\site-packages\tensorflow\python\ops\gen_nn_ops.py", line 1042, in conv2d
data_format=data_format, dilations=dilations, name=name)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 787, in _apply_op_helper
op_def=op_def)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\lib\site-packages\tensorflow\python\framework\ops.py", line 3392, in create_op
op_def=op_def)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\lib\site-packages\tensorflow\python\framework\ops.py", line 1718, in init
self._traceback = self._graph._extract_stack() # pylint: disable=protected-access

InternalError (see above for traceback): Blas SGEMM launch failed : m=19044, n=4, k=32
[[Node: pnet2/conv4-2/Conv2D = Conv2D[T=DT_FLOAT, data_format="NHWC", dilations=[1, 1, 1, 1], padding="SAME", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true, _device="/job:localhost/replica:0/task:0/device:GPU:0"](pnet2/PReLU3/add, pnet2/conv4-2/weights/read)]]

Exception while initialization: Traceback (most recent call last):
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\lib\site-packages\tensorflow\python\client\session.py", line 1322, in _do_call
return fn(*args)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\lib\site-packages\tensorflow\python\client\session.py", line 1307, in _run_fn
options, feed_dict, fetch_list, target_list, run_metadata)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\lib\site-packages\tensorflow\python\client\session.py", line 1409, in _call_tf_sessionrun
run_metadata)
tensorflow.python.framework.errors_impl.InternalError: Blas SGEMM launch failed : m=19044, n=4, k=32
[[Node: pnet2/conv4-2/Conv2D = Conv2D[T=DT_FLOAT, data_format="NHWC", dilations=[1, 1, 1, 1], padding="SAME", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true, _device="/job:localhost/replica:0/task:0/device:GPU:0"](pnet2/PReLU3/add, pnet2/conv4-2/weights/read)]]
[[Node: pnet2/conv4-2/BiasAdd/_203 = _Recvclient_terminated=false, recv_device="/job:localhost/replica:0/task:0/device:CPU:0", send_device="/job:localhost/replica:0/task:0/device:GPU:0", send_device_incarnation=1, tensor_name="edge_87_pnet2/conv4-2/BiasAdd", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/device:CPU:0"]]

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\DeepFaceLab\utils\SubprocessorBase.py", line 215, in subprocess
fail_message = self.onClientInitialize(client_dict)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\DeepFaceLab\mainscripts\Extractor.py", line 251, in onClientInitialize
self.e.enter()
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\DeepFaceLab\facelib\MTCExtractor.py", line 48, in enter
faces, pnts = detect_face ( np.zeros ( (self.scale_to, self.scale_to, 3)), self.min_face_size, self.pnet_fun, self.rnet_fun, self.onet_fun, [ self.thresh1, self.thresh2, self.thresh3 ], self.scale_factor )
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\DeepFaceLab\facelib\mtcnn.py", line 309, in detect_face
out = pnet([img_y])
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\lib\site-packages\keras\backend\tensorflow_backend.py", line 2482, in call
**self.session_kwargs)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\lib\site-packages\tensorflow\python\client\session.py", line 900, in run
run_metadata_ptr)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\lib\site-packages\tensorflow\python\client\session.py", line 1135, in _run
feed_dict_tensor, options, run_metadata)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\lib\site-packages\tensorflow\python\client\session.py", line 1316, in _do_run
run_metadata)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\lib\site-packages\tensorflow\python\client\session.py", line 1335, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.InternalError: Blas SGEMM launch failed : m=19044, n=4, k=32
[[Node: pnet2/conv4-2/Conv2D = Conv2D[T=DT_FLOAT, data_format="NHWC", dilations=[1, 1, 1, 1], padding="SAME", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true, _device="/job:localhost/replica:0/task:0/device:GPU:0"](pnet2/PReLU3/add, pnet2/conv4-2/weights/read)]]
[[Node: pnet2/conv4-2/BiasAdd/_203 = _Recvclient_terminated=false, recv_device="/job:localhost/replica:0/task:0/device:CPU:0", send_device="/job:localhost/replica:0/task:0/device:GPU:0", send_device_incarnation=1, tensor_name="edge_87_pnet2/conv4-2/BiasAdd", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/device:CPU:0"]]

Caused by op 'pnet2/conv4-2/Conv2D', defined at:
File "", line 1, in
File "multiprocessing\spawn.py", line 105, in spawn_main
File "multiprocessing\spawn.py", line 118, in _main
File "multiprocessing\process.py", line 258, in _bootstrap
File "multiprocessing\process.py", line 93, in run
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\DeepFaceLab\utils\SubprocessorBase.py", line 215, in subprocess
fail_message = self.onClientInitialize(client_dict)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\DeepFaceLab\mainscripts\Extractor.py", line 251, in onClientInitialize
self.e.enter()
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\DeepFaceLab\facelib\MTCExtractor.py", line 33, in enter
pnet2 = PNet(self.tf, {'data':data})
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\DeepFaceLab\facelib\mtcnn.py", line 75, in init
self.setup()
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\DeepFaceLab\facelib\mtcnn.py", line 232, in setup
.conv(1, 1, 4, 1, 1, relu=False, name='conv4-2'))
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\DeepFaceLab\facelib\mtcnn.py", line 52, in layer_decorated
layer_output = op(self, layer_input, *args, **kwargs)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\DeepFaceLab\facelib\mtcnn.py", line 158, in conv
output = convolve(inp, kernel)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\DeepFaceLab\facelib\mtcnn.py", line 154, in
convolve = lambda i, k: self.tf.nn.conv2d(i, k, [1, s_h, s_w, 1], padding=padding)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\lib\site-packages\tensorflow\python\ops\gen_nn_ops.py", line 1042, in conv2d
data_format=data_format, dilations=dilations, name=name)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 787, in _apply_op_helper
op_def=op_def)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\lib\site-packages\tensorflow\python\framework\ops.py", line 3392, in create_op
op_def=op_def)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\lib\site-packages\tensorflow\python\framework\ops.py", line 1718, in init
self._traceback = self._graph._extract_stack() # pylint: disable=protected-access

InternalError (see above for traceback): Blas SGEMM launch failed : m=19044, n=4, k=32
[[Node: pnet2/conv4-2/Conv2D = Conv2D[T=DT_FLOAT, data_format="NHWC", dilations=[1, 1, 1, 1], padding="SAME", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true, _device="/job:localhost/replica:0/task:0/device:GPU:0"](pnet2/PReLU3/add, pnet2/conv4-2/weights/read)]]
[[Node: pnet2/conv4-2/BiasAdd/_203 = _Recvclient_terminated=false, recv_device="/job:localhost/replica:0/task:0/device:CPU:0", send_device="/job:localhost/replica:0/task:0/device:GPU:0", send_device_incarnation=1, tensor_name="edge_87_pnet2/conv4-2/BiasAdd", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/device:CPU:0"]]

Exception while initialization: Traceback (most recent call last):
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\lib\site-packages\tensorflow\python\client\session.py", line 1322, in _do_call
return fn(*args)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\lib\site-packages\tensorflow\python\client\session.py", line 1307, in _run_fn
options, feed_dict, fetch_list, target_list, run_metadata)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\lib\site-packages\tensorflow\python\client\session.py", line 1409, in _call_tf_sessionrun
run_metadata)
tensorflow.python.framework.errors_impl.InternalError: Blas SGEMM launch failed : m=19044, n=4, k=32
[[Node: pnet2/conv4-2/Conv2D = Conv2D[T=DT_FLOAT, data_format="NHWC", dilations=[1, 1, 1, 1], padding="SAME", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true, _device="/job:localhost/replica:0/task:0/device:GPU:0"](pnet2/PReLU3/add, pnet2/conv4-2/weights/read)]]
[[Node: pnet2/conv4-2/BiasAdd/_203 = _Recvclient_terminated=false, recv_device="/job:localhost/replica:0/task:0/device:CPU:0", send_device="/job:localhost/replica:0/task:0/device:GPU:0", send_device_incarnation=1, tensor_name="edge_87_pnet2/conv4-2/BiasAdd", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/device:CPU:0"]]

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\DeepFaceLab\utils\SubprocessorBase.py", line 215, in subprocess
fail_message = self.onClientInitialize(client_dict)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\DeepFaceLab\mainscripts\Extractor.py", line 251, in onClientInitialize
self.e.enter()
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\DeepFaceLab\facelib\MTCExtractor.py", line 48, in enter
faces, pnts = detect_face ( np.zeros ( (self.scale_to, self.scale_to, 3)), self.min_face_size, self.pnet_fun, self.rnet_fun, self.onet_fun, [ self.thresh1, self.thresh2, self.thresh3 ], self.scale_factor )
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\DeepFaceLab\facelib\mtcnn.py", line 309, in detect_face
out = pnet([img_y])
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\lib\site-packages\keras\backend\tensorflow_backend.py", line 2482, in call
**self.session_kwargs)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\lib\site-packages\tensorflow\python\client\session.py", line 900, in run
run_metadata_ptr)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\lib\site-packages\tensorflow\python\client\session.py", line 1135, in _run
feed_dict_tensor, options, run_metadata)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\lib\site-packages\tensorflow\python\client\session.py", line 1316, in _do_run
run_metadata)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\lib\site-packages\tensorflow\python\client\session.py", line 1335, in _do_call
raise type(e)(node_def, op, message)
tensorflow.python.framework.errors_impl.InternalError: Blas SGEMM launch failed : m=19044, n=4, k=32
[[Node: pnet2/conv4-2/Conv2D = Conv2D[T=DT_FLOAT, data_format="NHWC", dilations=[1, 1, 1, 1], padding="SAME", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true, _device="/job:localhost/replica:0/task:0/device:GPU:0"](pnet2/PReLU3/add, pnet2/conv4-2/weights/read)]]
[[Node: pnet2/conv4-2/BiasAdd/_203 = _Recvclient_terminated=false, recv_device="/job:localhost/replica:0/task:0/device:CPU:0", send_device="/job:localhost/replica:0/task:0/device:GPU:0", send_device_incarnation=1, tensor_name="edge_87_pnet2/conv4-2/BiasAdd", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/device:CPU:0"]]

Caused by op 'pnet2/conv4-2/Conv2D', defined at:
File "", line 1, in
File "multiprocessing\spawn.py", line 105, in spawn_main
File "multiprocessing\spawn.py", line 118, in _main
File "multiprocessing\process.py", line 258, in _bootstrap
File "multiprocessing\process.py", line 93, in run
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\DeepFaceLab\utils\SubprocessorBase.py", line 215, in subprocess
fail_message = self.onClientInitialize(client_dict)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\DeepFaceLab\mainscripts\Extractor.py", line 251, in onClientInitialize
self.e.enter()
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\DeepFaceLab\facelib\MTCExtractor.py", line 33, in enter
pnet2 = PNet(self.tf, {'data':data})
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\DeepFaceLab\facelib\mtcnn.py", line 75, in init
self.setup()
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\DeepFaceLab\facelib\mtcnn.py", line 232, in setup
.conv(1, 1, 4, 1, 1, relu=False, name='conv4-2'))
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\DeepFaceLab\facelib\mtcnn.py", line 52, in layer_decorated
layer_output = op(self, layer_input, *args, **kwargs)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\DeepFaceLab\facelib\mtcnn.py", line 158, in conv
output = convolve(inp, kernel)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\DeepFaceLab\facelib\mtcnn.py", line 154, in
convolve = lambda i, k: self.tf.nn.conv2d(i, k, [1, s_h, s_w, 1], padding=padding)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\lib\site-packages\tensorflow\python\ops\gen_nn_ops.py", line 1042, in conv2d
data_format=data_format, dilations=dilations, name=name)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 787, in _apply_op_helper
op_def=op_def)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\lib\site-packages\tensorflow\python\framework\ops.py", line 3392, in create_op
op_def=op_def)
File "C:\deepfake\DeepFaceLabTorrent_internal\bin\lib\site-packages\tensorflow\python\framework\ops.py", line 1718, in init
self._traceback = self._graph._extract_stack() # pylint: disable=protected-access

InternalError (see above for traceback): Blas SGEMM launch failed : m=19044, n=4, k=32
[[Node: pnet2/conv4-2/Conv2D = Conv2D[T=DT_FLOAT, data_format="NHWC", dilations=[1, 1, 1, 1], padding="SAME", strides=[1, 1, 1, 1], use_cudnn_on_gpu=true, _device="/job:localhost/replica:0/task:0/device:GPU:0"](pnet2/PReLU3/add, pnet2/conv4-2/weights/read)]]
[[Node: pnet2/conv4-2/BiasAdd/_203 = _Recvclient_terminated=false, recv_device="/job:localhost/replica:0/task:0/device:CPU:0", send_device="/job:localhost/replica:0/task:0/device:GPU:0", send_device_incarnation=1, tensor_name="edge_87_pnet2/conv4-2/BiasAdd", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/device:CPU:0"]]

You have no capable GPUs. Try to close programs which can consume VRAM, and run again.
None
Performing 2nd pass...
Running on GeForce RTX 2070.
0it [00:00, ?it/s]

Images found: 167
Faces detected: 0

Thanks for your effort!

from deepfacelab.

modules dlib and tf should be upgraded in order to work with last cuda.

from deepfacelab.

Looking forward to your update.

from deepfacelab.

sry I have no time for that

from deepfacelab.

waiting tf 1.13.0 for fix

from deepfacelab.