sorry, I can't see any differences between pytorch regular ReLU and customized GuidedBackpropReLU, Who can teach me? Thanks.
here is the code, and the outputs from them are the same, WHY? If, they are same, then why we use GuidedBackpropReLU?
import torch
from torch.autograd import Function
class GuidedBackpropReLU(Function):
'''特殊的ReLU,区别在于反向传播时候只考虑大于零的输入和大于零的梯度'''
@staticmethod
def forward(ctx, input_img): # torch.Size([1, 64, 112, 112])
positive_mask = (input_img > 0).type_as(input_img) # torch.Size([1, 64, 112, 112])
# output = torch.addcmul(torch.zeros(input_img.size()).type_as(input_img), input_img, positive_mask)
output = input_img * positive_mask # 这行代码和上一行的功能相同
ctx.save_for_backward(input_img, output)
return output # torch.Size([1, 64, 112, 112])
# # 上部分定义的函数功能和以下定义的函数一致
# @staticmethod
# def forward(ctx, input_img): # torch.Size([1, 64, 112, 112])
# output = torch.clamp(input_img, min=0.0)
# # print('函数中的输入张量requires_grad',input_img.requires_grad)
# ctx.save_for_backward(input_img, output)
# return output # torch.Size([1, 64, 112, 112])
@staticmethod
def backward(ctx, grad_output): # torch.Size([1, 2048, 7, 7])
input_img, output = ctx.saved_tensors # torch.Size([1, 2048, 7, 7]) torch.Size([1, 2048, 7, 7])
# grad_input = None # 这行代码没作用
positive_mask_1 = (input_img > 0).type_as(grad_output) # torch.Size([1, 2048, 7, 7]) 输入的特征大于零
positive_mask_2 = (grad_output > 0).type_as(grad_output) # torch.Size([1, 2048, 7, 7]) 梯度大于零
grad_input = torch.addcmul(
torch.zeros(input_img.size()).type_as(input_img),
torch.addcmul(
torch.zeros(input_img.size()).type_as(input_img),
grad_output,
positive_mask_1
),
positive_mask_2
)
# grad_input = grad_output * positive_mask_1 * positive_mask_2 # 这行代码的作用和上一行代码相同
return grad_input
torch.manual_seed(seed=20200910)
size = (100,500)
input_data_1 = input = torch.randn(*size, requires_grad=True)
torch.manual_seed(seed=20200910)
input_data_2 = input = torch.randn(*size, requires_grad=True)
torch.manual_seed(seed=20200910)
input_data_3 = input = torch.randn(*size, requires_grad=True)
print('这三个输入数据的维度分别是:', input_data_1.shape, input_data_2.shape, input_data_3.shape)
# print(input_data_1)
# print(input_data_2)
# print(input_data_3)
loss_1 = torch.sum(torch.nn.ReLU()(input_data_1))
loss_2 = torch.sum(torch.nn.functional.relu(input_data_2))
loss_3 = torch.sum(GuidedBackpropReLU.apply(input_data_3))
loss_1.backward()
loss_2.backward()
loss_3.backward()
print(loss_1, loss_2, loss_3)
print(loss_1.item(), loss_2.item(), loss_3.item())
print('三个损失值是否相等', loss_1.item() == loss_2.item() == loss_3.item())
print('简略打印三个梯度信息...')
print(input_data_1.grad)
print(input_data_2.grad)
print(input_data_3.grad)
print('这三个梯度的维度分别是:', input_data_1.grad.shape, input_data_2.grad.shape, input_data_3.grad.shape)
print('检查这三个梯度是否两两相等...')
print(torch.equal(input_data_1.grad, input_data_2.grad))
print(torch.equal(input_data_1.grad, input_data_3.grad))
print(torch.equal(input_data_2.grad, input_data_3.grad))
Windows PowerShell
版权所有 (C) Microsoft Corporation。保留所有权利。
尝试新的跨平台 PowerShell https://aka.ms/pscore6
加载个人及系统配置文件用了 850 毫秒。
(base) PS C:\Users\chenxuqi\Desktop\News4cxq\test4cxq> conda activate ssd4pytorch1_2_0
(ssd4pytorch1_2_0) PS C:\Users\chenxuqi\Desktop\News4cxq\test4cxq> & 'D:\Anaconda3\envs\ssd4pytorch1_2_0\python.exe' 'c:\Users\chenxuqi\.vscode\extensions\ms-python.python-2021.1.502429796\pythonFiles\lib\python\debugpy\launcher' '60985' '--' 'c:\Users\chenxuqi\Desktop\News4cxq\test4cxq\testReLU.py'
这三个输入数据的维度分别是: torch.Size([100, 500]) torch.Size([100, 500]) torch.Size([100, 500])
tensor(19912.3828, grad_fn=<SumBackward0>) tensor(19912.3828, grad_fn=<SumBackward0>) tensor(19912.3828, grad_fn=<SumBackward0>)
19912.3828125 19912.3828125 19912.3828125
三个损失值是否相等 True
简略打印三个梯度信息...
tensor([[1., 1., 1., ..., 1., 1., 1.],
[1., 0., 0., ..., 0., 1., 1.],
[0., 1., 0., ..., 1., 1., 1.],
...,
[0., 1., 1., ..., 1., 0., 1.],
[0., 1., 1., ..., 1., 1., 1.],
[0., 0., 1., ..., 1., 1., 0.]])
tensor([[1., 1., 1., ..., 1., 1., 1.],
[1., 0., 0., ..., 0., 1., 1.],
[0., 1., 0., ..., 1., 1., 1.],
...,
[0., 1., 1., ..., 1., 0., 1.],
[0., 1., 1., ..., 1., 1., 1.],
[0., 0., 1., ..., 1., 1., 0.]])
tensor([[1., 1., 1., ..., 1., 1., 1.],
[1., 0., 0., ..., 0., 1., 1.],
[0., 1., 0., ..., 1., 1., 1.],
...,
[0., 1., 1., ..., 1., 0., 1.],
[0., 1., 1., ..., 1., 1., 1.],
[0., 0., 1., ..., 1., 1., 0.]])
这三个梯度的维度分别是: torch.Size([100, 500]) torch.Size([100, 500]) torch.Size([100, 500])
检查这三个梯度是否两两相等...
True
True
True
(ssd4pytorch1_2_0) PS C:\Users\chenxuqi\Desktop\News4cxq\test4cxq>