对pytorch中的梯度更新方法详解

tensor([[[ 1., 1.],……torch.Size([2, 2, 2])
0 2 * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 
tensor([[[ 2., 2.],…… torch.Size([2, 2, 2])
1 2 * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 
tensor([[[ 3., 3.],…… torch.Size([2, 2, 2])
2 2 * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * 

tensor([[[ 3., 3.],…… torch.Size([2, 2, 2])

tensor([[[ 3., 3.],…… torch.Size([2, 2, 2])

import torch
import torch.nn as nn
from torch.autograd import Variable


class model(nn.Module):
 def __init__(self, w):
  super(model, self).__init__()
  self.w = w

 def forward(self, xx):
  b, c, _, _ = xx.shape
  # extra = xx.device.index + 1 ## 实验（一）
  y = xx.reshape(b, -1).mm(self.w.cuda(xx.device).reshape(-1, 2) * extra)
  return y.reshape(len(xx), -1)


g = 1
x = Variable(torch.ones(2, 1, 2, 2))
# x[1] += 1 ## 实验（二）
w = Variable(torch.ones(2, 2, 2) * 2, requires_grad=True)
# optim = torch.optim.SGD({'params': x},
lr = 0.01
momentum = 0.9
M = model(w)

M = torch.nn.DataParallel(M, device_ids=range(g))

for i in range(3):
 b = len(x)
 z = M(x)
 zz = z.sum(1)
 l = (zz - Variable(torch.ones(b).cuda())).mean()
 # zz.backward(Variable(torch.ones(b).cuda()))
 l.backward()
 print(w.grad, w.grad.shape)
 # w.grad.zero_() ## 实验（三）
 print(i, b, '* * ' * 20)