PyTorch使用Tricks:Dropout,R-Dropout和Multi-Sample Dropout方式

目录

• 1、为什么使用Dropout？
• 2、Dropout的拓展1：R-Dropout
• 3、Dropout的拓展2：Multi-Sample Dropout
• 4、Dropout的拓展3：DropConnect
• 5、Dropout的拓展4：Standout
• 6、Dropout的拓展5：Gaussian Dropout
• 总结

import torch
import torch.nn as nn
 
class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.layer1 = nn.Linear(null, 20)
        self.layer2 = nn.Linear(null, 20)
        self.layer3 = nn.Linear(null, 4)
        self.dropout = nn.Dropout(p=0.5)
 
    def forward(self, x):
        x = F.relu(self.layer1(x))
        x = self.dropout(x)
        x = F.relu(self.layer2(x))
        x = self.dropout(x)
        return self.layer3(x)

import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
 
class RDropoutNN(nn.Module):
    def __init__(self):
        super(RDropoutNN, self).__init__()
        self.fc1 = nn.Linear(null, 256)
        self.dropout = nn.Dropout(0.5)
        self.fc2 = nn.Linear(null, 10)
 
    def forward(self, x, with_dropout=True):
        x = F.relu(self.fc1(x))
        if with_dropout:
            x = self.dropout(x)
        x = self.fc2(x)
        return x
 
def compute_kl_loss(p, q):
    p_log_q = F.kl_div(F.log_softmax(p, dim=1), F.softmax(q, dim=1), reduction='sum')
    q_log_p = F.kl_div(F.log_softmax(q, dim=1), F.softmax(p, dim=1), reduction='sum')
    return (p_log_q + q_log_p) / 2
 
# 假设
model = RDropoutNN()
optimizer = optim.Adam(model.parameters(), lr=0.001)
criterion = nn.CrossEntropyLoss()
 
# 假设有一个数据加载器
# for inputs, labels in data_loader:
    # 模拟数据
inputs = torch.randn(null, 784)  # 假设的输入
labels = torch.randint(null, 10, (null,))  # 假设的标签
 
# 清零梯度
optimizer.zero_grad()
 
# 两次前向传播，每次都使用Dropout
outputs1 = model(inputs, with_dropout=True)
outputs2 = model(inputs, with_dropout=True)
 
# 计算原始损失
loss1 = criterion(outputs1, labels)
loss2 = criterion(outputs2, labels)
original_loss = (loss1 + loss2) / 2
 
# 计算R-Dropout正则化项
kl_loss = compute_kl_loss(outputs1, outputs2)
 
# 最终损失
lambda_kl = 0.5  # R-Dropout正则化项的权重
total_loss = original_loss + lambda_kl * kl_loss
 
# 反向传播和优化
total_loss.backward()
optimizer.step()

class MultiSampleDropout(nn.Module):
    def __init__(self, dropout_rate, num_samples):
        super(MultiSampleDropout, self).__init__()
        self.dropout_rate = dropout_rate
        self.num_samples = num_samples
 
    def forward(self, x):
        outputs = []
        for _ in range(self.num_samples):
            output = F.dropout(x, self.dropout_rate, training=self.training)
            outputs.append(output)
        return torch.mean(torch.stack(outputs), dim=0)

import torch
import torch.nn as nn
import torch.nn.functional as F
 
class DropConnect(nn.Module):
    def __init__(self, input_dim, output_dim, drop_prob=0.5):
        super(DropConnect, self).__init__()
        self.drop_prob = drop_prob
        self.weight = nn.Parameter(torch.randn(input_dim, output_dim))
        self.bias = nn.Parameter(torch.randn(output_dim))
    
    def forward(self, x):
        if self.training:
            # 生成与权重相同形状的掩码
            mask = torch.rand(self.weight.size()) > self.drop_prob
            # 应用DropConnect：用掩码乘以权重
            drop_weight = self.weight * mask.float().to(self.weight.device)
        else:
            # 在测试时不应用DropConnect，但要调整权重以反映丢弃率
            drop_weight = self.weight * (1 - self.drop_prob)
        
        return F.linear(x, drop_weight, self.bias)
 
# 使用DropConnect层的示例网络
class ExampleNet(nn.Module):
    def __init__(self):
        super(ExampleNet, self).__init__()
        self.dropconnect_layer = DropConnect(null, 10, drop_prob=0.5)
        self.fc2 = nn.Linear(null, 2)
    
    def forward(self, x):
        x = F.relu(self.dropconnect_layer(x))
        x = self.fc2(x)
        return x
 
# 示例使用
model = ExampleNet()
input = torch.randn(null, 20)  # 假设的输入
output = model(input)
print(output)

import torch
import torch.nn as nn
 
class Standout(nn.Module):
    def __init__(self, input_dim, output_dim):
        super(Standout, self).__init__()
        self.fc = nn.Linear(input_dim, output_dim)
        # 使用一个简单的全连接层来生成保留概率
        self.prob_fc = nn.Linear(input_dim, output_dim)
    
    def forward(self, x):
        # 计算正常的前向传播
        x_out = self.fc(x)
        
        if self.training:
            # 计算每个神经元的保留概率
            probs = torch.sigmoid(self.prob_fc(x))
            # 生成与激活大小相同的二值掩码
            mask = torch.bernoulli(probs).to(x.device)
            # 应用mask
            x_out = x_out * mask
        # 注意：在评估模式下不应用Standout
        
        return x_out

import torch
import torch.nn as nn
 
class GaussianDropout(nn.Module):
    def __init__(self, sigma=0.1):
        super(GaussianDropout, self).__init__()
        self.sigma = sigma
 
    def forward(self, x):
        if self.training:
            # 生成服从高斯分布 N(null, sigma^2) 的随机变量
            gaussian_noise = torch.normal(1.0, self.sigma, size=x.size()).to(x.device)
            # 应用高斯 dropout
            return x * gaussian_noise
        else:
            # 测试阶段不使用 dropout
            return x
 
# 使用示例
layer = GaussianDropout(sigma=0.1)
input_tensor = torch.randn(null, 20)  # 假设的输入
output = layer(input_tensor)