softmax回归是logistic回归在多分类问题上的推广
原理
网络架构：
常用的方式是独热编码：
如果下面这样，会使得分类器更倾向于把奶牛和耗牛预测到一起，因为预测为海公牛惩罚更大，这样是不合理的。
损失函数：交叉熵损失（这个函数的由来是最小化负对数似然）
如果预测奶牛的概率是1，则log1=0，损失为0，这是我们想要的。也就是，正确概率越大，损失越小。推广到多个样本：
将模型公式代入损失函数，得到关于w,b的损失函数：
然后使用梯度下降法，梯度是：
手动实现
这个代码可以在vscode而不是jupyter notebook中运行。
import torch
import torchvision
from torch.utils import data
from torchvision import transforms
import matplotlib.pyplot as plt
#加载fashion_mnist数据集
def load_data_fashion_mnist(batch_size, resize=None):
"""下载Fashion-MNIST数据集，然后将其加载到内存中"""
trans = [transforms.ToTensor()]
if resize:
trans.insert(0, transforms.Resize(resize))
trans = transforms.Compose(trans)
mnist_train = torchvision.datasets.FashionMNIST(root="../data", train=True, transform=trans, download=True)
mnist_test = torchvision.datasets.FashionMNIST(root="../data", train=False, transform=trans, download=True)
#print(len(mnist_train),len(mnist_test))
return (data.DataLoader(mnist_train, batch_size, shuffle=True),
data.DataLoader(mnist_test, batch_size, shuffle=False)) #windows下不能多进程,linux下可以
#labels数字转为文字标签？
def get_fashion_mnist_labels(labels):
text_labels=['t-shirt', 'trouser', 'pullover', 'dress', 'coat','sandal', 'shirt', 'sneaker', 'bag', 'ankle boot']
return [text_labels[int(i)] for i in labels]
#展示数据集图片的函数
def show_images(imgs, num_rows, num_cols, titles=None, scale=1.5):
"""绘制图像列表"""
figsize = (num_cols * scale, num_rows * scale)
_, axes = plt.subplots(num_rows, num_cols, figsize=figsize)
axes = axes.flatten()
for i, (ax, img) in enumerate(zip(axes, imgs)):
if torch.is_tensor(img):
# 图⽚张量
ax.imshow(img.numpy())
else:
# PIL图⽚
ax.imshow(img)
ax.axes.get_xaxis().set_visible(False)
ax.axes.get_yaxis().set_visible(False)
if titles:
ax.set_title(titles[i])
plt.show()
return axes
#展示数据集，没什么用
# trans = transforms.ToTensor()
# mnist_train = torchvision.datasets.FashionMNIST(root="../data", train=True, transform=trans, download=True)
# X, y = next(iter(data.DataLoader(mnist_train, batch_size=18)))
# show_images(X.reshape(18, 28, 28), 2, 9, titles=get_fashion_mnist_labels(y))
#用于绘图，设置轴
def set_axes(axes, xlabel, ylabel, xlim, ylim, xscale, yscale, legend):
"""设置matplotlib的轴"""
axes.set_xlabel(xlabel)
axes.set_ylabel(ylabel)
axes.set_xscale(xscale)
axes.set_yscale(yscale)
axes.set_xlim(xlim)
axes.set_ylim(ylim)
if legend:
axes.legend(legend)
axes.grid()
#绘制学习的情况
class Animator:
"""在动画中绘制数据"""
def __init__(self, xlabel=None, ylabel=None, legend=None, xlim=None,
ylim=None, xscale='linear', yscale='linear',
fmts=('-', 'm--', 'g-.', 'r:'), nrows=1, ncols=1,
figsize=(3.5, 2.5)):
# 增量地绘制多条线
if legend is None:
legend = []
self.fig, self.axes = plt.subplots(nrows, ncols, figsize=figsize)
if nrows * ncols == 1:
self.axes = [self.axes, ]
# 使⽤lambda函数捕获参数
self.config_axes = lambda: set_axes(self.axes[0], xlabel, ylabel, xlim, ylim, xscale, yscale, legend)
self.X, self.Y, self.fmts = None, None, fmts
def add(self, x, y):
# 向图表中添加多个数据点
if not hasattr(y, "__len__"):
y = [y]
n = len(y)
if not hasattr(x, "__len__"):
x = [x] * n
if not self.X:
self.X = [[] for _ in range(n)]
if not self.Y:
self.Y = [[] for _ in range(n)]
for i, (a, b) in enumerate(zip(x, y)):
if a is not None and b is not None:
self.X[i].append(a)
self.Y[i].append(b)
self.axes[0].cla()
for x, y, fmt in zip(self.X, self.Y, self.fmts):
self.axes[0].plot(x, y, fmt)
self.config_axes()
#display.display(self.fig)
# 通过以下两行代码实现了在PyCharm中显示动图
plt.draw()
#plt.pause(interval=0.001)
#display.clear_output(wait=True)
#批大小
batch_size = 256
#训练和测试的迭代器
train_iter, test_iter = load_data_fashion_mnist(batch_size)
# for X, y in train_iter:
# print(X.shape, X.dtype, y.shape, y.dtype)
# break
num_inputs = 784
num_outputs = 10
#初始化权重和偏置
W = torch.normal(0, 0.01, size=(num_inputs, num_outputs), requires_grad=True)#784*10的权重矩阵
b = torch.zeros(num_outputs, requires_grad=True)
# softmax操作
def softmax(X):
X_exp = torch.exp(X)
partition = X_exp.sum(1, keepdim=True)
return X_exp / partition # 这⾥应⽤了⼴播机制
#网络
def net(X):
return softmax(torch.matmul(X.reshape((-1, W.shape[0])), W) + b)
# 损失函数 交叉熵
def cross_entropy(y_hat, y):
return - torch.log(y_hat[range(len(y_hat)), y])
#精度计算函数
def accuracy(y_hat, y):
"""计算预测正确的数量"""
if len(y_hat.shape) > 1 and y_hat.shape[1] > 1:
y_hat = y_hat.argmax(axis=1)
cmp = y_hat.type(y.dtype) == y
return float(cmp.type(y.dtype).sum())
class Accumulator:
"""在n个变量上累加"""
def __init__(self, n):
self.data = [0.0] * n
def add(self, *args):
self.data = [a + float(b) for a, b in zip(self.data, args)]
def reset(self):
self.data = [0.0] * len(self.data)
def __getitem__(self, idx):
return self.data[idx]
#精度计算函数搭配accumulator使用
def evaluate_accuracy(net, data_iter):
"""计算在指定数据集上模型的精度"""
if isinstance(net, torch.nn.Module):
net.eval() # 将模型设置为评估模式
metric = Accumulator(2) # 正确预测数、预测总数
with torch.no_grad():
for X, y in data_iter:
metric.add(accuracy(net(X), y), y.numel())
return metric[0] / metric[1]
#训练单轮
def train_epoch_ch3(net, train_iter, loss, updater):
"""训练模型⼀个迭代周期（定义⻅第3章）"""
# 将模型设置为训练模式
if isinstance(net, torch.nn.Module):
net.train()
# 训练损失总和、训练准确度总和、样本数
metric = Accumulator(3)
for X, y in train_iter:
# 计算梯度并更新参数
y_hat = net(X)
l = loss(y_hat, y)
if isinstance(updater, torch.optim.Optimizer):
# 使⽤PyTorch内置的优化器和损失函数
updater.zero_grad() #清除梯度
l.mean().backward() #反向传播
updater.step()
else:
# 使⽤定制的优化器和损失函数
l.sum().backward()
updater(X.shape[0])
metric.add(float(l.sum()), accuracy(y_hat, y), y.numel())
# 返回训练损失和训练精度
return metric[0] / metric[2], metric[1] / metric[2]
#训练
def train_ch3(net, train_iter, test_iter, loss, num_epochs, updater): #@save
"""训练模型（定义⻅第3章）"""
animator = Animator(xlabel='epoch', xlim=[1, num_epochs], ylim=[0.3, 0.9],
legend=['train loss', 'train acc', 'test acc'])
for epoch in range(num_epochs):
train_metrics = train_epoch_ch3(net, train_iter, loss, updater)
test_acc = evaluate_accuracy(net, test_iter)
animator.add(epoch + 1, train_metrics + (test_acc,))
train_loss, train_acc = train_metrics
assert train_loss < 0.5, train_loss
assert train_acc <= 1 and train_acc > 0.7, train_acc
assert test_acc <= 1 and test_acc > 0.7, test_acc
lr = 0.1
#优化算法 小批量随机梯度下降（Stochastic Gradient Descent）
def sgd(params,lr,batch_size):
with torch.no_grad():
for param in params:
param-=lr*param.grad/batch_size
param.grad.zero_()
#更新器使用sgd优化算法
def updater(batch_size):
return sgd([W, b], lr, batch_size)
num_epochs = 10
train_ch3(net, train_iter, test_iter, cross_entropy, num_epochs, updater)
plt.show()
def predict_ch3(net, test_iter, n=6):
"""预测标签（定义⻅第3章）"""
for X, y in test_iter:
break
trues = get_fashion_mnist_labels(y)
preds = get_fashion_mnist_labels(net(X).argmax(axis=1))
titles = [true +'\n' + pred for true, pred in zip(trues, preds)]
show_images(X[0:n].reshape((n, 28, 28)), 1, n, titles=titles[0:n])
predict_ch3(net, test_iter)
课后题：
- softmax函数可能会导致什么问题？
softmax运算可能造成溢出，因为分母要计算多个exp的值求和 - cross_entropy 是根据交叉熵损失函数的定义实现的。这个实现可能有什么问题？
y^中若某行最大的值也接近0的话，loss的值也可能造成溢出， - 什么解决方案来解决上述两个问题？
nllloss和log_softmax一起使用。softmax之后的损失函数是交叉熵，这很合理。但是softmax有可能上溢，交叉熵也有可能溢出(-log0)，所以可以用Log_softmax和负对数似然估计nll loss 配套使用。log_softmax就是对softmax取对数，nll就是直接取反，比交叉熵还方便。
结果：
调库实现
import torch
from torch import nn
from torchvision import transforms
import torchvision
from torch.utils import data
import matplotlib.pyplot as plt
#加载fashion_mnist数据集
def load_data_fashion_mnist(batch_size, resize=None):
"""下载Fashion-MNIST数据集，然后将其加载到内存中"""
trans = [transforms.ToTensor()]
if resize:
trans.insert(0, transforms.Resize(resize))
trans = transforms.Compose(trans)
mnist_train = torchvision.datasets.FashionMNIST(root="../data", train=True, transform=trans, download=True)
mnist_test = torchvision.datasets.FashionMNIST(root="../data", train=False, transform=trans, download=True)
#print(len(mnist_train),len(mnist_test))
return (data.DataLoader(mnist_train, batch_size, shuffle=True),
data.DataLoader(mnist_test, batch_size, shuffle=False)) #windows下不能多进程,linux下可以
#批大小
batch_size = 256
#训练和测试的迭代器
train_iter, test_iter = load_data_fashion_mnist(batch_size)
net=nn.Sequential(nn.Flatten(),nn.Linear(784,10))
#初始化模型参数
def init_weights(m):
if type(m)==nn.Linear:
nn.init.normal_(m.weight,std=0.01)
net.apply(init_weights)
loss=nn.CrossEntropyLoss(reduction='none')
trainer=torch.optim.SGD(net.parameters(),lr=0.1)
def set_axes(axes, xlabel, ylabel, xlim, ylim, xscale, yscale, legend):
"""设置matplotlib的轴"""
axes.set_xlabel(xlabel)
axes.set_ylabel(ylabel)
axes.set_xscale(xscale)
axes.set_yscale(yscale)
axes.set_xlim(xlim)
axes.set_ylim(ylim)
if legend:
axes.legend(legend)
axes.grid()
class Animator:
"""在动画中绘制数据"""
def __init__(self, xlabel=None, ylabel=None, legend=None, xlim=None,
ylim=None, xscale='linear', yscale='linear',
fmts=('-', 'm--', 'g-.', 'r:'), nrows=1, ncols=1,
figsize=(3.5, 2.5)):
# 增量地绘制多条线
if legend is None:
legend = []
self.fig, self.axes = plt.subplots(nrows, ncols, figsize=figsize)
if nrows * ncols == 1:
self.axes = [self.axes, ]
# 使⽤lambda函数捕获参数
self.config_axes = lambda: set_axes(self.axes[0], xlabel, ylabel, xlim, ylim, xscale, yscale, legend)
self.X, self.Y, self.fmts = None, None, fmts
def add(self, x, y):
# 向图表中添加多个数据点
if not hasattr(y, "__len__"):
y = [y]
n = len(y)
if not hasattr(x, "__len__"):
x = [x] * n
if not self.X:
self.X = [[] for _ in range(n)]
if not self.Y:
self.Y = [[] for _ in range(n)]
for i, (a, b) in enumerate(zip(x, y)):
if a is not None and b is not None:
self.X[i].append(a)
self.Y[i].append(b)
self.axes[0].cla()
for x, y, fmt in zip(self.X, self.Y, self.fmts):
self.axes[0].plot(x, y, fmt)
self.config_axes()
#display.display(self.fig)
# 通过以下两行代码实现了在PyCharm中显示动图
plt.draw()
#plt.pause(interval=0.001)
#display.clear_output(wait=True)
#精度计算函数
def accuracy(y_hat, y):
"""计算预测正确的数量"""
if len(y_hat.shape) > 1 and y_hat.shape[1] > 1:
y_hat = y_hat.argmax(axis=1)
cmp = y_hat.type(y.dtype) == y
return float(cmp.type(y.dtype).sum())
class Accumulator:
"""在n个变量上累加"""
def __init__(self, n):
self.data = [0.0] * n
def add(self, *args):
self.data = [a + float(b) for a, b in zip(self.data, args)]
def reset(self):
self.data = [0.0] * len(self.data)
def __getitem__(self, idx):
return self.data[idx]
def evaluate_accuracy(net, data_iter):
"""计算在指定数据集上模型的精度"""
if isinstance(net, torch.nn.Module):
net.eval() # 将模型设置为评估模式
metric = Accumulator(2) # 正确预测数、预测总数
with torch.no_grad():
for X, y in data_iter:
metric.add(accuracy(net(X), y), y.numel())
return metric[0] / metric[1]
#训练单轮
def train_epoch_ch3(net, train_iter, loss, updater):
"""训练模型⼀个迭代周期（定义⻅第3章）"""
# 将模型设置为训练模式
if isinstance(net, torch.nn.Module):
net.train()
# 训练损失总和、训练准确度总和、样本数
metric = Accumulator(3)
for X, y in train_iter:
# 计算梯度并更新参数
y_hat = net(X)
l = loss(y_hat, y)
if isinstance(updater, torch.optim.Optimizer):
# 使⽤PyTorch内置的优化器和损失函数
updater.zero_grad() #清除梯度
l.mean().backward() #反向传播
updater.step()
else:
# 使⽤定制的优化器和损失函数
l.sum().backward()
updater(X.shape[0])
metric.add(float(l.sum()), accuracy(y_hat, y), y.numel())
# 返回训练损失和训练精度
return metric[0] / metric[2], metric[1] / metric[2]
#训练
def train_ch3(net, train_iter, test_iter, loss, num_epochs, updater): #@save
"""训练模型（定义⻅第3章）"""
animator = Animator(xlabel='epoch', xlim=[1, num_epochs], ylim=[0.3, 0.9],
legend=['train loss', 'train acc', 'test acc'])
for epoch in range(num_epochs):
train_metrics = train_epoch_ch3(net, train_iter, loss, updater)
test_acc = evaluate_accuracy(net, test_iter)
animator.add(epoch + 1, train_metrics + (test_acc,))
train_loss, train_acc = train_metrics
assert train_loss < 0.5, train_loss
assert train_acc <= 1 and train_acc > 0.7, train_acc
assert test_acc <= 1 and test_acc > 0.7, test_acc
num_epochs=10
train_ch3(net,train_iter,test_iter,loss,num_epochs,trainer)
plt.show()
