第一个：使用numpy写的神经元训练
import numpy as np
np.random.seed(0)
data = np.array([[0, 0], [0, 1], [1, 0], [1, 1]])
target = np.array([[0], [1], [0], [1]])
weights_0_1 = np.random.rand(2, 3)
weights_1_2 = np.random.rand(3, 1)
for i in range(10):
# Predict
layer_1 = data.dot(weights_0_1)
layer_2 = layer_1.dot(weights_1_2)
# Compare
diff = (layer_2 - target)
sqdiff = (diff * diff)
loss = sqdiff.sum(0) # mean squared error loss
# Learn: this is the backpropagation piece
layer_1_grad = diff.dot(weights_1_2.transpose())
weight_1_2_update = layer_1.transpose().dot(diff)
weight_0_1_update = data.transpose().dot(layer_1_grad)
weights_1_2 -= weight_1_2_update * 0.1
weights_0_1 -= weight_0_1_update * 0.1
print(loss[0])
'''
5.066439994622395
0.4959907791902342
0.4180671892167177
0.35298133007809646
0.2972549636567377
0.2492326038163328
0.20785392075862477
0.17231260916265176
0.14193744536652986
0.11613979792168384
'''
第二个：使用张量重写的神经元训练
import numpy as np
class Tensor(object):
def __init__(self, data,
autograd=False,
creators=None,
creation_op=None,
id=None):
self.data = np.array(data)
self.autograd = autograd
self.grad = None
if (id is None):
self.id = np.random.randint(0, 100000)
else:
self.id = id
self.creators = creators
self.creation_op = creation_op
self.children = {}
if (creators is not None):
for c in creators:
if (self.id not in c.children):
c.children[self.id] = 1
else:
c.children[self.id] += 1
def all_children_grads_accounted_for(self):
for id, cnt in self.children.items():
if (cnt != 0):
return False
return True
def backward(self, grad=None, grad_origin=None):
if (self.autograd):
if (grad is None):
grad = Tensor(np.ones_like(self.data))
if (grad_origin is not None):
if (self.children[grad_origin.id] == 0):
raise Exception("cannot backprop more than once")
else:
self.children[grad_origin.id] -= 1
if (self.grad is None):
self.grad = grad
else:
self.grad += grad
# grads must not have grads of their own
assert grad.autograd == False
# only continue backpropping if there's something to
# backprop into and if all gradients (from children)
# are accounted for override waiting for children if
# "backprop" was called on this variable directly
if (self.creators is not None and
(self.all_children_grads_accounted_for() or
grad_origin is None)):
if (self.creation_op == "add"):
self.creators[0].backward(self.grad, self)
self.creators[1].backward(self.grad, self)
if (self.creation_op == "sub"):
self.creators[0].backward(Tensor(self.grad.data), self)
self.creators[1].backward(Tensor(self.grad.__neg__().data), self)
if (self.creation_op == "mul"):
new = self.grad * self.creators[1]
self.creators[0].backward(new, self)
new = self.grad * self.creators[0]
self.creators[1].backward(new, self)
if (self.creation_op == "mm"):
c0 = self.creators[0]
c1 = self.creators[1]
new = self.grad.mm(c1.transpose())
c0.backward(new)
new = self.grad.transpose().mm(c0).transpose()
c1.backward(new)
if (self.creation_op == "transpose"):
self.creators[0].backward(self.grad.transpose())
if ("sum" in self.creation_op):
dim = int(self.creation_op.split("_")[1])
self.creators[0].backward(self.grad.expand(dim,
self.creators[0].data.shape[dim]))
if ("expand" in self.creation_op):
dim = int(self.creation_op.split("_")[1])
self.creators[0].backward(self.grad.sum(dim))
if (self.creation_op == "neg"):
self.creators[0].backward(self.grad.__neg__())
#加法
def __add__(self, other):
if (self.autograd and other.autograd):
return Tensor(self.data + other.data,
autograd=True,
creators=[self, other],
creation_op="add")
return Tensor(self.data + other.data)
#取负
def __neg__(self):
if (self.autograd):
return Tensor(self.data * -1,
autograd=True,
creators=[self],
creation_op="neg")
return Tensor(self.data * -1)
#减法
def __sub__(self, other):
if (self.autograd and other.autograd):
return Tensor(self.data - other.data,
autograd=True,
creators=[self, other],
creation_op="sub")
return Tensor(self.data - other.data)
#乘法
def __mul__(self, other):
if (self.autograd and other.autograd):
return Tensor(self.data * other.data,
autograd=True,
creators=[self, other],
creation_op="mul")
return Tensor(self.data * other.data)
#求和
def sum(self, dim):
if (self.autograd):
return Tensor(self.data.sum(dim), #即 Tensor 对象所存储的 numpy 数组数据）在指定维度 dim 上进行求和操作
autograd=True,
creators=[self],
creation_op="sum_" + str(dim))
return Tensor(self.data.sum(dim))
#扩展
def expand(self, dim, copies):
trans_cmd = list(range(0, len(self.data.shape)))
trans_cmd.insert(dim, len(self.data.shape))
new_data = self.data.repeat(copies).reshape(list(self.data.shape) + [copies]).transpose(trans_cmd)
if (self.autograd):
return Tensor(new_data,
autograd=True,
creators=[self],
creation_op="expand_" + str(dim))
return Tensor(new_data)
#转置
def transpose(self):
if (self.autograd):
return Tensor(self.data.transpose(),
autograd=True,
creators=[self],
creation_op="transpose")
return Tensor(self.data.transpose())
#矩阵乘法
def mm(self, x):
if (self.autograd):
return Tensor(self.data.dot(x.data),
autograd=True,
creators=[self, x],
creation_op="mm")
return Tensor(self.data.dot(x.data))
def __repr__(self):
return str(self.data.__repr__())
def __str__(self):
return str(self.data.__str__())
np.random.seed(0)
data = Tensor(np.array([[0, 0], [0, 1], [1, 0], [1, 1]]), autograd=True)
target = Tensor(np.array([[0], [1], [0], [1]]), autograd=True)
w = list()
w.append(Tensor(np.random.rand(2, 3), autograd=True))
w.append(Tensor(np.random.rand(3, 1), autograd=True))
for i in range(10):
# Predict
pred = data.mm(w[0]).mm(w[1])
# Compare
loss = ((pred - target) * (pred - target)).sum(0)
# Learn
loss.backward(Tensor(np.ones_like(loss.data)))
for w_ in w:
w_.data -= w_.grad.data * 0.1
w_.grad.data *= 0
print(loss)
'''
[0.58128304]
[0.48988149]
[0.41375111]
[0.34489412]
[0.28210124]
[0.2254484]
[0.17538853]
[0.1324231]
[0.09682769]
[0.06849361]
'''