下面是我从cs231n上整理的神经网络的入门实现，麻雀虽小，五脏俱全，基本上神经网络涉及到的知识点都有在代码中体现。
理论看上千万遍，不如看一遍源码跑一跑。
源码上我已经加了很多注释，结合代码看一遍很容易理解。
最后可视化权重的图：
主文件，用来训练调参
two_layer_net.py
1 # coding: utf-8
2
3 # 实现一个简单的神经网络并在CIFAR10上测试性能
4
5 import numpy as np
6 import matplotlib.pyplot as plt
7 from neural_net import TwoLayerNet
8 from data_utils import load_CIFAR10
9 from vis_utils import visualize_grid
10
11 def get_CIFAR10_data(num_training=49000, num_validation=1000, num_test=1000):
12 cifar10_dir = ‘cs231n/datasets/cifar-10-batches-py‘
13 X_train, y_train, X_test, y_test = load_CIFAR10(cifar10_dir)
14
15 # 采样
16 mask = list(range(num_training, num_training + num_validation))
17 X_val = X_train[mask]
18 y_val = y_train[mask]
19 mask = list(range(num_training))
20 X_train = X_train[mask]
21 y_train = y_train[mask]
22 mask = list(range(num_test))
23 X_test = X_test[mask]
24 y_test = y_test[mask]
25
26 # 归一化操作：减去均值，使得数据以0为中心
27 mean_image = np.mean(X_train, axis=0)
28 X_train -= mean_image
29 X_val -= mean_image
30 X_test -= mean_image
31
32 X_train = X_train.reshape(num_training, -1)
33 X_val = X_val.reshape(num_validation, -1)
34 X_test = X_test.reshape(num_test, -1)
35
36 return X_train, y_train, X_val, y_val, X_test, y_test
37
38
39 X_train, y_train, X_val, y_val, X_test, y_test = get_CIFAR10_data()
40 print(‘Train data shape: ‘, X_train.shape)
41 print(‘Train labels shape: ‘, y_train.shape)
42 print(‘Validation data shape: ‘, X_val.shape)
43 print(‘Validation labels shape: ‘, y_val.shape)
44 print(‘Test data shape: ‘, X_test.shape)
45 print(‘Test labels shape: ‘, y_test.shape)
46
47
48 #第一次训练
49 input_size = 32 * 32 * 3
50 hidden_size = 50
51 num_classes = 10
52 net = TwoLayerNet(input_size, hidden_size, num_classes)
53 stats = net.train(X_train, y_train, X_val, y_val,
54 num_iters=1000, batch_size=200,
55 learning_rate=1e-4, learning_rate_decay=0.95,
56 reg=0.25, verbose=True)
57 val_acc = (net.predict(X_val) == y_val).mean()
58 print(‘Validation accuracy: ‘, val_acc)
59
60 #效果不太理想，debug
61
62 # 先画一下loss和正确率的曲线看一看
63 plt.subplot(2, 1, 1)
64 plt.plot(stats[‘loss_history‘])
65 plt.title(‘Loss history‘)
66 plt.xlabel(‘Iteration‘)
67 plt.ylabel(‘Loss‘)
68
69 plt.subplot(2, 1, 2)
70 plt.plot(stats[‘train_acc_history‘], label=‘train‘)
71 plt.plot(stats[‘val_acc_history‘], label=‘val‘)
72 plt.title(‘Classification accuracy history‘)
73 plt.xlabel(‘Epoch‘)
74 plt.ylabel(‘Clasification accuracy‘)
75 plt.show()
76
77
78
79 #可视化一下权重
80 def show_net_weights(net):
81 W1 = net.params[‘W1‘]
82 W1 = W1.reshape(32, 32, 3, -1).transpose(3, 0, 1, 2)
83 plt.imshow(visualize_grid(W1, padding=3).astype(‘uint8‘))
84 plt.gca().axis(‘off‘)
85 plt.show()
86
87 show_net_weights(net)
88
89
90 #通过上面的曲线我们可以看到基本上loss还在线性下降，表示我们的loss下降的还不够。
91 #一方面，我们可以加大学习率使loss更加快速的下降，另一方面，也可以增加迭代的次数，让loss继续下降。
92 #还有，在训练集和验证集上的正确率没有明显差距，表明网络的容量可能不够，可以尝试增加网络的复杂度使之拥有更强的表达能力。
93
94
95
96 #下面是我调出来的参数，实际上选了很久 ,在测试集上的正确率在55%左右
97 hidden_size = 150#[50,70,100,130]
98 learning_rates = 1e-3#np.array([0.5,1,1.5])*1e-3
99 regularization_strengths = 0.2#[0.1,0.2,0.3]
100 best_net = None
101 results = {}
102 best_val_acc = 0
103
104
105 for hs in hidden_size:
106 for lr in learning_rates:
107 for reg in regularization_strengths:
108
109 net = TwoLayerNet(input_size, hs, num_classes)
110 # Train the network
111 stats = net.train(X_train, y_train, X_val, y_val,
112 num_iters=3000, batch_size=200,
113 learning_rate=lr, learning_rate_decay=0.95,
114 reg= reg, verbose=False)
115 val_acc = (net.predict(X_val) == y_val).mean()
116 if val_acc > best_val_acc:
117 best_val_acc = val_acc
118 best_net = net
119 results[(hs,lr,reg)] = val_acc
120
121 plt.subplot(2, 1, 1)
122 plt.plot(stats[‘loss_history‘])
123 plt.title(‘Loss history‘)
124 plt.xlabel(‘Iteration‘)
125 plt.ylabel(‘Loss‘)
126
127 plt.subplot(2, 1, 2)
128 plt.plot(stats[‘train_acc_history‘], label=‘train‘)
129 plt.plot(stats[‘val_acc_history‘], label=‘val‘)
130 plt.title(‘Classification accuracy history‘)
131 plt.xlabel(‘Epoch‘)
132 plt.ylabel(‘Clasification accuracy‘)
133 plt.show()
134
135
136 for hs,lr, reg in sorted(results):
137 val_acc = results[(hs, lr, reg)]
138 print (‘hs %d lr %e reg %e val accuracy: %f‘ % (hs, lr, reg, val_acc))
139
140 print (‘best validation accuracy achieved during cross-validation: %f‘ % best_val_acc)
141
142
143 show_net_weights(best_net)
144 test_acc = (best_net.predict(X_test) == y_test).mean()
145 print(‘Test accuracy: ‘, test_acc)
定义神经网络和前向反向计算、损失函数、自动训练的类
neural_net.py
1 import numpy as np
2 import matplotlib.pyplot as plt
3
4 class TwoLayerNet(object):
5 """
6 两层的全连接网络。使用sotfmax损失函数和L2正则，非线性函数采用Relu函数。
7 网络结构：input - fully connected layer - ReLU - fully connected layer - softmax
8 """
9
10 def __init__(self, input_size, hidden_size, output_size, std=1e-4):
11 """
12 初始化模型。
13 初始化权重矩阵W和偏置b。这里b置为零，但是Alexnet论文中说采用Relu函数激活时b置为1可以更快的收敛。
14 参数都保存在self.params字典中。
15 键为：
16 W1 (D, H)
17 b1 (H,)
18 W2 (H, C)
19 b2 (C,)
20 D,H,C分别表示输入数据的维度，隐藏层大小，输出类别的个数
21 """
22 self.params = {}
23 self.params[‘W1‘] = std * np.random.randn(input_size, hidden_size)
24 self.params[‘b1‘] = np.zeros(hidden_size)
25 self.params[‘W2‘] = std * np.random.randn(hidden_size, output_size)
26 self.params[‘b2‘] = np.zeros(output_size)
27
28 def loss(self, X, y=None, reg=0.0):
29 """
30 如果是在训练过程,计算损失和梯度，如果是在测试过程，返回最后一层的输入,即每个类的得分。
31
32 Inputs:
33 - X (N, D). X[i] 为一个训练样本。
34 - y: 标签。如果为None则表示是在进行测试过程，否则是在进行训练过程。
35 - reg: Regularization strength.
36
37 Returns:
38 如果y=None，返回shape为(N, C)的矩阵，scores[i, c]表示输入i在c类上的得分。
39
40 如果y!=None, 返回一个tuple:
41 - loss: 包括数据损失和正则损失两部分。
42 - grads: 各个参数的梯度。
43 """
44
45 W1, b1 = self.params[‘W1‘], self.params[‘b1‘]
46 W2, b2 = self.params[‘W2‘], self.params[‘b2‘]
47 N, D = X.shape
48 C=b2.shape[0]
49
50 #forward pass
51 h1=np.maximum(0,np.dot(X,W1)+b1)
52 h2=np.dot(h1,W2)+b2
53 scores=h2
54
55 if y is None:
56 return scores
57
58 # 计算loss
59 shift_scores=scores-np.max(scores,axis=1).reshape(-1,1)
60 exp_scores=np.exp(shift_scores)
61 softmax_out=exp_scores/np.sum(exp_scores,axis=1).reshape(-1,1)
62 loss=np.sum(-np.log(softmax_out[range(N),y]))/N+reg * (np.sum(W1 * W1) + np.sum(W2 * W2))
63 print(np.sum(-np.log(softmax_out[range(N),y]))/N,reg * (np.sum(W1 * W1) + np.sum(W2 * W2)))
64
65 # Backward pass: 计算梯度，梯度的计算就是链式求导的过程
66 grads = {}
67
68 dscores = softmax_out.copy()
69 dscores[range(N),y]-=1
70 dscores /= N
71
72 grads[‘W2‘]=np.dot(h1.T,dscores)+2*reg*W2
73 grads[‘b2‘]=np.sum(dscores,axis=0)
74
75 dh=np.dot(dscores,W2.T)
76 d_max=(h1>0)*dh
77
78 grads[‘W1‘] = X.T.dot(d_max) + 2*reg * W1
79 grads[‘b1‘] = np.sum(d_max, axis = 0)
80
81 return loss, grads
82
83 def train(self, X, y, X_val, y_val,
84 learning_rate=1e-3, learning_rate_decay=0.95,
85 reg=5e-6, num_iters=100,
86 batch_size=200, verbose=False):
87 """
88 自动化训练过程。采用SGD优化。
89
90 Inputs:
91 - X (N, D)：训练输入。
92 - y (N,) ：标签。 y[i] = c 表示X[i]的类别下标是c。
93 - X_val (N_val, D)：验证集输入。
94 - y_val (N_val,)： 验证集标签。
95 - learning_rate:
96 - learning_rate_decay: 学习率的损失因子。
97 - reg: regularization strength。
98 - num_iters: 迭代次数。
99 - batch_size: 每次迭代的数据批大小。.
100 - verbose: 是否显示训练进度。
101 """
102 num_train = X.shape[0]
103 iterations_per_epoch = max(num_train / batch_size, 1)
104
105 loss_history = []
106 train_acc_history = []
107 val_acc_history = []
108
109 for it in range(num_iters):
110 #随机选择一批数据
111 idx = np.random.choice(num_train, batch_size, replace=True)
112 X_batch = X[idx]
113 y_batch = y[idx]
114 # 计算损失和梯度
115 loss, grads = self.loss(X_batch, y=y_batch, reg=reg)
116 loss_history.append(loss)
117 #更新参数
118 self.params[‘W2‘] += - learning_rate * grads[‘W2‘]
119 self.params[‘b2‘] += - learning_rate * grads[‘b2‘]
120 self.params[‘W1‘] += - learning_rate * grads[‘W1‘]
121 self.params[‘b1‘] += - learning_rate * grads[‘b1‘]
122 #可视化进度
123 if verbose and it % 100 == 0:
124 print(‘iteration %d / %d: loss %f‘ % (it, num_iters, loss))
125
126 # 每个epoch保存一次数据记录
127 if it % iterations_per_epoch == 0:
128 train_acc = (self.predict(X_batch) == y_batch).mean()
129 val_acc = (self.predict(X_val) == y_val).mean()
130 train_acc_history.append(train_acc)
131 val_acc_history.append(val_acc)
132 #学习率衰减
133 learning_rate *= learning_rate_decay
134 return {
135 ‘loss_history‘: loss_history,
136 ‘train_acc_history‘: train_acc_history,
137 ‘val_acc_history‘: val_acc_history,
138 }
139
140 def predict(self, X):
141 """
142 使用训练好的参数预测输入的标签。
143
144 Inputs:
145 - X (N, D)： 需要预测的输入。
146
147 Returns:
148 - y_pred (N,)：每个输入的预测分类下标。
149 """
150
151 h = np.maximum(0, X.dot(self.params[‘W1‘]) + self.params[‘b1‘])
152 scores = h.dot(self.params[‘W2‘]) + self.params[‘b2‘]
153 y_pred = np.argmax(scores, axis=1)
154
155 return y_pred
载入CIFAR10数据的函数
data_utils.py
1 from six.moves import cPickle as pickle
2 import numpy as np
3 import os
4 from scipy.misc import imread
5 import platform
6
7 def load_pickle(f):
8 version = platform.python_version_tuple()
9 if version[0] == ‘2‘:
10 return pickle.load(f)
11 elif version[0] == ‘3‘:
12 return pickle.load(f, encoding=‘latin1‘)
13 raise ValueError("invalid python version: {}".format(version))
14
15 def load_CIFAR_batch(filename):
16 """ CIRAR的数据是分批的，这个函数的功能是载入一批数据 """
17 with open(filename, ‘rb‘) as f:
18 datadict = load_pickle(f) #以二进制方式打开文件
19 X = datadict[‘data‘]
20 Y = datadict[‘labels‘]
21 X = X.reshape(10000, 3, 32, 32).transpose(0,2,3,1).astype("float")
22 Y = np.array(Y)
23 return X, Y
24
25 def load_CIFAR10(ROOT):
26 """ load 所有的数据 """
27 xs = []
28 ys = []
29 for b in range(1,6):
30 f = os.path.join(ROOT, ‘data_batch_%d‘ % (b, ))
31 X, Y = load_CIFAR_batch(f)
32 xs.append(X)
33 ys.append(Y)
34 Xtr = np.concatenate(xs)
35 Ytr = np.concatenate(ys)
36 del X, Y
37 Xte, Yte = load_CIFAR_batch(os.path.join(ROOT, ‘test_batch‘))
38 return Xtr, Ytr, Xte, Yte
可视化用到的函数
vis_utils.py
1 from math import sqrt, ceil 2 import numpy as np 3 4 def visualize_grid(Xs, ubound=255.0, padding=1): 5 """ 6 #把4维的数据显示在平面图上，也就是把(N, H, W, C)N张3通道的图片同时显示出来 7 8 Inputs: 9 - Xs:(N, H, W, C)shape的数据 10 - ubound: 像素会被放缩到【0，ubound】之间 11 - padding: 方块之间的间隔填充 12 """ 13 (N, H, W, C) = Xs.shape 14 grid_size = int(ceil(sqrt(N))) 15 grid_height = H * grid_size + padding * (grid_size - 1) 16 grid_width = W * grid_size + padding * (grid_size - 1) 17 grid = np.zeros((grid_height, grid_width, C)) 18 next_idx = 0 19 y0, y1 = 0, H 20 for y in range(grid_size): 21 x0, x1 = 0, W 22 for x in range(grid_size): 23 if next_idx < N: 24 img = Xs[next_idx] 25 low, high = np.min(img), np.max(img) 26 grid[y0:y1, x0:x1] = ubound * (img - low) / (high - low) 27 next_idx += 1 28 x0 += W + padding 29 x1 += W + padding 30 y0 += H + padding 31 y1 += H + padding 32 return grid
原文地址：https://www.cnblogs.com/super-JJboom/p/9749119.html
时间： 2024-10-08 14:36:05