AdaBoost算法

理论知识可参考：《统计学习方法》 （李航 著） 第八章

简单代码实现：

 1 from numpy import *
 2 import matplotlib.pyplot as plt
 3
 4 def loadSimpData():
 5     dataMat = matrix([[1,2.1],
 6         [2,1.1],
 7         [1.3,1],
 8         [1,1],
 9         [2,1]])
10     classLabels = [1.0, 1.0, -1.0, -1.0, 1.0]
11     return dataMat, classLabels
12
13 def stumpClassify(dataMatrix, dimen, threshVal, threshIneq):
14     retArray = ones((shape(dataMatrix)[0], 1))
15     if threshIneq == ‘lt‘:
16         retArray[dataMatrix[:, dimen] <= threshVal] = -1.0
17     else:
18         retArray[dataMatrix[:, dimen] > threshVal] = -1.0
19     return retArray
20
21 def buildStump(dataArr, classLabels, D):
22     dataMatrix = mat(dataArr); labelMat = mat(classLabels).T
23     m, n = shape(dataMatrix)
24     numSteps = 10.0; bestStump = {}; bestClasEst = mat(zeros((m, 1)))
25     minError = inf
26     for i in range(n):
27         rangeMin = dataMatrix[:, i].min(); rangeMax = dataMatrix[:,i].max();
28         stepSize = (rangeMax-rangeMin)/numSteps
29         for j in range(-1, int(numSteps)+1):
30             for inequal in [‘lt‘, ‘gt‘]:
31                 threshVal = (rangeMin + float(j) * stepSize)
32                 predictedVals = stumpClassify(dataMatrix, i, threshVal, inequal)
33                 errArr = mat(ones((m, 1)))
34                 errArr[predictedVals == labelMat] = 0
35                 weightedError = D.T * errArr
36                 #print("split: dim %d, thresh %.2f, thresh ineqal: %s, the weighted error is %.3f" % (i, threshVal, inequal, weightedError))
37                 if weightedError < minError:
38                     minError = weightedError
39                     bestClasEst = predictedVals.copy()
40                     bestStump[‘dim‘] = i
41                     bestStump[‘thresh‘] = threshVal
42                     bestStump[‘ineq‘] = inequal
43     return bestStump, minError, bestClasEst
44
45 def adaBoostTrainDS(dataArr, classLabels, numIt = 40):
46     weakClassArr = []
47     m = shape(dataArr)[0]
48     D = mat(ones((m, 1))/m)
49     aggClassEst = mat(zeros((m, 1)))
50     for i in range(numIt):
51         bestStump, error, classEst = buildStump(dataArr, classLabels, D)
52         print("D:", D.T)
53         alpha = float(0.5*log((1.0-error)/max(error, 1e-16)))
54         bestStump[‘alpha‘] = alpha
55         weakClassArr.append(bestStump)
56         print("classEst:", classEst)
57         expon = multiply(-1*alpha*mat(classLabels).T, classEst)
58         D = multiply(D, exp(expon))
59         D = D/D.sum()
60         aggClassEst += alpha*classEst
61         print("aggClassEst:", aggClassEst.T)
62         aggErrors = multiply(sign(aggClassEst) != mat(classLabels).T, ones((m, 1)))
63         errorRate = aggErrors.sum()/m
64         print("total error:", errorRate, "\n")
65         if errorRate == 0.0: break
66     return weakClassArr
67
68 dataMat, classLabels = loadSimpData()
69 D = mat(ones((5, 1))/5)
70 classifierArray = adaBoostTrainDS(dataMat, classLabels, 9)
71 print(classifierArray)