first version ada boost

Author: WenDesi

.idea/workspace.xml

@@ -0,0 +1,147 @@
+# encoding=utf-8
+# @Author: wendesi
+# @Date:   15-11-16
+# @Email:  [email protected]
+# @Last modified by:   wendesi
+# @Last modified time: 15-11-16
+
+import math
+
+class Sign(object):
+    def __init__(self,features,labels,w):
+        self.X = features
+        self.Y = labels
+        self.N = len(labels)
+
+        self.w = w
+
+    def _train_less_than_(self):
+        index = -1
+        error_score = 1000000
+
+        for i in xrange(self.N+1):
+            score = 0
+            for j in xrange(self.N):
+                val = -1
+                if j<i:
+                    val = 1
+
+                if val*self.Y[j]<0:
+                    score += self.w[j]
+
+            if score < error_score:
+                index = i
+                error_score = score
+
+        return index,error_score
+
+
+
+    def _train_more_than_(self):
+        index = -1
+        error_score = 1000000
+
+        for i in xrange(self.N+1):
+            score = 0
+            for j in xrange(self.N):
+                val = 1
+                if j<i:
+                    val = -1
+
+                if val*self.Y[j]<0:
+                    score += self.w[j]
+
+            if score < error_score:
+                index = i
+                error_score = score
+
+        return index,error_score
+
+    def train(self):
+        less_index,less_score = self._train_less_than_()
+        more_index,more_score = self._train_more_than_()
+
+        if less_score < more_score:
+            self.is_less = True
+            self.index = less_index
+            return less_score
+
+        else:
+            self.is_less = False
+            self.index = more_index
+            return more_score
+
+    def predict(self,feature):
+        if self.is_less:
+            if feature<self.index:
+                return 1.0
+            else:
+                return -1.0
+        else:
+            if feature<self.index:
+                return -1.0
+            else:
+                return 1.0
+
+
+class AdaBoost(object):
+
+    def __init__(self):
+        pass
+
+    def _init_parameters_(self,features,labels):
+        self.X = features
+        self.Y = labels
+
+        self.n = len(features[0])
+        self.N = len(features)
+        self.M = 100                            # 分类器数目
+
+        self.w = [1.0/self.N]*self.N
+        self.alpha = []
+        self.classifier = []
+
+    def _w_(self,index,classifier,i):
+        return self.w[i]*math.exp(-self.alpha[-1]*self.Y[i]*classifier.predict(self.X[i][index]))
+
+    def _Z_(self,index,classifier):
+        Z = 0
+
+        for i in xrange(self.N):
+            Z += self._w_(index,classifier,i)
+
+        return Z
+
+    def train(self,features,labels):
+
+        self._init_parameters_(features,labels)
+
+        for times in xrange(self.M):
+
+            best_classifier = (100000,None,None)        #(误差率,分类器,针对的特征)
+            for i in xrange(self.n):
+                features = map(lambda x:x[i],self.X)
+                classifier = Sign(features,self.Y,self.w)
+                error_score = classifier.train()
+
+                if error_score < best_classifier[0]:
+                    best_classifier = (error_score,classifier,i)
+
+            em = best_classifier[0]
+            self.alpha.append(1/2*math.log((1-em)/em))
+            self.classifier.append(best_classifier[1:])
+
+            Z = self._Z_(best_classifier[1],best_classifier[2])
+
+            for i in xrange(self.N):
+                self.w[i] = self._w_(best_classifier[1],best_classifier[2],i)/Z
+
+if __name__ == '__main__':
+    features = [[0],[1],[2],[3],[4],[5],[6],[7],[8],[9]]
+    labels = [1,1,1,-1,-1,-1,1,1,1,-1]
+
+
+
+
+    ada = AdaBoost()
+    ada.train(features,labels)

AdaBoost/adaboost.py

@@ -0,0 +1,6 @@
+# encoding=utf-8
+# @Author: wendesi
+# @Date:   15-11-16
+# @Email:  [email protected]
+# @Last modified by:   wendesi
+# @Last modified time: 15-11-16

AdaBoost/adaboost_fakedata.py

@@ -0,0 +1,136 @@
+# encoding=utf8
+import numpy as np
+import random
+import matplotlib
+import matplotlib.pyplot as plt
+
+N = 10 #生成训练数据的个数
+
+# AX=0 相当于matlab中 null(a','r')
+def null(a, rtol=1e-5):
+    u, s, v = np.linalg.svd(a)
+    rank = (s > rtol*s[0]).sum()
+    return rank, v[rank:].T.copy()
+
+# 符号函数，之后要进行向量化
+def sign(x):
+    if x > 0:
+        return 1
+    elif x == 0:
+        return 0
+    elif x < 0:
+        return -1
+#noisy=False，那么就会生成N的dim维的线性可分数据X，标签为y
+#noisy=True, 那么生成的数据是线性不可分的,标签为y
+def mk_data(N, noisy=False):
+    rang = [-10,10]
+    dim = 2
+
+    X=np.random.rand(dim,N)*(rang[1]-rang[0])+rang[0]
+
+    while True:
+        Xsample = np.concatenate((np.ones((1,dim)), np.random.rand(dim,dim)*(rang[1]-rang[0])+rang[0]))
+        k,w=null(Xsample.T)
+        y = sign(np.dot(w.T,np.concatenate((np.ones((1,N)), X))))
+        if np.all(y):
+            break
+
+    if noisy == True:
+        idx = random.sample(range(1,N), N/10)
+
+        for id in idx:
+            y[0][id] = -y[0][id]
+
+    return (X,y,w)
+
+def data_visualization(X,y,title):
+    class_1 = [[],[]]
+    class_2 = [[],[]]
+
+    size = len(y)
+
+    for i in xrange(size):
+        X_1 = X[0][i]
+        X_2 = X[1][i]
+
+        if y[i] == 1:
+            class_1[0].append(X_1)
+            class_1[1].append(X_2)
+        else:
+            class_2[0].append(X_1)
+            class_2[1].append(X_2)
+
+
+    plt.figure(figsize=(8, 6), dpi=80)
+    plt.title(title)
+
+    axes = plt.subplot(111)
+
+    type1 = axes.scatter(class_1[0], class_1[1], s=20, c='red')
+    type2 = axes.scatter(class_2[0], class_2[1], s=20, c='green')
+
+
+    plt.show()
+
+def rebuild_features(features):
+    size = len(features[0])
+
+    new_features = []
+    for i in xrange(size):
+        new_features.append([features[0][i],features[1][i]])
+
+    return new_features
+
+def generate_dataset(size, noisy = False, visualization = True):
+    global sign
+    sign = np.vectorize(sign)
+    X,y,w = mk_data(size,False)
+    y = list(y[0])
+
+    if visualization:
+        data_visualization(X,y,'all data')         #数据可视化
+
+    testset_size = int(len(y)*0.333)
+
+    indexes = [i for i in xrange(len(y))]
+    test_indexes = random.sample(indexes,testset_size)
+    train_indexes = list(set(indexes)-set(test_indexes))
+
+    trainset_features = [[],[]]
+    trainset_labels = []
+
+    testset_features = [[],[]]
+    testset_labels = []
+
+    for i in test_indexes:
+        testset_features[0].append(X[0][i])
+        testset_features[1].append(X[1][i])
+        testset_labels.append(y[i])
+
+
+    if visualization:
+        data_visualization(testset_features,testset_labels,'test set')
+
+    for i in train_indexes:
+        trainset_features[0].append(X[0][i])
+        trainset_features[1].append(X[1][i])
+        trainset_labels.append(y[i])
+
+    if visualization:
+        data_visualization(trainset_features,trainset_labels,'train set')
+
+    return rebuild_features(trainset_features),trainset_labels,rebuild_features(testset_features),testset_labels
+
+
+
+if __name__ == '__main__':
+
+    size = 1000
+    generate_dataset(size)
+
+    # generate_dataset
+    # print sign
+    # sign = np.vectorize(sign)
+    # X,y,w = mk_data(size,False)
+    #
+    # data_visualization(X,y)