《機器學習實戰》決策樹(ID3演算法)的分析與實現

#coding=utf-8
from math import log
import operator

def createDataSet():
    dataSet =[[1,1,'yes'],
              [1,1,'yes'],
              [1,0,'no'],
              [0,1,'no'],
              [0,1,'no']]  #建立資料集
    labels = ['no surfacing','flippers'] #分類屬性
    return dataSet,labels
'''
測試結果
[[1, 'no'], [1, 'no']]
>>> dataSet
[[1, 1, 'yes'], [1, 1, 'yes'], [1, 0, 'no'], [0, 1, 'no'], [0, 1, 'no']]
>>> labels
['no surfacing', 'flippers']
'''
def calcShannonEnt(dataSet):   #計算給定資料集的夏農熵
    numEntries = len(dataSet)  #求長度
    labelCounts = {}
    for featVec in dataSet:
        currentLabel = featVec[-1] #獲得標籤
        if currentLabel not in labelCounts.keys():  #如果標籤不在新定義的字典裡建立該標籤值
            labelCounts[currentLabel]=0
        labelCounts[currentLabel]+=1 #該類標籤下含有資料的個數
    shannonEnt = 0.0
    for key in labelCounts:
        prob = float(labelCounts[key])/numEntries  #同類標籤出現的概率
        shannonEnt -=prob*log(prob,2)   #以2為底求對數
    return shannonEnt
'''
測試結果
>>> trees.calcShannonEnt(dataSet)
0.9709505944546686
'''
def splitDataSet(dataSet,axis,value):  #劃分資料集,三個引數為帶劃分的資料集，劃分資料集的特徵，特徵的返回值
    retDataSet = []
    for featVec in dataSet:
        if featVec[axis] ==value:
            #將相同資料集特徵的抽取出來
            reducedFeatVec = featVec[:axis]
            reducedFeatVec.extend(featVec[axis+1:])
            retDataSet.append(reducedFeatVec)
    return retDataSet #返回一個列表
'''
測試結果
>>> trees.splitDataSet(dataSet,0,1)
[[1, 'yes'], [1, 'yes'], [0, 'no']] 
>>> trees.splitDataSet(dataSet,0,0)
[[1, 'no'], [1, 'no']]
'''          
def chooseBestFeatureToSplit(dataSet):  #選擇最好的資料集劃分方式
    numFeature = len(dataSet[0])-1
    baseEntropy = calcShannonEnt(dataSet)
    bestInfoGain = 0.0
    beatFeature = -1
    for i in range(numFeature):
        featureList = [example[i] for example in dataSet] #獲取第i個特徵所有的可能取值
        uniqueVals = set(featureList)  #從列表中建立集合，得到不重複的所有可能取值
        newEntropy = 0.0
        for value in uniqueVals:
            subDataSet = splitDataSet(dataSet,i,value)  #以i為資料集特徵，value為返回值，劃分資料集
            prob = len(subDataSet)/float(len(dataSet))   #資料集特徵為i的所佔的比例
            newEntropy +=prob * calcShannonEnt(subDataSet)   #計算每種資料集的資訊熵
        infoGain = baseEntropy- newEntropy
        #計算最好的資訊增益，增益越大說明所佔決策權越大
        if (infoGain > bestInfoGain):
            bestInfoGain = infoGain
            bestFeature = i
    return bestFeature
'''
測試結果
>>> trees.chooseBestFeatureToSplit(dataSet)
0
說明：第0個特徵是最好的用於劃分資料集的特徵
'''
def majorityCnt(classList):      #遞迴構建決策樹
    classCount = {}
    for vote in classList:
        if vote not in classCount.keys():
            classCount[vote]=0
        classCount[vote]+=1
    sortedClassCount = sorted(classCount.iteritems(),key =operator.itemgetter(1),reverse=True)#排序，True升序
    return sortedClassCount[0][0] #返回出現次數最多的
'''
測試結果

'''  
def createTree(dataSet,labels):     #建立樹的函式程式碼
    classList = [example[-1]  for example in dataSet]
    if classList.count(classList[0])==len(classList): #類別完全相同則停止劃分
        return classList[0]
    if len(dataSet[0]) ==1:            #遍歷完所有特徵值時返回出現次數最多的
        return majorityCnt(classList)
    bestFeat = chooseBestFeatureToSplit(dataSet)   #選擇最好的資料集劃分方式
    bestFeatLabel = labels[bestFeat]   #得到對應的標籤值
    myTree = {bestFeatLabel:{}}
    del(labels[bestFeat])      #清空labels[bestFeat],在下一次使用時清零
    featValues = [example[bestFeat] for example in dataSet] 
    uniqueVals = set(featValues)
    for value in uniqueVals:
        subLabels =labels[:]
        #遞迴呼叫建立決策樹函式
        myTree[bestFeatLabel][value]=createTree(splitDataSet(dataSet,bestFeat,value),subLabels)
    return myTree  
'''
測試結果
>>> reload(trees)

>>> dataSet,labels = trees.createDataSet()
>>> myTree = trees.createTree(dataSet,labels)
>>> myTree
{'no surfacing': {0: 'no', 1: {'flippers': {0: 'no', 1: 'yes'}}}}
'''