#-*- coding: utf-8 -*-
'''
Created on Oct 12, 2010
Decision Tree Source Code for Machine Learning in Action Ch. 3
@author: Peter Harrington
'''
from math import log
import operator
#训练数据集
def createDataSet():
#类别值有两个：yes,no
dataSet = [[1, 1, 'yes'],
[1, 1, 'yes'],
[1, 0, 'no'],
[0, 1, 'no'],
[0, 1, 'no']]
#两个特征名称
labels = ['no surfacing','flippers']
#change to discrete values
return dataSet, labels
#计算熵
def calcShannonEnt(dataSet):
#样本总数
numEntries = len(dataSet)
#dict字典数据类型，字典是由键对值组组成
labelCounts = {}
#featVec遍历dataSet的每一行
for featVec in dataSet:
#每行最后一列为类别标签
currentLabel = featVec[-1]
#***统计各类别中的样本数
#为所有可能取值建立字典<key,value>结构
#key表示类别，value表示出现次数
if currentLabel not in labelCounts.keys():
#当前key不在字典中，扩展字典
labelCounts[currentLabel] = 0
#当前类别出现一次，字典value值加1
labelCounts[currentLabel] += 1
#保存信息熵
shannonEnt = 0.0
#样本遍历完后，计算给类别占总样本数的比例
for key in labelCounts:
#各类别所占比例
prob = float(labelCounts[key])/numEntries
#熵计算，自然对数以2为底
shannonEnt -= prob * log(prob,2)
return shannonEnt
#*****按照给定特征值划分数据集
#dataSet：待划分的数据集
#axis：特征索引，按照该特征进行划分
#value：所选特征下的某个取值
def splitDataSet(dataSet, axis, value):
#定义新变量，保存划分后的数据集
retDataSet = []
#遍历数据集dataSet的每一行(条)数据
for featVec in dataSet:
#第axis所对应的特征值为要用于划分的特征值
if featVec[axis] == value:
#取特征列[0]~[axis-1]
reducedFeatVec = featVec[:axis]
#取特征列[axis+1]~[最后一列]
reducedFeatVec.extend(featVec[axis+1:])
#裁剪掉 [axis]列的特征值
retDataSet.append(reducedFeatVec)
return retDataSet
#*****选择最好的特征划分数据集，即选信息增益最大的特征划分
#dataSet：待划分的数据集
def chooseBestFeatureToSplit(dataSet):
#特征维数
numFeatures = len(dataSet[0]) - 1
#数据集的整体熵
baseEntropy = calcShannonEnt(dataSet)
#保存最大信息增益，初始化为0
bestInfoGain = 0.0;
#信息增益最大的特征，初始化为-1列特征
bestFeature = -1
#遍历所有特征
for i in range(numFeatures):
#取第i列特征
featList = [example[i] for example in dataSet]
#第i列特征下的所有不重复取值
uniqueVals = set(featList)
#条件熵，初始化为0
newEntropy = 0.0
#遍历某个特征下的所有值
for value in uniqueVals:
#将dataSet按照第i列特征==value划分成子数据集subDataSet
subDataSet = splitDataSet(dataSet, i, value)
#按第i列特征划分后的各子集在总数据集dataSet所占的比例
prob = len(subDataSet)/float(len(dataSet))
#按照第i列特征划分后的条件熵
newEntropy += prob * calcShannonEnt(subDataSet)
#信息增益=整体熵-条件熵
infoGain = baseEntropy - newEntropy
#找最大信息增益及所对应的特征
if (infoGain > bestInfoGain):
bestInfoGain = infoGain
bestFeature = i #最大信息增益对应的特征
return bestFeature
#所有特征用完后，以数据集中类别最多的类别作为最终类别
#classList:类别
def majorityCnt(classList):
#<key,value>字典结构，key存类别，value存类别对应的样本数
classCount={}
#遍历数据集中的类别
for vote in classList:
#若类别不在字典中，则添加到字典中去
if vote not in classCount.keys(): classCount[vote] = 0
#若某类别值出现一次，则累加1,即统计某类别下的样本数
classCount[vote] += 1
#按照classCount[vote]值从大到小排序（即各类别下样本数从大到小排序）
sortedClassCount = sorted(classCount.iteritems(), key=operator.itemgetter(1), reverse=True)
#返回含样本数最多的类别
return sortedClassCount[0][0]
#*****递归创建决策树
#dataSet：数据集
#labels：特征名称
def createTree(dataSet,labels):
#获取数据集最后一列，即类别标签
classList = [example[-1] for example in dataSet]
#*当划分的数据集属于同一类别则停止划分，返回该类别
if classList.count(classList[0]) == len(classList):
return classList[0]
#*划分的数据集已经没有特征值，返回出现次数多的类别
if len(dataSet[0]) == 1:
return majorityCnt(classList)
#*递归未终止
#选出最佳划分的特征所对应的索引，即信息增益最大的划分特征
bestFeat = chooseBestFeatureToSplit(dataSet)
#最佳划分的特征名称
bestFeatLabel = labels[bestFeat]
#将该特征名称作为根节点
myTree = {bestFeatLabel:{}}
#删除在原特征名称中的最佳划分特征名称
del(labels[bestFeat])
#取出最佳的划分特征列
featValues = [example[bestFeat] for example in dataSet]
#特征列下的不重复特征值（去重）
uniqueVals = set(featValues)
#遍历最佳划分特征下的特征值
for value in uniqueVals:
#获取删除最佳划分特征名称后的特征名称集合
subLabels = labels[:]
#对划分后的子集进行递归调用构建决策树，将递归调用的结果作为树的一个分支
myTree[bestFeatLabel][value] = createTree(splitDataSet(dataSet, bestFeat, value),subLabels)
return myTree
#*****使用决策树进行分类
#inputTree：训练好的决策树
#featLabels：特征的名称
#testVec：测试数据
def classify(inputTree,featLabels,testVec):
#存放决策树的根节点名称(比如no surfacing)
firstStr = inputTree.keys()[0]
print 'firstStr=',firstStr
print 'featLabels=',featLabels
#除根节点名称外的值
#即{0:'no',1:{'flippers':{0:'no',1:'yes'}}}
secondDict = inputTree[firstStr]
print 'secondDict=',secondDict
#index方法查找当前列表中第一个匹配firstStr变量的元素的索引
#即找到树根节点在所有特征列的第几列
featIndex = featLabels.index(firstStr)
print 'featIndex=',featIndex
#测试数据对应根节点所在特征下的特征值
key = testVec[featIndex]
#secondDict[0]='no'secondDict[1]={'flippers':{0:'no',1:'yes'}}
valueOfFeat = secondDict[key]
#判断valueOfFeat的类型
#valueOfFeat为dict字典类型，递归寻找
if isinstance(valueOfFeat, dict):
classLabel = classify(valueOfFeat, featLabels, testVec)
#valueOfFeat为数值，直接返回该值（最终类别）
#此处valueOfFeat=secondDict[key]='no'返回no
else: classLabel = valueOfFeat
#返回最终类别
return classLabel
def storeTree(inputTree,filename):
import pickle
fw = open(filename,'w')
pickle.dump(inputTree,fw)
fw.close()
def grabTree(filename):
import pickle
fr = open(filename)
return pickle.load(fr)
#==================================================================
####绘图：绘构建的树
# import matplotlib.pyplot as plt
#
# decisionNode = dict(boxstyle="sawtooth", fc="0.8")
# leafNode = dict(boxstyle="round4", fc="0.8")
# arrow_args = dict(arrowstyle="<-")
#
# def getNumLeafs(myTree):
# numLeafs = 0
# firstStr = myTree.keys()[0]
# secondDict = myTree[firstStr]
# for key in secondDict.keys():
# if type(secondDict[key]).__name__=='dict':#test to see if the nodes are dictonaires, if not they are leaf nodes
# numLeafs += getNumLeafs(secondDict[key])
# else: numLeafs +=1
# return numLeafs
#
# def getTreeDepth(myTree):
# maxDepth = 0
# firstStr = myTree.keys()[0]
# secondDict = myTree[firstStr]
# for key in secondDict.keys():
# if type(secondDict[key]).__name__=='dict':#test to see if the nodes are dictonaires, if not they are leaf nodes
# thisDepth = 1 + getTreeDepth(secondDict[key])
# else: thisDepth = 1
# if thisDepth > maxDepth: maxDepth = thisDepth
# return maxDepth
#
# def plotNode(nodeTxt, centerPt, parentPt, nodeType):
# createPlot.ax1.annotate(nodeTxt, xy=parentPt, xycoords='axes fraction',
# xytext=centerPt, textcoords='axes fraction',
# va="center", ha="center", bbox=nodeType, arrowprops=arrow_args )
#
# def plotMidText(cntrPt, parentPt, txtString):
# xMid = (parentPt[0]-cntrPt[0])/2.0 + cntrPt[0]
# yMid = (parentPt[1]-cntrPt[1])/2.0 + cntrPt[1]
# createPlot.ax1.text(xMid, yMid, txtString, va="center", ha="center", rotation=30)
#
# def plotTree(myTree, parentPt, nodeTxt):#if the first key tells you what feat was split on
# numLeafs = getNumLeafs(myTree) #this determines the x width of this tree
# depth = getTreeDepth(myTree)
# firstStr = myTree.keys()[0] #the text label for this node should be this
# cntrPt = (plotTree.xOff + (1.0 + float(numLeafs))/2.0/plotTree.totalW, plotTree.yOff)
# plotMidText(cntrPt, parentPt, nodeTxt)
# plotNode(firstStr, cntrPt, parentPt, decisionNode)
# secondDict = myTree[firstStr]
# plotTree.yOff = plotTree.yOff - 1.0/plotTree.totalD
# for key in secondDict.keys():
# if type(secondDict[key]).__name__=='dict':#test to see if the nodes are dictonaires, if not they are leaf nodes
# plotTree(secondDict[key],cntrPt,str(key)) #recursion
# else: #it's a leaf node print the leaf node
# plotTree.xOff = plotTree.xOff + 1.0/plotTree.totalW
# plotNode(secondDict[key], (plotTree.xOff, plotTree.yOff), cntrPt, leafNode)
# plotMidText((plotTree.xOff, plotTree.yOff), cntrPt, str(key))
# plotTree.yOff = plotTree.yOff + 1.0/plotTree.totalD
# #if you do get a dictonary you know it's a tree, and the first element will be another dict
#
# def createPlot(inTree):
# fig = plt.figure(1, facecolor='white')
# fig.clf()
# axprops = dict(xticks=[], yticks=[])
# createPlot.ax1 = plt.subplot(111, frameon=False, **axprops) #no ticks
# #createPlot.ax1 = plt.subplot(111, frameon=False) #ticks for demo puropses
# plotTree.totalW = float(getNumLeafs(inTree))
# plotTree.totalD = float(getTreeDepth(inTree))
# plotTree.xOff = -0.5/plotTree.totalW; plotTree.yOff = 1.0;
# plotTree(inTree, (0.5,1.0), '')
# plt.show()
##def createPlot():
## fig = plt.figure(1, facecolor='white')
## fig.clf()
## createPlot.ax1 = plt.subplot(111, frameon=False) #ticks for demo puropses
## plotNode('a decision node', (0.5, 0.1), (0.1, 0.5), decisionNode)
## plotNode('a leaf node', (0.8, 0.1), (0.3, 0.8), leafNode)
## plt.show()
##def retrieveTree(i):
## listOfTrees =[{'no surfacing': {0: 'no', 1: {'flippers': {0: 'no', 1: 'yes'}}}},
## {'no surfacing': {0: 'no', 1: {'flippers': {0: {'head': {0: 'no', 1: 'yes'}}, 1: 'no'}}}}
## ]
## return listOfTrees[i]
##
###createPlot(thisTree)
####绘图：绘构建的树
#==================================================================
if __name__ == "__main__":
mydat,labels=createDataSet()
featlabels=labels
mytree=createTree(mydat,labels)
#需要将初始得到的labels保存备份。
print mytree
#****隐形眼镜数据集
##
## fr=open('lenses.txt','r')
## lenses=[line.strip().split('\t')for line in fr.readlines()]
## lensesLabels=['age','prescript','astigmatic','tearRate']
## lensesTree=createTree(lenses,lensesLabels)
## print lensesTree
## createPlot(lensesTree) #绘图：构建的树
本文章为转载内容，我们尊重原作者对文章享有的著作权。如有内容错误或侵权问题，欢迎原作者联系我们进行内容更正或删除文章。
