Package rdkit :: Package ML :: Package KNN :: Module CrossValidate
[hide private]
[frames] | no frames]

Source Code for Module rdkit.ML.KNN.CrossValidate

  1  # 
  2  #  Copyright (C) 2000-2008  greg Landrum 
  3  # 
  4  """ handles doing cross validation with k-nearest neighbors model 
  5   
  6  and evaluation of individual models 
  7   
  8  """ 
  9  from __future__ import print_function 
 10   
 11  from rdkit.ML.Data import SplitData 
 12  from rdkit.ML.KNN import DistFunctions 
 13  from rdkit.ML.KNN.KNNClassificationModel import KNNClassificationModel 
 14  from rdkit.ML.KNN.KNNRegressionModel import KNNRegressionModel 
 15   
 16   
17 -def makeClassificationModel(numNeigh, attrs, distFunc):
18 return KNNClassificationModel(numNeigh, attrs, distFunc)
19 20
21 -def makeRegressionModel(numNeigh, attrs, distFunc):
22 return KNNRegressionModel(numNeigh, attrs, distFunc)
23 24
25 -def CrossValidate(knnMod, testExamples, appendExamples=0):
26 """ 27 Determines the classification error for the testExamples 28 29 **Arguments** 30 31 - tree: a decision tree (or anything supporting a _ClassifyExample()_ method) 32 33 - testExamples: a list of examples to be used for testing 34 35 - appendExamples: a toggle which is passed along to the tree as it does 36 the classification. The trees can use this to store the examples they 37 classify locally. 38 39 **Returns** 40 41 a 2-tuple consisting of: 42 """ 43 nTest = len(testExamples) 44 45 if isinstance(knnMod, KNNClassificationModel): 46 badExamples = [] 47 nBad = 0 48 for i in range(nTest): 49 testEx = testExamples[i] 50 trueRes = testEx[-1] 51 res = knnMod.ClassifyExample(testEx, appendExamples) 52 if (trueRes != res): 53 badExamples.append(testEx) 54 nBad += 1 55 return float(nBad) / nTest, badExamples 56 elif isinstance(knnMod, KNNRegressionModel): 57 devSum = 0.0 58 for i in range(nTest): 59 testEx = testExamples[i] 60 trueRes = testEx[-1] 61 res = knnMod.PredictExample(testEx, appendExamples) 62 devSum += abs(trueRes - res) 63 return devSum / nTest, None 64 raise ValueError("Unrecognized Model Type")
65 66
67 -def CrossValidationDriver(examples, attrs, nPossibleValues, numNeigh, 68 modelBuilder=makeClassificationModel, 69 distFunc=DistFunctions.EuclideanDist, holdOutFrac=0.3, silent=0, 70 calcTotalError=0, **kwargs):
71 """ Driver function for building a KNN model of a specified type 72 73 **Arguments** 74 75 - examples: the full set of examples 76 77 - numNeigh: number of neighbors for the KNN model (basically k in k-NN) 78 79 - knnModel: the type of KNN model (a classification vs regression model) 80 81 - holdOutFrac: the fraction of the data which should be reserved for the hold-out set 82 (used to calculate error) 83 84 - silent: a toggle used to control how much visual noise this makes as it goes 85 86 - calcTotalError: a toggle used to indicate whether the classification error 87 of the tree should be calculated using the entire data set (when true) or just 88 the training hold out set (when false) 89 """ 90 91 nTot = len(examples) 92 if not kwargs.get('replacementSelection', 0): 93 testIndices, trainIndices = SplitData.SplitIndices(nTot, holdOutFrac, silent=1, legacy=1, 94 replacement=0) 95 else: 96 testIndices, trainIndices = SplitData.SplitIndices(nTot, holdOutFrac, silent=1, legacy=0, 97 replacement=1) 98 trainExamples = [examples[x] for x in trainIndices] 99 testExamples = [examples[x] for x in testIndices] 100 101 nTrain = len(trainExamples) 102 103 if not silent: 104 print("Training with %d examples" % (nTrain)) 105 106 knnMod = modelBuilder(numNeigh, attrs, distFunc) 107 108 knnMod.SetTrainingExamples(trainExamples) 109 knnMod.SetTestExamples(testExamples) 110 111 if not calcTotalError: 112 xValError, _ = CrossValidate(knnMod, testExamples, appendExamples=1) 113 else: 114 xValError, _ = CrossValidate(knnMod, examples, appendExamples=0) 115 116 if not silent: 117 print('Validation error was %%%4.2f' % (100 * xValError)) 118 119 knnMod._trainIndices = trainIndices 120 return knnMod, xValError
121