private double computeInfoDiscrete(double[][] input, int[] output, double[] weight,
int attributeIndex, out List <int>[] partitions, out List <int> missingValues)
{
// Compute the information gain obtained by using
// this current attribute as the next decision node.
double info = 0;
IntRange valueRange = inputRanges[attributeIndex];
int numberOfDistinctValues = valueRange.Length + 1;
partitions = new List <int> [numberOfDistinctValues];
missingValues = new List <int>();
for (int j = 0; j < input.Length; j++)
{
if (Double.IsNaN(input[j][attributeIndex]))
{
missingValues.Add(j);
}
}
// For each possible value of the attribute
for (int i = 0; i < numberOfDistinctValues; i++)
{
int value = valueRange.Min + i;
// Partition the remaining data set
// according to the attribute values
var indicesInPartition = new List <int>();
double weightTotalSum = 0;
double weightSubsetSum = 0;
for (int j = 0; j < input.Length; j++)
{
double x = input[j][attributeIndex];
if (!Double.IsNaN(x) && x == value)
{
indicesInPartition.Add(j);
weightSubsetSum += weight[j];
}
weightTotalSum += weight[j];
}
// For each of the instances under responsibility
// of this node, check which have the same value
int[] outputSubset = output.Get(indicesInPartition);
double[] weightSubset = weight.Get(indicesInPartition);
// Check the entropy gain originating from this partitioning
double e = Measures.WeightedEntropy(outputSubset, weightSubset, Model.NumberOfClasses);
info += (weightSubsetSum / weightTotalSum) * e;
partitions[i] = indicesInPartition;
}
return(info);
}
private double computeInfoContinuous(double[][] input, int[] output, double[] weight,
int attributeIndex, out List <int>[] partitions, out List <int> missingValues, out double threshold)
{
// Compute the information gain obtained by using
// this current attribute as the next decision node.
double[] t = thresholds[attributeIndex];
double bestGain = Double.NegativeInfinity;
missingValues = new List <int>();
for (int j = 0; j < input.Length; j++)
{
if (Double.IsNaN(input[j][attributeIndex]))
{
missingValues.Add(j);
}
}
// If there are no possible thresholds that we can use
// to split the data (i.e. if all values are the same)
if (t.Length == 0)
{
// Then they all belong to the same partition
partitions = new[] { new List <int>(Vector.Range(input.Length)), null };
threshold = Double.NegativeInfinity;
return(bestGain);
}
partitions = null;
double bestThreshold = t[0];
var indicesBelowThreshold = new List <int>(input.Length);
var indicesAboveThreshold = new List <int>(input.Length);
var output1 = new List <int>(input.Length);
var output2 = new List <int>(input.Length);
var weights1 = new List <double>(input.Length);
var weights2 = new List <double>(input.Length);
// For each possible splitting point of the attribute
for (int i = 0; i < t.Length; i += splitStep)
{
// Partition the remaining data set
// according to the threshold value
double value = t[i];
for (int j = 0; j < input.Length; j++)
{
double x = input[j][attributeIndex];
if (Double.IsNaN(x))
{
continue;
}
else if (x <= value)
{
indicesBelowThreshold.Add(j);
output1.Add(output[j]);
weights1.Add(weight[j]);
}
else if (x > value)
{
indicesAboveThreshold.Add(j);
output2.Add(output[j]);
weights2.Add(weight[j]);
}
}
double weightSum = weight.Sum();
double p1 = weights1.Sum() / weightSum;
double p2 = weights2.Sum() / weightSum;
double splitGain =
-p1 *Measures.WeightedEntropy(output1, weights1, Model.NumberOfClasses) +
-p2 *Measures.WeightedEntropy(output2, weights2, Model.NumberOfClasses);
if (splitGain > bestGain)
{
bestThreshold = value;
bestGain = splitGain;
if (indicesBelowThreshold.Count == 0)
{
indicesBelowThreshold = null;
}
if (indicesAboveThreshold.Count == 0)
{
indicesAboveThreshold = null;
}
partitions = new[] { indicesBelowThreshold, indicesAboveThreshold };
indicesBelowThreshold = new List <int>(input.Length);
indicesAboveThreshold = new List <int>(input.Length);
}
else
{
indicesBelowThreshold.Clear();
indicesAboveThreshold.Clear();
}
output1.Clear();
output2.Clear();
weights1.Clear();
weights2.Clear();
}
threshold = bestThreshold;
return(bestGain);
}
private void split(DecisionNode root, double[][] inputs, int[] outputs, double[] weights, int height)
{
// 2. If all examples are for the same class, return the single-node
// tree with the output label corresponding to this common class.
double entropy = Measures.WeightedEntropy(outputs, weights, Model.NumberOfClasses);
if (entropy == 0)
{
if (outputs.Length > 0)
{
root.Output = outputs[0];
}
return;
}
// 3. If number of predicting attributes is empty, then return the single-node
// tree with the output label corresponding to the most common value of
// the target attributes in the examples.
// how many variables have been used less than the limit (if there is a limit)
int[] candidates = Matrix.Find(AttributeUsageCount, x => Join == 0 ? true : x < Join);
if (candidates.Length == 0 || (MaxHeight > 0 && height == MaxHeight) ||
(minimumSplitSize > 0 && inputs.Length < minimumSplitSize))
{
root.Output = Measures.WeightedMode(outputs, weights);
return;
}
// 4. Otherwise, try to select the attribute which
// best explains the data sample subset. If the tree
// is part of a random forest, only consider a percentage
// of the candidate attributes at each split point
if (MaxVariables > 0 && candidates.Length > MaxVariables)
{
candidates = Vector.Sample(candidates, MaxVariables);
}
var scores = new double[candidates.Length];
var thresholds = new double[candidates.Length];
var partitions = new List <int> [candidates.Length][];
if (ParallelOptions.MaxDegreeOfParallelism == 1)
{
// For each attribute in the data set
for (int i = 0; i < scores.Length; i++)
{
scores[i] = computeGainRatio(inputs, outputs, weights, candidates[i],
entropy, out partitions[i], out thresholds[i]);
}
}
else
{
// For each attribute in the data set
Parallel.For(0, scores.Length, ParallelOptions, i =>
{
scores[i] = computeGainRatio(inputs, outputs, weights, candidates[i],
entropy, out partitions[i], out thresholds[i]);
});
}
// Select the attribute with maximum gain ratio
int maxGainIndex; scores.Max(out maxGainIndex);
var maxGainPartition = partitions[maxGainIndex];
var maxGainAttribute = candidates[maxGainIndex];
var maxGainRange = inputRanges[maxGainAttribute];
var maxGainThreshold = thresholds[maxGainIndex];
// Mark this attribute as already used
AttributeUsageCount[maxGainAttribute]++;
double[][] inputSubset;
int[] outputSubset;
double[] weightSubset;
// Now, create next nodes and pass those partitions as their responsibilities.
if (Model.Attributes[maxGainAttribute].Nature == DecisionVariableKind.Discrete)
{
// This is a discrete nature attribute. We will branch at each
// possible value for the discrete variable and call recursion.
var children = new DecisionNode[maxGainPartition.Length];
// Create a branch for each possible value
for (int i = 0; i < children.Length; i++)
{
children[i] = new DecisionNode(Model)
{
Parent = root,
Value = i + maxGainRange.Min,
Comparison = ComparisonKind.Equal,
};
inputSubset = inputs.Get(maxGainPartition[i]);
outputSubset = outputs.Get(maxGainPartition[i]);
weightSubset = weights.Get(maxGainPartition[i]);
if (outputSubset.Length == 0)
{
//in this case the we have no samples for this category
//but we still want to be able to make a decision, so we will give the best of the current node as output
outputSubset = new int[1] {
Measures.WeightedMode(outputs, weights)
};
weightSubset = new double[1] {
1
}; //does not matter
}
split(children[i], inputSubset, outputSubset, weightSubset, height + 1); // recursion
}
root.Branches.AttributeIndex = maxGainAttribute;
root.Branches.AddRange(children);
}
else if (Model.Attributes[maxGainAttribute].Nature == DecisionVariableKind.Continuous)
{
List <int> partitionBelowThreshold = maxGainPartition[0];
List <int> partitionAboveThreshold = maxGainPartition[1];
if (partitionBelowThreshold != null && partitionAboveThreshold != null)
{
//Before we branch we test whether each node is big enough, we stop here and set it as a leaf node
if (partitionAboveThreshold.Count < minimumLeafSize || partitionBelowThreshold.Count < minimumLeafSize)
{
root.Output = Measures.WeightedMode(outputs, weights);
}
else
{
// This is a continuous nature attribute, and we achieved two partitions
// using the partitioning scheme. We will branch on two possible settings:
// either the value is greater than a currently detected optimal threshold
// or it is less.
DecisionNode[] children =
{
new DecisionNode(Model)
{
Parent = root, Value = maxGainThreshold,
Comparison = ComparisonKind.LessThanOrEqual
},
new DecisionNode(Model)
{
Parent = root, Value = maxGainThreshold,
Comparison = ComparisonKind.GreaterThan
}
};
// Create a branch for lower values
inputSubset = inputs.Get(partitionBelowThreshold);
outputSubset = outputs.Get(partitionBelowThreshold);
weightSubset = weights.Get(partitionBelowThreshold);
split(children[0], inputSubset, outputSubset, weightSubset, height + 1);
// Create a branch for higher values
inputSubset = inputs.Get(partitionAboveThreshold);
outputSubset = outputs.Get(partitionAboveThreshold);
weightSubset = weights.Get(partitionAboveThreshold);
split(children[1], inputSubset, outputSubset, weightSubset, height + 1);
root.Branches.AttributeIndex = maxGainAttribute;
root.Branches.AddRange(children);
}
}
else
{
// This is a continuous nature attribute, but all variables are equal
// to a constant. If there is only a constant value as the predictor
// and there are multiple output labels associated with this constant
// value, there isn't much we can do. This node will be a leaf.
// We will set the class label for this node as the
// majority of the currently selected output classes.
var outputIndices = partitionBelowThreshold ?? partitionAboveThreshold;
outputSubset = outputs.Get(outputIndices);
root.Output = Measures.Mode(outputSubset);
}
}
AttributeUsageCount[maxGainAttribute]--;
}