private static Tuple <TDecisionValue, double> HandleRegressionAndModelLeaf(
IDataVector <TDecisionValue> vector,
IDecisionTreeNode decisionTree,
double probabilitiesProductSoFar)
{
var regressionLeaf = decisionTree as IDecisionTreeRegressionAndModelLeaf;
var numericVector = vector.NumericVector.ToList();
numericVector.Insert(0, 1.0);
var vectorWithIntercept = Vector <double> .Build.DenseOfArray(numericVector.ToArray());
double predictedVal = 0.0;
if (regressionLeaf.ModelWeights != null)
{
predictedVal =
vectorWithIntercept.DotProduct(Vector <double> .Build.DenseOfArray(regressionLeaf.ModelWeights.ToArray()));
}
else
{
predictedVal = regressionLeaf.DecisionMeanValue;
}
return(new Tuple <TDecisionValue, double>(
(TDecisionValue)Convert.ChangeType(predictedVal, typeof(TDecisionValue)),
probabilitiesProductSoFar));
}
private void DoExtractPatterns(IDecisionTreeNode node,
List <SelectorContext> contexts, InstanceModel model,
Action <EmergingPattern> patternFound,
Feature classFeature)
{
if (node.IsLeaf)
{
EmergingPattern newPattern = Create(contexts, model, classFeature);
newPattern.Counts = node.Data;
newPattern.Supports = CalculateSupports(node.Data, classFeature);
newPattern.ClassValue = newPattern.Supports.ArgMax();
if (patternFound != null)
{
patternFound(newPattern);
}
}
else
{
for (int i = 0; i < node.Children.Length; i++)
{
SelectorContext context = new SelectorContext
{
Index = i,
Selector = node.ChildSelector,
};
contexts.Add(context);
DoExtractPatterns(node.Children[i], contexts, model, patternFound,
classFeature);
contexts.Remove(context);
}
}
}
/// <summary>
/// Creates a new decision node.
/// </summary>
/// <param name="test">
/// Function which will make the true or false decision.
/// </param>
/// <param name="trueNode">
/// DT node to return when the decision is true.
/// </param>
/// <param name="falseNode">
/// DT node to return when the decision is false.
/// </param>
public DecisionNode(
Func <bool> test,
IDecisionTreeNode trueNode, IDecisionTreeNode falseNode)
{
this.decisionFunc = test;
this.trueNode = trueNode;
this.falseNode = falseNode;
}
/// <summary>
/// Make a decision.
/// </summary>
/// <returns>
/// A DT node, which will depend wether the decision was true or false.
/// </returns>
public IDecisionTreeNode MakeDecision()
{
// Determine the branch to take depending on the decision function
IDecisionTreeNode branch = decisionFunc() ? trueNode : falseNode;
// Recursively return the branch returned by the selected branch
return(branch.MakeDecision());
}
private double[] ClassifyInstance(IDecisionTreeNode node, Instance instance, double instanceMembership)
{
if (node.IsLeaf)
{
return(node.Data.MultiplyBy(instanceMembership));
}
double[] childrenSelection = node.ChildSelector.Select(instance);
if (childrenSelection != null)
{
if (childrenSelection.Length != node.Children.Length)
{
throw new IndexOutOfRangeException("Child index is out of range");
}
double[] result = null;
for (int i = 0; i < childrenSelection.Length; i++)
{
double selection = childrenSelection[i];
if (selection > 0)
{
IDecisionTreeNode child = node.Children[i];
double[] childValue = ClassifyInstance(child, instance, instanceMembership);
if (result != null)
{
result = result.AddTo(childValue);
}
else
{
result = childValue;
}
}
}
return(result);
}
else
{
double[] result = null;
double totalNodeMembership = node.Data.Sum();
for (int i = 0; i < node.Children.Length; i++)
{
IDecisionTreeNode child = node.Children[i];
double childMembership = node.Children[i].Data.Sum();
double[] childValue = ClassifyInstance(child, instance, childMembership / totalNodeMembership * instanceMembership);
if (result != null)
{
result = result.AddTo(childValue);
}
else
{
result = childValue;
}
}
return(result);
}
}
public IDecisionTreeNode MakeDecision()
{
IDecisionTreeNode branch = GetBranch();
if (branch != null)
{
branch = branch.MakeDecision();
}
return(branch);
}
public static IDecisionTreeNode Do(this IDecisionTreeNode self,
IDecisionTreeNode node)
{
if (self is BaseDecision bd)
{
bd.TrueNode = node;
return(self);
}
log.Error("You can use it only with BaseDecision");
return(self);
}
public IPredictionModel BuildModel(IDataFrame dataFrame, string dependentFeatureName, IModelBuilderParams additionalParams)
{
if (!(additionalParams is IRandomForestModelBuilderParams))
{
throw new ArgumentException("Invalid parameters passed to Random Forest Model builder!");
}
var randomForestParams = additionalParams as IRandomForestModelBuilderParams;
var trees = new IDecisionTreeNode[randomForestParams.TreesCount];
var oobErrors = new double[randomForestParams.TreesCount];
var columnsCountToTake = featuresToUseCountCalculator(dataFrame.ColumnsCount - 1);
var featureColumns = dataFrame.ColumnNames.Except(new[] { dependentFeatureName }).ToList();
var randomizer = new Random();
var locker = new object();
Parallel.For(
0,
randomForestParams.TreesCount,
i =>
{
var localRandomizer = new Random(i.GetHashCode());
var randomlySelectedIndices =
Enumerable.Range(0, dataFrame.RowCount)
.Select(_ => localRandomizer.Next(0, dataFrame.RowCount))
.ToList();
var outOfBagIndices =
Enumerable.Range(0, dataFrame.RowCount).Except(randomlySelectedIndices).ToList();
var columnsToTake = new List <string>();
columnsToTake = featureColumns.Shuffle(localRandomizer).Take(columnsCountToTake).ToList();
columnsToTake.Add(dependentFeatureName);
var baggedTestData = dataFrame.Slice(randomlySelectedIndices, columnsToTake);
var oobTestData = dataFrame.Slice(outOfBagIndices, columnsToTake);
var oobExpected = oobTestData.GetColumnVector <TPredictionVal>(dependentFeatureName).Values;
var decisionTree = decisionTreeModelBuilder.BuildModel(
baggedTestData,
dependentFeatureName,
decisionTreeModelBuilderParamsFactory());
var prediction = decisionTreePredictor.Predict(oobTestData, decisionTree, dependentFeatureName);
//TODO: AAA !!! Later on add support for calculating variable importance!!!
var oobError = dataQualityMeasure.CalculateError(oobExpected, prediction);
trees[i] = decisionTree as IDecisionTreeNode;
oobErrors[i] = oobError;
});
return(new RandomForestModel(trees, oobErrors));
}
public void TestDecisionTreeStructure()
{
IDecisionTreeNode first = new DecisionTreeLeafNode(2);
var tree = new DecisionTree(first);
AreEqual(2, tree.Classify(new[] { 1, 2, 3 }));
var tests = new IDecisionTreeNode[]
{
new DecisionTreeLeafNode(0), new DecisionTreeLeafNode(0),
new DecisionTreeLeafNode(1), new DecisionTreeLeafNode(2)
};
first = new DecisionTreeInternalNode(1, tests);
tree = new DecisionTree(first);
int[][] x = { new[] { 0, 0 }, new[] { 0, 1 }, new[] { 0, 2 } };
for (var i = 0; i < 3; i++)
{
AreEqual(x[i][1], tree.Classify(x[i]));
}
}
private bool Prune(IDecisionTreeNode node)
{
if (node.IsLeaf)
{
return(true);
}
bool allPrunned = true;
foreach (IDecisionTreeNode child in node.Children)
{
bool childPrunned = Prune(child);
allPrunned = allPrunned && childPrunned;
}
if (allPrunned && PruneTester.CanPrune(node))
{
node.ChildSelector = null;
node.Children = null;
}
return(false);
}
public string ToString(int ident, InstanceModel model, int digits = -1)
{
// "-[2,3]\n -IntFeature<=4 [1,1]\n -IntFeature>4 [1,2]"
StringBuilder builder = new StringBuilder();
builder.Append(Data.ToStringEx(digits));
if (!IsLeaf)
{
for (int i = 0; i < Children.Length; i++)
{
IDecisionTreeNode child = Children[i];
builder.Append("\n");
builder.Append(' ', (ident + 1) * 3);
builder.Append("- ");
builder.Append(ChildSelector.ToString(model, i));
builder.Append(' ');
builder.Append(child.ToString(ident + 1, model));
}
}
return(builder.ToString());
}
// Create the decision tree
protected override void Start()
{
// Call base class Start()
base.Start();
// Create the leaf actions
IDecisionTreeNode seek = new ActionNode(SeekAction);
IDecisionTreeNode nothing = new ActionNode(StandStillAction);
IDecisionTreeNode patrol = new ActionNode(PatrolAction);
// Create the random decision behaviour node
RandomDecisionBehaviour rdb = new RandomDecisionBehaviour(
() => Random.value,
() => Time.frameCount,
randomDecisionDurationInFrames,
0.55f);
IDecisionTreeNode rndNode = new DecisionNode(
rdb.RandomDecision, seek, nothing);
// Create the root node
root = new DecisionNode(PlayerInSight, rndNode, patrol);
}
public IDecisionTreeLink GetChildLinkForChild(IDecisionTreeNode child)
{
return(this.LinksByChildren[child]);
}
private int ComputeLeaves(IDecisionTreeNode decisionTree)
{
return(decisionTree.IsLeaf ? 1 : decisionTree.Children.Sum(childs => ComputeLeaves(childs)));
}
private void FillNode(ref Dictionary <IDecisionTreeNode, double> validityIndexByNode, InstanceModel model, ref Dictionary <IDecisionTreeNode, IEnumerable <Tuple <Instance, double> > > instancesByNode,
Feature classFeature, ref Dictionary <IDecisionTreeNode, int> levelByNode, List <SelectorContext> currentContext, ref int leafCount)
{
IDecisionTreeNode node = null;
double bestIndexValue = Double.MinValue;
foreach (var currentNode in validityIndexByNode.Keys)
{
if (bestIndexValue < validityIndexByNode[currentNode])
{
bestIndexValue = validityIndexByNode[currentNode];
node = currentNode;
}
}
if (node != null)
{
int level = levelByNode[node];
var instances = instancesByNode[node];
int whichBetterToFind = 1;
if (OnSelectingWhichBetterSplit != null)
{
whichBetterToFind = OnSelectingWhichBetterSplit(node, level);
}
WiningSplitSelector winingSplitSelector = new WiningSplitSelector(whichBetterToFind)
{
CanAcceptChildSelector = this.CanAcceptChildSelector,
};
foreach (var feature in OnSelectingFeaturesToConsider(model.Features, level))
{
if (feature != classFeature)
{
ISplitIterator splitIterator = SplitIteratorProvider.GetSplitIterator(model, feature, classFeature);
if (splitIterator == null)
{
throw new InvalidOperationException(string.Format("Undefined iterator for feature {0}",
feature));
}
splitIterator.Initialize(feature, instances);
while (splitIterator.FindNext())
{
double currentGain = DistributionEvaluator.Evaluate(node.Data,
splitIterator.CurrentDistribution);
if (currentGain > MinimalSplitGain || leafCount < ClusterCount)
{
if (OnSplitEvaluation != null)
{
OnSplitEvaluation(node, splitIterator, currentContext);
}
winingSplitSelector.EvaluateThis(currentGain, splitIterator, level);
}
}
}
}
if (winingSplitSelector.IsWinner())
{
IChildSelector maxSelector = winingSplitSelector.WinningSelector;
node.ChildSelector = maxSelector;
node.Children = new IDecisionTreeNode[maxSelector.ChildrenCount];
var instancesPerChildNode =
childrenInstanceCreator.CreateChildrenInstances(instances, maxSelector, double.MinValue);
for (int i = 0; i < maxSelector.ChildrenCount; i++)
{
var childNode = new DecisionTreeNode {
Parent = node
};
node.Children[i] = childNode;
childNode.Data = winingSplitSelector.WinningDistribution[i];
SelectorContext context = null;
if (OnSplitEvaluation != null)
{
context = new SelectorContext
{
Index = i,
Selector = node.ChildSelector,
};
currentContext.Add(context);
}
double currentBestValidityIndex = double.MinValue;
foreach (var feature in OnSelectingFeaturesToConsider(model.Features, level))
{
if (feature != classFeature)
{
ISplitIterator splitIterator = SplitIteratorProvider.GetSplitIterator(model, feature, classFeature);
if (splitIterator == null)
{
throw new InvalidOperationException(string.Format("Undefined iterator for feature {0}",
feature));
}
splitIterator.Initialize(feature, instancesPerChildNode[i]);
while (splitIterator.FindNext())
{
double currentGain = DistributionEvaluator.Evaluate(node.Data,
splitIterator.CurrentDistribution);
if (currentGain > currentBestValidityIndex)
{
if (OnSplitEvaluation != null)
{
OnSplitEvaluation(node, splitIterator, currentContext);
}
currentBestValidityIndex = currentGain;
}
}
}
}
if (currentBestValidityIndex > validityIndexByNode[node] || leafCount < ClusterCount)
{
validityIndexByNode.Add(childNode, currentBestValidityIndex);
instancesByNode.Add(childNode, instancesPerChildNode[i]);
levelByNode.Add(childNode, level + 1);
}
if (OnSplitEvaluation != null)
{
currentContext.Remove(context);
}
}
validityIndexByNode.Remove(node);
instancesByNode.Remove(node);
levelByNode.Remove(node);
leafCount++;
if (leafCount < 4 * ClusterCount)
{
FillNode(ref validityIndexByNode, model, ref instancesByNode, classFeature, ref levelByNode, currentContext, ref leafCount);
}
}
}
}
public void Init(IDecisionTreeNode trueNode, IDecisionTreeNode falseNode)
{
TrueNode = trueNode;
FalseNode = falseNode;
}
private int ComputeSizeTree(IDecisionTreeNode decisionTree)
{
return(decisionTree.Children == null
? 0
: decisionTree.Children.Select(ComputeSizeTree).Concat(new[] { 0 }).Max() + 1);
}
public void SetDecisionTree(IDecisionTreeNode decisionTree) => this.decisionTree = CurrentDecision = decisionTree;
public DecisionTree(IDecisionTreeNode root)
{
_root = root;
}
private static IDecisionTreeNode IterativeDichotomiser3(int[,] x, int[] y, List <int> attributes)
{
var distinctLabelCount = y.Distinct().Count();
if (x.Length == 0)
{
return(new DecisionTreeLeafNode(0));
}
switch (distinctLabelCount)
{
case 1:
return(new DecisionTreeLeafNode(y[0]));
default:
{
if (attributes.Count == 0)
{
return(new DecisionTreeLeafNode(MajorLabel(y, distinctLabelCount)));
}
var attributeTest = BestAttribute(x, y, attributes);
if (attributeTest == -1)
{
return(new DecisionTreeLeafNode(MajorLabel(y, distinctLabelCount)));
}
attributes.Remove(attributeTest);
var attributeStateCount = 0;
for (var i = 0; i < y.Length; i++)
{
if (x[i, attributeTest] > attributeStateCount)
{
attributeStateCount = x[i, attributeTest];
}
}
++attributeStateCount;
attributeStateCount = Math.Max(_discreteClasses * 2, attributeStateCount);
// Split data
var subX = new int[attributeStateCount][, ];
var subY = new int[attributeStateCount][];
var attributeSubXCount = new int[attributeStateCount];
for (var i = 0; i < y.Length; i++)
{
++attributeSubXCount[x[i, attributeTest]];
}
var attributeLength = x.Length / y.Length;
for (var i = 0; i < attributeStateCount; i++)
{
subX[i] = new int[attributeSubXCount[i], attributeLength];
subY[i] = new int[attributeSubXCount[i]];
}
var subXAttributeFormCount = new int[attributeStateCount];
for (var i = 0; i < y.Length; i++)
{
for (var j = 0; j < attributeLength; j++)
{
subX[x[i, attributeTest]][subXAttributeFormCount[x[i, attributeTest]], j] = x[i, j];
}
subY[x[i, attributeTest]][subXAttributeFormCount[x[i, attributeTest]]] = y[i];
++subXAttributeFormCount[x[i, attributeTest]];
}
// Create tests
var tests = new IDecisionTreeNode[attributeStateCount];
for (var i = 0; i < attributeStateCount; i++)
{
if (subX[i].Length == 0)
{
tests[i] = new DecisionTreeLeafNode(MajorLabel(y, distinctLabelCount));
}
else
{
tests[i] = IterativeDichotomiser3(subX[i], subY[i],
attributes.GetRange(0, attributes.Count));
}
}
IDecisionTreeNode root = new DecisionTreeInternalNode(attributeTest, tests);
return(root);
}
}
}
public LoggingDecorator(IDecisionTreeNode decision, DebugInfo debugInfo)
{
this.decision = decision;
this.debugInfo = debugInfo;
}
private static Tuple <TDecisionValue, double> HandleLeaf(IDataVector <TDecisionValue> vector, IDecisionTreeNode decisionTree, double probabilitiesProductSoFar)
{
if (decisionTree is IDecisionTreeRegressionAndModelLeaf)
{
return(HandleRegressionAndModelLeaf(vector, decisionTree, probabilitiesProductSoFar));
}
var classificationLeaf = decisionTree as IDecisionTreeLeaf;
return(new Tuple <TDecisionValue, double>((TDecisionValue)classificationLeaf.LeafValue, probabilitiesProductSoFar));
}
private Tuple <TDecisionValue, double> ProcessInstance(IDataVector <TDecisionValue> vector, IDecisionTreeNode decisionTree, double probabilitiesProductSoFar)
{
if (decisionTree is IDecisionTreeLeaf)
{
return(HandleLeaf(vector, decisionTree, probabilitiesProductSoFar));
}
var parentNode = decisionTree as IDecisionTreeParentNode;
if (parentNode is IBinaryDecisionTreeParentNode)
{
return(this.ProcessBinarySplit(vector, parentNode as IBinaryDecisionTreeParentNode, probabilitiesProductSoFar));
}
return(this.ProcessMultiValueSplit(vector, parentNode, probabilitiesProductSoFar));
}
public object GetTestResultForChild(IDecisionTreeNode child)
{
return(this.TestResultsByChildren[child]);
}
