/// <summary>
/// Apply a trained forest to some test data.
/// </summary>
/// <typeparam name="F">Type of split function</typeparam>
/// <param name="forest">Trained forest</param>
/// <param name="testData">Test data</param>
/// <returns>An array of class distributions, one per test data point</returns>
public static HistogramAggregator[] Test <F>(Forest <F, HistogramAggregator> forest, DataPointCollection testData) where F : IFeatureResponse
{
int nClasses = forest.GetTree(0).GetNode(0).TrainingDataStatistics.BinCount;
int[][] leafIndicesPerTree = forest.Apply(testData);
HistogramAggregator[] result = new HistogramAggregator[testData.Count()];
for (int i = 0; i < testData.Count(); i++)
{
// Aggregate statistics for this sample over all leaf nodes reached
result[i] = new HistogramAggregator(nClasses);
for (int t = 0; t < forest.TreeCount; t++)
{
int leafIndex = leafIndicesPerTree[t][i];
result[i].Aggregate(forest.GetTree(t).GetNode(leafIndex).TrainingDataStatistics);
}
}
return(result);
}
public static Bitmap Visualize(
Forest <AxisAlignedFeatureResponse, GaussianAggregator2d> forest,
DataPointCollection trainingData,
Size PlotSize,
PointF PlotDilation)
{
// Generate some test samples in a grid pattern (a useful basis for creating visualization images)
PlotCanvas plotCanvas = new PlotCanvas(trainingData.GetRange(0), trainingData.GetRange(1), PlotSize, PlotDilation);
// Apply the trained forest to the test data
Console.WriteLine("\nApplying the forest to test data...");
DataPointCollection testData = DataPointCollection.Generate2dGrid(plotCanvas.plotRangeX, PlotSize.Width, plotCanvas.plotRangeY, PlotSize.Height);
int[][] leafNodeIndices = forest.Apply(testData);
// Compute normalization factors per node
int nTrainingPoints = (int)(trainingData.Count()); // could also count over tree nodes if training data no longer accessible
double[][] normalizationFactors = new double[forest.TreeCount][];
for (int t = 0; t < forest.TreeCount; t++)
{
normalizationFactors[t] = new double[forest.GetTree(t).NodeCount];
ComputeNormalizationFactorsRecurse(forest.GetTree(t), 0, nTrainingPoints, new Bounds(2), normalizationFactors[t]);
}
Bitmap result = new Bitmap(PlotSize.Width, PlotSize.Height);
// Paint the test data
int index = 0;
for (int j = 0; j < PlotSize.Height; j++)
{
for (int i = 0; i < PlotSize.Width; i++)
{
// Map pixel coordinate (i,j) in visualization image back to point in input space
float x = plotCanvas.plotRangeX.Item1 + i * plotCanvas.stepX;
float y = plotCanvas.plotRangeY.Item1 + j * plotCanvas.stepY;
// Aggregate statistics for this sample over all trees
double probability = 0.0;
for (int t = 0; t < forest.TreeCount; t++)
{
int leafIndex = leafNodeIndices[t][index];
probability += normalizationFactors[t][leafIndex] * forest.GetTree(t).GetNode(leafIndex).TrainingDataStatistics.GetPdf().GetProbability(x, y);
}
probability /= forest.TreeCount;
// 'Gamma correct' probability density for better display
float l = (float)(LuminanceScaleFactor * Math.Pow(probability, Gamma));
if (l < 0)
{
l = 0;
}
else if (l > 255)
{
l = 255;
}
Color c = Color.FromArgb(255, (byte)(l), 0, 0);
result.SetPixel(i, j, c);
index++;
}
}
// Also plot the original training data
using (Graphics g = Graphics.FromImage(result))
{
g.InterpolationMode = System.Drawing.Drawing2D.InterpolationMode.HighQualityBicubic;
g.SmoothingMode = System.Drawing.Drawing2D.SmoothingMode.HighQuality;
for (int s = 0; s < trainingData.Count(); s++)
{
PointF x = new PointF(
(trainingData.GetDataPoint(s)[0] - plotCanvas.plotRangeX.Item1) / plotCanvas.stepX,
(trainingData.GetDataPoint(s)[1] - plotCanvas.plotRangeY.Item1) / plotCanvas.stepY);
RectangleF rectangle = new RectangleF(x.X - 2.0f, x.Y - 2.0f, 4.0f, 4.0f);
g.FillRectangle(new SolidBrush(DataPointColor), rectangle);
g.DrawRectangle(new Pen(Color.Black), rectangle.X, rectangle.Y, rectangle.Width, rectangle.Height);
}
}
return(result);
}
static void Main(string[] args)
{
if (args.Length == 0 || args[0] == "/?" || args[0].ToLower() == "help")
{
DisplayHelp();
return;
}
// These command line parameters are reused over several command line modes...
StringParameter trainingDataPath = new StringParameter("path", "Path of file containing training data.");
NaturalParameter T = new NaturalParameter("t", "No. of trees in the forest (default = {0}).", 10);
NaturalParameter D = new NaturalParameter("d", "Maximum tree levels (default = {0}).", 10, 20);
NaturalParameter F = new NaturalParameter("f", "No. of candidate feature responses per decision node (default = {0}).", 10);
NaturalParameter L = new NaturalParameter("l", "No. of candidate thresholds per feature response (default = {0}).", 1);
SingleParameter a = new SingleParameter("a", "The number of 'effective' prior observations (default = {0}).", true, false, 10.0f);
SingleParameter b = new SingleParameter("b", "The variance of the effective observations (default = {0}).", true, true, 400.0f);
SimpleSwitchParameter verboseSwitch = new SimpleSwitchParameter("Enables verbose progress indication.");
SingleParameter plotPaddingX = new SingleParameter("padx", "Pad plot horizontally (default = {0}).", true, false, 0.1f);
SingleParameter plotPaddingY = new SingleParameter("pady", "Pad plot vertically (default = {0}).", true, false, 0.1f);
EnumParameter split = new EnumParameter(
"s",
"Specify what kind of split function to use (default = {0}).",
new string[] { "axis", "linear" },
new string[] { "axis-aligned split", "linear split" },
"axis");
// Behaviour depends on command line mode...
string mode = args[0].ToLower(); // first argument defines the command line mode
if (mode == "clas" || mode == "class")
{
#region Supervised classification
CommandLineParser parser = new CommandLineParser();
parser.Command = "SW " + mode.ToUpper();
parser.AddArgument(trainingDataPath);
parser.AddSwitch("T", T);
parser.AddSwitch("D", D);
parser.AddSwitch("F", F);
parser.AddSwitch("L", L);
parser.AddSwitch("SPLIT", split);
parser.AddSwitch("PADX", plotPaddingX);
parser.AddSwitch("PADY", plotPaddingY);
parser.AddSwitch("VERBOSE", verboseSwitch);
// Default values up above should be fine here.
if (args.Length == 1)
{
parser.PrintHelp();
DisplayTextFiles(CLAS_DATA_PATH);
return;
}
if (parser.Parse(args, 1) == false)
{
return;
}
TrainingParameters trainingParameters = new TrainingParameters()
{
MaxDecisionLevels = D.Value - 1,
NumberOfCandidateFeatures = F.Value,
NumberOfCandidateThresholdsPerFeature = L.Value,
NumberOfTrees = T.Value,
Verbose = verboseSwitch.Used
};
PointF plotDilation = new PointF(plotPaddingX.Value, plotPaddingY.Value);
DataPointCollection trainingData = LoadTrainingData(
trainingDataPath.Value,
CLAS_DATA_PATH,
2,
DataDescriptor.HasClassLabels);
if (split.Value == "linear")
{
Forest <LinearFeatureResponse2d, HistogramAggregator> forest = ClassificationExample.Train(
trainingData,
new LinearFeatureFactory(),
trainingParameters);
using (Bitmap result = ClassificationExample.Visualize(forest, trainingData, new Size(300, 300), plotDilation))
{
ShowVisualizationImage(result);
}
}
else if (split.Value == "axis")
{
Forest <AxisAlignedFeatureResponse, HistogramAggregator> forest = ClassificationExample.Train(
trainingData,
new AxisAlignedFeatureFactory(),
trainingParameters);
using (Bitmap result = ClassificationExample.Visualize(forest, trainingData, new Size(300, 300), plotDilation))
{
ShowVisualizationImage(result);
}
}
#endregion
}
else if (mode == "density")
{
#region Density Estimation
CommandLineParser parser = new CommandLineParser();
parser.Command = "SW " + mode.ToUpper();
parser.AddArgument(trainingDataPath);
parser.AddSwitch("T", T);
parser.AddSwitch("D", D);
parser.AddSwitch("F", F);
parser.AddSwitch("L", L);
// For density estimation (and semi-supervised learning) we add
// a command line option to set the hyperparameters of the prior.
parser.AddSwitch("a", a);
parser.AddSwitch("b", b);
parser.AddSwitch("PADX", plotPaddingX);
parser.AddSwitch("PADY", plotPaddingY);
parser.AddSwitch("VERBOSE", verboseSwitch);
// Override default values for command line options.
T.Value = 1;
D.Value = 3;
F.Value = 5;
L.Value = 1;
a.Value = 0;
b.Value = 900;
if (args.Length == 1)
{
parser.PrintHelp();
DisplayTextFiles(DENSITY_DATA_PATH);
return;
}
if (parser.Parse(args, 1) == false)
{
return;
}
TrainingParameters parameters = new TrainingParameters()
{
MaxDecisionLevels = D.Value - 1,
NumberOfCandidateFeatures = F.Value,
NumberOfCandidateThresholdsPerFeature = L.Value,
NumberOfTrees = T.Value,
Verbose = verboseSwitch.Used
};
DataPointCollection trainingData = LoadTrainingData(
trainingDataPath.Value,
DENSITY_DATA_PATH,
2,
DataDescriptor.Unadorned);
Forest <AxisAlignedFeatureResponse, GaussianAggregator2d> forest = DensityEstimationExample.Train(trainingData, parameters, a.Value, b.Value);
PointF plotDilation = new PointF(plotPaddingX.Value, plotPaddingY.Value);
using (Bitmap result = DensityEstimationExample.Visualize(forest, trainingData, new Size(300, 300), plotDilation))
{
ShowVisualizationImage(result);
}
#endregion
}
else if (mode == "ssclas" || mode == "ssclas")
{
#region Semi-supervised classification
CommandLineParser parser = new CommandLineParser();
parser.Command = "SW " + mode.ToUpper();
parser.AddArgument(trainingDataPath);
parser.AddSwitch("T", T);
parser.AddSwitch("D", D);
parser.AddSwitch("F", F);
parser.AddSwitch("L", L);
parser.AddSwitch("split", split);
parser.AddSwitch("a", a);
parser.AddSwitch("b", b);
EnumParameter plotMode = new EnumParameter(
"plot",
"Determines what to plot",
new string[] { "density", "labels" },
new string[] { "plot recovered density estimate", "plot class likelihood" },
"labels");
parser.AddSwitch("plot", plotMode);
parser.AddSwitch("PADX", plotPaddingX);
parser.AddSwitch("PADY", plotPaddingY);
parser.AddSwitch("VERBOSE", verboseSwitch);
// Override default values for command line options.
T.Value = 10;
D.Value = 12 - 1;
F.Value = 30;
L.Value = 1;
if (args.Length == 1)
{
parser.PrintHelp();
DisplayTextFiles(SSCLAS_DATA_PATH);
return;
}
if (parser.Parse(args, 1) == false)
{
return;
}
DataPointCollection trainingData = LoadTrainingData(
trainingDataPath.Value,
SSCLAS_DATA_PATH,
2,
DataDescriptor.HasClassLabels);
TrainingParameters parameters = new TrainingParameters()
{
MaxDecisionLevels = D.Value - 1,
NumberOfCandidateFeatures = F.Value,
NumberOfCandidateThresholdsPerFeature = L.Value,
NumberOfTrees = T.Value,
Verbose = verboseSwitch.Used
};
Forest <LinearFeatureResponse2d, SemiSupervisedClassificationStatisticsAggregator> forest = SemiSupervisedClassificationExample.Train(
trainingData, parameters, a.Value, b.Value);
PointF plotPadding = new PointF(plotPaddingX.Value, plotPaddingY.Value);
if (plotMode.Value == "labels")
{
using (Bitmap result = SemiSupervisedClassificationExample.VisualizeLabels(forest, trainingData, new Size(300, 300), plotPadding))
{
ShowVisualizationImage(result);
}
}
else if (plotMode.Value == "density")
{
using (Bitmap result = SemiSupervisedClassificationExample.VisualizeDensity(forest, trainingData, new Size(300, 300), plotPadding))
{
ShowVisualizationImage(result);
}
}
#endregion
}
else if (mode == "regression")
{
#region Regression
CommandLineParser parser = new CommandLineParser();
parser.Command = "SW " + mode.ToUpper();
parser.AddArgument(trainingDataPath);
parser.AddSwitch("T", T);
parser.AddSwitch("D", D);
parser.AddSwitch("F", F);
parser.AddSwitch("L", L);
parser.AddSwitch("PADX", plotPaddingX);
parser.AddSwitch("PADY", plotPaddingY);
parser.AddSwitch("VERBOSE", verboseSwitch);
// Override default values for command line options
T.Value = 10;
D.Value = 2;
a.Value = 0; // prior turned off by default
b.Value = 900;
if (args.Length == 1)
{
parser.PrintHelp();
DisplayTextFiles(REGRESSION_DATA_PATH);
return;
}
if (parser.Parse(args, 1) == false)
{
return;
}
RegressionExample regressionDemo = new RegressionExample();
regressionDemo.PlotDilation.X = plotPaddingX.Value;
regressionDemo.PlotDilation.Y = plotPaddingY.Value;
regressionDemo.TrainingParameters = new TrainingParameters()
{
MaxDecisionLevels = D.Value - 1,
NumberOfCandidateFeatures = F.Value,
NumberOfCandidateThresholdsPerFeature = L.Value,
NumberOfTrees = T.Value,
Verbose = verboseSwitch.Used
};
DataPointCollection trainingData = LoadTrainingData(
trainingDataPath.Value,
REGRESSION_DATA_PATH,
1,
DataDescriptor.HasTargetValues);
using (Bitmap result = regressionDemo.Run(trainingData))
{
ShowVisualizationImage(result);
}
#endregion
}
else
{
Console.WriteLine("Unrecognized command line argument, try SW HELP.");
return;
}
}
public static Bitmap Visualize <F>(
Forest <F, HistogramAggregator> forest,
DataPointCollection trainingData,
Size PlotSize,
PointF PlotDilation) where F : IFeatureResponse
{
// Size PlotSize = new Size(300, 300), PointF PlotDilation = new PointF(0.1f, 0.1f)
// Generate some test samples in a grid pattern (a useful basis for creating visualization images)
PlotCanvas plotCanvas = new PlotCanvas(trainingData.GetRange(0), trainingData.GetRange(1), PlotSize, PlotDilation);
DataPointCollection testData = DataPointCollection.Generate2dGrid(plotCanvas.plotRangeX, PlotSize.Width, plotCanvas.plotRangeY, PlotSize.Height);
Console.WriteLine("\nApplying the forest to test data...");
int[][] leafNodeIndices = forest.Apply(testData);
// Form a palette of random colors, one per class
Color[] colors = new Color[Math.Max(trainingData.CountClasses(), 4)];
// First few colours are same as those in the book, remainder are random.
colors[0] = Color.FromArgb(183, 170, 8);
colors[1] = Color.FromArgb(194, 32, 14);
colors[2] = Color.FromArgb(4, 154, 10);
colors[3] = Color.FromArgb(13, 26, 188);
Color grey = Color.FromArgb(255, 127, 127, 127);
System.Random r = new Random(0); // same seed every time so colours will be consistent
for (int c = 4; c < colors.Length; c++)
{
colors[c] = Color.FromArgb(255, r.Next(0, 255), r.Next(0, 255), r.Next(0, 255));
}
// Create a visualization image
Bitmap result = new Bitmap(PlotSize.Width, PlotSize.Height);
// For each pixel...
int index = 0;
for (int j = 0; j < PlotSize.Height; j++)
{
for (int i = 0; i < PlotSize.Width; i++)
{
// Aggregate statistics for this sample over all leaf nodes reached
HistogramAggregator h = new HistogramAggregator(trainingData.CountClasses());
for (int t = 0; t < forest.TreeCount; t++)
{
int leafIndex = leafNodeIndices[t][index];
h.Aggregate(forest.GetTree(t).GetNode(leafIndex).TrainingDataStatistics);
}
// Let's muddy the colors with grey where the entropy is high.
float mudiness = 0.5f * (float)(h.Entropy());
float R = 0.0f, G = 0.0f, B = 0.0f;
for (int b = 0; b < trainingData.CountClasses(); b++)
{
float p = (1.0f - mudiness) * h.GetProbability(b); // NB probabilities sum to 1.0 over the classes
R += colors[b].R * p;
G += colors[b].G * p;
B += colors[b].B * p;
}
R += grey.R * mudiness;
G += grey.G * mudiness;
B += grey.B * mudiness;
Color c = Color.FromArgb(255, (byte)(R), (byte)(G), (byte)(B));
result.SetPixel(i, j, c); // painfully slow but safe
index++;
}
}
// Also draw the original training data
using (Graphics g = Graphics.FromImage(result))
{
g.InterpolationMode = System.Drawing.Drawing2D.InterpolationMode.HighQualityBicubic;
g.SmoothingMode = System.Drawing.Drawing2D.SmoothingMode.HighQuality;
for (int s = 0; s < trainingData.Count(); s++)
{
PointF x = new PointF(
(trainingData.GetDataPoint(s)[0] - plotCanvas.plotRangeX.Item1) / plotCanvas.stepX,
(trainingData.GetDataPoint(s)[1] - plotCanvas.plotRangeY.Item1) / plotCanvas.stepY);
RectangleF rectangle = new RectangleF(x.X - 3.0f, x.Y - 3.0f, 6.0f, 6.0f);
g.FillRectangle(new SolidBrush(colors[trainingData.GetIntegerLabel(s)]), rectangle);
g.DrawRectangle(new Pen(Color.Black), rectangle.X, rectangle.Y, rectangle.Width, rectangle.Height);
}
}
return(result);
}
public static Bitmap VisualizeLabels(Forest <LinearFeatureResponse2d, SemiSupervisedClassificationStatisticsAggregator> forest, DataPointCollection trainingData, Size PlotSize, PointF PlotDilation)
{
// Generate some test samples in a grid pattern (a useful basis for creating visualization images)
PlotCanvas plotCanvas = new PlotCanvas(trainingData.GetRange(0), trainingData.GetRange(1), PlotSize, PlotDilation);
// Apply the trained forest to the test data
Console.WriteLine("\nApplying the forest to test data...");
DataPointCollection testData = DataPointCollection.Generate2dGrid(plotCanvas.plotRangeX, PlotSize.Width, plotCanvas.plotRangeY, PlotSize.Height);
int[][] leafNodeIndices = forest.Apply(testData);
Bitmap result = new Bitmap(PlotSize.Width, PlotSize.Height);
// Paint the test data
GaussianPdf2d[][] leafDistributions = new GaussianPdf2d[forest.TreeCount][];
for (int t = 0; t < forest.TreeCount; t++)
{
leafDistributions[t] = new GaussianPdf2d[forest.GetTree(t).NodeCount];
for (int i = 0; i < forest.GetTree(t).NodeCount; i++)
{
Node <LinearFeatureResponse2d, SemiSupervisedClassificationStatisticsAggregator> nodeCopy = forest.GetTree(t).GetNode(i);
if (nodeCopy.IsLeaf)
{
leafDistributions[t][i] = nodeCopy.TrainingDataStatistics.GaussianAggregator2d.GetPdf();
}
}
}
// Form a palette of random colors, one per class
Color[] colors = new Color[Math.Max(trainingData.CountClasses(), 4)];
// First few colours are same as those in the book, remainder are random.
colors[0] = Color.FromArgb(183, 170, 8);
colors[1] = Color.FromArgb(194, 32, 14);
colors[2] = Color.FromArgb(4, 154, 10);
colors[3] = Color.FromArgb(13, 26, 188);
Color grey = Color.FromArgb(255, 127, 127, 127);
System.Random r = new Random(0); // same seed every time so colours will be consistent
for (int c = 4; c < colors.Length; c++)
{
colors[c] = Color.FromArgb(255, r.Next(0, 255), r.Next(0, 255), r.Next(0, 255));
}
int index = 0;
for (int j = 0; j < PlotSize.Height; j++)
{
for (int i = 0; i < PlotSize.Width; i++)
{
// Aggregate statistics for this sample over all leaf nodes reached
HistogramAggregator h = new HistogramAggregator(trainingData.CountClasses());
for (int t = 0; t < forest.TreeCount; t++)
{
int leafIndex = leafNodeIndices[t][index];
SemiSupervisedClassificationStatisticsAggregator a = forest.GetTree(t).GetNode(leafIndex).TrainingDataStatistics;
h.Aggregate(a.HistogramAggregator);
}
// Let's muddy the colors with a little grey where entropy is high.
float mudiness = 0.5f * (float)(h.Entropy());
float R = 0.0f, G = 0.0f, B = 0.0f;
for (int b = 0; b < trainingData.CountClasses(); b++)
{
float p = (1.0f - mudiness) * h.GetProbability(b); // NB probabilities sum to 1.0 over the classes
R += colors[b].R * p;
G += colors[b].G * p;
B += colors[b].B * p;
}
R += grey.R * mudiness;
G += grey.G * mudiness;
B += grey.B * mudiness;
Color c = Color.FromArgb(255, (byte)(R), (byte)(G), (byte)(B));
result.SetPixel(i, j, c);
index++;
}
}
PaintTrainingData(trainingData, plotCanvas, result);
return(result);
}
