private static void decisionTree(double[][] inputs, int[] outputs)
{
// In our problem, we have 2 classes (samples can be either
// positive or negative), and 2 continuous-valued inputs.
DecisionTree tree = new DecisionTree(inputs: new[]
{
DecisionVariable.Continuous("X"),
DecisionVariable.Continuous("Y")
}, classes: 2);
C45Learning teacher = new C45Learning(tree);
// The C4.5 algorithm expects the class labels to
// range from 0 to k, so we convert -1 to be zero:
//
outputs = outputs.Apply(x => x < 0 ? 0 : x);
double error = teacher.Run(inputs, outputs);
// Classify the samples using the model
int[] answers = inputs.Apply(tree.Compute);
// Plot the results
ScatterplotBox.Show("Expected results", inputs, outputs);
ScatterplotBox.Show("Decision Tree results", inputs, answers)
.Hold();
}
public static TrainingSet[] GenerateTrainingSets(IEnumerable<KeyValuePair<User, double[]>> studentsAndMarks, string[] normalRecords, string[] anomalies)
{
var countOfEntries = normalRecords.Length + anomalies.Length;
var inputData = new double[countOfEntries][];
var outputData = new int[countOfEntries];
var counter = 0;
foreach (var studentAndMarks in studentsAndMarks)
{
if (normalRecords.Contains(studentAndMarks.Key.OpenId))
{
inputData[counter] = studentAndMarks.Value;
outputData[counter++] = 1;
}
if (!anomalies.Contains(studentAndMarks.Key.OpenId))
{
continue;
}
inputData[counter] = studentAndMarks.Value;
outputData[counter++] = 0;
}
var countOfFeatures = studentsAndMarks.ElementAt(0).Value.Length;
var features = new DecisionVariable[countOfFeatures];
features[0] = new DecisionVariable("0", DecisionAttributeKind.Continuous, new AForge.DoubleRange(80, 1200));
for (var i = 1; i < countOfFeatures; i++)
{
features[i] = new DecisionVariable(i.ToString(), DecisionAttributeKind.Continuous, new AForge.DoubleRange(0, 10));
}
// Create the Decision tree with only 2 result values
var tree = new DecisionTree(features, 2);
// Creates a new instance of the C4.5 learning algorithm
var c45 = new C45Learning(tree);
// Learn the decision tree
var error = c45.Run(inputData, outputData);
// Split all data into normal and anomalies
var setOfNormalRecords = studentsAndMarks.Where(x => tree.Compute(x.Value) == 1);
var setOfAnomalies = studentsAndMarks.Where(x => tree.Compute(x.Value) == 0);
// Split normal records into 2 groups (one for training set and one for anomaly detection ocurency detection)
var setOfNormalRecordsList = setOfNormalRecords.ToList();
var splitCount = setOfNormalRecordsList.Count * 2 / 3;
var setOfNormalRecordsTr1 = setOfNormalRecordsList.GetRange(0, splitCount);
var setOfNormalRecordsTr2 = setOfNormalRecordsList.GetRange(splitCount, setOfNormalRecordsList.Count - splitCount);
// Create Training Sets
var trSetNormalFirst = CreateTrainingSetFromResources(setOfNormalRecordsTr1);
var trSetNormalSecond = CreateTrainingSetFromResources(setOfNormalRecordsTr2);
var trSetAnomalies = CreateTrainingSetFromResources(setOfAnomalies);
return new[] { trSetNormalFirst, trSetNormalSecond, trSetAnomalies };
}
/*************************** Primary Methods *******************************/
public double learnDecisionTreeModel(DataSet trainSet)
{
// Convert TrainSet --> TrainDataTable
this.convertToTrainIntputTable(trainSet);
// C4.5 Decision Tree Algorithm
double learningError;
C45Learning c45 = new C45Learning(this.descisionTree);
learningError = c45.Run(this.trainInputArray, this.trainOutputVector);
return learningError;
}
public static void CreateMitchellExample(out DecisionTree tree, out double[][] inputs, out int[] outputs)
{
DataTable data = new DataTable("Mitchell's Tennis Example");
data.Columns.Add("Day", typeof(string));
data.Columns.Add("Outlook", typeof(string));
data.Columns.Add("Temperature", typeof(double));
data.Columns.Add("Humidity", typeof(double));
data.Columns.Add("Wind", typeof(string));
data.Columns.Add("PlayTennis", typeof(string));
data.Rows.Add("D1", "Sunny", 85, 85, "Weak", "No");
data.Rows.Add("D2", "Sunny", 80, 90, "Strong", "No");
data.Rows.Add("D3", "Overcast", 83, 78, "Weak", "Yes");
data.Rows.Add("D4", "Rain", 70, 96, "Weak", "Yes");
data.Rows.Add("D5", "Rain", 68, 80, "Weak", "Yes");
data.Rows.Add("D6", "Rain", 65, 70, "Strong", "No");
data.Rows.Add("D7", "Overcast", 64, 65, "Strong", "Yes");
data.Rows.Add("D8", "Sunny", 72, 95, "Weak", "No");
data.Rows.Add("D9", "Sunny", 69, 70, "Weak", "Yes");
data.Rows.Add("D10", "Rain", 75, 80, "Weak", "Yes");
data.Rows.Add("D11", "Sunny", 75, 70, "Strong", "Yes");
data.Rows.Add("D12", "Overcast", 72, 90, "Strong", "Yes");
data.Rows.Add("D13", "Overcast", 81, 75, "Weak", "Yes");
data.Rows.Add("D14", "Rain", 71, 80, "Strong", "No");
// Create a new codification codebook to
// convert strings into integer symbols
Codification codebook = new Codification(data);
DecisionVariable[] attributes =
{
new DecisionVariable("Outlook", codebook["Outlook"].Symbols), // 3 possible values (Sunny, overcast, rain)
new DecisionVariable("Temperature", DecisionVariableKind.Continuous), // continuous values
new DecisionVariable("Humidity", DecisionVariableKind.Continuous), // continuous values
new DecisionVariable("Wind", codebook["Wind"].Symbols) // 2 possible values (Weak, strong)
};
int classCount = codebook["PlayTennis"].Symbols; // 2 possible values (yes, no)
tree = new DecisionTree(attributes, classCount);
C45Learning c45 = new C45Learning(tree);
// Extract symbols from data and train the classifier
DataTable symbols = codebook.Apply(data);
inputs = symbols.ToArray("Outlook", "Temperature", "Humidity", "Wind");
outputs = symbols.ToArray<int>("PlayTennis");
double error = c45.Run(inputs, outputs);
}
//public static C45Model CreateC45Model(Codification codification)
//{
// int lastIndex = codification.Columns.Count - 1;
// List<DecisionVariable> attributes = new List<DecisionVariable>();
// for (int indexColumn = 0; indexColumn < lastIndex; indexColumn++)
// {
// attributes.Add(new DecisionVariable(codification.Columns[indexColumn].ColumnName,
// codification[indexColumn].Symbols));
// }
// C45Model model = new C45Model(new DecisionTree(attributes.ToArray(), 2));
// return model;
//}
//public C45Model(DecisionTree tree)
//{
// this.Tree = tree;
//}
// Trainning decision tree with C4.5 algorithm
public override void TrainningModel(TrainningData trainningData)
{
// Get data for trainning tree
Codification codification = trainningData.CodificationData;
double[][] inputs = trainningData.TrainningAttributes;
int[] outputs = trainningData.ClassificationAttribute;
// Create tree
this.Tree = this.CreateDecisionTree(codification);
//var attributes = DecisionVariable.FromCodebook(codification, inputColumns);
//DecisionTree tree = new DecisionTree(attributes, outputClasses: 5);
// Creates a new instance of the C4.5 learning algorithm
C45Learning c45 = new C45Learning(this.Tree);
// Learn the decision tree
double error = c45.Run(inputs, outputs);
}
public void ConsistencyTest1()
{
double[,] random = Matrix.Random(1000, 10, 0.0, 1.0);
double[][] samples = random.ToJagged();
int[] outputs = new int[1000];
for (int i = 0; i < samples.Length; i++)
{
if (samples[i][0] > 0.8)
outputs[i] = 1;
}
DecisionVariable[] vars = new DecisionVariable[10];
for (int i = 0; i < vars.Length; i++)
vars[i] = new DecisionVariable(i.ToString(), DecisionVariableKind.Continuous);
DecisionTree tree = new DecisionTree(vars, 2);
C45Learning teacher = new C45Learning(tree);
double error = teacher.Run(samples, outputs);
Assert.AreEqual(0, error);
Assert.AreEqual(2, tree.Root.Branches.Count);
Assert.IsTrue(tree.Root.Branches[0].IsLeaf);
Assert.IsTrue(tree.Root.Branches[1].IsLeaf);
}
public void same_input_different_output()
{
double[][] inputs = new double[][] {
new double[] { 1 },
new double[] { 0 },
new double[] { 2 },
new double[] { 3 },
new double[] { 0 },
};
int[] outputs = new int[] {
11,
00,
22,
33,
01
};
DecisionVariable[] variables = { new DecisionVariable("x", DecisionVariableKind.Continuous) };
DecisionTree decisionTree = new DecisionTree(variables, 34);
C45Learning c45Learning = new C45Learning(decisionTree)
{
Join = 10,
MaxHeight = 10
};
c45Learning.Run(inputs, outputs); // System.AggregateException thrown here
int[] actual = decisionTree.Decide(inputs);
Assert.AreEqual(11, actual[0]);
Assert.AreEqual(00, actual[1]);
Assert.AreEqual(22, actual[2]);
Assert.AreEqual(33, actual[3]);
Assert.AreEqual(00, actual[4]);
}
public void AttributeReuseTest1()
{
string[][] text = Resources.iris_data.Split(
new[] { '\n' }, StringSplitOptions.RemoveEmptyEntries)
.Apply(x => x.Split(','));
double[][] inputs = new double[text.Length][];
for (int i = 0; i < inputs.Length; i++)
inputs[i] = text[i].First(4).Convert(s => Double.Parse(s, System.Globalization.CultureInfo.InvariantCulture));
string[] labels = text.GetColumn(4);
Codification codebook = new Codification("Label", labels);
int[] outputs = codebook.Translate("Label", labels);
DecisionVariable[] features =
{
new DecisionVariable("sepal length", DecisionVariableKind.Continuous),
new DecisionVariable("sepal width", DecisionVariableKind.Continuous),
new DecisionVariable("petal length", DecisionVariableKind.Continuous),
new DecisionVariable("petal width", DecisionVariableKind.Continuous),
};
DecisionTree tree = new DecisionTree(features, codebook.Columns[0].Symbols);
C45Learning teacher = new C45Learning(tree);
teacher.Join = 3;
double error = teacher.Run(inputs, outputs);
Assert.AreEqual(0.02, error, 1e-10);
DecisionSet rules = tree.ToRules();
double newError = ComputeError(rules, inputs, outputs);
Assert.AreEqual(0.02, newError, 1e-10);
string ruleText = rules.ToString(codebook,
System.Globalization.CultureInfo.InvariantCulture);
// TODO: implement this assertion properly, actually checking
// the text contents once the feature is completely finished.
Assert.AreEqual(600, ruleText.Length);
}
public void IrisDatasetTest()
{
#region doc_iris
// In this example, we will process the famous Fisher's Iris dataset in
// which the task is to classify weather the features of an Iris flower
// belongs to an Iris setosa, an Iris versicolor, or an Iris virginica:
//
// - https://en.wikipedia.org/wiki/Iris_flower_data_set
//
// First, let's load the dataset into an array of text that we can process
string[][] text = Resources.iris_data.Split(new[] { '\n' }, StringSplitOptions.RemoveEmptyEntries).Apply(x => x.Split(','));
// The first four columns contain the flower features
double[][] inputs = text.GetColumns(0, 1, 2, 3).To<double[][]>();
// The last column contains the expected flower type
string[] labels = text.GetColumn(4);
// Since the labels are represented as text, the first step is to convert
// those text labels into integer class labels, so we can process them
// more easily. For this, we will create a codebook to encode class labels:
//
var codebook = new Codification("Output", labels);
// With the codebook, we can convert the labels:
int[] outputs = codebook.Translate("Output", labels);
// Let's declare the names of our input variables:
DecisionVariable[] features =
{
new DecisionVariable("sepal length", DecisionVariableKind.Continuous),
new DecisionVariable("sepal width", DecisionVariableKind.Continuous),
new DecisionVariable("petal length", DecisionVariableKind.Continuous),
new DecisionVariable("petal width", DecisionVariableKind.Continuous),
};
// Now, we can finally create our tree for the 3 classes:
var tree = new DecisionTree(inputs: features, classes: 3);
// And we can use the C4.5 for learning:
var teacher = new C45Learning(tree);
// And finally induce the tree:
teacher.Learn(inputs, outputs);
// To get the estimated class labels, we can use
int[] predicted = tree.Decide(inputs);
// And the classification error can be computed as
double error = new ZeroOneLoss(outputs) // 0.0266
{
Mean = true
}.Loss(tree.Decide(inputs));
// Moreover, we may decide to convert our tree to a set of rules:
DecisionSet rules = tree.ToRules();
// And using the codebook, we can inspect the tree reasoning:
string ruleText = rules.ToString(codebook, "Output",
System.Globalization.CultureInfo.InvariantCulture);
// The output is:
string expected = @"Iris-setosa =: (petal length <= 2.45)
Iris-versicolor =: (petal length > 2.45) && (petal width <= 1.75) && (sepal length <= 7.05) && (sepal width <= 2.85)
Iris-versicolor =: (petal length > 2.45) && (petal width <= 1.75) && (sepal length <= 7.05) && (sepal width > 2.85)
Iris-versicolor =: (petal length > 2.45) && (petal width > 1.75) && (sepal length <= 5.95) && (sepal width > 3.05)
Iris-virginica =: (petal length > 2.45) && (petal width <= 1.75) && (sepal length > 7.05)
Iris-virginica =: (petal length > 2.45) && (petal width > 1.75) && (sepal length > 5.95)
Iris-virginica =: (petal length > 2.45) && (petal width > 1.75) && (sepal length <= 5.95) && (sepal width <= 3.05)
";
#endregion
Assert.AreEqual(0.026666666666666668, error, 1e-10);
Assert.AreEqual(4, tree.NumberOfInputs);
Assert.AreEqual(3, tree.NumberOfOutputs);
double newError = ComputeError(rules, inputs, outputs);
Assert.AreEqual(0.026666666666666668, newError, 1e-10);
Assert.AreEqual(expected, ruleText);
}
private string C45(DataTable tbl)
{
int classCount = 2;
Codification codebook = new Codification(tbl);
DecisionVariable[] attributes ={
new DecisionVariable("Clump Thickness",10),
new DecisionVariable("Uniformity of Cell Size",10),new DecisionVariable("Uniformity of Cell Shape",10),
new DecisionVariable("Marginal Adhesion",10),new DecisionVariable("Single Epithelial Cell Size",10),
new DecisionVariable("Bare Nuclei",10),new DecisionVariable("Bland Chromatin",10),
new DecisionVariable("Normal Nucleoli",10),new DecisionVariable("Mitoses",10),
};
DecisionTree tree = new DecisionTree(attributes, classCount);
// ID3Learning id3learning = new ID3Learning(tree);
// Translate our training data into integer symbols using our codebook:
DataTable symbols = codebook.Apply(tbl);
double[][] inputs = symbols.ToIntArray("Clump Thickness", "Uniformity of Cell Size", "Uniformity of Cell Shape", "Marginal Adhesion", "Single Epithelial Cell Size", "Bare Nuclei", "Bland Chromatin", "Normal Nucleoli", "Mitoses").ToDouble();
int[] outputs = symbols.ToIntArray("Class").GetColumn(0);
// symbols.
// id3learning.Run(inputs, outputs);
// Now, let's create the C4.5 algorithm
C45Learning c45 = new C45Learning(tree);
// and learn a decision tree. The value of
// the error variable below should be 0.
//
double error = c45.Run(inputs, outputs);
// To compute a decision for one of the input points,
// such as the 25-th example in the set, we can use
//
int y = tree.Compute(inputs[5]);
// Finally, we can also convert our tree to a native
// function, improving efficiency considerably, with
//
Func<double[], int> func = tree.ToExpression().Compile();
// Again, to compute a new decision, we can just use
//
int z = func(inputs[5]);
int[] query = codebook.Translate(inputlar[0], inputlar[1], inputlar[2], inputlar[3],
inputlar[4], inputlar[5], inputlar[6], inputlar[7], inputlar[8]);
int output = tree.Compute(query);
string answer = codebook.Translate("Class", output);
return answer;
// throw new NotImplementedException();
}
public DecisionTree MakeDecisionTree()
{
DecisionTree tree = new DecisionTree(MakeAttributes(), 2);
C45Learning c45 = new C45Learning(tree);
c45.Run(trainingSet.Inputs, trainingSet.Outputs);
return tree;
}
public void Run(String filename)
{
ReadFile(filename);
// Now, we have to convert the textual, categorical data found
// in the table to a more manageable discrete representation.
//
// For this, we will create a codebook to translate text to
// discrete integer symbols:
//
Codification codebook = new Codification(data);
// And then convert all data into symbols
//
DataTable symbols = codebook.Apply(data);
for (int i = 0; i < inputColumns.Count; i++)
if (inputTypes[i] == "string")
CreateDic(inputColumns[i], symbols);
CreateDic(outputColumn, symbols);
double[][] inputs = (from p in symbols.AsEnumerable()
select GetInputRow(p)
).Cast<double[]>().ToArray();
int[] outputs = (from p in symbols.AsEnumerable()
select GetIndex(outputColumn, p[outputColumn].ToString())).Cast<int>().ToArray();
// From now on, we can start creating the decision tree.
//
var attributes = DecisionVariable.FromCodebook(codebook, inputColumns.ToArray());
DecisionTree tree = new DecisionTree(attributes, 5); //outputClasses: 5
// Now, let's create the C4.5 algorithm
C45Learning c45 = new C45Learning(tree);
// and learn a decision tree. The value of
// the error variable below should be 0.
//
double error = c45.Run(inputs, outputs);
// To compute a decision for one of the input points,
// such as the 25-th example in the set, we can use
//
//int y = tree.Compute(inputs[25]);
// Finally, we can also convert our tree to a native
// function, improving efficiency considerably, with
//
//Func<double[], int> func = tree.ToExpression().Compile();
// Again, to compute a new decision, we can just use
//
//int z = func(inputs[25]);
var expression = tree.ToExpression();
Console.WriteLine(tree.ToCode("ClassTest"));
DecisionSet s = tree.ToRules();
Console.WriteLine(s.ToString());
}
private double run(double[][] inputs, int[] output)
{
int rows = inputs.Length;
int cols = inputs[0].Length;
int classes = output.DistinctCount();
int colsPerTree;
if (CoverageRatio == 0)
{
colsPerTree = (int)(System.Math.Sqrt(cols));
}
else
{
colsPerTree = (int)(cols * CoverageRatio);
}
var trees = forest.Trees;
int[] idx = Classes.Random(output, classes, trees.Length);
Parallel.For(0, trees.Length, i =>
{
var x = inputs.Get(idx);
var y = output.Get(idx);
var c45 = new C45Learning(forest.Trees[i])
{
MaxVariables = colsPerTree,
Join = 100
};
c45.Learn(x, y);
});
return 0;
}
public void LargeSampleTest2()
{
Accord.Math.Tools.SetupGenerator(0);
double[][] dataSamples = Matrix.Random(500, 3, 0.0, 10.0).ToJagged();
int[] target = Matrix.Random(500, 1, 0.0, 2.0).ToInt32().GetColumn(0);
DecisionVariable[] features =
{
new DecisionVariable("Outlook", DecisionVariableKind.Continuous),
new DecisionVariable("Temperature", DecisionVariableKind.Continuous),
new DecisionVariable("Humidity", DecisionVariableKind.Continuous),
};
DecisionTree tree = new DecisionTree(features, 2);
C45Learning teacher = new C45Learning(tree);
double error = teacher.Run(dataSamples, target);
foreach (var node in tree)
{
if (node.IsLeaf)
Assert.IsNotNull(node.Output);
}
Assert.IsTrue(error < 0.50);
}
/// <summary>
/// Создание дерева
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void btnCreate_Click(object sender, EventArgs e)
{
try
{
if (dgvLearningSource.DataSource == null)
{
MessageBox.Show("Загрузите данные");
return;
}
if (Tree_property.Property_View == true)
{
Tree_property.ShowDialog();
}
// Завершаем операцию с DataGridView
dgvLearningSource.EndEdit();
#region Алгоритм С4.5
///
///Алгоритм С4.5
///
if (Tree_property.Alg == "C4.5")
{
// // создаем матрицу из data table
double[,] sourceMatrix = (dgvLearningSource.DataSource as DataTable).ToMatrix(out sourceColumns);
C45Learning c45;
// получаем входные значения
double[][] inputs = sourceMatrix.Submatrix(null, 0, Tree_property.Coun_In - 1).ToArray();
// получаем выходные значения
int[] outputs = sourceMatrix.GetColumn(Tree_property.Coun_Out - 1).ToInt32();
DecisionVariable[] attributes = new DecisionVariable[Tree_property.Coun_In];
for (int j = 0; j < Tree_property.Coun_In; j++)
{
attributes[j] = new DecisionVariable(dgvLearningSource.Columns[j].Name, DecisionAttributeKind.Continuous);
}
// создаем дерево решений
tree = new DecisionTree(attributes, 60);
c45 = new C45Learning(tree);
double error = c45.Run(inputs, outputs);
}
#endregion
#region Алгоритм ID3
///
///Алгоритм ID3
///
if (Tree_property.Alg == "ID3")
{
// создаем матрицу из дататыйбл
int[][] arr = (dgvLearningSource.DataSource as DataTable).ToIntArray(sourceColumns);
int[,] sourceMatrix = arr.ToMatrix();
//// получаем входные значения
int[][] inputs = sourceMatrix.Submatrix(null, 0, Tree_property.Coun_In - 1).ToArray();
//// получаем выходные значения
int[] outputs = sourceMatrix.GetColumn(Tree_property.Coun_In - 1);
DecisionVariable[] attributes = new DecisionVariable[Tree_property.Coun_In];
for (int j = 0; j < Tree_property.Coun_In; j++)
{
attributes[j] = new DecisionVariable(j.ToString(), DecisionAttributeKind.Continuous);
}
// создаем дерево решений
tree = new DecisionTree(attributes, 60);
ID3Learning id3learning = new ID3Learning(tree);
double error = id3learning.Run(inputs, outputs);
}
#endregion
asd.Dispose();
asd.Close();
Drawing dr = new Drawing();
dr.recursion(tree.Root, tree.Root.Branches, 0);
dr.Save_();
asd = new Tree_View();
asd.userControl11.Load_f(Application.StartupPath);
System.Linq.Expressions.Expression df = tree.ToExpression();
// выбираем tabe page для просмотра дерева
tabControl.SelectTab(tabOverview);
// отображаем построенной дереыыо решений
decisionTreeView1.TreeSource = tree;
try
{
File.Copy(@".\Resources\recursion.png", @".\Resources\recursion2.png", true);
}
catch
{
}
using (Stream s = File.OpenRead(@".\Resources\recursion2.png"))
{
pictureBox1.Image = Image.FromStream(s);
}
}
catch (Exception t)
{
MessageBox.Show(t.Message);
}
}
public void ArgumentCheck1()
{
double[][] samples =
{
new [] { 0, 2, 4.0 },
new [] { 1, 5, 2.0 },
null,
new [] { 1, 5, 6.0 },
};
int[] outputs =
{
1, 1, 0, 0
};
DecisionVariable[] vars = new DecisionVariable[3];
for (int i = 0; i < vars.Length; i++)
vars[i] = DecisionVariable.Continuous(i.ToString());
DecisionTree tree = new DecisionTree(vars, 2);
var teacher = new C45Learning(tree);
bool thrown = false;
try { double error = teacher.Run(samples, outputs); }
catch (ArgumentNullException) { thrown = true; }
Assert.IsTrue(thrown);
}
private void btnCreate_Click(object sender, EventArgs e)
{
if (dgvLearningSource.DataSource == null)
{
MessageBox.Show("Please load some data first.");
return;
}
// Finishes and save any pending changes to the given data
dgvLearningSource.EndEdit();
// Creates a matrix from the source data table
double[,] sourceMatrix = (dgvLearningSource.DataSource as DataTable).ToMatrix(out sourceColumns);
// Perform classification
C45Learning c45;
// Get only the input vector values
double[][] inputs = sourceMatrix.Submatrix(null, 0, 1).ToArray();
// Get only the label outputs
int[] outputs = sourceMatrix.GetColumn(2).ToInt32();
DecisionVariable[] attributes =
{
new DecisionVariable("x", DecisionVariableKind.Continuous),
new DecisionVariable("y", DecisionVariableKind.Continuous),
};
// Create the Decision tree
tree = new DecisionTree(attributes, 2);
// Creates a new instance of the C4.5 learning algorithm
c45 = new C45Learning(tree);
// Learn the decision tree
double error = c45.Run(inputs, outputs);
// Show the learned tree in the view
decisionTreeView1.TreeSource = tree;
// Draw the separating surface
var ranges = Matrix.Range(sourceMatrix, 0);
double[][] map = Matrix.CartesianProduct(
Matrix.Interval(ranges[0], 0.05),
Matrix.Interval(ranges[1], 0.05));
var result = map.Apply(tree.Compute).Apply(Math.Sign);
var graph = map.ToMatrix().InsertColumn(result.ToDouble());
CreateScatterplot(zedGraphControl2, graph);
}
public void new_method_create_tree()
{
string[][] text = Resources.iris_data.Split(new[] { '\n' }, StringSplitOptions.RemoveEmptyEntries).Apply(x => x.Split(','));
double[][] inputs = text.GetColumns(0, 1, 2, 3).To<double[][]>();
string[] labels = text.GetColumn(4);
var codebook = new Codification("Output", labels);
int[] outputs = codebook.Translate("Output", labels);
// And we can use the C4.5 for learning:
var teacher = new C45Learning();
// And finally induce the tree:
var tree = teacher.Learn(inputs, outputs);
// To get the estimated class labels, we can use
int[] predicted = tree.Decide(inputs);
// And the classification error can be computed as
double error = new ZeroOneLoss(outputs) // 0.0266
{
Mean = true
}.Loss(tree.Decide(inputs));
// Moreover, we may decide to convert our tree to a set of rules:
DecisionSet rules = tree.ToRules();
// And using the codebook, we can inspect the tree reasoning:
string ruleText = rules.ToString(codebook, "Output",
System.Globalization.CultureInfo.InvariantCulture);
// The output is:
string expected = @"Iris-setosa =: (2 <= 2.45)
Iris-versicolor =: (2 > 2.45) && (3 <= 1.75) && (0 <= 7.05) && (1 <= 2.85)
Iris-versicolor =: (2 > 2.45) && (3 <= 1.75) && (0 <= 7.05) && (1 > 2.85)
Iris-versicolor =: (2 > 2.45) && (3 > 1.75) && (0 <= 5.95) && (1 > 3.05)
Iris-virginica =: (2 > 2.45) && (3 <= 1.75) && (0 > 7.05)
Iris-virginica =: (2 > 2.45) && (3 > 1.75) && (0 > 5.95)
Iris-virginica =: (2 > 2.45) && (3 > 1.75) && (0 <= 5.95) && (1 <= 3.05)
";
Assert.AreEqual(0.026666666666666668, error, 1e-10);
double newError = ComputeError(rules, inputs, outputs);
Assert.AreEqual(0.026666666666666668, newError, 1e-10);
Assert.AreEqual(expected, ruleText);
}
// Returns Error as a percentage. Best is 0.0, Worst is 1.0
public double TreeLearn()
{
c45 = new C45Learning(tree);
return c45.Run(Inputs, Outputs);
}
public void same_input_different_output_minimal()
{
double[][] inputs = new double[][] {
new double[] { 0 },
new double[] { 0 }
};
int[] outputs = new int[] {
1,
0
};
DecisionVariable[] variables = { new DecisionVariable("x", DecisionVariableKind.Continuous) };
DecisionTree decisionTree = new DecisionTree(variables, 2);
C45Learning c45Learning = new C45Learning(decisionTree);
c45Learning.Run(inputs, outputs); // System.AggregateException thrown here
Assert.AreEqual(decisionTree.Decide(new[] { 0 }), 0);
}
public void LargeRunTest()
{
#region doc_nursery
// This example uses the Nursery Database available from the University of
// California Irvine repository of machine learning databases, available at
//
// http://archive.ics.uci.edu/ml/machine-learning-databases/nursery/nursery.names
//
// The description paragraph is listed as follows.
//
// Nursery Database was derived from a hierarchical decision model
// originally developed to rank applications for nursery schools. It
// was used during several years in 1980's when there was excessive
// enrollment to these schools in Ljubljana, Slovenia, and the
// rejected applications frequently needed an objective
// explanation. The final decision depended on three subproblems:
// occupation of parents and child's nursery, family structure and
// financial standing, and social and health picture of the family.
// The model was developed within expert system shell for decision
// making DEX (M. Bohanec, V. Rajkovic: Expert system for decision
// making. Sistemica 1(1), pp. 145-157, 1990.).
//
// Let's begin by loading the raw data. This string variable contains
// the contents of the nursery.data file as a single, continuous text.
//
string nurseryData = Resources.nursery;
// Those are the input columns available in the data
//
string[] inputColumns =
{
"parents", "has_nurs", "form", "children",
"housing", "finance", "social", "health"
};
// And this is the output, the last column of the data.
//
string outputColumn = "output";
// Let's populate a data table with this information.
//
DataTable table = new DataTable("Nursery");
table.Columns.Add(inputColumns);
table.Columns.Add(outputColumn);
string[] lines = nurseryData.Split(
new[] { Environment.NewLine }, StringSplitOptions.None);
foreach (var line in lines)
table.Rows.Add(line.Split(','));
// Now, we have to convert the textual, categorical data found
// in the table to a more manageable discrete representation.
//
// For this, we will create a codebook to translate text to
// discrete integer symbols:
//
Codification codebook = new Codification(table);
// And then convert all data into symbols
//
DataTable symbols = codebook.Apply(table);
double[][] inputs = symbols.ToArray(inputColumns);
int[] outputs = symbols.ToArray<int>(outputColumn);
// From now on, we can start creating the decision tree.
//
var attributes = DecisionVariable.FromCodebook(codebook, inputColumns);
DecisionTree tree = new DecisionTree(attributes, classes: 5);
// Now, let's create the C4.5 algorithm
C45Learning c45 = new C45Learning(tree);
// and learn a decision tree. The value of
// the error variable below should be 0.
//
double error = c45.Run(inputs, outputs);
// To compute a decision for one of the input points,
// such as the 25-th example in the set, we can use
//
int y = tree.Compute(inputs[25]);
#endregion
Assert.AreEqual(0, error);
for (int i = 0; i < inputs.Length; i++)
{
int expected = outputs[i];
int actual = tree.Compute(inputs[i]);
Assert.AreEqual(expected, actual);
}
#if !NET35
// Finally, we can also convert our tree to a native
// function, improving efficiency considerably, with
//
Func<double[], int> func = tree.ToExpression().Compile();
// Again, to compute a new decision, we can just use
//
int z = func(inputs[25]);
for (int i = 0; i < inputs.Length; i++)
{
int expected = outputs[i];
int actual = func(inputs[i]);
Assert.AreEqual(expected, actual);
}
#endif
}
public static DecisionTree createNurseryExample(out double[][] inputs, out int[] outputs, int first)
{
string nurseryData = Resources.nursery;
string[] inputColumns =
{
"parents", "has_nurs", "form", "children",
"housing", "finance", "social", "health"
};
string outputColumn = "output";
DataTable table = new DataTable("Nursery");
table.Columns.Add(inputColumns);
table.Columns.Add(outputColumn);
string[] lines = nurseryData.Split(
new[] { Environment.NewLine }, StringSplitOptions.None);
foreach (var line in lines)
table.Rows.Add(line.Split(','));
Codification codebook = new Codification(table);
DataTable symbols = codebook.Apply(table);
inputs = symbols.ToArray(inputColumns);
outputs = symbols.ToArray<int>(outputColumn);
var attributes = DecisionVariable.FromCodebook(codebook, inputColumns);
var tree = new DecisionTree(attributes, outputClasses: 5);
C45Learning c45 = new C45Learning(tree);
double error = c45.Run(inputs.Submatrix(first), outputs.Submatrix(first));
Assert.AreEqual(0, error);
return tree;
}
public void ConstantDiscreteVariableTest()
{
DecisionTree tree;
double[][] inputs;
int[] outputs;
DataTable data = new DataTable("Degenerated Tennis Example");
data.Columns.Add("Day", typeof(string));
data.Columns.Add("Outlook", typeof(string));
data.Columns.Add("Temperature", typeof(double));
data.Columns.Add("Humidity", typeof(double));
data.Columns.Add("Wind", typeof(string));
data.Columns.Add("PlayTennis", typeof(string));
data.Rows.Add("D1", "Sunny", 50, 85, "Weak", "No");
data.Rows.Add("D2", "Sunny", 50, 90, "Weak", "No");
data.Rows.Add("D3", "Overcast", 83, 78, "Weak", "Yes");
data.Rows.Add("D4", "Rain", 70, 96, "Weak", "Yes");
data.Rows.Add("D5", "Rain", 68, 80, "Weak", "Yes");
data.Rows.Add("D6", "Rain", 65, 70, "Weak", "No");
data.Rows.Add("D7", "Overcast", 64, 65, "Weak", "Yes");
data.Rows.Add("D8", "Sunny", 50, 95, "Weak", "No");
data.Rows.Add("D9", "Sunny", 69, 70, "Weak", "Yes");
data.Rows.Add("D10", "Rain", 75, 80, "Weak", "Yes");
data.Rows.Add("D11", "Sunny", 75, 70, "Weak", "Yes");
data.Rows.Add("D12", "Overcast", 72, 90, "Weak", "Yes");
data.Rows.Add("D13", "Overcast", 81, 75, "Weak", "Yes");
data.Rows.Add("D14", "Rain", 50, 80, "Weak", "No");
// Create a new codification codebook to
// convert strings into integer symbols
Codification codebook = new Codification(data);
DecisionVariable[] attributes =
{
new DecisionVariable("Outlook", codebook["Outlook"].Symbols), // 3 possible values (Sunny, overcast, rain)
new DecisionVariable("Temperature", DecisionVariableKind.Continuous), // continuous values
new DecisionVariable("Humidity", DecisionVariableKind.Continuous), // continuous values
new DecisionVariable("Wind", codebook["Wind"].Symbols + 1) // 1 possible value (Weak)
};
int classCount = codebook["PlayTennis"].Symbols; // 2 possible values (yes, no)
tree = new DecisionTree(attributes, classCount);
C45Learning c45 = new C45Learning(tree);
// Extract symbols from data and train the classifier
DataTable symbols = codebook.Apply(data);
inputs = symbols.ToArray("Outlook", "Temperature", "Humidity", "Wind");
outputs = symbols.ToArray<int>("PlayTennis");
double error = c45.Run(inputs, outputs);
for (int i = 0; i < inputs.Length; i++)
{
int y = tree.Compute(inputs[i]);
Assert.AreEqual(outputs[i], y);
}
}
/// <summary>
/// Creates and learns a Decision Tree to recognize the
/// previously loaded dataset using the current settings.
/// </summary>
///
private void btnCreate_Click(object sender, EventArgs e)
{
if (dgvLearningSource.DataSource == null)
{
MessageBox.Show("Please load some data first.");
return;
}
// Finishes and save any pending changes to the given data
dgvLearningSource.EndEdit();
// Creates a matrix from the entire source data table
double[,] table = (dgvLearningSource.DataSource as DataTable).ToMatrix(out columnNames);
// Get only the input vector values (first two columns)
double[][] inputs = table.GetColumns(0, 1).ToArray();
// Get only the output labels (last column)
int[] outputs = table.GetColumn(2).ToInt32();
// Specify the input variables
DecisionVariable[] variables =
{
new DecisionVariable("x", DecisionVariableKind.Continuous),
new DecisionVariable("y", DecisionVariableKind.Continuous),
};
// Create the discrete Decision tree
tree = new DecisionTree(variables, 2);
// Create the C4.5 learning algorithm
C45Learning c45 = new C45Learning(tree);
// Learn the decision tree using C4.5
double error = c45.Run(inputs, outputs);
// Show the learned tree in the view
decisionTreeView1.TreeSource = tree;
// Get the ranges for each variable (X and Y)
DoubleRange[] ranges = Matrix.Range(table, 0);
// Generate a Cartesian coordinate system
double[][] map = Matrix.CartesianProduct(
Matrix.Interval(ranges[0], 0.05),
Matrix.Interval(ranges[1], 0.05));
// Classify each point in the Cartesian coordinate system
double[] result = map.Apply(tree.Compute).ToDouble();
double[,] surface = map.ToMatrix().InsertColumn(result);
CreateScatterplot(zedGraphControl2, surface);
lbStatus.Text = "Learning finished! Click the other tabs to explore results!";
}
/// <summary>
/// The main entry point for the program
/// </summary>
public static void Main()
{
try
{
#region Exploratory Data Analysis Explanation
/*
John Tukey coined the term Exploratory Data Analysis in his seminal book of the same name. There really is not a prescribed way to do an EDA.
Tools I use for EDA include Microsoft Excel, plots and visual inspection of the data. Without creating an early bias, gut feelings do play a role in a good EDA.
Some objectives of EDA are to:
• Identify the types of data in the dataset
• Examine the statistical properties of the data
• Look for invalid data (may need Domain or Subject Matter experts)
• Understand the provenance of the data
• Aide in the selection of appropriate statistical tools and techniques
For our diabetes dataset, notice that there is both quantitative and qualitative data. Note that the result or outcome variable (which indicates if the person has
diabetes) is nominal data with only two states. This is called dichotomous or binary categorical data which rules out some machine learning algorithms and directs
us to others.
*/
#endregion
// Because of time constraints, the loading of the DataTables and EDA is complete.
XmlConfigurator.Configure();
Logger.Info("Exploratory Data Analysis");
FileInfo fi = new FileInfo("training.csv");
DataTable training = DataTableCsvConvertor.GetDataTableFromCsv(fi);
fi = new FileInfo("test.csv");
DataTable test = DataTableCsvConvertor.GetDataTableFromCsv(fi);
// Print out the first few table rows.
Head.PrintHead(training);
//Logger.Info(string.Empty);
//BasicStatistics.BasicStats(training); // For most EDA's Basic Descriptive statistics are important, but this outputs a lot of information
#region Data Imputation & Cleanup Explanation
/*
Keep in mind that Machine Learning algorithms operate on numerical data only, something will have to be done with the data is text or NULL. Also predictor
variables(aka features or columns of data) that do not vary will not be predictive and may need to be removed. Due to time constraints the EDA, ETL (Extract, Transform and Load)
and data cleaning is already completed in the solution. For this analysis, the HeartRate column because it is all NULL and remove any rows of data that contain NULLs.
*/
#endregion
// Delete any columns that are not needed.
training.Columns.Remove("HeartRate");
test.Columns.Remove("HeartRate");
// How to handle rows containing missing or NA data - data imputation or deletion?
training = DataImputation.RemoveMissing(training);
test = DataImputation.RemoveMissing(test);
Codification codebook = new Codification(training);
int outputClasses = 2;
string[] inputColumns =
{
"Gender", "YearOfBirth", "SmokingEffectiveYear", "NISTcode", "Height", "Weight", "BMI", "SystolicBP", "DiastolicBP", "RespiratoryRate", "Temperature"
};
string outputColumn = "DMIndicator";
// Translate our training data into integer symbols using our codebook:
DataTable symbols = codebook.Apply(training);
double[][] inputs = symbols.ToArray(inputColumns);
int[] outputs = Matrix.ToArray<int>(training, outputColumn);
#region Decision Tree Overview
/*
Decision Trees are very powerful, especially with a binary classification model, and are somewhat resistant to over-fitting the data.
Additionally, they are intuitive to explain to stakeholders.
*/
#endregion
Logger.Info(string.Empty);
Logger.Info("Decision Tree");
DecisionVariable[] attributes =
{
new DecisionVariable("Gender", 2), // 2 possible values (Male, Female)
new DecisionVariable("YearOfBirth", DecisionVariableKind.Continuous),
new DecisionVariable("SmokingEffectiveYear", DecisionVariableKind.Continuous),
new DecisionVariable("NISTcode", DecisionVariableKind.Continuous),
new DecisionVariable("Height", DecisionVariableKind.Continuous),
new DecisionVariable("Weight", DecisionVariableKind.Continuous),
new DecisionVariable("BMI", DecisionVariableKind.Continuous),
new DecisionVariable("SystolicBP", DecisionVariableKind.Continuous),
new DecisionVariable("DiastolicBP", DecisionVariableKind.Continuous),
new DecisionVariable("RespiratoryRate", DecisionVariableKind.Continuous),
new DecisionVariable("Temperature", DecisionVariableKind.Continuous)
};
DecisionTree tree = new DecisionTree(attributes, outputClasses);
C45Learning c45learning = new C45Learning(tree);
// Learn the training instances!
c45learning.Run(inputs, outputs);
// The next two lines are optional to save the model into IL for future use.
// Convert to an expression tree
var expression = tree.ToExpression();
// Compiles the expression to IL
var func = expression.Compile();
#region Evaluation Explanation
/*
To evaluate the model, now use each row of the test dataset to predict the output variable (DMIndicator) using the DecisionTree’s compute method passing in the same
variables that were used to train the model. Store the test dataset’s value of DMIndicator and the predicted value in a DataTable and integer collection for future
validation of the model.
*/
#endregion
Evaluator.Evaluate(test, tree);
#region Validation Explanation
/*
There are many ways to validate models, but we will use a confusion matrix because it is intuitive and a very accepted way to validate binary classification models.
Most conveniently the Accord.Net has a ConfusionMatrix class to create this matrix for you. Passing in the collection of integers of predicted and actual values
stored earlier to the ConfusionMatrix class and output the matrix and accuracy.
*/
#endregion
Validator.Validate(test, tree);
#region Support Vector Machine Overview
/*
Support Vector Machines are powerful classification machine learning algorithms with very few knobs to turn. The kernel of the SVM can be exchanged to use
a number of different mathematical algorithms including polynomials, neural networks and Gaussian functions.
*/
#endregion
Logger.Info(string.Empty);
Logger.Info("Support Vector Machine");
// Add SVM code here
IKernel kernel = new Linear();
// Create the Multi-class Support Vector Machine using the selected Kernel
int inputDimension = inputs[0].Length;
var ksvm = new MulticlassSupportVectorMachine(inputDimension, kernel, outputClasses);
// Create the learning algorithm using the machine and the training data
var ml = new MulticlassSupportVectorLearning(ksvm, inputs, outputs)
{
Algorithm = (svm, classInputs, classOutputs, i, j) =>
{
return new SequentialMinimalOptimization(svm, classInputs, classOutputs)
{
CacheSize = 0
};
}
};
double svmError = ml.Run();
#region Evaluation Explanation
/*
To evaluate the model, now use each row of the test dataset to predict the output variable (DMIndicator) using the DecisionTree’s compute method passing in the same
variables that were used to train the model. Store the test dataset’s value of DMIndicator and the predicted value in a DataTable and integer collection for future
validation of the model.
*/
#endregion
Evaluator.Evaluate(test, ksvm);
#region Validation Explanation
/*
There are many ways to validate models, but we will use a confusion matrix because it is intuitive and a very accepted way to validate binary classification models.
Most conveniently the Accord.Net has a ConfusionMatrix class to create this matrix for you. Passing in the collection of integers of predicted and actual values
stored earlier to the ConfusionMatrix class and output the matrix and accuracy.
*/
#endregion
Validator.Validate(test, ksvm);
}
catch (Exception ex)
{
Logger.Error(ex.ToString());
}
}
