public void RunTest3()
{
// Example XOR problem
double[][] inputs =
{
new double[] { 0, 0 }, // 0 xor 0: 1 (label +1)
new double[] { 0, 1 }, // 0 xor 1: 0 (label -1)
new double[] { 1, 0 }, // 1 xor 0: 0 (label -1)
new double[] { 1, 1 } // 1 xor 1: 1 (label +1)
};
// Dichotomy SVM outputs should be given as [-1;+1]
int[] labels =
{
1, -1, -1, 1
};
// Create a Kernel Support Vector Machine for the given inputs
KernelSupportVectorMachine svm = new KernelSupportVectorMachine(new Gaussian(0.1), inputs[0].Length);
// Instantiate a new learning algorithm for SVMs
SequentialMinimalOptimization smo = new SequentialMinimalOptimization(svm, inputs, labels);
// Set up the learning algorithm
smo.Complexity = 1.0;
// Run the learning algorithm
double error = smo.Run();
Assert.IsFalse(svm.IsProbabilistic);
// Instantiate the probabilistic learning calibration
var calibration = new ProbabilisticOutputCalibration(svm, inputs, labels);
// Run the calibration algorithm
double loglikelihood = calibration.Run();
Assert.IsTrue(svm.IsProbabilistic);
// Compute the decision output for one of the input vectors,
// while also retrieving the probability of the answer
double probability;
int decision = svm.Compute(inputs[0], out probability);
// At this point, decision is +1 with a probability of 75%
Assert.AreEqual(1, decision);
Assert.AreEqual(0.74999975815069375, probability, 1e-10);
}
private static void sparseMachineProbabilistic(Sparse <double>[] inputs, double[] doubleOutputs)
{
// The dataset has output labels as 4 and 2. We have to convert them
// into negative and positive labels so they can be properly processed.
//
bool[] outputs = doubleOutputs.Apply(x => x == 2.0 ? false : true);
// Create a learning algorithm for Sparse data. The first generic argument
// of the learning algorithm below is the chosen kernel function, and the
// second is the type of inputs the machine should accept. Note that, using
// those interfaces, it is possible to define custom kernel functions that
// operate directly on double[], string[], graphs, trees or any object:
var teacher = new LinearDualCoordinateDescent <Linear, Sparse <double> >()
{
Loss = Loss.L2,
Complexity = 1000, // Create a hard-margin SVM
Tolerance = 1e-5
};
// Use the learning algorithm to Learn
var svm = teacher.Learn(inputs, outputs);
// Create a probabilistic calibration algorithm based on Platt's method:
var calibration = new ProbabilisticOutputCalibration <Linear, Sparse <double> >()
{
Model = svm
};
// Let's say that instead of having our data as bool[], we would
// have received it as double[] containing the actual probabilities
// associated with each sample:
doubleOutputs.Apply(x => x == 2.0 ? 0.05 : 0.87, result: doubleOutputs);
// Calibrate the SVM using Platt's method
svm = calibration.Learn(inputs, doubleOutputs);
// Compute the machine's answers
bool[] answers = svm.Decide(inputs);
// Compute the machine's probabilities
double[] prob = svm.Probability(inputs);
// Create a confusion matrix to show the machine's performance
var m = new ConfusionMatrix(predicted: answers, expected: outputs);
// Show it onscreen
DataGridBox.Show(new ConfusionMatrixView(m)).Hold();
}
public void RunTest1()
{
double[][] inputs =
{
new double[] { -1, -1 },
new double[] { -1, 1 },
new double[] { 1, -1 },
new double[] { 1, 1 }
};
int[] outputs =
{
-1,
1,
1,
-1
};
KernelSupportVectorMachine svm = new KernelSupportVectorMachine(new Gaussian(3.6), 2);
var smo = new SequentialMinimalOptimization(svm, inputs, outputs);
double error1 = smo.Run();
Assert.AreEqual(0, error1);
double[] distances = new double[outputs.Length];
for (int i = 0; i < outputs.Length; i++)
{
int y = svm.Compute(inputs[i], out distances[i]);
Assert.AreEqual(outputs[i], y);
}
var target = new ProbabilisticOutputCalibration(svm, inputs, outputs);
double ll0 = target.LogLikelihood(inputs, outputs);
double ll1 = target.Run();
double ll2 = target.LogLikelihood(inputs, outputs);
Assert.AreEqual(5.5451735748694571, ll1);
Assert.AreEqual(ll1, ll2);
Assert.IsTrue(ll1 > ll0);
double[] newdistances = new double[outputs.Length];
for (int i = 0; i < outputs.Length; i++)
{
int y = svm.Compute(inputs[i], out newdistances[i]);
Assert.AreEqual(outputs[i], y);
}
double[] probs = new double[outputs.Length];
for (int i = 0; i < outputs.Length; i++)
{
int y;
probs[i] = svm.ToMulticlass().Probability(inputs[i], out y);
Assert.AreEqual(outputs[i], y == 1 ? 1 : -1);
}
Assert.AreEqual(0.25, probs[0], 1e-5);
Assert.AreEqual(0.75, probs[1], 1e-5);
Assert.AreEqual(0.75, probs[2], 1e-5);
Assert.AreEqual(0.25, probs[3], 1e-5);
foreach (var p in probs)
{
Assert.IsFalse(Double.IsNaN(p));
}
}
static void Main(string[] args)
{
if (args.Length != 1)
{
WriteLine("Usage:\n\n\ttrain.exe <folder path>");
return;
}
WriteLine("Platform: " + IntPtr.Size);
var stemmer = new SnowballStemmer();
using (var zip = new ZipOutputStream(
OpenWrite(
Combine(
GetDirectoryName(args[0]),
GetFileName(args[0]) + ".cl"))))
{
WriteLine("Reading dictionary...");
var dicPath = Combine(args[0], "dic.txt");
var dicText = File.Exists(dicPath) ? ReadAllLines(dicPath) : new string[0];
WriteLine("Building dictionary...");
var text = ReadAllLines(Combine(args[0], "text.txt"));
var dic = new Dictionary(dicText.Concat(text
.SelectMany(jd => jd.StemText(stemmer))
.ToLookup(w => w.Stemmed)
.Select(l => new { Stemmed = l.Key, Count = l.Count() })
.OrderByDescending(w => w.Count)
.Select(w => w.Stemmed))
.Take(5000)
.ToArray());
zip.PutNextEntry(new ZipEntry("dic.txt"));
using (var writer = new StreamWriter(zip, UTF8, 4096, true))
dic.WriteTo(writer);
zip.CloseEntry();
WriteLine("Vectorizing...");
var input = new double[text.Length][];
for (int i = 0; i < text.Length; i++)
{
input[i] = dic.Vectorize(new Document(text[i], stemmer));
}
var classifiers = from f in GetFiles(args[0])
where GetFileName(f) != "text.txt" && GetFileName(f) != "dic.txt"
where GetExtension(f) == ".txt"
let ll = ReadAllLines(f)
let ld = ll.Distinct().ToArray()
let li = ld
.Select((l, i) => new { l, i })
.ToDictionary(x => x.l, x => x.i)
select new
{
Name = GetFileNameWithoutExtension(f),
Labels = ld,
Output = ll.Select(l => li[l]).ToArray()
};
foreach (var classifier in classifiers)
{
Write($"Training: {classifier.Name}");
IKernel kernel = new Linear();
var machine = new MulticlassSupportVectorMachine(input[0].Length, kernel, classifier.Labels.Length);
var teacher = new MulticlassSupportVectorLearning(machine, input, classifier.Output);
teacher.Algorithm = (svm, classInputs, classOutputs, i, j) =>
{
//var sequentialMinimalOptimization = new SequentialMinimalOptimization(svm, classInputs, classOutputs);
//sequentialMinimalOptimization.Run();
var linearCoordinateDescent = new LinearCoordinateDescent(svm, classInputs, classOutputs);
linearCoordinateDescent.Run();
var probabilisticOutputLearning = new ProbabilisticOutputCalibration(svm, classInputs, classOutputs);
return(probabilisticOutputLearning);
};
WriteLine($", error = {teacher.Run()}");
zip.PutNextEntry(new ZipEntry(classifier.Name + ".svm"));
using (var stream = new MemoryStream())
{
machine.Save(stream);
stream.Position = 0;
stream.CopyTo(zip);
}
zip.CloseEntry();
zip.PutNextEntry(new ZipEntry(classifier.Name + ".txt"));
using (var writer = new StreamWriter(zip, UTF8, 4096, true))
foreach (var lable in classifier.Labels)
{
writer.WriteLine(lable);
}
zip.CloseEntry();
}
WriteLine("Done.");
}
}
public void Train(DataPackage data, CancellationToken token)
{
if (data is null)
{
throw new ArgumentNullException(nameof(data));
}
log.Debug("Training with {0} records", data.Y.Length);
standardizer = Standardizer.GetNumericStandardizer(data.X);
var xTraining = data.X;
var yTraining = data.Y;
var xTesting = xTraining;
var yTesting = yTraining;
int testSize = 100;
if (xTraining.Length > testSize * 4)
{
var training = xTraining.Length - testSize;
xTesting = xTraining.Skip(training).ToArray();
yTesting = yTraining.Skip(training).ToArray();
xTraining = xTraining.Take(training).ToArray();
yTraining = yTraining.Take(training).ToArray();
}
xTraining = standardizer.StandardizeAll(xTraining);
// Instantiate a new Grid Search algorithm for Kernel Support Vector Machines
var gridsearch = new GridSearch <SupportVectorMachine <Gaussian>, double[], int>()
{
// Here we can specify the range of the parameters to be included in the search
ParameterRanges = new GridSearchRangeCollection
{
new GridSearchRange("complexity", new [] { 0.001, 0.01, 0.1, 1, 10 }),
new GridSearchRange("gamma", new [] { 0.001, 0.01, 0.1, 1 })
},
// Indicate how learning algorithms for the models should be created
Learner = p => new SequentialMinimalOptimization <Gaussian>
{
Complexity = p["complexity"],
Kernel = new Gaussian
{
Gamma = p["gamma"]
}
},
// Define how the performance of the models should be measured
Loss = (actual, expected, m) => new ZeroOneLoss(expected).Loss(actual)
};
gridsearch.Token = token;
var randomized = new Random().Shuffle(xTraining, yTraining).ToArray();
yTraining = randomized[1].Cast <int>().ToArray();
xTraining = randomized[0].Cast <double[]>().ToArray();
var result = gridsearch.Learn(xTraining, yTraining);
// Get the best SVM found during the parameter search
SupportVectorMachine <Gaussian> svm = result.BestModel;
// Instantiate the probabilistic calibration (using Platt's scaling)
var calibration = new ProbabilisticOutputCalibration <Gaussian>(svm);
// Run the calibration algorithm
calibration.Learn(xTraining, yTraining); // returns the same machine
model = calibration.Model;
var predicted = ClassifyInternal(xTraining);
var confusionMatrix = new GeneralConfusionMatrix(classes: 2, expected: yTraining, predicted: predicted);
log.Debug("Performance on training dataset . F1(0):{0} F1(1):{1}", confusionMatrix.PerClassMatrices[0].FScore, confusionMatrix.PerClassMatrices[1].FScore);
predicted = Classify(xTesting);
confusionMatrix = new GeneralConfusionMatrix(classes: 2, expected: yTesting, predicted: predicted);
TestSetPerformance = confusionMatrix;
log.Debug("Performance on testing dataset . F1(0):{0} F1(1):{1}", confusionMatrix.PerClassMatrices[0].FScore, confusionMatrix.PerClassMatrices[1].FScore);
}
public void learn_test()
{
#region doc_learn
double[][] inputs = // Example XOR problem
{
new double[] { 0, 0 }, // 0 xor 0: 1 (label +1)
new double[] { 0, 1 }, // 0 xor 1: 0 (label -1)
new double[] { 1, 0 }, // 1 xor 0: 0 (label -1)
new double[] { 1, 1 } // 1 xor 1: 1 (label +1)
};
int[] outputs = // XOR outputs
{
1, 0, 0, 1
};
// Instantiate a new SMO learning algorithm for SVMs
var smo = new SequentialMinimalOptimization <Gaussian>()
{
Kernel = new Gaussian(0.1),
Complexity = 1.0
};
// Learn a SVM using the algorithm
var svm = smo.Learn(inputs, outputs);
// Predict labels for each input sample
bool[] predicted = svm.Decide(inputs);
// Compute classification error
double error = new ZeroOneLoss(outputs).Loss(predicted);
// Instantiate the probabilistic calibration (using Platt's scaling)
var calibration = new ProbabilisticOutputCalibration <Gaussian>(svm);
// Run the calibration algorithm
calibration.Learn(inputs, outputs); // returns the same machine
// Predict probabilities of each input sample
double[] probabilities = svm.Probability(inputs);
// Compute the error based on a hard decision
double loss = new BinaryCrossEntropyLoss(outputs).Loss(probabilities);
// Compute the decision output for one of the input vectors,
// while also retrieving the probability of the answer
bool decision;
double probability = svm.Probability(inputs[0], out decision);
#endregion
// At this point, decision is +1 with a probability of 75%
Assert.AreEqual(true, decision);
Assert.AreEqual(0, error);
Assert.AreEqual(5.5451735748925355, loss);
Assert.AreEqual(0.74999975815069375, probability, 1e-10);
Assert.IsTrue(svm.IsProbabilistic);
Assert.AreEqual(-1.0986109988055595, svm.Weights[0]);
Assert.AreEqual(1.0986109988055595, svm.Weights[1]);
Assert.AreEqual(-1.0986109988055595, svm.Weights[2]);
Assert.AreEqual(1.0986109988055595, svm.Weights[3]);
}
