/// <summary>
/// Samples weights from the learned weight distribution of the Bayes point machine classifier.
/// </summary>
/// <typeparam name="TTrainingSettings">The type of the settings for training.</typeparam>
/// <param name="classifier">The Bayes point machine used to sample weights from.</param>
/// <returns>The samples from the weight distribution of the Bayes point machine classifier.</returns>
/// <remarks>Adapted from MicrosoftResearch.Infer.Learners</remarks>
private IEnumerable <Vector> SampleWeights(
IBayesPointMachineClassifier <
IList <Vector>,
int,
IList <string>,
string,
IDictionary <string, double>,
BayesPointMachineClassifierTrainingSettings,
BinaryBayesPointMachineClassifierPredictionSettings <string> > classifier)
{
Debug.Assert(classifier != null, "The classifier must not be null.");
IReadOnlyList <IReadOnlyList <Gaussian> > weightPosteriorDistributions = classifier.WeightPosteriorDistributions;
int classCount = weightPosteriorDistributions.Count < 2 ? 2 : weightPosteriorDistributions.Count;
int featureCount = weightPosteriorDistributions[0].Count;
var samples = new Vector[classCount - 1];
for (int c = 0; c < classCount - 1; c++)
{
var sample = Vector.Zero(featureCount);
for (int f = 0; f < featureCount; f++)
{
sample[f] = weightPosteriorDistributions[c][f].Sample();
}
samples[c] = sample;
}
return(samples);
}
public override void Evaluate(
string inputModelFileName,
string reportFileName,
string positiveClassLabel,
string groundTruthFileName,
string predictionsFileName,
string weightsFileName,
string calibrationCurveFileName,
string precisionRecallCurveFileName,
string rocCurveFileName)
{
IBayesPointMachineClassifier <
IList <Vector>, int, IList <string>, string, IDictionary <string, double>,
BayesPointMachineClassifierTrainingSettings,
BinaryBayesPointMachineClassifierPredictionSettings <string> > classifier = null;
// Validate
_validate.Evaluate(
inputModelFileName: inputModelFileName,
reportFileName: reportFileName,
groundTruthFileName: groundTruthFileName,
predictionsFileName: predictionsFileName,
weightsFileName: weightsFileName,
calibrationCurveFileName: calibrationCurveFileName,
precisionRecallCurveFileName: precisionRecallCurveFileName,
rocCurveFileName: rocCurveFileName);
// Load model
if (string.IsNullOrEmpty(inputModelFileName))
{
classifier =
BayesPointMachineClassifier.LoadBinaryClassifier <
IList <Vector>, int, IList <string>, string, IDictionary <string, double> >
(inputModelFileName);
}
else
{
classifier = _classifier;
}
EvaluationReportsMapped evaluationReports = new EvaluationReportsMapped(
classifier,
_evaluator,
_x,
_y,
_yPredicDistrib,
_yPredicLabel,
reportFileName,
positiveClassLabel,
groundTruthFileName,
predictionsFileName,
weightsFileName,
calibrationCurveFileName,
precisionRecallCurveFileName,
rocCurveFileName);
}
/// <summary>
/// Samples weights from the learned weight distribution of the Bayes point machine classifier.
/// </summary>
/// <typeparam name="TTrainingSettings">The type of the settings for training.</typeparam>
/// <param name="classifier">The Bayes point machine used to sample weights from.</param>
/// <param name="samplesFile">The name of the file to which the weights will be written.</param>
public static void SampleWeights <TTrainingSettings>(
IBayesPointMachineClassifier <IList <LabeledFeatureValues>, LabeledFeatureValues, IList <LabelDistribution>, string, IDictionary <string, double>, TTrainingSettings, IBayesPointMachineClassifierPredictionSettings <string> > classifier,
string samplesFile)
where TTrainingSettings : BayesPointMachineClassifierTrainingSettings
{
// Sample weights
var samples = SampleWeights(classifier);
// Write samples to file
if (!string.IsNullOrEmpty(samplesFile))
{
ClassifierPersistenceUtils.SaveVectors(samplesFile, samples);
}
}
/// <summary>
/// Samples weights from the learned weight distribution of the Bayes point machine classifier.
/// </summary>
/// <param name="fileName">The name of the file to write the weights to.</param>
/// <typeparam name="TTrainingSettings">The type of the settings for training.</typeparam>
/// <param name="classifier">The Bayes point machine used to sample weights from.</param>
/// <param name="samplesFile">The name of the file to which the weights will be written.</param>
/// <remarks>Adapted from MicrosoftResearch.Infer.Learners</remarks>
public void SampleWeights(
string fileName,
IBayesPointMachineClassifier <
IList <Vector>,
int,
IList <string>,
string,
IDictionary <string, double>,
BayesPointMachineClassifierTrainingSettings,
BinaryBayesPointMachineClassifierPredictionSettings <string> > classifier)
{
// Sample weights
IEnumerable <Vector> samples = SampleWeights(classifier);
// Write samples to file
using (var writer = new StreamWriter(fileName))
{
writer.WriteLine(string.Join(",", samples));
}
}
public override IDictionary <string, double> PredictInstance(
string inputModelFileName,
DistributionName distributionName,
InferenceAlgorithm inferenceEngineAlgorithm,
int instance,
double noise)
{
TraceListeners.Log(TraceEventType.Warning, 0,
"Advanced setting will not be used: " +
"distributionName, inferenceEngineAlgorithm & noise.", false, true);
// Validate
_validate.PredictInstance(
inputModelFileName: inputModelFileName,
instance: instance,
numObservations: _numObservations);
IBayesPointMachineClassifier <
IList <Vector>, int, IList <string>, string, IDictionary <string, double>,
BayesPointMachineClassifierTrainingSettings,
BinaryBayesPointMachineClassifierPredictionSettings <string> > classifier = null;
// Load model
if (string.IsNullOrEmpty(inputModelFileName))
{
classifier =
BayesPointMachineClassifier.LoadBinaryClassifier <
IList <Vector>, int, IList <string>, string, IDictionary <string, double> >
(inputModelFileName);
}
else
{
classifier = _classifier;
}
IDictionary <string, double> yPredicted =
classifier.PredictDistribution(instance, _x);
// string yPredicLabel = classifier.Predict(instance, _x);
return(yPredicted);
}
public override void TrainIncremental(
string inputModelFileName,
string outputModelFileName,
int iterationCount,
bool computeModelEvidence,
int batchCount,
DistributionName distributionName,
InferenceAlgorithm inferenceEngineAlgorithm,
double noise)
{
TraceListeners.Log(TraceEventType.Warning, 0,
"Advanced setting will not be used: " +
"distributionName, inferenceEngineAlgorithm & noise.", false, true);
// Validate
_validate.TrainIncremental(
inputModelFileName: inputModelFileName,
outputModelFileName: outputModelFileName,
iterationCount: iterationCount,
batchCount: batchCount);
// Load model
IBayesPointMachineClassifier <
IList <Vector>, int, IList <string>, string, IDictionary <string, double>,
BayesPointMachineClassifierTrainingSettings,
BinaryBayesPointMachineClassifierPredictionSettings <string> > classifier =
BayesPointMachineClassifier.LoadBinaryClassifier <
IList <Vector>, int, IList <string>, string, IDictionary <string, double> >
(inputModelFileName);
// Set settings
classifier.Settings.Training.ComputeModelEvidence = computeModelEvidence;
classifier.Settings.Training.IterationCount = iterationCount;
classifier.Settings.Training.BatchCount = batchCount;
// train
classifier.TrainIncremental(_x, _y);
classifier.Save(outputModelFileName);
}
public BPMMapped(
string[] labels)
{
Debug.Assert(labels != null, "The labels must not be null.");
Debug.Assert(labels.Length == 2, "The labels must have two possible values.");
// Initialise the validations
_validate = new Validate();
// Create a BPM from the mapping
_mapping = new GenericClassifierMapping(labels);
_classifier = BayesPointMachineClassifier.CreateBinaryClassifier(_mapping);
// Evaluator mapping
var evaluatorMapping = _mapping.ForEvaluation();
_evaluator = new ClassifierEvaluator
<IList <Vector>, int, IList <string>, string>(evaluatorMapping);
// Other initialisations
_availableDatasetName = new DatasetName();
_numObservations = 0;
_numFeatures = 0;
}
public override void Predict(
string inputModelFileName,
DistributionName distributionName,
InferenceAlgorithm inferenceEngineAlgorithm,
double noise)
{
TraceListeners.Log(TraceEventType.Warning, 0,
"Advanced setting will not be used: " +
"distributionName, inferenceEngineAlgorithm & noise.", false, true);
// Validate
_validate.Predict(inputModelFileName);
// Define the classifier
IBayesPointMachineClassifier <
IList <Vector>, int, IList <string>, string, IDictionary <string, double>,
BayesPointMachineClassifierTrainingSettings,
BinaryBayesPointMachineClassifierPredictionSettings <string> > classifier = null;
// Load model
if (string.IsNullOrEmpty(inputModelFileName))
{
classifier =
BayesPointMachineClassifier.LoadBinaryClassifier <
IList <Vector>, int, IList <string>, string, IDictionary <string, double> >
(inputModelFileName);
}
else
{
classifier = _classifier;
}
_validate.ValidatePredict(_x, _x);
_yPredicDistrib = classifier.PredictDistribution(_x);
_yPredicLabel = classifier.Predict(_x);
}
/// <summary>
///
/// </summary>
/// <param name="evaluator"></param>
/// <param name="x"></param>
/// <param name="y"></param>
/// <param name="yPredicDistrib"></param>
/// <param name="yPredicLabel"></param>
/// <param name="reportFileName"></param>
/// <param name="positiveClassLabel"></param>
/// <param name="calibrationCurveFileName"></param>
/// <param name="rocCurveFileName"></param>
/// <param name="groundTruthFileName"></param>
/// <param name="predictionsFileName"></param>
/// <remarks>Adapted from MicrosoftResearch.Infer.Learners</remarks>
public EvaluationReportsMapped(
IBayesPointMachineClassifier <
IList <Vector>,
int,
IList <string>,
string,
IDictionary <string, double>,
BayesPointMachineClassifierTrainingSettings,
BinaryBayesPointMachineClassifierPredictionSettings <string> > classifier,
ClassifierEvaluator <IList <Vector>, int, IList <string>, string> evaluator,
Vector[] x,
IList <string> y,
IEnumerable <IDictionary <string, double> > yPredicDistrib,
IEnumerable <string> yPredicLabel,
string reportFileName,
string positiveClassLabel,
string groundTruthFileName = "",
string predictionsFileName = "",
string weightsFileName = "",
string calibrationCurveFileName = "",
string precisionRecallCurveFileName = "",
string rocCurveFileName = "")
{
Debug.Assert(classifier != null, "The classifier must not be null.");
Debug.Assert(evaluator != null, "The evaluator must not be null.");
Debug.Assert(x != null, "The feature vector must not be null.");
Debug.Assert(y != null, "The targe variable must not be null.");
Debug.Assert(yPredicDistrib != null, "The predictive distribution must not be null.");
Debug.Assert(yPredicLabel != null, "The predicted labels must not be null.");
Debug.Assert(!string.IsNullOrEmpty(reportFileName), "The report file name must not be null/empty.");
Debug.Assert(!string.IsNullOrEmpty(positiveClassLabel), "The positive class label must not be null/empty.");
// Write evaluation report header information
if (!string.IsNullOrEmpty(reportFileName))
{
using (var writer = new StreamWriter(reportFileName))
{
this.WriteReportHeader(writer, groundTruthFileName, predictionsFileName);
this.WriteReport(writer, evaluator, x, y, yPredicDistrib, yPredicLabel);
}
}
// Write the prediction distribution for all labels
if (!string.IsNullOrEmpty(predictionsFileName))
{
SaveLabelDistributions(predictionsFileName, yPredicDistrib);
}
// Compute and write the empirical probability calibration curve
if (!string.IsNullOrEmpty(calibrationCurveFileName))
{
this.WriteCalibrationCurve(calibrationCurveFileName, evaluator, x, y, yPredicDistrib, positiveClassLabel);
}
// Compute and write the precision-recall curve
if (!string.IsNullOrEmpty(precisionRecallCurveFileName))
{
this.WritePrecisionRecallCurve(precisionRecallCurveFileName, evaluator, x, y, yPredicDistrib, positiveClassLabel);
}
// Compute and write the receiver operating characteristic curve
if (!string.IsNullOrEmpty(rocCurveFileName))
{
this.WriteRocCurve(rocCurveFileName, evaluator, x, y, yPredicDistrib, positiveClassLabel);
}
// Compute and write the weights
if (!string.IsNullOrEmpty(weightsFileName))
{
this.SampleWeights(weightsFileName, classifier);
}
}
/// <summary>
/// Diagnoses the Bayes point machine classifier on the specified data set.
/// </summary>
/// <typeparam name="TTrainingSettings">The type of the settings for training.</typeparam>
/// <param name="classifier">The Bayes point machine classifier.</param>
/// <param name="trainingSet">The dataset.</param>
/// <param name="maxParameterChangesFileName">The name of the file to store the maximum parameter differences.</param>
/// <param name="modelFileName">The name of the file to store the trained Bayes point machine model.</param>
public static void DiagnoseClassifier <TTrainingSettings>(
IBayesPointMachineClassifier <IList <LabeledFeatureValues>, LabeledFeatureValues, IList <LabelDistribution>, string, IDictionary <string, double>, TTrainingSettings, IBayesPointMachineClassifierPredictionSettings <string> > classifier,
IList <LabeledFeatureValues> trainingSet,
string maxParameterChangesFileName,
string modelFileName)
where TTrainingSettings : BayesPointMachineClassifierTrainingSettings
{
// Create prior distributions over weights
int classCount = trainingSet[0].LabelDistribution.LabelSet.Count;
int featureCount = trainingSet[0].GetDenseFeatureVector().Count;
var priorWeightDistributions = Util.ArrayInit(classCount, c => Util.ArrayInit(featureCount, f => new Gaussian(0.0, 1.0)));
// Create IterationChanged handler
var watch = new Stopwatch();
classifier.IterationChanged += (sender, eventArgs) =>
{
watch.Stop();
double maxParameterChange = MaxDiff(eventArgs.WeightPosteriorDistributions, priorWeightDistributions);
if (!string.IsNullOrEmpty(maxParameterChangesFileName))
{
SaveMaximumParameterDifference(
maxParameterChangesFileName,
eventArgs.CompletedIterationCount,
maxParameterChange,
watch.ElapsedMilliseconds);
}
Console.WriteLine(
"[{0}] Iteration {1,-4} dp = {2,-20} dt = {3,5}ms",
DateTime.Now.ToLongTimeString(),
eventArgs.CompletedIterationCount,
maxParameterChange,
watch.ElapsedMilliseconds);
// Copy weight marginals
for (int c = 0; c < eventArgs.WeightPosteriorDistributions.Count; c++)
{
for (int f = 0; f < eventArgs.WeightPosteriorDistributions[c].Count; f++)
{
priorWeightDistributions[c][f] = eventArgs.WeightPosteriorDistributions[c][f];
}
}
watch.Restart();
};
// Write file header
if (!string.IsNullOrEmpty(maxParameterChangesFileName))
{
using (var writer = new StreamWriter(maxParameterChangesFileName))
{
writer.WriteLine("# time, # iteration, # maximum absolute parameter difference, # iteration time in milliseconds");
}
}
// Train the Bayes point machine classifier
Console.WriteLine("[{0}] Starting training...", DateTime.Now.ToLongTimeString());
watch.Start();
classifier.Train(trainingSet);
// Compute evidence
if (classifier.Settings.Training.ComputeModelEvidence)
{
Console.WriteLine("Log evidence = {0,10:0.0000}", classifier.LogModelEvidence);
}
// Save trained model
if (!string.IsNullOrEmpty(modelFileName))
{
classifier.Save(modelFileName);
}
}
