public ApplicationProtocolClassificationStatisticsMeter EpiClasify(
AppIdentDataSource appIdentDataSource,
FeatureSelector featureSelector,
AppIdentTestContext appIdentTestContext = null)
{
return(this.EpiClasify(appIdentDataSource, featureSelector, out var epiEvaluator, appIdentTestContext));
}
private static void RandomForest(AppIdentService appIdentService, AppIdentDataSource appIdentDataSource, AppIdentTestContext context)
{
Console.WriteLine($"{DateTime.Now} Running feature elimination with trashold { context.FeatureSelectionTreshold}.");
var featureSelector = appIdentService.EliminateCorelatedFeatures(appIdentDataSource, context.FeatureSelectionTreshold, context);
GridSearchParameterCollection bestParameters;
if (!File.Exists(context.BestParametersFilePath))
{
Console.WriteLine($"{DateTime.Now} Looking for best parameters.");
bestParameters = appIdentService.RandomForestGetBestParameters(appIdentDataSource, featureSelector, context);
}
else
{
Console.WriteLine($"{DateTime.Now} Loading best parameters.");
context.Load <GridSearchParameterCollection>(context.BestParametersFilePath, out bestParameters);
}
Console.WriteLine($"{DateTime.Now} Running cross validation.");
var classificationStatisticsMeter =
appIdentService.RandomForestCrossValidation(appIdentDataSource, context.FeatureSelector, bestParameters, context.CrossValidationFolds, context);
Console.WriteLine($"{DateTime.Now} Cross validation results:");
classificationStatisticsMeter.PrintResults();
var model = context.Model;
Console.WriteLine($"{DateTime.Now} Running classification");
appIdentService.AccordClassify(appIdentDataSource, model, context.FeatureSelector, context);
Console.WriteLine($"{DateTime.Now} Classification results:");
classificationStatisticsMeter.PrintResults();
context.Save();
}
public AppIdentDataSource CreateAppIdentDataSource(IEnumerable <L7Conversation> l7Conversations, int minFlows = 1, double trainingToClassifyingRatio = 1)
{
var appIdentDataSource = new AppIdentDataSource();
appIdentDataSource.Initialize(l7Conversations, minFlows, trainingToClassifyingRatio);
return(appIdentDataSource);
}
private static void Epi(AppIdentService appIdentService, AppIdentDataSource appIdentDataSource, AppIdentTestContext context)
{
Console.WriteLine($"{DateTime.Now} Running feature elimination with trashold { context.FeatureSelectionTreshold}.");
var featureSelector = appIdentService.EliminateCorelatedFeatures(appIdentDataSource, context.FeatureSelectionTreshold, context);
Console.WriteLine($"{DateTime.Now} Running classification");
var classificationStatisticsMeter = appIdentService.EpiClasify(appIdentDataSource, featureSelector, context);
Console.WriteLine($"{DateTime.Now} Classification results:");
classificationStatisticsMeter.PrintResults();
}
private static void Bayesian(AppIdentService appIdentService, AppIdentDataSource appIdentDataSource, AppIdentTestContext context)
{
Console.WriteLine($"{DateTime.Now} Running feature elimination with trashold { context.FeatureSelectionTreshold}.");
var featureSelector = appIdentService.EliminateCorelatedFeatures(appIdentDataSource, context.FeatureSelectionTreshold, context);
Console.WriteLine($"{DateTime.Now} Running classification.");
var classificationStatisticsMeter = appIdentService.BayesianClassify(appIdentDataSource, context.TrainingToVerificationRation, 0.99, context);
Console.WriteLine($"{DateTime.Now} Classification results:");
classificationStatisticsMeter.PrintResults();
context.Save();
}
public GridSearchParameterCollection RandomForestGetBestParameters(
AppIdentDataSource appIdentDataSource,
FeatureSelector featureSelector,
AppIdentTestContext appIdentTestContext = null)
{
var accordAppIdent = new AccordAppIdent();
var appIdentAcordSource = this.GetAppIdentAcordSource(appIdentDataSource.TrainingSet, featureSelector);
accordAppIdent.GetBestRandomForestsWithGridSearch(appIdentAcordSource, out var bestParameters, out var minError);
appIdentTestContext?.Save(bestParameters);
return(bestParameters);
}
public ApplicationProtocolClassificationStatisticsMeter EpiClasify(
AppIdentDataSource appIdentDataSource,
FeatureSelector featureSelector,
out EPIEvaluator epiEvaluator,
AppIdentTestContext appIdentTestContext = null)
{
//this.CreateDatasetAndTestset(appIdentDataSource, trainingToVerificationRatio, out var trainingSet, out var verificationSet);
epiEvaluator = new EPIEvaluator(featureSelector);
epiEvaluator.CreateApplicationProtocolModels(appIdentDataSource.TrainingSet);
var precMeasure = epiEvaluator.ComputeStatistics(appIdentDataSource.VerificationSet);
appIdentTestContext?.Save(precMeasure);
return(precMeasure);
}
public ApplicationProtocolClassificationStatisticsMeter AccordClassify(
AppIdentDataSource appIdentDataSource,
MulticlassClassifierBase model,
FeatureSelector featureSelector,
AppIdentTestContext appIdentTestContext)
{
var precMeasure = new ApplicationProtocolClassificationStatisticsMeter();
var appIdentAcordSource = this.GetAppIdentAcordSource(appIdentDataSource.VerificationSet, featureSelector);
var predictedValues = model.Decide(appIdentAcordSource.Samples);
for (var j = 0; j < predictedValues.Length; j++)
{
precMeasure.UpdateStatistics(appIdentAcordSource.LabelsFromInteges[predictedValues[j]], appIdentAcordSource.Labels[j]);
}
appIdentTestContext.Save(precMeasure);
return(precMeasure);
}
public ApplicationProtocolClassificationStatisticsMeter RandomForestCrossValidation(
AppIdentDataSource appIdentDataSource,
FeatureSelector featureSelector,
GridSearchParameterCollection bestParameters,
int folds,
AppIdentTestContext appIdentTestContext)
{
var precMeasure = new ApplicationProtocolClassificationStatisticsMeter();
var accordAppIdent = new AccordAppIdent();
var appIdentAcordSource = this.GetAppIdentAcordSource(appIdentDataSource.TrainingSet, featureSelector);
var cvResults = accordAppIdent.GetCrossValidationResultsOfRandomForestModel(appIdentAcordSource, bestParameters, folds);
Console.WriteLine("### CV Results ###");
Console.WriteLine("\n### Training stats ###");
Console.WriteLine(">> model error mean: {0}\n>> model std: {1}", Math.Round(cvResults.Training.Mean, 6), Math.Round(cvResults.Training.StandardDeviation, 6));
Console.WriteLine("\n### Validation stats ###");
Console.WriteLine(">> model error mean: {0}\n>> model std: {1}", Math.Round(cvResults.Validation.Mean, 6), Math.Round(cvResults.Validation.StandardDeviation, 6));
var minErorr = cvResults.Validation.Values.Min();
var bestIndex = cvResults.Validation.Values.IndexOf(minErorr);
var classifier = cvResults.Models[bestIndex];
var model = classifier.Model;
var labels = appIdentAcordSource.Labels.Distinct();
var modelFilePath = appIdentTestContext.Save(model, labels);
var validationDataSource = classifier.Tag as AccordAppIdent.ValidationDataSource;
var predictedValues = classifier.Model.Decide(validationDataSource.ValidationInputs);
for (var j = 0; j < predictedValues.Length; j++)
{
precMeasure.UpdateStatistics(appIdentAcordSource.LabelsFromInteges[predictedValues[j]],
appIdentAcordSource.LabelsFromInteges[validationDataSource.ValidationOutputs[j]]);
}
appIdentTestContext.SaveCrossValidation(precMeasure);
return(precMeasure);
}
public ApplicationProtocolClassificationStatisticsMeter BayesianClassify(AppIdentDataSource appIdentDataSource, double trainingToVerificationRatio, double precisionTrashHold, AppIdentTestContext appIdentTestContext = null)
{
var precMeasure = new ApplicationProtocolClassificationStatisticsMeter();
//this.CreateDatasetAndTestset(appIdentDataSource, trainingToVerificationRatio, out var trainingSet, out var verificationSet);
var classifier = new NaiveBayesClassifier(appIdentDataSource.TrainingSet);
foreach (var featureVector in appIdentDataSource.VerificationSet)
{
var appTag = featureVector.Label.Replace("_", "").Replace("-", "");
featureVector.Label = "Unknown";
classifier.Normalizator.Normalize(featureVector);
var cl = classifier.ClassifierModel.Predict(featureVector, true);
if (cl.Precision > precisionTrashHold)
{
precMeasure.UpdateStatistics(cl.Label, appTag);
}
}
appIdentTestContext?.Save(precMeasure);
return(precMeasure);
}
//public void CreateDatasetAndTestset(
// AppIdentDataSource appIdentDataSource,
// double trainingToClassifyingRatio,
// out List<FeatureVector> trainingSet,
// out List<FeatureVector> verificationSet)
//{
// trainingSet = new List<FeatureVector>();
// verificationSet = new List<FeatureVector>();
// var groupedFeatureVectors = from featureVector in appIdentDataSource.FeatureVectors
// group featureVector by featureVector.Label
// into featureVectors
// orderby featureVectors.Key
// select featureVectors;
// //todo this can me managed more randomly
// foreach(var gc in groupedFeatureVectors)
// {
// var conves = gc.ToList();
// var ratioIndex = (int) (conves.Count * trainingToClassifyingRatio);
// var testingDataCount = conves.Count - ratioIndex;
// trainingSet.AddRange(conves.GetRange(0, ratioIndex));
// verificationSet.AddRange(conves.GetRange(ratioIndex, testingDataCount));
// }
//}
public ApplicationProtocolClassificationStatisticsMeter DecisionTreeClassify(
AppIdentDataSource appIdentDataSource,
double trainingToVerificationRatio,
double precisionTrashHold)
{
var precMeasure = new ApplicationProtocolClassificationStatisticsMeter();
//this.CreateDatasetAndTestset(appIdentDataSource, trainingToVerificationRatio, out var trainingSet, out var verificationSet);
var classifier = new DecisionTreeClassifier(appIdentDataSource.TrainingSet);
foreach (var feature in appIdentDataSource.VerificationSet)
{
var appTag = feature.Label;
feature.Label = "Unknown";
classifier.Normalizator.Normalize(feature);
var cl = classifier.ClassifierModel.Predict(feature);
if (cl.Precision > precisionTrashHold)
{
precMeasure.UpdateStatistics(feature.Label, appTag);
}
}
return(precMeasure);
}
public FeatureSelector EliminateCorelatedFeatures(AppIdentDataSource appIdentDataSource, double trashold, AppIdentTestContext appIdentTestContext)
{
var appIdentAcordSource = this.GetAppIdentAcordSource(appIdentDataSource.TrainingSet, new FeatureSelector());
return(this.EliminateCorelatedFeatures(appIdentAcordSource, trashold, appIdentTestContext));
}
