public ApplicationProtocolClassificationStatisticsMeter EpiClasify( AppIdentDataSource appIdentDataSource, FeatureSelector featureSelector, AppIdentTestContext appIdentTestContext = null) { return(this.EpiClasify(appIdentDataSource, featureSelector, out var epiEvaluator, appIdentTestContext)); }
private static void RandomForest(AppIdentService appIdentService, AppIdentDataSource appIdentDataSource, AppIdentTestContext context) { Console.WriteLine($"{DateTime.Now} Running feature elimination with trashold { context.FeatureSelectionTreshold}."); var featureSelector = appIdentService.EliminateCorelatedFeatures(appIdentDataSource, context.FeatureSelectionTreshold, context); GridSearchParameterCollection bestParameters; if (!File.Exists(context.BestParametersFilePath)) { Console.WriteLine($"{DateTime.Now} Looking for best parameters."); bestParameters = appIdentService.RandomForestGetBestParameters(appIdentDataSource, featureSelector, context); } else { Console.WriteLine($"{DateTime.Now} Loading best parameters."); context.Load <GridSearchParameterCollection>(context.BestParametersFilePath, out bestParameters); } Console.WriteLine($"{DateTime.Now} Running cross validation."); var classificationStatisticsMeter = appIdentService.RandomForestCrossValidation(appIdentDataSource, context.FeatureSelector, bestParameters, context.CrossValidationFolds, context); Console.WriteLine($"{DateTime.Now} Cross validation results:"); classificationStatisticsMeter.PrintResults(); var model = context.Model; Console.WriteLine($"{DateTime.Now} Running classification"); appIdentService.AccordClassify(appIdentDataSource, model, context.FeatureSelector, context); Console.WriteLine($"{DateTime.Now} Classification results:"); classificationStatisticsMeter.PrintResults(); context.Save(); }
public AppIdentDataSource CreateAppIdentDataSource(IEnumerable <L7Conversation> l7Conversations, int minFlows = 1, double trainingToClassifyingRatio = 1) { var appIdentDataSource = new AppIdentDataSource(); appIdentDataSource.Initialize(l7Conversations, minFlows, trainingToClassifyingRatio); return(appIdentDataSource); }
private static void Epi(AppIdentService appIdentService, AppIdentDataSource appIdentDataSource, AppIdentTestContext context) { Console.WriteLine($"{DateTime.Now} Running feature elimination with trashold { context.FeatureSelectionTreshold}."); var featureSelector = appIdentService.EliminateCorelatedFeatures(appIdentDataSource, context.FeatureSelectionTreshold, context); Console.WriteLine($"{DateTime.Now} Running classification"); var classificationStatisticsMeter = appIdentService.EpiClasify(appIdentDataSource, featureSelector, context); Console.WriteLine($"{DateTime.Now} Classification results:"); classificationStatisticsMeter.PrintResults(); }
private static void Bayesian(AppIdentService appIdentService, AppIdentDataSource appIdentDataSource, AppIdentTestContext context) { Console.WriteLine($"{DateTime.Now} Running feature elimination with trashold { context.FeatureSelectionTreshold}."); var featureSelector = appIdentService.EliminateCorelatedFeatures(appIdentDataSource, context.FeatureSelectionTreshold, context); Console.WriteLine($"{DateTime.Now} Running classification."); var classificationStatisticsMeter = appIdentService.BayesianClassify(appIdentDataSource, context.TrainingToVerificationRation, 0.99, context); Console.WriteLine($"{DateTime.Now} Classification results:"); classificationStatisticsMeter.PrintResults(); context.Save(); }
public GridSearchParameterCollection RandomForestGetBestParameters( AppIdentDataSource appIdentDataSource, FeatureSelector featureSelector, AppIdentTestContext appIdentTestContext = null) { var accordAppIdent = new AccordAppIdent(); var appIdentAcordSource = this.GetAppIdentAcordSource(appIdentDataSource.TrainingSet, featureSelector); accordAppIdent.GetBestRandomForestsWithGridSearch(appIdentAcordSource, out var bestParameters, out var minError); appIdentTestContext?.Save(bestParameters); return(bestParameters); }
public ApplicationProtocolClassificationStatisticsMeter EpiClasify( AppIdentDataSource appIdentDataSource, FeatureSelector featureSelector, out EPIEvaluator epiEvaluator, AppIdentTestContext appIdentTestContext = null) { //this.CreateDatasetAndTestset(appIdentDataSource, trainingToVerificationRatio, out var trainingSet, out var verificationSet); epiEvaluator = new EPIEvaluator(featureSelector); epiEvaluator.CreateApplicationProtocolModels(appIdentDataSource.TrainingSet); var precMeasure = epiEvaluator.ComputeStatistics(appIdentDataSource.VerificationSet); appIdentTestContext?.Save(precMeasure); return(precMeasure); }
public ApplicationProtocolClassificationStatisticsMeter AccordClassify( AppIdentDataSource appIdentDataSource, MulticlassClassifierBase model, FeatureSelector featureSelector, AppIdentTestContext appIdentTestContext) { var precMeasure = new ApplicationProtocolClassificationStatisticsMeter(); var appIdentAcordSource = this.GetAppIdentAcordSource(appIdentDataSource.VerificationSet, featureSelector); var predictedValues = model.Decide(appIdentAcordSource.Samples); for (var j = 0; j < predictedValues.Length; j++) { precMeasure.UpdateStatistics(appIdentAcordSource.LabelsFromInteges[predictedValues[j]], appIdentAcordSource.Labels[j]); } appIdentTestContext.Save(precMeasure); return(precMeasure); }
public ApplicationProtocolClassificationStatisticsMeter RandomForestCrossValidation( AppIdentDataSource appIdentDataSource, FeatureSelector featureSelector, GridSearchParameterCollection bestParameters, int folds, AppIdentTestContext appIdentTestContext) { var precMeasure = new ApplicationProtocolClassificationStatisticsMeter(); var accordAppIdent = new AccordAppIdent(); var appIdentAcordSource = this.GetAppIdentAcordSource(appIdentDataSource.TrainingSet, featureSelector); var cvResults = accordAppIdent.GetCrossValidationResultsOfRandomForestModel(appIdentAcordSource, bestParameters, folds); Console.WriteLine("### CV Results ###"); Console.WriteLine("\n### Training stats ###"); Console.WriteLine(">> model error mean: {0}\n>> model std: {1}", Math.Round(cvResults.Training.Mean, 6), Math.Round(cvResults.Training.StandardDeviation, 6)); Console.WriteLine("\n### Validation stats ###"); Console.WriteLine(">> model error mean: {0}\n>> model std: {1}", Math.Round(cvResults.Validation.Mean, 6), Math.Round(cvResults.Validation.StandardDeviation, 6)); var minErorr = cvResults.Validation.Values.Min(); var bestIndex = cvResults.Validation.Values.IndexOf(minErorr); var classifier = cvResults.Models[bestIndex]; var model = classifier.Model; var labels = appIdentAcordSource.Labels.Distinct(); var modelFilePath = appIdentTestContext.Save(model, labels); var validationDataSource = classifier.Tag as AccordAppIdent.ValidationDataSource; var predictedValues = classifier.Model.Decide(validationDataSource.ValidationInputs); for (var j = 0; j < predictedValues.Length; j++) { precMeasure.UpdateStatistics(appIdentAcordSource.LabelsFromInteges[predictedValues[j]], appIdentAcordSource.LabelsFromInteges[validationDataSource.ValidationOutputs[j]]); } appIdentTestContext.SaveCrossValidation(precMeasure); return(precMeasure); }
public ApplicationProtocolClassificationStatisticsMeter BayesianClassify(AppIdentDataSource appIdentDataSource, double trainingToVerificationRatio, double precisionTrashHold, AppIdentTestContext appIdentTestContext = null) { var precMeasure = new ApplicationProtocolClassificationStatisticsMeter(); //this.CreateDatasetAndTestset(appIdentDataSource, trainingToVerificationRatio, out var trainingSet, out var verificationSet); var classifier = new NaiveBayesClassifier(appIdentDataSource.TrainingSet); foreach (var featureVector in appIdentDataSource.VerificationSet) { var appTag = featureVector.Label.Replace("_", "").Replace("-", ""); featureVector.Label = "Unknown"; classifier.Normalizator.Normalize(featureVector); var cl = classifier.ClassifierModel.Predict(featureVector, true); if (cl.Precision > precisionTrashHold) { precMeasure.UpdateStatistics(cl.Label, appTag); } } appIdentTestContext?.Save(precMeasure); return(precMeasure); }
//public void CreateDatasetAndTestset( // AppIdentDataSource appIdentDataSource, // double trainingToClassifyingRatio, // out List<FeatureVector> trainingSet, // out List<FeatureVector> verificationSet) //{ // trainingSet = new List<FeatureVector>(); // verificationSet = new List<FeatureVector>(); // var groupedFeatureVectors = from featureVector in appIdentDataSource.FeatureVectors // group featureVector by featureVector.Label // into featureVectors // orderby featureVectors.Key // select featureVectors; // //todo this can me managed more randomly // foreach(var gc in groupedFeatureVectors) // { // var conves = gc.ToList(); // var ratioIndex = (int) (conves.Count * trainingToClassifyingRatio); // var testingDataCount = conves.Count - ratioIndex; // trainingSet.AddRange(conves.GetRange(0, ratioIndex)); // verificationSet.AddRange(conves.GetRange(ratioIndex, testingDataCount)); // } //} public ApplicationProtocolClassificationStatisticsMeter DecisionTreeClassify( AppIdentDataSource appIdentDataSource, double trainingToVerificationRatio, double precisionTrashHold) { var precMeasure = new ApplicationProtocolClassificationStatisticsMeter(); //this.CreateDatasetAndTestset(appIdentDataSource, trainingToVerificationRatio, out var trainingSet, out var verificationSet); var classifier = new DecisionTreeClassifier(appIdentDataSource.TrainingSet); foreach (var feature in appIdentDataSource.VerificationSet) { var appTag = feature.Label; feature.Label = "Unknown"; classifier.Normalizator.Normalize(feature); var cl = classifier.ClassifierModel.Predict(feature); if (cl.Precision > precisionTrashHold) { precMeasure.UpdateStatistics(feature.Label, appTag); } } return(precMeasure); }
public FeatureSelector EliminateCorelatedFeatures(AppIdentDataSource appIdentDataSource, double trashold, AppIdentTestContext appIdentTestContext) { var appIdentAcordSource = this.GetAppIdentAcordSource(appIdentDataSource.TrainingSet, new FeatureSelector()); return(this.EliminateCorelatedFeatures(appIdentAcordSource, trashold, appIdentTestContext)); }