public override void ComputeFeatureForProtocolModel(IFeatureCollectionWrapper <FeatureBase> featureValues) { //this.ModelBytePairsValues = (from feature in featureValues as IFeatureCollectionWrapper<BytePairsReoccuringBase> // group feature by feature.FeatureBytePairsValue // into grp // select grp.Key).Where(grp => grp != null).ToArray(); var featuresTyped = featureValues as IFeatureCollectionWrapper <BytePairsReoccuringBase>; var hashAndCounts = featuresTyped.GroupBy(feature => feature.FeatureBytePairsValue, new ArrayEqualityComparer <byte>()).Select(g => new Entropy.EntropyItem <byte[]> { Key = g.Key, Count = g.Count() }).ToArray(); this.ModelBytePairsValues = hashAndCounts.Select(hashAndCount => hashAndCount.Key).ToArray(); var normEnth = 0.0; var n = hashAndCounts.Count(); if (n != 1) { var enth = Entropy.Calculate(hashAndCounts); normEnth = enth / Math.Log(n); } this.Weight = 1 - normEnth; }
// https://en.wikipedia.org/wiki/Coefficient_of_variation public static double WeightUsingCoefficientOfVariation(IFeatureCollectionWrapper <FeatureBase> featureValues) { var features = featureValues.Where(feature => !feature.FeatureValue.Equals(-1.0)); if (!features.Any() || features.Count() < 2) { return(0); } var stdDev = features.StandardDeviation(feature => feature.FeatureValue); if (stdDev.Equals(0)) { return(1); } // /Math.Sqrt(features.Count()); } var average = features.Average(feature => feature.FeatureValue); if (stdDev.CompareTo(average) > 0) { return(0); } //average / stdDev; } return(stdDev / average); }
public static double WeightUsingVarianceAndStandarDeviation(IFeatureCollectionWrapper <FeatureBase> featureValues) { var features = featureValues.Where(feature => !feature.FeatureValue.Equals(-1.0)); if (!features.Any() || features.Count() < 2) { return(0); } //if (featureValues.Count() < 2) { return 1; } var stdDev = features.StandardDeviation(feature => feature.FeatureValue); if (stdDev.Equals(0)) { return(1 / Math.Sqrt(features.Count())); } var variance = features.Variance(feature => feature.FeatureValue); if (stdDev.CompareTo(variance) > 0) { return(1 - (stdDev)); } return(1 - (stdDev / variance)); }
public override void ComputeFeatureForProtocolModel(IFeatureCollectionWrapper <FeatureBase> featureValues) { this.FeatureValue = FeatureMetrics.FeatureMetricAverage(featureValues); this.Weight = WeightMetrics.WeightUsingStandarDeviation(featureValues); if (this.FeatureValue < 50) { this.Weight = 1; } }
public static double IsValueInStatsMetric(FeatureBase value, IFeatureCollectionWrapper <FeatureBase> featureValues) { if (value.FeatureValue.Equals(-1.0) || !featureValues.Any()) { return(0); } return(featureValues.Any(feature => value.FeatureValue.Equals(feature.FeatureValue))? 0.0 : 1.0); }
public override void ComputeFeatureForProtocolModel(IFeatureCollectionWrapper <FeatureBase> featureValues) { this.FeatureValue = FeatureMetrics.FeatureMetricAverage(featureValues); if (this.FeatureValue == 0) { this.Weight = 0; } else { this.Weight = WeightMetrics.WeightUsingNormEntropy(featureValues); } }
public static double WeightProbabilityMetric(IFeatureCollectionWrapper <FeatureBase> trainingFeatureVector) { var mostVal = (from i in trainingFeatureVector group i by i.FeatureValue into grp orderby grp.Count() descending select grp).First(); var result = (double)mostVal.Count() / trainingFeatureVector.Count(); //Console.WriteLine("Weight: " + result + "\n" + "Count " + trainingFeatureVector.Count()); return(result); }
public static double WeightUsingVariance(IFeatureCollectionWrapper <FeatureBase> featureValues) { if (featureValues.Count() < 2) { return(1); } var variance = featureValues.Variance(feature => feature.FeatureValue); if (variance.Equals(0)) { return((double)1 / featureValues.Count()); } return(1 / variance); }
public static double WeightUsingStandarDeviation(IFeatureCollectionWrapper <FeatureBase> featureValues) { var features = featureValues.Where(feature => !feature.FeatureValue.Equals(-1.0)); if (!features.Any() || features.Count() < 2) { return(0); } //if (featureValues.Count() < 2) { return 1; } var stdDev = features.StandardDeviation(feature => feature.FeatureValue); // if(stdDev.Equals(0)) { return 1;} return(1 / Math.Sqrt(1 + stdDev)); }
public override void ComputeFeatureForProtocolModel(IFeatureCollectionWrapper <FeatureBase> featureValues) { var firstSample = featureValues.FirstOrDefault() as TransportProtocolType; foreach (var featureValue in featureValues) { if (((TransportProtocolType)featureValue).L4ProtocolType != firstSample.L4ProtocolType) { throw new InvalidOperationException($"Mandatory condition have not been met: {this.GetType().Name}"); } } this.L4ProtocolType = firstSample.L4ProtocolType; this.FeatureValue = (double)this.L4ProtocolType; this.Weight = 1; }
public static double First3BEqualMetric(FeatureBase value, IFeatureCollectionWrapper <FeatureBase> featureValues) { if (value.FeatureValue.Equals(-1.0) || !featureValues.Any()) { return(0); } var tmpTrue = featureValues.Count(x => x.FeatureValue.Equals(1.0)); var tmpFalse = featureValues.Count(x => x.FeatureValue.Equals(0.0)); Console.WriteLine("3Bequal " + tmpTrue + " " + tmpFalse + " FeatureValue: " + value.FeatureValue); if ((tmpTrue > tmpFalse && value.FeatureValue.Equals(1.0)) || (tmpTrue < tmpFalse && value.FeatureValue.Equals(0.0))) { return(0.0); } return(1.0); }
public static double WeightUsingNormEntropy(IFeatureCollectionWrapper <FeatureBase> featureValues) { var features = featureValues.Where(feature => !feature.FeatureValue.Equals(-1.0)); if (!features.Any() || features.Count() == 1) { return(0); } var values = (from i in featureValues group i by i.FeatureValue into grp orderby grp.Count() descending select grp); //if (featureValues.Count() == 1) { return 1; } int classes; var entropyValue = Entropy.Calculate(features.Select(feature => feature.FeatureValue), out classes); if (entropyValue.Equals(0)) { return(1); //(double)1 / Math.Sqrt(classes); } var result = 1 - entropyValue / Math.Log(classes); if (result.Equals(0)) { return(result); } //return WeightUsingCoefficientOfVariation(featureValues); if (values.Count() < Math.Sqrt(classes)) { var coeffOfVar = WeightUsingCoefficientOfVariation(featureValues); if (!coeffOfVar.Equals(0)) { return(coeffOfVar); } //return coeffOfVar; } return(result); }
public static double DistanceMetric(FeatureBase value, IFeatureCollectionWrapper <FeatureBase> featureValues) { if (value.FeatureValue.Equals(-1.0) || !featureValues.Any()) { return(0); } var result = Math.Abs(value.FeatureValue - featureValues.First().FeatureValue); foreach (var stat in featureValues.Skip(1)) { var d = Math.Abs(value.FeatureValue - stat.FeatureValue); if (d < result) { result = d; } } //Console.WriteLine("Distance " + result + " value: " + value.FeatureValue); return(result); }
public override void ComputeFeatureForProtocolModel(IFeatureCollectionWrapper <FeatureBase> featureValues) { var trueValues = featureValues.Count(feature => feature.FeatureValue.Equals(1.0)); var falseValues = featureValues.Count(feature => feature.FeatureValue.Equals(0.0)); this.FeatureValue = (trueValues > falseValues)? 1 : 0; if (trueValues == 0 || falseValues == 0) { this.Weight = 1; } // /(double)Math.Sqrt(trueValues+falseValues); } else if (trueValues < falseValues) { this.Weight = 1 - (trueValues / (double)falseValues); } else { this.Weight = 1 - (falseValues / (double)trueValues); } }
/// <summary> /// http://math.stackexchange.com/questions/395121/how-entropy-scales-with-sample-size /// </summary> /// <param name="featureValues"></param> public override void ComputeFeatureForProtocolModel(IFeatureCollectionWrapper <FeatureBase> featureValues) { var featuresTyped = featureValues as IFeatureCollectionWrapper <First4BytesHashBase>; var hashAndCounts = featuresTyped.GroupBy(feature => feature.FeatureHashValue, new ArrayEqualityComparer <byte>()).Select(g => new Entropy.EntropyItem <byte[]> { Key = g.Key, Count = g.Count() }).ToArray(); this.ModelHashValues = hashAndCounts.Select(hashAndCount => hashAndCount.Key).ToArray(); var normEnth = 0.0; var n = hashAndCounts.Count(); if (n != 1) { var enth = Entropy.Calculate(hashAndCounts); normEnth = enth / Math.Log(n); } this.Weight = 1 - normEnth; }
public static double FeatureMetricAverage(IFeatureCollectionWrapper <FeatureBase> featureValues) { var features = featureValues.Where(feature => !feature.FeatureValue.Equals(-1.0)).ToArray(); return(!features.Any()? 0 : features.Average(feature => feature.FeatureValue)); }