public override void ComputeFeatureForProtocolModel(IFeatureCollectionWrapper <FeatureBase> featureValues)
{
//this.ModelBytePairsValues = (from feature in featureValues as IFeatureCollectionWrapper<BytePairsReoccuringBase>
// group feature by feature.FeatureBytePairsValue
// into grp
// select grp.Key).Where(grp => grp != null).ToArray();
var featuresTyped = featureValues as IFeatureCollectionWrapper <BytePairsReoccuringBase>;
var hashAndCounts = featuresTyped.GroupBy(feature => feature.FeatureBytePairsValue, new ArrayEqualityComparer <byte>()).Select(g => new Entropy.EntropyItem <byte[]>
{
Key = g.Key,
Count = g.Count()
}).ToArray();
this.ModelBytePairsValues = hashAndCounts.Select(hashAndCount => hashAndCount.Key).ToArray();
var normEnth = 0.0;
var n = hashAndCounts.Count();
if (n != 1)
{
var enth = Entropy.Calculate(hashAndCounts);
normEnth = enth / Math.Log(n);
}
this.Weight = 1 - normEnth;
}
// https://en.wikipedia.org/wiki/Coefficient_of_variation
public static double WeightUsingCoefficientOfVariation(IFeatureCollectionWrapper <FeatureBase> featureValues)
{
var features = featureValues.Where(feature => !feature.FeatureValue.Equals(-1.0));
if (!features.Any() || features.Count() < 2)
{
return(0);
}
var stdDev = features.StandardDeviation(feature => feature.FeatureValue);
if (stdDev.Equals(0))
{
return(1);
} // /Math.Sqrt(features.Count()); }
var average = features.Average(feature => feature.FeatureValue);
if (stdDev.CompareTo(average) > 0)
{
return(0);
} //average / stdDev; }
return(stdDev / average);
}
public static double WeightUsingVarianceAndStandarDeviation(IFeatureCollectionWrapper <FeatureBase> featureValues)
{
var features = featureValues.Where(feature => !feature.FeatureValue.Equals(-1.0));
if (!features.Any() || features.Count() < 2)
{
return(0);
}
//if (featureValues.Count() < 2) { return 1; }
var stdDev = features.StandardDeviation(feature => feature.FeatureValue);
if (stdDev.Equals(0))
{
return(1 / Math.Sqrt(features.Count()));
}
var variance = features.Variance(feature => feature.FeatureValue);
if (stdDev.CompareTo(variance) > 0)
{
return(1 - (stdDev));
}
return(1 - (stdDev / variance));
}
public override void ComputeFeatureForProtocolModel(IFeatureCollectionWrapper <FeatureBase> featureValues)
{
this.FeatureValue = FeatureMetrics.FeatureMetricAverage(featureValues);
this.Weight = WeightMetrics.WeightUsingStandarDeviation(featureValues);
if (this.FeatureValue < 50)
{
this.Weight = 1;
}
}
public static double IsValueInStatsMetric(FeatureBase value, IFeatureCollectionWrapper <FeatureBase> featureValues)
{
if (value.FeatureValue.Equals(-1.0) || !featureValues.Any())
{
return(0);
}
return(featureValues.Any(feature => value.FeatureValue.Equals(feature.FeatureValue))? 0.0 : 1.0);
}
public override void ComputeFeatureForProtocolModel(IFeatureCollectionWrapper <FeatureBase> featureValues)
{
this.FeatureValue = FeatureMetrics.FeatureMetricAverage(featureValues);
if (this.FeatureValue == 0)
{
this.Weight = 0;
}
else
{
this.Weight = WeightMetrics.WeightUsingNormEntropy(featureValues);
}
}
public static double WeightProbabilityMetric(IFeatureCollectionWrapper <FeatureBase> trainingFeatureVector)
{
var mostVal = (from i in trainingFeatureVector
group i by i.FeatureValue
into grp
orderby grp.Count() descending
select grp).First();
var result = (double)mostVal.Count() / trainingFeatureVector.Count();
//Console.WriteLine("Weight: " + result + "\n" + "Count " + trainingFeatureVector.Count());
return(result);
}
public static double WeightUsingVariance(IFeatureCollectionWrapper <FeatureBase> featureValues)
{
if (featureValues.Count() < 2)
{
return(1);
}
var variance = featureValues.Variance(feature => feature.FeatureValue);
if (variance.Equals(0))
{
return((double)1 / featureValues.Count());
}
return(1 / variance);
}
public static double WeightUsingStandarDeviation(IFeatureCollectionWrapper <FeatureBase> featureValues)
{
var features = featureValues.Where(feature => !feature.FeatureValue.Equals(-1.0));
if (!features.Any() || features.Count() < 2)
{
return(0);
}
//if (featureValues.Count() < 2) { return 1; }
var stdDev = features.StandardDeviation(feature => feature.FeatureValue);
// if(stdDev.Equals(0)) { return 1;}
return(1 / Math.Sqrt(1 + stdDev));
}
public override void ComputeFeatureForProtocolModel(IFeatureCollectionWrapper <FeatureBase> featureValues)
{
var firstSample = featureValues.FirstOrDefault() as TransportProtocolType;
foreach (var featureValue in featureValues)
{
if (((TransportProtocolType)featureValue).L4ProtocolType != firstSample.L4ProtocolType)
{
throw new InvalidOperationException($"Mandatory condition have not been met: {this.GetType().Name}");
}
}
this.L4ProtocolType = firstSample.L4ProtocolType;
this.FeatureValue = (double)this.L4ProtocolType;
this.Weight = 1;
}
public static double First3BEqualMetric(FeatureBase value, IFeatureCollectionWrapper <FeatureBase> featureValues)
{
if (value.FeatureValue.Equals(-1.0) || !featureValues.Any())
{
return(0);
}
var tmpTrue = featureValues.Count(x => x.FeatureValue.Equals(1.0));
var tmpFalse = featureValues.Count(x => x.FeatureValue.Equals(0.0));
Console.WriteLine("3Bequal " + tmpTrue + " " + tmpFalse + " FeatureValue: " + value.FeatureValue);
if ((tmpTrue > tmpFalse && value.FeatureValue.Equals(1.0)) || (tmpTrue < tmpFalse && value.FeatureValue.Equals(0.0)))
{
return(0.0);
}
return(1.0);
}
public static double WeightUsingNormEntropy(IFeatureCollectionWrapper <FeatureBase> featureValues)
{
var features = featureValues.Where(feature => !feature.FeatureValue.Equals(-1.0));
if (!features.Any() || features.Count() == 1)
{
return(0);
}
var values = (from i in featureValues
group i by i.FeatureValue
into grp
orderby grp.Count() descending
select grp);
//if (featureValues.Count() == 1) { return 1; }
int classes;
var entropyValue = Entropy.Calculate(features.Select(feature => feature.FeatureValue), out classes);
if (entropyValue.Equals(0))
{
return(1); //(double)1 / Math.Sqrt(classes);
}
var result = 1 - entropyValue / Math.Log(classes);
if (result.Equals(0))
{
return(result);
}
//return WeightUsingCoefficientOfVariation(featureValues);
if (values.Count() < Math.Sqrt(classes))
{
var coeffOfVar = WeightUsingCoefficientOfVariation(featureValues);
if (!coeffOfVar.Equals(0))
{
return(coeffOfVar);
}
//return coeffOfVar;
}
return(result);
}
public static double DistanceMetric(FeatureBase value, IFeatureCollectionWrapper <FeatureBase> featureValues)
{
if (value.FeatureValue.Equals(-1.0) || !featureValues.Any())
{
return(0);
}
var result = Math.Abs(value.FeatureValue - featureValues.First().FeatureValue);
foreach (var stat in featureValues.Skip(1))
{
var d = Math.Abs(value.FeatureValue - stat.FeatureValue);
if (d < result)
{
result = d;
}
}
//Console.WriteLine("Distance " + result + " value: " + value.FeatureValue);
return(result);
}
public override void ComputeFeatureForProtocolModel(IFeatureCollectionWrapper <FeatureBase> featureValues)
{
var trueValues = featureValues.Count(feature => feature.FeatureValue.Equals(1.0));
var falseValues = featureValues.Count(feature => feature.FeatureValue.Equals(0.0));
this.FeatureValue = (trueValues > falseValues)? 1 : 0;
if (trueValues == 0 || falseValues == 0)
{
this.Weight = 1;
} // /(double)Math.Sqrt(trueValues+falseValues); }
else if (trueValues < falseValues)
{
this.Weight = 1 - (trueValues / (double)falseValues);
}
else
{
this.Weight = 1 - (falseValues / (double)trueValues);
}
}
/// <summary>
/// http://math.stackexchange.com/questions/395121/how-entropy-scales-with-sample-size
/// </summary>
/// <param name="featureValues"></param>
public override void ComputeFeatureForProtocolModel(IFeatureCollectionWrapper <FeatureBase> featureValues)
{
var featuresTyped = featureValues as IFeatureCollectionWrapper <First4BytesHashBase>;
var hashAndCounts = featuresTyped.GroupBy(feature => feature.FeatureHashValue, new ArrayEqualityComparer <byte>()).Select(g => new Entropy.EntropyItem <byte[]>
{
Key = g.Key,
Count = g.Count()
}).ToArray();
this.ModelHashValues = hashAndCounts.Select(hashAndCount => hashAndCount.Key).ToArray();
var normEnth = 0.0;
var n = hashAndCounts.Count();
if (n != 1)
{
var enth = Entropy.Calculate(hashAndCounts);
normEnth = enth / Math.Log(n);
}
this.Weight = 1 - normEnth;
}
public static double FeatureMetricAverage(IFeatureCollectionWrapper <FeatureBase> featureValues)
{
var features = featureValues.Where(feature => !feature.FeatureValue.Equals(-1.0)).ToArray();
return(!features.Any()? 0 : features.Average(feature => feature.FeatureValue));
}
