/// <summary>
/// This method required training sets for algorithm training and return PxFormula with setted algorithm values.
/// </summary>
/// <param name="trainingSet">This is TrainingSet class object. Must contain training set with ONLY normal records.</param>
/// <param name="nonAnomaliesSet">This is TrainingSet class object. Must contain training set with ONLY normal records.</param>
/// <param name="anomaliesSet">This is TrainingSet class object. Must contain training set with ONLY ANOMALY records.</param>
/// <returns></returns>
public PxFormula TrainAlgorithm(TrainingSet trainingSet, TrainingSet nonAnomaliesSet, TrainingSet anomaliesSet)
{
var formula = CalcFormula(trainingSet);
this.accuracy.CalcAccuracy(nonAnomaliesSet, anomaliesSet, formula);
return formula;
}
private static TrainingSet createTrainingSetFromResources(IEnumerable<KeyValuePair<User, double[]>> studentsAndMarks)
{
TrainingSet resultTrainingSet = new TrainingSet(studentsAndMarks.First().Value.Length);
foreach (KeyValuePair<User, double[]> studentAndMark in studentsAndMarks)
{
resultTrainingSet.addRecord(studentAndMark.Value);
}
return resultTrainingSet;
}
// For demo
public static IEnumerable<KeyValuePair<KeyValuePair<User, double[]>, bool>> runAlg(IEnumerable<KeyValuePair<User, double[]>> studentsAndMarks, TrainingSet trainingSet1, TrainingSet trainingSet2Normal, TrainingSet trainingSet2Anomalies)
{
AnomalyDetectionAlgorithm train = new AnomalyDetectionAlgorithm();
var formula = train.trainAlgorithm(trainingSet1, trainingSet2Normal, trainingSet2Anomalies);
List<KeyValuePair<KeyValuePair<User, double[]>, bool>> res = new List<KeyValuePair<KeyValuePair<User, double[]>, bool>>();
foreach (KeyValuePair<User, double[]> studentAndMark in studentsAndMarks)
{
bool isAnomaly = formula.isAnomaly(studentAndMark.Value);
res.Add(new KeyValuePair<KeyValuePair<User, double[]>, bool>(studentAndMark, isAnomaly));
}
return res;
}
/// <summary>
/// This method creates PxFormula class object and set in it calculated nu vector and sigma matrix values.
/// </summary>
/// <param name="trainingSet">This is TrainingSet class object. Must contain training set with ONLY normal records.</param>
/// <returns>PxFormula class object with setted nu vector and sigma matrix values.</returns>
private static PxFormula CalcFormula(TrainingSet trainingSet)
{
var trainingSetCount = trainingSet.GetCountOfRecords();
var size = trainingSet.GetAllRecords().First().Length;
// calculate nu vector value
var nu = new double[size];
nu = trainingSet.GetAllRecords().Aggregate(nu, (current, featureVector) => current.Add(featureVector));
nu = nu.Divide(trainingSetCount);
// calculate sigme matrix
var sigma = new double[size, size];
sigma = trainingSet.GetAllRecords().Select(featureVector => featureVector.Subtract(nu)).Aggregate(sigma, (current, xNu) => current.Add(Matrix.ColumnVector(xNu).Multiply(Matrix.RowVector(xNu))));
sigma = sigma.Divide(trainingSetCount);
// return PxFormula class object with setted nu and sigma values
return new PxFormula(nu, sigma);
}
/// <summary>
/// This method creates PxFormula class object and set in it calculated nu vector and sigma matrix values.
/// </summary>
/// <param name="trainingSet">This is TrainingSet class object. Must contain training set with ONLY normal records.</param>
/// <returns>PxFormula class object with setted nu vector and sigma matrix values.</returns>
private static PxFormula CalcFormula(TrainingSet trainingSet)
{
var trainingSetCount = trainingSet.GetCountOfRecords();
var size = trainingSet.GetAllRecords().First().Length;
// calculate nu vector value
var nu = new double[size];
nu = trainingSet.GetAllRecords().Aggregate(nu, (current, featureVector) => current.Add(featureVector));
nu = nu.Divide(trainingSetCount);
// calculate sigme matrix
var sigma = new double[size, size];
sigma = trainingSet.GetAllRecords().Select(featureVector => featureVector.Subtract(nu)).Aggregate(sigma, (current, xNu) => current.Add(Matrix.ColumnVector(xNu).Multiply(Matrix.RowVector(xNu))));
sigma = sigma.Divide(trainingSetCount);
// return PxFormula class object with setted nu and sigma values
return(new PxFormula(nu, sigma));
}
public void TrainingSetAdd()
{
var set = new TrainingSet(3);
set.AddRecord(new[] { 3.0, 4.0, 5.0 });
set.AddRecord(new[] { 3.0, 4.0, 5.0 });
set.AddRecord(new[] { 3.0, 4.0, 5.0 });
Assert.AreEqual(set.GetCountOfRecords(), 3);
try
{
set.AddRecord(new[] { 3.0, 4.0, 5.0, 6.0 });
}
catch (Exception)
{
Assert.Pass();
return;
}
Assert.Fail();
}
/// <summary>
/// This method creates PxFormula class object and set in it calculated nu vector and sigma matrix values.
/// </summary>
/// <param name="trainingSet">This is TrainingSet class object. Must contain training set with ONLY normal records.</param>
/// <returns>PxFormula class object with setted nu vector and sigma matrix values.</returns>
private PxFormula calcFormula (TrainingSet trainingSet)
{
int training_set_count = trainingSet.getCountOfRecords();
int size = trainingSet.getAllRecords().First().Length;
// calculate nu vector value
double[] nu = new double[size];
foreach (double[] feature_vector in trainingSet.getAllRecords())
{
nu = Matrix.Add(nu, feature_vector);
}
nu = Matrix.Divide(nu,training_set_count);
// calculate sigme matrix
double[,] sigma = new double[size, size];
foreach (double[] feature_vector in trainingSet.getAllRecords())
{
double[] x_nu = Matrix.Subtract(feature_vector, nu);
sigma = Matrix.Add(sigma, Matrix.Multiply(Matrix.ColumnVector(x_nu), Matrix.RowVector(x_nu)));
}
sigma = Matrix.Divide(sigma, training_set_count);
// return PxFormula class object with setted nu and sigma values
return new PxFormula(nu, sigma);
}
public void TrainingSetGetDimmensionCount()
{
var set = new TrainingSet(3);
Assert.AreEqual(set.GetDimensionsCount(), 3);
}
/// <summary>
/// Train and run anomaly detection algorithm for passes students.
/// </summary>
/// <param name="studentsAndMarks">all students, that we want to check</param>
/// <param name="trainingSet1">Training set</param>
/// <param name="trainingSet2Normal">Cross validation set with only normal records</param>
/// <param name="trainingSet2Anomalies">Cross validation set with inly anomalies records</param>
/// <returns></returns>
public static IEnumerable<KeyValuePair<KeyValuePair<User, double[]>, bool>> RunAlg(IEnumerable<KeyValuePair<User, double[]>> studentsAndMarks, TrainingSet trainingSet1, TrainingSet trainingSet2Normal, TrainingSet trainingSet2Anomalies)
{
var train = new AnomalyDetectionAlgorithm();
var formula = train.TrainAlgorithm(trainingSet1, trainingSet2Normal, trainingSet2Anomalies);
return (from studentAndMark in studentsAndMarks
let isAnomaly = formula.IsAnomaly(studentAndMark.Value)
select new KeyValuePair<KeyValuePair<User, double[]>, bool>(studentAndMark, isAnomaly)).ToList();
}
/// <summary>
/// Train and run anomaly detection algorithm for passes students.
/// </summary>
/// <param name="studentsAndMarks">all students, that we want to check</param>
/// <param name="trainingSet1">Training set</param>
/// <param name="trainingSet2Normal">Cross validation set with only normal records</param>
/// <param name="trainingSet2Anomalies">Cross validation set with inly anomalies records</param>
/// <returns></returns>
public static IEnumerable <KeyValuePair <KeyValuePair <User, double[]>, bool> > RunAlg(IEnumerable <KeyValuePair <User, double[]> > studentsAndMarks, TrainingSet trainingSet1, TrainingSet trainingSet2Normal, TrainingSet trainingSet2Anomalies)
{
var train = new AnomalyDetectionAlgorithm();
var formula = train.TrainAlgorithm(trainingSet1, trainingSet2Normal, trainingSet2Anomalies);
return((from studentAndMark in studentsAndMarks
let isAnomaly = formula.IsAnomaly(studentAndMark.Value)
select new KeyValuePair <KeyValuePair <User, double[]>, bool>(studentAndMark, isAnomaly)).ToList());
}
/// <summary>
/// This method required training sets for algorithm training and return PxFormula with setted algorithm values.
/// </summary>
/// <param name="trainingSet">This is TrainingSet class object. Must contain training set with ONLY normal records.</param>
/// <param name="nonAnomaliesSet">This is TrainingSet class object. Must contain training set with ONLY normal records.</param>
/// <param name="anomaliesSet">This is TrainingSet class object. Must contain training set with ONLY ANOMALY records.</param>
/// <returns></returns>
public PxFormula TrainAlgorithm(TrainingSet trainingSet, TrainingSet nonAnomaliesSet, TrainingSet anomaliesSet)
{
var formula = CalcFormula(trainingSet);
this.accuracy.CalcAccuracy(nonAnomaliesSet, anomaliesSet, formula);
return(formula);
}
// For demo
public static IEnumerable <KeyValuePair <KeyValuePair <User, double[]>, bool> > runAlg(IEnumerable <KeyValuePair <User, double[]> > studentsAndMarks, TrainingSet trainingSet1, TrainingSet trainingSet2Normal, TrainingSet trainingSet2Anomalies)
{
AnomalyDetectionAlgorithm train = new AnomalyDetectionAlgorithm();
var formula = train.trainAlgorithm(trainingSet1, trainingSet2Normal, trainingSet2Anomalies);
List <KeyValuePair <KeyValuePair <User, double[]>, bool> > res = new List <KeyValuePair <KeyValuePair <User, double[]>, bool> >();
foreach (KeyValuePair <User, double[]> studentAndMark in studentsAndMarks)
{
bool isAnomaly = formula.isAnomaly(studentAndMark.Value);
res.Add(new KeyValuePair <KeyValuePair <User, double[]>, bool>(studentAndMark, isAnomaly));
}
return(res);
}
/// <summary>
/// This method required training sets for algorithm training and return PxFormula with setted algorithm values.
/// </summary>
/// <param name="trainingSet">This is TrainingSet class object. Must contain training set with ONLY normal records.</param>
/// <param name="nonAnomaliesSet">This is TrainingSet class object. Must contain training set with ONLY normal records.</param>
/// <param name="anomaliesSet">This is TrainingSet class object. Must contain training set with ONLY ANOMALY records.</param>
/// <returns></returns>
public PxFormula trainAlgorithm(TrainingSet trainingSet, TrainingSet nonAnomaliesSet, TrainingSet anomaliesSet)
{
PxFormula formula = calcFormula(trainingSet);
accuracy.calcAccuracy(nonAnomaliesSet, anomaliesSet, formula);
return(formula);
}
/// <summary>
/// This method required training sets for algorithm training and return PxFormula with setted algorithm values.
/// </summary>
/// <param name="trainingSet">This is TrainingSet class object. Must contain training set with ONLY normal records.</param>
/// <param name="nonAnomaliesSet">This is TrainingSet class object. Must contain training set with ONLY normal records.</param>
/// <param name="anomaliesSet">This is TrainingSet class object. Must contain training set with ONLY ANOMALY records.</param>
/// <returns></returns>
public PxFormula trainAlgorithm (TrainingSet trainingSet, TrainingSet nonAnomaliesSet, TrainingSet anomaliesSet)
{
PxFormula formula = calcFormula(trainingSet);
accuracy.calcAccuracy(nonAnomaliesSet, anomaliesSet, formula);
return formula;
}