public void Test_Training()
{
double[][] clusterCentars = new double[3][];
clusterCentars[0] = new double[] { 5.0, 5.0 };
clusterCentars[1] = new double[] { 15.0, 15.0 };
clusterCentars[2] = new double[] { 30.0, 30.0 };
string[] attributes = new string[] { "Height", "Weight" };
var rawData = Helpers.CreateSampleData(clusterCentars, 2, 10000, 0.5);
int numAttributes = attributes.Length; // 2 in this demo (height,weight)
int numClusters = 3; // vary this to experiment (must be between 2 and number data tuples)
int maxCount = 300; // trial and error
ClusteringSettings clusterSettings = new ClusteringSettings(maxCount, numClusters, numAttributes, KmeansAlgorithm: 1, Replace: true);
AnomalyDetectionAPI kmeanApi = new AnomalyDetectionAPI(clusterSettings); //AnomalyDetectionAPI(clusterSettings), Constractor should not be null when run Training()
AnomalyDetectionResponse response;
for (int i = 0; i < 5; i++)
{
response = kmeanApi.Training(rawData, clusterCentars);
Assert.True(response.Code == 0);
}
//Save Cluster and Instance in json
response = kmeanApi.Save("interface.json");
Assert.True(response.Code == 0);
}
public void Test_Save()
{
double[][] clusterCenters = new double[3][];
clusterCenters[0] = new double[] { 5.0, 5.0 };
clusterCenters[1] = new double[] { 15.0, 15.0 };
clusterCenters[2] = new double[] { 30.0, 30.0 };
string[] attributes = new string[] { "Height", "Weight" };
int numAttributes = attributes.Length; // 2 in this demo (height,weight)
int numClusters = 3; // vary this to experiment (must be between 2 and number data tuples)
int maxCount = 300; // trial and error
ClusteringSettings settings = new ClusteringSettings(maxCount, numClusters, numAttributes, KmeansAlgorithm: 2);
// Creates learning api object
LearningApi api = new LearningApi(loadDescriptor());
// creates data
var rawData = Helpers.CreateSampleData(clusterCenters, 2, 10000, 0.5);
KMeansAlgorithm kMeans = new KMeansAlgorithm(settings);
// train
var response = kMeans.Run(rawData, api.Context);
string fileName = "Test01.json";
kMeans.Save(rootFolder + fileName);
}
public void Test()
{
AnomalyDetectionAPI AnoDet_Api = new AnomalyDetectionAPI(null, 0);
ClusteringSettings Settings;
SaveLoadSettings SaveObject;
SaveLoadSettings LoadObject;
AnomalyDetectionResponse ImportData;
//TODO: Remove user specific paths from application.
string FilePath = @"C:\Users\mhoshen\Desktop\DataSet\SampleDataSet.csv";
double[][] RawData = Helpers.cSVtoDoubleJaggedArray(FilePath);
string SavePath = @"C:\Users\mhoshen\Desktop\DataSet" + "json";
ImportData = SaveLoadSettings.JSON_Settings(SavePath, out SaveObject, true);
string LoadimpPath = @"C:\Users\mhoshen\Desktop\DataSet" + ".json";
ImportData = SaveLoadSettings.JSON_Settings(LoadimpPath, out LoadObject, true);
int kmeansMaxIterations = 5;
int numClusters = 2;
int numOfAttributes = 2;
if (LoadimpPath.Contains("DataSet"))
{
Settings = new ClusteringSettings(RawData, kmeansMaxIterations, numClusters, numOfAttributes, SaveObject, Replace: true);
}
else
{
Settings = new ClusteringSettings(RawData, kmeansMaxIterations, numClusters, numOfAttributes, SaveObject, 1, false, LoadObject, Replace: true);
}
ImportData = AnoDet_Api.ImportNewDataForClustering(Settings);
}
public void Test_FunctionRecognitionModuleSave(int MinNoiseForPrediction, int MaxNoiseForPrediction)
{
#region Train and Save
var batch = 100;
var funcData = FunctionGenerator.CreateFunction(500, 2, 2 * Math.PI / 100);
LearningApi api = new LearningApi();
api.UseActionModule <object, double[][]>((notUsed, ctx) =>
{
var similarFuncData = FunctionGenerator.CreateSimilarFromReferenceFunc(funcData.ToArray(), 7, 10);
double[][] formattedData = formatData(similarFuncData);
return(formattedData);
});
double[][] initCentroids = new double[4][];
initCentroids[0] = new double[] { 1.53, 0.63 };
initCentroids[1] = new double[] { 4.68, -0.63 };
initCentroids[2] = new double[] { 7.85, 0.62 };
initCentroids[3] = new double[] { 10.99, -0.64 };
ClusteringSettings settings = new ClusteringSettings(0, numClusters: 4, numDims: 2, KmeansAlgorithm: 2, initialCentroids: initCentroids, tolerance: 0)
{
KmeansMaxIterations = 1000
};
api.UseKMeansFunctionRecognitionModule(settings);
KMeansFunctionRecognitonScore res;
while (batch-- > 0)
{
res = api.RunBatch() as KMeansFunctionRecognitonScore;
}
api.Save("sinusmodel");
#endregion
#region Load And Predict
var api2 = LearningApi.Load("sinusmodel");
var noisedFunc = FunctionGenerator.CreateSimilarFromReferenceFunc(funcData.ToArray(), MinNoiseForPrediction, MaxNoiseForPrediction);
double[][] data = formatData(noisedFunc);
var predictionResult = api2.Algorithm.Predict(data, null) as KMeansFunctionRecognitionResult;
// TRUE positives
if (MaxNoiseForPrediction <= 10)
{
Assert.True(predictionResult.Loss == 1.0);
}
// TRUE negatives
else if (MaxNoiseForPrediction >= 25)
{
Assert.False(predictionResult.Loss == 1.0);
}
#endregion
}
public void TestClusterCalcuation()
{
//
// In test we know where are positions of centroids.
// We will now create data around known centroids and let alorithm
// find centroids.
double[][] clusterCentars = new double[3][];
clusterCentars[0] = new double[] { 5.0, 5.0 };
clusterCentars[1] = new double[] { 15.0, 15.0 };
clusterCentars[2] = new double[] { 30.0, 30.0 };
string[] attributes = new string[] { "Height", "Weight" };
var rawData = Helpers.CreateSampleData(clusterCentars, 2, 10000, 0.5);
int numAttributes = attributes.Length; // 2 in this demo (height,weight)
int numClusters = 3; // vary this to experiment (must be between 2 and number data tuples)
int maxCount = 300; // trial and error
SaveLoadSettings persistenceProviderSettings;
var resp = SaveLoadSettings.JSON_Settings("model.json", out persistenceProviderSettings, false);
AnomalyDetectionAPI kmeanApi = new AnomalyDetectionAPI(rawData, numClusters);
ClusteringSettings clusterSettings = new ClusteringSettings(rawData, maxCount, numClusters, numAttributes, persistenceProviderSettings, KmeansAlgorithm: 1, Replace: true);
AnomalyDetectionResponse response = kmeanApi.ImportNewDataForClustering(clusterSettings);
Assert.True(response.Code == 0);
int detectedCluster;
double[] Sample = new double[] { 26, 28 };
CheckingSampleSettings SampleSettings = new CheckingSampleSettings(null, Sample, 3);
response = kmeanApi.CheckSample(SampleSettings, out detectedCluster);
Assert.True(response.Code == 0);
Assert.True(detectedCluster == 0);
double[] Sample2 = new double[] { 150, 16 };
CheckingSampleSettings SampleSettings2 = new CheckingSampleSettings(null, Sample2, 3);
response = kmeanApi.CheckSample(SampleSettings2, out detectedCluster);
Assert.True(response.Code == 1);
Assert.True(detectedCluster == -1);// Out of all clusters.
double[] Sample3 = new double[] { 16, 14 };
CheckingSampleSettings SampleSettings3 = new CheckingSampleSettings(null, Sample3, 3);
response = kmeanApi.CheckSample(SampleSettings3, out detectedCluster);
Assert.True(response.Code == 0);
Assert.True(detectedCluster == 1);
double[] Sample4 = new double[] { 6, 4 };
CheckingSampleSettings SampleSettings4 = new CheckingSampleSettings(null, Sample4, 3);
response = kmeanApi.CheckSample(SampleSettings4, out detectedCluster);
Assert.True(response.Code == 0);
Assert.True(detectedCluster == 2);
}
public void Test_LoadSave()
{
string moduleName = "test-action";
double[][] clusterCentars = new double[3][];
clusterCentars[0] = new double[] { 5.0, 5.0 };
clusterCentars[1] = new double[] { 15.0, 15.0 };
clusterCentars[2] = new double[] { 30.0, 30.0 };
string[] attributes = new string[] { "Height", "Weight" };
int numAttributes = attributes.Length; // 2 in this demo (height,weight)
int numClusters = 3; // vary this to experiment (must be between 2 and number data tuples)
int maxCount = 300; // trial and error
ClusteringSettings settings = new ClusteringSettings(maxCount, numClusters, numAttributes, KmeansAlgorithm: 1);
// Creates learning api object
LearningApi api = new LearningApi(loadDescriptor());
//
// Defines action method, which will generate training data.
api.UseActionModule <object, double[][]>((data, ctx) =>
{
var rawData = Helpers.CreateSampleData(clusterCentars, 2, 10000, 0.5);
return(rawData);
}, moduleName);
api.UseKMeans(settings);
var resp = api.Run() as KMeansScore;
Assert.True(resp.Model.Clusters != null);
Assert.True(resp.Model.Clusters.Length == clusterCentars.Length);
var result = api.Algorithm.Predict(clusterCentars, api.Context) as KMeansResult;
Assert.True(result.PredictedClusters[0] == 0);
Assert.True(result.PredictedClusters[1] == 1);
Assert.True(result.PredictedClusters[2] == 2);
// This is where trained model is saved.
api.Save(nameof(TestLoadSave));
// Loads the saved model.
var loadedApi = LearningApi.Load(nameof(TestLoadSave));
//
// Because we have used action method in the LearningApi, we will have to setup it again.
// This is not required because API design limitation. It is restriction of .NET framework. It cannot persist code.
loadedApi.ReplaceActionModule <object, double[][]>(moduleName, (data, ctx) =>
{
var rawData = Helpers.CreateSampleData(clusterCentars, 2, 10000, 0.5);
return(rawData);
});
loadedApi.Run();
}
public void ContinuousTrainData()
{
double[][] initialCentroids = new double[4][];
initialCentroids[0] = new double[] { 0.2, -4.0 };
initialCentroids[1] = new double[] { 0.2, -6.0 };
initialCentroids[2] = new double[] { 0.4, -4.0 };
initialCentroids[3] = new double[] { 0.4, -6.0 };
string[] attributes = new string[] { "x", "y" };
int numAttributes = attributes.Length; // 2 in this demo (x,y)
int numClusters = 4;
int maxCount = 300;
SaveLoadSettings sett;
var resp = SaveLoadSettings.JSON_Settings(@"C:\Data\Function1.json", out sett, true);
AnomalyDetectionAPI kmeanApi = new AnomalyDetectionAPI(null, numClusters);
LearningApi api = new LearningApi(loadMetaData1());
api.UseActionModule <object[][], object[][]>((input, ctx) =>
{
var rawDatalist = getRealDataSample(@"C:\Data\Function1.csv").ToList();
double[][] oldSamples;
var nn = kmeanApi.GetPreviousSamples(sett, out oldSamples);
if (oldSamples != null)
{
foreach (var old in oldSamples)
{
var row = old.Cast <object>().ToArray();
rawDatalist.Add(row);
}
}
return(rawDatalist.ToArray());
});
//this call must be first in the pipeline
api.UseDefaultDataMapper();
api.UseGaussNormalizer();
var rawData = api.Run() as double[][];
Helpers.WriteToCSVFile(rawData);
ClusteringSettings Settings = new ClusteringSettings(rawData, maxCount, numClusters, numAttributes, sett, KmeansAlgorithm: 1, InitialGuess: true, Replace: true);
AnomalyDetectionResponse response = kmeanApi.ImportNewDataForClustering(Settings);
}
public void Test_LoadSave()
{
string moduleName = "test-action";
double[][] clusterCentars = new double[3][];
clusterCentars[0] = new double[] { 5.0, 5.0 };
clusterCentars[1] = new double[] { 15.0, 15.0 };
clusterCentars[2] = new double[] { 30.0, 30.0 };
string[] attributes = new string[] { "Height", "Weight" };
int numAttributes = attributes.Length; // 2 in this demo (height,weight)
int numClusters = 3; // vary this to experiment (must be between 2 and number data tuples)
int maxCount = 300; // trial and error
ClusteringSettings settings = new ClusteringSettings(maxCount, numClusters, numAttributes, KmeansAlgorithm: 1);
// Creates learning api object
LearningApi api = new LearningApi(loadDescriptor());
api.UseActionModule <object, double[][]>((data, ctx) =>
{
var rawData = Helpers.CreateSampleData(clusterCentars, 2, 10000, 0.5);
return(rawData);
}, moduleName);
api.UseKMeans(settings);
var resp = api.Run() as KMeansScore;
Assert.True(resp.Model.Clusters != null);
Assert.True(resp.Model.Clusters.Length == clusterCentars.Length);
var result = api.Algorithm.Predict(clusterCentars, api.Context) as KMeansResult;
Assert.True(result.PredictedClusters[0] == 0);
Assert.True(result.PredictedClusters[1] == 1);
Assert.True(result.PredictedClusters[2] == 2);
api.Save(nameof(LoadSaveTests));
var loadedApi = LearningApi.Load(nameof(LoadSaveTests));
loadedApi.ReplaceActionModule <object, double[][]>(moduleName, (data, ctx) =>
{
var rawData = Helpers.CreateSampleData(clusterCentars, 2, 10000, 0.5);
return(rawData);
});
loadedApi.Run();
}
//Main functions
public AnomalyDetectionResponse ImportNewDataForClustering(ClusteringSettings Settings)
{
Random rnd = new Random();
int n = rnd.Next(1);
if (n == 0)
{
return(new AnomalyDetectionResponse(0, "Clustering Complete. K-means converged at iteration: 5"));
}
else
{
return(new AnomalyDetectionResponse(0, "Clustering Complete. K-means stopped at the maximum allowed iteration: 300"));
}
}
public void Training()
{
int cnt = 0;
double[][] initialCentroids = new double[4][];
initialCentroids[0] = new double[] { 0.4, 25.0 };
initialCentroids[1] = new double[] { 0.4, 15.0 };
initialCentroids[2] = new double[] { 0.6, 15.0 };
initialCentroids[3] = new double[] { 0.6, 25.0 };
string[] attributes = new string[] { "x", "y" };
int numAttributes = attributes.Length;
int numClusters = 4;
int maxCount = 300;
ClusteringSettings clusterSettings = new ClusteringSettings(maxCount, numClusters, numAttributes, KmeansAlgorithm: 1, Replace: true);
AnomalyDetectionAPI kmeanApi = new AnomalyDetectionAPI(clusterSettings); //AnomalyDetectionAPI(clusterSettings), Constractor should not be null when run Training()
AnomalyDetectionResponse response;
// Creates learning api object
LearningApi api = new LearningApi(loadMetaData1());
api.UseActionModule <object[][], object[][]>((input, ctx) =>
{
var rawDataArray = getData(cnt);
return(rawDataArray);
});
api.UseDefaultDataMapper();
api.UseGaussNormalizer();
//
for (int i = 0; i < 15; i++)
{
cnt = i;
var rawData = api.Run() as double[][];
response = kmeanApi.Training(rawData, initialCentroids);
Helpers.WriteToCSVFile(kmeanApi.GetCentroid(), $"Data\\Centroid{i}.csv");
//response = kmeanApi.Save($"Function{i}.json");
}
}
public void ContinuousTrainData2()
{
int cnt = 0;
double[][] initialCentroids = new double[4][];
initialCentroids[0] = new double[] { 40.0, 10.0 };
initialCentroids[1] = new double[] { 20.0, 10.0 };
initialCentroids[2] = new double[] { 40.0, 20.0 };
initialCentroids[3] = new double[] { 20.0, 20.0 };
string[] attributes = new string[] { "x", "y" };
int numAttributes = attributes.Length; // 2 in this demo (x,y)
int numClusters = 4;
int maxCount = 300;
SaveLoadSettings sett;
var resp = SaveLoadSettings.JSON_Settings(@"C:\Data\Function1.json", out sett, true);
AnomalyDetectionAPI kmeanApi = new AnomalyDetectionAPI(null, numClusters, initialCentroids);
LearningApi api = new LearningApi(loadMetaData1());
api.UseActionModule <object[][], object[][]>((input, ctx) =>
{
var rawDatalist = getData(cnt);
return(rawDatalist);
});
//this call must be first in the pipeline
api.UseDefaultDataMapper();
api.UseGaussNormalizer();
for (int i = 0; i < 15; i++)
{
cnt = i;
var rawData = api.Run() as double[][];
ClusteringSettings Settings = new ClusteringSettings(rawData, maxCount, numClusters, numAttributes, sett, KmeansAlgorithm: 1, InitialGuess: true, Replace: true);
AnomalyDetectionResponse response = kmeanApi.ImportNewDataForClustering(Settings);
}
}
/// <summary>
/// Provide settings for K-Means algorithm.
/// </summary>
public MouseGestureRecognizer()
{
m_LearningApi = new LearningApi();
m_LearningApi.UseActionModule <object, double[][]>((notUsed, ctx) =>
{
return(m_CurrentData);
});
double[][] initCentroids = new double[2][];
initCentroids[0] = new double[] { 0.0, 0.0, 0.0 };
initCentroids[1] = new double[] { 400.0, 0.0, 0.0 };
m_Settings = new ClusteringSettings(KmeansMaxIterations: 1000, numClusters: 2, numDims: 3, KmeansAlgorithm: 2, initialCentroids: initCentroids, tolerance: 0);
m_LearningApi.UseKMeansFunctionRecognitionModule(m_Settings);
}
public void Test_GetClusters()
{
double[][] clusterCentars = new double[3][];
clusterCentars[0] = new double[] { 5.0, 5.0 };
clusterCentars[1] = new double[] { 15.0, 15.0 };
clusterCentars[2] = new double[] { 30.0, 30.0 };
double[][] initialCentroids = new double[3][];
clusterCentars[0] = new double[] { 5.0, 5.0 };
clusterCentars[1] = new double[] { 15.0, 15.0 };
clusterCentars[2] = new double[] { 30.0, 30.0 };
string[] attributes = new string[] { "Height", "Weight" };
var rawData = Helpers.CreateSampleData(clusterCentars, 2, 10000, 0.5);
int numAttributes = attributes.Length; // 2 in this demo (height,weight)
int numClusters = 3; // vary this to experiment (must be between 2 and number data tuples)
int maxCount = 300; // trial and error
ClusteringSettings Settings = new ClusteringSettings(maxCount, numClusters, numAttributes, KmeansAlgorithm: 1, InitialGuess: true, Replace: true);
AnomalyDetectionAPI kmeanApi = new AnomalyDetectionAPI(Settings);
AnomalyDetectionResponse response = kmeanApi.Training(rawData, clusterCentars);
Assert.True(response.Code == 0);
Cluster[] clusters;
response = kmeanApi.GetClusters(null, out clusters); // Path can be null if GetClusters() runs after Training()
Assert.True(response.Code == 0);
Assert.True(clusters.Length != 0);
Assert.True(clusters != null);
AnomalyDetectionAPI kApi = new AnomalyDetectionAPI();
string filePath = $"{Directory.GetCurrentDirectory()}\\Cluster Result\\CheckSample.json";
Cluster[] clusters1;
response = kmeanApi.GetClusters(null, out clusters1);
Assert.True(response.Code == 0);
Assert.True(clusters.Length != 0);
Assert.True(clusters != null);
}
public AnomalyDetectionResponse Training([FromBody] TrainingContent trainingContent)
{
ClusteringSettings clusterSettings = new ClusteringSettings(trainingContent.KmeansMaxIterations, trainingContent.NumOfClusters, trainingContent.NumOfAttributes, KmeansAlgorithm: 1, Replace: true);
AnomalyDetectionAPI kmeanApi = new AnomalyDetectionAPI(clusterSettings);
AnomalyDetectionResponse response = null;
try
{
double[][] rawData = null;
foreach (var file in trainingContent.CsvFilePaths)
{
rawData = dataProvider(file);
response = kmeanApi.Training(rawData);
}
if (response.Code == 0)
{
response = kmeanApi.Save(trainingContent.SavePath);
}
return(response);
}
catch (Exception Ex)
{
if (Ex is System.IO.FileNotFoundException)
{
response = new AnomalyDetectionResponse(200, "File not found");
}
else if (Ex is System.IO.DirectoryNotFoundException)
{
response = new AnomalyDetectionResponse(204, "Directory not found");
}
else
{
response = new AnomalyDetectionResponse(202, "File cannot be loaded");
}
return(response);
}
}
public void TestObjestWithCentroid()
{
double[][] initialCentroids = new double[4][];
initialCentroids[0] = new double[] { 0.2, -4.0 };
initialCentroids[1] = new double[] { 0.2, -6.0 };
initialCentroids[2] = new double[] { 0.4, -4.0 };
initialCentroids[3] = new double[] { 0.4, -6.0 };
string[] attributes = new string[] { "x", "y" };
var data = getRealDataSample(@"C:\Data\Function1.csv");
List <double[]> list = new List <double[]>();
foreach (var n in data)
{
double[] d = new double[n.Length];
for (int i = 0; i < n.Length; i++)
{
d[i] = double.Parse(n[i].ToString());
}
list.Add(d);
}
var rawData = list.ToArray();
int numAttributes = attributes.Length; // 2 in this demo (height,weight)
int numClusters = 4; // vary this to experiment (must be between 2 and number data tuples)
int maxCount = 300; // trial and error
SaveLoadSettings sett;
var resp = SaveLoadSettings.JSON_Settings(@"C:\Data\Function1.json", out sett, true);
AnomalyDetectionAPI kmeanApi = new AnomalyDetectionAPI(rawData, numClusters);
ClusteringSettings Settings = new ClusteringSettings(rawData, maxCount, numClusters, numAttributes, sett, KmeansAlgorithm: 1, InitialGuess: true, Replace: true);
AnomalyDetectionResponse response = kmeanApi.ImportNewDataForClustering(Settings);
}
public void Test_OptimalNumberOfCLusters()
{
// directory to load
string loadDirectory = rootFolder + "Functions\\";
string FunctionName = "SIN X"; //without extension
string savePath = rootFolder + "Optimal Clusters\\" + FunctionName + " Results.csv";
double[][] function = Helpers.LoadFunctionData(loadDirectory + FunctionName + "\\" + FunctionName + ".csv");
function = TestFunctionGenerators.normalizeData(function);
int numAttributes = 2; // 2 in this demo (height,weight)
int numClusters = 0; // vary this to experiment (must be between 2 and number data tuples)
int maxCount = 300; // trial and error
ClusteringSettings settings = new ClusteringSettings(maxCount, numClusters, numAttributes, KmeansAlgorithm: 2);
// Creates learning api object
LearningApi api = new LearningApi();
api.UseActionModule <object, double[][]>((data, ctx) =>
{
return(KMeansAlgorithm.transposeFunction(function));
});
api.UseKMeans(settings);
// train
var resp = api.Run() as KMeansScore;
Assert.True(resp.Model.NumberOfClusters > 1);
double[][] OptimalClustersResults = new double[4][];
OptimalClustersResults[0] = new double[] { 2, 3, 4, 5, 6, 7, 8, 9, 10 };
OptimalClustersResults[1] = resp.Model.D;
OptimalClustersResults[2] = resp.Model.DPrime;
OptimalClustersResults[3] = resp.Model.Fmin;
Helpers.Write2CSVFile(OptimalClustersResults, savePath);
// implement
}
public AnomalyDetectionResponse Training(string[] csvFilePath, string savePath, string loadimpPath, int numClusters, int numOfAttributes, int kmeansMaxIterations)
{
ClusteringSettings clusterSettings = new ClusteringSettings(kmeansMaxIterations, numClusters, numOfAttributes, KmeansAlgorithm: 1, Replace: true);
AnomalyDetectionAPI kmeanApi = new AnomalyDetectionAPI(clusterSettings);
AnomalyDetectionResponse response;
try
{
double[][] rawData = null;
foreach (var file in csvFilePath)
{
rawData = dataProvider(file);
}
response = kmeanApi.Training(rawData);
response = kmeanApi.Save(savePath);
return(response);
}
catch (Exception Ex)
{
if (Ex is System.IO.FileNotFoundException)
{
response = new AnomalyDetectionResponse(200, "File not found");
}
else if (Ex is System.IO.DirectoryNotFoundException)
{
response = new AnomalyDetectionResponse(204, "Directory not found");
}
else
{
response = new AnomalyDetectionResponse(202, "File cannot be loaded");
}
return(response);
}
}
public void Test_BalancedClusters()
{
// prepare the 2d functions for clustering
double[][] mFun = Helpers.cSVtoDoubleJaggedArray(@"C:\Users\KywAnn\Desktop\Anomaly Detection Functions Normalized\Sine 3 HP\Sine 3 HP.csv");
// normalize functions
mFun = NormalizeData(mFun);
// load the original function
double[][] rawData = getSimilarFunctionsData(mFun, 1);
// Settings for the K-Means Alg
int maxCount = 500;
int maxNumberOfClusters = 5;
int minNumberOfClusters = 2;
int numAttributes = 2;
int KAlg = 2;
// recommended number of clusters
int recNumClusters;
ClusteringSettings clusterSettings = new ClusteringSettings(maxCount, 2, numAttributes, KmeansAlgorithm: KAlg, Replace: true);
AnomalyDetectionAPI kmeanApi = new AnomalyDetectionAPI(clusterSettings);
// get the suggested number of clusters bas on the balanced clusters method
AnomalyDetectionResponse ar = kmeanApi.RecommendedNumberOfClusters(rawData, maxCount, numAttributes, maxNumberOfClusters, minNumberOfClusters, 3, null, out recNumClusters, kmeansAlgorithm: KAlg);
}
public void TestWithNormalize_GaussAndCentroid()
{
double[][] initialCentroids = new double[4][];
initialCentroids[0] = new double[] { 0.2, -4.0 };
initialCentroids[1] = new double[] { 0.2, -6.0 };
initialCentroids[2] = new double[] { 0.4, -4.0 };
initialCentroids[3] = new double[] { 0.4, -6.0 };
string[] attributes = new string[] { "x", "y" };
// Creates learning api object
LearningApi api = new LearningApi(loadMetaData1());
//Real dataset must be defined as object type, because data can be numeric, binary and classification
api.UseActionModule <object[][], object[][]>((input, ctx) =>
{
return(getRealDataSample(@"C:\Data\Function15.csv"));
});
//this call must be first in the pipeline
api.UseDefaultDataMapper();
api.UseGaussNormalizer();
var rawData = api.Run() as double[][];
int numAttributes = attributes.Length; // 2 in this demo (height,weight)
int numClusters = 4; // vary this to experiment (must be between 2 and number data tuples)
int maxCount = 300; // trial and error
SaveLoadSettings sett;
var resp = SaveLoadSettings.JSON_Settings(@"C:\Data\Function15.json", out sett, true);
AnomalyDetectionAPI kmeanApi = new AnomalyDetectionAPI(rawData, numClusters);
ClusteringSettings Settings = new ClusteringSettings(rawData, maxCount, numClusters, numAttributes, sett, KmeansAlgorithm: 1, InitialGuess: true, Replace: true);
AnomalyDetectionResponse response = kmeanApi.ImportNewDataForClustering(Settings);
}
public void Test_OptimalNumberOfClusters_TwoFunctions()
{
int numAttributes = 3;
int MinNumClusters = 2;
int MaxNumClusters = 10;
int maxCount = 300;
double[] Fmins;
// directory to load
string loadDirectory = rootFolder + "Functions\\";
string functionA = "SIN_SIN X";
string functionB = "SIN_COS X";
// load Data
double[][] loadedSimFunctions1 = Helpers.LoadFunctionData(loadDirectory + functionA + "\\NRP5-10\\" + functionA + " SimilarFunctions Normalized NRP5-10.csv");
double[][] loadedSimFunctions2 = Helpers.LoadFunctionData(loadDirectory + functionB + "\\NRP5-10\\" + functionB + " SimilarFunctions Normalized NRP5-10.csv");
ClusteringSettings settings = new ClusteringSettings(maxCount, MinNumClusters, numAttributes, KmeansAlgorithm: 2);
int recNumClusters = KMeansFunctionRecognitionAlgorithm.OptimalNumberOfClusters_TwoFunctions(loadedSimFunctions1, loadedSimFunctions2, settings, MinNumClusters, MaxNumClusters, out Fmins);
}
public KMeansFunctionRecognitionAlgorithm(ClusteringSettings settings)
{
this.Settings = settings;
this.Score = new KMeansFunctionRecognitonScore();
this.Score.Centroids = new double[settings.NumberOfClusters][];
if (this.Settings.FuncRecogMethod == 1)
{
this.Score.InClusterMaxDistance = new double[settings.NumberOfClusters];
this.Score.MaxCentroid = null;
this.Score.MinCentroid = null;
for (int i = 0; i < settings.NumberOfClusters; i++)
{
this.Score.Centroids[i] = new double[settings.NumOfDimensions];
}
}
else
{
this.Score.InClusterMaxDistance = null;
this.Score.MaxCentroid = new double[settings.NumberOfClusters][];
this.Score.MinCentroid = new double[settings.NumberOfClusters][];
for (int i = 0; i < settings.NumberOfClusters; i++)
{
this.Score.MaxCentroid[i] = new double[settings.NumOfDimensions];
this.Score.MinCentroid[i] = new double[settings.NumOfDimensions];
this.Score.Centroids[i] = new double[settings.NumOfDimensions];
for (int dim = 0; dim < settings.NumOfDimensions; dim++)
{
this.Score.MaxCentroid[i][dim] = double.MinValue;
this.Score.MinCentroid[i][dim] = double.MaxValue;
}
}
}
}
public static int OptimalNumberOfClusters_TwoFunctions(double[][] functions1, double[][] functions2, ClusteringSettings settings, int MinNumClusters, int MaxNumClusters, out double[] Fmins_k)
{
if (MinNumClusters < 2)
{
MinNumClusters = 2;
}
double[][] oneFunction = new double[settings.NumOfDimensions][];
int numFun = 0;
double dist;
int score = 0;
KMeansFunctionRecognitionAlgorithm kMeansFR1, kMeansFR2;
KMeansFunctionRecognitonScore res1, res2;
double Fmax = 0;
double Fmin = double.MaxValue;
double[] Fmins;
Fmins_k = new double[MaxNumClusters - MinNumClusters + 1];
double F = 0;
int cluster = -1;
for (int k = MinNumClusters; k <= MaxNumClusters; k++)
{
settings.InitialCentroids = null;
settings.NumberOfClusters = k;
kMeansFR1 = new KMeansFunctionRecognitionAlgorithm(settings);
kMeansFR2 = new KMeansFunctionRecognitionAlgorithm(settings);
res1 = new KMeansFunctionRecognitonScore();
res2 = new KMeansFunctionRecognitonScore();
numFun = functions1.Length / settings.NumOfDimensions;
for (int f = 0; f < numFun; f++)
{
oneFunction = KMeansAlgorithm.transposeFunction(KMeansAlgorithm.selectFunction(functions1, f + 1, settings.NumOfDimensions));
res1 = kMeansFR1.Run(oneFunction, null) as KMeansFunctionRecognitonScore;
}
numFun = functions2.Length / settings.NumOfDimensions;
settings.InitialCentroids = null;
for (int f = 0; f < numFun; f++)
{
oneFunction = KMeansAlgorithm.transposeFunction(KMeansAlgorithm.selectFunction(functions2, f + 1, settings.NumOfDimensions));
res2 = kMeansFR2.Run(oneFunction, null) as KMeansFunctionRecognitonScore;
}
Fmins = new double[k];
// check if there is a non intersection cluster
for (int i = 0; i < k; i++)
{
Fmin = double.MaxValue;
score = 0;
for (int j = 0; j < k; j++)
{
dist = KMeansAlgorithm.calculateDistance(res1.Centroids[i], res2.Centroids[j]);
if (dist > res1.InClusterMaxDistance[i] + res2.InClusterMaxDistance[j])
{
score++;
}
}
if (score == k)
{
//calculate F;
for (int j = 0; j < k; j++)
{
F = KMeansAlgorithm.calculateDistance(res1.Centroids[i], res2.Centroids[j]) / (res1.InClusterMaxDistance[i] + res2.InClusterMaxDistance[j]);
// select min F among the two functions
if (F < Fmin)
{
Fmin = F;
}
}
}
//save Fmin of each centroid
if (Fmin != double.MaxValue)
{
Fmins[i] = Fmin;
}
}
// save max Fmin per number of clusters
for (int i = 0; i < k; i++)
{
if (Fmins[i] > Fmins_k[k - MinNumClusters])
{
Fmins_k[k - MinNumClusters] = Fmins[i];
}
}
}
//select max F among different number of cluters
for (int i = 0; i < Fmins_k.Length; i++)
{
if (Fmins_k[i] > Fmax)
{
Fmax = Fmins_k[i];
cluster = i + MinNumClusters;
}
}
return(cluster);
}
public void Test_CheckSample()
{
double[][] clusterCentars = new double[3][];
clusterCentars[0] = new double[] { 5.0, 5.0 };
clusterCentars[1] = new double[] { 15.0, 15.0 };
clusterCentars[2] = new double[] { 30.0, 30.0 };
double[][] initialCentroids = new double[3][];
clusterCentars[0] = new double[] { 5.0, 5.0 };
clusterCentars[1] = new double[] { 15.0, 15.0 };
clusterCentars[2] = new double[] { 30.0, 30.0 };
string[] attributes = new string[] { "Height", "Weight" };
var rawData = Helpers.CreateSampleData(clusterCentars, 2, 10000, 0.5);
int numAttributes = attributes.Length; // 2 in this demo (height,weight)
int numClusters = 3; // vary this to experiment (must be between 2 and number data tuples)
int maxCount = 300; // trial and error
ClusteringSettings Settings = new ClusteringSettings(maxCount, numClusters, numAttributes, KmeansAlgorithm: 1, InitialGuess: true, Replace: true);
AnomalyDetectionAPI kmeanApi = new AnomalyDetectionAPI(Settings);
AnomalyDetectionResponse response = kmeanApi.Training(rawData, clusterCentars);
Assert.True(response.Code == 0);
response = kmeanApi.Save("CheckSample.json");
Assert.True(response.Code == 0);
int detectedCluster;
double[] Sample = new double[] { 26, 28 };
CheckingSampleSettings SampleSettings = new CheckingSampleSettings(null, Sample, 3); //If path is null you should run Training()
response = kmeanApi.CheckSample(SampleSettings, out detectedCluster);
Assert.True(response.Code == 0);
Assert.True(detectedCluster == 2);
AnomalyDetectionAPI kApi = new AnomalyDetectionAPI();
string filePath = $"{Directory.GetCurrentDirectory()}\\Instance Result\\CheckSample.json";
double[] Sample2 = new double[] { 150, 16 };
CheckingSampleSettings SampleSettings2 = new CheckingSampleSettings(filePath, Sample2, 3);
response = kApi.CheckSample(SampleSettings2, out detectedCluster);
Assert.True(response.Code == 1);
Assert.True(detectedCluster == -1);// Out of all clusters.
double[] Sample3 = new double[] { 16, 14 };
CheckingSampleSettings SampleSettings3 = new CheckingSampleSettings(filePath, Sample3, 3);
response = kApi.CheckSample(SampleSettings3, out detectedCluster);
Assert.True(response.Code == 0);
Assert.True(detectedCluster == 1);
double[] Sample4 = new double[] { 6, 4 };
CheckingSampleSettings SampleSettings4 = new CheckingSampleSettings(filePath, Sample4, 3);
response = kApi.CheckSample(SampleSettings4, out detectedCluster);
Assert.True(response.Code == 0);
Assert.True(detectedCluster == 0);
}
public void Test_TrainingSimilarFunctions()
{
// Settings to import the functions (NRP should match the desired loading file)
string FunctionName = "SIN 2X"; //without extension
string directory = rootFolder + FunctionName + "\\";
int NRPmin = 5;
int NRPmax = 10;
string NRP = NRPmin + "-" + NRPmax;
// Settings for the K-Means Alg
int maxCount = 500;
int numClusters = 10;
int numAttributes = 2;
int KAlg = 2;
int Runs = 1;
// prepare the functions for clustering
// Holds the data of all function. Attribute of every function contains data in a row.
// N dimensions of fnctionmeans N rows per function.
//double[][] allFunctionsData = Helpers.LoadFunctionData(directory + "\\NRP" + NRP + "\\" + FunctionName + " SimilarFunctions Normalized NRP" + NRP + ".csv");
double[][] allFunctionsData = Helpers.LoadFunctionData(directory + "\\NRP" + NRP + "\\" + FunctionName + " SimilarFunctions NRP" + NRP + ".csv");
int numFunc = allFunctionsData.Length / numAttributes;
// Creates learning api object
LearningApi api;
ClusteringSettings clusterSettings;
double[][] lastCalculatedCentroids = null;
double[][] Centroids = null;
// original Centroids
double[][] oCentroids;
// matched Centroids
double[][] mCentroids;
for (int k = 2; k < numClusters + 1; k++)
{
oCentroids = new double[k][];
clusterSettings = new ClusteringSettings(maxCount, k, numAttributes, KmeansAlgorithm: KAlg);
for (int j = 0; j < Runs; j++)
{
// save directory
//string savePath = directory + "NRP" + NRP + "\\" + FunctionName + " SimilarFunctions Normalized Centroids NRP" + NRP + " KA" + KAlg + " C" + k + " I" + maxCount + " R" + (j + 1) + ".csv";
string savePath = directory + "NRP" + NRP + "\\" + FunctionName + " SimilarFunctions Centroids NRP" + NRP + " KA" + KAlg + " C" + k + " I" + maxCount + " R" + (j + 1) + ".csv";
lastCalculatedCentroids = null;
for (int funcIndx = 0; funcIndx < numFunc; funcIndx++)
{
// Get data of specific function with indec funcIndx.
double[][] rawData = getSimilarFunctionsData(allFunctionsData, numAttributes, funcIndx + 1);
api = new LearningApi();
api.UseActionModule <object, double[][]>((data, ctx) =>
{
return(rawData);
});
clusterSettings.InitialCentroids = lastCalculatedCentroids;
api.UseKMeans(clusterSettings);
// train
var resp = api.Run() as KMeansScore;
// get resulting centroids
lastCalculatedCentroids = new double[k][];
for (int i = 0; i < k; i++)
{
lastCalculatedCentroids[i] = resp.Model.Clusters[i].Centroid;
}
Centroids = lastCalculatedCentroids;
/*
* // match the centroids centroids
* if (funcIndx == 0)
* {
* oCentroids = Centroids;
* mCentroids = Centroids;
* }
* else
* {
* mCentroids = matchCentroids(Centroids, oCentroids);
* }*/
// save centroids
if (funcIndx == 0)
{
// save or overwrite
Helpers.Write2CSVFile(Centroids, savePath);
}
else
{
// append
Helpers.Write2CSVFile(Centroids, savePath, true);
}
}
}
}
}
public void Test_TrainPartials()
{
double[][] clusterCenters = new double[3][];
clusterCenters[0] = new double[] { 5.0, 5.0 };
clusterCenters[1] = new double[] { 15.0, 15.0 };
clusterCenters[2] = new double[] { 30.0, 30.0 };
double[][] clusterCenters2 = new double[3][];
clusterCenters2[0] = new double[] { 6, 5 };
clusterCenters2[1] = new double[] { 17, 18 };
clusterCenters2[2] = new double[] { 28, 30 };
string[] attributes = new string[] { "Height", "Weight" };
int numAttributes = attributes.Length; // 2 in this demo (height,weight)
int numClusters = 3; // vary this to experiment (must be between 2 and number data tuples)
int maxCount = 300; // trial and error
double[][] apiResp1Centroid = new double[numClusters][];
double[] apiResp1MaxDistance = new double[numClusters];
double[] apiResp1NumSamples = new double[numClusters];
ClusteringSettings settings = new ClusteringSettings(maxCount, numClusters, numAttributes, KmeansAlgorithm: 2);
// Creates learning api object
LearningApi api = new LearningApi(loadDescriptor());
LearningApi api2 = new LearningApi(loadDescriptor());
double[][] rawData = Helpers.CreateSampleData(clusterCenters, 2, 10000, 0.5);
double[][] rawData2 = Helpers.CreateSampleData(clusterCenters2, 2, 5000, 0.5);
int runNum = 0;
api.UseActionModule <object, double[][]>((data, ctx) =>
{
if (runNum == 0)
{
return(rawData);
}
else
{
return(rawData2);
}
});
api2.UseActionModule <object, double[][]>((data, ctx) =>
{
return(rawData2);
});
// start api2 that runs only second raw data (rawData2)
api2.UseKMeans(settings);
// train
var api2Resp = api2.Run() as KMeansScore;
Assert.True(api2Resp.Model.Clusters != null);
Assert.True(api2Resp.Model.Clusters.Length == clusterCenters.Length);
// start api that runs first raw data (rawData) and save results in variables
api.UseKMeans(settings);
// train
var apiResp = api.Run() as KMeansScore;
Assert.True(apiResp.Model.Clusters != null);
Assert.True(apiResp.Model.Clusters.Length == clusterCenters.Length);
// save first run results in variables
for (int i = 0; i < numClusters; i++)
{
apiResp1Centroid[i] = apiResp.Model.Clusters[i].Centroid;
apiResp1MaxDistance[i] = apiResp.Model.Clusters[i].InClusterMaxDistance;
apiResp1NumSamples[i] = apiResp.Model.Clusters[i].NumberOfSamples;
}
/// run with new data
runNum++;
// continue partial api run using second raw data (rawData2)
settings = new ClusteringSettings(maxCount, numClusters, numAttributes, KmeansAlgorithm: 2, initialCentroids: apiResp1Centroid);
// train
apiResp = api.Run() as KMeansScore;
Assert.True(apiResp.Model.Clusters != null);
Assert.True(apiResp.Model.Clusters.Length == clusterCenters.Length);
//// compare results
double f, res;
for (int i = 0; i < numClusters; i++)
{
// partial formula f*res
f = (double)1 / apiResp.Model.Clusters[i].NumberOfSamples;
for (int j = 0; j < numAttributes; j++)
{
res = apiResp1Centroid[i][j] * apiResp1NumSamples[i] + api2Resp.Model.Clusters[i].Centroid[j] * api2Resp.Model.Clusters[i].NumberOfSamples;
// partial centroid check
Assert.True(apiResp.Model.Clusters[i].Centroid[j] == f * res);
}
// max distance in cluster check
Assert.True(apiResp.Model.Clusters[i].InClusterMaxDistance >= apiResp1MaxDistance[i] + KMeansAlgorithm.calculateDistance(apiResp1Centroid[i], apiResp.Model.Clusters[i].Centroid));
}
}
public KmeansDataController()
{
if (numofclstrs == 0)
{
numofclstrs = 3;
}
int numberofclusters = 3;
/// desired number of clusters
int numberofattributes = 2;
/// X,Y,Z components
double[][] result = MockApi.MockApi.CSVtoDoubleJaggedArray(@"F:\ProData\data.csv");
/// CSV to Array Conversion
if (System.IO.File.Exists(@"F:\ProData\data.csv"))
{
numberofattributes = result[0].Length;
}
/// Number of attributes are 3 for 3D and 2 for 2D
IAnomalyDetectionApi X = new AnomalyDetectionApi.AnomalyDetectionAPI(result, numofclstrs);
ClusteringSettings Y = new ClusteringSettings(result, 10, numofclstrs, numberofattributes, @"F:\ProData\cluster.json", Replace: true);
/// Set number of clusters, max iterations and give data in result
X.ImportNewDataForClustering(Y);
/// CheckSample is a function that detect to which cluster the given sample belongs to.
ClusteringResults[] r;
/// All the Kmeans results needed for plotting
AnomalyDetectionResponse Z = X.GetResults(@"F:\ProData\cluster.json", out r);
/// Codes for Errors and Success
//if (PgaeLoadFlag == 1) //use this only for first time page load
//{
if (r != null) // in case result contains some value we need to add results
{
if (kmeansdata.IsValueCreated == true) //if already points are created we need to reinitiallize
{
checksample = kmeansdata.Value[0].CheckSampleResult;
kmeansdata = new Lazy <List <KmeansData> >();
}
int i = 0; //Counts number of Samples we have
for (int j = 0; j < numofclstrs; j++) //Runs for numberofclusters defined earlier
{
for (int k = 0; k < r[j].ClusterData.Length; k++) // add the data in web Api
{
if (numberofattributes == 3) // add data for 2D
{
kmeansdata.Value.Add(new KmeansData {
ID = i, ClustersDataLength = r[j].ClusterData.Length, TotalCentroid = r.Length, CentroidIdNumber = j, CentroidX = r[j].Centroid[0], CentroidY = r[j].Centroid[1], CentroidZ = r[j].Centroid[2], Xaxis = r[j].ClusterData[k][0], Yaxis = r[j].ClusterData[k][1], Zaxis = r[j].ClusterData[k][2], ClusterDataDistanceToCentroid = r[j].ClusterDataDistanceToCentroid[k], ClusterDataOriginalIndex = r[j].ClusterDataOriginalIndex[k], ClusterOfNearestForeignSample = r[j].ClusterOfNearestForeignSample, DistanceToNearestClusterCentroid = r[j].DistanceToNearestClusterCentroid, DistanceToNearestForeignSample = r[j].DistanceToNearestForeignSample, DistanceToNearestForeignSampleInNearestCluster = r[j].DistanceToNearestForeignSampleInNearestCluster, InClusterFarthestSampleX = r[j].InClusterFarthestSample[0], InClusterFarthestSampleY = r[j].InClusterFarthestSample[1], InClusterFarthestSampleZ = r[j].InClusterFarthestSample[2], InClusterFarthestSampleIndex = r[j].InClusterFarthestSampleIndex, InClusterMaxDistance = r[j].InClusterMaxDistance, MeanX = r[j].Mean[0], MeanY = r[j].Mean[1], MeanZ = r[j].Mean[2], NearestCluster = r[j].NearestCluster, NearestForeignSampleX = r[j].NearestForeignSample[0], NearestForeignSampleY = r[j].NearestForeignSample[1], NearestForeignSampleZ = r[j].NearestForeignSample[2], NearestForeignSampleInNearestClusterX = r[j].NearestForeignSampleInNearestCluster[0], NearestForeignSampleInNearestClusterY = r[j].NearestForeignSampleInNearestCluster[1], NearestForeignSampleInNearestClusterZ = r[j].NearestForeignSampleInNearestCluster[2], NumberOfAttribs = numberofattributes
});
}
if (numberofattributes == 2) // add data for 3D
{
kmeansdata.Value.Add(new KmeansData {
ID = i, ClustersDataLength = r[j].ClusterData.Length, TotalCentroid = r.Length, CentroidIdNumber = j, CentroidX = r[j].Centroid[0], CentroidY = r[j].Centroid[1], Xaxis = r[j].ClusterData[k][0], Yaxis = r[j].ClusterData[k][1], ClusterDataDistanceToCentroid = r[j].ClusterDataDistanceToCentroid[k], ClusterDataOriginalIndex = r[j].ClusterDataOriginalIndex[k], ClusterOfNearestForeignSample = r[j].ClusterOfNearestForeignSample, DistanceToNearestClusterCentroid = r[j].DistanceToNearestClusterCentroid, DistanceToNearestForeignSample = r[j].DistanceToNearestForeignSample, DistanceToNearestForeignSampleInNearestCluster = r[j].DistanceToNearestForeignSampleInNearestCluster, InClusterFarthestSampleX = r[j].InClusterFarthestSample[0], InClusterFarthestSampleY = r[j].InClusterFarthestSample[1], InClusterFarthestSampleIndex = r[j].InClusterFarthestSampleIndex, InClusterMaxDistance = r[j].InClusterMaxDistance, MeanX = r[j].Mean[0], MeanY = r[j].Mean[1], NearestCluster = r[j].NearestCluster, NearestForeignSampleX = r[j].NearestForeignSample[0], NearestForeignSampleY = r[j].NearestForeignSample[1], NearestForeignSampleInNearestClusterX = r[j].NearestForeignSampleInNearestCluster[0], NearestForeignSampleInNearestClusterY = r[j].NearestForeignSampleInNearestCluster[1], NumberOfAttribs = numberofattributes
});
}
i++;
}
}
// PgaeLoadFlag++;
//}
kmeansdata.Value[0].CheckSampleResult = checksample;
}
}
// POST api/product
/*
* public void KmeansDataAdd(KmeansData data) //post method
* {
*
* data.ID = ProductID;
* kmeansdata.Value.Add(data); //add the post product data to the product list
* ProductID++;
* //instead of adding product data to the static product list you can save data to the database.
* }
* /*/
public void KmeansDataCheck(KmeansData data) //post method
{
if (data.NumberOfClstrs != 0 && data.NumberOfClstrs != null)
{
numofclstrs = data.NumberOfClstrs;
// if (System.IO.File.Exists(@"F:\ProData\cluster.json"))// in case some files exist already delete them and upload the current one
// {
// System.IO.File.Delete(@"F:\ProData\cluster.json");
// System.IO.File.Delete(@"F:\ProData\Result\cluster.json");
// }
int numberofattributes = 2;
/// X,Y,Z components
double[][] result = MockApi.MockApi.CSVtoDoubleJaggedArray(@"F:\ProData\data.csv");
/// CSV to Array Conversion
if (System.IO.File.Exists(@"F:\ProData\data.csv"))
{
numberofattributes = result[0].Length;
}
/// Number of attributes are 3 for 3D and 2 for 2D
IAnomalyDetectionApi X = new AnomalyDetectionApi.AnomalyDetectionAPI(result, numofclstrs);
ClusteringSettings Y = new ClusteringSettings(result, 10, numofclstrs, numberofattributes, @"F:\ProData\cluster.json", Replace: true);
/// Set number of clusters, max iterations and give data in result
AnomalyDetectionResponse AB = X.ImportNewDataForClustering(Y);
/// CheckSample is a function that detect to which cluster the given sample belongs to.
ClusteringResults[] r;
/// All the Kmeans results needed for plotting
AnomalyDetectionResponse Z = X.GetResults(@"F:\ProData\cluster.json", out r);
/// Codes for Errors and Success
//if (PgaeLoadFlag == 1) //use this only for first time page load
//{
// if (r != null)// in case result contains some value we need to add results
// {
if (kmeansdata.IsValueCreated == true) //if already points are created we need to reinitiallize
{
checksample = kmeansdata.Value[0].CheckSampleResult;
kmeansdata = new Lazy <List <KmeansData> >();
}
int i = 0; //Counts number of Samples we have
for (int j = 0; j < numofclstrs; j++) //Runs for numberofclusters defined earlier
{
for (int k = 0; k < r[j].ClusterData.Length; k++) // add the data in web Api
{
if (numberofattributes == 3) // add data for 2D
{
kmeansdata.Value.Add(new KmeansData {
ID = i, ClustersDataLength = r[j].ClusterData.Length, TotalCentroid = r.Length, CentroidIdNumber = j, CentroidX = r[j].Centroid[0], CentroidY = r[j].Centroid[1], CentroidZ = r[j].Centroid[2], Xaxis = r[j].ClusterData[k][0], Yaxis = r[j].ClusterData[k][1], Zaxis = r[j].ClusterData[k][2], ClusterDataDistanceToCentroid = r[j].ClusterDataDistanceToCentroid[k], ClusterDataOriginalIndex = r[j].ClusterDataOriginalIndex[k], ClusterOfNearestForeignSample = r[j].ClusterOfNearestForeignSample, DistanceToNearestClusterCentroid = r[j].DistanceToNearestClusterCentroid, DistanceToNearestForeignSample = r[j].DistanceToNearestForeignSample, DistanceToNearestForeignSampleInNearestCluster = r[j].DistanceToNearestForeignSampleInNearestCluster, InClusterFarthestSampleX = r[j].InClusterFarthestSample[0], InClusterFarthestSampleY = r[j].InClusterFarthestSample[1], InClusterFarthestSampleZ = r[j].InClusterFarthestSample[2], InClusterFarthestSampleIndex = r[j].InClusterFarthestSampleIndex, InClusterMaxDistance = r[j].InClusterMaxDistance, MeanX = r[j].Mean[0], MeanY = r[j].Mean[1], MeanZ = r[j].Mean[2], NearestCluster = r[j].NearestCluster, NearestForeignSampleX = r[j].NearestForeignSample[0], NearestForeignSampleY = r[j].NearestForeignSample[1], NearestForeignSampleZ = r[j].NearestForeignSample[2], NearestForeignSampleInNearestClusterX = r[j].NearestForeignSampleInNearestCluster[0], NearestForeignSampleInNearestClusterY = r[j].NearestForeignSampleInNearestCluster[1], NearestForeignSampleInNearestClusterZ = r[j].NearestForeignSampleInNearestCluster[2], NumberOfAttribs = numberofattributes
});
}
if (numberofattributes == 2) // add data for 3D
{
kmeansdata.Value.Add(new KmeansData {
ID = i, ClustersDataLength = r[j].ClusterData.Length, TotalCentroid = r.Length, CentroidIdNumber = j, CentroidX = r[j].Centroid[0], CentroidY = r[j].Centroid[1], Xaxis = r[j].ClusterData[k][0], Yaxis = r[j].ClusterData[k][1], ClusterDataDistanceToCentroid = r[j].ClusterDataDistanceToCentroid[k], ClusterDataOriginalIndex = r[j].ClusterDataOriginalIndex[k], ClusterOfNearestForeignSample = r[j].ClusterOfNearestForeignSample, DistanceToNearestClusterCentroid = r[j].DistanceToNearestClusterCentroid, DistanceToNearestForeignSample = r[j].DistanceToNearestForeignSample, DistanceToNearestForeignSampleInNearestCluster = r[j].DistanceToNearestForeignSampleInNearestCluster, InClusterFarthestSampleX = r[j].InClusterFarthestSample[0], InClusterFarthestSampleY = r[j].InClusterFarthestSample[1], InClusterFarthestSampleIndex = r[j].InClusterFarthestSampleIndex, InClusterMaxDistance = r[j].InClusterMaxDistance, MeanX = r[j].Mean[0], MeanY = r[j].Mean[1], NearestCluster = r[j].NearestCluster, NearestForeignSampleX = r[j].NearestForeignSample[0], NearestForeignSampleY = r[j].NearestForeignSample[1], NearestForeignSampleInNearestClusterX = r[j].NearestForeignSampleInNearestCluster[0], NearestForeignSampleInNearestClusterY = r[j].NearestForeignSampleInNearestCluster[1], NumberOfAttribs = numberofattributes
});
}
i++;
}
}
// PgaeLoadFlag++;
//}
kmeansdata.Value[0].CheckSampleResult = checksample;
}
// }
else
{
numofclstrs = 3;
int numberofclusters = 3;
/// desired number of clusters
int numberofattributes = 3;
/// X,Y,Z components
double[][] result = MockApi.MockApi.CSVtoDoubleJaggedArray(@"F:\ProData\data.csv");
/// CSV to Array Conversion
if (System.IO.File.Exists(@"F:\ProData\data.csv"))
{
numberofattributes = result[0].Length;
}
if (numberofattributes == data.ClustersDataLength)
{
/// Number of attributes are 3 for 3D and 2 for 2D
IAnomalyDetectionApi X = new AnomalyDetectionApi.AnomalyDetectionAPI(result, numofclstrs);
ClusteringSettings Y = new ClusteringSettings(result, 10, numofclstrs, numberofattributes, @"F:\ProData\cluster.json");
/// Set number of clusters, max iterations and give data in result
X.ImportNewDataForClustering(Y);
/// CheckSample is a function that detect to which cluster the given sample belongs to.
ClusteringResults[] r;
/// All the Kmeans results needed for plotting
AnomalyDetectionResponse Z = X.GetResults(@"F:\ProData\cluster.json", out r);
if (numberofattributes == 3)
{
double[] sample = new double[3] {
data.XaxisCheck, data.YaxisCheck, data.ZaxisCheck
};
CheckingSampleSettings A = new CheckingSampleSettings(@"F:\ProData\cluster.json", sample, 10);
int N;
Z = X.CheckSample(A, out N);
data.CheckSampleResult = Z.Message;
kmeansdata.Value[0].CheckSampleResult = Z.Message;
}
if (numberofattributes == 2)
{
double[] sample = new double[2] {
data.XaxisCheck, data.YaxisCheck
};
CheckingSampleSettings A = new CheckingSampleSettings(@"F:\ProData\cluster.json", sample, 10);
int N;
Z = X.CheckSample(A, out N);
data.CheckSampleResult = Z.Message;
kmeansdata.Value[0].CheckSampleResult = Z.Message;
}
}
else
{
kmeansdata.Value[0].CheckSampleResult = "Your Data attributes are invalid ! Please Enter number of Cordinates according to the File Uploaded !";
}
}
//kmeansdata.Value.Add.();
//data.XaxisCheck = ProductID;
//kmeansdata.Value.Add(data); //add the post product data to the product list
//ProductID++;
//instead of adding product data to the static product list you can save data to the database.
}
public void GenerateAndTest()
{
// Settings to Generate Similar functions
string FunctionName = "Function0";
string directory = @"C:\Users\KywAnn\Desktop\Anomaly Detection Functions Normalized\" + FunctionName + "\\";
// number of similar functions
int NumSimFunc = 999;
// added noise level (distortion) between -NL & NL
int NL = 1;
// Settings for the K-Means Alg
int maxCount = 500;
int numClusters = 6;
int numAttributes = 2;
int KAlg = 2;
int Runs = 5;
// generate the similar functions
GenerateSimilarFunctions(directory + FunctionName + ".csv", NumSimFunc, NL);
// prepare the 2d functions for clustering
double[][] mFun = Helpers.cSVtoDoubleJaggedArray(directory + "\\NL" + NL + "\\" + FunctionName + " SimilarFunctions NL" + NL + ".csv");
// normalize the functions
mFun = NormalizeData(mFun);
// save the normalized similar functions
Helpers.Write2CSVFile(mFun, directory + "\\NL" + NL + "\\" + FunctionName + " SimilarFunctions Normalized NL" + NL + ".csv");
int NumFun = mFun.Length - 1;
ClusteringSettings clusterSettings;
AnomalyDetectionAPI kmeanApi;
AnomalyDetectionResponse response;
double[][] Centroids;
// original Centroids
double[][] oCentroids;
// matched Centroids
double[][] mCentroids;
for (int k = 2; k < numClusters + 1; k++)
{
oCentroids = new double[k][];
clusterSettings = new ClusteringSettings(maxCount, k, numAttributes, KmeansAlgorithm: KAlg, Replace: true);
kmeanApi = new AnomalyDetectionAPI(clusterSettings);
for (int j = 0; j < Runs; j++)
{
// save directory
string SavePath = directory + "NL" + NL + "\\" + FunctionName + " SimilarFunctions Centroids NL" + NL + " KA" + KAlg + " C" + k + " I" + maxCount + " R" + (j + 1) + ".csv";
for (int i = 0; i < NumFun; i++)
{
// cluster each function
double[][] rawData = getSimilarFunctionsData(mFun, i + 1);
response = kmeanApi.Training(rawData);
Centroids = kmeanApi.GetCentroid();
// match the centroids centroids
if (i == 0)
{
oCentroids = Centroids;
mCentroids = Centroids;
}
else
{
mCentroids = matchCentroids(Centroids, oCentroids);
}
// save centroids
if (i == 0)
{
// save or overwrite
Helpers.Write2CSVFile(mCentroids, SavePath);
}
else
{
// append
Helpers.Write2CSVFile(mCentroids, SavePath, true);
}
}
// save in a different format to plot results easily in excel
Special2DWrite2CSV(SavePath, k);
}
}
}
public void Test_FunctionRecognition()
{
int numCluster = 2;
// a value in % representing the tolerance to possible outliers
double tolerance = 0;
// directory to load
string loadDirectory = rootFolder + "Functions\\";
// directory to save
string saveDirectory = rootFolder + "Function Recognition\\";
//Assert.True(File.Exists("KMeans\\TestFiles\\TestFile01.csv"), "Expected file was not deployed to unit test foilder.");
// functions' paths
string[] FunctionPaths = new string[]
{
loadDirectory + "TestFile01\\NRP10\\TestFile01 SimilarFunctions Normalized Centroids NRP10 KA2 C" + numCluster + " I500 R1.csv",
loadDirectory + "TestFile02\\NRP10\\TestFile02 SimilarFunctions Normalized Centroids NRP10 KA2 C" + numCluster + " I500 R1.csv"
};
int numTrainFun = 800;
int numTestFun = 200;
Tuple <double[][], double[]> trainedClusters = formClusters(FunctionPaths[0], numCluster, numTrainFun);
// save the formed clusters
Helpers.Write2CSVFile(trainedClusters.Item1, saveDirectory + "Calculated Centroids.csv");
double[][] tempMaxDistance = new double[1][];
tempMaxDistance[0] = trainedClusters.Item2;
Helpers.Write2CSVFile(tempMaxDistance, saveDirectory + "Calculated Max Distance.csv");
// start testing for function recognition
// combine testing data
double[][] testingCentroids = new double[FunctionPaths.Length * numTestFun * numCluster][];
double[][] loadedCentroids;
int testingCentroidsOffset = numTrainFun * numCluster;
for (int i = 0; i < FunctionPaths.Length; i++)
{
loadedCentroids = Helpers.LoadFunctionData(FunctionPaths[i]);
for (int j = 0; j < numTestFun * numCluster; j++)
{
testingCentroids[i * numTestFun * numCluster + j] = loadedCentroids[j + testingCentroidsOffset];
}
// only needed for to avoid training centroids
testingCentroidsOffset = 0;
}
// save the testing centroids
Helpers.Write2CSVFile(testingCentroids, saveDirectory + "Testing Centroids.csv");
// check functions
//KMeans kMeans = new KMeans();
//kMeans.setTrivialClusters(numCluster, trainedClusters.Item1, trainedClusters.Item2);
// Creates learning api object
LearningApi api = new LearningApi();
api.UseActionModule <object, double[][]>((data, ctx) =>
{
return(null);
});
// basic settings for prediction
ClusteringSettings settings = new ClusteringSettings(0, numCluster, 0, tolerance: tolerance);
// construct trivial clusters
api.UseKMeans(settings, trainedClusters.Item1, trainedClusters.Item2);
double[] funResults = patternTesting(api, settings.NumberOfClusters, testingCentroids);
// save results
double[][] tempFunResults = new double[1][];
tempFunResults[0] = funResults;
Helpers.Write2CSVFile(tempFunResults, saveDirectory + "Results.csv");
}
public void Test_OptimalNumberOfCLustersBasic()
{
double[][] clusterCenters = new double[3][];
clusterCenters[0] = new double[] { 5.0, 5.0 };
clusterCenters[1] = new double[] { 15.0, 15.0 };
clusterCenters[2] = new double[] { 30.0, 30.0 };
string[] attributes = new string[] { "Height", "Weight" };
int numAttributes = attributes.Length; // 2 in this demo (height,weight)
int numClusters = 0; // vary this to experiment (must be between 2 and number data tuples)
int maxCount = 300; // trial and error
ClusteringSettings settings = new ClusteringSettings(maxCount, numClusters, numAttributes, KmeansAlgorithm: 2);
// Creates learning api object
LearningApi api = new LearningApi();
api.UseActionModule <object, double[][]>((data, ctx) =>
{
var rawData = Helpers.CreateSampleData(clusterCenters, 2, 10000, 0.5);
return(rawData);
});
api.UseKMeans(settings);
// train
var resp = api.Run() as KMeansScore;
Assert.True(resp.Model.NumberOfClusters > 1);
int points = 150;
var delta = 2 * Math.PI / 100;
List <double[]> rows = LearningFoundation.Helpers.FunctionGenerator.CreateFunction(points, 2, delta);
double[][] rawData2 = new double[points][];
for (int i = 0; i < points; i++)
{
rawData2[i] = new double[2];
for (int j = 0; j < 2; j++)
{
rawData2[i][j] = rows[j][i];
}
}
// Creates learning api object
LearningApi api2 = new LearningApi();
api2.UseActionModule <object, double[][]>((data, ctx) =>
{
return(rawData2);
});
api2.UseKMeans(settings);
// train
var resp2 = api2.Run() as KMeansScore;
Assert.True(resp2.Model.NumberOfClusters > 1);
}
