private static void PerformanceEvaluation()
{
var modelDirectory = Model.GetModelDirectory();
var modelFiles = Directory.GetFiles(modelDirectory).Where(file => !file.EndsWith("_cscc") && !file.Equals("training"));
var keys = ContextModel <ContextInfo> .Load(Path.Combine(modelDirectory, "training")).GetAllTypes();
var models = new List <ContextModel <ContextInfo> >();
foreach (var modelFile in modelFiles)
{
var model = ContextModel <ContextInfo> .Load(modelFile);
model.KeepTypes(keys);
model.RemoveDuplicates();
models.Add(model);
}
Console.WriteLine("Models loaded...");
var precision = 0.0;
var recall = 0.0;
for (int i = 0; i < models.Count; i++)
{
var trainingFolds = models.Where((foldIndices, foldIndex) => foldIndex != i).ToArray();
var trainingModel = ContextModel <ContextInfo> .Combine(trainingFolds);
trainingModel.RemoveDuplicates();
Console.WriteLine("Training model created...");
var validationFold = models[i].Contexts.SelectMany(kvp => kvp.Value.Select(ci => new Tuple <string, ContextInfo>(kvp.Key, ci))).Take(1000);
var cscc = new GeneCSCC.GeneCSCC(trainingModel);
var validationHits = 0.0;
var recallHits = 0.0;
var validations = 0;
foreach (var validation in validationFold)
{
if (!trainingModel.Contexts.ContainsKey(validation.Item1))
{
continue;
}
var predictions = cscc.GetPredictions(validation.Item2, validation.Item1);
validations++;
if (predictions.Count == 0)
{
continue;
}
recallHits++;
if (validation.Item2.Invocation.Equals(predictions[0]))
{
validationHits++;
}
}
precision += validationHits / validations;
recall += recallHits / validations;
Console.WriteLine(validationHits / validations);
}
Console.WriteLine("Precision: {0} Recall: {1}", precision / models.Count, recall / models.Count);
Console.ReadKey();
}
