public static void EvaluateGeneCscc()
{
var model = ContextModel <ContextInfo> .Load(Path.Combine(Model.GetModelDirectory(), "training"));
var cscc = new GeneCSCC.GeneCSCC(model);
var rand = new Random();
var unorderedQueryData = model.Contexts.SelectMany(kvp => kvp.Value.Select(c => new Tuple <string, ContextInfo>(kvp.Key, c))).ToList();
unorderedQueryData.Shuffle(RandomProvider.GetThreadRandom());
var queryData = unorderedQueryData.Take(3000);
var sw = new Stopwatch();
sw.Start();
foreach (var query in queryData)
{
cscc.GetPredictions(query.Item2, query.Item1);
}
sw.Stop();
Console.WriteLine(
$"Queries: {model.Contexts.Sum(kvp => kvp.Value.Count)} Inference speed: {(double) sw.Elapsed.Milliseconds/3000}");
}
private static void TestGene()
{
var training = ContextModel <ContextInfo> .Load(Path.Combine(Model.GetModelDirectory(), "Source"));
var model = new GeneCSCC.GeneCSCC(training);
var validation = ContextModel <ContextInfo> .Load(Path.Combine(Model.GetModelDirectory(), "NewtonsoftJson-master")).Contexts;
var list = validation.SelectMany(kvp => kvp.Value.Select(ci => new Tuple <string, ContextInfo>(kvp.Key, ci))).ToList();
var validationError = 0.0;
var validations = 0;
for (var i = 0; i < validation.Count; i++)
{
if (!training.Contexts.ContainsKey(list[i].Item1))
{
continue;
}
var predictions = model.GetPredictions(list[i].Item2, list[i].Item1);
validations++;
if (predictions.Count == 0)
{
continue;
}
if (list[i].Item2.Invocation.Equals(predictions[0]))
{
validationError++;
}
}
Console.WriteLine(validationError / validations);
Console.ReadKey();
}
private static void PerformanceEvaluation()
{
var modelDirectory = Model.GetModelDirectory();
var modelFiles = Directory.GetFiles(modelDirectory).Where(file => !file.EndsWith("_cscc") && !file.Equals("training"));
var keys = ContextModel <ContextInfo> .Load(Path.Combine(modelDirectory, "training")).GetAllTypes();
var models = new List <ContextModel <ContextInfo> >();
foreach (var modelFile in modelFiles)
{
var model = ContextModel <ContextInfo> .Load(modelFile);
model.KeepTypes(keys);
model.RemoveDuplicates();
models.Add(model);
}
Console.WriteLine("Models loaded...");
var precision = 0.0;
var recall = 0.0;
for (int i = 0; i < models.Count; i++)
{
var trainingFolds = models.Where((foldIndices, foldIndex) => foldIndex != i).ToArray();
var trainingModel = ContextModel <ContextInfo> .Combine(trainingFolds);
trainingModel.RemoveDuplicates();
Console.WriteLine("Training model created...");
var validationFold = models[i].Contexts.SelectMany(kvp => kvp.Value.Select(ci => new Tuple <string, ContextInfo>(kvp.Key, ci))).Take(1000);
var cscc = new GeneCSCC.GeneCSCC(trainingModel);
var validationHits = 0.0;
var recallHits = 0.0;
var validations = 0;
foreach (var validation in validationFold)
{
if (!trainingModel.Contexts.ContainsKey(validation.Item1))
{
continue;
}
var predictions = cscc.GetPredictions(validation.Item2, validation.Item1);
validations++;
if (predictions.Count == 0)
{
continue;
}
recallHits++;
if (validation.Item2.Invocation.Equals(predictions[0]))
{
validationHits++;
}
}
precision += validationHits / validations;
recall += recallHits / validations;
Console.WriteLine(validationHits / validations);
}
Console.WriteLine("Precision: {0} Recall: {1}", precision / models.Count, recall / models.Count);
Console.ReadKey();
}