public static void EvaluateCscc()
{
var model = ContextModel <CsccContextInfo> .Load(Path.Combine(Model.GetModelDirectory(), "training_cscc"));
var cscc = new CSCC.CSCC(model);
var rand = new Random();
var unorderedQueryData = model.Contexts.SelectMany(kvp => kvp.Value.Select(c => new Tuple <string, CsccContextInfo>(kvp.Key, c))).ToList();
unorderedQueryData.Shuffle(RandomProvider.GetThreadRandom());
var queryData = unorderedQueryData.Take(3000);
foreach (var query in queryData)
{
cscc.GetPredictions(query.Item2, query.Item1);
}
var sw = new Stopwatch();
sw.Start();
foreach (var query in queryData)
{
cscc.GetPredictions(query.Item2, query.Item1);
}
sw.Stop();
Console.WriteLine(string.Format("Queries: {0} Inference speed: {1}",
model.Contexts.Sum(kvp => kvp.Value.Count), (double)sw.Elapsed.Milliseconds / 3000));
}
public CrossValidationResult Evaluate()
{
var crossvalidation = new KFoldCrossValidation(_inputs.Count, 10);
crossvalidation.Evaluation = delegate(int k, int[] indicesTrain, int[] indicesValidation)
{
var trainingInputs = _inputs.SubArray(indicesTrain);
var trainingModel = new Dictionary <string, List <CsccContextInfo> >();
foreach (var trainingInput in trainingInputs)
{
trainingModel.ChainedAdd(trainingInput.Item1, trainingInput.Item2);
}
var validationInputs = _inputs.SubArray(indicesValidation);
var validationOutputs = _outputs.SubArray(indicesValidation);
var cscc = new CSCC(new ContextModel <CsccContextInfo>(trainingModel));
var top1Matches = 0.0;
var top3Matches = 0.0;
var predictionsMade = 0.0;
/*var sw = new Stopwatch();
* sw.Start();*/
for (var i = 0; i < validationInputs.Length; i++)
{
var predictions = cscc.GetPredictions(validationInputs[i].Item2, validationInputs[i].Item1);
if (predictions.Count == 0)
{
continue;
}
predictionsMade++;
if (validationOutputs[i].Validate(predictions[0]) == ValidationInfo.Result.Match)
{
top1Matches++;
top3Matches++;
}
else if (predictions.Count > 1 && validationOutputs[i].Validate(predictions[1]) == ValidationInfo.Result.Match)
{
top3Matches++;
}
else if (predictions.Count > 2 && validationOutputs[i].Validate(predictions[2]) == ValidationInfo.Result.Match)
{
top3Matches++;
}
}
//sw.Stop();
//Console.WriteLine("Elapsed time: " + sw.ElapsedMilliseconds / validationInputs.Length);
return(new PredictionQualityValues(top3Matches / validationInputs.Length, predictionsMade / validationInputs.Length));
};
var predictionQualities = crossvalidation.Compute();
return(predictionQualities);
}
/*private static void TestCV()
* {
* var model = new TrainingModel<CsccContextInfo>(GetContexts(@"C:\Users\M\Desktop\CSharp_Source\CSharp_Source\roslyn-master"));
* var trainingModel = new EvaluationModel(model.Contexts);
* Console.WriteLine(trainingModel.Evaluate());
* Console.ReadKey();
* }*/
private static void Test()
{
var training = ContextModel <CsccContextInfo> .Load(Path.Combine(Model.GetModelDirectory(), "Source_cscc"));
var model = new CSCC.CSCC(training);
var validation = ContextModel <CsccContextInfo> .Load(Path.Combine(Model.GetModelDirectory(), "NewtonsoftJson-master_cscc")).Contexts;
var list = validation.SelectMany(kvp => kvp.Value.Select(ci => new Tuple <string, CsccContextInfo>(kvp.Key, ci))).ToList();
var validationError = 0.0;
var validations = 0;
for (var i = 0; i < validation.Count; i++)
{
if (!training.Contexts.ContainsKey(list[i].Item1))
{
continue;
}
var predictions = model.GetPredictions(list[i].Item2, list[i].Item1);
validations++;
if (predictions.Count == 0)
{
continue;
}
if (list[i].Item2.Invocation.Equals(predictions[0]))
{
validationError++;
}
}
Console.WriteLine(validationError / validations);
Console.ReadKey();
}