public List <String> GetModelResults(Version pVersion, VersionStatusModel pModel)
{
if (!String.IsNullOrWhiteSpace(pModel.ModelFileName))
{
//get dir
var dir = GetVersionModelsDirectory(pVersion);
//get results file (*.csv)
var modelsdir = System.IO.Path.Combine(dir.FullName, pModel.ModelFileName);
var file = new DirectoryInfo(modelsdir).GetFiles().FirstOrDefault(t => t.Name.Contains("results.csv"));
if (file != null)
{
return(System.IO.File.ReadAllLines(file.FullName).ToList());
}
}
return(null);
}
public List <VersionStatusModelMetricItem> EvaluateModel(Conductor_Shared.Version pVersion, VersionStatusModel pModel)
{
try
{
//check if version defines a special bucketing algorithm
int bucketing = BucketingType;
switch (pVersion.DatasetType)
{
case DatasetType.Generic:
bucketing = 1; break;
case DatasetType.Cargo2000:
bucketing = 2; break;
default:
bucketing = 1; break;
}
//get raw data
var data = fsManager.GetModelResults(pVersion, pModel);
if (data != null && data.Count > 1)
{
//generate lines
List <Line> lines = Line.GetLinesFromData(data, pVersion.DatasetType == DatasetType.Generic);
//generate buckets from lines
List <Bucket> buckets = Bucketing.CreateBuckets(BucketGranularity, BucketingType, lines);
List <VersionStatusModelMetricItem> metrics = new List <VersionStatusModelMetricItem>();
if (buckets.Any(t => !double.IsNaN(t.MCC)))
{
metrics.Add(new VersionStatusModelMetricItem()
{
Name = "MCC", Value = buckets.Where(t => !double.IsNaN(t.MCC)).Average(t => t.MCC)
});
}
if (buckets.Any(t => !double.IsNaN(t.Accuracy)))
{
metrics.Add(new VersionStatusModelMetricItem()
{
Name = "Accuracy", Value = buckets.Where(t => !double.IsNaN(t.Accuracy)).Average(t => t.Accuracy)
});
}
if (buckets.Any(t => !double.IsNaN(t.Precision)))
{
metrics.Add(new VersionStatusModelMetricItem()
{
Name = "Precision", Value = buckets.Where(t => !double.IsNaN(t.Precision)).Average(t => t.Precision)
});
}
if (buckets.Any(t => !double.IsNaN(t.Recall)))
{
metrics.Add(new VersionStatusModelMetricItem()
{
Name = "Recall", Value = buckets.Where(t => !double.IsNaN(t.Recall)).Average(t => t.Recall)
});
}
if (buckets.Any(t => !double.IsNaN(t.FMeasure)))
{
metrics.Add(new VersionStatusModelMetricItem()
{
Name = "FMetric", Value = buckets.Where(t => !double.IsNaN(t.FMeasure)).Average(t => t.FMeasure)
});
}
if (buckets.Any(t => !double.IsNaN(t.Specificity)))
{
metrics.Add(new VersionStatusModelMetricItem()
{
Name = "Speceficity", Value = buckets.Where(t => !double.IsNaN(t.Specificity)).Average(t => t.Specificity)
});
}
if (buckets.Any(t => !double.IsNaN(t.NegativePredictedValue)))
{
metrics.Add(new VersionStatusModelMetricItem()
{
Name = "Negative Predictions", Value = buckets.Where(t => !double.IsNaN(t.NegativePredictedValue)).Average(t => t.NegativePredictedValue)
});
}
if (buckets.Any(t => !double.IsNaN(t.FalsePositiveRate)))
{
metrics.Add(new VersionStatusModelMetricItem()
{
Name = "False Positive Rate", Value = buckets.Where(t => !double.IsNaN(t.FalsePositiveRate)).Average(t => t.FalsePositiveRate)
});
}
if (buckets.Any(t => !double.IsNaN(t.MAE)))
{
metrics.Add(new VersionStatusModelMetricItem()
{
Name = "MAE", Value = buckets.Where(t => !double.IsNaN(t.MAE)).Average(t => t.MAE)
});
}
if (buckets.Any(t => !double.IsNaN(t.MSE)))
{
metrics.Add(new VersionStatusModelMetricItem()
{
Name = "MSE", Value = buckets.Where(t => !double.IsNaN(t.MSE)).Average(t => t.MSE)
});
}
if (buckets.Any(t => !double.IsNaN(t.RMSE)))
{
metrics.Add(new VersionStatusModelMetricItem()
{
Name = "RMSE", Value = buckets.Where(t => !double.IsNaN(t.RMSE)).Average(t => t.RMSE)
});
}
if (buckets.Any(t => !double.IsNaN(t.RAE)))
{
metrics.Add(new VersionStatusModelMetricItem()
{
Name = "RAE", Value = buckets.Where(t => !double.IsNaN(t.RAE)).Average(t => t.RAE)
});
}
if (buckets.Any(t => !double.IsNaN(t.RSE)))
{
metrics.Add(new VersionStatusModelMetricItem()
{
Name = "RSE", Value = buckets.Where(t => !double.IsNaN(t.RSE)).Average(t => t.RSE)
});
}
if (buckets.Any(t => !double.IsNaN(t.RRSE)))
{
metrics.Add(new VersionStatusModelMetricItem()
{
Name = "RRSE", Value = buckets.Where(t => !double.IsNaN(t.RRSE)).Average(t => t.RRSE)
});
}
return(metrics);
}
}
catch (Exception ex)
{
NotifyNewLogMessageEvent($"something went wrong while evaluating {pVersion.ToString()} {pModel.ModelFileName}: {ex.Message}");
}
return(null);
}
