public DirectoryInfo GetVersionErrorlogsDirectory(Version pVersion)
{
var path = System.IO.Path.Combine(AssetsPath, pVersion.ToString(), "errorlogs");
CreateIfNotExists(path);
return(new DirectoryInfo(path));
}
//604.800.000 = 7d, 86.400.000s = 24h; 43.200.000s = 12h; 1.200.000s = 20m
public WorkItem(Version pVersion, List <File> pFiles, WorkType pWorkType, Guid pGuid)
{
//specify default values
this.Version = pVersion;
this.WorkType = pWorkType;
StartDate = DateTime.MinValue;
WorkItemID = pGuid;
Files = pFiles;
}
public void SaveVersionToFile(Version pVersion)
{
XmlSerializer serializer = new XmlSerializer(typeof(Conductor_Shared.Version));
var path = GetVersionRootDirectory(pVersion).FullName;
using (System.IO.FileStream stream = new System.IO.FileStream(Path.Combine(path, "version.xml"), System.IO.FileMode.Create))
{
serializer.Serialize(stream, pVersion);
stream.Close();
}
}
public bool CheckIfFileExists(Version pVersion, String pGUID)
{
//check directory
var dir = System.IO.Path.Combine(AssetsPath, pVersion.ToString(), "models", pGUID);
if (!System.IO.Directory.Exists(dir))
{
return(false);
}
//check if file exists
return(new DirectoryInfo(dir).GetFiles().Any(t => t.Extension == "h5"));
}
public void WriteErrorlogToFilesystem(Version pVersion, List <String> pLogmessage, Guid pErrorguid)
{
var path = this.GetVersionErrorlogsDirectory(pVersion).FullName;
//create up to 10 error files per guid
for (int i = 0; i < 10; i++)
{
var filepath = System.IO.Path.Combine(path, $"{pErrorguid.ToString()}-{i}.log");
if (!System.IO.File.Exists(filepath))
{
//write if file not exist
System.IO.File.WriteAllLines(filepath, pLogmessage);
break;
}
}
}
public List <String> GetModelResults(Version pVersion, VersionStatusModel pModel)
{
if (!String.IsNullOrWhiteSpace(pModel.ModelFileName))
{
//get dir
var dir = GetVersionModelsDirectory(pVersion);
//get results file (*.csv)
var modelsdir = System.IO.Path.Combine(dir.FullName, pModel.ModelFileName);
var file = new DirectoryInfo(modelsdir).GetFiles().FirstOrDefault(t => t.Name.Contains("results.csv"));
if (file != null)
{
return(System.IO.File.ReadAllLines(file.FullName).ToList());
}
}
return(null);
}
public List <VersionStatusModelMetricItem> EvaluateModel(Conductor_Shared.Version pVersion, VersionStatusModel pModel)
{
try
{
//check if version defines a special bucketing algorithm
int bucketing = BucketingType;
switch (pVersion.DatasetType)
{
case DatasetType.Generic:
bucketing = 1; break;
case DatasetType.Cargo2000:
bucketing = 2; break;
default:
bucketing = 1; break;
}
//get raw data
var data = fsManager.GetModelResults(pVersion, pModel);
if (data != null && data.Count > 1)
{
//generate lines
List <Line> lines = Line.GetLinesFromData(data, pVersion.DatasetType == DatasetType.Generic);
//generate buckets from lines
List <Bucket> buckets = Bucketing.CreateBuckets(BucketGranularity, BucketingType, lines);
List <VersionStatusModelMetricItem> metrics = new List <VersionStatusModelMetricItem>();
if (buckets.Any(t => !double.IsNaN(t.MCC)))
{
metrics.Add(new VersionStatusModelMetricItem()
{
Name = "MCC", Value = buckets.Where(t => !double.IsNaN(t.MCC)).Average(t => t.MCC)
});
}
if (buckets.Any(t => !double.IsNaN(t.Accuracy)))
{
metrics.Add(new VersionStatusModelMetricItem()
{
Name = "Accuracy", Value = buckets.Where(t => !double.IsNaN(t.Accuracy)).Average(t => t.Accuracy)
});
}
if (buckets.Any(t => !double.IsNaN(t.Precision)))
{
metrics.Add(new VersionStatusModelMetricItem()
{
Name = "Precision", Value = buckets.Where(t => !double.IsNaN(t.Precision)).Average(t => t.Precision)
});
}
if (buckets.Any(t => !double.IsNaN(t.Recall)))
{
metrics.Add(new VersionStatusModelMetricItem()
{
Name = "Recall", Value = buckets.Where(t => !double.IsNaN(t.Recall)).Average(t => t.Recall)
});
}
if (buckets.Any(t => !double.IsNaN(t.FMeasure)))
{
metrics.Add(new VersionStatusModelMetricItem()
{
Name = "FMetric", Value = buckets.Where(t => !double.IsNaN(t.FMeasure)).Average(t => t.FMeasure)
});
}
if (buckets.Any(t => !double.IsNaN(t.Specificity)))
{
metrics.Add(new VersionStatusModelMetricItem()
{
Name = "Speceficity", Value = buckets.Where(t => !double.IsNaN(t.Specificity)).Average(t => t.Specificity)
});
}
if (buckets.Any(t => !double.IsNaN(t.NegativePredictedValue)))
{
metrics.Add(new VersionStatusModelMetricItem()
{
Name = "Negative Predictions", Value = buckets.Where(t => !double.IsNaN(t.NegativePredictedValue)).Average(t => t.NegativePredictedValue)
});
}
if (buckets.Any(t => !double.IsNaN(t.FalsePositiveRate)))
{
metrics.Add(new VersionStatusModelMetricItem()
{
Name = "False Positive Rate", Value = buckets.Where(t => !double.IsNaN(t.FalsePositiveRate)).Average(t => t.FalsePositiveRate)
});
}
if (buckets.Any(t => !double.IsNaN(t.MAE)))
{
metrics.Add(new VersionStatusModelMetricItem()
{
Name = "MAE", Value = buckets.Where(t => !double.IsNaN(t.MAE)).Average(t => t.MAE)
});
}
if (buckets.Any(t => !double.IsNaN(t.MSE)))
{
metrics.Add(new VersionStatusModelMetricItem()
{
Name = "MSE", Value = buckets.Where(t => !double.IsNaN(t.MSE)).Average(t => t.MSE)
});
}
if (buckets.Any(t => !double.IsNaN(t.RMSE)))
{
metrics.Add(new VersionStatusModelMetricItem()
{
Name = "RMSE", Value = buckets.Where(t => !double.IsNaN(t.RMSE)).Average(t => t.RMSE)
});
}
if (buckets.Any(t => !double.IsNaN(t.RAE)))
{
metrics.Add(new VersionStatusModelMetricItem()
{
Name = "RAE", Value = buckets.Where(t => !double.IsNaN(t.RAE)).Average(t => t.RAE)
});
}
if (buckets.Any(t => !double.IsNaN(t.RSE)))
{
metrics.Add(new VersionStatusModelMetricItem()
{
Name = "RSE", Value = buckets.Where(t => !double.IsNaN(t.RSE)).Average(t => t.RSE)
});
}
if (buckets.Any(t => !double.IsNaN(t.RRSE)))
{
metrics.Add(new VersionStatusModelMetricItem()
{
Name = "RRSE", Value = buckets.Where(t => !double.IsNaN(t.RRSE)).Average(t => t.RRSE)
});
}
return(metrics);
}
}
catch (Exception ex)
{
NotifyNewLogMessageEvent($"something went wrong while evaluating {pVersion.ToString()} {pModel.ModelFileName}: {ex.Message}");
}
return(null);
}
public bool CheckIfResultExists(Version pVersion, String pGUID)
{
return(_filesystemManager.CheckIfFileExists(pVersion, pGUID));
}
