public static string FindConcurrentDataXMLfile(string ImageFilename, string basePath = "",
bool searchFilesRecursively = true,
ServiceTools.DatetimeExtractionMethod method = ServiceTools.DatetimeExtractionMethod.Filename)
{
if (basePath == "")
{
basePath = Directory.GetCurrentDirectory();
basePath = basePath +
((basePath.Last() == Path.DirectorySeparatorChar)
? ("")
: (Path.DirectorySeparatorChar.ToString()));
basePath += "results" + Path.DirectorySeparatorChar;
}
string nearestConcurrentDataXMLfilename = "";
string currImgFilename = ImageFilename;
ServiceTools.DatetimeExtractionMethod _method = method;
#region find most close concurrent data XML file
List <FileDatetimeInfo> lImagesConcurrentDataFilesInfo = new List <FileDatetimeInfo>();
List <string> filesListConcurrentData =
Directory.EnumerateFiles(basePath,
ConventionalTransitions.ImageConcurrentDataFilesNamesPattern(),
(searchFilesRecursively) ? (SearchOption.AllDirectories) : (SearchOption.TopDirectoryOnly))
.ToList();
foreach (string strConcDataXMLFile in filesListConcurrentData)
{
try
{
lImagesConcurrentDataFilesInfo.Add(new FileDatetimeInfo(strConcDataXMLFile,
_method));
}
catch (Exception ex)
{
continue;
}
}
string currImgFilenameWOext = Path.GetFileNameWithoutExtension(currImgFilename);
string ptrn = @"(devID\d)";
Regex rgxp = new Regex(ptrn, RegexOptions.IgnoreCase);
string strCurrImgDT = rgxp.Replace(currImgFilenameWOext.Substring(4), "");
//2015-12-16T06-01-38
strCurrImgDT = strCurrImgDT.Substring(0, 11) + strCurrImgDT.Substring(11).Replace("-", ":");
DateTime currImgDT = DateTime.Parse(strCurrImgDT, null,
DateTimeStyles.AdjustToUniversal);
DateTime.SpecifyKind(currImgDT, DateTimeKind.Utc);
FileDatetimeInfo nearestConcurrentDataXMLfileInfo = lImagesConcurrentDataFilesInfo.Aggregate((cDtFinfo1, cDtFinfo2) =>
{
TimeSpan tspan1 = new TimeSpan(Math.Abs((cDtFinfo1.datetime - currImgDT).Ticks));
TimeSpan tspan2 = new TimeSpan(Math.Abs((cDtFinfo2.datetime - currImgDT).Ticks));
return((tspan1 <= tspan2) ? (cDtFinfo1) : (cDtFinfo2));
});
if (new TimeSpan(Math.Abs((nearestConcurrentDataXMLfileInfo.datetime - currImgDT).Ticks)) >=
new TimeSpan(0, 3, 0))
{
//theLogWindow = ServiceTools.LogAText(theLogWindow,
// "couldn`t find close enough concurrent data file for image:" + Environment.NewLine +
// currImgFilename + Environment.NewLine + "closest concurrent data file is:" +
// Environment.NewLine + nearestConcurrentDataXMLfileInfo.filename + Environment.NewLine +
// "with date-time value " + nearestConcurrentDataXMLfileInfo.datetime.ToString("o"));
nearestConcurrentDataXMLfilename = "";
}
else
{
nearestConcurrentDataXMLfilename = nearestConcurrentDataXMLfileInfo.filename;
}
#endregion find most close concurrent data
return(nearestConcurrentDataXMLfilename);
}
public async Task<SunDiskCondition> CalcSDC_NN(string SDC_NNconfigFile, string SDC_NNtrainedParametersFile, string NormMeansFile, string NormRangeFile)
{
if (!File.Exists(SDC_NNconfigFile))
{
throw new FileNotFoundException("couldn`t find the file specified: " + SDC_NNconfigFile);
}
if (!File.Exists(SDC_NNtrainedParametersFile))
{
throw new FileNotFoundException("couldn`t find the file specified: " + SDC_NNtrainedParametersFile);
}
if (!File.Exists(NormMeansFile))
{
throw new FileNotFoundException("couldn`t find the file specified: " + NormMeansFile);
}
if (!File.Exists(NormRangeFile))
{
throw new FileNotFoundException("couldn`t find the file specified: " + NormRangeFile);
}
if ((ImagesRoundMasksXMLfilesMappingList == "") || (ImagesRoundMasksXMLfilesMappingList == null))
{
ImagesRoundMasksXMLfilesMappingList = Directory.GetCurrentDirectory() + Path.DirectorySeparatorChar + "settings" +
Path.DirectorySeparatorChar +
"ImagesRoundMasksXMLfilesMappingList.csv";
}
if (!File.Exists(ImagesRoundMasksXMLfilesMappingList))
{
throw new FileNotFoundException("couldn`t find the file specified: " + ImagesRoundMasksXMLfilesMappingList);
}
#region read or calculate GrIxYRGB stats
if (Directory.Exists(imageYRGBstatsXMLdataFilesDirectory))
{
List<string> foundXMLfiles = Directory.EnumerateFiles(imageYRGBstatsXMLdataFilesDirectory,
ConventionalTransitions.ImageGrIxYRGBstatsDataFileName(ImageFilename, "", false),
SearchOption.AllDirectories).ToList();
if (foundXMLfiles.Any())
{
currImageStatsData =
(SkyImageIndexesStatsData)
ServiceTools.ReadObjectFromXML(foundXMLfiles[0], typeof(SkyImageIndexesStatsData));
}
}
if (currImageStatsData == null)
{
Task<SkyImageIndexesStatsData> tskImageStatsCalculation = new Task<SkyImageIndexesStatsData>(() =>
{
Dictionary<string, object> optionalParameters = new Dictionary<string, object>();
optionalParameters.Add("ImagesRoundMasksXMLfilesMappingList", ImagesRoundMasksXMLfilesMappingList);
Stopwatch sw = new Stopwatch();
sw.Start();
optionalParameters.Add("Stopwatch", sw);
ImageStatsDataCalculationResult currImageProcessingResult = null;
try
{
currImageProcessingResult = ImageProcessing.CalculateImageStatsData(ImageFilename,
optionalParameters);
}
catch (Exception ex)
{
return null;
}
if (currImageProcessingResult == null)
{
return null;
}
else
{
string strImageGrIxYRGBDataFileName =
ConventionalTransitions.ImageGrIxYRGBstatsDataFileName(ImageFilename,
imageYRGBstatsXMLdataFilesDirectory);
ServiceTools.WriteObjectToXML(currImageProcessingResult.grixyrgbStatsData,
strImageGrIxYRGBDataFileName);
// currImageProcessingResult.stopwatch.Stop();
// string currentFullFileName = currImageProcessingResult.imgFilename;
// string strPerfCountersData = currentFullFileName + ";" +
// currImageProcessingResult.stopwatch.ElapsedMilliseconds + ";" +
// (currImageProcessingResult.procTotalProcessorTimeEnd -
// currImageProcessingResult.procTotalProcessorTimeStart)
// .TotalMilliseconds +
// Environment.NewLine;
// ServiceTools.logToTextFile(strPerformanceCountersStatsFile, strPerfCountersData, true);
currImageStatsData = currImageProcessingResult.grixyrgbStatsData;
return currImageProcessingResult.grixyrgbStatsData;
}
});
currImageStatsData = await tskImageStatsCalculation;
}
if (currImageStatsData == null)
{
// theLogWindow = ServiceTools.LogAText(theLogWindow, "ERROR searching concurrent data for image. Will not proceed.");
throw new Exception("ERROR searching concurrent data for image. Will not proceed.");
}
#endregion read or calculate GrIxYRGB stats
ConcurrentData nearestConcurrentData = null;
#region search for concurrent data
Task<ConcurrentData> tskImageConcurrentDataSearching = new Task<ConcurrentData>(() =>
{
string currImgFilename = Path.GetFileNameWithoutExtension(ImageFilename);
string ptrn = @"(devID\d)";
Regex rgxp = new Regex(ptrn, RegexOptions.IgnoreCase);
string strCurrImgDT = rgxp.Replace(currImgFilename.Substring(4), "");
//2015-12-16T06-01-38
strCurrImgDT = strCurrImgDT.Substring(0, 11) + strCurrImgDT.Substring(11).Replace("-", ":");
DateTime currImgDT = DateTime.Parse(strCurrImgDT, null,
System.Globalization.DateTimeStyles.AdjustToUniversal);
List<Tuple<string, ConcurrentData>> lImagesConcurrentData =
new List<Tuple<string, ConcurrentData>>();
List<string> filesListConcurrentData =
Directory.EnumerateFiles(ConcurrentDataXMLfilesBasePath,
ConventionalTransitions.ImageConcurrentDataFilesNamesPattern(),
SearchOption.AllDirectories)
.ToList();
List<Tuple<string, DateTime>> XMLfilesListConcurrentData = filesListConcurrentData.ConvertAll(
strXMLfilename =>
{
string xmlFile1DateTimeString =
Path.GetFileNameWithoutExtension(strXMLfilename).Replace("data-", "").Substring(0, 19);
xmlFile1DateTimeString = xmlFile1DateTimeString.Substring(0, 11) + xmlFile1DateTimeString.Substring(11).Replace("-", ":");
DateTime dt1 = DateTime.Parse(xmlFile1DateTimeString);
return new Tuple<string, DateTime>(strXMLfilename, dt1);
});
string nearestConcurrentDataFileName = XMLfilesListConcurrentData.Aggregate((cDt1, cDt2) =>
{
TimeSpan tspan1 = new TimeSpan(Math.Abs((cDt1.Item2 - currImgDT).Ticks));
TimeSpan tspan2 = new TimeSpan(Math.Abs((cDt2.Item2 - currImgDT).Ticks));
return ((tspan1 <= tspan2) ? (cDt1) : (cDt2));
}).Item1;
Dictionary<string, object> currDict = ServiceTools.ReadDictionaryFromXML(nearestConcurrentDataFileName);
currDict.Add("XMLfileName", Path.GetFileName(nearestConcurrentDataFileName));
ConcurrentData nearestConcurrentDataObtained = new ConcurrentData(currDict);
if (new TimeSpan(Math.Abs((nearestConcurrentDataObtained.datetimeUTC - currImgDT).Ticks)) >=
new TimeSpan(0, 2, 0))
{
//theLogWindow = ServiceTools.LogAText(theLogWindow,
// "couldn`t find close enough concurrent data file for image:" + Environment.NewLine +
// bgwCurrImageFInfo.FullName + Environment.NewLine + "closest concurrent data file is:" +
// Environment.NewLine + nearestConcurrentData.filename + Environment.NewLine +
// "with date-time value " + nearestConcurrentData.datetimeUTC.ToString("o"));
nearestConcurrentDataObtained = null;
}
return nearestConcurrentDataObtained;
});
nearestConcurrentData = await tskImageConcurrentDataSearching;
if (nearestConcurrentData == null)
{
// theLogWindow = ServiceTools.LogAText(theLogWindow, "ERROR searching concurrent data for image. Will not proceed.");
throw new Exception("ERROR searching concurrent data for image. Will not proceed.");
}
#endregion search for concurrent data
DenseVector dvMeans = (DenseVector)((DenseMatrix)ServiceTools.ReadDataFromCSV(NormMeansFile, 0, ",")).Row(0);
DenseVector dvRanges = (DenseVector)((DenseMatrix)ServiceTools.ReadDataFromCSV(NormRangeFile, 0, ",")).Row(0);
DenseVector dvThetaValues = (DenseVector)ServiceTools.ReadDataFromCSV(SDC_NNtrainedParametersFile, 0, ",");
List<int> NNlayersConfig =
new List<double>(((DenseMatrix)ServiceTools.ReadDataFromCSV(SDC_NNconfigFile, 0, ",")).Row(0)).ConvertAll
(dVal => Convert.ToInt32(dVal));
List<double> decisionProbabilities = new List<double>();
return PredictSDC_NN(currImageStatsData, nearestConcurrentData, NNlayersConfig, dvThetaValues, dvMeans,
dvRanges, out decisionProbabilities);
}
public async Task <int> CalcTCC_NN(string SDC_NNconfigFile, string SDC_NNtrainedParametersFile,
string SDC_NormMeansFile,
string SDC_NormRangeFile, string TCC_NNconfigFile, string TCC_NNtrainedParametersFile,
string TCC_NormMeansFile, string TCC_NormRangeFile, string TCC_ExcludingVarsFile)
{
#region check files existence
if (!File.Exists(SDC_NNconfigFile))
{
throw new FileNotFoundException("couldn`t find the file specified: " + SDC_NNconfigFile);
}
if (!File.Exists(SDC_NNtrainedParametersFile))
{
throw new FileNotFoundException("couldn`t find the file specified: " + SDC_NNtrainedParametersFile);
}
if (!File.Exists(SDC_NormMeansFile))
{
throw new FileNotFoundException("couldn`t find the file specified: " + SDC_NormMeansFile);
}
if (!File.Exists(SDC_NormRangeFile))
{
throw new FileNotFoundException("couldn`t find the file specified: " + SDC_NormRangeFile);
}
if (!File.Exists(TCC_NNconfigFile))
{
throw new FileNotFoundException("couldn`t find the file specified: " + TCC_NNconfigFile);
}
if (!File.Exists(TCC_NNtrainedParametersFile))
{
throw new FileNotFoundException("couldn`t find the file specified: " + TCC_NNtrainedParametersFile);
}
if (!File.Exists(TCC_NormMeansFile))
{
throw new FileNotFoundException("couldn`t find the file specified: " + TCC_NormMeansFile);
}
if (!File.Exists(TCC_NormRangeFile))
{
throw new FileNotFoundException("couldn`t find the file specified: " + TCC_NormRangeFile);
}
if (!File.Exists(TCC_ExcludingVarsFile))
{
throw new FileNotFoundException("couldn`t find the file specified: " + TCC_ExcludingVarsFile);
}
if ((ImagesRoundMasksXMLfilesMappingList == "") || (ImagesRoundMasksXMLfilesMappingList == null))
{
ImagesRoundMasksXMLfilesMappingList = Directory.GetCurrentDirectory() + Path.DirectorySeparatorChar + "settings" +
Path.DirectorySeparatorChar +
"ImagesRoundMasksXMLfilesMappingList.csv";
}
if (!File.Exists(ImagesRoundMasksXMLfilesMappingList))
{
throw new FileNotFoundException("couldn`t find the file specified: " + ImagesRoundMasksXMLfilesMappingList);
}
#endregion check files existence
#region read or calculate GrIxYRGB stats
if (Directory.Exists(imageYRGBstatsXMLdataFilesDirectory))
{
List <string> foundXMLfiles = Directory.EnumerateFiles(imageYRGBstatsXMLdataFilesDirectory,
ConventionalTransitions.ImageGrIxYRGBstatsDataFileName(ImageFilename, "", false),
SearchOption.AllDirectories).ToList();
if (foundXMLfiles.Any())
{
currImageStatsData =
(SkyImageIndexesStatsData)
ServiceTools.ReadObjectFromXML(foundXMLfiles[0], typeof(SkyImageIndexesStatsData));
}
}
if (currImageStatsData == null)
{
Task <SkyImageIndexesStatsData> tskImageStatsCalculation = new Task <SkyImageIndexesStatsData>(() =>
{
Dictionary <string, object> optionalParameters = new Dictionary <string, object>();
optionalParameters.Add("ImagesRoundMasksXMLfilesMappingList", ImagesRoundMasksXMLfilesMappingList);
Stopwatch sw = new Stopwatch();
sw.Start();
optionalParameters.Add("Stopwatch", sw);
ImageStatsDataCalculationResult currImageProcessingResult = null;
try
{
currImageProcessingResult = ImageProcessing.CalculateImageStatsData(ImageFilename,
optionalParameters);
}
catch (Exception ex)
{
return(null);
}
if (currImageProcessingResult == null)
{
return(null);
}
else
{
string strImageGrIxYRGBDataFileName =
ConventionalTransitions.ImageGrIxYRGBstatsDataFileName(ImageFilename,
imageYRGBstatsXMLdataFilesDirectory);
ServiceTools.WriteObjectToXML(currImageProcessingResult.grixyrgbStatsData,
strImageGrIxYRGBDataFileName);
currImageStatsData = currImageProcessingResult.grixyrgbStatsData;
return(currImageProcessingResult.grixyrgbStatsData);
}
});
currImageStatsData = await tskImageStatsCalculation;
}
if (currImageStatsData == null)
{
// theLogWindow = ServiceTools.LogAText(theLogWindow, "ERROR searching concurrent data for image. Will not proceed.");
throw new Exception("ERROR searching concurrent data for image. Will not proceed.");
}
#endregion read or calculate GrIxYRGB stats
ConcurrentData nearestConcurrentData = null;
#region search for concurrent data
Task <ConcurrentData> tskImageConcurrentDataSearching = new Task <ConcurrentData>(() =>
{
string currImgFilename = Path.GetFileNameWithoutExtension(ImageFilename);
string ptrn = @"(devID\d)";
Regex rgxp = new Regex(ptrn, RegexOptions.IgnoreCase);
string strCurrImgDT = rgxp.Replace(currImgFilename.Substring(4), "");
//2015-12-16T06-01-38
strCurrImgDT = strCurrImgDT.Substring(0, 11) + strCurrImgDT.Substring(11).Replace("-", ":");
DateTime currImgDT = DateTime.Parse(strCurrImgDT, null,
System.Globalization.DateTimeStyles.AdjustToUniversal);
List <Tuple <string, ConcurrentData> > lImagesConcurrentData =
new List <Tuple <string, ConcurrentData> >();
List <string> filesListConcurrentData =
Directory.EnumerateFiles(ConcurrentDataXMLfilesBasePath,
ConventionalTransitions.ImageConcurrentDataFilesNamesPattern(),
SearchOption.AllDirectories)
.ToList();
List <Tuple <string, DateTime> > XMLfilesListConcurrentData = filesListConcurrentData.ConvertAll(
strXMLfilename =>
{
string xmlFile1DateTimeString =
Path.GetFileNameWithoutExtension(strXMLfilename).Replace("data-", "").Substring(0, 19);
xmlFile1DateTimeString = xmlFile1DateTimeString.Substring(0, 11) + xmlFile1DateTimeString.Substring(11).Replace("-", ":");
DateTime dt1 = DateTime.Parse(xmlFile1DateTimeString);
return(new Tuple <string, DateTime>(strXMLfilename, dt1));
});
string nearestConcurrentDataFileName = XMLfilesListConcurrentData.Aggregate((cDt1, cDt2) =>
{
TimeSpan tspan1 = new TimeSpan(Math.Abs((cDt1.Item2 - currImgDT).Ticks));
TimeSpan tspan2 = new TimeSpan(Math.Abs((cDt2.Item2 - currImgDT).Ticks));
return((tspan1 <= tspan2) ? (cDt1) : (cDt2));
}).Item1;
Dictionary <string, object> currDict = ServiceTools.ReadDictionaryFromXML(nearestConcurrentDataFileName);
currDict.Add("XMLfileName", Path.GetFileName(nearestConcurrentDataFileName));
ConcurrentData nearestConcurrentDataObtained = new ConcurrentData(currDict);
if (new TimeSpan(Math.Abs((nearestConcurrentDataObtained.datetimeUTC - currImgDT).Ticks)) >=
new TimeSpan(0, 2, 0))
{
nearestConcurrentDataObtained = null;
}
return(nearestConcurrentDataObtained);
});
nearestConcurrentData = await tskImageConcurrentDataSearching;
if (nearestConcurrentData == null)
{
// theLogWindow = ServiceTools.LogAText(theLogWindow, "ERROR searching concurrent data for image. Will not proceed.");
throw new Exception("ERROR searching concurrent data for image. Will not proceed.");
}
#endregion search for concurrent data
DenseVector dvSDCmeans = (DenseVector)((DenseMatrix)ServiceTools.ReadDataFromCSV(SDC_NormMeansFile, 0, ",")).Row(0);
DenseVector dvSDCranges = (DenseVector)((DenseMatrix)ServiceTools.ReadDataFromCSV(SDC_NormRangeFile, 0, ",")).Row(0);
DenseVector dvSDCthetaValues = (DenseVector)ServiceTools.ReadDataFromCSV(SDC_NNtrainedParametersFile, 0, ",");
List <int> SDCnnLayersConfig =
new List <double>(((DenseMatrix)ServiceTools.ReadDataFromCSV(SDC_NNconfigFile, 0, ",")).Row(0)).ConvertAll
(dVal => Convert.ToInt32(dVal));
DenseVector dvTCCmeans = (DenseVector)((DenseMatrix)ServiceTools.ReadDataFromCSV(TCC_NormMeansFile, 0, ",")).Row(0);
DenseVector dvTCCranges = (DenseVector)((DenseMatrix)ServiceTools.ReadDataFromCSV(TCC_NormRangeFile, 0, ",")).Row(0);
DenseVector dvTCCthetaValues = (DenseVector)ServiceTools.ReadDataFromCSV(TCC_NNtrainedParametersFile, 0, ",");
List <int> TCCnnLayersConfig =
new List <double>(((DenseMatrix)ServiceTools.ReadDataFromCSV(TCC_NNconfigFile, 0, ",")).Row(0)).ConvertAll
(dVal => Convert.ToInt32(dVal));
List <int> TCCnnConfigVarsToExclude =
new List <double>(((DenseMatrix)ServiceTools.ReadDataFromCSV(TCC_ExcludingVarsFile, 0, ",")).Row(0)).ConvertAll
(dVal => Convert.ToInt32(dVal));
List <double> TCCdecisionProbabilities = null;
return(PredictTCC_NN(currImageStatsData, nearestConcurrentData, SDCnnLayersConfig, dvSDCthetaValues,
dvSDCmeans,
dvSDCranges, TCCnnLayersConfig, dvTCCthetaValues, dvTCCmeans, dvTCCranges, TCCnnConfigVarsToExclude,
out TCCdecisionProbabilities));
}
