private void ProcessImage(ImageStatsCollectingData srcData)
{
Interlocked.Increment(ref totalFilesProcessed);
int perc = Convert.ToInt32(100.0d * (double)totalFilesProcessed / (double)totalFilesCountToProcess);
Console.WriteLine(DateTime.Now.ToString("s") + " : " + perc + "% : started processing file " + Environment.NewLine + srcData.filename);
Dictionary <string, object> optionalParameters = new Dictionary <string, object>();
optionalParameters.Add("logFileName", errorLogFilename);
// найти и записать данные GPS
GPSdata currimageGPS = ServiceTools.FindProperGPSdataForImage(srcData.filename, null, defaultProperties,
ref NVdataFilesAlreadyReadDateTimeSpans, ref NVdataFilesAlreadyReadData);
if (currimageGPS != null)
{
ServiceTools.WriteObjectToXML(currimageGPS,
ConventionalTransitions.ConcurrentGPSdataFileName(srcData.filename, strConcurrentGPSdataXMLfilesPath));
}
// найти и записать данные SDC и Cloud Cover
DateTime curDateTime = GetImageDateTime(srcData.filename);
if (!lMissionObservedData.Any())
{
return;
}
lMissionObservedData.Sort((obsRecord1, obsRecord2) =>
{
double dev1 = Math.Abs((obsRecord1.dateTime - curDateTime).TotalMilliseconds);
double dev2 = Math.Abs((obsRecord2.dateTime - curDateTime).TotalMilliseconds);
return((dev1 >= dev2) ? (1) : (-1));
});
MissionsObservedData closestObservedDatum = lMissionObservedData[0];
if ((closestObservedDatum.dateTime - curDateTime).TotalSeconds > 600)
{
return;
}
SunDiskConditionData currImageSDC = new SunDiskConditionData()
{
filename = srcData.filename,
sunDiskCondition = closestObservedDatum.SDC
};
ServiceTools.WriteObjectToXML(currImageSDC,
ConventionalTransitions.SunDiskConditionFileName(srcData.filename, SunDiskConditionXMLdataFilesDirectory));
// find grixyrgbStatsXMLfile
SkyImageIndexesStatsData currImageStatsData = null;
string currImageStatsDataXMLfile = "";
if (Directory.Exists(imageYRGBstatsXMLdataFilesDirectory))
{
List <string> foundXMLfiles = Directory.EnumerateFiles(imageYRGBstatsXMLdataFilesDirectory,
ConventionalTransitions.ImageGrIxYRGBstatsDataFileName(srcData.filename, "", false),
SearchOption.AllDirectories).ToList();
if (foundXMLfiles.Any())
{
// возьмем первый попавшийся
currImageStatsDataXMLfile = foundXMLfiles[0];
currImageStatsData =
(SkyImageIndexesStatsData)
ServiceTools.ReadObjectFromXML(currImageStatsDataXMLfile, typeof(SkyImageIndexesStatsData));
}
}
SkyImagesDataWith_Concurrent_Stats_CloudCover_SDC currImageData = new SkyImagesDataWith_Concurrent_Stats_CloudCover_SDC
()
{
skyImageFullFileName = srcData.filename,
skyImageFileName = Path.GetFileName(srcData.filename),
currImageDateTime = curDateTime,
observedCloudCoverData = new ObservedClCoverData()
{
dt = closestObservedDatum.dateTime,
CloudCoverTotal = closestObservedDatum.CloudCoverTotal,
CloudCoverLower = closestObservedDatum.CloudCoverLower
},
concurrentDataXMLfile = "",
concurrentData = new ConcurrentData()
{
filename = "",
datetimeUTC = currimageGPS.DateTimeUTC,
GPSdata = "",
GPSLat = currimageGPS.Lat,
GPSLon = currimageGPS.Lon,
GPSDateTimeUTC = currimageGPS.DateTimeUTC,
PressurePa = closestObservedDatum.pressure,
gps = currimageGPS
},
grixyrgbStatsXMLfile = currImageStatsDataXMLfile,
grixyrgbStats = currImageStatsData,
SDCvalue = closestObservedDatum.SDC
};
ServiceTools.WriteObjectToXML(currImageData,
ConventionalTransitions.SkyImagesDataWithConcurrentStatsCloudCoverAndSDC_FileName(srcData.filename,
strSkyImagesDataWithConcurrentStatsCloudCoverAndSDCDirectory));
}
//private void ImageProcessing(ImagesProcessingData ipdt)
//{
//}
private void EnumerateFilesToProcess()
{
string directory = Path.GetDirectoryName(inputBasePath);
string filemask = "*.jpg";
List <string> filesList =
new List <string>(Directory.EnumerateFiles(directory, filemask,
bEnumerateFilesRecursively ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly));
#region filter by camID
//...devID1.jpg
string ptrnCamID = "devid" + CamIDtoProcess + ".jpg";
filesList = filesList.Where(fname => fname.ToLower().Contains(ptrnCamID)).ToList();
#endregion
Console.WriteLine("found " + filesList.Count + " images.");
#region list, read and map image stats files
Console.WriteLine("filtering by ready-to-use GrIxYRGB XML files...");
List <string> statsFilesList =
new List <string>(Directory.EnumerateFiles(imageYRGBstatsXMLdataFilesDirectory, ConventionalTransitions.ImageGrIxYRGBstatsFileNamesPattern(),
bEnumerateFilesRecursively ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly));
List <string> statsFilesListWOpath = statsFilesList.ConvertAll(Path.GetFileName);
Console.WriteLine("found " + statsFilesList.Count + " XML stats files in directory " + Environment.NewLine +
imageYRGBstatsXMLdataFilesDirectory + Environment.NewLine + "by mask " +
Environment.NewLine + ConventionalTransitions.ImageGrIxYRGBstatsFileNamesPattern());
int removed =
filesList.RemoveAll(
fname =>
!statsFilesListWOpath.Contains(ConventionalTransitions.ImageGrIxYRGBstatsDataFileName(fname, "",
false)));
Console.WriteLine("removed " + removed + " items (couldn`t find stats data files). Remains " + filesList.Count + " to process.");
if (!filesList.Any())
{
Console.WriteLine("There is no " + filemask + " files that sutisfy settings specified. Processing will not be started.");
return;
}
lStatsProcessing = filesList.ConvertAll(strImgFname =>
{
ImagesProcessingData retVal = new ImagesProcessingData()
{
filename = strImgFname
};
return(retVal);
});
#if DEBUG
//lStatsProcessing = lStatsProcessing.Where((ipd, ind) => ind < 10).ToList();
#endif
Console.WriteLine("started reading and mapping stats data");
int totalFilesCountToRead = lStatsProcessing.Count;
int filesRead = 0;
int currProgressPerc = 0;
foreach (ImagesProcessingData ipdt in lStatsProcessing)
{
ipdt.grixyrgbStatsXMLfile =
statsFilesList.First(
statsFname =>
statsFname.Contains(ConventionalTransitions.ImageGrIxYRGBstatsDataFileName(ipdt.filename, "",
false)));
ipdt.grixyrgbStats =
ServiceTools.ReadObjectFromXML(ipdt.grixyrgbStatsXMLfile, typeof(SkyImageIndexesStatsData)) as
SkyImageIndexesStatsData;
#region calculate and report progress
filesRead++;
double progress = 100.0d * (double)filesRead / (double)totalFilesCountToRead;
if (progress - (double)currProgressPerc > 1.0d)
{
currProgressPerc = Convert.ToInt32(progress);
Console.WriteLine("read " + currProgressPerc + "%");
}
#endregion calculate and report progress
}
#endregion
#region list, read and map concurrent data
List <string> concurrentDataFilesList =
Directory.EnumerateFiles(ConcurrentDataXMLfilesDirectory,
ConventionalTransitions.ImageConcurrentDataFilesNamesPattern(),
bEnumerateFilesRecursively ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly).ToList();
List <ConcurrentData> lConcurrentData = null;
#region reading
Console.WriteLine("started concurrent data reading");
totalFilesCountToRead = concurrentDataFilesList.Count;
filesRead = 0;
currProgressPerc = 0;
List <Dictionary <string, object> > lDictionariesConcurrentData =
new List <Dictionary <string, object> >();
foreach (string strConcDataXMLFile in concurrentDataFilesList)
{
Dictionary <string, object> currDict = ServiceTools.ReadDictionaryFromXML(strConcDataXMLFile);
currDict.Add("XMLfileName", Path.GetFileName(strConcDataXMLFile));
lDictionariesConcurrentData.Add(currDict);
#region calculate and report progress
filesRead++;
double progress = 100.0d * (double)filesRead / (double)totalFilesCountToRead;
if (progress - (double)currProgressPerc > 1.0d)
{
currProgressPerc = Convert.ToInt32(progress);
Console.WriteLine("read " + currProgressPerc + "%");
}
#endregion calculate and report progress
}
lDictionariesConcurrentData.RemoveAll(dict => dict == null);
lConcurrentData =
lDictionariesConcurrentData.ConvertAll <ConcurrentData>(dict =>
{
ConcurrentData retVal = null;
try
{
retVal = new ConcurrentData(dict);
}
catch (Exception ex)
{
string strError = "couldn`t parse XML file " + dict["XMLfileName"] + " : " +
Environment.NewLine + ex.Message;
Console.WriteLine(strError);
}
return(retVal);
});
lConcurrentData.RemoveAll(val => val == null);
#endregion reading
#region mapping
// map obtained concurrent data to images by its datetime
Console.WriteLine("concurrent data mapping started");
lStatsProcessing = lStatsProcessing.ConvertAll(ipdt =>
{
string currImgFilename = ipdt.filename;
currImgFilename = Path.GetFileNameWithoutExtension(currImgFilename);
string ptrn = @"(devID\d)";
Regex rgxp = new Regex(ptrn, RegexOptions.IgnoreCase);
string strCurrImgDT = rgxp.Replace(currImgFilename.Substring(4), "");
//2015-12-16T06-01-38
strCurrImgDT = strCurrImgDT.Substring(0, 11) + strCurrImgDT.Substring(11).Replace("-", ":");
DateTime currImgDT = DateTime.Parse(strCurrImgDT, null,
System.Globalization.DateTimeStyles.AdjustToUniversal);
ConcurrentData nearestConcurrentData = lConcurrentData.Aggregate((cDt1, cDt2) =>
{
TimeSpan tspan1 = new TimeSpan(Math.Abs((cDt1.datetimeUTC - currImgDT).Ticks));
TimeSpan tspan2 = new TimeSpan(Math.Abs((cDt2.datetimeUTC - currImgDT).Ticks));
return((tspan1 <= tspan2) ? (cDt1) : (cDt2));
});
if (new TimeSpan(Math.Abs((nearestConcurrentData.datetimeUTC - currImgDT).Ticks)) >=
TimeSpanForConcurrentDataMappingTolerance)
{
string strError = "couldn`t find close enough concurrent data file for image:" + Environment.NewLine +
currImgFilename + Environment.NewLine + "closest concurrent data file is:" +
Environment.NewLine + nearestConcurrentData.filename + Environment.NewLine +
"with date-time value " + nearestConcurrentData.datetimeUTC.ToString("o");
Console.WriteLine(strError);
nearestConcurrentData = null;
}
ipdt.concurrentData = nearestConcurrentData;
if (nearestConcurrentData != null)
{
ipdt.concurrentDataXMLfile = nearestConcurrentData.filename;
}
return(ipdt);
});
#endregion mapping
removed = lStatsProcessing.RemoveAll(ipdt => ipdt.concurrentData == null);
Console.WriteLine("removed " + removed + " items (couldn`t find concurrent data). " + lStatsProcessing.Count + " files remains to process.");
#endregion list, read and map concurrent data
if (!lStatsProcessing.Any())
{
Console.WriteLine("There is no files that sutisfy settings specified and have all required concurrent data (stats or GPS etc.). Processing will not be proceeded.");
return;
}
#region Predict SDC values using pre-trained NN parameters
string csvHeader = lStatsProcessing[0].grixyrgbStats.CSVHeader() +
",SunElevationDeg,SunAzimuthDeg,sunDiskCondition";
List <string> lCSVheader = csvHeader.Split(',').ToList();
List <int> columnsToDelete =
lCSVheader.Select((str, idx) => new Tuple <int, string>(idx, str))
.Where(tpl => tpl.Item2.ToLower().Contains("filename")).ToList().ConvertAll(tpl => tpl.Item1);
List <List <string> > lCalculatedData = lStatsProcessing.ConvertAll(dt =>
{
string currImageALLstatsDataCSVWithConcurrentData = dt.grixyrgbStats.ToCSV() + "," +
dt.concurrentData.gps.SunZenithAzimuth()
.ElevationAngle.ToString()
.Replace(",", ".") + "," +
dt.concurrentData.gps.SunZenithAzimuth()
.Azimuth.ToString()
.Replace(",", ".");
List <string> retVal = currImageALLstatsDataCSVWithConcurrentData.Split(',').ToList();
retVal = retVal.Where((str, idx) => !columnsToDelete.Contains(idx)).ToList();
return(retVal);
});
List <DenseVector> lDV_objects_features =
lCalculatedData.ConvertAll(
list =>
DenseVector.OfEnumerable(list.ConvertAll <double>(str => Convert.ToDouble(str.Replace(".", ",")))));
DenseVector dvMeans = (DenseVector)((DenseMatrix)ServiceTools.ReadDataFromCSV(NormMeansFile, 0, ",")).Row(0);
DenseVector dvRanges = (DenseVector)((DenseMatrix)ServiceTools.ReadDataFromCSV(NormRangeFile, 0, ",")).Row(0);
lDV_objects_features = lDV_objects_features.ConvertAll(dv =>
{
DenseVector dvShifted = dv - dvMeans;
DenseVector dvNormed = (DenseVector)dvShifted.PointwiseDivide(dvRanges);
return(dvNormed);
});
DenseMatrix dmObjectsFeatures = DenseMatrix.OfRowVectors(lDV_objects_features);
DenseVector dvThetaValues = (DenseVector)ServiceTools.ReadDataFromCSV(NNtrainedParametersFile, 0, ",");
List <int> NNlayersConfig =
new List <double>(((DenseMatrix)ServiceTools.ReadDataFromCSV(NNconfigFile, 0, ",")).Row(0)).ConvertAll
(dVal => Convert.ToInt32(dVal));
List <List <double> > lDecisionProbabilities = null;
List <SunDiskCondition> predictedSDClist =
NNclassificatorPredictor <SunDiskCondition> .NNpredict(dmObjectsFeatures, dvThetaValues, NNlayersConfig,
out lDecisionProbabilities, SunDiskConditionData.MatlabEnumeratedSDCorderedList()).ToList();
//List<SunDiskCondition> predictedSDClist = predictedSDC.ConvertAll(sdcInt =>
//{
// switch (sdcInt)
// {
// case 4:
// return SunDiskCondition.NoSun;
// break;
// case 1:
// return SunDiskCondition.Sun0;
// break;
// case 2:
// return SunDiskCondition.Sun1;
// break;
// case 3:
// return SunDiskCondition.Sun2;
// break;
// default:
// return SunDiskCondition.Defect;
// }
//});
string strToShow = "SDC values probabilities: " + Environment.NewLine +
"| No Sun | Sun_0 | Sun_1 | Sun_2 | Detected |" + Environment.NewLine;
foreach (List <double> lDecisionProbability in lDecisionProbabilities)
{
strToShow += "| " + lDecisionProbability[3].ToString("F4") +
" | " + lDecisionProbability[0].ToString("F4") +
" | " + lDecisionProbability[1].ToString("F4") +
" | " + lDecisionProbability[2].ToString("F4") + " |" +
predictedSDClist[lDecisionProbabilities.IndexOf(lDecisionProbability)] + "|" +
Environment.NewLine;
}
ServiceTools.logToTextFile(errorLogFilename, strToShow, true, false);
#endregion
//lStatsProcessing =
// lStatsProcessing.Where((ipd, idx) => predictedSDClist[idx] == SunDiskCondition.Sun2).ToList();
lStatsProcessing =
lStatsProcessing.Where((ipd, idx) => predictedSDClist[idx] == sdcFilter).ToList();
Console.WriteLine("Detected " + lStatsProcessing.Count + " images with SDC = " + sdcFilter.ToString());
if (!lStatsProcessing.Any())
{
Console.WriteLine("There is no files with SDC = Sun2. Processing will not be proceeded.");
return;
}
Console.WriteLine("finished enumerating and filtering files. Files to process: " + lStatsProcessing.Count);
}
public void Start(string[] args)
{
readDefaultProperties();
List <string> argsList = new List <string>(args);
if (argsList.Find(str => str == "--recursive") != null)
{
bEnumerateFilesRecursively = true;
}
if (argsList.Find(str => str == "-y") != null)
{
bStartWithoutConfirmation = true;
}
// --filter-by-observed-cloud-cover-records
// bFilterByObservedCloudCoverRecords
if (argsList.Find(str => str == "--filter-by-observed-cloud-cover-records") != null)
{
bFilterByObservedCloudCoverRecords = true;
}
// sdcFilter
if (argsList.Where(str => str.Contains("--sdc=")).Count() > 0)
{
string foundArg = argsList.Where(str => str.Contains("--sdc=")).ToList()[0];
string strValue = foundArg.Replace("--sdc=", "");
//sdcFilter
if (strValue == "none")
{
sdcFilter = SunDiskCondition.NoSun;
}
else if (strValue == "0")
{
sdcFilter = SunDiskCondition.Sun0;
}
else if (strValue == "1")
{
sdcFilter = SunDiskCondition.Sun1;
}
else if (strValue == "2")
{
sdcFilter = SunDiskCondition.Sun2;
}
else
{
sdcFilter = SunDiskCondition.Sun2;
}
}
else
{
Console.WriteLine("SDC filter is not specified. Filtering by SDC will not applied.");
sdcFilter = SunDiskCondition.Undefined; // Не применять фильтрацию
}
if (argsList.Where(str => str.Contains("--camera-id=")).Count() > 0)
{
string foundArg = argsList.Where(str => str.Contains("--camera-id=")).ToList()[0];
string strValue = foundArg.Replace("--camera-id=", "");
CamIDtoProcess = Convert.ToInt32(strValue);
if ((CamIDtoProcess != 1) && (CamIDtoProcess != 2))
{
Console.WriteLine("camera ID out of range detected. I will not filter by camera ID.");
CamIDtoProcess = 0;
}
}
else
{
Console.WriteLine("camera ID out of range detected. I will not filter by camera ID");
CamIDtoProcess = 0; // will not filter
}
if (!bStartWithoutConfirmation)
{
Console.Write("Start with the mentioned properties? [y/n] ");
string strReply = Console.ReadLine();
if (strReply.ToLower().First() != 'y')
{
Console.WriteLine("\nWill not proceed due to user interruprion.");
Console.WriteLine("===FINISHED===");
Console.ReadKey();
return;
}
}
outputDataFile = strOutputDirectory + Path.GetFileNameWithoutExtension(outputDataFile) + "-" + sdcFilter.ToString() + "-camID" +
CamIDtoProcess + ".xml";
string outputCSVfile = strOutputDirectory + Path.GetFileNameWithoutExtension(outputDataFile) + "-" + sdcFilter.ToString() + "-camID" +
CamIDtoProcess + ".csv";
Console.WriteLine("getting files list");
#region Enumerating files
string directory = Path.GetDirectoryName(inputBasePath);
string filemask = "*.jpg";
List <string> filesList =
new List <string>(Directory.EnumerateFiles(directory, filemask,
bEnumerateFilesRecursively ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly));
#region filter by camID
//...devID1.jpg
if (CamIDtoProcess > 0)
{
string ptrnCamID = "devid" + CamIDtoProcess + ".jpg";
filesList = filesList.Where(fname => fname.ToLower().Contains(ptrnCamID)).ToList();
}
#endregion
Console.WriteLine("found " + filesList.Count + " images.");
#region try to find concurrent and stats data already assembled into a small set of files
List <string> assembledDataFilesList =
Directory.EnumerateFiles(ConcurrentDataXMLfilesDirectory,
"*.xml", SearchOption.TopDirectoryOnly).ToList();
List <ImagesProcessingData> lReadAssembledData = new List <ImagesProcessingData>();
foreach (string strAssembledDataXMlfName in assembledDataFilesList)
{
try
{
List <ImagesProcessingData> currFileContent =
ServiceTools.ReadObjectFromXML(strAssembledDataXMlfName, typeof(List <ImagesProcessingData>)) as
List <ImagesProcessingData>;
lReadAssembledData.AddRange(currFileContent);
}
catch (Exception ex)
{
continue;
}
}
if (lReadAssembledData.Any())
{
Console.WriteLine("Found pre-assembled ImagesProcessingData XML files: ");
foreach (string s in assembledDataFilesList)
{
Console.WriteLine(s);
}
Console.WriteLine("Read records from this set: " + lReadAssembledData.Count);
Console.WriteLine("Images to process originally: " + filesList.Count);
Console.WriteLine("Should I use these pre-assembled data? (y/n): ");
string ans = Console.ReadKey().KeyChar.ToString();
if (ans == "y")
{
lStatsProcessing = lReadAssembledData;
}
}
#endregion try to find data already compiled into a small set of files
if (!lStatsProcessing.Any())
{
#region list, read and map image stats files
Console.WriteLine("filtering by ready-to-use GrIxYRGB XML files...");
List <string> statsFilesList =
new List <string>(Directory.EnumerateFiles(imageYRGBstatsXMLdataFilesDirectory, ConventionalTransitions.ImageGrIxYRGBstatsFileNamesPattern(),
bEnumerateFilesRecursively ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly));
List <string> statsFilesListWOpath = statsFilesList.ConvertAll(Path.GetFileName);
Console.WriteLine("found " + statsFilesList.Count + " XML stats files in directory " + Environment.NewLine +
imageYRGBstatsXMLdataFilesDirectory + Environment.NewLine + "by mask " +
Environment.NewLine + ConventionalTransitions.ImageGrIxYRGBstatsFileNamesPattern());
int removed =
filesList.RemoveAll(
fname =>
!statsFilesListWOpath.Contains(ConventionalTransitions.ImageGrIxYRGBstatsDataFileName(fname, "",
false)));
Console.WriteLine("removed " + removed + " items (couldn`t find stats data files). Remains " + filesList.Count + " to process.");
if (!filesList.Any())
{
Console.WriteLine("There is no " + filemask + " files that sutisfy settings specified. Processing will not be started.");
return;
}
lStatsProcessing = filesList.ConvertAll(strImgFname =>
{
ImagesProcessingData retVal = new ImagesProcessingData()
{
filename = strImgFname
};
return(retVal);
});
//#if DEBUG
// lStatsProcessing = lStatsProcessing.Where((ipd, ind) => ind < 10).ToList();
//#endif
Console.WriteLine("started reading and mapping stats data");
int totalFilesCountToRead = lStatsProcessing.Count;
int filesRead = 0;
int currProgressPerc = 0;
foreach (ImagesProcessingData ipdt in lStatsProcessing)
{
ipdt.grixyrgbStatsXMLfile =
statsFilesList.First(
statsFname =>
statsFname.Contains(ConventionalTransitions.ImageGrIxYRGBstatsDataFileName(ipdt.filename, "",
false)));
ipdt.grixyrgbStats =
ServiceTools.ReadObjectFromXML(ipdt.grixyrgbStatsXMLfile, typeof(SkyImageIndexesStatsData)) as
SkyImageIndexesStatsData;
#region calculate and report progress
filesRead++;
double progress = 100.0d * (double)filesRead / (double)totalFilesCountToRead;
if (progress - (double)currProgressPerc > 1.0d)
{
currProgressPerc = Convert.ToInt32(progress);
Console.WriteLine("read " + currProgressPerc + "%");
}
#endregion calculate and report progress
}
#endregion
#region list, read and map concurrent data
List <ConcurrentData> lConcurrentData = null;
List <string> concurrentDataFilesList =
Directory.EnumerateFiles(ConcurrentDataXMLfilesDirectory,
ConventionalTransitions.ImageConcurrentDataFilesNamesPattern(),
bEnumerateFilesRecursively ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly).ToList();
#region reading
Console.WriteLine("started concurrent data reading");
totalFilesCountToRead = concurrentDataFilesList.Count;
filesRead = 0;
currProgressPerc = 0;
List <Dictionary <string, object> > lDictionariesConcurrentData =
new List <Dictionary <string, object> >();
foreach (string strConcDataXMLFile in concurrentDataFilesList)
{
Dictionary <string, object> currDict = ServiceTools.ReadDictionaryFromXML(strConcDataXMLFile);
currDict.Add("XMLfileName", Path.GetFileName(strConcDataXMLFile));
lDictionariesConcurrentData.Add(currDict);
#region calculate and report progress
filesRead++;
double progress = 100.0d * (double)filesRead / (double)totalFilesCountToRead;
if (progress - (double)currProgressPerc > 1.0d)
{
currProgressPerc = Convert.ToInt32(progress);
Console.WriteLine("read " + currProgressPerc + "%");
}
#endregion calculate and report progress
}
lDictionariesConcurrentData.RemoveAll(dict => dict == null);
lConcurrentData =
lDictionariesConcurrentData.ConvertAll <ConcurrentData>(dict =>
{
ConcurrentData retVal = null;
try
{
retVal = new ConcurrentData(dict);
}
catch (Exception ex)
{
string strError = "couldn`t parse XML file " + dict["XMLfileName"] + " : " +
Environment.NewLine + ex.Message;
Console.WriteLine(strError);
}
return(retVal);
});
lConcurrentData.RemoveAll(val => val == null);
#endregion reading
#region mapping
// map obtained concurrent data to images by its datetime
Console.WriteLine("concurrent data mapping started");
lStatsProcessing = lStatsProcessing.ConvertAll(ipdt =>
{
string currImgFilename = ipdt.filename;
currImgFilename = Path.GetFileNameWithoutExtension(currImgFilename);
DateTime currImgDT = ConventionalTransitions.DateTimeOfSkyImageFilename(currImgFilename);
ConcurrentData nearestConcurrentData = lConcurrentData.Aggregate((cDt1, cDt2) =>
{
TimeSpan tspan1 = new TimeSpan(Math.Abs((cDt1.datetimeUTC - currImgDT).Ticks));
TimeSpan tspan2 = new TimeSpan(Math.Abs((cDt2.datetimeUTC - currImgDT).Ticks));
return((tspan1 <= tspan2) ? (cDt1) : (cDt2));
});
if (new TimeSpan(Math.Abs((nearestConcurrentData.datetimeUTC - currImgDT).Ticks)) >=
TimeSpanForConcurrentDataMappingTolerance)
{
string strError = "couldn`t find close enough concurrent data file for image:" + Environment.NewLine +
currImgFilename + Environment.NewLine + "closest concurrent data file is:" +
Environment.NewLine + nearestConcurrentData.filename + Environment.NewLine +
"with date-time value " + nearestConcurrentData.datetimeUTC.ToString("o");
Console.WriteLine(strError);
nearestConcurrentData = null;
}
ipdt.concurrentData = nearestConcurrentData;
if (nearestConcurrentData != null)
{
ipdt.concurrentDataXMLfile = nearestConcurrentData.filename;
}
return(ipdt);
});
#endregion mapping
removed = lStatsProcessing.RemoveAll(ipdt => ipdt.concurrentData == null);
Console.WriteLine("removed " + removed + " items (couldn`t find concurrent data). " + lStatsProcessing.Count + " files remains to process.");
#endregion list, read and map concurrent data
}
if (!lStatsProcessing.Any())
{
Console.WriteLine("There is no files that sutisfy settings specified and have all required concurrent data (stats or GPS etc.). Processing will not be proceeded.");
return;
}
#region Filter by SDC values predicting it using pre-trained NN parameters
#region //
//string csvHeader = lStatsProcessing[0].grixyrgbStats.CSVHeader() +
// ",SunElevationDeg,SunAzimuthDeg,sunDiskCondition";
//List<string> lCSVheader = csvHeader.Split(',').ToList();
//List<int> columnsToDelete =
// lCSVheader.Select((str, idx) => new Tuple<int, string>(idx, str))
// .Where(tpl => tpl.Item2.ToLower().Contains("filename")).ToList().ConvertAll(tpl => tpl.Item1);
//List<List<string>> lCalculatedData = lStatsProcessing.ConvertAll(dt =>
//{
// string currImageALLstatsDataCSVWithConcurrentData = dt.grixyrgbStats.ToCSV() + "," +
// dt.concurrentData.gps.SunZenithAzimuth()
// .ElevationAngle.ToString()
// .Replace(",", ".") + "," +
// dt.concurrentData.gps.SunZenithAzimuth()
// .Azimuth.ToString()
// .Replace(",", ".");
// List<string> retVal = currImageALLstatsDataCSVWithConcurrentData.Split(',').ToList();
// retVal = retVal.Where((str, idx) => !columnsToDelete.Contains(idx)).ToList();
// return retVal;
//});
//List<DenseVector> lDV_objects_features =
// lCalculatedData.ConvertAll(
// list =>
// DenseVector.OfEnumerable(list.ConvertAll<double>(str => Convert.ToDouble(str.Replace(".", ",")))));
#endregion
DenseVector dvMeans = (DenseVector)((DenseMatrix)ServiceTools.ReadDataFromCSV(NormMeansFile, 0, ",")).Row(0);
DenseVector dvRanges = (DenseVector)((DenseMatrix)ServiceTools.ReadDataFromCSV(NormRangeFile, 0, ",")).Row(0);
#region //
//lDV_objects_features = lDV_objects_features.ConvertAll(dv =>
//{
// DenseVector dvShifted = dv - dvMeans;
// DenseVector dvNormed = (DenseVector)dvShifted.PointwiseDivide(dvRanges);
// return dvNormed;
//});
//DenseMatrix dmObjectsFeatures = DenseMatrix.OfRowVectors(lDV_objects_features);
#endregion
DenseVector dvThetaValues = (DenseVector)ServiceTools.ReadDataFromCSV(NNtrainedParametersFile, 0, ",");
List <int> NNlayersConfig =
new List <double>(((DenseMatrix)ServiceTools.ReadDataFromCSV(NNconfigFile, 0, ",")).Row(0)).ConvertAll
(dVal => Convert.ToInt32(dVal));
#region //
// List<List<double>> lDecisionProbabilities = null;
#endregion
List <Tuple <ImagesProcessingData, List <SDCdecisionProbability>, SunDiskCondition> > lTplsPredictedSDClist =
new List <Tuple <ImagesProcessingData, List <SDCdecisionProbability>, SunDiskCondition> >();
List <List <double> > SDCdecisionProbabilitiesListDoubles = new List <List <double> >();
List <SunDiskCondition> imagesSDCpredicted = SDCpredictorNN.PredictSDC_NN(lStatsProcessing, NNlayersConfig,
dvThetaValues, dvMeans, dvRanges, out SDCdecisionProbabilitiesListDoubles);
List <List <SDCdecisionProbability> > SDCdecisionProbabilitiesLists =
SDCdecisionProbabilitiesListDoubles.ConvertAll(
currSDCdecisionProbabilities =>
currSDCdecisionProbabilities.Select((dProb, idx) => new SDCdecisionProbability()
{
sdc = SunDiskConditionData.MatlabSDCenum(idx + 1),
sdcDecisionProbability = dProb
}).ToList());
lTplsPredictedSDClist =
lStatsProcessing.Zip(
SDCdecisionProbabilitiesLists.Zip(imagesSDCpredicted,
(lDecProb, sdcPredicted) =>
new Tuple <List <SDCdecisionProbability>, SunDiskCondition>(lDecProb, sdcPredicted)).ToList(),
(ipd, tpl) =>
new Tuple <ImagesProcessingData, List <SDCdecisionProbability>, SunDiskCondition>(ipd, tpl.Item1,
tpl.Item2)).ToList();
#region //
//foreach (ImagesProcessingData dt in lStatsProcessing)
//{
// List<double> currSDCdecisionProbabilities = new List<double>();
// SunDiskCondition currSDC = SDCpredictorNN.PredictSDC_NN(dt.grixyrgbStats, dt.concurrentData,
// NNlayersConfig, dvThetaValues, dvMeans, dvRanges, out currSDCdecisionProbabilities);
// List<SDCdecisionProbability> currSDCdecisionProbabilitiesList = currSDCdecisionProbabilities.Select((dProb, idx) => new SDCdecisionProbability()
// {
// sdc = SunDiskConditionData.MatlabSDCenum(idx + 1),
// sdcDecisionProbability = dProb
// }).ToList();
// lTplsPredictedSDClist.Add(
// new Tuple<ImagesProcessingData, List<SDCdecisionProbability>, SunDiskCondition>(dt,
// currSDCdecisionProbabilitiesList, currSDC));
//}
#endregion
#region //
//List<int> predictedSDC =
// NNclassificatorPredictor.NNpredict(dmObjectsFeatures, dvThetaValues, NNlayersConfig,
// out lDecisionProbabilities).ToList();
//List<SunDiskCondition> predictedSDClist = predictedSDC.ConvertAll(sdcInt =>
//{
// switch (sdcInt)
// {
// case 4:
// return SunDiskCondition.NoSun;
// break;
// case 1:
// return SunDiskCondition.Sun0;
// break;
// case 2:
// return SunDiskCondition.Sun1;
// break;
// case 3:
// return SunDiskCondition.Sun2;
// break;
// default:
// return SunDiskCondition.Defect;
// }
//});
//List<Tuple<ImagesProcessingData, SunDiskCondition>> lTplsPredictedSDClist =
// predictedSDClist.Zip(lStatsProcessing,
// (sdc, ipd) => new Tuple<ImagesProcessingData, SunDiskCondition>(ipd, sdc)).ToList();
#endregion
#region output obtained SDC data to log file
string strToShow = "SDC values probabilities: " + Environment.NewLine +
"| NoSun | Sun0 | Sun1 | Sun2 |" + Environment.NewLine;
foreach (Tuple <ImagesProcessingData, List <SDCdecisionProbability>, SunDiskCondition> tpl in lTplsPredictedSDClist)
{
List <SDCdecisionProbability> currSDCdecisionProbabilitiesList = tpl.Item2;
strToShow += "|" +
String.Format("{0,9}",
(currSDCdecisionProbabilitiesList.First(
prob => prob.sdc == SunDiskCondition.NoSun).sdcDecisionProbability * 100.0d)
.ToString("F2") + "%") + "|" +
String.Format("{0,9}",
(currSDCdecisionProbabilitiesList.First(
prob => prob.sdc == SunDiskCondition.Sun0).sdcDecisionProbability * 100.0d)
.ToString("F2") + "%") + "|" +
String.Format("{0,9}",
(currSDCdecisionProbabilitiesList.First(
prob => prob.sdc == SunDiskCondition.Sun1).sdcDecisionProbability * 100.0d)
.ToString("F2") + "%") + "|" +
String.Format("{0,9}",
(currSDCdecisionProbabilitiesList.First(
prob => prob.sdc == SunDiskCondition.Sun2).sdcDecisionProbability * 100.0d)
.ToString("F2") + "%") + "|" + Environment.NewLine;
}
ServiceTools.logToTextFile(errorLogFilename, strToShow, true, false);
#endregion output obtained SDC data to log file
#region filter by SDC value if needed
if (sdcFilter != SunDiskCondition.Undefined)
{
lStatsProcessing = lStatsProcessing.Where((ipd, idx) => lTplsPredictedSDClist[idx].Item3 == sdcFilter).ToList();
Console.WriteLine("Detected " + lStatsProcessing.Count + " images with SDC = " + sdcFilter.ToString());
}
#endregion filter by SDC value if needed
#endregion Filter by SDC values predicting it using pre-trained NN parameters
#region ObservedCloudCoverDataCSVfile
if (bFilterByObservedCloudCoverRecords)
{
Console.WriteLine("Reading observed cloud cover data CSV file...");
if (!File.Exists(ObservedCloudCoverDataCSVfile))
{
Console.WriteLine("Unable to read observed data CSV file: " + ObservedCloudCoverDataCSVfile);
return;
}
List <List <string> > lCSVfileContents = ServiceTools.ReadDataFromCSV(ObservedCloudCoverDataCSVfile, 1,
true);
if (!lCSVfileContents.Any())
{
Console.WriteLine("The observed cloud cover CSV file seems to be empty: " +
ObservedCloudCoverDataCSVfile);
return;
}
List <ObservedClCoverData> lObservedData =
lCSVfileContents.ConvertAll(lStr => new ObservedClCoverData(lStr));
List <SkyImagesDataWith_Concurrent_Stats_CloudCover_SDC> lImagesFilteredByAvailableObservedData =
new List <SkyImagesDataWith_Concurrent_Stats_CloudCover_SDC>();
#region filter images by available observed data using DateTimeFilterTolerance
Console.WriteLine("Filtering by observed data available...");
foreach (ObservedClCoverData observedData in lObservedData)
{
DateTime currObservedDatumDateTime = observedData.dt;
List <SkyImagesDataWith_Concurrent_Stats_CloudCover_SDC> lImagesCloseToCurrObservedDatum = lStatsProcessing
.Where(ipd =>
{
TimeSpan tspan =
new TimeSpan(
Math.Abs(
(ConventionalTransitions.DateTimeOfSkyImageFilename(ipd.filename) -
currObservedDatumDateTime).Ticks));
return(tspan <= DateTimeFilterTolerance);
})
.ToList()
.ConvertAll(ifd => new SkyImagesDataWith_Concurrent_Stats_CloudCover_SDC()
{
// observedData
// ifd
skyImageFullFileName = ifd.filename,
skyImageFileName = Path.GetFileName(ifd.filename),
currImageDateTime = ConventionalTransitions.DateTimeOfSkyImageFilename(ifd.filename),
observedCloudCoverData = observedData,
concurrentDataXMLfile = ifd.concurrentDataXMLfile,
concurrentData = ifd.concurrentData,
grixyrgbStatsXMLfile = ifd.grixyrgbStatsXMLfile,
grixyrgbStats = ifd.grixyrgbStats,
SDCvalue = lTplsPredictedSDClist.First(tpl => tpl.Item1 == ifd).Item3,
SDCprobabilities = lTplsPredictedSDClist.First(tpl => tpl.Item1 == ifd).Item2
});
lImagesFilteredByAvailableObservedData.AddRange(lImagesCloseToCurrObservedDatum);
}
#endregion filter images by available observed data using DateTimeFilterTolerance
if (!lImagesFilteredByAvailableObservedData.Any())
{
Console.WriteLine(
"There is no images remain after filtering using all available data. Output will be empty.");
}
ServiceTools.WriteObjectToXML(lImagesFilteredByAvailableObservedData, outputDataFile);
#region Сформируем и запишем данные в CSV-файл
// Здесь есть данные по наблюдаемому CloudCover
string csvHeader = lImagesFilteredByAvailableObservedData[0].grixyrgbStats.CSVHeader() +
",SunElevationDeg,SunAzimuthDeg,ObservedTotalCloudCover,ObservedLowerCloudCover,SDC,SDCprobabilityNoSun,SDCprobabilitySun0,SDCprobabilitySun1,SDCprobabilitySun2";
List <string> lCSVoutputData = lImagesFilteredByAvailableObservedData.ConvertAll(ifd =>
{
// все стат. предикторы - как для SDC
// данные CloudCover
string retVal = "";
//ImagesProcessingData dt = tpl.Item2;
//ObservedClCoverData clCov = tpl.Item1;
retVal = ifd.grixyrgbStats.ToCSV() + "," +
ifd.concurrentData.gps.SunZenithAzimuth().ElevationAngle.ToString().Replace(",", ".") + "," +
ifd.concurrentData.gps.SunZenithAzimuth().Azimuth.ToString().Replace(",", ".") + "," +
ifd.observedCloudCoverData.CloudCoverTotal.ToString() + "," +
ifd.observedCloudCoverData.CloudCoverLower.ToString() + "," +
ifd.SDCvalue.ToString() + "," +
ifd.SDCprobabilities.First(prob => prob.sdc == SunDiskCondition.NoSun)
.sdcDecisionProbability.ToString().Replace(",", ".") + "," +
ifd.SDCprobabilities.First(prob => prob.sdc == SunDiskCondition.Sun0)
.sdcDecisionProbability.ToString().Replace(",", ".") + "," +
ifd.SDCprobabilities.First(prob => prob.sdc == SunDiskCondition.Sun1)
.sdcDecisionProbability.ToString().Replace(",", ".") + "," +
ifd.SDCprobabilities.First(prob => prob.sdc == SunDiskCondition.Sun2)
.sdcDecisionProbability.ToString().Replace(",", ".");
return(retVal);
});
string strToOutputToCSVfile = string.Join(Environment.NewLine, lCSVoutputData);
ServiceTools.logToTextFile(outputCSVfile, csvHeader + Environment.NewLine, true, false);
ServiceTools.logToTextFile(outputCSVfile, strToOutputToCSVfile, true, false);
#endregion Сформируем и запишем данные в CSV-файл
}
else
{
ServiceTools.WriteObjectToXML(lStatsProcessing, outputDataFile);
#region Сформируем и запишем данные в CSV-файл
// здесь нет данных по наблюдаемому Cloud Cover
// не надо нам такое. Оставим все это только в виде XML-файла.
#endregion Сформируем и запишем данные в CSV-файл
}
#endregion ObservedCloudCoverDataCSVfile
#endregion Enumerating files and output to XML and CSV file
Console.WriteLine("saved output data to file: " + Environment.NewLine + outputDataFile + Environment.NewLine +
Environment.NewLine);
Console.WriteLine("===FINISHED===");
Console.ReadKey();
}
public static List<SunDiskCondition> PredictSDC_NN(
List<Tuple<SkyImageIndexesStatsData, ConcurrentData>> lTplInputData, IEnumerable<int> SDC_NNconfig,
IEnumerable<double> SDC_NNtrainedParameters,
IEnumerable<double> NNfeturesNormMeans, IEnumerable<double> NNfeaturesNormRange,
out List<List<double>> decisionProbabilitiesList)
{
List<string> lImagesALLstatsDataCSVWithConcurrentData = lTplInputData.ConvertAll(tpl =>
{
SkyImageIndexesStatsData imageStats = tpl.Item1;
ConcurrentData snapshotConcurrentData = tpl.Item2;
return imageStats.ToCSV() + "," +
snapshotConcurrentData.gps.SunZenithAzimuth().ElevationAngle.ToString().Replace(",", ".") + "," +
snapshotConcurrentData.gps.SunZenithAzimuth().Azimuth.ToString().Replace(",", ".");
});
// string currImageALLstatsDataCSVWithConcurrentData =
string csvHeader = lTplInputData.First().Item1.CSVHeader() +
",SunElevationDeg,SunAzimuthDeg,sunDiskCondition";
List<string> lCalculatedData = new List<string>();
// lCalculatedData.Add(currImageALLstatsDataCSVWithConcurrentData);
lCalculatedData.AddRange(lImagesALLstatsDataCSVWithConcurrentData);
List<List<string>> csvFileContentStrings =
lCalculatedData.ConvertAll(str => str.Split(',').ToList()).ToList();
List<string> lCSVheader = csvHeader.Split(',').ToList();
List<int> columnsToDelete =
lCSVheader.Select((str, idx) => new Tuple<int, string>(idx, str))
.Where(tpl => tpl.Item2.ToLower().Contains("filename")).ToList().ConvertAll(tpl => tpl.Item1);
List<List<string>> csvFileContentStringsFiltered = new List<List<string>>();
foreach (List<string> listDataStrings in csvFileContentStrings)
{
csvFileContentStringsFiltered.Add(
listDataStrings.Where((str, idx) => !columnsToDelete.Contains(idx)).ToList());
}
List<List<string>> csvFileContentStringsFiltered_wo_sdc = csvFileContentStringsFiltered;
List<DenseVector> lDV_objects_features =
csvFileContentStringsFiltered_wo_sdc.ConvertAll(
list =>
DenseVector.OfEnumerable(list.ConvertAll<double>(str => Convert.ToDouble(str.Replace(".", ",")))));
DenseVector dvMeans = DenseVector.OfEnumerable(NNfeturesNormMeans);
DenseVector dvRanges = DenseVector.OfEnumerable(NNfeaturesNormRange);
DenseVector dvThetaValues = DenseVector.OfEnumerable(SDC_NNtrainedParameters);
List<int> NNlayersConfig = SDC_NNconfig.ToList();
lDV_objects_features = lDV_objects_features.ConvertAll(dv =>
{
DenseVector dvShifted = dv - dvMeans;
DenseVector dvNormed = (DenseVector)dvShifted.PointwiseDivide(dvRanges);
return dvNormed;
});
DenseMatrix dmObjectsFeatures = DenseMatrix.OfRowVectors(lDV_objects_features);
List<List<double>> lDecisionProbabilities = null;
List<SunDiskCondition> predictedSDClist =
NNclassificatorPredictor<SunDiskCondition>.NNpredict(dmObjectsFeatures, dvThetaValues, NNlayersConfig,
out lDecisionProbabilities, SunDiskConditionData.MatlabEnumeratedSDCorderedList()).ToList();
//List<SunDiskCondition> predictedSDClist =
// predictedSDC.ConvertAll(sdcInt => SunDiskConditionData.MatlabSDCenum(sdcInt));
decisionProbabilitiesList = lDecisionProbabilities;
return predictedSDClist;
}
public void Start(string[] args)
{
readDefaultProperties();
List <string> argsList = new List <string>(args);
string filename = argsList.Last();
if (!File.Exists(filename))
{
Console.WriteLine("Couldn`t find input file \"" + filename + "\"");
return;
}
CommonTools.PrintDictionaryToConsole(defaultProperties,
"Default settings specified in file \"" + defaultPropertiesXMLfileName + "\"");
if (!File.Exists(NNconfigFile) || !File.Exists(NNtrainedParametersFile) || !File.Exists(NormMeansFile) || !File.Exists(NormRangeFile))
{
Console.WriteLine(
"couldn`t find at least one of pre-calculated NN parameters file specified in settings: ");
return;
}
List <List <string> > csvFileContentStrings = ServiceTools.ReadDataFromCSV(filename, 0, true, ",");
List <string> csvFileHeader = csvFileContentStrings[0];
csvFileContentStrings = csvFileContentStrings.Where((list, idx) => idx > 0).ToList();
List <int> columnsToDelete =
csvFileHeader.Select((str, idx) => new Tuple <int, string>(idx, str))
.Where(tpl => tpl.Item2.ToLower().Contains("filename")).ToList().ConvertAll(tpl => tpl.Item1);
List <List <string> > csvFileContentStringsFiltered = new List <List <string> >();
foreach (List <string> listDataStrings in csvFileContentStrings)
{
csvFileContentStringsFiltered.Add(
listDataStrings.Where((str, idx) => !columnsToDelete.Contains(idx)).ToList());
}
List <SunDiskCondition> trueAnswers =
csvFileContentStringsFiltered.ConvertAll(
lstr => (SunDiskCondition)Enum.Parse(typeof(SunDiskCondition), lstr.Last()));
//List<int> trueAnswersInt = trueAnswers.ConvertAll(sdc => SunDiskConditionData.MatlabNumeralSDC(sdc));
List <List <string> > csvFileContentStringsFiltered_wo_sdc =
csvFileContentStringsFiltered.ConvertAll(list => list.Where((val, idx) => idx < list.Count - 1).ToList());
List <DenseVector> lDV_objects_features =
csvFileContentStringsFiltered_wo_sdc.ConvertAll(
list =>
DenseVector.OfEnumerable(list.ConvertAll <double>(str => Convert.ToDouble(str.Replace(".", ",")))));
DenseVector dvMeans = (DenseVector)((DenseMatrix)ServiceTools.ReadDataFromCSV(NormMeansFile, 0, ",")).Row(0);
DenseVector dvRanges = (DenseVector)((DenseMatrix)ServiceTools.ReadDataFromCSV(NormRangeFile, 0, ",")).Row(0);
lDV_objects_features = lDV_objects_features.ConvertAll(dv =>
{
DenseVector dvShifted = dv - dvMeans;
DenseVector dvNormed = (DenseVector)dvShifted.PointwiseDivide(dvRanges);
return(dvNormed);
});
DenseMatrix dmObjectsFeatures = DenseMatrix.OfRowVectors(lDV_objects_features);
DenseVector dvThetaValues = (DenseVector)ServiceTools.ReadDataFromCSV(NNtrainedParametersFile, 0, ",");
List <int> NNlayersConfig =
new List <double>(((DenseMatrix)ServiceTools.ReadDataFromCSV(NNconfigFile, 0, ",")).Row(0)).ConvertAll
(dVal => Convert.ToInt32(dVal));
List <SunDiskCondition> predictedSDC =
NNclassificatorPredictor <SunDiskCondition> .NNpredict(dmObjectsFeatures, dvThetaValues, NNlayersConfig,
SunDiskConditionData.MatlabEnumeratedSDCorderedList()).ToList();
List <Tuple <SunDiskCondition, SunDiskCondition> > PredictedVStrue = predictedSDC.Zip(trueAnswers,
(predVal, trueVal) => new Tuple <SunDiskCondition, SunDiskCondition>(predVal, trueVal)).ToList();
Console.WriteLine("=== Prediction result vs true ===");
foreach (Tuple <SunDiskCondition, SunDiskCondition> tpl in PredictedVStrue)
{
Console.WriteLine("pred: " + tpl.Item1.ToString() + ":" + tpl.Item2.ToString() + " :true");
}
double accuracy = 100.0d * ((double)PredictedVStrue.Count(tpl => tpl.Item1 == tpl.Item2)) /
(double)PredictedVStrue.Count();
Console.WriteLine("accuracy: " + accuracy);
Console.WriteLine("Finished. Press any key...");
Console.ReadKey();
}
public static int PredictTCC_NN(SkyImageIndexesStatsData imageStats, ConcurrentData snapshotConcurrentData,
IEnumerable <int> SDC_NNconfig, IEnumerable <double> SDC_NNtrainedParameters,
IEnumerable <double> SDC_NNfeturesNormMeans,
IEnumerable <double> SDC_NNfeaturesNormRange, IEnumerable <int> TCCnnLayersConfig,
IEnumerable <double> TCC_NNtrainedParameters, IEnumerable <double> TCC_NNfeturesNormMeans,
IEnumerable <double> TCC_NNfeaturesNormRange, IEnumerable <int> TCCnnConfigVarsToExclude,
out List <double> TCCdecisionProbabilities)
{
string currImageALLstatsDataCSVWithConcurrentData = imageStats.ToCSV() + "," +
snapshotConcurrentData.gps.SunZenithAzimuth().ElevationAngle.ToString().Replace(",", ".") + "," +
snapshotConcurrentData.gps.SunZenithAzimuth().Azimuth.ToString().Replace(",", ".");
List <string> lCalculatedData = new List <string>();
lCalculatedData.Add(currImageALLstatsDataCSVWithConcurrentData);
string csvHeader = imageStats.CSVHeader();
List <string> lCSVheader = csvHeader.Split(',').ToList();
List <int> columnsToDelete =
lCSVheader.Select((str, idx) => new Tuple <int, string>(idx, str))
.Where(tpl => tpl.Item2.ToLower().Contains("filename")).ToList().ConvertAll(tpl => tpl.Item1);
List <List <string> > csvFileContentStrings =
lCalculatedData.ConvertAll(str => str.Split(',').ToList()).ToList();
List <List <string> > csvFileContentStringsFiltered = new List <List <string> >();
foreach (List <string> listDataStrings in csvFileContentStrings)
{
csvFileContentStringsFiltered.Add(
listDataStrings.Where((str, idx) => !columnsToDelete.Contains(idx)).ToList());
}
#region SDC prediction
List <List <string> > csvFileContentStringsFiltered_wo_CC = csvFileContentStringsFiltered;
List <DenseVector> lDV_objects_features =
csvFileContentStringsFiltered_wo_CC.ConvertAll(
list =>
DenseVector.OfEnumerable(list.ConvertAll <double>(CommonTools.ParseDouble)));
DenseMatrix dmSDCpredictionObjectsFeatures = DenseMatrix.OfRows(lDV_objects_features);
// DenseVector dvMeans = DenseVector.OfEnumerable(SDC_NNfeturesNormMeans);
// DenseVector dvRanges = DenseVector.OfEnumerable(SDC_NNfeaturesNormRange);
DenseMatrix dmSDCpredictionObjectsFeaturesNormed =
ANNservice.FeatureNormalization(dmSDCpredictionObjectsFeatures, SDC_NNfeturesNormMeans,
SDC_NNfeaturesNormRange);
#region Predict SDC
List <int> sdcMatlabValues = new List <int>();
List <double> lSDCpredictionProbabilities = new List <double>();
SunDiskCondition sdc = SDCpredictorNN.PredictSDC_NN(imageStats, snapshotConcurrentData, SDC_NNconfig,
SDC_NNtrainedParameters, SDC_NNfeturesNormMeans, SDC_NNfeaturesNormRange, out lSDCpredictionProbabilities);
sdcMatlabValues.Add(SunDiskConditionData.MatlabNumeralSDC(sdc));
//lDV_objects_features = lDV_objects_features.Zip(sdcMatlabValues, (dv, intSDC) =>
//{
// List<double> lFeaturesWithSDCdata = lDV_objects_features[0].ToList();
// lFeaturesWithSDCdata.Add((double)intSDC);
// return DenseVector.OfEnumerable(lFeaturesWithSDCdata);
//}).ToList();
#endregion Predict SDC
#endregion SDC prediction
DenseMatrix dmTCCpredictionObjectsFeatures = dmSDCpredictionObjectsFeatures.Copy();
List <List <double> > rowsSDCprobabilitiesPerObject = new List <List <double> >();
rowsSDCprobabilitiesPerObject.Add(lSDCpredictionProbabilities);
DenseMatrix dmToAppend = DenseMatrix.OfRows(rowsSDCprobabilitiesPerObject);
dmTCCpredictionObjectsFeatures = (DenseMatrix)dmTCCpredictionObjectsFeatures.Append(dmToAppend);
// remove vars listed in TCCnnConfigVarsToExclude
List <int> TCCnnConfigVarsToExcludeIndexes = TCCnnConfigVarsToExclude.ToList();
TCCnnConfigVarsToExcludeIndexes = TCCnnConfigVarsToExcludeIndexes.ConvertAll(i => i - 1); // based_1 indexes to based_0
DenseMatrix dmTCCpredictionObjectsFeatures_RemovedExcludingFeatures =
dmTCCpredictionObjectsFeatures.RemoveColumns(TCCnnConfigVarsToExcludeIndexes);
DenseMatrix dmTCCpredictionObjectsFeatures_RemovedExcludingFeatures_Normed =
ANNservice.FeatureNormalization(dmTCCpredictionObjectsFeatures_RemovedExcludingFeatures,
TCC_NNfeturesNormMeans,
TCC_NNfeaturesNormRange);
List <List <double> > lTCCdecisionProbabilities = null;
List <int> TCCvaluesSet = new List <int>();
for (int i = 0; i < 9; i++)
{
TCCvaluesSet.Add(i);
}
List <int> predictedTCC =
NNclassificatorPredictor <int> .NNpredict(dmTCCpredictionObjectsFeatures_RemovedExcludingFeatures_Normed,
TCC_NNtrainedParameters, TCCnnLayersConfig, out lTCCdecisionProbabilities, TCCvaluesSet).ToList();
// Matlab trained TCC model: classes 1-9
//predictedTCC = predictedTCC.ConvertAll(iVal => iVal - 1);
TCCdecisionProbabilities = lTCCdecisionProbabilities[0];
return(predictedTCC[0]);
}
