public void SaveClusteringData(string filenameTemplate)
{
FileInfo fInfo = new FileInfo(filenameTemplate);
string dir = fInfo.DirectoryName;
if (!ServiceTools.CheckIfDirectoryExists(dir))
{
return;
}
if (fInfo.Exists)
{
// не трогаем уже существующий файл
return;
}
if (foundClassesContours.Count == 0)
{
return;
}
Rectangle gridRect = new Rectangle(0, 0, dmDensityMesh.ColumnCount, dmDensityMesh.RowCount);
RectangleF realDataRect = new RectangleF((float)minXval, (float)minYval, (float)(maxXval - minXval),
(float)(maxYval - minYval));
foreach (Contour <Point> foundClassContour in foundClassesContours)
{
ClusteringData currClusteringData = new ClusteringData(foundClassContour, gridRect, realDataRect);
string currClassFName = filenameTemplate +
"-X" + currClusteringData.ptdClusterMassCenter.X.ToString("e").Replace(",", "_") +
"-Y" + currClusteringData.ptdClusterMassCenter.Y.ToString("e").Replace(",", "_") + ".xml";
ServiceTools.WriteObjectToXML(currClusteringData, currClassFName);
}
}
private void ProcessImage(ImageStatsCollectingData srcData)
{
Interlocked.Increment(ref totalFilesProcessed);
int perc = Convert.ToInt32(100.0d * (double)totalFilesProcessed / (double)totalFilesCountToProcess);
Console.WriteLine(DateTime.Now.ToString("s") + " : " + perc + "% : started processing file " + Environment.NewLine + srcData.filename);
Dictionary <string, object> optionalParameters = new Dictionary <string, object>();
optionalParameters.Add("logFileName", errorLogFilename);
// найти и записать данные GPS
GPSdata currimageGPS = ServiceTools.FindProperGPSdataForImage(srcData.filename, null, defaultProperties,
ref NVdataFilesAlreadyReadDateTimeSpans, ref NVdataFilesAlreadyReadData);
if (currimageGPS != null)
{
ServiceTools.WriteObjectToXML(currimageGPS,
ConventionalTransitions.ConcurrentGPSdataFileName(srcData.filename, strConcurrentGPSdataXMLfilesPath));
}
// найти и записать данные SDC и Cloud Cover
DateTime curDateTime = GetImageDateTime(srcData.filename);
if (!lMissionObservedData.Any())
{
return;
}
lMissionObservedData.Sort((obsRecord1, obsRecord2) =>
{
double dev1 = Math.Abs((obsRecord1.dateTime - curDateTime).TotalMilliseconds);
double dev2 = Math.Abs((obsRecord2.dateTime - curDateTime).TotalMilliseconds);
return((dev1 >= dev2) ? (1) : (-1));
});
MissionsObservedData closestObservedDatum = lMissionObservedData[0];
if ((closestObservedDatum.dateTime - curDateTime).TotalSeconds > 600)
{
return;
}
SunDiskConditionData currImageSDC = new SunDiskConditionData()
{
filename = srcData.filename,
sunDiskCondition = closestObservedDatum.SDC
};
ServiceTools.WriteObjectToXML(currImageSDC,
ConventionalTransitions.SunDiskConditionFileName(srcData.filename, SunDiskConditionXMLdataFilesDirectory));
// find grixyrgbStatsXMLfile
SkyImageIndexesStatsData currImageStatsData = null;
string currImageStatsDataXMLfile = "";
if (Directory.Exists(imageYRGBstatsXMLdataFilesDirectory))
{
List <string> foundXMLfiles = Directory.EnumerateFiles(imageYRGBstatsXMLdataFilesDirectory,
ConventionalTransitions.ImageGrIxYRGBstatsDataFileName(srcData.filename, "", false),
SearchOption.AllDirectories).ToList();
if (foundXMLfiles.Any())
{
// возьмем первый попавшийся
currImageStatsDataXMLfile = foundXMLfiles[0];
currImageStatsData =
(SkyImageIndexesStatsData)
ServiceTools.ReadObjectFromXML(currImageStatsDataXMLfile, typeof(SkyImageIndexesStatsData));
}
}
SkyImagesDataWith_Concurrent_Stats_CloudCover_SDC currImageData = new SkyImagesDataWith_Concurrent_Stats_CloudCover_SDC
()
{
skyImageFullFileName = srcData.filename,
skyImageFileName = Path.GetFileName(srcData.filename),
currImageDateTime = curDateTime,
observedCloudCoverData = new ObservedClCoverData()
{
dt = closestObservedDatum.dateTime,
CloudCoverTotal = closestObservedDatum.CloudCoverTotal,
CloudCoverLower = closestObservedDatum.CloudCoverLower
},
concurrentDataXMLfile = "",
concurrentData = new ConcurrentData()
{
filename = "",
datetimeUTC = currimageGPS.DateTimeUTC,
GPSdata = "",
GPSLat = currimageGPS.Lat,
GPSLon = currimageGPS.Lon,
GPSDateTimeUTC = currimageGPS.DateTimeUTC,
PressurePa = closestObservedDatum.pressure,
gps = currimageGPS
},
grixyrgbStatsXMLfile = currImageStatsDataXMLfile,
grixyrgbStats = currImageStatsData,
SDCvalue = closestObservedDatum.SDC
};
ServiceTools.WriteObjectToXML(currImageData,
ConventionalTransitions.SkyImagesDataWithConcurrentStatsCloudCoverAndSDC_FileName(srcData.filename,
strSkyImagesDataWithConcurrentStatsCloudCoverAndSDCDirectory));
}
public void Start(string[] args)
{
readDefaultProperties();
List <string> argsList = new List <string>(args);
if (argsList.Find(str => str == "--recursive") != null)
{
bEnumerateFilesRecursively = true;
}
if (argsList.Find(str => str == "-y") != null)
{
bStartWithoutConfirmation = true;
if (defaultProperties.ContainsKey("StartWithoutConfirmation"))
{
defaultProperties["StartWithoutConfirmation"] = bStartWithoutConfirmation;
}
else
{
defaultProperties.Add("StartWithoutConfirmation", bStartWithoutConfirmation);
}
}
// sdcFilter
if (argsList.Where(str => str.Contains("--sdc=")).Count() > 0)
{
string foundArg = argsList.Where(str => str.Contains("--sdc=")).ToList()[0];
string strValue = foundArg.Replace("--sdc=", "");
//sdcFilter
if (strValue == "none")
{
sdcFilter = SunDiskCondition.NoSun;
}
else if (strValue == "0")
{
sdcFilter = SunDiskCondition.Sun0;
}
else if (strValue == "1")
{
sdcFilter = SunDiskCondition.Sun1;
}
else if (strValue == "2")
{
sdcFilter = SunDiskCondition.Sun2;
}
else
{
sdcFilter = SunDiskCondition.Sun2;
}
}
else
{
sdcFilter = SunDiskCondition.Sun2;
}
if (argsList.Where(str => str.Contains("--camera-id=")).Count() > 0)
{
string foundArg = argsList.Where(str => str.Contains("--camera-id=")).ToList()[0];
string strValue = foundArg.Replace("--camera-id=", "");
CamIDtoProcess = Convert.ToInt32(strValue);
if ((CamIDtoProcess != 1) && (CamIDtoProcess != 2))
{
Console.WriteLine("camera ID out of range detected. I will use default id=1");
CamIDtoProcess = 1;
}
}
else
{
Console.WriteLine("camera ID out of range detected. I will use default id=1");
CamIDtoProcess = 1;
}
if (!bStartWithoutConfirmation)
{
Console.Write("Start with the mentioned properties? [y/n] ");
string strReply = Console.ReadLine();
if (strReply.ToLower().First() != 'y')
{
Console.WriteLine("\nWill not proceed due to user interruprion.");
Console.WriteLine("===FINISHED===");
Console.ReadKey();
return;
}
}
outputDataFile = strOutputDirectory + Path.GetFileNameWithoutExtension(outputDataFile) + "-" + sdcFilter.ToString() + "-camID" +
CamIDtoProcess + ".xml";
Console.WriteLine("getting files list");
EnumerateFilesToProcess();
//var options = new ParallelOptions();
//options.MaxDegreeOfParallelism = maxConcurrentProcessingThreads;
//Parallel.ForEach(lStatsProcessing, options, item => { ImageProcessing(item); });
//List<object> lDataToSave = lStatsProcessing.ConvertAll(ipdt => (object) ipdt);
ServiceTools.WriteObjectToXML(lStatsProcessing, outputDataFile);
Console.WriteLine("saved output data to file: " + Environment.NewLine + outputDataFile + Environment.NewLine +
Environment.NewLine);
Console.WriteLine("===FINISHED===");
Console.ReadKey();
}
private void ProcessImage(ImageStatsCollectingData srcData)
{
Interlocked.Increment(ref totalFilesProcessed);
int perc = Convert.ToInt32(100.0d * (double)totalFilesProcessed / (double)totalFilesCountToProcess);
Console.WriteLine(DateTime.Now.ToString("s") + " : " + perc + "% : started processing file " + Environment.NewLine + srcData.filename);
Dictionary <string, object> optionalParameters = new Dictionary <string, object>();
optionalParameters.Add("ImagesRoundMasksXMLfilesMappingList", ImagesRoundMasksXMLfilesMappingList);
Stopwatch sw = new Stopwatch();
sw.Start();
optionalParameters.Add("Stopwatch", sw);
optionalParameters.Add("logFileName", errorLogFilename);
ImageStatsDataCalculationResult currImageProcessingResult =
ImageProcessing.CalculateImageStatsData(srcData.filename, optionalParameters);
currImageProcessingResult.stopwatch.Stop();
if (currImageProcessingResult.calcResult)
{
string currentFullFileName = currImageProcessingResult.imgFilename;
string strPerfCountersData = currentFullFileName + ";" +
currImageProcessingResult.stopwatch.ElapsedMilliseconds + ";" +
(currImageProcessingResult.procTotalProcessorTimeEnd -
currImageProcessingResult.procTotalProcessorTimeStart).TotalMilliseconds +
Environment.NewLine;
ServiceTools.logToTextFile(strPerformanceCountersStatsFile, strPerfCountersData, true);
//string strImageGrIxMedianP5DataFileName =
// ConventionalTransitions.ImageGrIxMedianP5DataFileName(currentFullFileName, imageMP5statsXMLdataFilesDirectory);
//ServiceTools.WriteObjectToXML(currImageProcessingResult.mp5Result, strImageGrIxMedianP5DataFileName);
string strImageGrIxYRGBDataFileName =
ConventionalTransitions.ImageGrIxYRGBstatsDataFileName(currentFullFileName, imageYRGBstatsXMLdataFilesDirectory, true, currPath2Process);
ServiceTools.WriteObjectToXML(currImageProcessingResult.grixyrgbStatsData, strImageGrIxYRGBDataFileName);
ImageStatsCollectingData foundDataObj =
lStatsCalculation.Find(obj => obj.filename == currentFullFileName);
foundDataObj.State = ImageStatsCollectingState.Finished;
foundDataObj.GrIxMedianValue = currImageProcessingResult.mp5Result.GrIxStatsMedian;
foundDataObj.GrIxPerc5Value = currImageProcessingResult.mp5Result.GrIxStatsPerc5;
Console.WriteLine(DateTime.Now.ToString("s") + " : finished processing file " + Environment.NewLine + currentFullFileName);
}
else
{
string currentFullFileName = currImageProcessingResult.imgFilename;
ImageStatsCollectingData foundDataObj =
lStatsCalculation.Find(obj => obj.filename == currentFullFileName);
foundDataObj.State = ImageStatsCollectingState.Error;
Console.WriteLine("ERROR processing file " + Path.GetFileName(currentFullFileName));
try
{
//report full error to error log file
#region report error
#if (DEBUG && MONO)
ServiceTools.logToTextFile(errorLogFilename,
"Error processing file: " + Environment.NewLine + currentFullFileName +
Environment.NewLine + "messages: " +
ServiceTools.GetExceptionMessages(currImageProcessingResult.exception) + Environment.NewLine +
"Stack trace: " + Environment.NewLine +
Environment.StackTrace + Environment.NewLine + Environment.NewLine, true, true);
#else
#if MONO
ServiceTools.logToTextFile(errorLogFilename,
"Error processing file: " + Environment.NewLine + currentFullFileName +
Environment.NewLine + "messages: " +
ServiceTools.GetExceptionMessages(currImageProcessingResult.exception) + Environment.NewLine +
"Stack trace: " + Environment.NewLine +
Environment.StackTrace + Environment.NewLine + Environment.NewLine, true, true);
#else
ServiceTools.logToTextFile(errorLogFilename,
"Error processing file: " + Environment.NewLine + currentFullFileName +
Environment.NewLine + "message: " +
ServiceTools.GetExceptionMessages(currImageProcessingResult.exception) + Environment.NewLine +
ServiceTools.CurrentCodeLineDescription() + Environment.NewLine + "Stack trace: " + Environment.NewLine +
Environment.StackTrace + Environment.NewLine + Environment.NewLine, true, true);
#endif
#endif
#endregion report error
}
catch (Exception ex)
{
return;
}
}
}
public void Start(string[] args)
{
readDefaultProperties();
List <string> argsList = new List <string>(args);
if (argsList.Find(str => str == "--recursive") != null)
{
bEnumerateFilesRecursively = true;
}
if (argsList.Find(str => str == "-y") != null)
{
bStartWithoutConfirmation = true;
}
// --filter-by-observed-cloud-cover-records
// bFilterByObservedCloudCoverRecords
if (argsList.Find(str => str == "--filter-by-observed-cloud-cover-records") != null)
{
bFilterByObservedCloudCoverRecords = true;
}
// sdcFilter
if (argsList.Where(str => str.Contains("--sdc=")).Count() > 0)
{
string foundArg = argsList.Where(str => str.Contains("--sdc=")).ToList()[0];
string strValue = foundArg.Replace("--sdc=", "");
//sdcFilter
if (strValue == "none")
{
sdcFilter = SunDiskCondition.NoSun;
}
else if (strValue == "0")
{
sdcFilter = SunDiskCondition.Sun0;
}
else if (strValue == "1")
{
sdcFilter = SunDiskCondition.Sun1;
}
else if (strValue == "2")
{
sdcFilter = SunDiskCondition.Sun2;
}
else
{
sdcFilter = SunDiskCondition.Sun2;
}
}
else
{
Console.WriteLine("SDC filter is not specified. Filtering by SDC will not applied.");
sdcFilter = SunDiskCondition.Undefined; // Не применять фильтрацию
}
if (argsList.Where(str => str.Contains("--camera-id=")).Count() > 0)
{
string foundArg = argsList.Where(str => str.Contains("--camera-id=")).ToList()[0];
string strValue = foundArg.Replace("--camera-id=", "");
CamIDtoProcess = Convert.ToInt32(strValue);
if ((CamIDtoProcess != 1) && (CamIDtoProcess != 2))
{
Console.WriteLine("camera ID out of range detected. I will not filter by camera ID.");
CamIDtoProcess = 0;
}
}
else
{
Console.WriteLine("camera ID out of range detected. I will not filter by camera ID");
CamIDtoProcess = 0; // will not filter
}
if (!bStartWithoutConfirmation)
{
Console.Write("Start with the mentioned properties? [y/n] ");
string strReply = Console.ReadLine();
if (strReply.ToLower().First() != 'y')
{
Console.WriteLine("\nWill not proceed due to user interruprion.");
Console.WriteLine("===FINISHED===");
Console.ReadKey();
return;
}
}
outputDataFile = strOutputDirectory + Path.GetFileNameWithoutExtension(outputDataFile) + "-" + sdcFilter.ToString() + "-camID" +
CamIDtoProcess + ".xml";
string outputCSVfile = strOutputDirectory + Path.GetFileNameWithoutExtension(outputDataFile) + "-" + sdcFilter.ToString() + "-camID" +
CamIDtoProcess + ".csv";
Console.WriteLine("getting files list");
#region Enumerating files
string directory = Path.GetDirectoryName(inputBasePath);
string filemask = "*.jpg";
List <string> filesList =
new List <string>(Directory.EnumerateFiles(directory, filemask,
bEnumerateFilesRecursively ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly));
#region filter by camID
//...devID1.jpg
if (CamIDtoProcess > 0)
{
string ptrnCamID = "devid" + CamIDtoProcess + ".jpg";
filesList = filesList.Where(fname => fname.ToLower().Contains(ptrnCamID)).ToList();
}
#endregion
Console.WriteLine("found " + filesList.Count + " images.");
#region try to find concurrent and stats data already assembled into a small set of files
List <string> assembledDataFilesList =
Directory.EnumerateFiles(ConcurrentDataXMLfilesDirectory,
"*.xml", SearchOption.TopDirectoryOnly).ToList();
List <ImagesProcessingData> lReadAssembledData = new List <ImagesProcessingData>();
foreach (string strAssembledDataXMlfName in assembledDataFilesList)
{
try
{
List <ImagesProcessingData> currFileContent =
ServiceTools.ReadObjectFromXML(strAssembledDataXMlfName, typeof(List <ImagesProcessingData>)) as
List <ImagesProcessingData>;
lReadAssembledData.AddRange(currFileContent);
}
catch (Exception ex)
{
continue;
}
}
if (lReadAssembledData.Any())
{
Console.WriteLine("Found pre-assembled ImagesProcessingData XML files: ");
foreach (string s in assembledDataFilesList)
{
Console.WriteLine(s);
}
Console.WriteLine("Read records from this set: " + lReadAssembledData.Count);
Console.WriteLine("Images to process originally: " + filesList.Count);
Console.WriteLine("Should I use these pre-assembled data? (y/n): ");
string ans = Console.ReadKey().KeyChar.ToString();
if (ans == "y")
{
lStatsProcessing = lReadAssembledData;
}
}
#endregion try to find data already compiled into a small set of files
if (!lStatsProcessing.Any())
{
#region list, read and map image stats files
Console.WriteLine("filtering by ready-to-use GrIxYRGB XML files...");
List <string> statsFilesList =
new List <string>(Directory.EnumerateFiles(imageYRGBstatsXMLdataFilesDirectory, ConventionalTransitions.ImageGrIxYRGBstatsFileNamesPattern(),
bEnumerateFilesRecursively ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly));
List <string> statsFilesListWOpath = statsFilesList.ConvertAll(Path.GetFileName);
Console.WriteLine("found " + statsFilesList.Count + " XML stats files in directory " + Environment.NewLine +
imageYRGBstatsXMLdataFilesDirectory + Environment.NewLine + "by mask " +
Environment.NewLine + ConventionalTransitions.ImageGrIxYRGBstatsFileNamesPattern());
int removed =
filesList.RemoveAll(
fname =>
!statsFilesListWOpath.Contains(ConventionalTransitions.ImageGrIxYRGBstatsDataFileName(fname, "",
false)));
Console.WriteLine("removed " + removed + " items (couldn`t find stats data files). Remains " + filesList.Count + " to process.");
if (!filesList.Any())
{
Console.WriteLine("There is no " + filemask + " files that sutisfy settings specified. Processing will not be started.");
return;
}
lStatsProcessing = filesList.ConvertAll(strImgFname =>
{
ImagesProcessingData retVal = new ImagesProcessingData()
{
filename = strImgFname
};
return(retVal);
});
//#if DEBUG
// lStatsProcessing = lStatsProcessing.Where((ipd, ind) => ind < 10).ToList();
//#endif
Console.WriteLine("started reading and mapping stats data");
int totalFilesCountToRead = lStatsProcessing.Count;
int filesRead = 0;
int currProgressPerc = 0;
foreach (ImagesProcessingData ipdt in lStatsProcessing)
{
ipdt.grixyrgbStatsXMLfile =
statsFilesList.First(
statsFname =>
statsFname.Contains(ConventionalTransitions.ImageGrIxYRGBstatsDataFileName(ipdt.filename, "",
false)));
ipdt.grixyrgbStats =
ServiceTools.ReadObjectFromXML(ipdt.grixyrgbStatsXMLfile, typeof(SkyImageIndexesStatsData)) as
SkyImageIndexesStatsData;
#region calculate and report progress
filesRead++;
double progress = 100.0d * (double)filesRead / (double)totalFilesCountToRead;
if (progress - (double)currProgressPerc > 1.0d)
{
currProgressPerc = Convert.ToInt32(progress);
Console.WriteLine("read " + currProgressPerc + "%");
}
#endregion calculate and report progress
}
#endregion
#region list, read and map concurrent data
List <ConcurrentData> lConcurrentData = null;
List <string> concurrentDataFilesList =
Directory.EnumerateFiles(ConcurrentDataXMLfilesDirectory,
ConventionalTransitions.ImageConcurrentDataFilesNamesPattern(),
bEnumerateFilesRecursively ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly).ToList();
#region reading
Console.WriteLine("started concurrent data reading");
totalFilesCountToRead = concurrentDataFilesList.Count;
filesRead = 0;
currProgressPerc = 0;
List <Dictionary <string, object> > lDictionariesConcurrentData =
new List <Dictionary <string, object> >();
foreach (string strConcDataXMLFile in concurrentDataFilesList)
{
Dictionary <string, object> currDict = ServiceTools.ReadDictionaryFromXML(strConcDataXMLFile);
currDict.Add("XMLfileName", Path.GetFileName(strConcDataXMLFile));
lDictionariesConcurrentData.Add(currDict);
#region calculate and report progress
filesRead++;
double progress = 100.0d * (double)filesRead / (double)totalFilesCountToRead;
if (progress - (double)currProgressPerc > 1.0d)
{
currProgressPerc = Convert.ToInt32(progress);
Console.WriteLine("read " + currProgressPerc + "%");
}
#endregion calculate and report progress
}
lDictionariesConcurrentData.RemoveAll(dict => dict == null);
lConcurrentData =
lDictionariesConcurrentData.ConvertAll <ConcurrentData>(dict =>
{
ConcurrentData retVal = null;
try
{
retVal = new ConcurrentData(dict);
}
catch (Exception ex)
{
string strError = "couldn`t parse XML file " + dict["XMLfileName"] + " : " +
Environment.NewLine + ex.Message;
Console.WriteLine(strError);
}
return(retVal);
});
lConcurrentData.RemoveAll(val => val == null);
#endregion reading
#region mapping
// map obtained concurrent data to images by its datetime
Console.WriteLine("concurrent data mapping started");
lStatsProcessing = lStatsProcessing.ConvertAll(ipdt =>
{
string currImgFilename = ipdt.filename;
currImgFilename = Path.GetFileNameWithoutExtension(currImgFilename);
DateTime currImgDT = ConventionalTransitions.DateTimeOfSkyImageFilename(currImgFilename);
ConcurrentData nearestConcurrentData = lConcurrentData.Aggregate((cDt1, cDt2) =>
{
TimeSpan tspan1 = new TimeSpan(Math.Abs((cDt1.datetimeUTC - currImgDT).Ticks));
TimeSpan tspan2 = new TimeSpan(Math.Abs((cDt2.datetimeUTC - currImgDT).Ticks));
return((tspan1 <= tspan2) ? (cDt1) : (cDt2));
});
if (new TimeSpan(Math.Abs((nearestConcurrentData.datetimeUTC - currImgDT).Ticks)) >=
TimeSpanForConcurrentDataMappingTolerance)
{
string strError = "couldn`t find close enough concurrent data file for image:" + Environment.NewLine +
currImgFilename + Environment.NewLine + "closest concurrent data file is:" +
Environment.NewLine + nearestConcurrentData.filename + Environment.NewLine +
"with date-time value " + nearestConcurrentData.datetimeUTC.ToString("o");
Console.WriteLine(strError);
nearestConcurrentData = null;
}
ipdt.concurrentData = nearestConcurrentData;
if (nearestConcurrentData != null)
{
ipdt.concurrentDataXMLfile = nearestConcurrentData.filename;
}
return(ipdt);
});
#endregion mapping
removed = lStatsProcessing.RemoveAll(ipdt => ipdt.concurrentData == null);
Console.WriteLine("removed " + removed + " items (couldn`t find concurrent data). " + lStatsProcessing.Count + " files remains to process.");
#endregion list, read and map concurrent data
}
if (!lStatsProcessing.Any())
{
Console.WriteLine("There is no files that sutisfy settings specified and have all required concurrent data (stats or GPS etc.). Processing will not be proceeded.");
return;
}
#region Filter by SDC values predicting it using pre-trained NN parameters
#region //
//string csvHeader = lStatsProcessing[0].grixyrgbStats.CSVHeader() +
// ",SunElevationDeg,SunAzimuthDeg,sunDiskCondition";
//List<string> lCSVheader = csvHeader.Split(',').ToList();
//List<int> columnsToDelete =
// lCSVheader.Select((str, idx) => new Tuple<int, string>(idx, str))
// .Where(tpl => tpl.Item2.ToLower().Contains("filename")).ToList().ConvertAll(tpl => tpl.Item1);
//List<List<string>> lCalculatedData = lStatsProcessing.ConvertAll(dt =>
//{
// string currImageALLstatsDataCSVWithConcurrentData = dt.grixyrgbStats.ToCSV() + "," +
// dt.concurrentData.gps.SunZenithAzimuth()
// .ElevationAngle.ToString()
// .Replace(",", ".") + "," +
// dt.concurrentData.gps.SunZenithAzimuth()
// .Azimuth.ToString()
// .Replace(",", ".");
// List<string> retVal = currImageALLstatsDataCSVWithConcurrentData.Split(',').ToList();
// retVal = retVal.Where((str, idx) => !columnsToDelete.Contains(idx)).ToList();
// return retVal;
//});
//List<DenseVector> lDV_objects_features =
// lCalculatedData.ConvertAll(
// list =>
// DenseVector.OfEnumerable(list.ConvertAll<double>(str => Convert.ToDouble(str.Replace(".", ",")))));
#endregion
DenseVector dvMeans = (DenseVector)((DenseMatrix)ServiceTools.ReadDataFromCSV(NormMeansFile, 0, ",")).Row(0);
DenseVector dvRanges = (DenseVector)((DenseMatrix)ServiceTools.ReadDataFromCSV(NormRangeFile, 0, ",")).Row(0);
#region //
//lDV_objects_features = lDV_objects_features.ConvertAll(dv =>
//{
// DenseVector dvShifted = dv - dvMeans;
// DenseVector dvNormed = (DenseVector)dvShifted.PointwiseDivide(dvRanges);
// return dvNormed;
//});
//DenseMatrix dmObjectsFeatures = DenseMatrix.OfRowVectors(lDV_objects_features);
#endregion
DenseVector dvThetaValues = (DenseVector)ServiceTools.ReadDataFromCSV(NNtrainedParametersFile, 0, ",");
List <int> NNlayersConfig =
new List <double>(((DenseMatrix)ServiceTools.ReadDataFromCSV(NNconfigFile, 0, ",")).Row(0)).ConvertAll
(dVal => Convert.ToInt32(dVal));
#region //
// List<List<double>> lDecisionProbabilities = null;
#endregion
List <Tuple <ImagesProcessingData, List <SDCdecisionProbability>, SunDiskCondition> > lTplsPredictedSDClist =
new List <Tuple <ImagesProcessingData, List <SDCdecisionProbability>, SunDiskCondition> >();
List <List <double> > SDCdecisionProbabilitiesListDoubles = new List <List <double> >();
List <SunDiskCondition> imagesSDCpredicted = SDCpredictorNN.PredictSDC_NN(lStatsProcessing, NNlayersConfig,
dvThetaValues, dvMeans, dvRanges, out SDCdecisionProbabilitiesListDoubles);
List <List <SDCdecisionProbability> > SDCdecisionProbabilitiesLists =
SDCdecisionProbabilitiesListDoubles.ConvertAll(
currSDCdecisionProbabilities =>
currSDCdecisionProbabilities.Select((dProb, idx) => new SDCdecisionProbability()
{
sdc = SunDiskConditionData.MatlabSDCenum(idx + 1),
sdcDecisionProbability = dProb
}).ToList());
lTplsPredictedSDClist =
lStatsProcessing.Zip(
SDCdecisionProbabilitiesLists.Zip(imagesSDCpredicted,
(lDecProb, sdcPredicted) =>
new Tuple <List <SDCdecisionProbability>, SunDiskCondition>(lDecProb, sdcPredicted)).ToList(),
(ipd, tpl) =>
new Tuple <ImagesProcessingData, List <SDCdecisionProbability>, SunDiskCondition>(ipd, tpl.Item1,
tpl.Item2)).ToList();
#region //
//foreach (ImagesProcessingData dt in lStatsProcessing)
//{
// List<double> currSDCdecisionProbabilities = new List<double>();
// SunDiskCondition currSDC = SDCpredictorNN.PredictSDC_NN(dt.grixyrgbStats, dt.concurrentData,
// NNlayersConfig, dvThetaValues, dvMeans, dvRanges, out currSDCdecisionProbabilities);
// List<SDCdecisionProbability> currSDCdecisionProbabilitiesList = currSDCdecisionProbabilities.Select((dProb, idx) => new SDCdecisionProbability()
// {
// sdc = SunDiskConditionData.MatlabSDCenum(idx + 1),
// sdcDecisionProbability = dProb
// }).ToList();
// lTplsPredictedSDClist.Add(
// new Tuple<ImagesProcessingData, List<SDCdecisionProbability>, SunDiskCondition>(dt,
// currSDCdecisionProbabilitiesList, currSDC));
//}
#endregion
#region //
//List<int> predictedSDC =
// NNclassificatorPredictor.NNpredict(dmObjectsFeatures, dvThetaValues, NNlayersConfig,
// out lDecisionProbabilities).ToList();
//List<SunDiskCondition> predictedSDClist = predictedSDC.ConvertAll(sdcInt =>
//{
// switch (sdcInt)
// {
// case 4:
// return SunDiskCondition.NoSun;
// break;
// case 1:
// return SunDiskCondition.Sun0;
// break;
// case 2:
// return SunDiskCondition.Sun1;
// break;
// case 3:
// return SunDiskCondition.Sun2;
// break;
// default:
// return SunDiskCondition.Defect;
// }
//});
//List<Tuple<ImagesProcessingData, SunDiskCondition>> lTplsPredictedSDClist =
// predictedSDClist.Zip(lStatsProcessing,
// (sdc, ipd) => new Tuple<ImagesProcessingData, SunDiskCondition>(ipd, sdc)).ToList();
#endregion
#region output obtained SDC data to log file
string strToShow = "SDC values probabilities: " + Environment.NewLine +
"| NoSun | Sun0 | Sun1 | Sun2 |" + Environment.NewLine;
foreach (Tuple <ImagesProcessingData, List <SDCdecisionProbability>, SunDiskCondition> tpl in lTplsPredictedSDClist)
{
List <SDCdecisionProbability> currSDCdecisionProbabilitiesList = tpl.Item2;
strToShow += "|" +
String.Format("{0,9}",
(currSDCdecisionProbabilitiesList.First(
prob => prob.sdc == SunDiskCondition.NoSun).sdcDecisionProbability * 100.0d)
.ToString("F2") + "%") + "|" +
String.Format("{0,9}",
(currSDCdecisionProbabilitiesList.First(
prob => prob.sdc == SunDiskCondition.Sun0).sdcDecisionProbability * 100.0d)
.ToString("F2") + "%") + "|" +
String.Format("{0,9}",
(currSDCdecisionProbabilitiesList.First(
prob => prob.sdc == SunDiskCondition.Sun1).sdcDecisionProbability * 100.0d)
.ToString("F2") + "%") + "|" +
String.Format("{0,9}",
(currSDCdecisionProbabilitiesList.First(
prob => prob.sdc == SunDiskCondition.Sun2).sdcDecisionProbability * 100.0d)
.ToString("F2") + "%") + "|" + Environment.NewLine;
}
ServiceTools.logToTextFile(errorLogFilename, strToShow, true, false);
#endregion output obtained SDC data to log file
#region filter by SDC value if needed
if (sdcFilter != SunDiskCondition.Undefined)
{
lStatsProcessing = lStatsProcessing.Where((ipd, idx) => lTplsPredictedSDClist[idx].Item3 == sdcFilter).ToList();
Console.WriteLine("Detected " + lStatsProcessing.Count + " images with SDC = " + sdcFilter.ToString());
}
#endregion filter by SDC value if needed
#endregion Filter by SDC values predicting it using pre-trained NN parameters
#region ObservedCloudCoverDataCSVfile
if (bFilterByObservedCloudCoverRecords)
{
Console.WriteLine("Reading observed cloud cover data CSV file...");
if (!File.Exists(ObservedCloudCoverDataCSVfile))
{
Console.WriteLine("Unable to read observed data CSV file: " + ObservedCloudCoverDataCSVfile);
return;
}
List <List <string> > lCSVfileContents = ServiceTools.ReadDataFromCSV(ObservedCloudCoverDataCSVfile, 1,
true);
if (!lCSVfileContents.Any())
{
Console.WriteLine("The observed cloud cover CSV file seems to be empty: " +
ObservedCloudCoverDataCSVfile);
return;
}
List <ObservedClCoverData> lObservedData =
lCSVfileContents.ConvertAll(lStr => new ObservedClCoverData(lStr));
List <SkyImagesDataWith_Concurrent_Stats_CloudCover_SDC> lImagesFilteredByAvailableObservedData =
new List <SkyImagesDataWith_Concurrent_Stats_CloudCover_SDC>();
#region filter images by available observed data using DateTimeFilterTolerance
Console.WriteLine("Filtering by observed data available...");
foreach (ObservedClCoverData observedData in lObservedData)
{
DateTime currObservedDatumDateTime = observedData.dt;
List <SkyImagesDataWith_Concurrent_Stats_CloudCover_SDC> lImagesCloseToCurrObservedDatum = lStatsProcessing
.Where(ipd =>
{
TimeSpan tspan =
new TimeSpan(
Math.Abs(
(ConventionalTransitions.DateTimeOfSkyImageFilename(ipd.filename) -
currObservedDatumDateTime).Ticks));
return(tspan <= DateTimeFilterTolerance);
})
.ToList()
.ConvertAll(ifd => new SkyImagesDataWith_Concurrent_Stats_CloudCover_SDC()
{
// observedData
// ifd
skyImageFullFileName = ifd.filename,
skyImageFileName = Path.GetFileName(ifd.filename),
currImageDateTime = ConventionalTransitions.DateTimeOfSkyImageFilename(ifd.filename),
observedCloudCoverData = observedData,
concurrentDataXMLfile = ifd.concurrentDataXMLfile,
concurrentData = ifd.concurrentData,
grixyrgbStatsXMLfile = ifd.grixyrgbStatsXMLfile,
grixyrgbStats = ifd.grixyrgbStats,
SDCvalue = lTplsPredictedSDClist.First(tpl => tpl.Item1 == ifd).Item3,
SDCprobabilities = lTplsPredictedSDClist.First(tpl => tpl.Item1 == ifd).Item2
});
lImagesFilteredByAvailableObservedData.AddRange(lImagesCloseToCurrObservedDatum);
}
#endregion filter images by available observed data using DateTimeFilterTolerance
if (!lImagesFilteredByAvailableObservedData.Any())
{
Console.WriteLine(
"There is no images remain after filtering using all available data. Output will be empty.");
}
ServiceTools.WriteObjectToXML(lImagesFilteredByAvailableObservedData, outputDataFile);
#region Сформируем и запишем данные в CSV-файл
// Здесь есть данные по наблюдаемому CloudCover
string csvHeader = lImagesFilteredByAvailableObservedData[0].grixyrgbStats.CSVHeader() +
",SunElevationDeg,SunAzimuthDeg,ObservedTotalCloudCover,ObservedLowerCloudCover,SDC,SDCprobabilityNoSun,SDCprobabilitySun0,SDCprobabilitySun1,SDCprobabilitySun2";
List <string> lCSVoutputData = lImagesFilteredByAvailableObservedData.ConvertAll(ifd =>
{
// все стат. предикторы - как для SDC
// данные CloudCover
string retVal = "";
//ImagesProcessingData dt = tpl.Item2;
//ObservedClCoverData clCov = tpl.Item1;
retVal = ifd.grixyrgbStats.ToCSV() + "," +
ifd.concurrentData.gps.SunZenithAzimuth().ElevationAngle.ToString().Replace(",", ".") + "," +
ifd.concurrentData.gps.SunZenithAzimuth().Azimuth.ToString().Replace(",", ".") + "," +
ifd.observedCloudCoverData.CloudCoverTotal.ToString() + "," +
ifd.observedCloudCoverData.CloudCoverLower.ToString() + "," +
ifd.SDCvalue.ToString() + "," +
ifd.SDCprobabilities.First(prob => prob.sdc == SunDiskCondition.NoSun)
.sdcDecisionProbability.ToString().Replace(",", ".") + "," +
ifd.SDCprobabilities.First(prob => prob.sdc == SunDiskCondition.Sun0)
.sdcDecisionProbability.ToString().Replace(",", ".") + "," +
ifd.SDCprobabilities.First(prob => prob.sdc == SunDiskCondition.Sun1)
.sdcDecisionProbability.ToString().Replace(",", ".") + "," +
ifd.SDCprobabilities.First(prob => prob.sdc == SunDiskCondition.Sun2)
.sdcDecisionProbability.ToString().Replace(",", ".");
return(retVal);
});
string strToOutputToCSVfile = string.Join(Environment.NewLine, lCSVoutputData);
ServiceTools.logToTextFile(outputCSVfile, csvHeader + Environment.NewLine, true, false);
ServiceTools.logToTextFile(outputCSVfile, strToOutputToCSVfile, true, false);
#endregion Сформируем и запишем данные в CSV-файл
}
else
{
ServiceTools.WriteObjectToXML(lStatsProcessing, outputDataFile);
#region Сформируем и запишем данные в CSV-файл
// здесь нет данных по наблюдаемому Cloud Cover
// не надо нам такое. Оставим все это только в виде XML-файла.
#endregion Сформируем и запишем данные в CSV-файл
}
#endregion ObservedCloudCoverDataCSVfile
#endregion Enumerating files and output to XML and CSV file
Console.WriteLine("saved output data to file: " + Environment.NewLine + outputDataFile + Environment.NewLine +
Environment.NewLine);
Console.WriteLine("===FINISHED===");
Console.ReadKey();
}
private void CurrFileProcessing(ImagesConvertingData dat)
{
string currentFullFileName = dat.filename;
try
{
List <List <string> > csvFileContents = ServiceTools.ReadDataFromCSV(currentFullFileName, 3, true, ";", Environment.NewLine);
List <double> lCenterXValues = csvFileContents.ConvertAll <double>(lStr => Convert.ToDouble(lStr[1]));
List <double> lCenterYValues = csvFileContents.ConvertAll <double>(lStr => Convert.ToDouble(lStr[2]));
List <double> lRadiiValues = csvFileContents.ConvertAll <double>(lStr => Convert.ToDouble(lStr[3]));
// filter default values
// filteringMarks = "те, которые оставляем"
List <bool> filteringMarks = csvFileContents.ConvertAll <bool>(lStr => true);
if (filterDefaultDoubleValueCenterX > 0.0d)
{
List <bool> filteringMarksAdd = lCenterXValues.ConvertAll <bool>(dVal => dVal != filterDefaultDoubleValueCenterX);
filteringMarks =
(filteringMarks.Zip <bool, bool, bool>(filteringMarksAdd, (bVal1, bVal2) => bVal1 && bVal2))
.ToList();
}
if (filterDefaultDoubleValueCenterY > 0.0d)
{
List <bool> filteringMarksAdd = lCenterYValues.ConvertAll <bool>(dVal => dVal != filterDefaultDoubleValueCenterY);
filteringMarks =
(filteringMarks.Zip <bool, bool, bool>(filteringMarksAdd, (bVal1, bVal2) => bVal1 && bVal2))
.ToList();
}
if (filterDefaultDoubleValueRadius > 0.0d)
{
List <bool> filteringMarksAdd = lRadiiValues.ConvertAll <bool>(dVal => dVal != filterDefaultDoubleValueRadius);
filteringMarks =
(filteringMarks.Zip <bool, bool, bool>(filteringMarksAdd, (bVal1, bVal2) => bVal1 && bVal2))
.ToList();
}
List <int> indexes =
filteringMarks
.Select((val, idx) => new { val, idx })
.Where(x => x.val)
.Select(x => x.idx)
.ToList();
lCenterXValues = lCenterXValues
.Select((dVal, idx) => new { dVal, idx })
.Where(x => indexes.Contains(x.idx))
.Select(x => x.dVal).ToList();
lCenterYValues = lCenterYValues
.Select((dVal, idx) => new { dVal, idx })
.Where(x => indexes.Contains(x.idx))
.Select(x => x.dVal).ToList();
lRadiiValues = lRadiiValues
.Select((dVal, idx) => new { dVal, idx })
.Where(x => indexes.Contains(x.idx))
.Select(x => x.dVal).ToList();
DescriptiveStatistics stats = new DescriptiveStatistics(lCenterXValues, true);
double dCenterXvalue = stats.Mean;
stats = new DescriptiveStatistics(lCenterYValues, true);
double dCenterYvalue = stats.Mean;
stats = new DescriptiveStatistics(lRadiiValues, true);
double dRadiusvalue = stats.Mean;
RoundData rd = new RoundData(dCenterXvalue, dCenterYvalue, dRadiusvalue);
string xmlFilename = Path.GetDirectoryName(currentFullFileName);
xmlFilename += (xmlFilename.Last() == Path.DirectorySeparatorChar)
? ("")
: (Path.DirectorySeparatorChar.ToString());
xmlFilename += Path.GetFileNameWithoutExtension(currentFullFileName) + "-RoundImagemask.xml";
ServiceTools.WriteObjectToXML(rd, xmlFilename);
Console.WriteLine("finished processing file " + Environment.NewLine + currentFullFileName);
}
catch (Exception ex)
{
#region report
#if DEBUG
Console.WriteLine("exception has been thrown: " + ex.Message + Environment.NewLine +
ServiceTools.CurrentCodeLineDescription());
#else
ServiceTools.logToTextFile(errorLogFilename,
"exception has been thrown: " + ex.Message + Environment.NewLine +
ServiceTools.CurrentCodeLineDescription(), true, true);
#endif
#endregion report
}
}
// private List<bool> bgwFinished = new List<bool>();
public void CollectPositioningData()
{
#if DEBUG
int concurrentFilesProcessingCount = 1;
#else
int concurrentFilesProcessingCount = 2;
try
{
concurrentFilesProcessingCount = Convert.ToInt32(defaultProperties["MaxConcurrentFilesProcessingCount"]);
}
catch (Exception ex)
{
theLogWindow = ServiceTools.LogAText(theLogWindow,
"ERROR! exception thrown: " + ex.Message + Environment.NewLine +
"MaxConcurrentFilesProcessingCount value can`t be read. " + Environment.NewLine +
"Using default value = " + concurrentFilesProcessingCount);
}
#endif
theLogWindow = ServiceTools.LogAText(theLogWindow, "started on " + DateTime.UtcNow.ToString("s"));
string strImagesDir = ParentForm.richTextBox1.Text;
DirectoryInfo imagesDir = new DirectoryInfo(strImagesDir);
if (!imagesDir.Exists)
{
theLogWindow = ServiceTools.LogAText(theLogWindow,
"Операция не выполнена. Не найдена директория:" + Environment.NewLine + strImagesDir +
Environment.NewLine, true);
return;
}
// оказалось, что маска - case-insensitive
lImagesFileInfos = new List <FileInfo>(imagesDir.GetFiles("*.jpg", SearchOption.TopDirectoryOnly));
List <FileInfo> lImagesFileInfoWithExistingSunDiskInfo = lImagesFileInfos.ConvertAll <FileInfo>(fInfoSrc =>
{
string sunDiskInfoFileName = fInfoSrc.DirectoryName + "\\" +
Path.GetFileNameWithoutExtension(fInfoSrc.FullName) + "-SunDiskInfo.xml";
if (File.Exists(sunDiskInfoFileName))
{
return(fInfoSrc);
}
else
{
return(null);
}
});
lImagesFileInfoWithExistingSunDiskInfo.RemoveAll(fInfo => fInfo == null);
theLogWindow = ServiceTools.LogAText(theLogWindow,
"files already processed before: " + lImagesFileInfoWithExistingSunDiskInfo.Count);
lImagesFileInfos.RemoveAll(fInfo => lImagesFileInfoWithExistingSunDiskInfo.Contains(fInfo));
int filesCountToProcess = lImagesFileInfos.Count;
int filesCountAlreadyProcessed = lImagesFileInfoWithExistingSunDiskInfo.Count;
lImagesFileInfos.AddRange(lImagesFileInfoWithExistingSunDiskInfo);
//List<BackgroundWorker> bgwList = new List<BackgroundWorker>();
//for (int i = 0; i < 2; i++)
//{
// //bgwFinished.Add(true);
// //bgwList.Add(null);
//}
int currDataIdx = 1;
foreach (FileInfo finfo in lImagesFileInfos)
{
//int currentBgwID = -1;
while ((cbgBgwList.Count >= concurrentFilesProcessingCount) && (!hasToStopCollecting))
{
Application.DoEvents();
Thread.Sleep(0);
}
//while ((bgwFinished.Sum(boolVal => (boolVal) ? ((int)0) : ((int)1)) == bgwFinished.Count) && (!hasToStopCollecting))
//{
// Application.DoEvents();
// Thread.Sleep(0);
//}
if (hasToStopCollecting)
{
break;
}
//for (int i = 0; i < concurrentFilesProcessingCount; i++)
//{
theLogWindow = ServiceTools.LogAText(theLogWindow,
"" + currDataIdx + " / " + filesCountToProcess + " (+ " + filesCountAlreadyProcessed +
" processed before)");
theLogWindow = ServiceTools.LogAText(theLogWindow, Environment.NewLine + "starting: " + finfo.Name);
//object[] BGWorker2Args = new object[] { finfo, defaultProperties, currentBgwID };
object[] BGWorker2Args = new object[] { finfo, defaultProperties };
BackgroundWorker currBgw = new BackgroundWorker();
// bgwList[currentBgwID] = currBgw;
currBgw.DoWork += currBgw_DoWork;
currBgw.RunWorkerCompleted += currBgw_RunWorkerCompleted;
currBgw.RunWorkerAsync(BGWorker2Args);
cbgBgwList.Add(finfo.FullName);
if (filesCountToProcess > 0)
{
int progress = Convert.ToInt32(100.0d * (double)currDataIdx / (double)filesCountToProcess);
ThreadSafeOperations.UpdateProgressBar(ParentForm.pbUniversalProgressBar, progress);
Interlocked.Increment(ref currDataIdx);
}
}
while (cbgBgwList.Count > 0)
{
Application.DoEvents();
Thread.Sleep(0);
}
if (hasToStopCollecting)
{
return;
}
// сохраним на будущее
string strMedianPerc5StatsXMLFileName = (string)defaultProperties["DefaultMedianPerc5StatsXMLFile"];
FileInfo MedianPerc5StatsXMLFileInfo = new FileInfo(strMedianPerc5StatsXMLFileName);
string strMedianPerc5StatsXMLFilePath = MedianPerc5StatsXMLFileInfo.DirectoryName;
strMedianPerc5StatsXMLFilePath += (strMedianPerc5StatsXMLFilePath.Last() == '\\') ? ("") : ("\\");
string computedDeviationsXMLfilesPath = strMedianPerc5StatsXMLFilePath + "azimuth-dev-stats\\";
if (!ServiceTools.CheckIfDirectoryExists(computedDeviationsXMLfilesPath))
{
theLogWindow = ServiceTools.LogAText(theLogWindow,
"couldn`t locate or create directory " + computedDeviationsXMLfilesPath);
return;
}
string computedAzimuthDeviationsXMLFile = computedDeviationsXMLfilesPath + "PreComputedSunAnglesDeviationsData.xml";
List <AngleSunDeviationCalcResult> lResList =
new List <AngleSunDeviationCalcResult>(cbAnglesSunDeviationCalcResults);
ServiceTools.WriteObjectToXML(lResList, computedAzimuthDeviationsXMLFile);
ServiceTools.WriteDictionaryToXml(defaultProperties, defaultPropertiesXMLfileName, false);
// теперь посчитаем статистику
List <AngleSunDeviationCalcResult> azimuthResults = new List <AngleSunDeviationCalcResult>(lResList);
List <AngleSunDeviationCalcResult> zenithResults = new List <AngleSunDeviationCalcResult>(lResList);
azimuthResults.RemoveAll(
azimuthSunDeviationCalcResult =>
((!azimuthSunDeviationCalcResult.calculationSucceeded) ||
(!azimuthSunDeviationCalcResult.AzimuthDeviationCanBeComputed)));
zenithResults.RemoveAll(
sunDeviationCalcResult =>
((!sunDeviationCalcResult.calculationSucceeded) ||
(!sunDeviationCalcResult.ZenithDeviationCanBeComputed)));
List <double> azimuthDevValues =
azimuthResults.ConvertAll <double>(
azimuthSunDeviationCalcResult => azimuthSunDeviationCalcResult.computedAzimuthDeviationDeg);
List <double> zenithDevValues =
zenithResults.ConvertAll <double>(
azimuthSunDeviationCalcResult => azimuthSunDeviationCalcResult.computedZenithDeviationDeg);
ServiceTools.ExecMethodInSeparateThread(ParentForm, delegate()
{
HistogramDataAndProperties histAzimuthData =
new HistogramDataAndProperties(DenseVector.OfEnumerable(azimuthDevValues), 100);
HistogramCalcAndShowForm hAzimuthForm = new HistogramCalcAndShowForm("azimuth sun angle deviations (deg.)",
defaultProperties);
hAzimuthForm.HistToRepresent = histAzimuthData;
hAzimuthForm.Show();
hAzimuthForm.Represent();
});
ServiceTools.ExecMethodInSeparateThread(ParentForm, delegate()
{
HistogramDataAndProperties histZenithData =
new HistogramDataAndProperties(DenseVector.OfEnumerable(zenithDevValues), 100);
HistogramCalcAndShowForm hZenithForm = new HistogramCalcAndShowForm("zenith sun angle deviations (deg.)",
defaultProperties);
hZenithForm.HistToRepresent = histZenithData;
hZenithForm.Show();
hZenithForm.Represent();
});
ThreadSafeOperations.UpdateProgressBar(ParentForm.pbUniversalProgressBar, 0);
}
private AngleSunDeviationCalcResult CalculateDevDataForImage(FileInfo finfo, Dictionary <string, object> defaultProperties, LogWindow currImageLogWindow, bool showOnlyErrors = false)
{
FileInfo currFileInfo = finfo;
Dictionary <string, object> defaultProps = defaultProperties;
AngleSunDeviationCalcResult retRes = new AngleSunDeviationCalcResult()
{
fileName = currFileInfo.FullName,
};
GPSdata gps = ServiceTools.FindProperGPSdataForImage(currFileInfo.FullName, theLogWindow, defaultProperties,
ref NVdataFilesAlreadyReadDateTimeSpans, ref NVdataFilesAlreadyReadData);
if (gps == null)
{
theLogWindow = ServiceTools.LogAText(theLogWindow, "Couldn`t find GPS data for this image.");
retRes.calculationSucceeded = false;
retRes.resultMessage = "Couldn`t find GPS data for this image.";
return(retRes);
}
#region // obsolete
//// определяем дату-время файла
//DateTime curDateTime = DateTime.UtcNow;
//Image anImage = Image.FromFile(currFileInfo.FullName);
//ImageInfo newIInfo = new ImageInfo(anImage);
//int minute = 0;
//String dateTime = (String)newIInfo.getValueByKey("ExifDTOrig");
//if (dateTime == null)
//{
// //попробуем вытащить из имени файла
// string strDateTime = currFileInfo.Name;
// strDateTime = strDateTime.Substring(4, 19);
// dateTime = strDateTime;
//}
//try
//{
// curDateTime = CommonTools.DateTimeOfString(dateTime);
//}
//catch (Exception)
//{
// retRes.calculationSucceeded = false;
// retRes.resultMessage = "couldn`t get date/time for file: " + Environment.NewLine + currFileInfo.Name;
// return retRes;
//}
//curDateTime = DateTime.SpecifyKind(curDateTime, DateTimeKind.Utc);
//GPSdata neededGPSdata = new GPSdata();
//string currPath = currFileInfo.DirectoryName;
//string navFilesPath = defaultProps["IoffeMeteoNavFilesDirectory"] as string;
//List<IoffeVesselDualNavDataConverted> lAllNavData = new List<IoffeVesselDualNavDataConverted>();
//string[] sNavFilenames = Directory.GetFiles(navFilesPath, "*.nv2", SearchOption.AllDirectories);
//if (!sNavFilenames.Any())
//{
// retRes.calculationSucceeded = false;
// retRes.resultMessage = "Не найдено файлов данных навигации в директории " + navFilesPath;
// return retRes;
//}
//else
//{
// foreach (string navFilename in sNavFilenames)
// {
// Tuple<DateTime, DateTime> timeSpan =
// IoffeVesselNavDataReader.GetNavFileDateTimeMargins(navFilename);
// if (timeSpan == null)
// {
// continue;
// }
// if ((curDateTime < timeSpan.Item1) || (curDateTime > timeSpan.Item2))
// {
// continue;
// }
// List<IoffeVesselDualNavDataConverted> dataHasBeenRead = IoffeVesselNavDataReader.ReadNavFile(navFilename);
// if (dataHasBeenRead == null)
// {
// continue;
// }
// Application.DoEvents();
// lAllNavData.AddRange(dataHasBeenRead);
// }
//}
//lAllNavData.Sort((gpsRecord1, gpsRecord2) =>
//{
// double dev1 = Math.Abs((gpsRecord1.gps.dateTimeUTC - curDateTime).TotalMilliseconds);
// double dev2 = Math.Abs((gpsRecord2.gps.dateTimeUTC - curDateTime).TotalMilliseconds);
// return (dev1 >= dev2) ? (1) : (-1);
//});
//neededGPSdata = lAllNavData[0].gps;
#endregion // obsolete
retRes.gpsData = gps;
#region obsolete
//double lat = gps.LatDec;
//double lon = gps.LonDec;
//SPA spaCalc = new SPA(curDateTime.Year, curDateTime.Month, curDateTime.Day, curDateTime.Hour,
// curDateTime.Minute, curDateTime.Second, (float)lon, (float)lat,
// (float)SPAConst.DeltaT(curDateTime));
//int res = spaCalc.spa_calculate();
//AzimuthZenithAngle sunPositionSPAext = new AzimuthZenithAngle(spaCalc.spa.azimuth,
// spaCalc.spa.zenith);
#endregion obsolete
AzimuthZenithAngle sunPositionSPAext = gps.SunZenithAzimuth();
if (!showOnlyErrors)
{
currImageLogWindow = ServiceTools.LogAText(currImageLogWindow,
"SPA ext sun position for " + gps.dateTimeUTC.ToString("s") + ": " + sunPositionSPAext);
}
retRes.sunSPAcomputedPosition = sunPositionSPAext;
Image <Bgr, Byte> img2process = new Image <Bgr, byte>(currFileInfo.FullName);
img2process = ImageProcessing.ImageResizer(img2process, Convert.ToInt32(defaultProps["DefaultMaxImageSize"]));
Image <Bgr, Byte> LocalProcessingImage = ImageProcessing.SquareImageDimensions(img2process);
RoundData sunRoundData = RoundData.nullRoundData();
//посмотрим, нет ли уже имеющихся данных о положении и размере солнечного диска на изображении
string sunDiskInfoFileName = ConventionalTransitions.SunDiskInfoFileName(currFileInfo.FullName);
//string sunDiskInfoFileName = currFileInfo.DirectoryName + "\\" +
// Path.GetFileNameWithoutExtension(currFileInfo.FullName) + "-SunDiskInfo.xml";
RoundData existingRoundData = RoundData.nullRoundData();
Size imgSizeUnderExistingRoundData = LocalProcessingImage.Bitmap.Size;
object existingRoundDataObj = ServiceTools.ReadObjectFromXML(sunDiskInfoFileName, typeof(RoundDataWithUnderlyingImgSize));
if (existingRoundDataObj != null)
{
existingRoundData = ((RoundDataWithUnderlyingImgSize)existingRoundDataObj).circle;
imgSizeUnderExistingRoundData = ((RoundDataWithUnderlyingImgSize)existingRoundDataObj).imgSize;
}
double currScale = (double)LocalProcessingImage.Width / (double)imgSizeUnderExistingRoundData.Width;
if (currScale != 1.0d)
{
existingRoundData.DCenterX *= currScale;
existingRoundData.DCenterY *= currScale;
existingRoundData.DRadius *= currScale;
}
if (!existingRoundData.IsNull)
{
sunRoundData = existingRoundData;
}
ImageProcessing imgP = new ImageProcessing(LocalProcessingImage, true);
if (sunRoundData.IsNull)
{
SkyCloudClassification classificator = new SkyCloudClassification(img2process, defaultProperties);
classificator.verbosityLevel = 0;
classificator.ParentForm = ParentForm;
classificator.theLogWindow = currImageLogWindow;
classificator.ClassificationMethod = ClassificationMethods.GrIx;
classificator.isCalculatingUsingBgWorker = false;
// classificator.SelfWorker = currBGWsender as BackgroundWorker;
classificator.defaultOutputDataDirectory = (string)defaultProps["DefaultDataFilesLocation"];
classificator.theStdDevMarginValueDefiningSkyCloudSeparation =
Convert.ToDouble(defaultProps["GrIxDefaultSkyCloudMarginWithoutSun"]);
classificator.sourceImageFileName = currFileInfo.FullName;
retRes.imageEdgesDetected = new RoundDataWithUnderlyingImgSize()
{
circle = imgP.imageRD,
imgSize = LocalProcessingImage.Size,
};
DenseMatrix dmProcessingData = (DenseMatrix)imgP.eval("grix").Clone();
try
{
sunRoundData = classificator.DetectSunWithSerieOfArcs(imgP, dmProcessingData);
if (!sunRoundData.IsNull)
{
RoundDataWithUnderlyingImgSize infoToSave = new RoundDataWithUnderlyingImgSize()
{
circle = sunRoundData,
imgSize = LocalProcessingImage.Size,
};
ServiceTools.WriteObjectToXML(infoToSave, sunDiskInfoFileName);
}
}
catch (Exception ex)
{
retRes.calculationSucceeded = false;
retRes.resultMessage = ex.Message;
return(retRes);
}
ServiceTools.FlushMemory();
}
if (sunRoundData.IsNull)
{
throw new Exception(finfo.Name + ": couldn`t detect sun position");
}
else
{
retRes.sunDiskDetectedPosition = new RoundDataWithUnderlyingImgSize()
{
circle = sunRoundData,
imgSize = LocalProcessingImage.Size,
};
}
RoundData imageDetectedRound = imgP.imageRD.Copy();
retRes.imageEdgesDetected = new RoundDataWithUnderlyingImgSize()
{
circle = imageDetectedRound,
imgSize = LocalProcessingImage.Size,
};
try
{
double dev = retRes.computedAzimuthDeviation;
retRes.calculationSucceeded = true;
}
catch (Exception ex)
{
retRes.calculationSucceeded = false;
retRes.resultMessage = ex.Message;
return(retRes);
}
return(retRes);
}
private void btnCreateList_Click(object sender, EventArgs e)
{
theLogWindow = ServiceTools.LogAText(theLogWindow, "Started processing...");
BackgroundWorker bgwCreateList = new BackgroundWorker();
bgwCreateList.DoWork += (bgwSender, bgwArgs) =>
{
List <ImageFileDescription> lImagesFilesList = new List <ImageFileDescription>();
List <ImagesProcessingData> lImagesAllConcurrentData = new List <ImagesProcessingData>();
List <ObservedClCoverData> lObservedData = new List <ObservedClCoverData>();
#region check available data
//ImagesBaseSourcePath = "";
//ConcurrentAndStatsXMLfilesDir = "";
//ObservedDataCSVfile = "";
//DestinationPath = "";
//DateTimeFilterTolerance = new TimeSpan(0, 5, 0);
#region ImagesBaseSourcePath
theLogWindow = ServiceTools.LogAText(theLogWindow, "Started enumerating images...");
if (!Directory.Exists(ImagesBaseSourcePath))
{
theLogWindow = ServiceTools.LogAText(theLogWindow,
"Unable to find sky-images base source path. Please check if it does exist and contain at least one sky-image *.jpg file.");
FINISHED();
return;
}
lImagesFilesList =
Directory.GetFiles(ImagesBaseSourcePath, "*.jpg", SearchOption.AllDirectories)
.ToList()
.ConvertAll(str => new ImageFileDescription(str));
if (!lImagesFilesList.Any())
{
theLogWindow = ServiceTools.LogAText(theLogWindow,
"Unable to find any sky-images. Please check if at least one sky-image exists in base source path.");
FINISHED();
return;
}
#endregion ImagesBaseSourcePath
#region ConcurrentAndStatsXMLfilesDir
theLogWindow = ServiceTools.LogAText(theLogWindow, "Started reading concurrent data and stats data using existing XML files...");
if (!Directory.Exists(ConcurrentAndStatsXMLfilesDir))
{
theLogWindow = ServiceTools.LogAText(theLogWindow,
"Unable to find sky-images pre-calculated concurrent data XML files directory: " +
Environment.NewLine + ConcurrentAndStatsXMLfilesDir + Environment.NewLine +
". Please check if it does exist.");
FINISHED();
return;
}
#region read all concurrent data using all-included XML files
List <string> lXMLfiles =
Directory.GetFiles(ConcurrentAndStatsXMLfilesDir, "ImagesCameraPositioning-stats-*-camID?.xml", SearchOption.AllDirectories).ToList();
if (!lXMLfiles.Any())
{
theLogWindow = ServiceTools.LogAText(theLogWindow,
"Unable to find any XML file satisfied the following mask: \"ImagesCameraPositioning-stats-*-camID?.xml\". Please check if at least one XML of that kind does exist in directory " +
ConcurrentAndStatsXMLfilesDir);
FINISHED();
return;
}
foreach (string xmlFile in lXMLfiles)
{
theLogWindow = ServiceTools.LogAText(theLogWindow, "trying to read and parse file " + xmlFile);
try
{
List <ImagesProcessingData> currFileContents =
ServiceTools.ReadObjectFromXML(xmlFile, typeof(List <ImagesProcessingData>)) as
List <ImagesProcessingData>;
if (currFileContents != null)
{
lImagesAllConcurrentData.AddRange(currFileContents);
}
}
catch (Exception ex)
{
theLogWindow = ServiceTools.LogAText(theLogWindow,
"ERROR! Unable to read data from file: " + xmlFile + Environment.NewLine + ex.Message);
continue;
}
}
if (!lImagesAllConcurrentData.Any())
{
theLogWindow = ServiceTools.LogAText(theLogWindow,
"Unable to read any concurrent data. Please check the directory for files containing valid concurrent data XML files: " + Environment.NewLine +
"directory: " + ConcurrentAndStatsXMLfilesDir + Environment.NewLine +
"XML files mask: \"ImagesCameraPositioning-stats-*-camID?.xml\"");
FINISHED();
return;
}
#endregion read all concurrent data using all-included XML files
#endregion ConcurrentAndStatsXMLfilesDir
#region ObservedDataCSVfile
theLogWindow = ServiceTools.LogAText(theLogWindow, "Started reading observed data CSV file...");
if (!File.Exists(ObservedDataCSVfile))
{
theLogWindow = ServiceTools.LogAText(theLogWindow,
"Unable to read observed data CSV file: " + ObservedDataCSVfile);
FINISHED();
return;
}
List <List <string> > lCSVfileContents = ServiceTools.ReadDataFromCSV(ObservedDataCSVfile, 1, true);
if (!lCSVfileContents.Any())
{
theLogWindow = ServiceTools.LogAText(theLogWindow,
"Unable to read observed data CSV file: " + ObservedDataCSVfile);
FINISHED();
return;
}
lObservedData = lCSVfileContents.ConvertAll(lStr => new ObservedClCoverData(lStr));
#endregion ObservedDataCSVfile
#region DestinationPath
if (!Directory.Exists(DestinationPath))
{
theLogWindow = ServiceTools.LogAText(theLogWindow,
"Unable to find the output directory: " + DestinationPath);
FINISHED();
return;
}
#endregion DestinationPath
#endregion check available data
List <Tuple <ObservedClCoverData, ImageFileDescription> > lImagesFilteredByAvailableObservedData =
new List <Tuple <ObservedClCoverData, ImageFileDescription> >();
#region filter images by available observed data using DateTimeFilterTolerance
theLogWindow = ServiceTools.LogAText(theLogWindow, "Filtering by observed data available...");
foreach (ObservedClCoverData observedData in lObservedData)
{
DateTime currObservedDatumDateTime = observedData.dt;
List <Tuple <ObservedClCoverData, ImageFileDescription> > lImagesCloseToCurrObservedDatum = lImagesFilesList
.Where(ifd =>
{
TimeSpan tspan =
new TimeSpan(Math.Abs((ifd.currImageDateTime - currObservedDatumDateTime).Ticks));
return(tspan <= DateTimeFilterTolerance);
})
.ToList()
.ConvertAll(ifd => new Tuple <ObservedClCoverData, ImageFileDescription>(observedData, ifd));
lImagesFilteredByAvailableObservedData.AddRange(lImagesCloseToCurrObservedDatum);
}
#endregion filter images by available observed data using DateTimeFilterTolerance
List <SkyImagesDataWith_Concurrent_Stats_CloudCover> lImagesFilteredByAnyAvailableData =
new List <SkyImagesDataWith_Concurrent_Stats_CloudCover>();
#region map available stats data using image filename
theLogWindow = ServiceTools.LogAText(theLogWindow, "Mapping concurrent and stats data...");
lImagesFilteredByAnyAvailableData = lImagesFilteredByAvailableObservedData.ConvertAll(tpl =>
{
SkyImagesDataWith_Concurrent_Stats_CloudCover retVal = null;
try
{
ImagesProcessingData foundConcurrentData =
lImagesAllConcurrentData.Where(ipd => Path.GetFileName(ipd.filename) == tpl.Item2.fileName)
.ElementAt(0);
retVal = new SkyImagesDataWith_Concurrent_Stats_CloudCover()
{
skyImageFullFileName = tpl.Item2.fullFileName,
skyImageFileName = tpl.Item2.fileName,
currImageDateTime = tpl.Item2.currImageDateTime,
observedCloudCoverData = tpl.Item1,
concurrentDataXMLfile = foundConcurrentData.concurrentDataXMLfile,
concurrentData = foundConcurrentData.concurrentData,
grixyrgbStatsXMLfile = foundConcurrentData.grixyrgbStatsXMLfile,
grixyrgbStats = foundConcurrentData.grixyrgbStats
};
//(tpl.Item1, tpl.Item2, foundConcurrentData);
}
catch (Exception ex)
{
theLogWindow = ServiceTools.LogAText(theLogWindow,
"ERROR! Couldn`t find concurrent data for file " + Path.GetFileName(tpl.Item2.fileName) +
Environment.NewLine + ex.Message);
}
return(retVal);
});
lImagesFilteredByAnyAvailableData.RemoveAll(tpl => tpl == null);
if (!lImagesFilteredByAnyAvailableData.Any())
{
theLogWindow = ServiceTools.LogAText(theLogWindow,
"There is no images remain after filtering using all available data. Output will be empty.");
}
#endregion map available stats data using image filename
theLogWindow = ServiceTools.LogAText(theLogWindow, "Writing output list to file...");
string strOutputXMLfileName = DestinationPath +
((DestinationPath.Last() == Path.DirectorySeparatorChar)
? ("")
: (Path.DirectorySeparatorChar.ToString())) + "FilesListToDetectCloudCover.xml";
ServiceTools.WriteObjectToXML(lImagesFilteredByAnyAvailableData, strOutputXMLfileName);
theLogWindow = ServiceTools.LogAText(theLogWindow, "images list wrote to file: " + strOutputXMLfileName);
};
bgwCreateList.RunWorkerAsync();
}
private async void Bot_OnCallbackQuery(object sender, CallbackQueryEventArgs callbackQueryEventArgs)
{
CallbackQuery currQuery = callbackQueryEventArgs.CallbackQuery;
//search for dialog var choosing message
bool bFoundByVarChoosingMessage =
lDialogs.Any(dlg => dlg.varChoosingMessage.MessageId == currQuery.Message.MessageId);
List <UserDataFixingDialogs> lDialogsWithValueRequestMessages =
lDialogs.Where(dlg => dlg.valueRequestMessage != null).ToList();
bool bFoundByValueRequestMessage =
lDialogsWithValueRequestMessages.Any(
dlg => dlg.valueRequestMessage.MessageId == currQuery.Message.MessageId);
if (!bFoundByVarChoosingMessage && !bFoundByValueRequestMessage)
{
string replMessage = "Извините, кажется, я все забыл." + Environment.NewLine +
"Попробуйте заново запросить снимок, информацию о нем и исправить эти данные.";
await Bot.SendTextMessageAsync(currQuery.Message.Chat.Id, replMessage, true, false,
currQuery.Message.MessageId, BasicKeyboard());
return;
}
UserDataFixingDialogs currDialog = null;
if (bFoundByVarChoosingMessage)
{
currDialog =
lDialogs.First(
dlg => dlg.varChoosingMessage.MessageId == currQuery.Message.MessageId);
}
else if (bFoundByValueRequestMessage)
{
currDialog =
lDialogs.First(
dlg => dlg.valueRequestMessage.MessageId == currQuery.Message.MessageId);
}
if (currDialog.varname == null)
{
if ((currQuery.Data == "varnameSDC") || (currQuery.Data == "varnameTCC"))
{
currDialog.varname = currQuery.Data;
if (currDialog.varname == "varnameSDC")
{
InlineKeyboardMarkup kb = new InlineKeyboardMarkup(new[]
{
new InlineKeyboardButton[]
{
new InlineKeyboardButton("Cloudy", "SDC_Cloudy"),
new InlineKeyboardButton("Sun_0", "SDC_Sun_0"),
new InlineKeyboardButton("Sun_1", "SDC_Sun_1"),
new InlineKeyboardButton("Sun_2", "SDC_Sun_2"),
},
new InlineKeyboardButton[]
{
new InlineKeyboardButton("Defect", "SDC_Defect"),
}
});
Message sentMsg = await Bot.SendTextMessageAsync(currQuery.Message.Chat.Id, "Please choose value",
replyMarkup : kb);
currDialog.valueRequestMessage = sentMsg;
return;
}
else if (currDialog.varname == "varnameTCC")
{
InlineKeyboardMarkup kb = new InlineKeyboardMarkup(new[]
{
new InlineKeyboardButton[]
{
new InlineKeyboardButton("0", "TCC_value_0"),
new InlineKeyboardButton("1", "TCC_value_1"),
new InlineKeyboardButton("2", "TCC_value_2"),
new InlineKeyboardButton("3", "TCC_value_3"),
},
new InlineKeyboardButton[]
{
new InlineKeyboardButton("4", "TCC_value_4"),
new InlineKeyboardButton("5", "TCC_value_5"),
new InlineKeyboardButton("6", "TCC_value_6"),
new InlineKeyboardButton("7", "TCC_value_7"),
},
new InlineKeyboardButton[]
{
new InlineKeyboardButton("8", "TCC_value_8"),
new InlineKeyboardButton("Defect", "TCC_Defect"),
}
});
Message sentMsg = await Bot.SendTextMessageAsync(currQuery.Message.Chat.Id, "Please choose value",
replyMarkup : kb);
currDialog.valueRequestMessage = sentMsg;
return;
}
return;
}
else
{
string replMessage = "Извините, меня еще не научили разбирать такие сложные команды.";
await Bot.SendTextMessageAsync(currQuery.Message.Chat.Id, replMessage, true, false, currQuery.Message.MessageId, BasicKeyboard());
return;
}
}
else
{
if (currDialog.varname == "varnameSDC")
{
currDialog.varvalue = currQuery.Data;
string replMessage =
"Благодарим Вас за участие в обучении нашей модели (SDC). Ваше исправление отправлено на ревизию и будет учтено при дообучении.";
await
Bot.SendTextMessageAsync(currQuery.Message.Chat.Id, replMessage, true, false,
currQuery.Message.MessageId, BasicKeyboard());
ServiceTools.WriteObjectToXML(currDialog, Directory.GetCurrentDirectory() +
Path.DirectorySeparatorChar + "logs" +
Path.DirectorySeparatorChar +
"model_fix_" + currDialog.varname + "_" + currDialog.varChoosingMessage.MessageId +
".xml");
lDialogs.Remove(currDialog);
return;
}
else if (currDialog.varname == "varnameTCC")
{
currDialog.varvalue = currQuery.Data;
string replMessage =
"Благодарим Вас за участие в обучении нашей модели (TCC). Ваше исправление отправлено на ревизию и будет учтено при дообучении.";
await
Bot.SendTextMessageAsync(currQuery.Message.Chat.Id, replMessage, true, false,
currQuery.Message.MessageId, BasicKeyboard());
ServiceTools.WriteObjectToXML(currDialog, Directory.GetCurrentDirectory() +
Path.DirectorySeparatorChar + "logs" +
Path.DirectorySeparatorChar +
"model_fix_" + currDialog.varname + "_" + currDialog.varChoosingMessage.MessageId +
".xml");
lDialogs.Remove(currDialog);
return;
}
else
{
string replMessage = "Извините, меня еще не научили разбирать такие сложные команды.";
await Bot.SendTextMessageAsync(currQuery.Message.Chat.Id, replMessage, true, false, currQuery.Message.MessageId, BasicKeyboard());
return;
}
}
}