public static string FindConcurrentDataXMLfile(string ImageFilename, string basePath        = "",
                                                       bool searchFilesRecursively                  = true,
                                                       ServiceTools.DatetimeExtractionMethod method = ServiceTools.DatetimeExtractionMethod.Filename)
        {
            if (basePath == "")
            {
                basePath = Directory.GetCurrentDirectory();
                basePath = basePath +
                           ((basePath.Last() == Path.DirectorySeparatorChar)
                               ? ("")
                               : (Path.DirectorySeparatorChar.ToString()));
                basePath += "results" + Path.DirectorySeparatorChar;
            }


            string nearestConcurrentDataXMLfilename = "";



            string currImgFilename = ImageFilename;

            ServiceTools.DatetimeExtractionMethod _method = method;

            #region find most close concurrent data XML file

            List <FileDatetimeInfo> lImagesConcurrentDataFilesInfo = new List <FileDatetimeInfo>();

            List <string> filesListConcurrentData =
                Directory.EnumerateFiles(basePath,
                                         ConventionalTransitions.ImageConcurrentDataFilesNamesPattern(),
                                         (searchFilesRecursively) ? (SearchOption.AllDirectories) : (SearchOption.TopDirectoryOnly))
                .ToList();


            foreach (string strConcDataXMLFile in filesListConcurrentData)
            {
                try
                {
                    lImagesConcurrentDataFilesInfo.Add(new FileDatetimeInfo(strConcDataXMLFile,
                                                                            _method));
                }
                catch (Exception ex)
                {
                    continue;
                }
            }


            string currImgFilenameWOext = Path.GetFileNameWithoutExtension(currImgFilename);

            string ptrn = @"(devID\d)";
            Regex  rgxp = new Regex(ptrn, RegexOptions.IgnoreCase);

            string strCurrImgDT = rgxp.Replace(currImgFilenameWOext.Substring(4), "");
            //2015-12-16T06-01-38
            strCurrImgDT = strCurrImgDT.Substring(0, 11) + strCurrImgDT.Substring(11).Replace("-", ":");

            DateTime currImgDT = DateTime.Parse(strCurrImgDT, null,
                                                DateTimeStyles.AdjustToUniversal);
            DateTime.SpecifyKind(currImgDT, DateTimeKind.Utc);

            FileDatetimeInfo nearestConcurrentDataXMLfileInfo = lImagesConcurrentDataFilesInfo.Aggregate((cDtFinfo1, cDtFinfo2) =>
            {
                TimeSpan tspan1 = new TimeSpan(Math.Abs((cDtFinfo1.datetime - currImgDT).Ticks));
                TimeSpan tspan2 = new TimeSpan(Math.Abs((cDtFinfo2.datetime - currImgDT).Ticks));
                return((tspan1 <= tspan2) ? (cDtFinfo1) : (cDtFinfo2));
            });


            if (new TimeSpan(Math.Abs((nearestConcurrentDataXMLfileInfo.datetime - currImgDT).Ticks)) >=
                new TimeSpan(0, 3, 0))
            {
                //theLogWindow = ServiceTools.LogAText(theLogWindow,
                //    "couldn`t find close enough concurrent data file for image:" + Environment.NewLine +
                //    currImgFilename + Environment.NewLine + "closest concurrent data file is:" +
                //    Environment.NewLine + nearestConcurrentDataXMLfileInfo.filename + Environment.NewLine +
                //    "with date-time value " + nearestConcurrentDataXMLfileInfo.datetime.ToString("o"));
                nearestConcurrentDataXMLfilename = "";
            }
            else
            {
                nearestConcurrentDataXMLfilename = nearestConcurrentDataXMLfileInfo.filename;
            }


            #endregion find most close concurrent data


            return(nearestConcurrentDataXMLfilename);
        }
        public async Task<SunDiskCondition> CalcSDC_NN(string SDC_NNconfigFile, string SDC_NNtrainedParametersFile, string NormMeansFile, string NormRangeFile)
        {
            if (!File.Exists(SDC_NNconfigFile))
            {
                throw new FileNotFoundException("couldn`t find the file specified: " + SDC_NNconfigFile);
            }
            if (!File.Exists(SDC_NNtrainedParametersFile))
            {
                throw new FileNotFoundException("couldn`t find the file specified: " + SDC_NNtrainedParametersFile);
            }
            if (!File.Exists(NormMeansFile))
            {
                throw new FileNotFoundException("couldn`t find the file specified: " + NormMeansFile);
            }
            if (!File.Exists(NormRangeFile))
            {
                throw new FileNotFoundException("couldn`t find the file specified: " + NormRangeFile);
            }

            if ((ImagesRoundMasksXMLfilesMappingList == "") || (ImagesRoundMasksXMLfilesMappingList == null))
            {
                ImagesRoundMasksXMLfilesMappingList = Directory.GetCurrentDirectory() + Path.DirectorySeparatorChar + "settings" +
                                          Path.DirectorySeparatorChar +
                                          "ImagesRoundMasksXMLfilesMappingList.csv";
            }
            if (!File.Exists(ImagesRoundMasksXMLfilesMappingList))
            {
                throw new FileNotFoundException("couldn`t find the file specified: " + ImagesRoundMasksXMLfilesMappingList);
            }



            #region read or calculate GrIxYRGB stats

            if (Directory.Exists(imageYRGBstatsXMLdataFilesDirectory))
            {
                List<string> foundXMLfiles = Directory.EnumerateFiles(imageYRGBstatsXMLdataFilesDirectory,
                    ConventionalTransitions.ImageGrIxYRGBstatsDataFileName(ImageFilename, "", false),
                    SearchOption.AllDirectories).ToList();
                if (foundXMLfiles.Any())
                {
                    currImageStatsData =
                        (SkyImageIndexesStatsData)
                            ServiceTools.ReadObjectFromXML(foundXMLfiles[0], typeof(SkyImageIndexesStatsData));
                }
            }


            if (currImageStatsData == null)
            {
                Task<SkyImageIndexesStatsData> tskImageStatsCalculation = new Task<SkyImageIndexesStatsData>(() =>
                {
                    Dictionary<string, object> optionalParameters = new Dictionary<string, object>();
                    optionalParameters.Add("ImagesRoundMasksXMLfilesMappingList", ImagesRoundMasksXMLfilesMappingList);
                    Stopwatch sw = new Stopwatch();
                    sw.Start();
                    optionalParameters.Add("Stopwatch", sw);

                    ImageStatsDataCalculationResult currImageProcessingResult = null;
                    try
                    {
                        currImageProcessingResult = ImageProcessing.CalculateImageStatsData(ImageFilename,
                            optionalParameters);
                    }
                    catch (Exception ex)
                    {
                        return null;
                    }



                    if (currImageProcessingResult == null)
                    {
                        return null;
                    }
                    else
                    {
                        string strImageGrIxYRGBDataFileName =
                            ConventionalTransitions.ImageGrIxYRGBstatsDataFileName(ImageFilename,
                                imageYRGBstatsXMLdataFilesDirectory);
                        ServiceTools.WriteObjectToXML(currImageProcessingResult.grixyrgbStatsData,
                            strImageGrIxYRGBDataFileName);

                        // currImageProcessingResult.stopwatch.Stop();
                        // string currentFullFileName = currImageProcessingResult.imgFilename;
                        // string strPerfCountersData = currentFullFileName + ";" +
                        //                             currImageProcessingResult.stopwatch.ElapsedMilliseconds + ";" +
                        //                             (currImageProcessingResult.procTotalProcessorTimeEnd -
                        //                              currImageProcessingResult.procTotalProcessorTimeStart)
                        //                                 .TotalMilliseconds +
                        //                             Environment.NewLine;
                        // ServiceTools.logToTextFile(strPerformanceCountersStatsFile, strPerfCountersData, true);

                        currImageStatsData = currImageProcessingResult.grixyrgbStatsData;



                        return currImageProcessingResult.grixyrgbStatsData;
                    }
                });

                currImageStatsData = await tskImageStatsCalculation;
            }

            if (currImageStatsData == null)
            {
                // theLogWindow = ServiceTools.LogAText(theLogWindow, "ERROR searching concurrent data for image. Will not proceed.");
                throw new Exception("ERROR searching concurrent data for image. Will not proceed.");
            }

            #endregion read or calculate GrIxYRGB stats


            ConcurrentData nearestConcurrentData = null;



            #region search for concurrent data

            Task<ConcurrentData> tskImageConcurrentDataSearching = new Task<ConcurrentData>(() =>
            {
                string currImgFilename = Path.GetFileNameWithoutExtension(ImageFilename);
                string ptrn = @"(devID\d)";
                Regex rgxp = new Regex(ptrn, RegexOptions.IgnoreCase);

                string strCurrImgDT = rgxp.Replace(currImgFilename.Substring(4), "");
                //2015-12-16T06-01-38
                strCurrImgDT = strCurrImgDT.Substring(0, 11) + strCurrImgDT.Substring(11).Replace("-", ":");

                DateTime currImgDT = DateTime.Parse(strCurrImgDT, null,
                    System.Globalization.DateTimeStyles.AdjustToUniversal);



                List<Tuple<string, ConcurrentData>> lImagesConcurrentData =
                    new List<Tuple<string, ConcurrentData>>();

                List<string> filesListConcurrentData =
                    Directory.EnumerateFiles(ConcurrentDataXMLfilesBasePath,
                        ConventionalTransitions.ImageConcurrentDataFilesNamesPattern(),
                        SearchOption.AllDirectories)
                        .ToList();

                List<Tuple<string, DateTime>> XMLfilesListConcurrentData = filesListConcurrentData.ConvertAll(
                    strXMLfilename =>
                    {
                        string xmlFile1DateTimeString =
                            Path.GetFileNameWithoutExtension(strXMLfilename).Replace("data-", "").Substring(0, 19);
                        xmlFile1DateTimeString = xmlFile1DateTimeString.Substring(0, 11) + xmlFile1DateTimeString.Substring(11).Replace("-", ":");
                        DateTime dt1 = DateTime.Parse(xmlFile1DateTimeString);
                        return new Tuple<string, DateTime>(strXMLfilename, dt1);
                    });

                string nearestConcurrentDataFileName = XMLfilesListConcurrentData.Aggregate((cDt1, cDt2) =>
                {
                    TimeSpan tspan1 = new TimeSpan(Math.Abs((cDt1.Item2 - currImgDT).Ticks));
                    TimeSpan tspan2 = new TimeSpan(Math.Abs((cDt2.Item2 - currImgDT).Ticks));
                    return ((tspan1 <= tspan2) ? (cDt1) : (cDt2));
                }).Item1;

                Dictionary<string, object> currDict = ServiceTools.ReadDictionaryFromXML(nearestConcurrentDataFileName);
                currDict.Add("XMLfileName", Path.GetFileName(nearestConcurrentDataFileName));
                ConcurrentData nearestConcurrentDataObtained = new ConcurrentData(currDict);

                if (new TimeSpan(Math.Abs((nearestConcurrentDataObtained.datetimeUTC - currImgDT).Ticks)) >=
                    new TimeSpan(0, 2, 0))
                {

                    //theLogWindow = ServiceTools.LogAText(theLogWindow,
                    //    "couldn`t find close enough concurrent data file for image:" + Environment.NewLine +
                    //    bgwCurrImageFInfo.FullName + Environment.NewLine + "closest concurrent data file is:" +
                    //    Environment.NewLine + nearestConcurrentData.filename + Environment.NewLine +
                    //    "with date-time value " + nearestConcurrentData.datetimeUTC.ToString("o"));
                    nearestConcurrentDataObtained = null;
                }

                return nearestConcurrentDataObtained;
            });

            nearestConcurrentData = await tskImageConcurrentDataSearching;

            if (nearestConcurrentData == null)
            {
                // theLogWindow = ServiceTools.LogAText(theLogWindow, "ERROR searching concurrent data for image. Will not proceed.");
                throw new Exception("ERROR searching concurrent data for image. Will not proceed.");
            }

            #endregion search for concurrent data


            DenseVector dvMeans = (DenseVector)((DenseMatrix)ServiceTools.ReadDataFromCSV(NormMeansFile, 0, ",")).Row(0);
            DenseVector dvRanges = (DenseVector)((DenseMatrix)ServiceTools.ReadDataFromCSV(NormRangeFile, 0, ",")).Row(0);
            DenseVector dvThetaValues = (DenseVector)ServiceTools.ReadDataFromCSV(SDC_NNtrainedParametersFile, 0, ",");
            List<int> NNlayersConfig =
                new List<double>(((DenseMatrix)ServiceTools.ReadDataFromCSV(SDC_NNconfigFile, 0, ",")).Row(0)).ConvertAll
                    (dVal => Convert.ToInt32(dVal));
            List<double> decisionProbabilities = new List<double>();
            return PredictSDC_NN(currImageStatsData, nearestConcurrentData, NNlayersConfig, dvThetaValues, dvMeans,
                dvRanges, out decisionProbabilities);
        }
Пример #3
0
        public async Task <int> CalcTCC_NN(string SDC_NNconfigFile, string SDC_NNtrainedParametersFile,
                                           string SDC_NormMeansFile,
                                           string SDC_NormRangeFile, string TCC_NNconfigFile, string TCC_NNtrainedParametersFile,
                                           string TCC_NormMeansFile, string TCC_NormRangeFile, string TCC_ExcludingVarsFile)
        {
            #region check files existence
            if (!File.Exists(SDC_NNconfigFile))
            {
                throw new FileNotFoundException("couldn`t find the file specified: " + SDC_NNconfigFile);
            }
            if (!File.Exists(SDC_NNtrainedParametersFile))
            {
                throw new FileNotFoundException("couldn`t find the file specified: " + SDC_NNtrainedParametersFile);
            }
            if (!File.Exists(SDC_NormMeansFile))
            {
                throw new FileNotFoundException("couldn`t find the file specified: " + SDC_NormMeansFile);
            }
            if (!File.Exists(SDC_NormRangeFile))
            {
                throw new FileNotFoundException("couldn`t find the file specified: " + SDC_NormRangeFile);
            }

            if (!File.Exists(TCC_NNconfigFile))
            {
                throw new FileNotFoundException("couldn`t find the file specified: " + TCC_NNconfigFile);
            }
            if (!File.Exists(TCC_NNtrainedParametersFile))
            {
                throw new FileNotFoundException("couldn`t find the file specified: " + TCC_NNtrainedParametersFile);
            }
            if (!File.Exists(TCC_NormMeansFile))
            {
                throw new FileNotFoundException("couldn`t find the file specified: " + TCC_NormMeansFile);
            }
            if (!File.Exists(TCC_NormRangeFile))
            {
                throw new FileNotFoundException("couldn`t find the file specified: " + TCC_NormRangeFile);
            }
            if (!File.Exists(TCC_ExcludingVarsFile))
            {
                throw new FileNotFoundException("couldn`t find the file specified: " + TCC_ExcludingVarsFile);
            }

            if ((ImagesRoundMasksXMLfilesMappingList == "") || (ImagesRoundMasksXMLfilesMappingList == null))
            {
                ImagesRoundMasksXMLfilesMappingList = Directory.GetCurrentDirectory() + Path.DirectorySeparatorChar + "settings" +
                                                      Path.DirectorySeparatorChar +
                                                      "ImagesRoundMasksXMLfilesMappingList.csv";
            }
            if (!File.Exists(ImagesRoundMasksXMLfilesMappingList))
            {
                throw new FileNotFoundException("couldn`t find the file specified: " + ImagesRoundMasksXMLfilesMappingList);
            }
            #endregion check files existence



            #region read or calculate GrIxYRGB stats

            if (Directory.Exists(imageYRGBstatsXMLdataFilesDirectory))
            {
                List <string> foundXMLfiles = Directory.EnumerateFiles(imageYRGBstatsXMLdataFilesDirectory,
                                                                       ConventionalTransitions.ImageGrIxYRGBstatsDataFileName(ImageFilename, "", false),
                                                                       SearchOption.AllDirectories).ToList();
                if (foundXMLfiles.Any())
                {
                    currImageStatsData =
                        (SkyImageIndexesStatsData)
                        ServiceTools.ReadObjectFromXML(foundXMLfiles[0], typeof(SkyImageIndexesStatsData));
                }
            }


            if (currImageStatsData == null)
            {
                Task <SkyImageIndexesStatsData> tskImageStatsCalculation = new Task <SkyImageIndexesStatsData>(() =>
                {
                    Dictionary <string, object> optionalParameters = new Dictionary <string, object>();
                    optionalParameters.Add("ImagesRoundMasksXMLfilesMappingList", ImagesRoundMasksXMLfilesMappingList);
                    Stopwatch sw = new Stopwatch();
                    sw.Start();
                    optionalParameters.Add("Stopwatch", sw);

                    ImageStatsDataCalculationResult currImageProcessingResult = null;
                    try
                    {
                        currImageProcessingResult = ImageProcessing.CalculateImageStatsData(ImageFilename,
                                                                                            optionalParameters);
                    }
                    catch (Exception ex)
                    {
                        return(null);
                    }



                    if (currImageProcessingResult == null)
                    {
                        return(null);
                    }
                    else
                    {
                        string strImageGrIxYRGBDataFileName =
                            ConventionalTransitions.ImageGrIxYRGBstatsDataFileName(ImageFilename,
                                                                                   imageYRGBstatsXMLdataFilesDirectory);
                        ServiceTools.WriteObjectToXML(currImageProcessingResult.grixyrgbStatsData,
                                                      strImageGrIxYRGBDataFileName);

                        currImageStatsData = currImageProcessingResult.grixyrgbStatsData;



                        return(currImageProcessingResult.grixyrgbStatsData);
                    }
                });

                currImageStatsData = await tskImageStatsCalculation;
            }

            if (currImageStatsData == null)
            {
                // theLogWindow = ServiceTools.LogAText(theLogWindow, "ERROR searching concurrent data for image. Will not proceed.");
                throw new Exception("ERROR searching concurrent data for image. Will not proceed.");
            }

            #endregion read or calculate GrIxYRGB stats


            ConcurrentData nearestConcurrentData = null;



            #region search for concurrent data

            Task <ConcurrentData> tskImageConcurrentDataSearching = new Task <ConcurrentData>(() =>
            {
                string currImgFilename = Path.GetFileNameWithoutExtension(ImageFilename);
                string ptrn            = @"(devID\d)";
                Regex rgxp             = new Regex(ptrn, RegexOptions.IgnoreCase);

                string strCurrImgDT = rgxp.Replace(currImgFilename.Substring(4), "");
                //2015-12-16T06-01-38
                strCurrImgDT = strCurrImgDT.Substring(0, 11) + strCurrImgDT.Substring(11).Replace("-", ":");

                DateTime currImgDT = DateTime.Parse(strCurrImgDT, null,
                                                    System.Globalization.DateTimeStyles.AdjustToUniversal);



                List <Tuple <string, ConcurrentData> > lImagesConcurrentData =
                    new List <Tuple <string, ConcurrentData> >();

                List <string> filesListConcurrentData =
                    Directory.EnumerateFiles(ConcurrentDataXMLfilesBasePath,
                                             ConventionalTransitions.ImageConcurrentDataFilesNamesPattern(),
                                             SearchOption.AllDirectories)
                    .ToList();

                List <Tuple <string, DateTime> > XMLfilesListConcurrentData = filesListConcurrentData.ConvertAll(
                    strXMLfilename =>
                {
                    string xmlFile1DateTimeString =
                        Path.GetFileNameWithoutExtension(strXMLfilename).Replace("data-", "").Substring(0, 19);
                    xmlFile1DateTimeString = xmlFile1DateTimeString.Substring(0, 11) + xmlFile1DateTimeString.Substring(11).Replace("-", ":");
                    DateTime dt1           = DateTime.Parse(xmlFile1DateTimeString);
                    return(new Tuple <string, DateTime>(strXMLfilename, dt1));
                });

                string nearestConcurrentDataFileName = XMLfilesListConcurrentData.Aggregate((cDt1, cDt2) =>
                {
                    TimeSpan tspan1 = new TimeSpan(Math.Abs((cDt1.Item2 - currImgDT).Ticks));
                    TimeSpan tspan2 = new TimeSpan(Math.Abs((cDt2.Item2 - currImgDT).Ticks));
                    return((tspan1 <= tspan2) ? (cDt1) : (cDt2));
                }).Item1;

                Dictionary <string, object> currDict = ServiceTools.ReadDictionaryFromXML(nearestConcurrentDataFileName);
                currDict.Add("XMLfileName", Path.GetFileName(nearestConcurrentDataFileName));
                ConcurrentData nearestConcurrentDataObtained = new ConcurrentData(currDict);

                if (new TimeSpan(Math.Abs((nearestConcurrentDataObtained.datetimeUTC - currImgDT).Ticks)) >=
                    new TimeSpan(0, 2, 0))
                {
                    nearestConcurrentDataObtained = null;
                }

                return(nearestConcurrentDataObtained);
            });

            nearestConcurrentData = await tskImageConcurrentDataSearching;

            if (nearestConcurrentData == null)
            {
                // theLogWindow = ServiceTools.LogAText(theLogWindow, "ERROR searching concurrent data for image. Will not proceed.");
                throw new Exception("ERROR searching concurrent data for image. Will not proceed.");
            }

            #endregion search for concurrent data



            DenseVector dvSDCmeans        = (DenseVector)((DenseMatrix)ServiceTools.ReadDataFromCSV(SDC_NormMeansFile, 0, ",")).Row(0);
            DenseVector dvSDCranges       = (DenseVector)((DenseMatrix)ServiceTools.ReadDataFromCSV(SDC_NormRangeFile, 0, ",")).Row(0);
            DenseVector dvSDCthetaValues  = (DenseVector)ServiceTools.ReadDataFromCSV(SDC_NNtrainedParametersFile, 0, ",");
            List <int>  SDCnnLayersConfig =
                new List <double>(((DenseMatrix)ServiceTools.ReadDataFromCSV(SDC_NNconfigFile, 0, ",")).Row(0)).ConvertAll
                    (dVal => Convert.ToInt32(dVal));

            DenseVector dvTCCmeans        = (DenseVector)((DenseMatrix)ServiceTools.ReadDataFromCSV(TCC_NormMeansFile, 0, ",")).Row(0);
            DenseVector dvTCCranges       = (DenseVector)((DenseMatrix)ServiceTools.ReadDataFromCSV(TCC_NormRangeFile, 0, ",")).Row(0);
            DenseVector dvTCCthetaValues  = (DenseVector)ServiceTools.ReadDataFromCSV(TCC_NNtrainedParametersFile, 0, ",");
            List <int>  TCCnnLayersConfig =
                new List <double>(((DenseMatrix)ServiceTools.ReadDataFromCSV(TCC_NNconfigFile, 0, ",")).Row(0)).ConvertAll
                    (dVal => Convert.ToInt32(dVal));
            List <int> TCCnnConfigVarsToExclude =
                new List <double>(((DenseMatrix)ServiceTools.ReadDataFromCSV(TCC_ExcludingVarsFile, 0, ",")).Row(0)).ConvertAll
                    (dVal => Convert.ToInt32(dVal));


            List <double> TCCdecisionProbabilities = null;
            return(PredictTCC_NN(currImageStatsData, nearestConcurrentData, SDCnnLayersConfig, dvSDCthetaValues,
                                 dvSDCmeans,
                                 dvSDCranges, TCCnnLayersConfig, dvTCCthetaValues, dvTCCmeans, dvTCCranges, TCCnnConfigVarsToExclude,
                                 out TCCdecisionProbabilities));
        }