private void BotOnReceiveError(object sender, ReceiveErrorEventArgs e) { Debugger.Break(); ApiRequestException ex = e.ApiRequestException; Console.WriteLine(ex.Message); ServiceTools.logToTextFile(errorFilename, "got API exception: " + ex.Message + Environment.NewLine + "Error code: " + ex.ErrorCode + Environment.NewLine + ex.StackTrace, true, true); }
public void Start(string[] args) { readDefaultProperties(); List <string> argsList = new List <string>(args); if (argsList.Any(str => str.Contains("--convert-existing-stats="))) { string foundArg = argsList.Find(str => str.Substring(0, 25) == "--convert-existing-stats="); FilesToProcessMask = foundArg.Replace("--convert-existing-stats=", ""); Console.WriteLine("files mask to process: \"" + FilesToProcessMask + "\""); } if (argsList.Find(str => str == "--recursive") != null) { cbSearchImagesTopDirectoryOnly = false; } if (argsList.Find(str => str.Substring(0, 38) == "--filter-default-CenterX-double-value=") != null) { string foundArg = argsList.Find(str => str.Substring(0, 38) == "--filter-default-CenterX-double-value="); string strValue = foundArg.Replace("--filter-default-CenterX-double-value=", ""); filterDefaultDoubleValueCenterX = Convert.ToDouble(strValue.Replace(".", ",")); } if (argsList.Find(str => str.Substring(0, 38) == "--filter-default-CenterY-double-value=") != null) { string foundArg = argsList.Find(str => str.Substring(0, 38) == "--filter-default-CenterY-double-value="); string strValue = foundArg.Replace("--filter-default-CenterY-double-value=", ""); filterDefaultDoubleValueCenterY = Convert.ToDouble(strValue.Replace(".", ",")); } if (argsList.Find(str => str.Substring(0, 37) == "--filter-default-Radius-double-value=") != null) { string foundArg = argsList.Find(str => str.Substring(0, 37) == "--filter-default-Radius-double-value="); string strValue = foundArg.Replace("--filter-default-Radius-double-value=", ""); filterDefaultDoubleValueRadius = Convert.ToDouble(strValue.Replace(".", ",")); } ServiceTools.logToTextFile(outputDataFile, "processing data files using pattern: " + FilesToProcessMask + Environment.NewLine, true); Console.WriteLine("getting files list"); EnumerateFilesToProcess(); foreach (ImagesConvertingData enumObj in lStatsConversion) { CurrFileProcessing(enumObj); } }
public void Start(string[] args) { readDefaultProperties(); List <string> argsList = new List <string>(args); if (argsList.Find(str => str.Substring(0, 7) == "--mask=") != null) { string foundArg = argsList.Find(str => str.Substring(0, 7) == "--mask="); FilesToProcessMask = foundArg.Substring(7); Console.WriteLine("files mask to process: \"" + FilesToProcessMask + "\""); } if (argsList.Find(str => str == "--recursive") != null) { cbSearchImagesTopDirectoryOnly = false; } outputDataFile = strOutputDirectory + outputDataFile; ServiceTools.logToTextFile(outputDataFile, "processing images using filename mask: " + FilesToProcessMask + Environment.NewLine, true); Console.WriteLine("getting files list"); EnumerateFilesToProcess(); ServiceTools.logToTextFile(outputDataFile, "filename;centerX;centerY;radius" + Environment.NewLine, true); var options = new ParallelOptions(); options.MaxDegreeOfParallelism = maxConcurrentBackgroundWorkers; Parallel.ForEach(lStatsCalculation, options, item => { CurrImageProcessing(item); }); }
private void swapResponseLog(object sender) { string filename1; string strtowrite; byte[] info2write; int curDevID = 1; if (sender == btnSwapResponseLog1) { curDevID = 1; } else if (sender == btnSwapResponseLog2) { curDevID = 2; } filename1 = Directory.GetCurrentDirectory() + "\\ResponseLog-" + "devID" + curDevID + "-" + DateTime.Now.Year + "-" + DateTime.Now.Month + "-" + DateTime.Now.Day + "-" + DateTime.Now.Hour + "-" + DateTime.Now.Minute + "-" + DateTime.Now.Second + ".log"; ServiceTools.logToTextFile(filename1, tbResponseLog1.Text, true); ThreadSafeOperations.SetTextTB(tbResponseLog1, "", false); }
private void EnumerateFilesToProcess() { string directory = Path.GetDirectoryName(FilesToProcessMask); string filemask = FilesToProcessMask.Replace(directory + Path.DirectorySeparatorChar, ""); List <string> filesList = new List <string>(Directory.EnumerateFiles(directory, filemask, cbSearchImagesTopDirectoryOnly ? SearchOption.TopDirectoryOnly : SearchOption.AllDirectories)); if (!filesList.Any()) { Console.WriteLine("There is no images that sutisfy specified settings. Processing will not be started."); return; } int intFinalIndex = filesList.Count; int idx = 0; int currPerc = 0; int prevPerc = 0; foreach (string fName in filesList) { Console.Write("adding file to calculation queue:" + fName + "\r"); idx++; AddImageToCalculationQueue(fName); } Console.WriteLine("finished enumerating files. Files to process: " + lStatsCalculation.Count); ServiceTools.logToTextFile(outputDataFile, "files count to process: " + lStatsCalculation.Count + Environment.NewLine, true); }
private void CurrImageProcessing(ImageStatsCollectingData dat) { string strSaveControlImagesToPath = ""; string currentFullFileName = dat.filename; if (defaultProperties.ContainsKey("SaveControlImagesToPath")) { strSaveControlImagesToPath = (string)defaultProperties["SaveControlImagesToPath"]; } else { strSaveControlImagesToPath = "same as source image"; } try { Image <Bgr, byte> currImg = new Image <Bgr, byte>(currentFullFileName); ImageProcessing imgP = new ImageProcessing(currImg, true); #region debugging output control images #if DEBUG if (strSaveControlImagesToPath != "") { string strControlImageFileName = ""; if (strSaveControlImagesToPath == "same as source image") { strControlImageFileName = Path.GetDirectoryName(currentFullFileName); strControlImageFileName += ((strControlImageFileName.Last() == Path.DirectorySeparatorChar) ? ("") : (Path.DirectorySeparatorChar.ToString())); strControlImageFileName += Path.GetFileNameWithoutExtension(currentFullFileName) + "-control.jpg"; } else { if (Directory.Exists(strSaveControlImagesToPath)) { strControlImageFileName = strSaveControlImagesToPath + ((strSaveControlImagesToPath.Last() == Path.DirectorySeparatorChar) ? ("") : (Path.DirectorySeparatorChar.ToString())) + Path.GetFileNameWithoutExtension(currentFullFileName) + "-control.jpg"; } else { strControlImageFileName = ""; } } if (strControlImageFileName != "") { // save control image imgP.significantMaskImageOctLined.Save(strControlImageFileName); } } #endif #endregion debugging output control images string csvDataStr = Path.GetFileName(currentFullFileName) + ";" + imgP.imageRD.ToCSVstring(";"); ServiceTools.logToTextFile(outputDataFile, csvDataStr + Environment.NewLine, true); ImageStatsCollectingData foundDataObj = lStatsCalculation.Find(obj => obj.filename == currentFullFileName); foundDataObj.State = ImageGrIxStatsCollectingState.Finished; foundDataObj.rd = imgP.imageRD; Console.WriteLine("finished processing file " + Environment.NewLine + currentFullFileName); Console.WriteLine("remains: " + lStatsCalculation.FindAll( imgStats => imgStats.State == ImageGrIxStatsCollectingState.Queued).Count + " of " + lStatsCalculation.Count); } catch (Exception ex) { return; } }
private Image <Bgr, byte> FillGraphImage(Size imgSize) { string curDirPath = Directory.GetCurrentDirectory() + "\\logs\\"; DirectoryInfo dirInfo = new DirectoryInfo(curDirPath); List <Dictionary <string, object> > lReadData = new List <Dictionary <string, object> >(); if (defaultGraphsTimeSpan) { graphsTimeSpan = new Tuple <DateTime, DateTime>(DateTime.UtcNow.AddDays(-1), DateTime.UtcNow); } GraphVariablesTypes currVarType = GraphVariablesTypes.none; if (rbtnPressureGraph.Checked) { currVarType = GraphVariablesTypes.Pressure; } if (rbtnAirTempGraph.Checked) { currVarType = GraphVariablesTypes.AirTemp; } if (rbtnWaterTempGraph.Checked) { currVarType = GraphVariablesTypes.WaterTemp; } if (rbtnWindSpeedGraph.Checked) { currVarType = GraphVariablesTypes.WindSpeed; } if ((fRenderer == null) || (!prevGraphsTimeSpan.Equals(graphsTimeSpan) || currVarType != prevGraphVariable)) { fRenderer = new MultipleScatterAndFunctionsRepresentation(imgSize); switch (currVarType) { case GraphVariablesTypes.Pressure: { currValueColor = new Bgr(Color.Blue); break; } case GraphVariablesTypes.AirTemp: { currValueColor = new Bgr(Color.Red); break; } case GraphVariablesTypes.WaterTemp: { currValueColor = new Bgr(Color.RoyalBlue); break; } case GraphVariablesTypes.WindSpeed: { currValueColor = new Bgr(Color.Gray); break; } default: { currValueColor = new Bgr(Color.Blue); break; } } } else if (fRenderer != null) { if (fRenderer.TheImage.Size != imgSize) { fRenderer.ResizeCanvas(imgSize); } } if (!prevGraphsTimeSpan.Equals(graphsTimeSpan)) { IEnumerable <string> ncFileNames = Directory.EnumerateFiles(curDirPath, "IoffeVesselInfoStream-MeteoDataLog-*.nc", SearchOption.TopDirectoryOnly); foreach (string ncFileName in ncFileNames) { Tuple <DateTime, DateTime> currFileDateTimeRange = null; try { currFileDateTimeRange = ServiceTools.GetNetCDFfileTimeStampsRange(ncFileName); } catch (Exception ex) { #region report #if DEBUG ServiceTools.ExecMethodInSeparateThread(this, () => { theLogWindow = ServiceTools.LogAText(theLogWindow, "an exception has been thrown during file reading: " + Environment.NewLine + ncFileName + Environment.NewLine + "message: " + ex.Message + Environment.NewLine + ServiceTools.CurrentCodeLineDescription()); }); #else ServiceTools.ExecMethodInSeparateThread(this, () => { ServiceTools.logToTextFile(errorLogFilename, "an exception has been thrown during file reading: " + Environment.NewLine + ncFileName + Environment.NewLine + "message: " + ex.Message + Environment.NewLine + ServiceTools.CurrentCodeLineDescription(), true, true); }); #endif #endregion report } if (currFileDateTimeRange == null) { continue; } if ((currFileDateTimeRange.Item1 >= graphsTimeSpan.Item1) && (currFileDateTimeRange.Item1 <= graphsTimeSpan.Item2) || (currFileDateTimeRange.Item2 >= graphsTimeSpan.Item1) && (currFileDateTimeRange.Item2 <= graphsTimeSpan.Item2)) { Dictionary <string, object> dictFileData = null; try { dictFileData = NetCDFoperations.ReadDataFromFile(ncFileName); } catch (Exception ex) { #region report #if DEBUG ServiceTools.ExecMethodInSeparateThread(this, () => { theLogWindow = ServiceTools.LogAText(theLogWindow, "an exception has been thrown during file reading: " + Environment.NewLine + ncFileName + Environment.NewLine + "message: " + ex.Message + Environment.NewLine + ServiceTools.CurrentCodeLineDescription()); }); #else ServiceTools.ExecMethodInSeparateThread(this, () => { ServiceTools.logToTextFile(errorLogFilename, "an exception has been thrown during file reading: " + Environment.NewLine + ncFileName + Environment.NewLine + "message: " + ex.Message + Environment.NewLine + ServiceTools.CurrentCodeLineDescription(), true, true); }); #endif #endregion report } if (dictFileData != null) { lReadData.Add(dictFileData); } } } foreach (Dictionary <string, object> currFileDataDict in lReadData) { if (currFileDataDict == null) { continue; } string varNameDateTime = "DateTime"; List <long> currFileDateTimeLongTicksList = new List <long>((currFileDataDict[varNameDateTime] as long[])); List <DateTime> currFileDateTimeList = currFileDateTimeLongTicksList.ConvertAll(longVal => new DateTime(longVal)); string varNameMeteoData = "MeteoData"; List <MeteoData> currFileMeteoDataList = MeteoData.OfDenseMatrix(currFileDataDict[varNameMeteoData] as DenseMatrix); if (tsMeteoDataForGraphs == null) { try { tsMeteoDataForGraphs = new TimeSeries <MeteoData>(currFileMeteoDataList, currFileDateTimeList, true); } catch (Exception ex) { #region report #if DEBUG ServiceTools.ExecMethodInSeparateThread(this, () => { theLogWindow = ServiceTools.LogAText(theLogWindow, "couldn`t create timeseries: exception has been thrown" + Environment.NewLine + ServiceTools.CurrentCodeLineDescription() + Environment.NewLine + "message: " + ex.Message); }); #else ServiceTools.ExecMethodInSeparateThread(this, () => { ServiceTools.logToTextFile(errorLogFilename, "couldn`t create timeseries: exception has been thrown" + Environment.NewLine + ServiceTools.CurrentCodeLineDescription() + Environment.NewLine + "message: " + ex.Message, true, true); }); #endif #endregion report } } else { try { tsMeteoDataForGraphs.AddSubseriaData(currFileMeteoDataList, currFileDateTimeList, true); } catch (Exception ex) { #region report #if DEBUG ServiceTools.ExecMethodInSeparateThread(this, () => { theLogWindow = ServiceTools.LogAText(theLogWindow, "couldn`t create timeseries: exception has been thrown" + Environment.NewLine + ServiceTools.CurrentCodeLineDescription() + Environment.NewLine + "message: " + ex.Message); }); #else ServiceTools.ExecMethodInSeparateThread(this, () => { ServiceTools.logToTextFile(errorLogFilename, "couldn`t create timeseries: exception has been thrown" + Environment.NewLine + ServiceTools.CurrentCodeLineDescription() + Environment.NewLine + "message: " + ex.Message, true, true); }); #endif #endregion report } } } if (tsMeteoDataForGraphs == null) { return(null); } tsMeteoDataForGraphs.SortByTimeStamps(); tsMeteoDataForGraphs.RemoveDuplicatedTimeStamps(); DateTime utcNow = DateTime.UtcNow; if (defaultGraphsTimeSpan) { tsMeteoDataForGraphs.RemoveValues(dt => (utcNow - dt).TotalSeconds > 86400); } else { tsMeteoDataForGraphs.RemoveValues( dt => !((dt >= graphsTimeSpan.Item1) && (dt <= graphsTimeSpan.Item2))); } List <TimeSeries <MeteoData> > subSeriesBy1Minute = tsMeteoDataForGraphs.SplitByTimeSpan(new TimeSpan(0, 1, 0)); List <double> lSubSeriesEntriesCount = subSeriesBy1Minute.ConvertAll(subs => (double)subs.Count); DescriptiveStatistics statsCounts = new DescriptiveStatistics(lSubSeriesEntriesCount); aveMinuteEntriesCount = Convert.ToInt32(statsCounts.Mean); // = tsMeteoData.TimeStamps.ConvertAll(dt => (dt - maxDateTime).TotalSeconds); } List <MeteoData> meteoDataList = tsMeteoDataForGraphs.DataValues; DateTime maxDateTime = tsMeteoDataForGraphs.TimeStamps.Max(); if ((currVarType != prevGraphVariable) || !prevGraphsTimeSpan.Equals(graphsTimeSpan)) { double minVarValue = 0.0d; double maxVarValue = 1.0d; List <double> currVarToShowValues = new List <double>(); switch (currVarType) { case GraphVariablesTypes.Pressure: { currVarToShowValues = meteoDataList.ConvertAll(mdt => mdt.pressure); TimeSeries <double> currVarTS = new TimeSeries <double>(currVarToShowValues, tsMeteoDataForGraphs.TimeStamps); currVarTS.RemoveValues(dVal => dVal <= 900.0d); currVarToShowValues = new List <double>(currVarTS.DataValues); currFileSecondsList = currVarTS.TimeStamps.ConvertAll(dt => (dt - maxDateTime).TotalSeconds); fRenderer.yAxisValuesConversionToRepresentTicksValues = new Func <double, string>(dVal => dVal.ToString("F1")); break; } case GraphVariablesTypes.AirTemp: { currVarToShowValues = meteoDataList.ConvertAll(mdt => mdt.airTemperature); TimeSeries <double> currVarTS = new TimeSeries <double>(currVarToShowValues, tsMeteoDataForGraphs.TimeStamps); currVarTS.RemoveValues(dVal => ((dVal < -20.0d) || (dVal > 50.0d))); currVarToShowValues = new List <double>(currVarTS.DataValues); currFileSecondsList = currVarTS.TimeStamps.ConvertAll(dt => (dt - maxDateTime).TotalSeconds); fRenderer.yAxisValuesConversionToRepresentTicksValues = new Func <double, string>(dVal => dVal.ToString("F2")); break; } case GraphVariablesTypes.WaterTemp: { currVarToShowValues = meteoDataList.ConvertAll(mdt => mdt.waterTemperature); TimeSeries <double> currVarTS = new TimeSeries <double>(currVarToShowValues, tsMeteoDataForGraphs.TimeStamps); currVarTS.RemoveValues(dVal => ((dVal < -20.0d) || (dVal > 50.0d))); currVarToShowValues = new List <double>(currVarTS.DataValues); currFileSecondsList = currVarTS.TimeStamps.ConvertAll(dt => (dt - maxDateTime).TotalSeconds); fRenderer.yAxisValuesConversionToRepresentTicksValues = new Func <double, string>(dVal => dVal.ToString("F2")); break; } case GraphVariablesTypes.WindSpeed: { currVarToShowValues = meteoDataList.ConvertAll(mdt => mdt.windSpeed); TimeSeries <double> currVarTS = new TimeSeries <double>(currVarToShowValues, tsMeteoDataForGraphs.TimeStamps); currVarTS.RemoveValues(dVal => ((dVal < 0.0d) || (dVal > 50.0d))); currVarToShowValues = new List <double>(currVarTS.DataValues); currFileSecondsList = currVarTS.TimeStamps.ConvertAll(dt => (dt - maxDateTime).TotalSeconds); fRenderer.yAxisValuesConversionToRepresentTicksValues = new Func <double, string>(dVal => dVal.ToString("F1")); break; } default: return(null); } dvVarValues = DenseVector.OfEnumerable(currVarToShowValues); dvVarValues = dvVarValues.Conv(Extensions.StandardConvolutionKernels.gauss, aveMinuteEntriesCount * 10); } fRenderer.dvScatterFuncValues.Add(dvVarValues); fRenderer.dvScatterXSpace.Add(DenseVector.OfEnumerable(currFileSecondsList)); fRenderer.xAxisValuesConversionToRepresentTicksValues = (dValSec) => { DateTime currDT = tsMeteoDataForGraphs.TimeStamps.Max().AddSeconds(dValSec); return(currDT.ToString("yyyy-MM-dd" + Environment.NewLine + "HH:mm")); }; fRenderer.scatterLineColors.Add(currValueColor); fRenderer.scatterDrawingVariants.Add(SequencesDrawingVariants.polyline); fRenderer.xSpaceMin = currFileSecondsList.Min(); fRenderer.xSpaceMax = currFileSecondsList.Max(); fRenderer.overallFuncMin = dvVarValues.Min(); fRenderer.overallFuncMax = dvVarValues.Max(); fRenderer.fixSpecifiedMargins = true; fRenderer.Represent(); Image <Bgr, byte> retImg = fRenderer.TheImage; // расположим надпись string strSign = "current value: " + dvVarValues.Last().ToString("F2"); List <TextBarImage> textBarsCases = new List <TextBarImage>(); TextBarImage tbimTopLeftSign = new TextBarImage(strSign, retImg); tbimTopLeftSign.PtSurroundingBarStart = new Point(fRenderer.LeftServiceSpaceGapX + tbimTopLeftSign.textHalfHeight, fRenderer.TopServiceSpaceGapY + tbimTopLeftSign.textHalfHeight); textBarsCases.Add(tbimTopLeftSign); TextBarImage tbimBtmLeftSign = new TextBarImage(strSign, retImg); tbimBtmLeftSign.PtSurroundingBarStart = new Point(fRenderer.LeftServiceSpaceGapX + tbimBtmLeftSign.textHalfHeight, retImg.Height - fRenderer.BtmServiceSpaceGapY - tbimBtmLeftSign.textHalfHeight - tbimBtmLeftSign.textHeight * 2); textBarsCases.Add(tbimBtmLeftSign); TextBarImage tbimTopRightSign = new TextBarImage(strSign, retImg); tbimTopRightSign.PtSurroundingBarStart = new Point( retImg.Width - fRenderer.RightServiceSpaceGapX - tbimTopRightSign.textHalfHeight - tbimTopRightSign.textBarSize.Width, fRenderer.TopServiceSpaceGapY + tbimTopLeftSign.textHalfHeight); textBarsCases.Add(tbimTopRightSign); TextBarImage tbimBtmRightSign = new TextBarImage(strSign, retImg); tbimBtmRightSign.PtSurroundingBarStart = new Point( retImg.Width - fRenderer.RightServiceSpaceGapX - tbimBtmRightSign.textHalfHeight - tbimBtmRightSign.textBarSize.Width, retImg.Height - fRenderer.BtmServiceSpaceGapY - tbimBtmRightSign.textHalfHeight - tbimBtmRightSign.textHeight * 2); textBarsCases.Add(tbimBtmRightSign); textBarsCases.Sort((case1, case2) => (case1.SubImageInTextRect.CountNonzero().Sum() > case2.SubImageInTextRect.CountNonzero().Sum()) ? 1 : -1); MCvFont theFont = new MCvFont(Emgu.CV.CvEnum.FONT.CV_FONT_HERSHEY_PLAIN, 2.0d, 2.0d) { thickness = 2, }; // retImg.Draw(strSign, textBarsCases[0].ptTextBaselineStart, Emgu.CV.CvEnum.FontFace.HersheyPlain, 2.0d, new Bgr(Color.Green), 2); retImg.Draw(strSign, ref theFont, textBarsCases[0].ptTextBaselineStart, new Bgr(Color.Green)); retImg.Draw(textBarsCases[0].rectSurroundingBar, new Bgr(Color.Green), 2); prevGraphsTimeSpan = graphsTimeSpan; prevGraphVariable = currVarType; return(retImg); }
public void Start(string[] args) { readDefaultProperties(); List <string> argsList = new List <string>(args); if (argsList.Find(str => str == "--recursive") != null) { bEnumerateFilesRecursively = true; } if (argsList.Find(str => str == "-y") != null) { bStartWithoutConfirmation = true; } // --filter-by-observed-cloud-cover-records // bFilterByObservedCloudCoverRecords if (argsList.Find(str => str == "--filter-by-observed-cloud-cover-records") != null) { bFilterByObservedCloudCoverRecords = true; } // sdcFilter if (argsList.Where(str => str.Contains("--sdc=")).Count() > 0) { string foundArg = argsList.Where(str => str.Contains("--sdc=")).ToList()[0]; string strValue = foundArg.Replace("--sdc=", ""); //sdcFilter if (strValue == "none") { sdcFilter = SunDiskCondition.NoSun; } else if (strValue == "0") { sdcFilter = SunDiskCondition.Sun0; } else if (strValue == "1") { sdcFilter = SunDiskCondition.Sun1; } else if (strValue == "2") { sdcFilter = SunDiskCondition.Sun2; } else { sdcFilter = SunDiskCondition.Sun2; } } else { Console.WriteLine("SDC filter is not specified. Filtering by SDC will not applied."); sdcFilter = SunDiskCondition.Undefined; // Не применять фильтрацию } if (argsList.Where(str => str.Contains("--camera-id=")).Count() > 0) { string foundArg = argsList.Where(str => str.Contains("--camera-id=")).ToList()[0]; string strValue = foundArg.Replace("--camera-id=", ""); CamIDtoProcess = Convert.ToInt32(strValue); if ((CamIDtoProcess != 1) && (CamIDtoProcess != 2)) { Console.WriteLine("camera ID out of range detected. I will not filter by camera ID."); CamIDtoProcess = 0; } } else { Console.WriteLine("camera ID out of range detected. I will not filter by camera ID"); CamIDtoProcess = 0; // will not filter } if (!bStartWithoutConfirmation) { Console.Write("Start with the mentioned properties? [y/n] "); string strReply = Console.ReadLine(); if (strReply.ToLower().First() != 'y') { Console.WriteLine("\nWill not proceed due to user interruprion."); Console.WriteLine("===FINISHED==="); Console.ReadKey(); return; } } outputDataFile = strOutputDirectory + Path.GetFileNameWithoutExtension(outputDataFile) + "-" + sdcFilter.ToString() + "-camID" + CamIDtoProcess + ".xml"; string outputCSVfile = strOutputDirectory + Path.GetFileNameWithoutExtension(outputDataFile) + "-" + sdcFilter.ToString() + "-camID" + CamIDtoProcess + ".csv"; Console.WriteLine("getting files list"); #region Enumerating files string directory = Path.GetDirectoryName(inputBasePath); string filemask = "*.jpg"; List <string> filesList = new List <string>(Directory.EnumerateFiles(directory, filemask, bEnumerateFilesRecursively ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly)); #region filter by camID //...devID1.jpg if (CamIDtoProcess > 0) { string ptrnCamID = "devid" + CamIDtoProcess + ".jpg"; filesList = filesList.Where(fname => fname.ToLower().Contains(ptrnCamID)).ToList(); } #endregion Console.WriteLine("found " + filesList.Count + " images."); #region try to find concurrent and stats data already assembled into a small set of files List <string> assembledDataFilesList = Directory.EnumerateFiles(ConcurrentDataXMLfilesDirectory, "*.xml", SearchOption.TopDirectoryOnly).ToList(); List <ImagesProcessingData> lReadAssembledData = new List <ImagesProcessingData>(); foreach (string strAssembledDataXMlfName in assembledDataFilesList) { try { List <ImagesProcessingData> currFileContent = ServiceTools.ReadObjectFromXML(strAssembledDataXMlfName, typeof(List <ImagesProcessingData>)) as List <ImagesProcessingData>; lReadAssembledData.AddRange(currFileContent); } catch (Exception ex) { continue; } } if (lReadAssembledData.Any()) { Console.WriteLine("Found pre-assembled ImagesProcessingData XML files: "); foreach (string s in assembledDataFilesList) { Console.WriteLine(s); } Console.WriteLine("Read records from this set: " + lReadAssembledData.Count); Console.WriteLine("Images to process originally: " + filesList.Count); Console.WriteLine("Should I use these pre-assembled data? (y/n): "); string ans = Console.ReadKey().KeyChar.ToString(); if (ans == "y") { lStatsProcessing = lReadAssembledData; } } #endregion try to find data already compiled into a small set of files if (!lStatsProcessing.Any()) { #region list, read and map image stats files Console.WriteLine("filtering by ready-to-use GrIxYRGB XML files..."); List <string> statsFilesList = new List <string>(Directory.EnumerateFiles(imageYRGBstatsXMLdataFilesDirectory, ConventionalTransitions.ImageGrIxYRGBstatsFileNamesPattern(), bEnumerateFilesRecursively ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly)); List <string> statsFilesListWOpath = statsFilesList.ConvertAll(Path.GetFileName); Console.WriteLine("found " + statsFilesList.Count + " XML stats files in directory " + Environment.NewLine + imageYRGBstatsXMLdataFilesDirectory + Environment.NewLine + "by mask " + Environment.NewLine + ConventionalTransitions.ImageGrIxYRGBstatsFileNamesPattern()); int removed = filesList.RemoveAll( fname => !statsFilesListWOpath.Contains(ConventionalTransitions.ImageGrIxYRGBstatsDataFileName(fname, "", false))); Console.WriteLine("removed " + removed + " items (couldn`t find stats data files). Remains " + filesList.Count + " to process."); if (!filesList.Any()) { Console.WriteLine("There is no " + filemask + " files that sutisfy settings specified. Processing will not be started."); return; } lStatsProcessing = filesList.ConvertAll(strImgFname => { ImagesProcessingData retVal = new ImagesProcessingData() { filename = strImgFname }; return(retVal); }); //#if DEBUG // lStatsProcessing = lStatsProcessing.Where((ipd, ind) => ind < 10).ToList(); //#endif Console.WriteLine("started reading and mapping stats data"); int totalFilesCountToRead = lStatsProcessing.Count; int filesRead = 0; int currProgressPerc = 0; foreach (ImagesProcessingData ipdt in lStatsProcessing) { ipdt.grixyrgbStatsXMLfile = statsFilesList.First( statsFname => statsFname.Contains(ConventionalTransitions.ImageGrIxYRGBstatsDataFileName(ipdt.filename, "", false))); ipdt.grixyrgbStats = ServiceTools.ReadObjectFromXML(ipdt.grixyrgbStatsXMLfile, typeof(SkyImageIndexesStatsData)) as SkyImageIndexesStatsData; #region calculate and report progress filesRead++; double progress = 100.0d * (double)filesRead / (double)totalFilesCountToRead; if (progress - (double)currProgressPerc > 1.0d) { currProgressPerc = Convert.ToInt32(progress); Console.WriteLine("read " + currProgressPerc + "%"); } #endregion calculate and report progress } #endregion #region list, read and map concurrent data List <ConcurrentData> lConcurrentData = null; List <string> concurrentDataFilesList = Directory.EnumerateFiles(ConcurrentDataXMLfilesDirectory, ConventionalTransitions.ImageConcurrentDataFilesNamesPattern(), bEnumerateFilesRecursively ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly).ToList(); #region reading Console.WriteLine("started concurrent data reading"); totalFilesCountToRead = concurrentDataFilesList.Count; filesRead = 0; currProgressPerc = 0; List <Dictionary <string, object> > lDictionariesConcurrentData = new List <Dictionary <string, object> >(); foreach (string strConcDataXMLFile in concurrentDataFilesList) { Dictionary <string, object> currDict = ServiceTools.ReadDictionaryFromXML(strConcDataXMLFile); currDict.Add("XMLfileName", Path.GetFileName(strConcDataXMLFile)); lDictionariesConcurrentData.Add(currDict); #region calculate and report progress filesRead++; double progress = 100.0d * (double)filesRead / (double)totalFilesCountToRead; if (progress - (double)currProgressPerc > 1.0d) { currProgressPerc = Convert.ToInt32(progress); Console.WriteLine("read " + currProgressPerc + "%"); } #endregion calculate and report progress } lDictionariesConcurrentData.RemoveAll(dict => dict == null); lConcurrentData = lDictionariesConcurrentData.ConvertAll <ConcurrentData>(dict => { ConcurrentData retVal = null; try { retVal = new ConcurrentData(dict); } catch (Exception ex) { string strError = "couldn`t parse XML file " + dict["XMLfileName"] + " : " + Environment.NewLine + ex.Message; Console.WriteLine(strError); } return(retVal); }); lConcurrentData.RemoveAll(val => val == null); #endregion reading #region mapping // map obtained concurrent data to images by its datetime Console.WriteLine("concurrent data mapping started"); lStatsProcessing = lStatsProcessing.ConvertAll(ipdt => { string currImgFilename = ipdt.filename; currImgFilename = Path.GetFileNameWithoutExtension(currImgFilename); DateTime currImgDT = ConventionalTransitions.DateTimeOfSkyImageFilename(currImgFilename); ConcurrentData nearestConcurrentData = lConcurrentData.Aggregate((cDt1, cDt2) => { TimeSpan tspan1 = new TimeSpan(Math.Abs((cDt1.datetimeUTC - currImgDT).Ticks)); TimeSpan tspan2 = new TimeSpan(Math.Abs((cDt2.datetimeUTC - currImgDT).Ticks)); return((tspan1 <= tspan2) ? (cDt1) : (cDt2)); }); if (new TimeSpan(Math.Abs((nearestConcurrentData.datetimeUTC - currImgDT).Ticks)) >= TimeSpanForConcurrentDataMappingTolerance) { string strError = "couldn`t find close enough concurrent data file for image:" + Environment.NewLine + currImgFilename + Environment.NewLine + "closest concurrent data file is:" + Environment.NewLine + nearestConcurrentData.filename + Environment.NewLine + "with date-time value " + nearestConcurrentData.datetimeUTC.ToString("o"); Console.WriteLine(strError); nearestConcurrentData = null; } ipdt.concurrentData = nearestConcurrentData; if (nearestConcurrentData != null) { ipdt.concurrentDataXMLfile = nearestConcurrentData.filename; } return(ipdt); }); #endregion mapping removed = lStatsProcessing.RemoveAll(ipdt => ipdt.concurrentData == null); Console.WriteLine("removed " + removed + " items (couldn`t find concurrent data). " + lStatsProcessing.Count + " files remains to process."); #endregion list, read and map concurrent data } if (!lStatsProcessing.Any()) { Console.WriteLine("There is no files that sutisfy settings specified and have all required concurrent data (stats or GPS etc.). Processing will not be proceeded."); return; } #region Filter by SDC values predicting it using pre-trained NN parameters #region // //string csvHeader = lStatsProcessing[0].grixyrgbStats.CSVHeader() + // ",SunElevationDeg,SunAzimuthDeg,sunDiskCondition"; //List<string> lCSVheader = csvHeader.Split(',').ToList(); //List<int> columnsToDelete = // lCSVheader.Select((str, idx) => new Tuple<int, string>(idx, str)) // .Where(tpl => tpl.Item2.ToLower().Contains("filename")).ToList().ConvertAll(tpl => tpl.Item1); //List<List<string>> lCalculatedData = lStatsProcessing.ConvertAll(dt => //{ // string currImageALLstatsDataCSVWithConcurrentData = dt.grixyrgbStats.ToCSV() + "," + // dt.concurrentData.gps.SunZenithAzimuth() // .ElevationAngle.ToString() // .Replace(",", ".") + "," + // dt.concurrentData.gps.SunZenithAzimuth() // .Azimuth.ToString() // .Replace(",", "."); // List<string> retVal = currImageALLstatsDataCSVWithConcurrentData.Split(',').ToList(); // retVal = retVal.Where((str, idx) => !columnsToDelete.Contains(idx)).ToList(); // return retVal; //}); //List<DenseVector> lDV_objects_features = // lCalculatedData.ConvertAll( // list => // DenseVector.OfEnumerable(list.ConvertAll<double>(str => Convert.ToDouble(str.Replace(".", ","))))); #endregion DenseVector dvMeans = (DenseVector)((DenseMatrix)ServiceTools.ReadDataFromCSV(NormMeansFile, 0, ",")).Row(0); DenseVector dvRanges = (DenseVector)((DenseMatrix)ServiceTools.ReadDataFromCSV(NormRangeFile, 0, ",")).Row(0); #region // //lDV_objects_features = lDV_objects_features.ConvertAll(dv => //{ // DenseVector dvShifted = dv - dvMeans; // DenseVector dvNormed = (DenseVector)dvShifted.PointwiseDivide(dvRanges); // return dvNormed; //}); //DenseMatrix dmObjectsFeatures = DenseMatrix.OfRowVectors(lDV_objects_features); #endregion DenseVector dvThetaValues = (DenseVector)ServiceTools.ReadDataFromCSV(NNtrainedParametersFile, 0, ","); List <int> NNlayersConfig = new List <double>(((DenseMatrix)ServiceTools.ReadDataFromCSV(NNconfigFile, 0, ",")).Row(0)).ConvertAll (dVal => Convert.ToInt32(dVal)); #region // // List<List<double>> lDecisionProbabilities = null; #endregion List <Tuple <ImagesProcessingData, List <SDCdecisionProbability>, SunDiskCondition> > lTplsPredictedSDClist = new List <Tuple <ImagesProcessingData, List <SDCdecisionProbability>, SunDiskCondition> >(); List <List <double> > SDCdecisionProbabilitiesListDoubles = new List <List <double> >(); List <SunDiskCondition> imagesSDCpredicted = SDCpredictorNN.PredictSDC_NN(lStatsProcessing, NNlayersConfig, dvThetaValues, dvMeans, dvRanges, out SDCdecisionProbabilitiesListDoubles); List <List <SDCdecisionProbability> > SDCdecisionProbabilitiesLists = SDCdecisionProbabilitiesListDoubles.ConvertAll( currSDCdecisionProbabilities => currSDCdecisionProbabilities.Select((dProb, idx) => new SDCdecisionProbability() { sdc = SunDiskConditionData.MatlabSDCenum(idx + 1), sdcDecisionProbability = dProb }).ToList()); lTplsPredictedSDClist = lStatsProcessing.Zip( SDCdecisionProbabilitiesLists.Zip(imagesSDCpredicted, (lDecProb, sdcPredicted) => new Tuple <List <SDCdecisionProbability>, SunDiskCondition>(lDecProb, sdcPredicted)).ToList(), (ipd, tpl) => new Tuple <ImagesProcessingData, List <SDCdecisionProbability>, SunDiskCondition>(ipd, tpl.Item1, tpl.Item2)).ToList(); #region // //foreach (ImagesProcessingData dt in lStatsProcessing) //{ // List<double> currSDCdecisionProbabilities = new List<double>(); // SunDiskCondition currSDC = SDCpredictorNN.PredictSDC_NN(dt.grixyrgbStats, dt.concurrentData, // NNlayersConfig, dvThetaValues, dvMeans, dvRanges, out currSDCdecisionProbabilities); // List<SDCdecisionProbability> currSDCdecisionProbabilitiesList = currSDCdecisionProbabilities.Select((dProb, idx) => new SDCdecisionProbability() // { // sdc = SunDiskConditionData.MatlabSDCenum(idx + 1), // sdcDecisionProbability = dProb // }).ToList(); // lTplsPredictedSDClist.Add( // new Tuple<ImagesProcessingData, List<SDCdecisionProbability>, SunDiskCondition>(dt, // currSDCdecisionProbabilitiesList, currSDC)); //} #endregion #region // //List<int> predictedSDC = // NNclassificatorPredictor.NNpredict(dmObjectsFeatures, dvThetaValues, NNlayersConfig, // out lDecisionProbabilities).ToList(); //List<SunDiskCondition> predictedSDClist = predictedSDC.ConvertAll(sdcInt => //{ // switch (sdcInt) // { // case 4: // return SunDiskCondition.NoSun; // break; // case 1: // return SunDiskCondition.Sun0; // break; // case 2: // return SunDiskCondition.Sun1; // break; // case 3: // return SunDiskCondition.Sun2; // break; // default: // return SunDiskCondition.Defect; // } //}); //List<Tuple<ImagesProcessingData, SunDiskCondition>> lTplsPredictedSDClist = // predictedSDClist.Zip(lStatsProcessing, // (sdc, ipd) => new Tuple<ImagesProcessingData, SunDiskCondition>(ipd, sdc)).ToList(); #endregion #region output obtained SDC data to log file string strToShow = "SDC values probabilities: " + Environment.NewLine + "| NoSun | Sun0 | Sun1 | Sun2 |" + Environment.NewLine; foreach (Tuple <ImagesProcessingData, List <SDCdecisionProbability>, SunDiskCondition> tpl in lTplsPredictedSDClist) { List <SDCdecisionProbability> currSDCdecisionProbabilitiesList = tpl.Item2; strToShow += "|" + String.Format("{0,9}", (currSDCdecisionProbabilitiesList.First( prob => prob.sdc == SunDiskCondition.NoSun).sdcDecisionProbability * 100.0d) .ToString("F2") + "%") + "|" + String.Format("{0,9}", (currSDCdecisionProbabilitiesList.First( prob => prob.sdc == SunDiskCondition.Sun0).sdcDecisionProbability * 100.0d) .ToString("F2") + "%") + "|" + String.Format("{0,9}", (currSDCdecisionProbabilitiesList.First( prob => prob.sdc == SunDiskCondition.Sun1).sdcDecisionProbability * 100.0d) .ToString("F2") + "%") + "|" + String.Format("{0,9}", (currSDCdecisionProbabilitiesList.First( prob => prob.sdc == SunDiskCondition.Sun2).sdcDecisionProbability * 100.0d) .ToString("F2") + "%") + "|" + Environment.NewLine; } ServiceTools.logToTextFile(errorLogFilename, strToShow, true, false); #endregion output obtained SDC data to log file #region filter by SDC value if needed if (sdcFilter != SunDiskCondition.Undefined) { lStatsProcessing = lStatsProcessing.Where((ipd, idx) => lTplsPredictedSDClist[idx].Item3 == sdcFilter).ToList(); Console.WriteLine("Detected " + lStatsProcessing.Count + " images with SDC = " + sdcFilter.ToString()); } #endregion filter by SDC value if needed #endregion Filter by SDC values predicting it using pre-trained NN parameters #region ObservedCloudCoverDataCSVfile if (bFilterByObservedCloudCoverRecords) { Console.WriteLine("Reading observed cloud cover data CSV file..."); if (!File.Exists(ObservedCloudCoverDataCSVfile)) { Console.WriteLine("Unable to read observed data CSV file: " + ObservedCloudCoverDataCSVfile); return; } List <List <string> > lCSVfileContents = ServiceTools.ReadDataFromCSV(ObservedCloudCoverDataCSVfile, 1, true); if (!lCSVfileContents.Any()) { Console.WriteLine("The observed cloud cover CSV file seems to be empty: " + ObservedCloudCoverDataCSVfile); return; } List <ObservedClCoverData> lObservedData = lCSVfileContents.ConvertAll(lStr => new ObservedClCoverData(lStr)); List <SkyImagesDataWith_Concurrent_Stats_CloudCover_SDC> lImagesFilteredByAvailableObservedData = new List <SkyImagesDataWith_Concurrent_Stats_CloudCover_SDC>(); #region filter images by available observed data using DateTimeFilterTolerance Console.WriteLine("Filtering by observed data available..."); foreach (ObservedClCoverData observedData in lObservedData) { DateTime currObservedDatumDateTime = observedData.dt; List <SkyImagesDataWith_Concurrent_Stats_CloudCover_SDC> lImagesCloseToCurrObservedDatum = lStatsProcessing .Where(ipd => { TimeSpan tspan = new TimeSpan( Math.Abs( (ConventionalTransitions.DateTimeOfSkyImageFilename(ipd.filename) - currObservedDatumDateTime).Ticks)); return(tspan <= DateTimeFilterTolerance); }) .ToList() .ConvertAll(ifd => new SkyImagesDataWith_Concurrent_Stats_CloudCover_SDC() { // observedData // ifd skyImageFullFileName = ifd.filename, skyImageFileName = Path.GetFileName(ifd.filename), currImageDateTime = ConventionalTransitions.DateTimeOfSkyImageFilename(ifd.filename), observedCloudCoverData = observedData, concurrentDataXMLfile = ifd.concurrentDataXMLfile, concurrentData = ifd.concurrentData, grixyrgbStatsXMLfile = ifd.grixyrgbStatsXMLfile, grixyrgbStats = ifd.grixyrgbStats, SDCvalue = lTplsPredictedSDClist.First(tpl => tpl.Item1 == ifd).Item3, SDCprobabilities = lTplsPredictedSDClist.First(tpl => tpl.Item1 == ifd).Item2 }); lImagesFilteredByAvailableObservedData.AddRange(lImagesCloseToCurrObservedDatum); } #endregion filter images by available observed data using DateTimeFilterTolerance if (!lImagesFilteredByAvailableObservedData.Any()) { Console.WriteLine( "There is no images remain after filtering using all available data. Output will be empty."); } ServiceTools.WriteObjectToXML(lImagesFilteredByAvailableObservedData, outputDataFile); #region Сформируем и запишем данные в CSV-файл // Здесь есть данные по наблюдаемому CloudCover string csvHeader = lImagesFilteredByAvailableObservedData[0].grixyrgbStats.CSVHeader() + ",SunElevationDeg,SunAzimuthDeg,ObservedTotalCloudCover,ObservedLowerCloudCover,SDC,SDCprobabilityNoSun,SDCprobabilitySun0,SDCprobabilitySun1,SDCprobabilitySun2"; List <string> lCSVoutputData = lImagesFilteredByAvailableObservedData.ConvertAll(ifd => { // все стат. предикторы - как для SDC // данные CloudCover string retVal = ""; //ImagesProcessingData dt = tpl.Item2; //ObservedClCoverData clCov = tpl.Item1; retVal = ifd.grixyrgbStats.ToCSV() + "," + ifd.concurrentData.gps.SunZenithAzimuth().ElevationAngle.ToString().Replace(",", ".") + "," + ifd.concurrentData.gps.SunZenithAzimuth().Azimuth.ToString().Replace(",", ".") + "," + ifd.observedCloudCoverData.CloudCoverTotal.ToString() + "," + ifd.observedCloudCoverData.CloudCoverLower.ToString() + "," + ifd.SDCvalue.ToString() + "," + ifd.SDCprobabilities.First(prob => prob.sdc == SunDiskCondition.NoSun) .sdcDecisionProbability.ToString().Replace(",", ".") + "," + ifd.SDCprobabilities.First(prob => prob.sdc == SunDiskCondition.Sun0) .sdcDecisionProbability.ToString().Replace(",", ".") + "," + ifd.SDCprobabilities.First(prob => prob.sdc == SunDiskCondition.Sun1) .sdcDecisionProbability.ToString().Replace(",", ".") + "," + ifd.SDCprobabilities.First(prob => prob.sdc == SunDiskCondition.Sun2) .sdcDecisionProbability.ToString().Replace(",", "."); return(retVal); }); string strToOutputToCSVfile = string.Join(Environment.NewLine, lCSVoutputData); ServiceTools.logToTextFile(outputCSVfile, csvHeader + Environment.NewLine, true, false); ServiceTools.logToTextFile(outputCSVfile, strToOutputToCSVfile, true, false); #endregion Сформируем и запишем данные в CSV-файл } else { ServiceTools.WriteObjectToXML(lStatsProcessing, outputDataFile); #region Сформируем и запишем данные в CSV-файл // здесь нет данных по наблюдаемому Cloud Cover // не надо нам такое. Оставим все это только в виде XML-файла. #endregion Сформируем и запишем данные в CSV-файл } #endregion ObservedCloudCoverDataCSVfile #endregion Enumerating files and output to XML and CSV file Console.WriteLine("saved output data to file: " + Environment.NewLine + outputDataFile + Environment.NewLine + Environment.NewLine); Console.WriteLine("===FINISHED==="); Console.ReadKey(); }
private void btnProcess_Click(object sender, EventArgs e) { string fName = richTextBox1.Text; if (!File.Exists(fName)) { theLogWindow = ServiceTools.LogAText(theLogWindow, "couldn`t find a file: " + fName); return; } try { xlApp = new Microsoft.Office.Interop.Excel.Application(); xlApp.Visible = true; wb = xlApp.Workbooks.Open(fName, Type.Missing, false); ws = wb.Sheets[1]; } catch (Exception) { CloseAll(); } List <Tuple <int, SunElevationTestDataRecord> > lRecords = new List <Tuple <int, SunElevationTestDataRecord> >(); int rowIdx = 2; double dDataTimeRead = 0.0d; while (true) { DateTime dtVal; SunElevationTestDataRecord currRec = new SunElevationTestDataRecord(); System.Windows.Forms.Application.DoEvents(); try { Range rngDT = ws.Cells[rowIdx, 1]; var dDataTimeReadValue = rngDT.Value2; if (dDataTimeReadValue == null) { theLogWindow = ServiceTools.LogAText(theLogWindow, "date-time value at row " + rowIdx + " is empty. Stopping reading."); CloseAll(); break; } dDataTimeRead = (double)(dDataTimeReadValue); if (dDataTimeRead == 0.0d) { theLogWindow = ServiceTools.LogAText(theLogWindow, "date-time value at row " + rowIdx + " is empty. Stopping reading."); CloseAll(); break; } } catch (Exception ex) { //CloseAll(); theLogWindow = ServiceTools.LogAText(theLogWindow, "====================="); theLogWindow = ServiceTools.LogAText(theLogWindow, ex.Message + Environment.NewLine + "for date-time value at row " + rowIdx); theLogWindow = ServiceTools.LogAText(theLogWindow, "====================="); lRecords.Add(new Tuple <int, SunElevationTestDataRecord>(rowIdx, currRec)); rowIdx++; continue; } dtVal = DateTime.FromOADate(dDataTimeRead); currRec.Dt = dtVal; double latVal = 0.0d; try { Range rngLat = ws.Cells[rowIdx, 2]; var rngLatVal = rngLat.Value2; if (rngLatVal == null) { theLogWindow = ServiceTools.LogAText(theLogWindow, "latitude value at row " + rowIdx + " is empty"); //rowIdx++; //continue; } else { latVal = (double)(rngLatVal); currRec.latDec = latVal; } } catch (Exception ex) { //CloseAll(); theLogWindow = ServiceTools.LogAText(theLogWindow, "====================="); theLogWindow = ServiceTools.LogAText(theLogWindow, ex.Message + Environment.NewLine + "for latitude value at row " + rowIdx); theLogWindow = ServiceTools.LogAText(theLogWindow, "====================="); lRecords.Add(new Tuple <int, SunElevationTestDataRecord>(rowIdx, currRec)); rowIdx++; continue; } double lonVal = 0.0d; try { Range rngLon = ws.Cells[rowIdx, 3]; var rngLonVal = rngLon.Value2; if (rngLonVal == null) { theLogWindow = ServiceTools.LogAText(theLogWindow, "longitude value at row " + rowIdx + " is empty"); //rowIdx++; //continue; } else { lonVal = (double)(rngLonVal); currRec.lonDec = lonVal; } } catch (Exception ex) { //CloseAll(); theLogWindow = ServiceTools.LogAText(theLogWindow, "====================="); theLogWindow = ServiceTools.LogAText(theLogWindow, ex.Message + Environment.NewLine + "for longitude value at row " + rowIdx); theLogWindow = ServiceTools.LogAText(theLogWindow, "====================="); lRecords.Add(new Tuple <int, SunElevationTestDataRecord>(rowIdx, currRec)); rowIdx++; continue; } double sunAltTestVal = 0.0d; try { Range rngSunAltTest = ws.Cells[rowIdx, 4]; var rngSunAltTestVal = rngSunAltTest.Value2; if (rngSunAltTestVal == null) { theLogWindow = ServiceTools.LogAText(theLogWindow, "testing sun elevation value at row " + rowIdx + " is empty"); //rowIdx++; //continue; } else { sunAltTestVal = (double)(rngSunAltTestVal); currRec.SunElevTest = sunAltTestVal; } } catch (Exception ex) { theLogWindow = ServiceTools.LogAText(theLogWindow, "====================="); theLogWindow = ServiceTools.LogAText(theLogWindow, ex.Message + Environment.NewLine + "for read sun elevation value at row " + rowIdx); theLogWindow = ServiceTools.LogAText(theLogWindow, "====================="); //CloseAll(); lRecords.Add(new Tuple <int, SunElevationTestDataRecord>(rowIdx, currRec)); rowIdx++; continue; } ThreadSafeOperations.SetText(lblLoadingStatus, "loading " + rowIdx + " row", false); lRecords.Add(new Tuple <int, SunElevationTestDataRecord>(rowIdx, currRec)); rowIdx++; } List <double> lDeviations = new List <double>(); string outFName = Path.GetDirectoryName(fName) + "\\" + Path.GetFileNameWithoutExtension(fName) + "-output.txt"; ServiceTools.logToTextFile(outFName, SunElevationTestDataRecord.TableFieldsHeader() + Environment.NewLine, true); foreach (Tuple <int, SunElevationTestDataRecord> tpl in lRecords) { rowIdx = tpl.Item1; SunElevationTestDataRecord currRec = tpl.Item2; if (currRec.WhetherAllValuesHasBeenRead) { SPA spaCalc = new SPA(currRec.Dt.Year, currRec.Dt.Month, currRec.Dt.Day, currRec.Dt.Hour, currRec.Dt.Minute, currRec.Dt.Second, (float)currRec.lonDec, (float)currRec.latDec, (float)SPAConst.DeltaT(currRec.Dt)); int res = spaCalc.spa_calculate(); AzimuthZenithAngle sunPositionSPAext = new AzimuthZenithAngle(spaCalc.spa.azimuth, spaCalc.spa.zenith); currRec.SunElevCalc = sunPositionSPAext.ElevationAngle; lDeviations.Add(currRec.SunElevCalc - currRec.SunElevTest); ServiceTools.logToTextFile(outFName, currRec.ToString() + Environment.NewLine, true); } else { ServiceTools.logToTextFile(outFName, currRec.ToString() + Environment.NewLine, true); } } HistogramDataAndProperties histData = new HistogramDataAndProperties(DenseVector.OfEnumerable(lDeviations), 20); HistogramCalcAndShowForm histForm = new HistogramCalcAndShowForm("", defaultProperties); histForm.HistToRepresent = histData; histForm.Show(); histForm.Represent(); CloseAll(); }
private async Task Run() { if (tgrm_token == "") { Console.WriteLine("telegram token for channel is not set. Processing will not proceed."); Console.WriteLine("Finished. Press any key..."); Console.ReadKey(); return; } var Bot = new Api(tgrm_token); var me = await Bot.GetMe(); Console.WriteLine("Hello my name is {0}", me.Username); var offset = 0; while (true) { var updates = await Bot.GetUpdates(offset); foreach (var update in updates) { if (update.Message.Type == MessageType.TextMessage) { if (ServiceTools.CheckIfDirectoryExists(Directory.GetCurrentDirectory() + Path.DirectorySeparatorChar + "logs")) { ServiceTools.logToTextFile(logFilename, "" + update.Message.Chat.Id + " : " + update.Message.Text + Environment.NewLine, true, true); } if (update.Message.Text == "/current_image") { string FilenameToSend = ""; Image <Bgr, byte> lastImagesCouple = CurrentImagesCouple(out FilenameToSend); string tmpFNameToSave = Path.GetTempPath(); tmpFNameToSave += (tmpFNameToSave.Last() == Path.DirectorySeparatorChar) ? ("") : (Path.DirectorySeparatorChar.ToString()); tmpFNameToSave += FilenameToSend; lastImagesCouple.Save(tmpFNameToSave); var fileStream = File.Open(tmpFNameToSave, FileMode.Open); Message sentMessage = await Bot.SendPhoto(update.Message.Chat.Id, new FileToSend(FilenameToSend, fileStream)); File.Delete(tmpFNameToSave); } else if (update.Message.Text == "/concurrent_info") { string lastConcurrentInfoFileName = GetLastConcurrentInfo(); Message sentMessage = await Bot.SendTextMessage(update.Message.Chat.Id, lastConcurrentInfoFileName, true, false, update.Message.MessageId); } else if (update.Message.Text == "/current_cc") { Task <string> tskObtainCC = ObtainLastImageCC(); //Message sentMessage1 = // await // Bot.SendTextMessage(update.Message.Chat.Id, // "Started calculations. Be patient please, it may take a few minutes.", true, // false, update.Message.MessageId); string strReply = await tskObtainCC; Message sentMessage2 = await Bot.SendTextMessage(update.Message.Chat.Id, strReply, true, false, update.Message.MessageId); } else if (update.Message.Text == "/meteo_info") { string strCurrentMeteoParametersString = await ObtainLatestMeteoParameters(); Message sentMessage = await Bot.SendTextMessage(update.Message.Chat.Id, strCurrentMeteoParametersString, true, false, update.Message.MessageId); } //else if (update.Message.Text == "/solar_irradiation") //{ // string strCurrentRadiometersString = await ObtainLatestRadiometersParameters(); // Message sentMessage = // await // Bot.SendTextMessage(update.Message.Chat.Id, strCurrentRadiometersString, true, false, // update.Message.MessageId); //} else if (update.Message.Text == "/start") { string strStartMessage = ServiceTools.ReadTextFromFile(Directory.GetCurrentDirectory() + Path.DirectorySeparatorChar + "settings" + Path.DirectorySeparatorChar + "BotStartMessage.txt"); Message sentMessage = await Bot.SendTextMessage(update.Message.Chat.Id, strStartMessage, true, false, update.Message.MessageId); Console.WriteLine("Echo Message: {0}", update.Message.Text); } else { // await Bot.SendChatAction(update.Message.Chat.Id, ChatAction.Typing); // await Task.Delay(200); var t = await Bot.SendTextMessage(update.Message.Chat.Id, "I can`t get your request. Please try again."); Console.WriteLine("Echo Message: {0}", update.Message.Text); } } if (update.Message.Type == MessageType.PhotoMessage) { var file = await Bot.GetFile(update.Message.Photo.LastOrDefault()?.FileId); Console.WriteLine("Received Photo: {0}", file.FilePath); var filename = IncomingImagesBasePath + file.FileId + "." + file.FilePath.Split('.').Last(); using (var profileImageStream = File.Open(filename, FileMode.Create)) { await file.FileStream.CopyToAsync(profileImageStream); } Console.WriteLine("Photo saved to: {0}", filename); if (ServiceTools.CheckIfDirectoryExists(Directory.GetCurrentDirectory() + Path.DirectorySeparatorChar + "logs")) { ServiceTools.logToTextFile(logFilename, "" + update.Message.Chat.Id + " : Received Photo: " + file.FilePath + Environment.NewLine + "\t\tSaved to file: " + filename + Environment.NewLine, true, true); } } offset = update.Id + 1; } await Task.Delay(200); if (NeedToStopFlag) { break; } } }
private void CurrFileProcessing(ImagesConvertingData dat) { string currentFullFileName = dat.filename; try { List <List <string> > csvFileContents = ServiceTools.ReadDataFromCSV(currentFullFileName, 3, true, ";", Environment.NewLine); List <double> lCenterXValues = csvFileContents.ConvertAll <double>(lStr => Convert.ToDouble(lStr[1])); List <double> lCenterYValues = csvFileContents.ConvertAll <double>(lStr => Convert.ToDouble(lStr[2])); List <double> lRadiiValues = csvFileContents.ConvertAll <double>(lStr => Convert.ToDouble(lStr[3])); // filter default values // filteringMarks = "те, которые оставляем" List <bool> filteringMarks = csvFileContents.ConvertAll <bool>(lStr => true); if (filterDefaultDoubleValueCenterX > 0.0d) { List <bool> filteringMarksAdd = lCenterXValues.ConvertAll <bool>(dVal => dVal != filterDefaultDoubleValueCenterX); filteringMarks = (filteringMarks.Zip <bool, bool, bool>(filteringMarksAdd, (bVal1, bVal2) => bVal1 && bVal2)) .ToList(); } if (filterDefaultDoubleValueCenterY > 0.0d) { List <bool> filteringMarksAdd = lCenterYValues.ConvertAll <bool>(dVal => dVal != filterDefaultDoubleValueCenterY); filteringMarks = (filteringMarks.Zip <bool, bool, bool>(filteringMarksAdd, (bVal1, bVal2) => bVal1 && bVal2)) .ToList(); } if (filterDefaultDoubleValueRadius > 0.0d) { List <bool> filteringMarksAdd = lRadiiValues.ConvertAll <bool>(dVal => dVal != filterDefaultDoubleValueRadius); filteringMarks = (filteringMarks.Zip <bool, bool, bool>(filteringMarksAdd, (bVal1, bVal2) => bVal1 && bVal2)) .ToList(); } List <int> indexes = filteringMarks .Select((val, idx) => new { val, idx }) .Where(x => x.val) .Select(x => x.idx) .ToList(); lCenterXValues = lCenterXValues .Select((dVal, idx) => new { dVal, idx }) .Where(x => indexes.Contains(x.idx)) .Select(x => x.dVal).ToList(); lCenterYValues = lCenterYValues .Select((dVal, idx) => new { dVal, idx }) .Where(x => indexes.Contains(x.idx)) .Select(x => x.dVal).ToList(); lRadiiValues = lRadiiValues .Select((dVal, idx) => new { dVal, idx }) .Where(x => indexes.Contains(x.idx)) .Select(x => x.dVal).ToList(); DescriptiveStatistics stats = new DescriptiveStatistics(lCenterXValues, true); double dCenterXvalue = stats.Mean; stats = new DescriptiveStatistics(lCenterYValues, true); double dCenterYvalue = stats.Mean; stats = new DescriptiveStatistics(lRadiiValues, true); double dRadiusvalue = stats.Mean; RoundData rd = new RoundData(dCenterXvalue, dCenterYvalue, dRadiusvalue); string xmlFilename = Path.GetDirectoryName(currentFullFileName); xmlFilename += (xmlFilename.Last() == Path.DirectorySeparatorChar) ? ("") : (Path.DirectorySeparatorChar.ToString()); xmlFilename += Path.GetFileNameWithoutExtension(currentFullFileName) + "-RoundImagemask.xml"; ServiceTools.WriteObjectToXML(rd, xmlFilename); Console.WriteLine("finished processing file " + Environment.NewLine + currentFullFileName); } catch (Exception ex) { #region report #if DEBUG Console.WriteLine("exception has been thrown: " + ex.Message + Environment.NewLine + ServiceTools.CurrentCodeLineDescription()); #else ServiceTools.logToTextFile(errorLogFilename, "exception has been thrown: " + ex.Message + Environment.NewLine + ServiceTools.CurrentCodeLineDescription(), true, true); #endif #endregion report } }
private void btnPerformExport_Click(object sender, EventArgs e) { string outPath = rtbExportDestinationDirectoryPath.Text; outPath += (outPath.Last() == '\\') ? ("") : ("\\"); if (cbExportMeteoData.Checked) { //export meteo data BackgroundWorker bgwReadMeteoData = new BackgroundWorker(); bgwReadMeteoData.WorkerSupportsCancellation = false; bgwReadMeteoData.WorkerReportsProgress = true; bgwReadMeteoData.DoWork += delegate(object currBGWsender, DoWorkEventArgs args) { BackgroundWorker selfWorker = currBGWsender as BackgroundWorker; TimeSeries <MeteoData> tsMeteoData = new TimeSeries <MeteoData>(); string[] meteoDataFiles = Directory.GetFiles(Directory.GetCurrentDirectory() + "\\logs\\", "*MeteoDataLog*.nc"); int totalFilesCount = meteoDataFiles.Count(); int readFiles = 0; foreach (string meteoDataFileName in meteoDataFiles) { ThreadSafeOperations.SetText(lblStatusBar, "reading " + Path.GetFileName(meteoDataFileName), false); Dictionary <string, object> currMeteoFileData = NetCDFoperations.ReadDataFromFile(meteoDataFileName); List <MeteoData> currFileMeteoData = MeteoData.OfDenseMatrix(currMeteoFileData["MeteoData"] as DenseMatrix); List <long> currFileDatetimeLong = new List <long>(currMeteoFileData["DateTime"] as long[]); List <DateTime> currFileDatetime = currFileDatetimeLong.ConvertAll <DateTime>(longDT => new DateTime(longDT)); tsMeteoData.AddSubseriaData(currFileMeteoData, currFileDatetime, true); readFiles++; selfWorker.ReportProgress(Convert.ToInt32(100.0d * readFiles / totalFilesCount)); } args.Result = new object[] { tsMeteoData }; }; bgwReadMeteoData.ProgressChanged += delegate(object currBGWsender, ProgressChangedEventArgs args) { ThreadSafeOperations.UpdateProgressBar(prbExportProgress, args.ProgressPercentage); }; bgwReadMeteoData.RunWorkerCompleted += delegate(object currBGWsender, RunWorkerCompletedEventArgs args) { ThreadSafeOperations.UpdateProgressBar(prbExportProgress, 0); TimeSeries <MeteoData> tsMeteoData = (args.Result as object[])[0] as TimeSeries <MeteoData>; List <Tuple <DateTime, MeteoData> > lTplMeteoData = tsMeteoData.TimeStamps.Zip(tsMeteoData.DataValues, (dt, dat) => new Tuple <DateTime, MeteoData>(dt, dat)).ToList(); if (cbExportFormatCSV.Checked) { List <string> tsMeteoDataCSV = lTplMeteoData.ConvertAll <string>(tpl => tpl.Item1.ToString("o") + "," + tpl.Item2.ToCSV()); string tsMeteoDataCSVForFile = String.Join(Environment.NewLine, tsMeteoDataCSV.ToArray <string>()); ServiceTools.logToTextFile(outPath + "MeteoData.csv", "DateTime," + lTplMeteoData[0].Item2.CSVHeader() + Environment.NewLine, false, false); ServiceTools.logToTextFile(outPath + "MeteoData.csv", tsMeteoDataCSVForFile, true, false); } }; ThreadSafeOperations.SetText(lblStatusBar, "meteo data read started", false); bgwReadMeteoData.RunWorkerAsync(); } }
private void btnProcessAccelerationTimeSeries_Click(object sender, EventArgs e) { if ((bgwCalculate != null) && (bgwCalculate.IsBusy)) { bgwCalculate.CancelAsync(); return; } //simpleMultipleImagesShow imagesRepresentingForm = new simpleMultipleImagesShow(); //imagesRepresentingForm.Show(); DoWorkEventHandler bgwCalculate_DoWorkHandler = delegate(object currBGWsender, DoWorkEventArgs args) { BackgroundWorker selfWorker = currBGWsender as BackgroundWorker; //simpleMultipleImagesShow multImagesRepresentingForm = (simpleMultipleImagesShow)((args.Argument as object[])[0]); //Type theShowImagesType = multImagesRepresentingForm.GetType(); //MethodInfo thePicturePlacingMethodInfo = theShowImagesType.GetMethod("PlaceAPicture"); int imageRepresentingCounter = 0; DateTime dbgDT = new DateTime(2014, 7, 9, 9, 0, 0); dbgDT = dbgDT.AddMinutes(33); DateTime dtSeriesStart = accSubseries[0].StartTime; double tsOverallSeriesDurationMillisec = (accSubseries[accSubseries.Count - 1].EndTime - accSubseries[0].StartTime).TotalMilliseconds; string strToWrite = " fileName ; lat ; lon ; date ; time ; time(s) since start ; period(s) ; spectrum amplitude"; ServiceTools.logToTextFile(strOutputDirectory + "\\100sData-spectra-maximums.dat", strToWrite + Environment.NewLine, true); foreach (TimeSeries <double> accSubseria in accSubseries) { if (selfWorker.CancellationPending) { break; } int startindex = 0; while (true) { int endIndex; if (selfWorker.CancellationPending) { break; } TimeSeries <double> currTimeSeria = accSubseria.SubSeria(startindex, new TimeSpan(1000000000), out endIndex); //100s currTimeSeria = currTimeSeria.InterpolateSeria(new TimeSpan(500000)); currTimeSeria = currTimeSeria.ExtractDataDeviationValues(); //обработать и оценить наличие выраженных периодов Complex[] sourceSignalArray = currTimeSeria.DataRealValuesComplexArray(); Fourier.Forward(sourceSignalArray); //Transform.FourierForward(sourceSignalArray); List <Complex> FourierTransformedSignal = new List <Complex>(sourceSignalArray); List <double> FourierTransformedSignalAmplitudes = FourierTransformedSignal.ConvertAll <double>( cVal => ((double.IsNaN(cVal.Magnitude)) || (double.IsInfinity(cVal.Magnitude))) ? (0.0d) : (cVal.Magnitude)); List <double> FourierTransformedSignalPeriods = new List <double>(); for (int ind = 0; ind < FourierTransformedSignalAmplitudes.Count; ind++) { FourierTransformedSignalPeriods.Add(currTimeSeria.TotalSeriaDuration.TotalSeconds / (double)ind); } FourierTransformedSignalAmplitudes = new List <double>( FourierTransformedSignalAmplitudes.Zip <double, double, double>( FourierTransformedSignalPeriods, (amp, periodSec) => ((double.IsNaN(periodSec)) || (double.IsInfinity(periodSec))) ? (0.0d) : (amp))); FourierTransformedSignalPeriods = FourierTransformedSignalPeriods.ConvertAll <double>( dval => ((double.IsNaN(dval)) || (double.IsInfinity(dval))) ? (0.0d) : (dval)); //проанализируем этот участок - есть ли выраженные пики по амплитуде конкретных частот //найти максимум в спектре и выдать данные об этом максимуме в файл // сначала отфильтруем периоды меньше 1с - для данных по динамике судна они несущественны FourierTransformedSignalAmplitudes = new List <double>( FourierTransformedSignalAmplitudes.Zip <double, double, double>( FourierTransformedSignalPeriods, (amp, periodSec) => (periodSec <= 1.0d) ? (0.0d) : (amp))); FourierTransformedSignalPeriods = FourierTransformedSignalPeriods.ConvertAll <double>(dVal => (dVal <= 1.0d) ? (0.0d) : (dVal)); DescriptiveStatistics currAmpsStat = new DescriptiveStatistics(FourierTransformedSignalAmplitudes); List <double> lAmpsOutstanding = FourierTransformedSignalAmplitudes.ConvertAll <double>(dVal => { if (dVal / currAmpsStat.Mean >= 100.0d) { return(dVal); } return(0.0d); }); List <double> lPeriodsOutstanding = new List <double>(FourierTransformedSignalPeriods.Zip <double, double, double>(lAmpsOutstanding, (per, amp) => (amp == 0.0d) ? (0.0d) : (per))); if (lAmpsOutstanding.Sum() > 0.0d) { MultipleScatterAndFunctionsRepresentation renderer = new MultipleScatterAndFunctionsRepresentation(2048, 1536); renderer.dvScatterXSpace.Add(currTimeSeria.TimeStampsValuesSeconds); renderer.dvScatterFuncValues.Add(currTimeSeria.dvDoubleDataValues); renderer.scatterLineColors.Add(new Bgr(255, 50, 50)); renderer.scatterDrawingVariants.Add(SequencesDrawingVariants.polyline); renderer.dvScatterXSpace.Add(DenseVector.OfEnumerable(FourierTransformedSignalPeriods)); renderer.dvScatterFuncValues.Add(DenseVector.OfEnumerable(FourierTransformedSignalAmplitudes)); renderer.scatterLineColors.Add(new Bgr(50, 255, 50)); renderer.scatterDrawingVariants.Add(SequencesDrawingVariants.squares); renderer.dvScatterXSpace.Add(DenseVector.OfEnumerable(lPeriodsOutstanding)); renderer.dvScatterFuncValues.Add(DenseVector.OfEnumerable(lAmpsOutstanding)); renderer.scatterLineColors.Add(new Bgr(50, 50, 255)); renderer.scatterDrawingVariants.Add(SequencesDrawingVariants.circles); renderer.Represent(); if (strOutputDirectory != "") { double maxAmp = lAmpsOutstanding.Max(); int idx = lAmpsOutstanding.FindIndex(dval => dval == maxAmp); double maxAmpPeriod = lPeriodsOutstanding[idx]; GPSdata gpsMark = gpsSeriaData.GetMostClose(currTimeSeria.StartTime).Item2; string fName = currTimeSeria.StartTime.ToString("s").Replace(":", "-") + "-100sData-spectrum.jpg"; renderer.SaveToImage(strOutputDirectory + "\\" + fName, true); strToWrite = "" + fName + " ; "; strToWrite += gpsMark.LatDec + " ; "; strToWrite += gpsMark.LonDec + " ; "; strToWrite += currTimeSeria.StartTime.Date.ToString("yyyy-MM-dd") + " ; "; strToWrite += currTimeSeria.StartTime.ToString("HH-mm-ss") + " ; "; strToWrite += (currTimeSeria.StartTime - dtSeriesStart).TotalSeconds + " ; "; strToWrite += maxAmpPeriod.ToString() + " ; "; strToWrite += maxAmp.ToString() + " ; "; ServiceTools.logToTextFile(strOutputDirectory + "\\100sData-spectra-maximums.dat", strToWrite + Environment.NewLine, true); ThreadSafeOperations.SetText(lblStatusString, "processing: " + currTimeSeria.StartTime.ToString("s"), false); } } if ((currTimeSeria.StartTime >= dbgDT) || (currTimeSeria.EndTime >= dbgDT)) { startindex++; startindex--; } if (endIndex == accSubseria.Count - 1) { break; } Application.DoEvents(); selfWorker.ReportProgress(Convert.ToInt32(100.0d * (currTimeSeria.EndTime - dtSeriesStart).TotalMilliseconds / tsOverallSeriesDurationMillisec)); startindex += Convert.ToInt32((endIndex - startindex) / 2.0d); } } }; RunWorkerCompletedEventHandler bgwCalculate_CompletedHandler = delegate(object currBGWCompletedSender, RunWorkerCompletedEventArgs args) { ThreadSafeOperations.ToggleButtonState(btnProcessAccelerationTimeSeries, true, "Process acceleration timeseries", false); }; ProgressChangedEventHandler bgwCalculate_ProgressChanged = delegate(object bgwDataReaderSender, ProgressChangedEventArgs args) { ThreadSafeOperations.UpdateProgressBar(prbReadingProcessingData, args.ProgressPercentage); }; ThreadSafeOperations.ToggleButtonState(btnProcessAccelerationTimeSeries, true, "STOP", true); bgwCalculate = new BackgroundWorker(); bgwCalculate.WorkerSupportsCancellation = true; bgwCalculate.WorkerReportsProgress = true; bgwCalculate.DoWork += bgwCalculate_DoWorkHandler; bgwCalculate.RunWorkerCompleted += bgwCalculate_CompletedHandler; bgwCalculate.ProgressChanged += bgwCalculate_ProgressChanged; //object[] bgwCalculateArgs = new object[] { imagesRepresentingForm }; object[] bgwCalculateArgs = new object[] { }; bgwCalculate.RunWorkerAsync(bgwCalculateArgs); }
//private void ImageProcessing(ImagesProcessingData ipdt) //{ //} private void EnumerateFilesToProcess() { string directory = Path.GetDirectoryName(inputBasePath); string filemask = "*.jpg"; List <string> filesList = new List <string>(Directory.EnumerateFiles(directory, filemask, bEnumerateFilesRecursively ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly)); #region filter by camID //...devID1.jpg string ptrnCamID = "devid" + CamIDtoProcess + ".jpg"; filesList = filesList.Where(fname => fname.ToLower().Contains(ptrnCamID)).ToList(); #endregion Console.WriteLine("found " + filesList.Count + " images."); #region list, read and map image stats files Console.WriteLine("filtering by ready-to-use GrIxYRGB XML files..."); List <string> statsFilesList = new List <string>(Directory.EnumerateFiles(imageYRGBstatsXMLdataFilesDirectory, ConventionalTransitions.ImageGrIxYRGBstatsFileNamesPattern(), bEnumerateFilesRecursively ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly)); List <string> statsFilesListWOpath = statsFilesList.ConvertAll(Path.GetFileName); Console.WriteLine("found " + statsFilesList.Count + " XML stats files in directory " + Environment.NewLine + imageYRGBstatsXMLdataFilesDirectory + Environment.NewLine + "by mask " + Environment.NewLine + ConventionalTransitions.ImageGrIxYRGBstatsFileNamesPattern()); int removed = filesList.RemoveAll( fname => !statsFilesListWOpath.Contains(ConventionalTransitions.ImageGrIxYRGBstatsDataFileName(fname, "", false))); Console.WriteLine("removed " + removed + " items (couldn`t find stats data files). Remains " + filesList.Count + " to process."); if (!filesList.Any()) { Console.WriteLine("There is no " + filemask + " files that sutisfy settings specified. Processing will not be started."); return; } lStatsProcessing = filesList.ConvertAll(strImgFname => { ImagesProcessingData retVal = new ImagesProcessingData() { filename = strImgFname }; return(retVal); }); #if DEBUG //lStatsProcessing = lStatsProcessing.Where((ipd, ind) => ind < 10).ToList(); #endif Console.WriteLine("started reading and mapping stats data"); int totalFilesCountToRead = lStatsProcessing.Count; int filesRead = 0; int currProgressPerc = 0; foreach (ImagesProcessingData ipdt in lStatsProcessing) { ipdt.grixyrgbStatsXMLfile = statsFilesList.First( statsFname => statsFname.Contains(ConventionalTransitions.ImageGrIxYRGBstatsDataFileName(ipdt.filename, "", false))); ipdt.grixyrgbStats = ServiceTools.ReadObjectFromXML(ipdt.grixyrgbStatsXMLfile, typeof(SkyImageIndexesStatsData)) as SkyImageIndexesStatsData; #region calculate and report progress filesRead++; double progress = 100.0d * (double)filesRead / (double)totalFilesCountToRead; if (progress - (double)currProgressPerc > 1.0d) { currProgressPerc = Convert.ToInt32(progress); Console.WriteLine("read " + currProgressPerc + "%"); } #endregion calculate and report progress } #endregion #region list, read and map concurrent data List <string> concurrentDataFilesList = Directory.EnumerateFiles(ConcurrentDataXMLfilesDirectory, ConventionalTransitions.ImageConcurrentDataFilesNamesPattern(), bEnumerateFilesRecursively ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly).ToList(); List <ConcurrentData> lConcurrentData = null; #region reading Console.WriteLine("started concurrent data reading"); totalFilesCountToRead = concurrentDataFilesList.Count; filesRead = 0; currProgressPerc = 0; List <Dictionary <string, object> > lDictionariesConcurrentData = new List <Dictionary <string, object> >(); foreach (string strConcDataXMLFile in concurrentDataFilesList) { Dictionary <string, object> currDict = ServiceTools.ReadDictionaryFromXML(strConcDataXMLFile); currDict.Add("XMLfileName", Path.GetFileName(strConcDataXMLFile)); lDictionariesConcurrentData.Add(currDict); #region calculate and report progress filesRead++; double progress = 100.0d * (double)filesRead / (double)totalFilesCountToRead; if (progress - (double)currProgressPerc > 1.0d) { currProgressPerc = Convert.ToInt32(progress); Console.WriteLine("read " + currProgressPerc + "%"); } #endregion calculate and report progress } lDictionariesConcurrentData.RemoveAll(dict => dict == null); lConcurrentData = lDictionariesConcurrentData.ConvertAll <ConcurrentData>(dict => { ConcurrentData retVal = null; try { retVal = new ConcurrentData(dict); } catch (Exception ex) { string strError = "couldn`t parse XML file " + dict["XMLfileName"] + " : " + Environment.NewLine + ex.Message; Console.WriteLine(strError); } return(retVal); }); lConcurrentData.RemoveAll(val => val == null); #endregion reading #region mapping // map obtained concurrent data to images by its datetime Console.WriteLine("concurrent data mapping started"); lStatsProcessing = lStatsProcessing.ConvertAll(ipdt => { string currImgFilename = ipdt.filename; currImgFilename = Path.GetFileNameWithoutExtension(currImgFilename); string ptrn = @"(devID\d)"; Regex rgxp = new Regex(ptrn, RegexOptions.IgnoreCase); string strCurrImgDT = rgxp.Replace(currImgFilename.Substring(4), ""); //2015-12-16T06-01-38 strCurrImgDT = strCurrImgDT.Substring(0, 11) + strCurrImgDT.Substring(11).Replace("-", ":"); DateTime currImgDT = DateTime.Parse(strCurrImgDT, null, System.Globalization.DateTimeStyles.AdjustToUniversal); ConcurrentData nearestConcurrentData = lConcurrentData.Aggregate((cDt1, cDt2) => { TimeSpan tspan1 = new TimeSpan(Math.Abs((cDt1.datetimeUTC - currImgDT).Ticks)); TimeSpan tspan2 = new TimeSpan(Math.Abs((cDt2.datetimeUTC - currImgDT).Ticks)); return((tspan1 <= tspan2) ? (cDt1) : (cDt2)); }); if (new TimeSpan(Math.Abs((nearestConcurrentData.datetimeUTC - currImgDT).Ticks)) >= TimeSpanForConcurrentDataMappingTolerance) { string strError = "couldn`t find close enough concurrent data file for image:" + Environment.NewLine + currImgFilename + Environment.NewLine + "closest concurrent data file is:" + Environment.NewLine + nearestConcurrentData.filename + Environment.NewLine + "with date-time value " + nearestConcurrentData.datetimeUTC.ToString("o"); Console.WriteLine(strError); nearestConcurrentData = null; } ipdt.concurrentData = nearestConcurrentData; if (nearestConcurrentData != null) { ipdt.concurrentDataXMLfile = nearestConcurrentData.filename; } return(ipdt); }); #endregion mapping removed = lStatsProcessing.RemoveAll(ipdt => ipdt.concurrentData == null); Console.WriteLine("removed " + removed + " items (couldn`t find concurrent data). " + lStatsProcessing.Count + " files remains to process."); #endregion list, read and map concurrent data if (!lStatsProcessing.Any()) { Console.WriteLine("There is no files that sutisfy settings specified and have all required concurrent data (stats or GPS etc.). Processing will not be proceeded."); return; } #region Predict SDC values using pre-trained NN parameters string csvHeader = lStatsProcessing[0].grixyrgbStats.CSVHeader() + ",SunElevationDeg,SunAzimuthDeg,sunDiskCondition"; List <string> lCSVheader = csvHeader.Split(',').ToList(); List <int> columnsToDelete = lCSVheader.Select((str, idx) => new Tuple <int, string>(idx, str)) .Where(tpl => tpl.Item2.ToLower().Contains("filename")).ToList().ConvertAll(tpl => tpl.Item1); List <List <string> > lCalculatedData = lStatsProcessing.ConvertAll(dt => { string currImageALLstatsDataCSVWithConcurrentData = dt.grixyrgbStats.ToCSV() + "," + dt.concurrentData.gps.SunZenithAzimuth() .ElevationAngle.ToString() .Replace(",", ".") + "," + dt.concurrentData.gps.SunZenithAzimuth() .Azimuth.ToString() .Replace(",", "."); List <string> retVal = currImageALLstatsDataCSVWithConcurrentData.Split(',').ToList(); retVal = retVal.Where((str, idx) => !columnsToDelete.Contains(idx)).ToList(); return(retVal); }); List <DenseVector> lDV_objects_features = lCalculatedData.ConvertAll( list => DenseVector.OfEnumerable(list.ConvertAll <double>(str => Convert.ToDouble(str.Replace(".", ","))))); DenseVector dvMeans = (DenseVector)((DenseMatrix)ServiceTools.ReadDataFromCSV(NormMeansFile, 0, ",")).Row(0); DenseVector dvRanges = (DenseVector)((DenseMatrix)ServiceTools.ReadDataFromCSV(NormRangeFile, 0, ",")).Row(0); lDV_objects_features = lDV_objects_features.ConvertAll(dv => { DenseVector dvShifted = dv - dvMeans; DenseVector dvNormed = (DenseVector)dvShifted.PointwiseDivide(dvRanges); return(dvNormed); }); DenseMatrix dmObjectsFeatures = DenseMatrix.OfRowVectors(lDV_objects_features); DenseVector dvThetaValues = (DenseVector)ServiceTools.ReadDataFromCSV(NNtrainedParametersFile, 0, ","); List <int> NNlayersConfig = new List <double>(((DenseMatrix)ServiceTools.ReadDataFromCSV(NNconfigFile, 0, ",")).Row(0)).ConvertAll (dVal => Convert.ToInt32(dVal)); List <List <double> > lDecisionProbabilities = null; List <SunDiskCondition> predictedSDClist = NNclassificatorPredictor <SunDiskCondition> .NNpredict(dmObjectsFeatures, dvThetaValues, NNlayersConfig, out lDecisionProbabilities, SunDiskConditionData.MatlabEnumeratedSDCorderedList()).ToList(); //List<SunDiskCondition> predictedSDClist = predictedSDC.ConvertAll(sdcInt => //{ // switch (sdcInt) // { // case 4: // return SunDiskCondition.NoSun; // break; // case 1: // return SunDiskCondition.Sun0; // break; // case 2: // return SunDiskCondition.Sun1; // break; // case 3: // return SunDiskCondition.Sun2; // break; // default: // return SunDiskCondition.Defect; // } //}); string strToShow = "SDC values probabilities: " + Environment.NewLine + "| No Sun | Sun_0 | Sun_1 | Sun_2 | Detected |" + Environment.NewLine; foreach (List <double> lDecisionProbability in lDecisionProbabilities) { strToShow += "| " + lDecisionProbability[3].ToString("F4") + " | " + lDecisionProbability[0].ToString("F4") + " | " + lDecisionProbability[1].ToString("F4") + " | " + lDecisionProbability[2].ToString("F4") + " |" + predictedSDClist[lDecisionProbabilities.IndexOf(lDecisionProbability)] + "|" + Environment.NewLine; } ServiceTools.logToTextFile(errorLogFilename, strToShow, true, false); #endregion //lStatsProcessing = // lStatsProcessing.Where((ipd, idx) => predictedSDClist[idx] == SunDiskCondition.Sun2).ToList(); lStatsProcessing = lStatsProcessing.Where((ipd, idx) => predictedSDClist[idx] == sdcFilter).ToList(); Console.WriteLine("Detected " + lStatsProcessing.Count + " images with SDC = " + sdcFilter.ToString()); if (!lStatsProcessing.Any()) { Console.WriteLine("There is no files with SDC = Sun2. Processing will not be proceeded."); return; } Console.WriteLine("finished enumerating and filtering files. Files to process: " + lStatsProcessing.Count); }
private void ProcessImage(ImageStatsCollectingData srcData) { Interlocked.Increment(ref totalFilesProcessed); int perc = Convert.ToInt32(100.0d * (double)totalFilesProcessed / (double)totalFilesCountToProcess); Console.WriteLine(DateTime.Now.ToString("s") + " : " + perc + "% : started processing file " + Environment.NewLine + srcData.filename); Dictionary <string, object> optionalParameters = new Dictionary <string, object>(); optionalParameters.Add("ImagesRoundMasksXMLfilesMappingList", ImagesRoundMasksXMLfilesMappingList); Stopwatch sw = new Stopwatch(); sw.Start(); optionalParameters.Add("Stopwatch", sw); optionalParameters.Add("logFileName", errorLogFilename); ImageStatsDataCalculationResult currImageProcessingResult = ImageProcessing.CalculateImageStatsData(srcData.filename, optionalParameters); currImageProcessingResult.stopwatch.Stop(); if (currImageProcessingResult.calcResult) { string currentFullFileName = currImageProcessingResult.imgFilename; string strPerfCountersData = currentFullFileName + ";" + currImageProcessingResult.stopwatch.ElapsedMilliseconds + ";" + (currImageProcessingResult.procTotalProcessorTimeEnd - currImageProcessingResult.procTotalProcessorTimeStart).TotalMilliseconds + Environment.NewLine; ServiceTools.logToTextFile(strPerformanceCountersStatsFile, strPerfCountersData, true); //string strImageGrIxMedianP5DataFileName = // ConventionalTransitions.ImageGrIxMedianP5DataFileName(currentFullFileName, imageMP5statsXMLdataFilesDirectory); //ServiceTools.WriteObjectToXML(currImageProcessingResult.mp5Result, strImageGrIxMedianP5DataFileName); string strImageGrIxYRGBDataFileName = ConventionalTransitions.ImageGrIxYRGBstatsDataFileName(currentFullFileName, imageYRGBstatsXMLdataFilesDirectory, true, currPath2Process); ServiceTools.WriteObjectToXML(currImageProcessingResult.grixyrgbStatsData, strImageGrIxYRGBDataFileName); ImageStatsCollectingData foundDataObj = lStatsCalculation.Find(obj => obj.filename == currentFullFileName); foundDataObj.State = ImageStatsCollectingState.Finished; foundDataObj.GrIxMedianValue = currImageProcessingResult.mp5Result.GrIxStatsMedian; foundDataObj.GrIxPerc5Value = currImageProcessingResult.mp5Result.GrIxStatsPerc5; Console.WriteLine(DateTime.Now.ToString("s") + " : finished processing file " + Environment.NewLine + currentFullFileName); } else { string currentFullFileName = currImageProcessingResult.imgFilename; ImageStatsCollectingData foundDataObj = lStatsCalculation.Find(obj => obj.filename == currentFullFileName); foundDataObj.State = ImageStatsCollectingState.Error; Console.WriteLine("ERROR processing file " + Path.GetFileName(currentFullFileName)); try { //report full error to error log file #region report error #if (DEBUG && MONO) ServiceTools.logToTextFile(errorLogFilename, "Error processing file: " + Environment.NewLine + currentFullFileName + Environment.NewLine + "messages: " + ServiceTools.GetExceptionMessages(currImageProcessingResult.exception) + Environment.NewLine + "Stack trace: " + Environment.NewLine + Environment.StackTrace + Environment.NewLine + Environment.NewLine, true, true); #else #if MONO ServiceTools.logToTextFile(errorLogFilename, "Error processing file: " + Environment.NewLine + currentFullFileName + Environment.NewLine + "messages: " + ServiceTools.GetExceptionMessages(currImageProcessingResult.exception) + Environment.NewLine + "Stack trace: " + Environment.NewLine + Environment.StackTrace + Environment.NewLine + Environment.NewLine, true, true); #else ServiceTools.logToTextFile(errorLogFilename, "Error processing file: " + Environment.NewLine + currentFullFileName + Environment.NewLine + "message: " + ServiceTools.GetExceptionMessages(currImageProcessingResult.exception) + Environment.NewLine + ServiceTools.CurrentCodeLineDescription() + Environment.NewLine + "Stack trace: " + Environment.NewLine + Environment.StackTrace + Environment.NewLine + Environment.NewLine, true, true); #endif #endif #endregion report error } catch (Exception ex) { return; } } }
private async void ConvertImage(ImageStatsCollectingData srcData) { Interlocked.Increment(ref totalFilesProcessed); int perc = Convert.ToInt32(100.0d * (double)totalFilesProcessed / (double)totalFilesCountToProcess); Console.WriteLine(DateTime.Now.ToString("s") + " : " + perc + "% : started processing file " + Environment.NewLine + srcData.filename); Dictionary <string, object> taskParameters = new Dictionary <string, object>(); foreach (string key in defaultProperties.Keys) { taskParameters.Add(key, defaultProperties[key]); } Stopwatch sw = new Stopwatch(); sw.Start(); taskParameters.Add("Stopwatch", sw); taskParameters.Add("logFileName", errorLogFilename); taskParameters.Add("currentFullFileName", srcData.filename); ImageStatsDataCalculationResult currImageConversionResult = await ConvertImageTask(taskParameters); #region process result currImageConversionResult.stopwatch.Stop(); if (currImageConversionResult.calcResult) { string currentFullFileName = currImageConversionResult.imgFilename; string strPerfCountersData = currentFullFileName + ";" + currImageConversionResult.stopwatch.ElapsedMilliseconds + ";" + (currImageConversionResult.procTotalProcessorTimeEnd - currImageConversionResult.procTotalProcessorTimeStart).TotalMilliseconds + Environment.NewLine; ServiceTools.logToTextFile(strPerformanceCountersStatsFile, strPerfCountersData, true); ImageStatsCollectingData foundDataObj = lStatsCalculation.Find(obj => obj.filename == currentFullFileName); foundDataObj.State = ImageStatsCollectingState.Finished; Console.WriteLine(DateTime.Now.ToString("s") + " : finished processing file " + Environment.NewLine + currentFullFileName); } else { string currentFullFileName = currImageConversionResult.imgFilename; ImageStatsCollectingData foundDataObj = lStatsCalculation.Find(obj => obj.filename == currentFullFileName); foundDataObj.State = ImageStatsCollectingState.Error; Console.WriteLine("ERROR processing file " + Path.GetFileName(currentFullFileName)); try { #region report error ServiceTools.logToTextFile(errorLogFilename, "Error processing file: " + Environment.NewLine + currentFullFileName + Environment.NewLine + "message: " + ServiceTools.GetExceptionMessages(currImageConversionResult.exception) + Environment.NewLine + ServiceTools.CurrentCodeLineDescription() + Environment.NewLine + "Stack trace: " + Environment.NewLine + Environment.StackTrace + Environment.NewLine + Environment.NewLine, true, true); #endregion report error } catch (Exception ex) { return; } } #endregion process result }
private void Bot_OnMessage(object sender, Telegram.Bot.Args.MessageEventArgs e) { Message currMsg = e.Message; if (currMsg.Type == MessageType.TextMessage) { Console.WriteLine("{0} from {1} ({2})", currMsg.Text, currMsg.Chat.Id, currMsg.Chat.Username); if (ServiceTools.CheckIfDirectoryExists(Directory.GetCurrentDirectory() + Path.DirectorySeparatorChar + "logs")) { ServiceTools.logToTextFile(logFilename, "" + currMsg.Chat.Id + " (" + currMsg.Chat.Username + ") : " + currMsg.Text + Environment.NewLine, true, true); } if ((currMsg.Text == "/current_image") || (currMsg.Text == "Image")) { Task <string> taskA = Task.Run(() => CurrentImagesCoupleImageFilename()); Task continuation = taskA.ContinueWith(antecedent => { string FilenameToSend = antecedent.Result; var fileStream = File.Open(FilenameToSend, FileMode.Open, FileAccess.Read, FileShare.Read); Bot.SendPhotoAsync(currMsg.Chat.Id, new FileToSend(FilenameToSend, fileStream)); }); } else if ((currMsg.Text == "/concurrent_info") || (currMsg.Text == "Info")) { Task <string> taskA = Task.Run(() => ReadLastConcurrentInfo()); Task continuation = taskA.ContinueWith(antecedent => { string strReply = antecedent.Result; Bot.SendTextMessageAsync(currMsg.Chat.Id, strReply, true, false, currMsg.MessageId, BasicKeyboard()); }); } else if ((currMsg.Text == "/current_cc") || (currMsg.Text == "TCC_SDC")) { Task <string> taskA = Task.Run(ReadCurrentCCinfo); Task continuation = taskA.ContinueWith(antecedent => { string strReply = antecedent.Result; Bot.SendTextMessageAsync(currMsg.Chat.Id, strReply, true, false, currMsg.MessageId, BasicKeyboard()); }); } else if ((currMsg.Text == "/meteo_info") || (currMsg.Text == "Meteo")) { Task <string> taskA = Task.Run(ObtainLatestMeteoParameters); Task continuation = taskA.ContinueWith(antecedent => { string strReply = antecedent.Result; Bot.SendTextMessageAsync(currMsg.Chat.Id, strReply, true, false, currMsg.MessageId, BasicKeyboard()); }); } else if (currMsg.Text == "/start") { Task taskA = Task.Run(() => { string strStartMessage = ServiceTools.ReadTextFromFile(Directory.GetCurrentDirectory() + Path.DirectorySeparatorChar + "settings" + Path.DirectorySeparatorChar + "BotStartMessage.txt"); Bot.SendTextMessageAsync(currMsg.Chat.Id, strStartMessage, true, false, currMsg.MessageId, BasicKeyboard()); Console.WriteLine("Echo Message: {0}", currMsg.Text); }); } else if (currMsg.Text.Contains("/fix")) { Console.WriteLine("Echo Message: {0}", currMsg.Text); Task <bool> taskA = Task.Run(() => RegisterFixStage1(currMsg)); } else if (currMsg.Text.Contains("/help") || currMsg.Text == "Help") { Task taskA = Task.Run(() => { string strStartMessage = ServiceTools.ReadTextFromFile(Directory.GetCurrentDirectory() + Path.DirectorySeparatorChar + "settings" + Path.DirectorySeparatorChar + "BotHelpMessage.txt"); Bot.SendTextMessageAsync(currMsg.Chat.Id, strStartMessage, true, false, currMsg.MessageId, BasicKeyboard()); Console.WriteLine("Echo Message: {0}", currMsg.Text); }); } else { // await Bot.SendChatAction(update.Message.Chat.Id, ChatAction.Typing); // await Task.Delay(200); Task taskA = Task.Run(() => { Bot.SendTextMessageAsync(currMsg.Chat.Id, "Sorry, I can`t understand you. Please try again using the following list of commands:" + Environment.NewLine + "/current_image" + Environment.NewLine + "/concurrent_info" + Environment.NewLine + "/current_cc" + Environment.NewLine + "/meteo_info" + Environment.NewLine + "/fix", replyMarkup: BasicKeyboard()); Console.WriteLine("Echo Message: {0}", currMsg.Text); }); } } }