public AutomaticReadData() { OnlyNewData = true; CalculateValue = true; ReaderProccess = new ReaderProccess(); Loop = true; }
/// <summary> /// Read data from file with option onlyNewData and calculateValue /// </summary> /// <param name="logger"></param> /// <param name="dataPath"></param> /// <param name="onlyNewData"></param> /// <param name="calculateValue"></param> /// <param name="refreshEntityContext"></param> public void ReadData(Logger logger, string dataPath, bool onlyNewData, bool calculateValue, ReaderProccess readerProccess) { try { Console.WriteLine("Thread[" + Thread.CurrentThread.ManagedThreadId + "]: Start read data file " + dataPath); ReaderProccess fileReaderProccess; if (readerProccess.FileReaderProccesses == null) { fileReaderProccess = readerProccess; } else { fileReaderProccess = new ReaderProccess(); fileReaderProccess.TotalRecord = FileUtils.CountTotalLines(dataPath); readerProccess.FileReaderProccesses.Add(fileReaderProccess); } fileReaderProccess.CurrentFile = dataPath; bool readFolder = !logger.DataPath.Equals(dataPath); var file = new FileInfo(dataPath); // Only read if file is modified if (onlyNewData) { if (readFolder) { if (logger.LastModifyDatetime >= file.LastWriteTime && logger.Meta.Contains(file.Name)) return; } else { if (logger.LastModifyDatetime >= file.LastWriteTime || logger.FileSize == file.Length) return; } } var streamReader = new StreamReader(dataPath, Encoding.UTF8); var spiliter = new[] { logger.Delimiter }; // read meta line string lineData = streamReader.ReadLine(); fileReaderProccess.ReadRecord++; if (String.IsNullOrEmpty(lineData)) { Console.WriteLine("File Empty!"); return; } // check type of data file int type = 0; string[] firstlineCells = lineData.Split(spiliter, StringSplitOptions.None); if (firstlineCells[0].ToInt32TryParse() > 0 && firstlineCells[1].ToInt32TryParse() > 0) { type = 1; } if (type == 0) { // Ignore 4 first lines for (int i = 0; i < 4; i++) { lineData = streamReader.ReadLine(); fileReaderProccess.ReadRecord++; } } // Begin to read data var entityConntext = new GeoViewerEntities(); var sensors = entityConntext.Sensors.Where(ent => ent.LoggerID == logger.LoggerID).OrderBy(ent => ent.ColumnIndex).ToList(); int line = 0; do { if (!string.IsNullOrEmpty(lineData.Trim())) { string[] cells = lineData.Split(spiliter, StringSplitOptions.None); var meaTime = type == 0 ? Convert.ToDateTime(CleanValue(cells[0])) : new DateTime(cells[1].ToInt32TryParse(), 1, 1).AddDays( cells[2].ToInt32TryParse()).AddMinutes( (int)(cells[3].ToInt32TryParse() * 60 / 100)); // Only read new record (by time) if (!(onlyNewData && !readFolder && meaTime >= logger.FirstLogDatetime && meaTime <= logger.LastLogDatetime)) { var valuesNeedCalc = new List<SensorValue>(); foreach (var sensor in sensors) { if (cells.Length > sensor.ColumnIndex) { // Check Value exist //var sensorValue = sensor.SensorValues.SingleOrDefault(ent => ent.MeaTime == meaTime); var sensorValue = entityConntext.SensorValues.SingleOrDefault(ent => ent.SensorID == sensor.SensorID && ent.MeaTime == meaTime); if (sensorValue == null) { sensorValue = new SensorValue(); sensorValue.SensorID = sensor.SensorID; sensorValue.MeaTime = meaTime; sensor.SensorValues.Add(sensorValue); } sensorValue.RawValue = CleanValue(cells[sensor.ColumnIndex]).ToDecimalTryParse(); valuesNeedCalc.Add(sensorValue); } } // Calc Value & Alarm Logger if (calculateValue) { foreach (var sensorValue in valuesNeedCalc) { CalculationBLL.Current.Calculating(sensorValue, true); } } } line++; if (line % 50 == 0) { entityConntext.SaveChanges(); entityConntext = new GeoViewerEntities(); sensors = entityConntext.Sensors.Where(ent => ent.LoggerID == logger.LoggerID).OrderBy(ent => ent.ColumnIndex).ToList(); Console.WriteLine("Thread[" + Thread.CurrentThread.ManagedThreadId + "]: Read success " + line + " lines in " + dataPath); } } // go to nextline lineData = streamReader.ReadLine(); fileReaderProccess.ReadRecord++; } while ((lineData != null)); streamReader.Close(); if (fileReaderProccess.ReadRecord > 0) { fileReaderProccess.ReadRecord--; } if (line > 0 && line % 50 != 0) { entityConntext.SaveChanges(); Console.WriteLine("Thread[" + Thread.CurrentThread.ManagedThreadId + "]: Read success " + line + " lines in " + dataPath); } Console.WriteLine("Thread[" + Thread.CurrentThread.ManagedThreadId + "]: End read data from " + dataPath); if (readerProccess.FileReaderProccesses != null) { readerProccess.ReadRecord += fileReaderProccess.ReadRecord; } } catch (Exception exception) { Console.WriteLine(exception.Message); } }
/// <summary> /// Read data for logger /// </summary> /// <param name="logger"></param> /// <param name="onlyNewData"></param> /// <param name="calculateValue"></param> /// <param name="refreshEntityContext"></param> public void ReadData(int loggerID, bool onlyNewData, bool calculateValue, ReaderProccess readerProccess) { using (var entityConntext = new GeoViewerEntities()) { Logger logger = entityConntext.Loggers.SingleOrDefault(ent => ent.LoggerID == loggerID); if (logger == null) return; // For test only Console.WriteLine("Thread[" + Thread.CurrentThread.ManagedThreadId + "]: Read data form logger " + logger.LoggerID); // End for test var fileInfo = new FileInfo(logger.DataPath); if (!fileInfo.Exists) { readerProccess.TotalRecord = FileUtils.CountTotalFolderLines(logger.DataPath); readerProccess.FileReaderProccesses = new List<ReaderProccess>(); readerProccess.CurrentFile = logger.DataPath; var directoryInfo = new DirectoryInfo(logger.DataPath); if (!directoryInfo.Exists) throw new Exception(Properties.Resources.Error_Data_FileNotFound); string meta = ""; if (logger.Meta.LastIndexOf("#") > 0) { meta = logger.Meta.Substring(0, logger.Meta.LastIndexOf("#") + 1); } foreach (var file in directoryInfo.GetFiles()) { ReadData(logger, file.FullName, onlyNewData, calculateValue, readerProccess); meta += file.Name + ","; } logger.Meta = meta; logger.FileSize = FileUtils.FolderSize(logger.DataPath); } else { readerProccess.TotalRecord = FileUtils.CountTotalLines(logger.DataPath); readerProccess.CurrentFile = logger.DataPath; ReadData(logger, logger.DataPath, onlyNewData, calculateValue, readerProccess); logger.FileSize = fileInfo.Length; } // Update last time read logger logger.LastModifyDatetime = DateTime.Now; // Update total record logger.TotalSensor = logger.Sensors.Count(); Sensor sensor = null; if (logger.TotalSensor > 0) { sensor = logger.Sensors.First(); logger.TotalRecord = sensor.SensorValues.LongCount(); } else { logger.TotalRecord = 0; } // Update first record meatime if (logger.TotalRecord > 0) { logger.FirstLogDatetime = sensor.SensorValues.OrderBy(ent => ent.MeaTime).First().MeaTime; logger.LastLogDatetime = sensor.SensorValues.OrderByDescending(ent => ent.MeaTime).First().MeaTime; } entityConntext.SaveChanges(); Console.WriteLine("Thread[" + Thread.CurrentThread.ManagedThreadId + "]: End Read data form logger " + logger.LoggerID + ". Total record " + logger.TotalRecord); } }
/// <summary> /// Read data for logger /// </summary> /// <param name="logger"></param> /// <param name="onlyNewData"></param> /// <param name="calculateValue"></param> /// <param name="refreshEntityContext"></param> public void ReadData(int loggerID, bool onlyNewData, bool calculateValue, ReaderProccess readerProccess) { using (var entityConntext = new GeoViewerEntities()) { Logger logger = entityConntext.Loggers.SingleOrDefault(ent => ent.LoggerID == loggerID); if (logger == null) { return; } // For test only Console.WriteLine("Thread[" + Thread.CurrentThread.ManagedThreadId + "]: Read data form logger " + logger.LoggerID); // End for test var fileInfo = new FileInfo(logger.DataPath); if (!fileInfo.Exists) { readerProccess.TotalRecord = FileUtils.CountTotalFolderLines(logger.DataPath); readerProccess.FileReaderProccesses = new List <ReaderProccess>(); readerProccess.CurrentFile = logger.DataPath; var directoryInfo = new DirectoryInfo(logger.DataPath); if (!directoryInfo.Exists) { throw new Exception(Properties.Resources.Error_Data_FileNotFound); } string meta = ""; if (logger.Meta.LastIndexOf("#") > 0) { meta = logger.Meta.Substring(0, logger.Meta.LastIndexOf("#") + 1); } foreach (var file in directoryInfo.GetFiles()) { ReadData(logger, file.FullName, onlyNewData, calculateValue, readerProccess); meta += file.Name + ","; } logger.Meta = meta; logger.FileSize = FileUtils.FolderSize(logger.DataPath); } else { readerProccess.TotalRecord = FileUtils.CountTotalLines(logger.DataPath); readerProccess.CurrentFile = logger.DataPath; ReadData(logger, logger.DataPath, onlyNewData, calculateValue, readerProccess); logger.FileSize = fileInfo.Length; } // Update last time read logger logger.LastModifyDatetime = DateTime.Now; // Update total record logger.TotalSensor = logger.Sensors.Count(); Sensor sensor = null; if (logger.TotalSensor > 0) { sensor = logger.Sensors.First(); logger.TotalRecord = sensor.SensorValues.LongCount(); } else { logger.TotalRecord = 0; } // Update first record meatime if (logger.TotalRecord > 0) { logger.FirstLogDatetime = sensor.SensorValues.OrderBy(ent => ent.MeaTime).First().MeaTime; logger.LastLogDatetime = sensor.SensorValues.OrderByDescending(ent => ent.MeaTime).First().MeaTime; } entityConntext.SaveChanges(); Console.WriteLine("Thread[" + Thread.CurrentThread.ManagedThreadId + "]: End Read data form logger " + logger.LoggerID + ". Total record " + logger.TotalRecord); } }
/// <summary> /// Read data from file with option onlyNewData and calculateValue /// </summary> /// <param name="logger"></param> /// <param name="dataPath"></param> /// <param name="onlyNewData"></param> /// <param name="calculateValue"></param> /// <param name="refreshEntityContext"></param> public void ReadData(Logger logger, string dataPath, bool onlyNewData, bool calculateValue, ReaderProccess readerProccess) { try { Console.WriteLine("Thread[" + Thread.CurrentThread.ManagedThreadId + "]: Start read data file " + dataPath); ReaderProccess fileReaderProccess; if (readerProccess.FileReaderProccesses == null) { fileReaderProccess = readerProccess; } else { fileReaderProccess = new ReaderProccess(); fileReaderProccess.TotalRecord = FileUtils.CountTotalLines(dataPath); readerProccess.FileReaderProccesses.Add(fileReaderProccess); } fileReaderProccess.CurrentFile = dataPath; bool readFolder = !logger.DataPath.Equals(dataPath); var file = new FileInfo(dataPath); // Only read if file is modified if (onlyNewData) { if (readFolder) { if (logger.LastModifyDatetime >= file.LastWriteTime && logger.Meta.Contains(file.Name)) { return; } } else { if (logger.LastModifyDatetime >= file.LastWriteTime || logger.FileSize == file.Length) { return; } } } var streamReader = new StreamReader(dataPath, Encoding.UTF8); var spiliter = new[] { logger.Delimiter }; // read meta line string lineData = streamReader.ReadLine(); fileReaderProccess.ReadRecord++; if (String.IsNullOrEmpty(lineData)) { Console.WriteLine("File Empty!"); return; } // check type of data file int type = 0; string[] firstlineCells = lineData.Split(spiliter, StringSplitOptions.None); if (firstlineCells[0].ToInt32TryParse() > 0 && firstlineCells[1].ToInt32TryParse() > 0) { type = 1; } if (type == 0) { // Ignore 4 first lines for (int i = 0; i < 4; i++) { lineData = streamReader.ReadLine(); fileReaderProccess.ReadRecord++; } } // Begin to read data var entityConntext = new GeoViewerEntities(); var sensors = entityConntext.Sensors.Where(ent => ent.LoggerID == logger.LoggerID).OrderBy(ent => ent.ColumnIndex).ToList(); int line = 0; do { if (!string.IsNullOrEmpty(lineData.Trim())) { string[] cells = lineData.Split(spiliter, StringSplitOptions.None); var meaTime = type == 0 ? Convert.ToDateTime(CleanValue(cells[0])) : new DateTime(cells[1].ToInt32TryParse(), 1, 1).AddDays( cells[2].ToInt32TryParse()).AddMinutes( (int)(cells[3].ToInt32TryParse() * 60 / 100)); // Only read new record (by time) if (!(onlyNewData && !readFolder && meaTime >= logger.FirstLogDatetime && meaTime <= logger.LastLogDatetime)) { var valuesNeedCalc = new List <SensorValue>(); foreach (var sensor in sensors) { if (cells.Length > sensor.ColumnIndex) { // Check Value exist //var sensorValue = sensor.SensorValues.SingleOrDefault(ent => ent.MeaTime == meaTime); var sensorValue = entityConntext.SensorValues.SingleOrDefault(ent => ent.SensorID == sensor.SensorID && ent.MeaTime == meaTime); if (sensorValue == null) { sensorValue = new SensorValue(); sensorValue.SensorID = sensor.SensorID; sensorValue.MeaTime = meaTime; sensor.SensorValues.Add(sensorValue); } sensorValue.RawValue = CleanValue(cells[sensor.ColumnIndex]).ToDecimalTryParse(); valuesNeedCalc.Add(sensorValue); } } // Calc Value & Alarm Logger if (calculateValue) { foreach (var sensorValue in valuesNeedCalc) { CalculationBLL.Current.Calculating(sensorValue, true); } } } line++; if (line % 50 == 0) { entityConntext.SaveChanges(); entityConntext = new GeoViewerEntities(); sensors = entityConntext.Sensors.Where(ent => ent.LoggerID == logger.LoggerID).OrderBy(ent => ent.ColumnIndex).ToList(); Console.WriteLine("Thread[" + Thread.CurrentThread.ManagedThreadId + "]: Read success " + line + " lines in " + dataPath); } } // go to nextline lineData = streamReader.ReadLine(); fileReaderProccess.ReadRecord++; } while ((lineData != null)); streamReader.Close(); if (fileReaderProccess.ReadRecord > 0) { fileReaderProccess.ReadRecord--; } if (line > 0 && line % 50 != 0) { entityConntext.SaveChanges(); Console.WriteLine("Thread[" + Thread.CurrentThread.ManagedThreadId + "]: Read success " + line + " lines in " + dataPath); } Console.WriteLine("Thread[" + Thread.CurrentThread.ManagedThreadId + "]: End read data from " + dataPath); if (readerProccess.FileReaderProccesses != null) { readerProccess.ReadRecord += fileReaderProccess.ReadRecord; } } catch (Exception exception) { Console.WriteLine(exception.Message); } }