// Opens the archive file in order to retrieve the statistics. private ArchiveFile OpenArchiveFile() { ArchiveFile file = new ArchiveFile { FileName = ArchiveFilePath, FileAccessMode = FileAccess.Read, StateFile = new StateFile { FileAccessMode = FileAccess.Read, FileName = StateFilePath }, IntercomFile = new IntercomFile { FileAccessMode = FileAccess.Read, FileName = IntercomFilePath }, MetadataFile = new MetadataFile { FileAccessMode = FileAccess.Read, FileName = MetadataFilePath, LoadOnOpen = true } }; file.Open(); return file; }
private Info GetHistoricFileInfo(string fileName) { Info fileInfo = null; try { // Validate the file name to determine whether the given file is actually a historic file if (!Regex.IsMatch(fileName, string.Format(".+_.+_to_.+\\{0}$", FileExtension))) return null; if (File.Exists(fileName)) { // We'll open the file and get relevant information about it. ArchiveFile historicArchiveFile = new ArchiveFile(); historicArchiveFile.FileName = fileName; historicArchiveFile.StateFile = m_stateFile; historicArchiveFile.IntercomFile = m_intercomFile; historicArchiveFile.MetadataFile = m_metadataFile; historicArchiveFile.FileAccessMode = FileAccess.Read; try { historicArchiveFile.Open(); fileInfo = new Info(fileName) { StartTimeTag = historicArchiveFile.Fat.FileStartTime, EndTimeTag = historicArchiveFile.Fat.FileEndTime }; } catch (Exception ex) { OnHistoricFileListBuildException(new InvalidOperationException(string.Format("Failed to open historic data file \"{0}\" due to exception: {1}", FilePath.GetFileName(fileName), ex.Message), ex)); } finally { historicArchiveFile.Dispose(); } } else { // We'll resolve to getting the file information from its name only if the file no longer exists // at the location. This will be the case when file is moved to a different location. In this // case the file information we provide is only as good as the file name. string datesString = FilePath.GetFileNameWithoutExtension(fileName).Substring((FilePath.GetFileNameWithoutExtension(m_fileName) + "_").Length); string[] fileStartEndDates = datesString.Split(new string[] { "_to_" }, StringSplitOptions.None); fileInfo = new Info(fileName); if (fileStartEndDates.Length == 2) { fileInfo.StartTimeTag = new TimeTag(Convert.ToDateTime(fileStartEndDates[0].Replace('!', ':'))); fileInfo.EndTimeTag = new TimeTag(Convert.ToDateTime(fileStartEndDates[1].Replace('!', ':'))); } } } catch (Exception ex) { OnHistoricFileListBuildException(new InvalidOperationException(string.Format("Failed during information access attempt for historic data file \"{0}\" due to exception: {1}", FilePath.GetFileName(fileName), ex.Message), ex)); } return fileInfo; }
private void WriteToHistoricArchiveFile(IDataPoint[] items) { // Wait until the historic file list has been built. if (m_buildHistoricFileListThread.IsAlive) m_buildHistoricFileListThread.Join(); Dictionary<Info, Dictionary<int, List<IDataPoint>>> historicFileData = new Dictionary<Info, Dictionary<int, List<IDataPoint>>>(); // Separate all point data into bins by historic file and point ID foreach (IDataPoint dataPoint in items) { try { Info historicFileInfo; Dictionary<int, List<IDataPoint>> sortedPointData; List<IDataPoint> pointData; lock (m_historicArchiveFiles) { // Attempt to find a historic archive file where the data point belongs historicFileInfo = m_historicArchiveFiles.Find(info => FindHistoricArchiveFileForWrite(info, dataPoint.Time)); } // If a historic file exists, sort the data point into the proper bin if ((object)historicFileInfo != null) { sortedPointData = historicFileData.GetOrAdd(historicFileInfo, info => new Dictionary<int, List<IDataPoint>>()); pointData = sortedPointData.GetOrAdd(dataPoint.HistorianID, id => new List<IDataPoint>()); pointData.Add(dataPoint); } } catch (Exception ex) { // Notify of the exception OnDataWriteException(ex); } } foreach (Info historicFileInfo in historicFileData.Keys) { Dictionary<int, List<IDataPoint>> sortedPointData = historicFileData[historicFileInfo]; int overflowBlocks = 0; using (ArchiveFile historicFile = new ArchiveFile()) { try { // Open the historic file historicFile.FileName = historicFileInfo.FileName; historicFile.StateFile = m_stateFile; historicFile.IntercomFile = m_intercomFile; historicFile.MetadataFile = m_metadataFile; historicFile.Open(); } catch (Exception ex) { // Notify of the exception OnDataWriteException(ex); // If we fail to open the historic file, // then we cannot write any of these data // points to it so we might as well move on continue; } // Calculate the number of additional data blocks needed to store all the data foreach (int pointID in sortedPointData.Keys) { try { ArchiveDataBlock lastDataBlock = historicFile.Fat.FindLastDataBlock(pointID); int blockCapacity = historicFile.DataBlockSize * 1024 / ArchiveDataPoint.FixedLength; int overflowPoints = sortedPointData[pointID].Count + (lastDataBlock?.SlotsUsed ?? 0) - (lastDataBlock?.Capacity ?? 0); overflowBlocks += (overflowPoints + blockCapacity - 1) / blockCapacity; } catch (Exception ex) { // Notify of the exception OnDataWriteException(ex); } } try { // Extend the file by the needed amount if (overflowBlocks > 0) historicFile.Fat.Extend(overflowBlocks); } catch (Exception ex) { // Notify of the exception OnDataWriteException(ex); } foreach (int pointID in sortedPointData.Keys) { try { ArchiveDataBlock historicFileBlock = null; // Sort the point data for the current point ID by time sortedPointData[pointID].Sort(); foreach (IDataPoint dataPoint in sortedPointData[pointID]) { if ((object)historicFileBlock == null || historicFileBlock.SlotsAvailable == 0) { // Request a new or previously used data block for point data historicFileBlock = historicFile.Fat.RequestDataBlock(pointID, dataPoint.Time, -1); } // Write the data point into the data block historicFileBlock.Write(dataPoint); historicFile.Fat.DataPointsReceived++; historicFile.Fat.DataPointsArchived++; } } catch (Exception ex) { // Notify of the exception OnDataWriteException(ex); } } try { // Save the file after all // data has been written to it historicFile.Save(); } catch (Exception ex) { // Notify of the exception OnDataWriteException(ex); } } } }
private void PrepareForRollover() { try { DriveInfo archiveDrive = new DriveInfo(Path.GetPathRoot(m_fileName).ToNonNullString()); // We'll start offloading historic files if we've reached the offload threshold. if (m_archiveOffloadMaxAge > 0) OffloadMaxAgedFiles(); if (archiveDrive.AvailableFreeSpace < archiveDrive.TotalSize * (1 - (m_archiveOffloadThreshold / 100))) OffloadHistoricFiles(); // Maintain maximum number of historic files, if configured to do so MaintainMaximumNumberOfHistoricFiles(); OnRolloverPreparationStart(); // Opening and closing a new archive file in "standby" mode will create a "standby" archive file. ArchiveFile standbyArchiveFile = new ArchiveFile(); standbyArchiveFile.FileName = StandbyArchiveFileName; standbyArchiveFile.FileSize = m_fileSize; standbyArchiveFile.DataBlockSize = m_dataBlockSize; standbyArchiveFile.StateFile = m_stateFile; standbyArchiveFile.IntercomFile = m_intercomFile; standbyArchiveFile.MetadataFile = m_metadataFile; try { standbyArchiveFile.Open(); } catch { string standbyFileName = standbyArchiveFile.FileName; standbyArchiveFile.Close(); // We didn't succeed in creating a "standby" archive file, so we'll delete it if it was created // partially (might happen if there isn't enough disk space or thread is aborted). This is to // ensure that this preparation processes is kicked off again until a valid "standby" archive // file is successfully created. DeleteFile(standbyFileName); throw; // Rethrow the exception so the appropriate action is taken. } finally { standbyArchiveFile.Dispose(); } OnRolloverPreparationComplete(); } catch (ThreadAbortException) { // This thread must die now... } catch (Exception ex) { OnRolloverPreparationException(ex); } }
// Read data implementation private IEnumerable<IDataPoint> ReadData(IEnumerable<int> historianIDs, TimeTag startTime, TimeTag endTime, IDataPoint resumeFrom, bool timeSorted) { // Yield to archive rollover process. m_rolloverWaitHandle.WaitOne(); // Ensure that the current file is open. if (!IsOpen) throw new InvalidOperationException(string.Format("\"{0}\" file is not open", m_fileName)); // Ensure that the current file is active. if (m_fileType != ArchiveFileType.Active) throw new InvalidOperationException("Data can only be directly read from files that are Active"); // Ensure that the start and end time are valid. if (startTime.CompareTo(endTime) > 0) throw new ArgumentException("End Time precedes Start Time in the specified time span"); List<Info> dataFiles = new List<Info>(); bool pendingRollover = false; bool usingActiveFile = false; if (startTime.CompareTo(m_fat.FileStartTime) < 0) { // Data is to be read from historic file(s) - make sure that the list has been built if (m_buildHistoricFileListThread.IsAlive) m_buildHistoricFileListThread.Join(); lock (m_historicArchiveFiles) { dataFiles.AddRange(m_historicArchiveFiles.FindAll(info => FindHistoricArchiveFileForRead(info, startTime, endTime))); } } if (endTime.CompareTo(m_fat.FileStartTime) >= 0) { // Data is to be read from the active file. Info activeFileInfo = new Info(m_fileName) { StartTimeTag = m_fat.FileStartTime, EndTimeTag = m_fat.FileEndTime }; dataFiles.Add(activeFileInfo); } // Read data from all qualifying files. foreach (Info dataFile in dataFiles) { ArchiveFile file = null; try { if (string.Compare(dataFile.FileName, m_fileName, StringComparison.OrdinalIgnoreCase) == 0) { // Read data from current file. usingActiveFile = true; file = this; // Atomically increment total number of readers for active file Interlocked.Increment(ref m_activeFileReaders); // Handle race conditions between rollover // and incrementing the active readers while (m_rolloverInProgress) { Interlocked.Decrement(ref m_activeFileReaders); m_rolloverWaitHandle.WaitOne(); Interlocked.Increment(ref m_activeFileReaders); } } else { // Read data from historic file. usingActiveFile = false; file = new ArchiveFile(); file.FileName = dataFile.FileName; file.StateFile = m_stateFile; file.IntercomFile = m_intercomFile; file.MetadataFile = m_metadataFile; file.FileAccessMode = FileAccess.Read; file.Open(); } // Create new data point scanner for the desired points in this file and given time range IArchiveFileScanner scanner; if (timeSorted) scanner = new TimeSortedArchiveFileScanner(); else scanner = new ArchiveFileScanner(); scanner.FileAllocationTable = file.Fat; scanner.HistorianIDs = historianIDs; scanner.StartTime = startTime; scanner.EndTime = endTime; scanner.ResumeFrom = resumeFrom; scanner.DataReadExceptionHandler = (sender, e) => OnDataReadException(e.Argument); // Reset resumeFrom to scan from beginning after picking up where left off from roll over resumeFrom = null; // Return data points foreach (IDataPoint dataPoint in scanner.Read()) { yield return dataPoint; // If a rollover needs to happen, we need to relinquish read lock and close file if (m_rolloverInProgress) { resumeFrom = dataPoint; pendingRollover = true; break; } } } finally { if (usingActiveFile) { // Atomically decrement active file reader count to signal in-process code that read is complete or yielded Interlocked.Decrement(ref m_activeFileReaders); } else if ((object)file != null) { file.Dispose(); } } if (pendingRollover) break; } if (pendingRollover) { // Recurse into this function with an updated start time and last read point ID so that read can // resume right where it left off - recursed function call will wait until rollover is complete foreach (IDataPoint dataPoint in ReadData(historianIDs, startTime, endTime, resumeFrom, timeSorted)) { yield return dataPoint; } } }
private static ArchiveFile OpenArchiveFile(string sourceFileName) { const string MetadataFileName = "{0}{1}_dbase.dat"; const string StateFileName = "{0}{1}_startup.dat"; const string IntercomFileName = "{0}scratch.dat"; string location = FilePath.GetDirectoryName(sourceFileName); string fileName = FilePath.GetFileName(sourceFileName); string instance = fileName.Substring(0, fileName.LastIndexOf("_archive", StringComparison.OrdinalIgnoreCase)); ArchiveFile file = new ArchiveFile { FileName = sourceFileName, FileAccessMode = FileAccess.ReadWrite, MonitorNewArchiveFiles = false, PersistSettings = false, LeadTimeTolerance = 10.0D * 365.0D * 24.0D * 60.0D, CompressData = false, ConserveMemory = false, StateFile = new StateFile { FileAccessMode = FileAccess.ReadWrite, FileName = string.Format(StateFileName, location, instance) }, IntercomFile = new IntercomFile { FileAccessMode = FileAccess.ReadWrite, FileName = string.Format(IntercomFileName, location) }, MetadataFile = new MetadataFile { FileAccessMode = FileAccess.ReadWrite, FileName = string.Format(MetadataFileName, location, instance), LegacyMode = MetadataFileLegacyMode.Disabled, LoadOnOpen = true } }; MetadataFile metadata = file.MetadataFile; file.MetadataFile.Open(); if (file.MetadataFile.RecordsOnDisk == 0) { Console.WriteLine("Building Metadata file..."); for (int i = 1; i <= MetaDataPoints; i++) { metadata.Write(i, new MetadataRecord(i, MetadataFileLegacyMode.Disabled) { Name = "point" + i, GeneralFlags = new MetadataRecordGeneralFlags { DataType = DataType.Analog, Enabled = true } }); } metadata.Save(); } file.Open(); file.SynchronizeStateFile(); return file; }
// Opens an archive file as read-only and returns the ArchiveFile object. private ArchiveFile OpenArchiveFile(string fileName, string offloadLocation) { const string metadataFileName = "{0}{1}_dbase.dat"; const string stateFileName = "{0}{1}_startup.dat"; const string intercomFileName = "{0}scratch.dat"; string archiveLocation = FilePath.GetDirectoryName(fileName); string archiveName = FilePath.GetFileName(fileName); string instance = archiveName.Substring(0, archiveName.LastIndexOf("_archive", StringComparison.OrdinalIgnoreCase)); ArchiveFile file = new ArchiveFile { FileName = fileName, FileAccessMode = FileAccess.Read, MonitorNewArchiveFiles = true, PersistSettings = false, ArchiveOffloadLocation = offloadLocation, StateFile = new StateFile { FileAccessMode = FileAccess.Read, FileName = string.Format(stateFileName, archiveLocation, instance) }, IntercomFile = new IntercomFile { FileAccessMode = FileAccess.Read, FileName = string.Format(intercomFileName, archiveLocation) }, MetadataFile = new MetadataFile { FileAccessMode = FileAccess.Read, FileName = string.Format(metadataFileName, archiveLocation, instance), LoadOnOpen = true } }; file.DataReadException += file_DataReadException; file.HistoricFileListBuildComplete += file_HistoricFileListBuildComplete; file.HistoricFileListBuildException += file_HistoricFileListBuildException; file.HistoricFileListBuildStart += file_HistoricFileListBuildStart; file.HistoricFileListUpdated += file_HistoricFileListUpdated; file.RolloverComplete += file_RolloverComplete; file.RolloverStart += file_RolloverStart; // Initialize the archive file (starts file watchers) file.Initialize(); // Open the active archive file file.Open(); // Start the roll-over watch timer m_rolloverWatcher.Start(); return file; }
private void WriteToHistoricArchiveFile(IDataPoint[] items) { // Wait until the historic file list has been built. if (m_buildHistoricFileListThread.IsAlive) m_buildHistoricFileListThread.Join(); Dictionary<int, List<IDataPoint>> sortedPointData = new Dictionary<int, List<IDataPoint>>(); // First we'll separate all point data by ID. for (int i = 0; i < items.Length; i++) { if (!sortedPointData.ContainsKey(items[i].HistorianID)) { sortedPointData.Add(items[i].HistorianID, new List<IDataPoint>()); } sortedPointData[items[i].HistorianID].Add(items[i]); } foreach (int pointID in sortedPointData.Keys) { // We'll sort the point data for the current point ID by time. sortedPointData[pointID].Sort(); ArchiveFile historicFile = null; ArchiveDataBlock historicFileBlock = null; try { for (int i = 0; i < sortedPointData[pointID].Count; i++) { if ((object)historicFile == null) { // We'll try to find a historic file when the current point data belongs. Info historicFileInfo; lock (m_historicArchiveFiles) { historicFileInfo = m_historicArchiveFiles.Find(info => FindHistoricArchiveFileForWrite(info, sortedPointData[pointID][i].Time)); } if ((object)historicFileInfo != null) { // Found a historic file where the data can be written. historicFile = new ArchiveFile(); historicFile.FileName = historicFileInfo.FileName; historicFile.StateFile = m_stateFile; historicFile.IntercomFile = m_intercomFile; historicFile.MetadataFile = m_metadataFile; historicFile.Open(); } } if ((object)historicFile != null) { if (sortedPointData[pointID][i].Time.CompareTo(historicFile.Fat.FileStartTime) >= 0 && sortedPointData[pointID][i].Time.CompareTo(historicFile.Fat.FileEndTime) <= 0) { // The current point data belongs to the current historic archive file. if ((object)historicFileBlock == null || historicFileBlock.SlotsAvailable == 0) { // Request a new or previously used data block for point data. historicFileBlock = historicFile.Fat.RequestDataBlock(pointID, sortedPointData[pointID][i].Time, -1); } historicFileBlock.Write(sortedPointData[pointID][i]); historicFile.Fat.DataPointsReceived++; historicFile.Fat.DataPointsArchived++; if (i == sortedPointData[pointID].Count() - 1) { // Last piece of data for the point, so we close the currently open file. historicFile.Save(); historicFile.Dispose(); historicFile = null; historicFileBlock = null; } } else { // The current point data doesn't belong to the current historic archive file, so we have // to write all the point data we have so far for the current historic archive file to it. i--; historicFile.Dispose(); historicFile = null; historicFileBlock = null; } } } } catch (Exception ex) { // Free-up used memory. if ((object)historicFile != null) { try { historicFile.Dispose(); } catch { } } // Notify of the exception. OnDataWriteException(ex); } } }