private static IEnumerable <FileEntry> ExtractFile(FileEntry fileEntry, ArchiveFileType archiveFileType) { switch (archiveFileType) { case ArchiveFileType.ZIP: return(ExtractZipFile(fileEntry)); case ArchiveFileType.GZIP: return(ExtractGZipFile(fileEntry)); case ArchiveFileType.TAR: return(ExtractTarFile(fileEntry)); case ArchiveFileType.XZ: return(ExtractXZFile(fileEntry)); case ArchiveFileType.BZIP2: return(ExtractBZip2File(fileEntry)); case ArchiveFileType.RAR: return(ExtractRarFile(fileEntry)); case ArchiveFileType.P7ZIP: return(Extract7ZipFile(fileEntry)); case ArchiveFileType.UNKNOWN: default: return(new[] { fileEntry }); } }
public ArchiveFile(IDataSource Source, string Name, ArchiveFileCompression Compression) { if (Name.EndsWith(".wav")) { //this.Source = new CompressedDataSource(Source, ArchiveFileType.Audio); this.Source = new Xgg.XggWrappedSource(Source); this.Name = Name.Replace(".wav", ".xgg"); } else { this.Source = Source; this.Name = Name; } if (this.Name.EndsWith(".xgg")) { Type = ArchiveFileType.Audio; } else if (this.Name.EndsWith(".png")) { Type = ArchiveFileType.Image; } else { this.Compression = Compression; } }
/// <summary> /// Remove any extractor (if set) for the given ArchiveFileType. /// </summary> /// <param name="targetType">The ArchiveFileType to remove the extractor for.</param> public void Unset(ArchiveFileType targetType) { if (Extractors.ContainsKey(targetType)) { Extractors.Remove(targetType); } }
/// <summary> /// Main entry point to start analysis; handles setting up rules, directory enumeration /// file type detection and handoff /// Pre: All Configure Methods have been called already and we are ready to SCAN /// </summary> /// <returns></returns> public int Run() { WriteOnce.SafeLog("AnalyzeCommand::Run", LogLevel.Trace); DateTime start = DateTime.Now; WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_RUNNING, "Analyze")); _appProfile.MetaData.TotalFiles = _srcfileList.Count();//updated for zipped files later // Iterate through all files and process against rules foreach (string filename in _srcfileList) { ArchiveFileType archiveFileType = MiniMagic.DetectFileType(filename); if (archiveFileType == ArchiveFileType.UNKNOWN)//not a known zipped file type { ProcessAsFile(filename); } else { UnZipAndProcess(filename, archiveFileType); } } WriteOnce.General("\r" + ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILES_PROCESSED_PCNT, 100)); WriteOnce.Operation(ErrMsg.GetString(ErrMsg.ID.CMD_PREPARING_REPORT)); //Prepare report results _appProfile.MetaData.LastUpdated = LastUpdated.ToString(); _appProfile.DateScanned = DateScanned.ToString(); _appProfile.PrepareReport(); TimeSpan timeSpan = start - DateTime.Now; WriteOnce.SafeLog(String.Format("Processing time: seconds:{0}", timeSpan.TotalSeconds * -1), LogLevel.Trace); FlushAll(); //wrapup result status if (_appProfile.MetaData.TotalFiles == _appProfile.MetaData.FilesSkipped) { WriteOnce.Error(ErrMsg.GetString(ErrMsg.ID.ANALYZE_NOSUPPORTED_FILETYPES)); } else if (_appProfile.MatchList.Count == 0) { WriteOnce.Error(ErrMsg.GetString(ErrMsg.ID.ANALYZE_NOPATTERNS)); } else { WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_COMPLETED, "Analyze")); if (!_arg_autoBrowserOpen) { WriteOnce.Any(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_OUTPUT_FILE, "output.html")); } } return(_appProfile.MatchList.Count() == 0 ? (int)ExitCode.NoMatches : (int)ExitCode.MatchesFound); }
public static IEnumerable <FileEntry> ExtractFile(string filename, ArchiveFileType archiveFileType) { _count = 0; //_files = new List<FileEntry>(); using var memoryStream = new MemoryStream(File.ReadAllBytes(filename)); //using var fileEntry = new FileEntry(filename, "", memoryStream); return(ExtractFile(new FileEntry(filename, "", memoryStream), archiveFileType)); //return _files; }
void UnZipAndProcess(string filename, ArchiveFileType archiveFileType) { //zip itself may be too huge for timely processing if (new FileInfo(filename).Length > WARN_ZIP_FILE_SIZE) { WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_FILESIZE_WARN)); } else { WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_PROCESSING)); } try { IEnumerable <FileEntry> files = Extractor.ExtractFile(filename); if (files.Count() > 0) { _appProfile.MetaData.TotalFiles += files.Count();//additive in case additional child zip files processed _appProfile.MetaData.PackageTypes.Add(ErrMsg.GetString(ErrMsg.ID.ANALYZE_COMPRESSED_FILETYPE)); foreach (FileEntry file in files) { if (file.Content.Length > MAX_FILESIZE) { WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILESIZE_SKIPPED, file.FullPath), LogLevel.Warn); _appProfile.MetaData.FilesSkipped++; continue; } //dup check vs Run() for zip contents; exclude sample, test or similar files by default or as specified in exclusion list if (!_arg_allowSampleFiles && _fileExclusionList.Any(v => file.FullPath.ToLower().Contains(v))) { WriteOnce.SafeLog("Part of excluded list: " + file.FullPath, LogLevel.Trace); WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILESIZE_SKIPPED, file.FullPath), LogLevel.Trace); _appProfile.MetaData.FilesSkipped++; continue; } WriteOnce.Log.Trace("processing zip file entry: " + file.FullPath); byte[] streamByteArray = file.Content.ToArray(); ProcessInMemory(file.FullPath, Encoding.UTF8.GetString(streamByteArray, 0, streamByteArray.Length), true); } } else { throw new OpException(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_ERROR, filename)); } } catch (Exception e) { string errmsg = ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_ERROR, filename); WriteOnce.Error(errmsg); throw new Exception(errmsg + e.Message + "\n" + e.StackTrace); } }
void UnZipAndProcess(string filename, ArchiveFileType archiveFileType) { // zip itself may be in excluded list i.e. sample, test or similar unless ignore filter requested if (_fileExclusionList != null && _fileExclusionList.Any(v => filename.ToLower().Contains(v))) { WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_EXCLUDED_TYPE_SKIPPED, filename), LogLevel.Warn); _appProfile.MetaData.FilesSkipped++; return; } //zip itself may be too huge for timely processing if (new FileInfo(filename).Length > WARN_ZIP_FILE_SIZE) { WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_FILESIZE_WARN)); } else { WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_PROCESSING)); } LastUpdated = File.GetLastWriteTime(filename); _appProfile.MetaData.PackageTypes.Add(ErrMsg.GetString(ErrMsg.ID.ANALYZE_COMPRESSED_FILETYPE)); try { IEnumerable <FileEntry> files = Extractor.ExtractFile(filename); if (files.Count() > 0) { _appProfile.MetaData.TotalFiles += files.Count();//additive in case additional child zip files processed foreach (FileEntry file in files) { //check uncompressed file passes standard checks LanguageInfo languageInfo = new LanguageInfo(); if (FileChecksPassed(file.FullPath, ref languageInfo, file.Content.Length)) { byte[] streamByteArray = file.Content.ToArray(); ProcessInMemory(file.FullPath, Encoding.UTF8.GetString(streamByteArray, 0, streamByteArray.Length), languageInfo); } } } else { WriteOnce.SafeLog(string.Format("Decompression found no files in {0}", filename), LogLevel.Warn);//zero results can be valid } } catch (Exception) { string errmsg = ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_ERROR, filename); WriteOnce.Error(errmsg); throw; } }
public Subfile(IDataSource source, string name, string archiveName, ArchiveFileType type, RequestedConversion requestedConversion) { if (!archiveName.EndsWith(".pp")) { archiveName += ".pp"; } ArchiveName = archiveName; Name = name; Source = source; Size = source.Size; Type = type; RequestedConversion = requestedConversion; EmulatedName = name; }
public CompressedDataSource(IDataSource basesource, ArchiveFileType type) { this.basesource = basesource; this.type = type; _md5 = basesource.Md5; _size = basesource.Size; /* * using (Stream stream = GetStream()) * { * _md5 = Utility.GetMd5(stream); * * _size = (uint)stream.Length; * }*/ }
void UnZipAndProcess(string filename, ArchiveFileType archiveFileType) { //zip itself may be too huge for timely processing if (new FileInfo(filename).Length > WARN_ZIP_FILE_SIZE) { WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_FILESIZE_WARN)); } else { WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_PROCESSING)); } LastUpdated = File.GetLastWriteTime(filename); _appProfile.MetaData.PackageTypes.Add(ErrMsg.GetString(ErrMsg.ID.ANALYZE_COMPRESSED_FILETYPE)); try { IEnumerable <FileEntry> files = Extractor.ExtractFile(filename); if (files.Count() > 0) { _appProfile.MetaData.TotalFiles += files.Count();//additive in case additional child zip files processed foreach (FileEntry file in files) { //check for supported language LanguageInfo languageInfo = new LanguageInfo(); if (FileChecksPassed(file.FullPath, ref languageInfo, file.Content.Length)) { byte[] streamByteArray = file.Content.ToArray(); ProcessInMemory(file.FullPath, Encoding.UTF8.GetString(streamByteArray, 0, streamByteArray.Length), languageInfo); } } } else { throw new OpException(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_ERROR, filename)); } } catch (Exception e) { string errmsg = ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_ERROR, filename); WriteOnce.Error(errmsg); throw new Exception(errmsg + e.Message + "\n" + e.StackTrace); } }
public void TestMiniMagic(string fileName, ArchiveFileType expectedArchiveFileType) { var path = Path.Combine(Directory.GetCurrentDirectory(), "TestData", fileName); using var fs = new FileStream(path, FileMode.Open); // Test just based on the content var fileEntry = new FileEntry("NoName", fs); // We make sure the expected type matches and we have reset the stream Assert.AreEqual(expectedArchiveFileType, MiniMagic.DetectFileType(fileEntry)); Assert.AreEqual(0, fileEntry.Content.Position); // Should also work if the stream doesn't start at 0 fileEntry.Content.Position = 10; Assert.AreEqual(expectedArchiveFileType, MiniMagic.DetectFileType(fileEntry)); Assert.AreEqual(10, fileEntry.Content.Position); }
public static IEnumerable <FileEntry> ExtractFile(string filename, ArchiveFileType archiveFileType) { try { if (!File.OpenRead(filename).CanRead) { throw new IOException($"ExtractFile called, but {filename} cannot be read."); } } catch (Exception) { //Logger.Trace("File {0} cannot be read, ignoring.", filename); return(Array.Empty <FileEntry>()); } using var memoryStream = new MemoryStream(File.ReadAllBytes(filename)); return(ExtractFile(new FileEntry(filename, "", memoryStream), archiveFileType)); }
private static IEnumerable <FileEntry> ExtractFile(FileEntry fileEntry, ArchiveFileType archiveFileType) { IEnumerable <FileEntry> results = new List <FileEntry>(); switch (archiveFileType) { case ArchiveFileType.ZIP: results = ExtractZipFile(fileEntry); break; case ArchiveFileType.GZIP: results = ExtractGZipFile(fileEntry); break; case ArchiveFileType.TAR: results = ExtractTarFile(fileEntry); break; case ArchiveFileType.XZ: results = ExtractXZFile(fileEntry); break; case ArchiveFileType.BZIP2: results = ExtractBZip2File(fileEntry); break; case ArchiveFileType.RAR: results = ExtractRarFile(fileEntry); break; case ArchiveFileType.P7ZIP: results = Extract7ZipFile(fileEntry); break; case ArchiveFileType.UNKNOWN: default: results = new[] { fileEntry }; break; } return(results); }
internal ArchiveFileSource(BinaryReader reader) { Type = (ArchiveFileType)reader.ReadByte(); Flags = (ArchiveFileFlags)reader.ReadByte(); Compression = (ArchiveFileCompression)reader.ReadByte(); reader.BaseStream.Seek(1, SeekOrigin.Current); _crc = reader.ReadUInt32(); _md5 = reader.ReadBytes(16); reader.BaseStream.Seek(48, SeekOrigin.Current); ushort len = reader.ReadUInt16(); string[] names = Encoding.Unicode.GetString(reader.ReadBytes(len)).Split('/'); ArchiveName = names[0]; Name = names[1]; offset = reader.ReadUInt64(); _size = reader.ReadUInt32(); length = reader.ReadUInt32(); ArchiveFilename = (reader.BaseStream as FileStream).Name; }
public static bool IsSupportedFormat(string filename) { ArchiveFileType archiveFileType = MiniMagic.DetectFileType(filename); return(archiveFileType != ArchiveFileType.UNKNOWN); }
/// <summary> /// Initializes a new instance of the <see cref="ArchiveFile"/> class. /// </summary> public ArchiveFile() : base() { m_fileName = DefaultFileName; m_fileType = DefaultFileType; m_fileSize = DefaultFileSize; m_fileAccessMode = DefaultFileAccessMode; m_dataBlockSize = DefaultDataBlockSize; m_rolloverOnFull = DefaultRolloverOnFull; m_rolloverPreparationThreshold = DefaultRolloverPreparationThreshold; m_fileOffloadLocation = DefaultFileOffloadLocation; m_fileOffloadCount = DefaultFileOffloadCount; m_fileOffloadThreshold = DefaultFileOffloadThreshold; m_leadTimeTolerance = DefaultLeadTimeTolerance; m_compressData = DefaultCompressData; m_discardOutOfSequenceData = DefaultDiscardOutOfSequenceData; m_cacheWrites = DefaultCacheWrites; m_conserveMemory = DefaultConserveMemory; m_persistSettings = DefaultPersistSettings; m_settingsCategory = DefaultSettingsCategory; m_delayedAlarmProcessing = new Dictionary<int, double>(); m_rolloverWaitHandle = new ManualResetEvent(true); m_rolloverPreparationThread = new Thread(PrepareForRollover); m_buildHistoricFileListThread = new Thread(BuildHistoricFileList); m_conserveMemoryTimer = new System.Timers.Timer(DataBlockCheckInterval); m_conserveMemoryTimer.Elapsed += ConserveMemoryTimer_Elapsed; m_historicDataQueue = ProcessQueue<ArchiveData>.CreateRealTimeQueue(WriteToHistoricArchiveFile); m_historicDataQueue.ProcessException += HistoricDataQueue_ProcessException; m_outOfSequenceDataQueue = ProcessQueue<ArchiveData>.CreateRealTimeQueue(InsertInCurrentArchiveFile); m_outOfSequenceDataQueue.ProcessException += OutOfSequenceDataQueue_ProcessException; m_currentLocationFileWatcher = new FileSystemWatcher(); m_currentLocationFileWatcher.IncludeSubdirectories = true; m_currentLocationFileWatcher.Renamed += FileWatcher_Renamed; m_currentLocationFileWatcher.Deleted += FileWatcher_Deleted; m_currentLocationFileWatcher.Created += FileWatcher_Created; m_offloadLocationFileWatcher = new FileSystemWatcher(); m_offloadLocationFileWatcher.IncludeSubdirectories = true; m_offloadLocationFileWatcher.Renamed += FileWatcher_Renamed; m_offloadLocationFileWatcher.Deleted += FileWatcher_Deleted; m_offloadLocationFileWatcher.Created += FileWatcher_Created; }
/// <summary> /// Opens the <see cref="ArchiveFile"/> for use. /// </summary> /// <exception cref="InvalidOperationException">One or all of the <see cref="StateFile"/>, <see cref="IntercomFile"/> or <see cref="MetadataFile"/> properties are not set.</exception> public void Open() { if (!IsOpen) { // Check for the existance of dependencies. if (m_stateFile == null || m_intercomFile == null | m_metadataFile == null) throw (new InvalidOperationException("One or more of the dependency files are not specified.")); // Validate file type against its name. m_fileName = m_fileName.ToLower(); if (Path.GetExtension(m_fileName) == StandbyFileExtension) m_fileType = ArchiveFileType.Standby; else if (Regex.IsMatch(m_fileName, string.Format(".+_.+_to_.+\\{0}$", FileExtension))) m_fileType = ArchiveFileType.Historic; else m_fileType = ArchiveFileType.Active; // Get the absolute path for the file name. m_fileName = FilePath.GetAbsolutePath(m_fileName); // Create the directory if it does not exist. if (!Directory.Exists(FilePath.GetDirectoryName(m_fileName))) Directory.CreateDirectory(FilePath.GetDirectoryName(m_fileName)); if (File.Exists(m_fileName)) { // File has been created already, so we just need to read it. m_fileStream = new FileStream(m_fileName, FileMode.Open, m_fileAccessMode, FileShare.ReadWrite); m_fat = new ArchiveFileAllocationTable(this); } else { // File does not exist, so we have to create it and initialize it. m_fileStream = new FileStream(m_fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite); m_fat = new ArchiveFileAllocationTable(this); m_fat.Save(); } // Don't proceed further for standby and historic files. if (m_fileType != ArchiveFileType.Active) return; // Start internal process queues. m_historicDataQueue.Start(); m_outOfSequenceDataQueue.Start(); // Open state file if closed. if (!m_stateFile.IsOpen) m_stateFile.Open(); //Open intercom file if closed. if (!m_intercomFile.IsOpen) m_intercomFile.Open(); // Open metadata file if closed. if (!m_metadataFile.IsOpen) m_metadataFile.Open(); // Create data block lookup list. if (m_stateFile.RecordsInMemory > 0) m_dataBlocks = new List<ArchiveDataBlock>(new ArchiveDataBlock[m_stateFile.RecordsInMemory]); else m_dataBlocks = new List<ArchiveDataBlock>(new ArchiveDataBlock[m_stateFile.RecordsOnDisk]); // Start the memory conservation process. if (m_conserveMemory) m_conserveMemoryTimer.Start(); // Ensure that "rollover in progress" is not set. IntercomRecord system = m_intercomFile.Read(1); system.RolloverInProgress = false; m_intercomFile.Write(1, system); // Start preparing the list of historic files. m_buildHistoricFileListThread = new Thread(BuildHistoricFileList); m_buildHistoricFileListThread.Priority = ThreadPriority.Lowest; m_buildHistoricFileListThread.Start(); // Start file watchers to monitor file system changes. m_currentLocationFileWatcher.Filter = HistoricFilesSearchPattern; m_currentLocationFileWatcher.Path = FilePath.GetDirectoryName(m_fileName); m_currentLocationFileWatcher.EnableRaisingEvents = true; if (Directory.Exists(m_fileOffloadLocation)) { m_offloadLocationFileWatcher.Filter = HistoricFilesSearchPattern; m_offloadLocationFileWatcher.Path = m_fileOffloadLocation; m_offloadLocationFileWatcher.EnableRaisingEvents = true; } } }
/// <summary> /// Creates a new data source from a processed byte array. /// </summary> /// <param name="data">The byte array to use.</param> /// <param name="compression">The compression that the byte array has used.</param> /// <param name="encoding">The encoding of the data.</param> public MemorySource(ReadOnlyMemory <byte> data, ArchiveFileType encoding) { Data = data; Encoding = encoding; }
/// <summary> /// Main entry point to start analysis from CLI; handles setting up rules, directory enumeration /// file type detection and handoff /// Pre: All Configure Methods have been called already and we are ready to SCAN /// </summary> /// <returns></returns> public override int Run() { WriteOnce.SafeLog("AnalyzeCommand::Run", LogLevel.Trace); WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_RUNNING, "analyze")); try { _appProfile.MetaData.TotalFiles = _srcfileList.Count();//updated for zipped files later // Iterate through all files and process against rules foreach (string filename in _srcfileList) { ArchiveFileType archiveFileType = ArchiveFileType.UNKNOWN; try //fix for #146 { archiveFileType = MiniMagic.DetectFileType(filename); } catch (Exception e) { WriteOnce.SafeLog(e.Message + "\n" + e.StackTrace, LogLevel.Error); //log details Exception f = new Exception(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILE_TYPE_OPEN, filename)); //report friendly version throw f; } if (archiveFileType == ArchiveFileType.UNKNOWN)//not a known zipped file type { ProcessAsFile(filename); } else { UnZipAndProcess(filename, archiveFileType); } } WriteOnce.General("\r" + ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILES_PROCESSED_PCNT, 100)); WriteOnce.Operation(ErrMsg.GetString(ErrMsg.ID.CMD_PREPARING_REPORT)); //Prepare report results _appProfile.MetaData.LastUpdated = LastUpdated.ToString(); _appProfile.DateScanned = DateScanned.ToString(); _appProfile.PrepareReport(); FlushAll(); //wrapup result status if (_appProfile.MetaData.TotalFiles == _appProfile.MetaData.FilesSkipped) { WriteOnce.Error(ErrMsg.GetString(ErrMsg.ID.ANALYZE_NOSUPPORTED_FILETYPES)); } else if (_appProfile.MatchList.Count == 0) { WriteOnce.Error(ErrMsg.GetString(ErrMsg.ID.ANALYZE_NOPATTERNS)); } else { WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_COMPLETED, "analyze")); } } catch (Exception e) { WriteOnce.Error(e.Message); //exit normaly for CLI callers and throw for DLL callers if (Utils.CLIExecutionContext) { return((int)ExitCode.CriticalError); } else { throw; } } finally { if (_arg_close_log_on_exit) { Utils.Logger = null; WriteOnce.Log = null; } } return(_appProfile.MatchList.Count() == 0 ? (int)ExitCode.NoMatches : (int)Utils.ExitCode.Success); }
/// <summary> /// Opens the <see cref="ArchiveFile"/> for use. /// </summary> /// <exception cref="InvalidOperationException">One or all of the <see cref="StateFile"/>, <see cref="IntercomFile"/> or <see cref="MetadataFile"/> properties are not set.</exception> public void Open() { if (!IsOpen) { // Check for the existence of dependencies. if ((object)m_stateFile == null || (object)m_intercomFile == null || (object)m_metadataFile == null) throw (new InvalidOperationException("One or more of the dependency files are not specified.")); // Validate file type against its name. if (Path.GetExtension(m_fileName).ToNonNullString().ToLower() == StandbyFileExtension) m_fileType = ArchiveFileType.Standby; else if (Regex.IsMatch(m_fileName.ToLower(), string.Format(".+_.+_to_.+\\{0}$", FileExtension))) m_fileType = ArchiveFileType.Historic; else m_fileType = ArchiveFileType.Active; // Get the absolute path for the file name. m_fileName = FilePath.GetAbsolutePath(m_fileName); // Create the directory if it does not exist. if (!Directory.Exists(FilePath.GetDirectoryName(m_fileName))) Directory.CreateDirectory(FilePath.GetDirectoryName(m_fileName)); // Validate a roll-over is not in progress when opening archive as read-only if (m_fileType == ArchiveFileType.Active && m_fileAccessMode == FileAccess.Read) { // Open intercom file if closed. if (!m_intercomFile.IsOpen) m_intercomFile.Open(); m_intercomFile.Load(); IntercomRecord record = m_intercomFile.Read(1); int waitCount = 0; while ((object)record != null && record.RolloverInProgress && waitCount < 30) { Thread.Sleep(1000); m_intercomFile.Load(); record = m_intercomFile.Read(1); waitCount++; } } OpenStream(); // Open state file if closed. if (!m_stateFile.IsOpen) m_stateFile.Open(); // Open intercom file if closed. if (!m_intercomFile.IsOpen) m_intercomFile.Open(); // Open metadata file if closed. if (!m_metadataFile.IsOpen) m_metadataFile.Open(); // Don't proceed further for standby and historic files. if (m_fileType != ArchiveFileType.Active) return; // Start internal process queues. m_currentDataQueue.Start(); m_historicDataQueue.Start(); m_outOfSequenceDataQueue.Start(); // Create data block lookup list. if (m_stateFile.RecordsInMemory > 0) m_dataBlocks = new List<ArchiveDataBlock>(new ArchiveDataBlock[m_stateFile.RecordsInMemory]); else m_dataBlocks = new List<ArchiveDataBlock>(new ArchiveDataBlock[m_stateFile.RecordsOnDisk]); // Validate the dependency files. SyncStateFile(null); if (m_intercomFile.FileAccessMode != FileAccess.Read) { // Ensure that "rollover in progress" is not set. IntercomRecord system = m_intercomFile.Read(1); if ((object)system == null) system = new IntercomRecord(1); system.RolloverInProgress = false; m_intercomFile.Write(1, system); } // Start the memory conservation process. if (m_conserveMemory) { m_conserveMemoryTimer.Interval = DataBlockCheckInterval; m_conserveMemoryTimer.Start(); } if (m_fileType == ArchiveFileType.Active) { // Start preparing the list of historic files. m_buildHistoricFileListThread = new Thread(BuildHistoricFileList); m_buildHistoricFileListThread.Priority = ThreadPriority.Lowest; m_buildHistoricFileListThread.Start(); // Start file watchers to monitor file system changes. if (m_monitorNewArchiveFiles) { if ((object)m_currentLocationFileWatcher != null) { m_currentLocationFileWatcher.Filter = HistoricFilesSearchPattern; m_currentLocationFileWatcher.Path = FilePath.GetDirectoryName(m_fileName); m_currentLocationFileWatcher.EnableRaisingEvents = true; } if (Directory.Exists(m_archiveOffloadLocation) && (object)m_offloadLocationFileWatcher != null) { m_offloadLocationFileWatcher.Filter = HistoricFilesSearchPattern; m_offloadLocationFileWatcher.Path = m_archiveOffloadLocation; m_offloadLocationFileWatcher.EnableRaisingEvents = true; } } } } }
/// <summary> /// Main entry point to start analysis; handles setting up rules, directory enumeration /// file type detection and handoff /// Pre: All Configure Methods have been called already and we are ready to SCAN /// </summary> /// <returns></returns> public int Run() { WriteOnce.SafeLog("AnalyzeCommand::Run", LogLevel.Trace); DateTime start = DateTime.Now; WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_RUNNING, "Analyze")); _appProfile.MetaData.TotalFiles = _srcfileList.Count();//updated for zipped files later // Iterate through all files and process against rules foreach (string filename in _srcfileList) { //exclude sample, test or similar files by default or as specified in exclusion list if (!_arg_allowSampleFiles && _fileExclusionList.Any(v => filename.ToLower().Contains(v))) { WriteOnce.SafeLog("Part of excluded list: " + filename, LogLevel.Trace); WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILESIZE_SKIPPED, filename), LogLevel.Trace); _appProfile.MetaData.FilesSkipped++; continue; } ArchiveFileType archiveFileType = MiniMagic.DetectFileType(filename); if (archiveFileType == ArchiveFileType.UNKNOWN) { ProcessAsFile(filename); } else { UnZipAndProcess(filename, archiveFileType); } } WriteOnce.General("\r" + ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILES_PROCESSED_PCNT, 100)); WriteOnce.Operation(ErrMsg.GetString(ErrMsg.ID.CMD_PREPARING_REPORT)); //Prepare report results _appProfile.MetaData.LastUpdated = LastUpdated.ToString(); _appProfile.DateScanned = DateScanned.ToString(); _appProfile.PrepareReport(); TimeSpan timeSpan = start - DateTime.Now; WriteOnce.SafeLog(String.Format("Processing time: seconds:{0}", timeSpan.TotalSeconds * -1), LogLevel.Trace); FlushAll(); //wrapup result status if (_appProfile.MetaData.TotalFiles == _appProfile.MetaData.FilesSkipped) { WriteOnce.Error(ErrMsg.GetString(ErrMsg.ID.ANALYZE_NOSUPPORTED_FILETYPES)); } else if (_appProfile.MatchList.Count == 0) { WriteOnce.Error(ErrMsg.GetString(ErrMsg.ID.ANALYZE_NOPATTERNS)); } else { WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_COMPLETED, "Analyze")); } return(_appProfile.MatchList.Count() == 0 ? (int)ExitCode.NoMatches : (int)ExitCode.MatchesFound); }
internal RequestedConversion(ArchiveFileType targetEncoding, object encodingParameters) { TargetEncoding = targetEncoding; EncodingParameters = encodingParameters; }
public static IEnumerable <FileEntry> ExtractFile(string filename, ArchiveFileType archiveFileType) { using var memoryStream = new MemoryStream(File.ReadAllBytes(filename)); return(ExtractFile(new FileEntry(filename, "", memoryStream), archiveFileType)); }
/// <summary> /// Initializes a new instance of the <see cref="ArchiveFile"/> class. /// </summary> public ArchiveFile() { m_fileName = DefaultFileName; m_fileType = DefaultFileType; m_fileSize = DefaultFileSize; m_fileAccessMode = DefaultFileAccessMode; m_dataBlockSize = DefaultDataBlockSize; m_rolloverPreparationThreshold = DefaultRolloverPreparationThreshold; m_archiveOffloadLocation = DefaultArchiveOffloadLocation; m_archiveOffloadCount = DefaultArchiveOffloadCount; m_archiveOffloadThreshold = DefaultArchiveOffloadThreshold; m_archiveOffloadMaxAge = DefaultArchiveOffloadMaxAge; m_maxHistoricArchiveFiles = DefaultMaxHistoricArchiveFiles; m_leadTimeTolerance = DefaultLeadTimeTolerance; m_compressData = DefaultCompressData; m_discardOutOfSequenceData = DefaultDiscardOutOfSequenceData; m_cacheWrites = DefaultCacheWrites; m_conserveMemory = DefaultConserveMemory; m_monitorNewArchiveFiles = DefaultMonitorNewArchiveFiles; m_persistSettings = DefaultPersistSettings; m_settingsCategory = DefaultSettingsCategory; m_delayedAlarmProcessing = new Dictionary<int, decimal>(); m_rolloverWaitHandle = new ManualResetEvent(true); m_rolloverPreparationThread = new Thread(PrepareForRollover); m_buildHistoricFileListThread = new Thread(BuildHistoricFileList); m_conserveMemoryTimer = new Timer(); m_conserveMemoryTimer.Elapsed += ConserveMemoryTimer_Elapsed; m_currentDataQueue = ProcessQueue<IDataPoint>.CreateRealTimeQueue(WriteToCurrentArchiveFile); m_currentDataQueue.ProcessException += CurrentDataQueue_ProcessException; m_historicDataQueue = ProcessQueue<IDataPoint>.CreateRealTimeQueue(WriteToHistoricArchiveFile); m_historicDataQueue.ProcessException += HistoricDataQueue_ProcessException; m_outOfSequenceDataQueue = ProcessQueue<IDataPoint>.CreateRealTimeQueue(InsertInCurrentArchiveFile); m_outOfSequenceDataQueue.ProcessException += OutOfSequenceDataQueue_ProcessException; }
/// <summary> /// Set a new Extractor for the given ArchiveFileType. /// </summary> /// <param name="targetType">The ArchiveFileType to assign this extractor to.</param> /// <param name="implementation">The ExtractorImplementation.</param> public void SetExtractor(ArchiveFileType targetType, AsyncExtractorInterface implementation) { Extractors[targetType] = implementation; }
public Subfile(IDataSource source, string name, string archiveName, ArchiveFileType type) : this(source, name, archiveName, type, null) { }