private static IEnumerable <FileEntry> ExtractFile(FileEntry fileEntry, ArchiveFileType archiveFileType)
{
switch (archiveFileType)
{
case ArchiveFileType.ZIP:
return(ExtractZipFile(fileEntry));
case ArchiveFileType.GZIP:
return(ExtractGZipFile(fileEntry));
case ArchiveFileType.TAR:
return(ExtractTarFile(fileEntry));
case ArchiveFileType.XZ:
return(ExtractXZFile(fileEntry));
case ArchiveFileType.BZIP2:
return(ExtractBZip2File(fileEntry));
case ArchiveFileType.RAR:
return(ExtractRarFile(fileEntry));
case ArchiveFileType.P7ZIP:
return(Extract7ZipFile(fileEntry));
case ArchiveFileType.UNKNOWN:
default:
return(new[] { fileEntry });
}
}
public ArchiveFile(IDataSource Source, string Name, ArchiveFileCompression Compression)
{
if (Name.EndsWith(".wav"))
{
//this.Source = new CompressedDataSource(Source, ArchiveFileType.Audio);
this.Source = new Xgg.XggWrappedSource(Source);
this.Name = Name.Replace(".wav", ".xgg");
}
else
{
this.Source = Source;
this.Name = Name;
}
if (this.Name.EndsWith(".xgg"))
{
Type = ArchiveFileType.Audio;
}
else if (this.Name.EndsWith(".png"))
{
Type = ArchiveFileType.Image;
}
else
{
this.Compression = Compression;
}
}
/// <summary>
/// Remove any extractor (if set) for the given ArchiveFileType.
/// </summary>
/// <param name="targetType">The ArchiveFileType to remove the extractor for.</param>
public void Unset(ArchiveFileType targetType)
{
if (Extractors.ContainsKey(targetType))
{
Extractors.Remove(targetType);
}
}
/// <summary>
/// Main entry point to start analysis; handles setting up rules, directory enumeration
/// file type detection and handoff
/// Pre: All Configure Methods have been called already and we are ready to SCAN
/// </summary>
/// <returns></returns>
public int Run()
{
WriteOnce.SafeLog("AnalyzeCommand::Run", LogLevel.Trace);
DateTime start = DateTime.Now;
WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_RUNNING, "Analyze"));
_appProfile.MetaData.TotalFiles = _srcfileList.Count();//updated for zipped files later
// Iterate through all files and process against rules
foreach (string filename in _srcfileList)
{
ArchiveFileType archiveFileType = MiniMagic.DetectFileType(filename);
if (archiveFileType == ArchiveFileType.UNKNOWN)//not a known zipped file type
{
ProcessAsFile(filename);
}
else
{
UnZipAndProcess(filename, archiveFileType);
}
}
WriteOnce.General("\r" + ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILES_PROCESSED_PCNT, 100));
WriteOnce.Operation(ErrMsg.GetString(ErrMsg.ID.CMD_PREPARING_REPORT));
//Prepare report results
_appProfile.MetaData.LastUpdated = LastUpdated.ToString();
_appProfile.DateScanned = DateScanned.ToString();
_appProfile.PrepareReport();
TimeSpan timeSpan = start - DateTime.Now;
WriteOnce.SafeLog(String.Format("Processing time: seconds:{0}", timeSpan.TotalSeconds * -1), LogLevel.Trace);
FlushAll();
//wrapup result status
if (_appProfile.MetaData.TotalFiles == _appProfile.MetaData.FilesSkipped)
{
WriteOnce.Error(ErrMsg.GetString(ErrMsg.ID.ANALYZE_NOSUPPORTED_FILETYPES));
}
else if (_appProfile.MatchList.Count == 0)
{
WriteOnce.Error(ErrMsg.GetString(ErrMsg.ID.ANALYZE_NOPATTERNS));
}
else
{
WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_COMPLETED, "Analyze"));
if (!_arg_autoBrowserOpen)
{
WriteOnce.Any(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_OUTPUT_FILE, "output.html"));
}
}
return(_appProfile.MatchList.Count() == 0 ? (int)ExitCode.NoMatches :
(int)ExitCode.MatchesFound);
}
public static IEnumerable <FileEntry> ExtractFile(string filename, ArchiveFileType archiveFileType)
{
_count = 0;
//_files = new List<FileEntry>();
using var memoryStream = new MemoryStream(File.ReadAllBytes(filename));
//using var fileEntry = new FileEntry(filename, "", memoryStream);
return(ExtractFile(new FileEntry(filename, "", memoryStream), archiveFileType));
//return _files;
}
void UnZipAndProcess(string filename, ArchiveFileType archiveFileType)
{
//zip itself may be too huge for timely processing
if (new FileInfo(filename).Length > WARN_ZIP_FILE_SIZE)
{
WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_FILESIZE_WARN));
}
else
{
WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_PROCESSING));
}
try
{
IEnumerable <FileEntry> files = Extractor.ExtractFile(filename);
if (files.Count() > 0)
{
_appProfile.MetaData.TotalFiles += files.Count();//additive in case additional child zip files processed
_appProfile.MetaData.PackageTypes.Add(ErrMsg.GetString(ErrMsg.ID.ANALYZE_COMPRESSED_FILETYPE));
foreach (FileEntry file in files)
{
if (file.Content.Length > MAX_FILESIZE)
{
WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILESIZE_SKIPPED, file.FullPath), LogLevel.Warn);
_appProfile.MetaData.FilesSkipped++;
continue;
}
//dup check vs Run() for zip contents; exclude sample, test or similar files by default or as specified in exclusion list
if (!_arg_allowSampleFiles && _fileExclusionList.Any(v => file.FullPath.ToLower().Contains(v)))
{
WriteOnce.SafeLog("Part of excluded list: " + file.FullPath, LogLevel.Trace);
WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILESIZE_SKIPPED, file.FullPath), LogLevel.Trace);
_appProfile.MetaData.FilesSkipped++;
continue;
}
WriteOnce.Log.Trace("processing zip file entry: " + file.FullPath);
byte[] streamByteArray = file.Content.ToArray();
ProcessInMemory(file.FullPath, Encoding.UTF8.GetString(streamByteArray, 0, streamByteArray.Length), true);
}
}
else
{
throw new OpException(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_ERROR, filename));
}
}
catch (Exception e)
{
string errmsg = ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_ERROR, filename);
WriteOnce.Error(errmsg);
throw new Exception(errmsg + e.Message + "\n" + e.StackTrace);
}
}
void UnZipAndProcess(string filename, ArchiveFileType archiveFileType)
{
// zip itself may be in excluded list i.e. sample, test or similar unless ignore filter requested
if (_fileExclusionList != null && _fileExclusionList.Any(v => filename.ToLower().Contains(v)))
{
WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_EXCLUDED_TYPE_SKIPPED, filename), LogLevel.Warn);
_appProfile.MetaData.FilesSkipped++;
return;
}
//zip itself may be too huge for timely processing
if (new FileInfo(filename).Length > WARN_ZIP_FILE_SIZE)
{
WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_FILESIZE_WARN));
}
else
{
WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_PROCESSING));
}
LastUpdated = File.GetLastWriteTime(filename);
_appProfile.MetaData.PackageTypes.Add(ErrMsg.GetString(ErrMsg.ID.ANALYZE_COMPRESSED_FILETYPE));
try
{
IEnumerable <FileEntry> files = Extractor.ExtractFile(filename);
if (files.Count() > 0)
{
_appProfile.MetaData.TotalFiles += files.Count();//additive in case additional child zip files processed
foreach (FileEntry file in files)
{
//check uncompressed file passes standard checks
LanguageInfo languageInfo = new LanguageInfo();
if (FileChecksPassed(file.FullPath, ref languageInfo, file.Content.Length))
{
byte[] streamByteArray = file.Content.ToArray();
ProcessInMemory(file.FullPath, Encoding.UTF8.GetString(streamByteArray, 0, streamByteArray.Length), languageInfo);
}
}
}
else
{
WriteOnce.SafeLog(string.Format("Decompression found no files in {0}", filename), LogLevel.Warn);//zero results can be valid
}
}
catch (Exception)
{
string errmsg = ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_ERROR, filename);
WriteOnce.Error(errmsg);
throw;
}
}
public Subfile(IDataSource source, string name, string archiveName, ArchiveFileType type, RequestedConversion requestedConversion)
{
if (!archiveName.EndsWith(".pp"))
{
archiveName += ".pp";
}
ArchiveName = archiveName;
Name = name;
Source = source;
Size = source.Size;
Type = type;
RequestedConversion = requestedConversion;
EmulatedName = name;
}
public CompressedDataSource(IDataSource basesource, ArchiveFileType type)
{
this.basesource = basesource;
this.type = type;
_md5 = basesource.Md5;
_size = basesource.Size;
/*
* using (Stream stream = GetStream())
* {
* _md5 = Utility.GetMd5(stream);
*
* _size = (uint)stream.Length;
* }*/
}
void UnZipAndProcess(string filename, ArchiveFileType archiveFileType)
{
//zip itself may be too huge for timely processing
if (new FileInfo(filename).Length > WARN_ZIP_FILE_SIZE)
{
WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_FILESIZE_WARN));
}
else
{
WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_PROCESSING));
}
LastUpdated = File.GetLastWriteTime(filename);
_appProfile.MetaData.PackageTypes.Add(ErrMsg.GetString(ErrMsg.ID.ANALYZE_COMPRESSED_FILETYPE));
try
{
IEnumerable <FileEntry> files = Extractor.ExtractFile(filename);
if (files.Count() > 0)
{
_appProfile.MetaData.TotalFiles += files.Count();//additive in case additional child zip files processed
foreach (FileEntry file in files)
{
//check for supported language
LanguageInfo languageInfo = new LanguageInfo();
if (FileChecksPassed(file.FullPath, ref languageInfo, file.Content.Length))
{
byte[] streamByteArray = file.Content.ToArray();
ProcessInMemory(file.FullPath, Encoding.UTF8.GetString(streamByteArray, 0, streamByteArray.Length), languageInfo);
}
}
}
else
{
throw new OpException(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_ERROR, filename));
}
}
catch (Exception e)
{
string errmsg = ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_ERROR, filename);
WriteOnce.Error(errmsg);
throw new Exception(errmsg + e.Message + "\n" + e.StackTrace);
}
}
public void TestMiniMagic(string fileName, ArchiveFileType expectedArchiveFileType)
{
var path = Path.Combine(Directory.GetCurrentDirectory(), "TestData", fileName);
using var fs = new FileStream(path, FileMode.Open);
// Test just based on the content
var fileEntry = new FileEntry("NoName", fs);
// We make sure the expected type matches and we have reset the stream
Assert.AreEqual(expectedArchiveFileType, MiniMagic.DetectFileType(fileEntry));
Assert.AreEqual(0, fileEntry.Content.Position);
// Should also work if the stream doesn't start at 0
fileEntry.Content.Position = 10;
Assert.AreEqual(expectedArchiveFileType, MiniMagic.DetectFileType(fileEntry));
Assert.AreEqual(10, fileEntry.Content.Position);
}
public static IEnumerable <FileEntry> ExtractFile(string filename, ArchiveFileType archiveFileType)
{
try
{
if (!File.OpenRead(filename).CanRead)
{
throw new IOException($"ExtractFile called, but {filename} cannot be read.");
}
}
catch (Exception)
{
//Logger.Trace("File {0} cannot be read, ignoring.", filename);
return(Array.Empty <FileEntry>());
}
using var memoryStream = new MemoryStream(File.ReadAllBytes(filename));
return(ExtractFile(new FileEntry(filename, "", memoryStream), archiveFileType));
}
private static IEnumerable <FileEntry> ExtractFile(FileEntry fileEntry, ArchiveFileType archiveFileType)
{
IEnumerable <FileEntry> results = new List <FileEntry>();
switch (archiveFileType)
{
case ArchiveFileType.ZIP:
results = ExtractZipFile(fileEntry);
break;
case ArchiveFileType.GZIP:
results = ExtractGZipFile(fileEntry);
break;
case ArchiveFileType.TAR:
results = ExtractTarFile(fileEntry);
break;
case ArchiveFileType.XZ:
results = ExtractXZFile(fileEntry);
break;
case ArchiveFileType.BZIP2:
results = ExtractBZip2File(fileEntry);
break;
case ArchiveFileType.RAR:
results = ExtractRarFile(fileEntry);
break;
case ArchiveFileType.P7ZIP:
results = Extract7ZipFile(fileEntry);
break;
case ArchiveFileType.UNKNOWN:
default:
results = new[] { fileEntry };
break;
}
return(results);
}
internal ArchiveFileSource(BinaryReader reader)
{
Type = (ArchiveFileType)reader.ReadByte();
Flags = (ArchiveFileFlags)reader.ReadByte();
Compression = (ArchiveFileCompression)reader.ReadByte();
reader.BaseStream.Seek(1, SeekOrigin.Current);
_crc = reader.ReadUInt32();
_md5 = reader.ReadBytes(16);
reader.BaseStream.Seek(48, SeekOrigin.Current);
ushort len = reader.ReadUInt16();
string[] names = Encoding.Unicode.GetString(reader.ReadBytes(len)).Split('/');
ArchiveName = names[0];
Name = names[1];
offset = reader.ReadUInt64();
_size = reader.ReadUInt32();
length = reader.ReadUInt32();
ArchiveFilename = (reader.BaseStream as FileStream).Name;
}
public static bool IsSupportedFormat(string filename)
{
ArchiveFileType archiveFileType = MiniMagic.DetectFileType(filename);
return(archiveFileType != ArchiveFileType.UNKNOWN);
}
/// <summary>
/// Initializes a new instance of the <see cref="ArchiveFile"/> class.
/// </summary>
public ArchiveFile()
: base()
{
m_fileName = DefaultFileName;
m_fileType = DefaultFileType;
m_fileSize = DefaultFileSize;
m_fileAccessMode = DefaultFileAccessMode;
m_dataBlockSize = DefaultDataBlockSize;
m_rolloverOnFull = DefaultRolloverOnFull;
m_rolloverPreparationThreshold = DefaultRolloverPreparationThreshold;
m_fileOffloadLocation = DefaultFileOffloadLocation;
m_fileOffloadCount = DefaultFileOffloadCount;
m_fileOffloadThreshold = DefaultFileOffloadThreshold;
m_leadTimeTolerance = DefaultLeadTimeTolerance;
m_compressData = DefaultCompressData;
m_discardOutOfSequenceData = DefaultDiscardOutOfSequenceData;
m_cacheWrites = DefaultCacheWrites;
m_conserveMemory = DefaultConserveMemory;
m_persistSettings = DefaultPersistSettings;
m_settingsCategory = DefaultSettingsCategory;
m_delayedAlarmProcessing = new Dictionary<int, double>();
m_rolloverWaitHandle = new ManualResetEvent(true);
m_rolloverPreparationThread = new Thread(PrepareForRollover);
m_buildHistoricFileListThread = new Thread(BuildHistoricFileList);
m_conserveMemoryTimer = new System.Timers.Timer(DataBlockCheckInterval);
m_conserveMemoryTimer.Elapsed += ConserveMemoryTimer_Elapsed;
m_historicDataQueue = ProcessQueue<ArchiveData>.CreateRealTimeQueue(WriteToHistoricArchiveFile);
m_historicDataQueue.ProcessException += HistoricDataQueue_ProcessException;
m_outOfSequenceDataQueue = ProcessQueue<ArchiveData>.CreateRealTimeQueue(InsertInCurrentArchiveFile);
m_outOfSequenceDataQueue.ProcessException += OutOfSequenceDataQueue_ProcessException;
m_currentLocationFileWatcher = new FileSystemWatcher();
m_currentLocationFileWatcher.IncludeSubdirectories = true;
m_currentLocationFileWatcher.Renamed += FileWatcher_Renamed;
m_currentLocationFileWatcher.Deleted += FileWatcher_Deleted;
m_currentLocationFileWatcher.Created += FileWatcher_Created;
m_offloadLocationFileWatcher = new FileSystemWatcher();
m_offloadLocationFileWatcher.IncludeSubdirectories = true;
m_offloadLocationFileWatcher.Renamed += FileWatcher_Renamed;
m_offloadLocationFileWatcher.Deleted += FileWatcher_Deleted;
m_offloadLocationFileWatcher.Created += FileWatcher_Created;
}
/// <summary>
/// Opens the <see cref="ArchiveFile"/> for use.
/// </summary>
/// <exception cref="InvalidOperationException">One or all of the <see cref="StateFile"/>, <see cref="IntercomFile"/> or <see cref="MetadataFile"/> properties are not set.</exception>
public void Open()
{
if (!IsOpen)
{
// Check for the existance of dependencies.
if (m_stateFile == null || m_intercomFile == null | m_metadataFile == null)
throw (new InvalidOperationException("One or more of the dependency files are not specified."));
// Validate file type against its name.
m_fileName = m_fileName.ToLower();
if (Path.GetExtension(m_fileName) == StandbyFileExtension)
m_fileType = ArchiveFileType.Standby;
else if (Regex.IsMatch(m_fileName, string.Format(".+_.+_to_.+\\{0}$", FileExtension)))
m_fileType = ArchiveFileType.Historic;
else
m_fileType = ArchiveFileType.Active;
// Get the absolute path for the file name.
m_fileName = FilePath.GetAbsolutePath(m_fileName);
// Create the directory if it does not exist.
if (!Directory.Exists(FilePath.GetDirectoryName(m_fileName)))
Directory.CreateDirectory(FilePath.GetDirectoryName(m_fileName));
if (File.Exists(m_fileName))
{
// File has been created already, so we just need to read it.
m_fileStream = new FileStream(m_fileName, FileMode.Open, m_fileAccessMode, FileShare.ReadWrite);
m_fat = new ArchiveFileAllocationTable(this);
}
else
{
// File does not exist, so we have to create it and initialize it.
m_fileStream = new FileStream(m_fileName, FileMode.Create, FileAccess.ReadWrite, FileShare.ReadWrite);
m_fat = new ArchiveFileAllocationTable(this);
m_fat.Save();
}
// Don't proceed further for standby and historic files.
if (m_fileType != ArchiveFileType.Active)
return;
// Start internal process queues.
m_historicDataQueue.Start();
m_outOfSequenceDataQueue.Start();
// Open state file if closed.
if (!m_stateFile.IsOpen)
m_stateFile.Open();
//Open intercom file if closed.
if (!m_intercomFile.IsOpen)
m_intercomFile.Open();
// Open metadata file if closed.
if (!m_metadataFile.IsOpen)
m_metadataFile.Open();
// Create data block lookup list.
if (m_stateFile.RecordsInMemory > 0)
m_dataBlocks = new List<ArchiveDataBlock>(new ArchiveDataBlock[m_stateFile.RecordsInMemory]);
else
m_dataBlocks = new List<ArchiveDataBlock>(new ArchiveDataBlock[m_stateFile.RecordsOnDisk]);
// Start the memory conservation process.
if (m_conserveMemory)
m_conserveMemoryTimer.Start();
// Ensure that "rollover in progress" is not set.
IntercomRecord system = m_intercomFile.Read(1);
system.RolloverInProgress = false;
m_intercomFile.Write(1, system);
// Start preparing the list of historic files.
m_buildHistoricFileListThread = new Thread(BuildHistoricFileList);
m_buildHistoricFileListThread.Priority = ThreadPriority.Lowest;
m_buildHistoricFileListThread.Start();
// Start file watchers to monitor file system changes.
m_currentLocationFileWatcher.Filter = HistoricFilesSearchPattern;
m_currentLocationFileWatcher.Path = FilePath.GetDirectoryName(m_fileName);
m_currentLocationFileWatcher.EnableRaisingEvents = true;
if (Directory.Exists(m_fileOffloadLocation))
{
m_offloadLocationFileWatcher.Filter = HistoricFilesSearchPattern;
m_offloadLocationFileWatcher.Path = m_fileOffloadLocation;
m_offloadLocationFileWatcher.EnableRaisingEvents = true;
}
}
}
/// <summary>
/// Creates a new data source from a processed byte array.
/// </summary>
/// <param name="data">The byte array to use.</param>
/// <param name="compression">The compression that the byte array has used.</param>
/// <param name="encoding">The encoding of the data.</param>
public MemorySource(ReadOnlyMemory <byte> data, ArchiveFileType encoding)
{
Data = data;
Encoding = encoding;
}
/// <summary>
/// Main entry point to start analysis from CLI; handles setting up rules, directory enumeration
/// file type detection and handoff
/// Pre: All Configure Methods have been called already and we are ready to SCAN
/// </summary>
/// <returns></returns>
public override int Run()
{
WriteOnce.SafeLog("AnalyzeCommand::Run", LogLevel.Trace);
WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_RUNNING, "analyze"));
try
{
_appProfile.MetaData.TotalFiles = _srcfileList.Count();//updated for zipped files later
// Iterate through all files and process against rules
foreach (string filename in _srcfileList)
{
ArchiveFileType archiveFileType = ArchiveFileType.UNKNOWN;
try //fix for #146
{
archiveFileType = MiniMagic.DetectFileType(filename);
}
catch (Exception e)
{
WriteOnce.SafeLog(e.Message + "\n" + e.StackTrace, LogLevel.Error); //log details
Exception f = new Exception(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILE_TYPE_OPEN, filename)); //report friendly version
throw f;
}
if (archiveFileType == ArchiveFileType.UNKNOWN)//not a known zipped file type
{
ProcessAsFile(filename);
}
else
{
UnZipAndProcess(filename, archiveFileType);
}
}
WriteOnce.General("\r" + ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILES_PROCESSED_PCNT, 100));
WriteOnce.Operation(ErrMsg.GetString(ErrMsg.ID.CMD_PREPARING_REPORT));
//Prepare report results
_appProfile.MetaData.LastUpdated = LastUpdated.ToString();
_appProfile.DateScanned = DateScanned.ToString();
_appProfile.PrepareReport();
FlushAll();
//wrapup result status
if (_appProfile.MetaData.TotalFiles == _appProfile.MetaData.FilesSkipped)
{
WriteOnce.Error(ErrMsg.GetString(ErrMsg.ID.ANALYZE_NOSUPPORTED_FILETYPES));
}
else if (_appProfile.MatchList.Count == 0)
{
WriteOnce.Error(ErrMsg.GetString(ErrMsg.ID.ANALYZE_NOPATTERNS));
}
else
{
WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_COMPLETED, "analyze"));
}
}
catch (Exception e)
{
WriteOnce.Error(e.Message);
//exit normaly for CLI callers and throw for DLL callers
if (Utils.CLIExecutionContext)
{
return((int)ExitCode.CriticalError);
}
else
{
throw;
}
}
finally
{
if (_arg_close_log_on_exit)
{
Utils.Logger = null;
WriteOnce.Log = null;
}
}
return(_appProfile.MatchList.Count() == 0 ? (int)ExitCode.NoMatches :
(int)Utils.ExitCode.Success);
}
/// <summary>
/// Opens the <see cref="ArchiveFile"/> for use.
/// </summary>
/// <exception cref="InvalidOperationException">One or all of the <see cref="StateFile"/>, <see cref="IntercomFile"/> or <see cref="MetadataFile"/> properties are not set.</exception>
public void Open()
{
if (!IsOpen)
{
// Check for the existence of dependencies.
if ((object)m_stateFile == null || (object)m_intercomFile == null || (object)m_metadataFile == null)
throw (new InvalidOperationException("One or more of the dependency files are not specified."));
// Validate file type against its name.
if (Path.GetExtension(m_fileName).ToNonNullString().ToLower() == StandbyFileExtension)
m_fileType = ArchiveFileType.Standby;
else if (Regex.IsMatch(m_fileName.ToLower(), string.Format(".+_.+_to_.+\\{0}$", FileExtension)))
m_fileType = ArchiveFileType.Historic;
else
m_fileType = ArchiveFileType.Active;
// Get the absolute path for the file name.
m_fileName = FilePath.GetAbsolutePath(m_fileName);
// Create the directory if it does not exist.
if (!Directory.Exists(FilePath.GetDirectoryName(m_fileName)))
Directory.CreateDirectory(FilePath.GetDirectoryName(m_fileName));
// Validate a roll-over is not in progress when opening archive as read-only
if (m_fileType == ArchiveFileType.Active && m_fileAccessMode == FileAccess.Read)
{
// Open intercom file if closed.
if (!m_intercomFile.IsOpen)
m_intercomFile.Open();
m_intercomFile.Load();
IntercomRecord record = m_intercomFile.Read(1);
int waitCount = 0;
while ((object)record != null && record.RolloverInProgress && waitCount < 30)
{
Thread.Sleep(1000);
m_intercomFile.Load();
record = m_intercomFile.Read(1);
waitCount++;
}
}
OpenStream();
// Open state file if closed.
if (!m_stateFile.IsOpen)
m_stateFile.Open();
// Open intercom file if closed.
if (!m_intercomFile.IsOpen)
m_intercomFile.Open();
// Open metadata file if closed.
if (!m_metadataFile.IsOpen)
m_metadataFile.Open();
// Don't proceed further for standby and historic files.
if (m_fileType != ArchiveFileType.Active)
return;
// Start internal process queues.
m_currentDataQueue.Start();
m_historicDataQueue.Start();
m_outOfSequenceDataQueue.Start();
// Create data block lookup list.
if (m_stateFile.RecordsInMemory > 0)
m_dataBlocks = new List<ArchiveDataBlock>(new ArchiveDataBlock[m_stateFile.RecordsInMemory]);
else
m_dataBlocks = new List<ArchiveDataBlock>(new ArchiveDataBlock[m_stateFile.RecordsOnDisk]);
// Validate the dependency files.
SyncStateFile(null);
if (m_intercomFile.FileAccessMode != FileAccess.Read)
{
// Ensure that "rollover in progress" is not set.
IntercomRecord system = m_intercomFile.Read(1);
if ((object)system == null)
system = new IntercomRecord(1);
system.RolloverInProgress = false;
m_intercomFile.Write(1, system);
}
// Start the memory conservation process.
if (m_conserveMemory)
{
m_conserveMemoryTimer.Interval = DataBlockCheckInterval;
m_conserveMemoryTimer.Start();
}
if (m_fileType == ArchiveFileType.Active)
{
// Start preparing the list of historic files.
m_buildHistoricFileListThread = new Thread(BuildHistoricFileList);
m_buildHistoricFileListThread.Priority = ThreadPriority.Lowest;
m_buildHistoricFileListThread.Start();
// Start file watchers to monitor file system changes.
if (m_monitorNewArchiveFiles)
{
if ((object)m_currentLocationFileWatcher != null)
{
m_currentLocationFileWatcher.Filter = HistoricFilesSearchPattern;
m_currentLocationFileWatcher.Path = FilePath.GetDirectoryName(m_fileName);
m_currentLocationFileWatcher.EnableRaisingEvents = true;
}
if (Directory.Exists(m_archiveOffloadLocation) && (object)m_offloadLocationFileWatcher != null)
{
m_offloadLocationFileWatcher.Filter = HistoricFilesSearchPattern;
m_offloadLocationFileWatcher.Path = m_archiveOffloadLocation;
m_offloadLocationFileWatcher.EnableRaisingEvents = true;
}
}
}
}
}
/// <summary>
/// Main entry point to start analysis; handles setting up rules, directory enumeration
/// file type detection and handoff
/// Pre: All Configure Methods have been called already and we are ready to SCAN
/// </summary>
/// <returns></returns>
public int Run()
{
WriteOnce.SafeLog("AnalyzeCommand::Run", LogLevel.Trace);
DateTime start = DateTime.Now;
WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_RUNNING, "Analyze"));
_appProfile.MetaData.TotalFiles = _srcfileList.Count();//updated for zipped files later
// Iterate through all files and process against rules
foreach (string filename in _srcfileList)
{
//exclude sample, test or similar files by default or as specified in exclusion list
if (!_arg_allowSampleFiles && _fileExclusionList.Any(v => filename.ToLower().Contains(v)))
{
WriteOnce.SafeLog("Part of excluded list: " + filename, LogLevel.Trace);
WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILESIZE_SKIPPED, filename), LogLevel.Trace);
_appProfile.MetaData.FilesSkipped++;
continue;
}
ArchiveFileType archiveFileType = MiniMagic.DetectFileType(filename);
if (archiveFileType == ArchiveFileType.UNKNOWN)
{
ProcessAsFile(filename);
}
else
{
UnZipAndProcess(filename, archiveFileType);
}
}
WriteOnce.General("\r" + ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILES_PROCESSED_PCNT, 100));
WriteOnce.Operation(ErrMsg.GetString(ErrMsg.ID.CMD_PREPARING_REPORT));
//Prepare report results
_appProfile.MetaData.LastUpdated = LastUpdated.ToString();
_appProfile.DateScanned = DateScanned.ToString();
_appProfile.PrepareReport();
TimeSpan timeSpan = start - DateTime.Now;
WriteOnce.SafeLog(String.Format("Processing time: seconds:{0}", timeSpan.TotalSeconds * -1), LogLevel.Trace);
FlushAll();
//wrapup result status
if (_appProfile.MetaData.TotalFiles == _appProfile.MetaData.FilesSkipped)
{
WriteOnce.Error(ErrMsg.GetString(ErrMsg.ID.ANALYZE_NOSUPPORTED_FILETYPES));
}
else if (_appProfile.MatchList.Count == 0)
{
WriteOnce.Error(ErrMsg.GetString(ErrMsg.ID.ANALYZE_NOPATTERNS));
}
else
{
WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_COMPLETED, "Analyze"));
}
return(_appProfile.MatchList.Count() == 0 ? (int)ExitCode.NoMatches :
(int)ExitCode.MatchesFound);
}
internal RequestedConversion(ArchiveFileType targetEncoding, object encodingParameters)
{
TargetEncoding = targetEncoding;
EncodingParameters = encodingParameters;
}
public static IEnumerable <FileEntry> ExtractFile(string filename, ArchiveFileType archiveFileType)
{
using var memoryStream = new MemoryStream(File.ReadAllBytes(filename));
return(ExtractFile(new FileEntry(filename, "", memoryStream), archiveFileType));
}
/// <summary>
/// Initializes a new instance of the <see cref="ArchiveFile"/> class.
/// </summary>
public ArchiveFile()
{
m_fileName = DefaultFileName;
m_fileType = DefaultFileType;
m_fileSize = DefaultFileSize;
m_fileAccessMode = DefaultFileAccessMode;
m_dataBlockSize = DefaultDataBlockSize;
m_rolloverPreparationThreshold = DefaultRolloverPreparationThreshold;
m_archiveOffloadLocation = DefaultArchiveOffloadLocation;
m_archiveOffloadCount = DefaultArchiveOffloadCount;
m_archiveOffloadThreshold = DefaultArchiveOffloadThreshold;
m_archiveOffloadMaxAge = DefaultArchiveOffloadMaxAge;
m_maxHistoricArchiveFiles = DefaultMaxHistoricArchiveFiles;
m_leadTimeTolerance = DefaultLeadTimeTolerance;
m_compressData = DefaultCompressData;
m_discardOutOfSequenceData = DefaultDiscardOutOfSequenceData;
m_cacheWrites = DefaultCacheWrites;
m_conserveMemory = DefaultConserveMemory;
m_monitorNewArchiveFiles = DefaultMonitorNewArchiveFiles;
m_persistSettings = DefaultPersistSettings;
m_settingsCategory = DefaultSettingsCategory;
m_delayedAlarmProcessing = new Dictionary<int, decimal>();
m_rolloverWaitHandle = new ManualResetEvent(true);
m_rolloverPreparationThread = new Thread(PrepareForRollover);
m_buildHistoricFileListThread = new Thread(BuildHistoricFileList);
m_conserveMemoryTimer = new Timer();
m_conserveMemoryTimer.Elapsed += ConserveMemoryTimer_Elapsed;
m_currentDataQueue = ProcessQueue<IDataPoint>.CreateRealTimeQueue(WriteToCurrentArchiveFile);
m_currentDataQueue.ProcessException += CurrentDataQueue_ProcessException;
m_historicDataQueue = ProcessQueue<IDataPoint>.CreateRealTimeQueue(WriteToHistoricArchiveFile);
m_historicDataQueue.ProcessException += HistoricDataQueue_ProcessException;
m_outOfSequenceDataQueue = ProcessQueue<IDataPoint>.CreateRealTimeQueue(InsertInCurrentArchiveFile);
m_outOfSequenceDataQueue.ProcessException += OutOfSequenceDataQueue_ProcessException;
}
/// <summary>
/// Set a new Extractor for the given ArchiveFileType.
/// </summary>
/// <param name="targetType">The ArchiveFileType to assign this extractor to.</param>
/// <param name="implementation">The ExtractorImplementation.</param>
public void SetExtractor(ArchiveFileType targetType, AsyncExtractorInterface implementation)
{
Extractors[targetType] = implementation;
}
public Subfile(IDataSource source, string name, string archiveName, ArchiveFileType type) : this(source, name, archiveName, type, null)
{
}
