/// <summary>
/// Main entry point to start analysis; handles setting up rules, directory enumeration
/// file type detection and handoff
/// Pre: All Configure Methods have been called already and we are ready to SCAN
/// </summary>
/// <returns></returns>
public int Run()
{
WriteOnce.SafeLog("AnalyzeCommand::Run", LogLevel.Trace);
DateTime start = DateTime.Now;
WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_RUNNING, "Analyze"));
_appProfile.MetaData.TotalFiles = _srcfileList.Count();//updated for zipped files later
// Iterate through all files and process against rules
foreach (string filename in _srcfileList)
{
var fileExtension = new FileInfo(filename).Extension;
if (COMPRESSED_EXTENSIONS.Any(fileExtension.Contains))
{
UnZipAndProcess(filename); //determine if file is a compressed item to unpackage for processing
}
else
{
ProcessAsFile(filename);
}
}
WriteOnce.General("\r" + ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILES_PROCESSED_PCNT, 100));
WriteOnce.Operation(ErrMsg.GetString(ErrMsg.ID.CMD_PREPARING_REPORT));
//Prepare report results
_appProfile.MetaData.LastUpdated = LastUpdated.ToString();
_appProfile.DateScanned = DateScanned.ToString();
_appProfile.PrepareReport();
TimeSpan timeSpan = start - DateTime.Now;
WriteOnce.SafeLog(String.Format("Processing time: seconds:{0}", timeSpan.TotalSeconds * -1), LogLevel.Trace);
FlushAll();
//wrapup result status
if (_appProfile.MetaData.TotalFiles == _appProfile.MetaData.FilesSkipped)
{
WriteOnce.Error(ErrMsg.GetString(ErrMsg.ID.ANALYZE_NOSUPPORTED_FILETYPES));
}
else if (_appProfile.MatchList.Count == 0)
{
WriteOnce.Error(ErrMsg.GetString(ErrMsg.ID.ANALYZE_NOPATTERNS));
}
else
{
WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_COMPLETED, "Analyze"));
}
//html report size warning
if (_arg_fileFormat == "html" && new FileInfo("output.html").Length > MAX_HTML_REPORT_FILE_SIZE)
{
WriteOnce.Info(ErrMsg.GetString(ErrMsg.ID.ANALYZE_REPORTSIZE_WARN));
}
return(_appProfile.MatchList.Count() == 0 ? (int)ExitCode.NoMatches :
(int)ExitCode.MatchesFound);
}
/// <summary>
/// Main entry point to start analysis; handles setting up rules, directory enumeration
/// file type detection and handoff
/// Pre: All Configure Methods have been called already and we are ready to SCAN
/// </summary>
/// <returns></returns>
public int Run()
{
WriteOnce.SafeLog("AnalyzeCommand::Run", LogLevel.Trace);
DateTime start = DateTime.Now;
WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_RUNNING, "Analyze"));
_appProfile.MetaData.TotalFiles = _srcfileList.Count();//updated for zipped files later
// Iterate through all files and process against rules
foreach (string filename in _srcfileList)
{
ArchiveFileType archiveFileType = MiniMagic.DetectFileType(filename);
if (archiveFileType == ArchiveFileType.UNKNOWN)//not a known zipped file type
{
ProcessAsFile(filename);
}
else
{
UnZipAndProcess(filename, archiveFileType);
}
}
WriteOnce.General("\r" + ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILES_PROCESSED_PCNT, 100));
WriteOnce.Operation(ErrMsg.GetString(ErrMsg.ID.CMD_PREPARING_REPORT));
//Prepare report results
_appProfile.MetaData.LastUpdated = LastUpdated.ToString();
_appProfile.DateScanned = DateScanned.ToString();
_appProfile.PrepareReport();
TimeSpan timeSpan = start - DateTime.Now;
WriteOnce.SafeLog(String.Format("Processing time: seconds:{0}", timeSpan.TotalSeconds * -1), LogLevel.Trace);
FlushAll();
//wrapup result status
if (_appProfile.MetaData.TotalFiles == _appProfile.MetaData.FilesSkipped)
{
WriteOnce.Error(ErrMsg.GetString(ErrMsg.ID.ANALYZE_NOSUPPORTED_FILETYPES));
}
else if (_appProfile.MatchList.Count == 0)
{
WriteOnce.Error(ErrMsg.GetString(ErrMsg.ID.ANALYZE_NOPATTERNS));
}
else
{
WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_COMPLETED, "Analyze"));
if (!_arg_autoBrowserOpen)
{
WriteOnce.Any(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_OUTPUT_FILE, "output.html"));
}
}
return(_appProfile.MatchList.Count() == 0 ? (int)ExitCode.NoMatches :
(int)ExitCode.MatchesFound);
}
void UnZipAndProcess(string filename, ArchiveFileType archiveFileType)
{
//zip itself may be too huge for timely processing
if (new FileInfo(filename).Length > WARN_ZIP_FILE_SIZE)
{
WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_FILESIZE_WARN));
}
else
{
WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_PROCESSING));
}
try
{
IEnumerable <FileEntry> files = Extractor.ExtractFile(filename);
if (files.Count() > 0)
{
_appProfile.MetaData.TotalFiles += files.Count();//additive in case additional child zip files processed
_appProfile.MetaData.PackageTypes.Add(ErrMsg.GetString(ErrMsg.ID.ANALYZE_COMPRESSED_FILETYPE));
foreach (FileEntry file in files)
{
if (file.Content.Length > MAX_FILESIZE)
{
WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILESIZE_SKIPPED, file.FullPath), LogLevel.Warn);
_appProfile.MetaData.FilesSkipped++;
continue;
}
//dup check vs Run() for zip contents; exclude sample, test or similar files by default or as specified in exclusion list
if (!_arg_allowSampleFiles && _fileExclusionList.Any(v => file.FullPath.ToLower().Contains(v)))
{
WriteOnce.SafeLog("Part of excluded list: " + file.FullPath, LogLevel.Trace);
WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILESIZE_SKIPPED, file.FullPath), LogLevel.Trace);
_appProfile.MetaData.FilesSkipped++;
continue;
}
WriteOnce.Log.Trace("processing zip file entry: " + file.FullPath);
byte[] streamByteArray = file.Content.ToArray();
ProcessInMemory(file.FullPath, Encoding.UTF8.GetString(streamByteArray, 0, streamByteArray.Length), true);
}
}
else
{
throw new OpException(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_ERROR, filename));
}
}
catch (Exception e)
{
string errmsg = ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_ERROR, filename);
WriteOnce.Error(errmsg);
throw new Exception(errmsg + e.Message + "\n" + e.StackTrace);
}
}
void UnZipAndProcess(string filename, ArchiveFileType archiveFileType)
{
// zip itself may be in excluded list i.e. sample, test or similar unless ignore filter requested
if (_fileExclusionList.Any(v => filename.ToLower().Contains(v)))
{
WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_EXCLUDED_TYPE_SKIPPED, filename), LogLevel.Warn);
_appProfile.MetaData.FilesSkipped++;
return;
}
//zip itself may be too huge for timely processing
if (new FileInfo(filename).Length > WARN_ZIP_FILE_SIZE)
{
WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_FILESIZE_WARN));
}
else
{
WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_PROCESSING));
}
LastUpdated = File.GetLastWriteTime(filename);
_appProfile.MetaData.PackageTypes.Add(ErrMsg.GetString(ErrMsg.ID.ANALYZE_COMPRESSED_FILETYPE));
try
{
IEnumerable <FileEntry> files = Extractor.ExtractFile(filename);
if (files.Count() > 0)
{
_appProfile.MetaData.TotalFiles += files.Count();//additive in case additional child zip files processed
foreach (FileEntry file in files)
{
//check uncompressed file passes standard checks
LanguageInfo languageInfo = new LanguageInfo();
if (FileChecksPassed(file.FullPath, ref languageInfo, file.Content.Length))
{
byte[] streamByteArray = file.Content.ToArray();
ProcessInMemory(file.FullPath, Encoding.UTF8.GetString(streamByteArray, 0, streamByteArray.Length), languageInfo);
}
}
}
else
{
WriteOnce.SafeLog(string.Format("Decompression found no files in {0}", filename), LogLevel.Warn);//zero results can be valid
}
}
catch (Exception e)
{
string errmsg = ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_ERROR, filename);
WriteOnce.Error(errmsg);
throw new Exception(errmsg + e.Message + "\n" + e.StackTrace);
}
}
/// <summary>
/// Program entry point which defines command verbs and options to running
/// </summary>
/// <param name="args"></param>
static int Main(string[] args)
{
int finalResult = -1;
WriteOnce.Verbosity = WriteOnce.ConsoleVerbosity.Medium;
try
{
WriteOnce.Info(GetVersionString());
var argsResult = Parser.Default.ParseArguments <AnalyzeCommandOptions,
TagDiffCommandOptions,
TagTestCommandOptions,
ExportTagsCommandOptions,
VerifyRulesCommandOptions>(args)
.MapResult(
(AnalyzeCommandOptions opts) => RunAnalyzeCommand(opts),
(TagDiffCommandOptions opts) => RunTagDiffCommand(opts),
(TagTestCommandOptions opts) => RunTagTestCommand(opts),
(ExportTagsCommandOptions opts) => RunExportTagsCommand(opts),
(VerifyRulesCommandOptions opts) => RunVerifyRulesCommand(opts),
errs => 1
);
finalResult = argsResult;
}
catch (OpException e)
{
if (Logger != null)
{
WriteOnce.Error(ErrMsg.FormatString(ErrMsg.ID.RUNTIME_ERROR_NAMED, e.Message));
Logger.Error($"Runtime error: {e.StackTrace}");
}
else
{
WriteOnce.Error(ErrMsg.FormatString(ErrMsg.ID.RUNTIME_ERROR_PRELOG, e.Message));
}
}
catch (Exception e)
{
if (Logger != null)
{
WriteOnce.Error(ErrMsg.FormatString(ErrMsg.ID.RUNTIME_ERROR_UNNAMED));
Logger.Error($"Runtime error: {e.StackTrace}");
}
else
{
WriteOnce.Error(ErrMsg.FormatString(ErrMsg.ID.RUNTIME_ERROR_PRELOG, e.Message));
}
}
return(finalResult);
}
void UnZipAndProcess(string filename, ArchiveFileType archiveFileType)
{
//zip itself may be too huge for timely processing
if (new FileInfo(filename).Length > WARN_ZIP_FILE_SIZE)
{
WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_FILESIZE_WARN));
}
else
{
WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_PROCESSING));
}
LastUpdated = File.GetLastWriteTime(filename);
_appProfile.MetaData.PackageTypes.Add(ErrMsg.GetString(ErrMsg.ID.ANALYZE_COMPRESSED_FILETYPE));
try
{
IEnumerable <FileEntry> files = Extractor.ExtractFile(filename);
if (files.Count() > 0)
{
_appProfile.MetaData.TotalFiles += files.Count();//additive in case additional child zip files processed
foreach (FileEntry file in files)
{
//check for supported language
LanguageInfo languageInfo = new LanguageInfo();
if (FileChecksPassed(file.FullPath, ref languageInfo, file.Content.Length))
{
byte[] streamByteArray = file.Content.ToArray();
ProcessInMemory(file.FullPath, Encoding.UTF8.GetString(streamByteArray, 0, streamByteArray.Length), languageInfo);
}
}
}
else
{
throw new OpException(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_ERROR, filename));
}
}
catch (Exception e)
{
string errmsg = ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_ERROR, filename);
WriteOnce.Error(errmsg);
throw new Exception(errmsg + e.Message + "\n" + e.StackTrace);
}
}
public int Run()
{
bool issues = false;
WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_RUNNING, "Verify Rules"));
//load [each] rules file separately to report out where a failure is happening
RuleSet rules = new RuleSet(WriteOnce.Log);
IEnumerable <string> fileListing = new List <string>();
foreach (string rulePath in _rulePaths)
{
if (Directory.Exists(rulePath))
{
fileListing = Directory.EnumerateFiles(rulePath, "*.json", SearchOption.AllDirectories);
}
else if (File.Exists(rulePath) && Path.GetExtension(rulePath) == ".json")
{
fileListing = new List <string>()
{
new string(rulePath)
}
}
;
else
{
throw new OpException(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_RULE_PATH, rulePath));
}
//test loading each file
foreach (string filename in fileListing)
{
try
{
rules.AddFile(filename);
WriteOnce.Info(string.Format("Rule file added {0}", filename), true, WriteOnce.ConsoleVerbosity.High);
}
catch (Exception e)
{
WriteOnce.Error(string.Format("Rule file add failed {0}", filename));
WriteOnce.SafeLog(e.Message + "\n" + e.StackTrace, NLog.LogLevel.Error);
issues = true;
}
}
}
//option to write validating data
if (_arg_consoleVerbosityLevel == WriteOnce.ConsoleVerbosity.High)
{
WritePartialRuleDetails(rules);
}
//final status report
if (issues)
{
WriteOnce.Any(ErrMsg.GetString(ErrMsg.ID.VERIFY_RULES_RESULTS_FAIL), true, ConsoleColor.Red, WriteOnce.ConsoleVerbosity.Low);
}
else
{
WriteOnce.Any(ErrMsg.GetString(ErrMsg.ID.VERIFY_RULES_RESULTS_SUCCESS), true, ConsoleColor.Green, WriteOnce.ConsoleVerbosity.Low);
}
WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_COMPLETED, "Verify Rules"));
WriteOnce.FlushAll();
if (!String.IsNullOrEmpty(_arg_outputFile))
{
WriteOnce.Any(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_OUTPUT_FILE, _arg_outputFile), true, ConsoleColor.Gray, WriteOnce.ConsoleVerbosity.Low);
}
return(issues ? (int)ExitCode.NotVerified : (int)ExitCode.Verified);
}
/// <summary>
/// Main entry point to start analysis; handles setting up rules, directory enumeration
/// file type detection and handoff
/// Pre: All Configure Methods have been called already and we are ready to SCAN
/// </summary>
/// <returns></returns>
public int Run()
{
WriteOnce.SafeLog("AnalyzeCommand::Run", LogLevel.Trace);
DateTime start = DateTime.Now;
WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_RUNNING, "Analyze"));
_appProfile.MetaData.TotalFiles = _srcfileList.Count();//updated for zipped files later
// Iterate through all files and process against rules
foreach (string filename in _srcfileList)
{
//exclude sample, test or similar files by default or as specified in exclusion list
if (!_arg_allowSampleFiles && _fileExclusionList.Any(v => filename.ToLower().Contains(v)))
{
WriteOnce.SafeLog("Part of excluded list: " + filename, LogLevel.Trace);
WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILESIZE_SKIPPED, filename), LogLevel.Trace);
_appProfile.MetaData.FilesSkipped++;
continue;
}
ArchiveFileType archiveFileType = MiniMagic.DetectFileType(filename);
if (archiveFileType == ArchiveFileType.UNKNOWN)
{
ProcessAsFile(filename);
}
else
{
UnZipAndProcess(filename, archiveFileType);
}
}
WriteOnce.General("\r" + ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILES_PROCESSED_PCNT, 100));
WriteOnce.Operation(ErrMsg.GetString(ErrMsg.ID.CMD_PREPARING_REPORT));
//Prepare report results
_appProfile.MetaData.LastUpdated = LastUpdated.ToString();
_appProfile.DateScanned = DateScanned.ToString();
_appProfile.PrepareReport();
TimeSpan timeSpan = start - DateTime.Now;
WriteOnce.SafeLog(String.Format("Processing time: seconds:{0}", timeSpan.TotalSeconds * -1), LogLevel.Trace);
FlushAll();
//wrapup result status
if (_appProfile.MetaData.TotalFiles == _appProfile.MetaData.FilesSkipped)
{
WriteOnce.Error(ErrMsg.GetString(ErrMsg.ID.ANALYZE_NOSUPPORTED_FILETYPES));
}
else if (_appProfile.MatchList.Count == 0)
{
WriteOnce.Error(ErrMsg.GetString(ErrMsg.ID.ANALYZE_NOPATTERNS));
}
else
{
WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_COMPLETED, "Analyze"));
}
return(_appProfile.MatchList.Count() == 0 ? (int)ExitCode.NoMatches :
(int)ExitCode.MatchesFound);
}