/// <summary> /// Main entry point to start analysis; handles setting up rules, directory enumeration /// file type detection and handoff /// Pre: All Configure Methods have been called already and we are ready to SCAN /// </summary> /// <returns></returns> public int Run() { WriteOnce.SafeLog("AnalyzeCommand::Run", LogLevel.Trace); DateTime start = DateTime.Now; WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_RUNNING, "Analyze")); _appProfile.MetaData.TotalFiles = _srcfileList.Count();//updated for zipped files later // Iterate through all files and process against rules foreach (string filename in _srcfileList) { var fileExtension = new FileInfo(filename).Extension; if (COMPRESSED_EXTENSIONS.Any(fileExtension.Contains)) { UnZipAndProcess(filename); //determine if file is a compressed item to unpackage for processing } else { ProcessAsFile(filename); } } WriteOnce.General("\r" + ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILES_PROCESSED_PCNT, 100)); WriteOnce.Operation(ErrMsg.GetString(ErrMsg.ID.CMD_PREPARING_REPORT)); //Prepare report results _appProfile.MetaData.LastUpdated = LastUpdated.ToString(); _appProfile.DateScanned = DateScanned.ToString(); _appProfile.PrepareReport(); TimeSpan timeSpan = start - DateTime.Now; WriteOnce.SafeLog(String.Format("Processing time: seconds:{0}", timeSpan.TotalSeconds * -1), LogLevel.Trace); FlushAll(); //wrapup result status if (_appProfile.MetaData.TotalFiles == _appProfile.MetaData.FilesSkipped) { WriteOnce.Error(ErrMsg.GetString(ErrMsg.ID.ANALYZE_NOSUPPORTED_FILETYPES)); } else if (_appProfile.MatchList.Count == 0) { WriteOnce.Error(ErrMsg.GetString(ErrMsg.ID.ANALYZE_NOPATTERNS)); } else { WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_COMPLETED, "Analyze")); } //html report size warning if (_arg_fileFormat == "html" && new FileInfo("output.html").Length > MAX_HTML_REPORT_FILE_SIZE) { WriteOnce.Info(ErrMsg.GetString(ErrMsg.ID.ANALYZE_REPORTSIZE_WARN)); } return(_appProfile.MatchList.Count() == 0 ? (int)ExitCode.NoMatches : (int)ExitCode.MatchesFound); }
/// <summary> /// Main entry point to start analysis; handles setting up rules, directory enumeration /// file type detection and handoff /// Pre: All Configure Methods have been called already and we are ready to SCAN /// </summary> /// <returns></returns> public int Run() { WriteOnce.SafeLog("AnalyzeCommand::Run", LogLevel.Trace); DateTime start = DateTime.Now; WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_RUNNING, "Analyze")); _appProfile.MetaData.TotalFiles = _srcfileList.Count();//updated for zipped files later // Iterate through all files and process against rules foreach (string filename in _srcfileList) { ArchiveFileType archiveFileType = MiniMagic.DetectFileType(filename); if (archiveFileType == ArchiveFileType.UNKNOWN)//not a known zipped file type { ProcessAsFile(filename); } else { UnZipAndProcess(filename, archiveFileType); } } WriteOnce.General("\r" + ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILES_PROCESSED_PCNT, 100)); WriteOnce.Operation(ErrMsg.GetString(ErrMsg.ID.CMD_PREPARING_REPORT)); //Prepare report results _appProfile.MetaData.LastUpdated = LastUpdated.ToString(); _appProfile.DateScanned = DateScanned.ToString(); _appProfile.PrepareReport(); TimeSpan timeSpan = start - DateTime.Now; WriteOnce.SafeLog(String.Format("Processing time: seconds:{0}", timeSpan.TotalSeconds * -1), LogLevel.Trace); FlushAll(); //wrapup result status if (_appProfile.MetaData.TotalFiles == _appProfile.MetaData.FilesSkipped) { WriteOnce.Error(ErrMsg.GetString(ErrMsg.ID.ANALYZE_NOSUPPORTED_FILETYPES)); } else if (_appProfile.MatchList.Count == 0) { WriteOnce.Error(ErrMsg.GetString(ErrMsg.ID.ANALYZE_NOPATTERNS)); } else { WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_COMPLETED, "Analyze")); if (!_arg_autoBrowserOpen) { WriteOnce.Any(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_OUTPUT_FILE, "output.html")); } } return(_appProfile.MatchList.Count() == 0 ? (int)ExitCode.NoMatches : (int)ExitCode.MatchesFound); }
void UnZipAndProcess(string filename, ArchiveFileType archiveFileType) { //zip itself may be too huge for timely processing if (new FileInfo(filename).Length > WARN_ZIP_FILE_SIZE) { WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_FILESIZE_WARN)); } else { WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_PROCESSING)); } try { IEnumerable <FileEntry> files = Extractor.ExtractFile(filename); if (files.Count() > 0) { _appProfile.MetaData.TotalFiles += files.Count();//additive in case additional child zip files processed _appProfile.MetaData.PackageTypes.Add(ErrMsg.GetString(ErrMsg.ID.ANALYZE_COMPRESSED_FILETYPE)); foreach (FileEntry file in files) { if (file.Content.Length > MAX_FILESIZE) { WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILESIZE_SKIPPED, file.FullPath), LogLevel.Warn); _appProfile.MetaData.FilesSkipped++; continue; } //dup check vs Run() for zip contents; exclude sample, test or similar files by default or as specified in exclusion list if (!_arg_allowSampleFiles && _fileExclusionList.Any(v => file.FullPath.ToLower().Contains(v))) { WriteOnce.SafeLog("Part of excluded list: " + file.FullPath, LogLevel.Trace); WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILESIZE_SKIPPED, file.FullPath), LogLevel.Trace); _appProfile.MetaData.FilesSkipped++; continue; } WriteOnce.Log.Trace("processing zip file entry: " + file.FullPath); byte[] streamByteArray = file.Content.ToArray(); ProcessInMemory(file.FullPath, Encoding.UTF8.GetString(streamByteArray, 0, streamByteArray.Length), true); } } else { throw new OpException(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_ERROR, filename)); } } catch (Exception e) { string errmsg = ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_ERROR, filename); WriteOnce.Error(errmsg); throw new Exception(errmsg + e.Message + "\n" + e.StackTrace); } }
void UnZipAndProcess(string filename, ArchiveFileType archiveFileType) { // zip itself may be in excluded list i.e. sample, test or similar unless ignore filter requested if (_fileExclusionList.Any(v => filename.ToLower().Contains(v))) { WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_EXCLUDED_TYPE_SKIPPED, filename), LogLevel.Warn); _appProfile.MetaData.FilesSkipped++; return; } //zip itself may be too huge for timely processing if (new FileInfo(filename).Length > WARN_ZIP_FILE_SIZE) { WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_FILESIZE_WARN)); } else { WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_PROCESSING)); } LastUpdated = File.GetLastWriteTime(filename); _appProfile.MetaData.PackageTypes.Add(ErrMsg.GetString(ErrMsg.ID.ANALYZE_COMPRESSED_FILETYPE)); try { IEnumerable <FileEntry> files = Extractor.ExtractFile(filename); if (files.Count() > 0) { _appProfile.MetaData.TotalFiles += files.Count();//additive in case additional child zip files processed foreach (FileEntry file in files) { //check uncompressed file passes standard checks LanguageInfo languageInfo = new LanguageInfo(); if (FileChecksPassed(file.FullPath, ref languageInfo, file.Content.Length)) { byte[] streamByteArray = file.Content.ToArray(); ProcessInMemory(file.FullPath, Encoding.UTF8.GetString(streamByteArray, 0, streamByteArray.Length), languageInfo); } } } else { WriteOnce.SafeLog(string.Format("Decompression found no files in {0}", filename), LogLevel.Warn);//zero results can be valid } } catch (Exception e) { string errmsg = ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_ERROR, filename); WriteOnce.Error(errmsg); throw new Exception(errmsg + e.Message + "\n" + e.StackTrace); } }
/// <summary> /// Program entry point which defines command verbs and options to running /// </summary> /// <param name="args"></param> static int Main(string[] args) { int finalResult = -1; WriteOnce.Verbosity = WriteOnce.ConsoleVerbosity.Medium; try { WriteOnce.Info(GetVersionString()); var argsResult = Parser.Default.ParseArguments <AnalyzeCommandOptions, TagDiffCommandOptions, TagTestCommandOptions, ExportTagsCommandOptions, VerifyRulesCommandOptions>(args) .MapResult( (AnalyzeCommandOptions opts) => RunAnalyzeCommand(opts), (TagDiffCommandOptions opts) => RunTagDiffCommand(opts), (TagTestCommandOptions opts) => RunTagTestCommand(opts), (ExportTagsCommandOptions opts) => RunExportTagsCommand(opts), (VerifyRulesCommandOptions opts) => RunVerifyRulesCommand(opts), errs => 1 ); finalResult = argsResult; } catch (OpException e) { if (Logger != null) { WriteOnce.Error(ErrMsg.FormatString(ErrMsg.ID.RUNTIME_ERROR_NAMED, e.Message)); Logger.Error($"Runtime error: {e.StackTrace}"); } else { WriteOnce.Error(ErrMsg.FormatString(ErrMsg.ID.RUNTIME_ERROR_PRELOG, e.Message)); } } catch (Exception e) { if (Logger != null) { WriteOnce.Error(ErrMsg.FormatString(ErrMsg.ID.RUNTIME_ERROR_UNNAMED)); Logger.Error($"Runtime error: {e.StackTrace}"); } else { WriteOnce.Error(ErrMsg.FormatString(ErrMsg.ID.RUNTIME_ERROR_PRELOG, e.Message)); } } return(finalResult); }
void UnZipAndProcess(string filename, ArchiveFileType archiveFileType) { //zip itself may be too huge for timely processing if (new FileInfo(filename).Length > WARN_ZIP_FILE_SIZE) { WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_FILESIZE_WARN)); } else { WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_PROCESSING)); } LastUpdated = File.GetLastWriteTime(filename); _appProfile.MetaData.PackageTypes.Add(ErrMsg.GetString(ErrMsg.ID.ANALYZE_COMPRESSED_FILETYPE)); try { IEnumerable <FileEntry> files = Extractor.ExtractFile(filename); if (files.Count() > 0) { _appProfile.MetaData.TotalFiles += files.Count();//additive in case additional child zip files processed foreach (FileEntry file in files) { //check for supported language LanguageInfo languageInfo = new LanguageInfo(); if (FileChecksPassed(file.FullPath, ref languageInfo, file.Content.Length)) { byte[] streamByteArray = file.Content.ToArray(); ProcessInMemory(file.FullPath, Encoding.UTF8.GetString(streamByteArray, 0, streamByteArray.Length), languageInfo); } } } else { throw new OpException(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_ERROR, filename)); } } catch (Exception e) { string errmsg = ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_ERROR, filename); WriteOnce.Error(errmsg); throw new Exception(errmsg + e.Message + "\n" + e.StackTrace); } }
public int Run() { bool issues = false; WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_RUNNING, "Verify Rules")); //load [each] rules file separately to report out where a failure is happening RuleSet rules = new RuleSet(WriteOnce.Log); IEnumerable <string> fileListing = new List <string>(); foreach (string rulePath in _rulePaths) { if (Directory.Exists(rulePath)) { fileListing = Directory.EnumerateFiles(rulePath, "*.json", SearchOption.AllDirectories); } else if (File.Exists(rulePath) && Path.GetExtension(rulePath) == ".json") { fileListing = new List <string>() { new string(rulePath) } } ; else { throw new OpException(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_RULE_PATH, rulePath)); } //test loading each file foreach (string filename in fileListing) { try { rules.AddFile(filename); WriteOnce.Info(string.Format("Rule file added {0}", filename), true, WriteOnce.ConsoleVerbosity.High); } catch (Exception e) { WriteOnce.Error(string.Format("Rule file add failed {0}", filename)); WriteOnce.SafeLog(e.Message + "\n" + e.StackTrace, NLog.LogLevel.Error); issues = true; } } } //option to write validating data if (_arg_consoleVerbosityLevel == WriteOnce.ConsoleVerbosity.High) { WritePartialRuleDetails(rules); } //final status report if (issues) { WriteOnce.Any(ErrMsg.GetString(ErrMsg.ID.VERIFY_RULES_RESULTS_FAIL), true, ConsoleColor.Red, WriteOnce.ConsoleVerbosity.Low); } else { WriteOnce.Any(ErrMsg.GetString(ErrMsg.ID.VERIFY_RULES_RESULTS_SUCCESS), true, ConsoleColor.Green, WriteOnce.ConsoleVerbosity.Low); } WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_COMPLETED, "Verify Rules")); WriteOnce.FlushAll(); if (!String.IsNullOrEmpty(_arg_outputFile)) { WriteOnce.Any(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_OUTPUT_FILE, _arg_outputFile), true, ConsoleColor.Gray, WriteOnce.ConsoleVerbosity.Low); } return(issues ? (int)ExitCode.NotVerified : (int)ExitCode.Verified); }
/// <summary> /// Main entry point to start analysis; handles setting up rules, directory enumeration /// file type detection and handoff /// Pre: All Configure Methods have been called already and we are ready to SCAN /// </summary> /// <returns></returns> public int Run() { WriteOnce.SafeLog("AnalyzeCommand::Run", LogLevel.Trace); DateTime start = DateTime.Now; WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_RUNNING, "Analyze")); _appProfile.MetaData.TotalFiles = _srcfileList.Count();//updated for zipped files later // Iterate through all files and process against rules foreach (string filename in _srcfileList) { //exclude sample, test or similar files by default or as specified in exclusion list if (!_arg_allowSampleFiles && _fileExclusionList.Any(v => filename.ToLower().Contains(v))) { WriteOnce.SafeLog("Part of excluded list: " + filename, LogLevel.Trace); WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILESIZE_SKIPPED, filename), LogLevel.Trace); _appProfile.MetaData.FilesSkipped++; continue; } ArchiveFileType archiveFileType = MiniMagic.DetectFileType(filename); if (archiveFileType == ArchiveFileType.UNKNOWN) { ProcessAsFile(filename); } else { UnZipAndProcess(filename, archiveFileType); } } WriteOnce.General("\r" + ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILES_PROCESSED_PCNT, 100)); WriteOnce.Operation(ErrMsg.GetString(ErrMsg.ID.CMD_PREPARING_REPORT)); //Prepare report results _appProfile.MetaData.LastUpdated = LastUpdated.ToString(); _appProfile.DateScanned = DateScanned.ToString(); _appProfile.PrepareReport(); TimeSpan timeSpan = start - DateTime.Now; WriteOnce.SafeLog(String.Format("Processing time: seconds:{0}", timeSpan.TotalSeconds * -1), LogLevel.Trace); FlushAll(); //wrapup result status if (_appProfile.MetaData.TotalFiles == _appProfile.MetaData.FilesSkipped) { WriteOnce.Error(ErrMsg.GetString(ErrMsg.ID.ANALYZE_NOSUPPORTED_FILETYPES)); } else if (_appProfile.MatchList.Count == 0) { WriteOnce.Error(ErrMsg.GetString(ErrMsg.ID.ANALYZE_NOPATTERNS)); } else { WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_COMPLETED, "Analyze")); } return(_appProfile.MatchList.Count() == 0 ? (int)ExitCode.NoMatches : (int)ExitCode.MatchesFound); }