/// <summary>
/// Simple validation on source path provided for scanning and preparation
/// </summary>
void ConfigSourcetoScan()
{
WriteOnce.SafeLog("AnalyzeCommand::ConfigSourcetoScan", LogLevel.Trace);
if (Directory.Exists(_arg_sourcePath))
{
try
{
_srcfileList = Directory.EnumerateFiles(_arg_sourcePath, "*.*", SearchOption.AllDirectories);
if (_srcfileList.Count() == 0)
{
throw new OpException(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_FILE_OR_DIR, _arg_sourcePath));
}
}
catch (Exception)
{
throw new OpException(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_FILE_OR_DIR, _arg_sourcePath));
}
}
else if (File.Exists(_arg_sourcePath)) //not a directory but make one for single flow
{
_srcfileList = new List <string>()
{
_arg_sourcePath
}
}
;
else
{
throw new OpException(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_FILE_OR_DIR, _arg_sourcePath));
}
}
/// <summary>
/// Expects user to supply all that apply
/// </summary>
void ConfigConfidenceFilters()
{
WriteOnce.SafeLog("AnalyzeCommand::ConfigConfidenceFilters", LogLevel.Trace);
//parse and verify confidence values
if (String.IsNullOrEmpty(_arg_confidenceFilters))
{
_arg_confidence = Confidence.High | Confidence.Medium; //excludes low by default
}
else
{
string[] confidences = _arg_confidenceFilters.Split(',');
foreach (string confidence in confidences)
{
Confidence single;
if (Enum.TryParse(confidence, true, out single))
{
_arg_confidence |= single;
}
else
{
throw new OpException(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_ARG_VALUE, "x"));
}
}
}
}
List <string> _fileExclusionList;//see exclusion list
public AnalyzeCommand(AnalyzeCommandOptions opts)
{
_arg_sourcePath = opts.SourcePath;
_arg_outputFile = opts.OutputFilePath;
_arg_fileFormat = opts.OutputFileFormat;
_arg_outputTextFormat = opts.TextOutputFormat;
_arg_outputUniqueTagsOnly = !opts.AllowDupTags;
_arg_allowSampleFiles = opts.AllowSampleFiles;
_arg_customRulesPath = opts.CustomRulesPath;
_arg_confidenceFilters = opts.ConfidenceFilters;
_arg_ignoreDefaultRules = opts.IgnoreDefaultRules;
_arg_simpleTagsOnly = opts.SimpleTagsOnly;
if (!Enum.TryParse(opts.ConsoleVerbosityLevel, true, out _arg_consoleVerbosityLevel))
{
throw new OpException(String.Format(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_ARG_VALUE, "-x")));
}
WriteOnce.Verbosity = _arg_consoleVerbosityLevel;
IgnoreMimeRegex = new Regex(@"^(audio|video)/.*$");
LastUpdated = DateTime.MinValue;
DateScanned = DateTime.Now;
ConfigOutput();
ConfigSourcetoScan();
ConfigConfidenceFilters();
ConfigRules();
_fileExclusionList = EXCLUDEMATCH_FILEPATH.ToList <string>();
}
/// Compares a set of rules against a source path...
/// Used for both RulesPresent and RulesNotePresent options
/// Focus is pass/fail not detailed comparison output -see Tagdiff for more
public TagTestCommand(TagTestCommandOptions opt)
{
_arg_srcPath = opt.SourcePath;
_arg_customRulesPath = opt.CustomRulesPath;
_arg_outputFile = opt.OutputFilePath;
if (!Enum.TryParse(opt.ConsoleVerbosityLevel, true, out _arg_consoleVerbosityLevel))
{
throw new OpException(String.Format(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_ARG_VALUE, "-x")));
}
WriteOnce.Verbosity = _arg_consoleVerbosityLevel;
if (string.IsNullOrEmpty(opt.TestType))
{
_arg_tagTestType = TagTestType.RulesPresent;
}
else if (!Enum.TryParse(opt.TestType, true, out _arg_tagTestType))
{
throw new OpException(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_ARG_VALUE, opt.TestType));
}
_arg_ignoreDefaultRules = opt.IgnoreDefaultRules;
_rulesSet = new RuleSet(Program.Logger);
if (string.IsNullOrEmpty(opt.CustomRulesPath) && _arg_ignoreDefaultRules)
{
throw new OpException(ErrMsg.GetString(ErrMsg.ID.CMD_NORULES_SPECIFIED));
}
ConfigureRules();
ConfigureOutput();
}
List <string> _fileExclusionList;//see exclusion list
public AnalyzeCommand(AnalyzeCommandOptions opts)
{
_arg_sourcePath = opts.SourcePath;
_arg_outputFile = opts.OutputFilePath;
_arg_fileFormat = opts.OutputFileFormat;
_arg_outputTextFormat = opts.TextOutputFormat;
_arg_outputUniqueTagsOnly = !opts.AllowDupTags;
_arg_customRulesPath = opts.CustomRulesPath;
_arg_confidenceFilters = opts.ConfidenceFilters;
_arg_ignoreDefaultRules = opts.IgnoreDefaultRules;
_arg_simpleTagsOnly = opts.SimpleTagsOnly;
_fileExclusionList = opts.FilePathExclusions.Split(",").ToList <string>();
if (_fileExclusionList.Contains("none") || _fileExclusionList.Contains("None"))
{
_fileExclusionList.Clear();
}
if (!Enum.TryParse(opts.ConsoleVerbosityLevel, true, out _arg_consoleVerbosityLevel))
{
throw new OpException(String.Format(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_ARG_VALUE, "-x")));
}
WriteOnce.Verbosity = _arg_consoleVerbosityLevel;
LastUpdated = DateTime.MinValue;
DateScanned = DateTime.Now;
ConfigOutput();
ConfigSourcetoScan();
ConfigConfidenceFilters();
ConfigRules();
_uniqueTagsControl = new HashSet <string>();
}
public static Writer GetWriter(string writerName, string defaultWritter, string format = null)
{
if (string.IsNullOrEmpty(writerName))
{
writerName = defaultWritter;
}
if (string.IsNullOrEmpty(writerName))
{
writerName = "text";
}
switch (writerName.ToLowerInvariant())
{
case "_dummy":
return(new DummyWriter());
case "json":
return(new JsonWriter());
case "text":
return(new SimpleTextWriter(format));
case "html":
return(new LiquidWriter());
default:
throw new OpException(String.Format(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_ARG_VALUE, "-f")));
}
}
/// <summary>
/// Main entry point to start analysis; handles setting up rules, directory enumeration
/// file type detection and handoff
/// Pre: All Configure Methods have been called already and we are ready to SCAN
/// </summary>
/// <returns></returns>
public int Run()
{
WriteOnce.SafeLog("AnalyzeCommand::Run", LogLevel.Trace);
DateTime start = DateTime.Now;
WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_RUNNING, "Analyze"));
_appProfile.MetaData.TotalFiles = _srcfileList.Count();//updated for zipped files later
// Iterate through all files and process against rules
foreach (string filename in _srcfileList)
{
var fileExtension = new FileInfo(filename).Extension;
if (COMPRESSED_EXTENSIONS.Any(fileExtension.Contains))
{
UnZipAndProcess(filename); //determine if file is a compressed item to unpackage for processing
}
else
{
ProcessAsFile(filename);
}
}
WriteOnce.General("\r" + ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILES_PROCESSED_PCNT, 100));
WriteOnce.Operation(ErrMsg.GetString(ErrMsg.ID.CMD_PREPARING_REPORT));
//Prepare report results
_appProfile.MetaData.LastUpdated = LastUpdated.ToString();
_appProfile.DateScanned = DateScanned.ToString();
_appProfile.PrepareReport();
TimeSpan timeSpan = start - DateTime.Now;
WriteOnce.SafeLog(String.Format("Processing time: seconds:{0}", timeSpan.TotalSeconds * -1), LogLevel.Trace);
FlushAll();
//wrapup result status
if (_appProfile.MetaData.TotalFiles == _appProfile.MetaData.FilesSkipped)
{
WriteOnce.Error(ErrMsg.GetString(ErrMsg.ID.ANALYZE_NOSUPPORTED_FILETYPES));
}
else if (_appProfile.MatchList.Count == 0)
{
WriteOnce.Error(ErrMsg.GetString(ErrMsg.ID.ANALYZE_NOPATTERNS));
}
else
{
WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_COMPLETED, "Analyze"));
}
//html report size warning
if (_arg_fileFormat == "html" && new FileInfo("output.html").Length > MAX_HTML_REPORT_FILE_SIZE)
{
WriteOnce.Info(ErrMsg.GetString(ErrMsg.ID.ANALYZE_REPORTSIZE_WARN));
}
return(_appProfile.MatchList.Count() == 0 ? (int)ExitCode.NoMatches :
(int)ExitCode.MatchesFound);
}
/// <summary>
/// Main entry point to start analysis; handles setting up rules, directory enumeration
/// file type detection and handoff
/// Pre: All Configure Methods have been called already and we are ready to SCAN
/// </summary>
/// <returns></returns>
public int Run()
{
WriteOnce.SafeLog("AnalyzeCommand::Run", LogLevel.Trace);
DateTime start = DateTime.Now;
WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_RUNNING, "Analyze"));
_appProfile.MetaData.TotalFiles = _srcfileList.Count();//updated for zipped files later
// Iterate through all files and process against rules
foreach (string filename in _srcfileList)
{
ArchiveFileType archiveFileType = MiniMagic.DetectFileType(filename);
if (archiveFileType == ArchiveFileType.UNKNOWN)//not a known zipped file type
{
ProcessAsFile(filename);
}
else
{
UnZipAndProcess(filename, archiveFileType);
}
}
WriteOnce.General("\r" + ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILES_PROCESSED_PCNT, 100));
WriteOnce.Operation(ErrMsg.GetString(ErrMsg.ID.CMD_PREPARING_REPORT));
//Prepare report results
_appProfile.MetaData.LastUpdated = LastUpdated.ToString();
_appProfile.DateScanned = DateScanned.ToString();
_appProfile.PrepareReport();
TimeSpan timeSpan = start - DateTime.Now;
WriteOnce.SafeLog(String.Format("Processing time: seconds:{0}", timeSpan.TotalSeconds * -1), LogLevel.Trace);
FlushAll();
//wrapup result status
if (_appProfile.MetaData.TotalFiles == _appProfile.MetaData.FilesSkipped)
{
WriteOnce.Error(ErrMsg.GetString(ErrMsg.ID.ANALYZE_NOSUPPORTED_FILETYPES));
}
else if (_appProfile.MatchList.Count == 0)
{
WriteOnce.Error(ErrMsg.GetString(ErrMsg.ID.ANALYZE_NOPATTERNS));
}
else
{
WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_COMPLETED, "Analyze"));
if (!_arg_autoBrowserOpen)
{
WriteOnce.Any(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_OUTPUT_FILE, "output.html"));
}
}
return(_appProfile.MatchList.Count() == 0 ? (int)ExitCode.NoMatches :
(int)ExitCode.MatchesFound);
}
void UnZipAndProcess(string filename, ArchiveFileType archiveFileType)
{
//zip itself may be too huge for timely processing
if (new FileInfo(filename).Length > WARN_ZIP_FILE_SIZE)
{
WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_FILESIZE_WARN));
}
else
{
WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_PROCESSING));
}
try
{
IEnumerable <FileEntry> files = Extractor.ExtractFile(filename);
if (files.Count() > 0)
{
_appProfile.MetaData.TotalFiles += files.Count();//additive in case additional child zip files processed
_appProfile.MetaData.PackageTypes.Add(ErrMsg.GetString(ErrMsg.ID.ANALYZE_COMPRESSED_FILETYPE));
foreach (FileEntry file in files)
{
if (file.Content.Length > MAX_FILESIZE)
{
WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILESIZE_SKIPPED, file.FullPath), LogLevel.Warn);
_appProfile.MetaData.FilesSkipped++;
continue;
}
//dup check vs Run() for zip contents; exclude sample, test or similar files by default or as specified in exclusion list
if (!_arg_allowSampleFiles && _fileExclusionList.Any(v => file.FullPath.ToLower().Contains(v)))
{
WriteOnce.SafeLog("Part of excluded list: " + file.FullPath, LogLevel.Trace);
WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILESIZE_SKIPPED, file.FullPath), LogLevel.Trace);
_appProfile.MetaData.FilesSkipped++;
continue;
}
WriteOnce.Log.Trace("processing zip file entry: " + file.FullPath);
byte[] streamByteArray = file.Content.ToArray();
ProcessInMemory(file.FullPath, Encoding.UTF8.GetString(streamByteArray, 0, streamByteArray.Length), true);
}
}
else
{
throw new OpException(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_ERROR, filename));
}
}
catch (Exception e)
{
string errmsg = ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_ERROR, filename);
WriteOnce.Error(errmsg);
throw new Exception(errmsg + e.Message + "\n" + e.StackTrace);
}
}
void UnZipAndProcess(string filename, ArchiveFileType archiveFileType)
{
// zip itself may be in excluded list i.e. sample, test or similar unless ignore filter requested
if (_fileExclusionList.Any(v => filename.ToLower().Contains(v)))
{
WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_EXCLUDED_TYPE_SKIPPED, filename), LogLevel.Warn);
_appProfile.MetaData.FilesSkipped++;
return;
}
//zip itself may be too huge for timely processing
if (new FileInfo(filename).Length > WARN_ZIP_FILE_SIZE)
{
WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_FILESIZE_WARN));
}
else
{
WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_PROCESSING));
}
LastUpdated = File.GetLastWriteTime(filename);
_appProfile.MetaData.PackageTypes.Add(ErrMsg.GetString(ErrMsg.ID.ANALYZE_COMPRESSED_FILETYPE));
try
{
IEnumerable <FileEntry> files = Extractor.ExtractFile(filename);
if (files.Count() > 0)
{
_appProfile.MetaData.TotalFiles += files.Count();//additive in case additional child zip files processed
foreach (FileEntry file in files)
{
//check uncompressed file passes standard checks
LanguageInfo languageInfo = new LanguageInfo();
if (FileChecksPassed(file.FullPath, ref languageInfo, file.Content.Length))
{
byte[] streamByteArray = file.Content.ToArray();
ProcessInMemory(file.FullPath, Encoding.UTF8.GetString(streamByteArray, 0, streamByteArray.Length), languageInfo);
}
}
}
else
{
WriteOnce.SafeLog(string.Format("Decompression found no files in {0}", filename), LogLevel.Warn);//zero results can be valid
}
}
catch (Exception e)
{
string errmsg = ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_ERROR, filename);
WriteOnce.Error(errmsg);
throw new Exception(errmsg + e.Message + "\n" + e.StackTrace);
}
}
/// <summary>
/// Program entry point which defines command verbs and options to running
/// </summary>
/// <param name="args"></param>
static int Main(string[] args)
{
int finalResult = -1;
WriteOnce.Verbosity = WriteOnce.ConsoleVerbosity.Medium;
try
{
WriteOnce.Info(GetVersionString());
var argsResult = Parser.Default.ParseArguments <AnalyzeCommandOptions,
TagDiffCommandOptions,
TagTestCommandOptions,
ExportTagsCommandOptions,
VerifyRulesCommandOptions>(args)
.MapResult(
(AnalyzeCommandOptions opts) => RunAnalyzeCommand(opts),
(TagDiffCommandOptions opts) => RunTagDiffCommand(opts),
(TagTestCommandOptions opts) => RunTagTestCommand(opts),
(ExportTagsCommandOptions opts) => RunExportTagsCommand(opts),
(VerifyRulesCommandOptions opts) => RunVerifyRulesCommand(opts),
errs => 1
);
finalResult = argsResult;
}
catch (OpException e)
{
if (Logger != null)
{
WriteOnce.Error(ErrMsg.FormatString(ErrMsg.ID.RUNTIME_ERROR_NAMED, e.Message));
Logger.Error($"Runtime error: {e.StackTrace}");
}
else
{
WriteOnce.Error(ErrMsg.FormatString(ErrMsg.ID.RUNTIME_ERROR_PRELOG, e.Message));
}
}
catch (Exception e)
{
if (Logger != null)
{
WriteOnce.Error(ErrMsg.FormatString(ErrMsg.ID.RUNTIME_ERROR_UNNAMED));
Logger.Error($"Runtime error: {e.StackTrace}");
}
else
{
WriteOnce.Error(ErrMsg.FormatString(ErrMsg.ID.RUNTIME_ERROR_PRELOG, e.Message));
}
}
return(finalResult);
}
public VerifyRulesCommand(VerifyRulesCommandOptions opt)
{
_arg_customRulesPath = opt.CustomRulesPath;
_arg_ignoreDefaultRules = opt.IgnoreDefaultRules;
_arg_outputFile = opt.OutputFilePath;
if (!Enum.TryParse(opt.ConsoleVerbosityLevel, true, out _arg_consoleVerbosityLevel))
{
throw new OpException(String.Format(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_ARG_VALUE, "-x")));
}
WriteOnce.Verbosity = _arg_consoleVerbosityLevel;
ConfigureOutput();
ConfigRules();
}
/// <summary>
/// Wrapper for files that are on disk and ready to read vs unzipped files which are not to allow separation of core
/// scan evaluation for use by decompression methods as well
/// </summary>
/// <param name="filename"></param>
void ProcessAsFile(string filename)
{
//check for supported language
LanguageInfo languageInfo = new LanguageInfo();
if (FileChecksPassed(filename, ref languageInfo))
{
LastUpdated = File.GetLastWriteTime(filename);
_appProfile.MetaData.PackageTypes.Add(ErrMsg.GetString(ErrMsg.ID.ANALYZE_UNCOMPRESSED_FILETYPE));
string fileText = File.ReadAllText(filename);
ProcessInMemory(filename, fileText, languageInfo);
}
}
/// <summary>
/// Wrapper for files that are on disk and ready to read to allow separation of core
/// scan evaluation for use by decompression methods as well
/// </summary>
/// <param name="filename"></param>
void ProcessAsFile(string filename)
{
if (File.Exists(filename))
{
_appProfile.MetaData.FileNames.Add(filename);
_appProfile.MetaData.PackageTypes.Add(ErrMsg.GetString(ErrMsg.ID.ANALYZE_UNCOMPRESSED_FILETYPE));
string fileText = File.ReadAllText(filename);
ProcessInMemory(filename, fileText);
}
else
{
throw new OpException(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_FILE_OR_DIR, filename));
}
}
void UnZipAndProcess(string filename, ArchiveFileType archiveFileType)
{
//zip itself may be too huge for timely processing
if (new FileInfo(filename).Length > WARN_ZIP_FILE_SIZE)
{
WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_FILESIZE_WARN));
}
else
{
WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_PROCESSING));
}
LastUpdated = File.GetLastWriteTime(filename);
_appProfile.MetaData.PackageTypes.Add(ErrMsg.GetString(ErrMsg.ID.ANALYZE_COMPRESSED_FILETYPE));
try
{
IEnumerable <FileEntry> files = Extractor.ExtractFile(filename);
if (files.Count() > 0)
{
_appProfile.MetaData.TotalFiles += files.Count();//additive in case additional child zip files processed
foreach (FileEntry file in files)
{
//check for supported language
LanguageInfo languageInfo = new LanguageInfo();
if (FileChecksPassed(file.FullPath, ref languageInfo, file.Content.Length))
{
byte[] streamByteArray = file.Content.ToArray();
ProcessInMemory(file.FullPath, Encoding.UTF8.GetString(streamByteArray, 0, streamByteArray.Length), languageInfo);
}
}
}
else
{
throw new OpException(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_ERROR, filename));
}
}
catch (Exception e)
{
string errmsg = ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_ERROR, filename);
WriteOnce.Error(errmsg);
throw new Exception(errmsg + e.Message + "\n" + e.StackTrace);
}
}
void ProcessTarGzFile(string filename)
{
WriteOnce.SafeLog(string.Format("Analyzing .tar.gz file: [{0}]", filename), LogLevel.Trace);
_appProfile.MetaData.TotalFiles = GetTarGzFileCount(filename);
using (var fileStream = new FileStream(filename, FileMode.Open, FileAccess.Read))
using (var gzipStream = new GZipInputStream(fileStream))
using (var memoryStream = new MemoryStream())
{
var tarStream = new TarInputStream(gzipStream);
TarEntry tarEntry;
while ((tarEntry = tarStream.GetNextEntry()) != null)
{
if (tarEntry.IsDirectory)
{
continue;
}
tarStream.CopyEntryContents(memoryStream);
if (tarEntry.Size > MAX_FILESIZE)
{
_appProfile.MetaData.FilesSkipped++;
WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILESIZE_SKIPPED, tarEntry.Name), LogLevel.Warn);
tarStream.Close();
continue;
}
var mimeType = MimeTypeMap.GetMimeType(Path.GetExtension(tarEntry.Name));
if (IgnoreMimeRegex.IsMatch(mimeType) && new FileInfo(tarEntry.Name).Extension != ".ts")
{
_appProfile.MetaData.FilesSkipped++;
WriteOnce.SafeLog(string.Format("Ignoring tar entry [{0}]", tarEntry.Name), LogLevel.Error);
}
else
{
//file name may contain slashes; remove prior to call
byte[] streamByteArray = memoryStream.ToArray();
ProcessInMemory(Path.GetFileName(tarEntry.Name), Encoding.UTF8.GetString(streamByteArray, 0, streamByteArray.Length));
}
memoryStream.SetLength(0); // Clear out the stream
}
tarStream.Close();
}
}
public static void OpenBrowser(string url)
{
if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
try
{
Process.Start(new ProcessStartInfo("cmd", $"/c start {url}"));
WriteOnce.General(ErrMsg.GetString(ErrMsg.ID.BROWSER_START_SUCCESS));
}
catch (Exception)
{
WriteOnce.General(ErrMsg.GetString(ErrMsg.ID.BROWSER_START_FAIL));
}
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
{
if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("BROWSER")))
{
try
{
Process.Start("xdg-open", url);
WriteOnce.General(ErrMsg.GetString(ErrMsg.ID.BROWSER_START_SUCCESS));
}
catch (Exception)
{
WriteOnce.SafeLog("Unable to open browser using BROWSER environment var", NLog.LogLevel.Error);
}
}
else
{
WriteOnce.General(ErrMsg.GetString(ErrMsg.ID.BROWSER_ENVIRONMENT_VAR));
}
}
else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
{
try
{
Process.Start("open", url);
WriteOnce.General(ErrMsg.GetString(ErrMsg.ID.BROWSER_START_SUCCESS));
}
catch (Exception)
{
WriteOnce.General(ErrMsg.GetString(ErrMsg.ID.BROWSER_START_FAIL));
}
}
}
void ProcessZipFile(string filename)
{
WriteOnce.SafeLog(string.Format("Analyzing .zip file: [{0}])", filename), LogLevel.Trace);
ZipFile zipFile;
using (var fileStream = new FileStream(filename, FileMode.Open, FileAccess.Read))
using (var memoryStream = new MemoryStream())
{
zipFile = new ZipFile(fileStream);
_appProfile.MetaData.TotalFiles = (int)zipFile.Count;
foreach (ZipEntry zipEntry in zipFile)
{
if (zipEntry.IsDirectory)
{
continue;
}
byte[] buffer = new byte[4096];
var zipStream = zipFile.GetInputStream(zipEntry);
if (zipEntry.Size > MAX_FILESIZE)
{
_appProfile.MetaData.FilesSkipped++;
WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILESIZE_SKIPPED, zipEntry.Name), LogLevel.Warn);
zipFile.Close();
continue;
}
StreamUtils.Copy(zipStream, memoryStream, buffer);
var mimeType = MimeTypeMap.GetMimeType(Path.GetExtension(zipEntry.Name));
if (IgnoreMimeRegex.IsMatch(mimeType) && new FileInfo(zipEntry.Name).Extension != ".ts")
{
_appProfile.MetaData.FilesSkipped++;
WriteOnce.SafeLog(string.Format("Ignoring zip entry [{0}]", zipEntry.Name), LogLevel.Error);
}
else
{
byte[] streamByteArray = memoryStream.ToArray();
ProcessInMemory(Path.GetFileName(zipEntry.Name), Encoding.UTF8.GetString(streamByteArray, 0, streamByteArray.Length));
}
memoryStream.SetLength(0); // Clear out the stream
}
zipFile.Close();
}
}
public TagDiffCommand(TagDiffCommandOptions opt)
{
_arg_src1 = opt.SourcePath1;
_arg_src2 = opt.SourcePath2;
_arg_rulesPath = opt.CustomRulesPath;
_arg_outputFile = opt.OutputFilePath;
if (!Enum.TryParse(opt.ConsoleVerbosityLevel, true, out _arg_consoleVerbosityLevel))
{
throw new OpException(String.Format(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_ARG_VALUE, "-x")));
}
WriteOnce.Verbosity = _arg_consoleVerbosityLevel;
if (!Enum.TryParse(opt.TestType, true, out _arg_tagTestType))
{
throw new OpException(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_ARG_VALUE, opt.TestType));
}
_arg_ignoreDefault = opt.IgnoreDefaultRules;
}
static void SetupLogging(string logFilePath, string logFileLevel)
{
var config = new NLog.Config.LoggingConfiguration();
if (String.IsNullOrEmpty(logFilePath))
{
logFilePath = Utils.GetPath(Utils.AppPath.defaultLog);
//if using default app log path clean up previous for convenience in reading
if (File.Exists(logFilePath))
{
File.Delete(logFilePath);
}
}
LogLevel log_level = LogLevel.Error;//default
try
{
log_level = LogLevel.FromString(logFileLevel);
}
catch (Exception)
{
throw new OpException(String.Format(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_ARG_VALUE, "-v")));
}
using (var fileTarget = new FileTarget()
{
Name = "LogFile",
FileName = logFilePath,
Layout = @"${date:universalTime=true:format=s} ${threadid} ${level:uppercase=true} - ${message}",
ForceMutexConcurrentWrites = true
})
{
config.AddTarget(fileTarget);
config.LoggingRules.Add(new LoggingRule("*", log_level, fileTarget));
}
LogManager.Configuration = config;
Logger = LogManager.GetCurrentClassLogger();
Logger.Info("[" + DateTime.Now.ToLocalTime() + "] //////////////////////////////////////////////////////////");
WriteOnce.Log = Logger;//allows log to be written to as well as console or output file
}
void ConfigRules()
{
_rulePaths = new List <string>();
if (!_arg_ignoreDefaultRules)
{
_rulePaths.Add(Utils.GetPath(Utils.AppPath.defaultRules));
}
if (!string.IsNullOrEmpty(_arg_customRulesPath))
{
_rulePaths.Add(_arg_customRulesPath);
}
if (_rulePaths.Count == 0)
{
throw new OpException(ErrMsg.GetString(ErrMsg.ID.CMD_NORULES_SPECIFIED));
}
//validation of paths is delayed for Run in this cmd
}
public void FlushAll()
{
if (_outputWriter != null)
{
_outputWriter.WriteApp(_appProfile);
if (_outputWriter.TextWriter != null && _arg_fileFormat != "html")
{
_outputWriter.FlushAndClose();//not required for html formal i.e. multiple files already closed
_outputWriter = null;
if (!String.IsNullOrEmpty(_arg_outputFile))
{
WriteOnce.Any(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_OUTPUT_FILE, _arg_outputFile));
}
else
{
WriteOnce.NewLine();
}
}
}
}
/// <summary>
/// Common validation called by ProcessAsFile and UnzipAndProcess to ensure same order and checks made
/// </summary>
/// <param name="filePath"></param>
/// <param name="languageInfo"></param>
/// <param name="fileLength">should be > zero if called from unzip method</param>
/// <returns></returns>
bool FileChecksPassed(string filePath, ref LanguageInfo languageInfo, long fileLength = 0)
{
_appProfile.MetaData.FileExtensions.Add(Path.GetExtension(filePath).Replace('.', ' ').TrimStart());
// 1. Skip files written in unknown language
if (!Language.FromFileName(filePath, ref languageInfo))
{
WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_LANGUAGE_NOTFOUND, filePath), LogLevel.Warn);
_appProfile.MetaData.FilesSkipped++;
return(false);
}
_appProfile.MetaData.AddLanguage(languageInfo.Name);
// 2. Skip excluded files i.e. sample, test or similar unless ignore filter requested
if (_fileExclusionList.Any(v => filePath.ToLower().Contains(v)))
{
WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_EXCLUDED_TYPE_SKIPPED, filePath), LogLevel.Warn);
_appProfile.MetaData.FilesSkipped++;
return(false);
}
// 3. Skip if exceeds file size limits
try
{
fileLength = fileLength <= 0 ? new FileInfo(filePath).Length : fileLength;
if (fileLength > MAX_FILESIZE)
{
WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILESIZE_SKIPPED, filePath), LogLevel.Warn);
_appProfile.MetaData.FilesSkipped++;
return(false);
}
}
catch (Exception)
{
throw new OpException(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_FILE_OR_DIR, filePath));
}
return(true);
}
bool CompareTags(string[] fileTags1, string[] fileTags2)
{
bool found = true;
//are all tags in file1 found in file2
foreach (string s1 in fileTags1)
{
if (!fileTags2.Contains(s1))
{
found = false;
WriteOnce.Result(s1, true, WriteOnce.ConsoleVerbosity.High);
}
}
//none missing
if (found)
{
WriteOnce.Result(ErrMsg.GetString(ErrMsg.ID.TAGTEST_RESULTS_NONE), true, WriteOnce.ConsoleVerbosity.High);
}
return(found);
}
public int Run()
{
WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_RUNNING, "Exporttags"));
SortedDictionary <string, string> uniqueTags = new SortedDictionary <string, string>();
foreach (Rule r in _rules)
{
//builds a list of unique tags
foreach (string t in r.Tags)
{
if (uniqueTags.ContainsKey(t))
{
continue;
}
else
{
uniqueTags.Add(t, t);
}
}
}
//separate loop so results are sorted (Sorted type)
foreach (string s in uniqueTags.Values)
{
WriteOnce.Result(s, true);
}
WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_COMPLETED, "Exporttags"));
WriteOnce.FlushAll();
if (!String.IsNullOrEmpty(_arg_outputFile))
{
WriteOnce.Any(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_OUTPUT_FILE, _arg_outputFile), true, ConsoleColor.Gray, WriteOnce.ConsoleVerbosity.Low);
}
return((int)ExitCode.Success);
}
/// <summary>
/// Wrapper for files that are on disk and ready to read to allow separation of core
/// scan evaluation for use by decompression methods as well
/// </summary>
/// <param name="filename"></param>
void ProcessAsFile(string filename)
{
try
{
if (new FileInfo(filename).Length > MAX_FILESIZE)
{
WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILESIZE_SKIPPED, filename), LogLevel.Warn);
_appProfile.MetaData.FilesSkipped++;
return;
}
}
catch (Exception)
{
throw new OpException(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_FILE_OR_DIR, filename));
}
_appProfile.MetaData.FileNames.Add(filename);
_appProfile.MetaData.PackageTypes.Add(ErrMsg.GetString(ErrMsg.ID.ANALYZE_UNCOMPRESSED_FILETYPE));
string fileText = File.ReadAllText(filename);
ProcessInMemory(filename, fileText);
}
void ConfigRules()
{
List <string> rulePaths = new List <string>();
if (!_arg_ignoreDefaultRules)
{
rulePaths.Add(Utils.GetPath(Utils.AppPath.defaultRules));
}
if (!string.IsNullOrEmpty(_arg_customRulesPath))
{
rulePaths.Add(_arg_customRulesPath);
}
foreach (string rulePath in rulePaths)
{
if (Directory.Exists(rulePath))
{
_rules.AddDirectory(rulePath);
}
else if (File.Exists(rulePath))
{
_rules.AddFile(rulePath);
}
else
{
throw new OpException(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_RULE_PATH, rulePath));
}
}
//error check based on ruleset not path enumeration
if (_rules.Count() == 0)
{
throw new OpException(ErrMsg.GetString(ErrMsg.ID.CMD_NORULES_SPECIFIED));
}
}
void UnZipAndProcess(string filename)
{
if (!File.Exists(filename))
{
throw new OpException(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_FILE_OR_DIR, filename));
}
// Ignore images and other junk like that
var fileExtension = new FileInfo(filename).Extension;
var mimeType = MimeTypeMap.GetMimeType(fileExtension);
bool mimeMatch = false;
if (!IgnoreMimeRegex.IsMatch(mimeType))
{
var isValidExtension = COMPRESSED_EXTENSIONS.Any(fileExtension.Contains);
if (isValidExtension || fileExtension == "ts")
{
mimeMatch = true;
}
else if (mimeType.Contains("zip", StringComparison.CurrentCultureIgnoreCase) || // Should have been caught in file extensions above, but still OK.
mimeType.Contains("tar", StringComparison.CurrentCultureIgnoreCase) ||
mimeType.Contains("compressed", StringComparison.CurrentCultureIgnoreCase))
{
mimeMatch = true;
}
if (mimeMatch)
{
// Now process the file
switch (fileExtension)
{
case ".tgz":
ProcessTarGzFile(filename);
break;
case ".gz":
if (filename.Contains(".tar.gz"))
{
ProcessTarGzFile(filename);
}
else
{
WriteOnce.SafeLog("no support for .gz unless .tar.gz: " + fileExtension, LogLevel.Warn);
_appProfile.MetaData.PackageTypes.Add("compressed-unsupported");
}
break;
case ".jar":
case ".zip":
ProcessZipFile(filename);
break;
case ".gem":
case ".tar":
case ".nupkg":
WriteOnce.SafeLog($"Processing of {fileExtension} not implemented yet.", LogLevel.Warn);
break;
default:
WriteOnce.SafeLog("no support for compressed type: " + fileExtension, LogLevel.Warn);
break;
}
_appProfile.MetaData.PackageTypes.Add("compressed");
}
else
{
_appProfile.MetaData.PackageTypes.Add("compressed-unsupported");
}
}
}
/// <summary>
/// Main WORKHORSE for analyzing file; called from file based or decompression functions
/// </summary>
/// <param name="filename"></param>
/// <param name="fileText"></param>
void ProcessInMemory(string filePath, string fileText)
{
#region quickvalidation
if (fileText.Length > MAX_FILESIZE)
{
WriteOnce.SafeLog("File too large: " + filePath, LogLevel.Trace);
WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILESIZE_SKIPPED, filePath), LogLevel.Error);
_appProfile.MetaData.FilesSkipped++;
return;
}
if (!_arg_allowSampleFiles && _fileExclusionList.Any(v => filePath.Contains(v)))
{
WriteOnce.SafeLog("Part of excluded list: " + filePath, LogLevel.Trace);
WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILESIZE_SKIPPED, filePath), LogLevel.Error);
_appProfile.MetaData.FilesSkipped++;
return;
}
//determine if file is a compressed item to unpackage for processing
string language = Language.FromFileName(filePath);
// Skip files written in unknown language
if (string.IsNullOrEmpty(language))
{
WriteOnce.SafeLog("Language not found for file: " + filePath, LogLevel.Trace);
language = Path.GetFileName(filePath);
_appProfile.MetaData.FilesSkipped++;
return;
}
else
{
WriteOnce.SafeLog("Preparing to process file: " + filePath, LogLevel.Trace);
}
#endregion
#region minorRollupTrackingAndProgress
_appProfile.MetaData.FilesAnalyzed++;
_appProfile.MetaData.AddLanguage(language);
_appProfile.MetaData.FileExtensions.Add(Path.GetExtension(filePath).Replace('.', ' ').TrimStart());
LastUpdated = File.GetLastWriteTime(filePath);
int totalFilesReviewed = _appProfile.MetaData.FilesAnalyzed + _appProfile.MetaData.FilesSkipped;
int percentCompleted = (int)((float)totalFilesReviewed / (float)_appProfile.MetaData.TotalFiles * 100);
WriteOnce.General("\r" + ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILES_PROCESSED_PCNT, percentCompleted), false);
#endregion
//process file against rules
Issue[] matches = _rulesProcessor.Analyze(fileText, language);
//if any matches found for this file...
if (matches.Count() > 0)
{
_appProfile.MetaData.FilesAffected++;
_appProfile.MetaData.TotalMatchesCount += matches.Count();
HashSet <string> uniqueTagsControl = new HashSet <string>();
// Iterate through each match issue
foreach (Issue match in matches)
{
WriteOnce.SafeLog(string.Format("Processing pattern matches for ruleId {0}, ruleName {1} file {2}", match.Rule.Id, match.Rule.Name, filePath), LogLevel.Trace);
//maintain a list of unique tags; multi-purpose but primarily for filtering -u option
bool dupTagFound = false;
foreach (string t in match.Rule.Tags)
{
dupTagFound = !uniqueTagsControl.Add(t);
}
//save all unique dependendencies even if Dependency tag pattern is not-unique
var tagPatternRegex = new Regex("Dependency.SourceInclude", RegexOptions.IgnoreCase);
String textMatch;
if (match.Rule.Tags.Any(v => tagPatternRegex.IsMatch(v)))
{
textMatch = ExtractDependency(fileText, match.Boundary.Index, match.PatternMatch, language);
}
else
{
textMatch = ExtractTextSample(fileText, match.Boundary.Index, match.Boundary.Length);
}
//wrap rule issue result to add metadata
MatchRecord record = new MatchRecord()
{
Filename = filePath,
Language = language,
Filesize = fileText.Length,
TextSample = textMatch,
Excerpt = ExtractExcerpt(fileText, match.StartLocation.Line),
Issue = match
};
//preserve issue level characteristics as rolled up meta data of interest
bool valid = _appProfile.MetaData.AddStandardProperties(record);
//bail after extracting any dependency unique items IF user requested
if (_arg_outputUniqueTagsOnly && dupTagFound)
{
continue;
}
else if (valid)
{
_appProfile.MatchList.Add(record);
}
}
}
else
{
WriteOnce.SafeLog("No pattern matches detected for file: " + filePath, LogLevel.Trace);
}
}
/// <summary>
/// Add default and/or custom rules paths
/// Iterate paths and add to ruleset
/// </summary>
void ConfigRules()
{
WriteOnce.SafeLog("AnalyzeCommand::ConfigRules", LogLevel.Trace);
RuleSet rulesSet = new RuleSet(Program.Logger);
List <string> rulePaths = new List <string>();
if (!_arg_ignoreDefaultRules)
{
rulePaths.Add(Utils.GetPath(Utils.AppPath.defaultRules));
}
if (!string.IsNullOrEmpty(_arg_customRulesPath))
{
rulePaths.Add(_arg_customRulesPath);
}
foreach (string rulePath in rulePaths)
{
if (Directory.Exists(rulePath))
{
rulesSet.AddDirectory(rulePath);
}
else if (File.Exists(rulePath))
{
rulesSet.AddFile(rulePath);
}
else
{
throw new OpException(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_RULE_PATH, rulePath));
}
}
//error check based on ruleset not path enumeration
if (rulesSet.Count() == 0)
{
throw new OpException(ErrMsg.GetString(ErrMsg.ID.CMD_NORULES_SPECIFIED));
}
//instantiate a RuleProcessor with the added rules and exception for dependency
_rulesProcessor = new RuleProcessor(rulesSet, _arg_confidence, _arg_outputUniqueTagsOnly, _arg_simpleTagsOnly, Program.Logger);
if (_arg_outputUniqueTagsOnly)
{
List <TagException> tagExceptions;
if (File.Exists(Utils.GetPath(Utils.AppPath.tagCounterPref)))
{
tagExceptions = JsonConvert.DeserializeObject <List <TagException> >(File.ReadAllText(Utils.GetPath(Utils.AppPath.tagCounterPref)));
string[] exceptions = new string[tagExceptions.Count];
for (int i = 0; i < tagExceptions.Count; i++)
{
exceptions[i] = tagExceptions[i].Tag;
}
_rulesProcessor.UniqueTagExceptions = exceptions;
}
}
_appProfile = new AppProfile(_arg_sourcePath, rulePaths, false, _arg_simpleTagsOnly, _arg_outputUniqueTagsOnly);
_appProfile.Args = "analyze -f " + _arg_fileFormat + " -u " + _arg_outputUniqueTagsOnly.ToString().ToLower() + " -v " +
WriteOnce.Verbosity.ToString() + " -x " + _arg_confidence + " -i " + _arg_ignoreDefaultRules.ToString().ToLower();
}
