예제 #1
0
        /// <summary>
        /// Simple validation on source path provided for scanning and preparation
        /// </summary>
        void ConfigSourcetoScan()
        {
            WriteOnce.SafeLog("AnalyzeCommand::ConfigSourcetoScan", LogLevel.Trace);

            if (Directory.Exists(_arg_sourcePath))
            {
                try
                {
                    _srcfileList = Directory.EnumerateFiles(_arg_sourcePath, "*.*", SearchOption.AllDirectories);
                    if (_srcfileList.Count() == 0)
                    {
                        throw new OpException(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_FILE_OR_DIR, _arg_sourcePath));
                    }
                }
                catch (Exception)
                {
                    throw new OpException(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_FILE_OR_DIR, _arg_sourcePath));
                }
            }
            else if (File.Exists(_arg_sourcePath)) //not a directory but make one for single flow
            {
                _srcfileList = new List <string>()
                {
                    _arg_sourcePath
                }
            }
            ;
            else
            {
                throw new OpException(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_FILE_OR_DIR, _arg_sourcePath));
            }
        }
예제 #2
0
 /// <summary>
 /// Expects user to supply all that apply
 /// </summary>
 void ConfigConfidenceFilters()
 {
     WriteOnce.SafeLog("AnalyzeCommand::ConfigConfidenceFilters", LogLevel.Trace);
     //parse and verify confidence values
     if (String.IsNullOrEmpty(_arg_confidenceFilters))
     {
         _arg_confidence = Confidence.High | Confidence.Medium; //excludes low by default
     }
     else
     {
         string[] confidences = _arg_confidenceFilters.Split(',');
         foreach (string confidence in confidences)
         {
             Confidence single;
             if (Enum.TryParse(confidence, true, out single))
             {
                 _arg_confidence |= single;
             }
             else
             {
                 throw new OpException(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_ARG_VALUE, "x"));
             }
         }
     }
 }
예제 #3
0
        List <string> _fileExclusionList;//see exclusion list


        public AnalyzeCommand(AnalyzeCommandOptions opts)
        {
            _arg_sourcePath           = opts.SourcePath;
            _arg_outputFile           = opts.OutputFilePath;
            _arg_fileFormat           = opts.OutputFileFormat;
            _arg_outputTextFormat     = opts.TextOutputFormat;
            _arg_outputUniqueTagsOnly = !opts.AllowDupTags;
            _arg_allowSampleFiles     = opts.AllowSampleFiles;
            _arg_customRulesPath      = opts.CustomRulesPath;
            _arg_confidenceFilters    = opts.ConfidenceFilters;
            _arg_ignoreDefaultRules   = opts.IgnoreDefaultRules;
            _arg_simpleTagsOnly       = opts.SimpleTagsOnly;
            if (!Enum.TryParse(opts.ConsoleVerbosityLevel, true, out _arg_consoleVerbosityLevel))
            {
                throw new OpException(String.Format(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_ARG_VALUE, "-x")));
            }
            WriteOnce.Verbosity = _arg_consoleVerbosityLevel;

            IgnoreMimeRegex = new Regex(@"^(audio|video)/.*$");

            LastUpdated = DateTime.MinValue;
            DateScanned = DateTime.Now;

            ConfigOutput();
            ConfigSourcetoScan();
            ConfigConfidenceFilters();
            ConfigRules();

            _fileExclusionList = EXCLUDEMATCH_FILEPATH.ToList <string>();
        }
예제 #4
0
        /// Compares a set of rules against a source path...
        /// Used for both RulesPresent and RulesNotePresent options
        /// Focus is pass/fail not detailed comparison output -see Tagdiff for more
        public TagTestCommand(TagTestCommandOptions opt)
        {
            _arg_srcPath         = opt.SourcePath;
            _arg_customRulesPath = opt.CustomRulesPath;
            _arg_outputFile      = opt.OutputFilePath;

            if (!Enum.TryParse(opt.ConsoleVerbosityLevel, true, out _arg_consoleVerbosityLevel))
            {
                throw new OpException(String.Format(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_ARG_VALUE, "-x")));
            }
            WriteOnce.Verbosity = _arg_consoleVerbosityLevel;

            if (string.IsNullOrEmpty(opt.TestType))
            {
                _arg_tagTestType = TagTestType.RulesPresent;
            }
            else if (!Enum.TryParse(opt.TestType, true, out _arg_tagTestType))
            {
                throw new OpException(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_ARG_VALUE, opt.TestType));
            }

            _arg_ignoreDefaultRules = opt.IgnoreDefaultRules;
            _rulesSet = new RuleSet(Program.Logger);

            if (string.IsNullOrEmpty(opt.CustomRulesPath) && _arg_ignoreDefaultRules)
            {
                throw new OpException(ErrMsg.GetString(ErrMsg.ID.CMD_NORULES_SPECIFIED));
            }

            ConfigureRules();
            ConfigureOutput();
        }
예제 #5
0
        List <string> _fileExclusionList;//see exclusion list


        public AnalyzeCommand(AnalyzeCommandOptions opts)
        {
            _arg_sourcePath           = opts.SourcePath;
            _arg_outputFile           = opts.OutputFilePath;
            _arg_fileFormat           = opts.OutputFileFormat;
            _arg_outputTextFormat     = opts.TextOutputFormat;
            _arg_outputUniqueTagsOnly = !opts.AllowDupTags;
            _arg_customRulesPath      = opts.CustomRulesPath;
            _arg_confidenceFilters    = opts.ConfidenceFilters;
            _arg_ignoreDefaultRules   = opts.IgnoreDefaultRules;
            _arg_simpleTagsOnly       = opts.SimpleTagsOnly;
            _fileExclusionList        = opts.FilePathExclusions.Split(",").ToList <string>();
            if (_fileExclusionList.Contains("none") || _fileExclusionList.Contains("None"))
            {
                _fileExclusionList.Clear();
            }

            if (!Enum.TryParse(opts.ConsoleVerbosityLevel, true, out _arg_consoleVerbosityLevel))
            {
                throw new OpException(String.Format(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_ARG_VALUE, "-x")));
            }
            WriteOnce.Verbosity = _arg_consoleVerbosityLevel;


            LastUpdated = DateTime.MinValue;
            DateScanned = DateTime.Now;

            ConfigOutput();
            ConfigSourcetoScan();
            ConfigConfidenceFilters();
            ConfigRules();

            _uniqueTagsControl = new HashSet <string>();
        }
예제 #6
0
        public static Writer GetWriter(string writerName, string defaultWritter, string format = null)
        {
            if (string.IsNullOrEmpty(writerName))
            {
                writerName = defaultWritter;
            }

            if (string.IsNullOrEmpty(writerName))
            {
                writerName = "text";
            }

            switch (writerName.ToLowerInvariant())
            {
            case "_dummy":
                return(new DummyWriter());

            case "json":
                return(new JsonWriter());

            case "text":
                return(new SimpleTextWriter(format));

            case "html":
                return(new LiquidWriter());

            default:
                throw new OpException(String.Format(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_ARG_VALUE, "-f")));
            }
        }
예제 #7
0
        /// <summary>
        /// Main entry point to start analysis; handles setting up rules, directory enumeration
        /// file type detection and handoff
        /// Pre: All Configure Methods have been called already and we are ready to SCAN
        /// </summary>
        /// <returns></returns>
        public int Run()
        {
            WriteOnce.SafeLog("AnalyzeCommand::Run", LogLevel.Trace);

            DateTime start = DateTime.Now;

            WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_RUNNING, "Analyze"));

            _appProfile.MetaData.TotalFiles = _srcfileList.Count();//updated for zipped files later

            // Iterate through all files and process against rules
            foreach (string filename in _srcfileList)
            {
                var fileExtension = new FileInfo(filename).Extension;
                if (COMPRESSED_EXTENSIONS.Any(fileExtension.Contains))
                {
                    UnZipAndProcess(filename); //determine if file is a compressed item to unpackage for processing
                }
                else
                {
                    ProcessAsFile(filename);
                }
            }

            WriteOnce.General("\r" + ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILES_PROCESSED_PCNT, 100));
            WriteOnce.Operation(ErrMsg.GetString(ErrMsg.ID.CMD_PREPARING_REPORT));

            //Prepare report results
            _appProfile.MetaData.LastUpdated = LastUpdated.ToString();
            _appProfile.DateScanned          = DateScanned.ToString();
            _appProfile.PrepareReport();
            TimeSpan timeSpan = start - DateTime.Now;

            WriteOnce.SafeLog(String.Format("Processing time: seconds:{0}", timeSpan.TotalSeconds * -1), LogLevel.Trace);
            FlushAll();

            //wrapup result status
            if (_appProfile.MetaData.TotalFiles == _appProfile.MetaData.FilesSkipped)
            {
                WriteOnce.Error(ErrMsg.GetString(ErrMsg.ID.ANALYZE_NOSUPPORTED_FILETYPES));
            }
            else if (_appProfile.MatchList.Count == 0)
            {
                WriteOnce.Error(ErrMsg.GetString(ErrMsg.ID.ANALYZE_NOPATTERNS));
            }
            else
            {
                WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_COMPLETED, "Analyze"));
            }

            //html report size warning
            if (_arg_fileFormat == "html" && new FileInfo("output.html").Length > MAX_HTML_REPORT_FILE_SIZE)
            {
                WriteOnce.Info(ErrMsg.GetString(ErrMsg.ID.ANALYZE_REPORTSIZE_WARN));
            }

            return(_appProfile.MatchList.Count() == 0 ? (int)ExitCode.NoMatches :
                   (int)ExitCode.MatchesFound);
        }
        /// <summary>
        /// Main entry point to start analysis; handles setting up rules, directory enumeration
        /// file type detection and handoff
        /// Pre: All Configure Methods have been called already and we are ready to SCAN
        /// </summary>
        /// <returns></returns>
        public int Run()
        {
            WriteOnce.SafeLog("AnalyzeCommand::Run", LogLevel.Trace);

            DateTime start = DateTime.Now;

            WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_RUNNING, "Analyze"));

            _appProfile.MetaData.TotalFiles = _srcfileList.Count();//updated for zipped files later

            // Iterate through all files and process against rules
            foreach (string filename in _srcfileList)
            {
                ArchiveFileType archiveFileType = MiniMagic.DetectFileType(filename);
                if (archiveFileType == ArchiveFileType.UNKNOWN)//not a known zipped file type
                {
                    ProcessAsFile(filename);
                }
                else
                {
                    UnZipAndProcess(filename, archiveFileType);
                }
            }

            WriteOnce.General("\r" + ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILES_PROCESSED_PCNT, 100));
            WriteOnce.Operation(ErrMsg.GetString(ErrMsg.ID.CMD_PREPARING_REPORT));

            //Prepare report results
            _appProfile.MetaData.LastUpdated = LastUpdated.ToString();
            _appProfile.DateScanned          = DateScanned.ToString();
            _appProfile.PrepareReport();
            TimeSpan timeSpan = start - DateTime.Now;

            WriteOnce.SafeLog(String.Format("Processing time: seconds:{0}", timeSpan.TotalSeconds * -1), LogLevel.Trace);
            FlushAll();

            //wrapup result status
            if (_appProfile.MetaData.TotalFiles == _appProfile.MetaData.FilesSkipped)
            {
                WriteOnce.Error(ErrMsg.GetString(ErrMsg.ID.ANALYZE_NOSUPPORTED_FILETYPES));
            }
            else if (_appProfile.MatchList.Count == 0)
            {
                WriteOnce.Error(ErrMsg.GetString(ErrMsg.ID.ANALYZE_NOPATTERNS));
            }
            else
            {
                WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_COMPLETED, "Analyze"));
                if (!_arg_autoBrowserOpen)
                {
                    WriteOnce.Any(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_OUTPUT_FILE, "output.html"));
                }
            }

            return(_appProfile.MatchList.Count() == 0 ? (int)ExitCode.NoMatches :
                   (int)ExitCode.MatchesFound);
        }
        void UnZipAndProcess(string filename, ArchiveFileType archiveFileType)
        {
            //zip itself may be too huge for timely processing
            if (new FileInfo(filename).Length > WARN_ZIP_FILE_SIZE)
            {
                WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_FILESIZE_WARN));
            }
            else
            {
                WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_PROCESSING));
            }

            try
            {
                IEnumerable <FileEntry> files = Extractor.ExtractFile(filename);

                if (files.Count() > 0)
                {
                    _appProfile.MetaData.TotalFiles += files.Count();//additive in case additional child zip files processed
                    _appProfile.MetaData.PackageTypes.Add(ErrMsg.GetString(ErrMsg.ID.ANALYZE_COMPRESSED_FILETYPE));

                    foreach (FileEntry file in files)
                    {
                        if (file.Content.Length > MAX_FILESIZE)
                        {
                            WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILESIZE_SKIPPED, file.FullPath), LogLevel.Warn);
                            _appProfile.MetaData.FilesSkipped++;
                            continue;
                        }

                        //dup check vs Run() for zip contents; exclude sample, test or similar files by default or as specified in exclusion list
                        if (!_arg_allowSampleFiles && _fileExclusionList.Any(v => file.FullPath.ToLower().Contains(v)))
                        {
                            WriteOnce.SafeLog("Part of excluded list: " + file.FullPath, LogLevel.Trace);
                            WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILESIZE_SKIPPED, file.FullPath), LogLevel.Trace);
                            _appProfile.MetaData.FilesSkipped++;
                            continue;
                        }

                        WriteOnce.Log.Trace("processing zip file entry: " + file.FullPath);
                        byte[] streamByteArray = file.Content.ToArray();
                        ProcessInMemory(file.FullPath, Encoding.UTF8.GetString(streamByteArray, 0, streamByteArray.Length), true);
                    }
                }
                else
                {
                    throw new OpException(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_ERROR, filename));
                }
            }
            catch (Exception e)
            {
                string errmsg = ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_ERROR, filename);
                WriteOnce.Error(errmsg);
                throw new Exception(errmsg + e.Message + "\n" + e.StackTrace);
            }
        }
        void UnZipAndProcess(string filename, ArchiveFileType archiveFileType)
        {
            // zip itself may be in excluded list i.e. sample, test or similar unless ignore filter requested
            if (_fileExclusionList.Any(v => filename.ToLower().Contains(v)))
            {
                WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_EXCLUDED_TYPE_SKIPPED, filename), LogLevel.Warn);
                _appProfile.MetaData.FilesSkipped++;
                return;
            }

            //zip itself may be too huge for timely processing
            if (new FileInfo(filename).Length > WARN_ZIP_FILE_SIZE)
            {
                WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_FILESIZE_WARN));
            }
            else
            {
                WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_PROCESSING));
            }

            LastUpdated = File.GetLastWriteTime(filename);
            _appProfile.MetaData.PackageTypes.Add(ErrMsg.GetString(ErrMsg.ID.ANALYZE_COMPRESSED_FILETYPE));

            try
            {
                IEnumerable <FileEntry> files = Extractor.ExtractFile(filename);

                if (files.Count() > 0)
                {
                    _appProfile.MetaData.TotalFiles += files.Count();//additive in case additional child zip files processed

                    foreach (FileEntry file in files)
                    {
                        //check uncompressed file passes standard checks
                        LanguageInfo languageInfo = new LanguageInfo();
                        if (FileChecksPassed(file.FullPath, ref languageInfo, file.Content.Length))
                        {
                            byte[] streamByteArray = file.Content.ToArray();
                            ProcessInMemory(file.FullPath, Encoding.UTF8.GetString(streamByteArray, 0, streamByteArray.Length), languageInfo);
                        }
                    }
                }
                else
                {
                    WriteOnce.SafeLog(string.Format("Decompression found no files in {0}", filename), LogLevel.Warn);//zero results can be valid
                }
            }
            catch (Exception e)
            {
                string errmsg = ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_ERROR, filename);
                WriteOnce.Error(errmsg);
                throw new Exception(errmsg + e.Message + "\n" + e.StackTrace);
            }
        }
예제 #11
0
        /// <summary>
        /// Program entry point which defines command verbs and options to running
        /// </summary>
        /// <param name="args"></param>
        static int Main(string[] args)
        {
            int finalResult = -1;

            WriteOnce.Verbosity = WriteOnce.ConsoleVerbosity.Medium;

            try
            {
                WriteOnce.Info(GetVersionString());
                var argsResult = Parser.Default.ParseArguments <AnalyzeCommandOptions,
                                                                TagDiffCommandOptions,
                                                                TagTestCommandOptions,
                                                                ExportTagsCommandOptions,
                                                                VerifyRulesCommandOptions>(args)
                                 .MapResult(
                    (AnalyzeCommandOptions opts) => RunAnalyzeCommand(opts),
                    (TagDiffCommandOptions opts) => RunTagDiffCommand(opts),
                    (TagTestCommandOptions opts) => RunTagTestCommand(opts),
                    (ExportTagsCommandOptions opts) => RunExportTagsCommand(opts),
                    (VerifyRulesCommandOptions opts) => RunVerifyRulesCommand(opts),
                    errs => 1
                    );

                finalResult = argsResult;
            }
            catch (OpException e)
            {
                if (Logger != null)
                {
                    WriteOnce.Error(ErrMsg.FormatString(ErrMsg.ID.RUNTIME_ERROR_NAMED, e.Message));
                    Logger.Error($"Runtime error: {e.StackTrace}");
                }
                else
                {
                    WriteOnce.Error(ErrMsg.FormatString(ErrMsg.ID.RUNTIME_ERROR_PRELOG, e.Message));
                }
            }
            catch (Exception e)
            {
                if (Logger != null)
                {
                    WriteOnce.Error(ErrMsg.FormatString(ErrMsg.ID.RUNTIME_ERROR_UNNAMED));
                    Logger.Error($"Runtime error: {e.StackTrace}");
                }
                else
                {
                    WriteOnce.Error(ErrMsg.FormatString(ErrMsg.ID.RUNTIME_ERROR_PRELOG, e.Message));
                }
            }

            return(finalResult);
        }
예제 #12
0
 public VerifyRulesCommand(VerifyRulesCommandOptions opt)
 {
     _arg_customRulesPath    = opt.CustomRulesPath;
     _arg_ignoreDefaultRules = opt.IgnoreDefaultRules;
     _arg_outputFile         = opt.OutputFilePath;
     if (!Enum.TryParse(opt.ConsoleVerbosityLevel, true, out _arg_consoleVerbosityLevel))
     {
         throw new OpException(String.Format(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_ARG_VALUE, "-x")));
     }
     WriteOnce.Verbosity = _arg_consoleVerbosityLevel;
     ConfigureOutput();
     ConfigRules();
 }
        /// <summary>
        /// Wrapper for files that are on disk and ready to read vs unzipped files which are not to allow separation of core
        /// scan evaluation for use by decompression methods as well
        /// </summary>
        /// <param name="filename"></param>
        void ProcessAsFile(string filename)
        {
            //check for supported language
            LanguageInfo languageInfo = new LanguageInfo();

            if (FileChecksPassed(filename, ref languageInfo))
            {
                LastUpdated = File.GetLastWriteTime(filename);
                _appProfile.MetaData.PackageTypes.Add(ErrMsg.GetString(ErrMsg.ID.ANALYZE_UNCOMPRESSED_FILETYPE));

                string fileText = File.ReadAllText(filename);
                ProcessInMemory(filename, fileText, languageInfo);
            }
        }
예제 #14
0
        /// <summary>
        /// Wrapper for files that are on disk and ready to read to allow separation of core
        /// scan evaluation for use by decompression methods as well
        /// </summary>
        /// <param name="filename"></param>
        void ProcessAsFile(string filename)
        {
            if (File.Exists(filename))
            {
                _appProfile.MetaData.FileNames.Add(filename);
                _appProfile.MetaData.PackageTypes.Add(ErrMsg.GetString(ErrMsg.ID.ANALYZE_UNCOMPRESSED_FILETYPE));

                string fileText = File.ReadAllText(filename);
                ProcessInMemory(filename, fileText);
            }
            else
            {
                throw new OpException(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_FILE_OR_DIR, filename));
            }
        }
예제 #15
0
        void UnZipAndProcess(string filename, ArchiveFileType archiveFileType)
        {
            //zip itself may be too huge for timely processing
            if (new FileInfo(filename).Length > WARN_ZIP_FILE_SIZE)
            {
                WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_FILESIZE_WARN));
            }
            else
            {
                WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_PROCESSING));
            }

            LastUpdated = File.GetLastWriteTime(filename);
            _appProfile.MetaData.PackageTypes.Add(ErrMsg.GetString(ErrMsg.ID.ANALYZE_COMPRESSED_FILETYPE));

            try
            {
                IEnumerable <FileEntry> files = Extractor.ExtractFile(filename);

                if (files.Count() > 0)
                {
                    _appProfile.MetaData.TotalFiles += files.Count();//additive in case additional child zip files processed

                    foreach (FileEntry file in files)
                    {
                        //check for supported language
                        LanguageInfo languageInfo = new LanguageInfo();
                        if (FileChecksPassed(file.FullPath, ref languageInfo, file.Content.Length))
                        {
                            byte[] streamByteArray = file.Content.ToArray();
                            ProcessInMemory(file.FullPath, Encoding.UTF8.GetString(streamByteArray, 0, streamByteArray.Length), languageInfo);
                        }
                    }
                }
                else
                {
                    throw new OpException(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_ERROR, filename));
                }
            }
            catch (Exception e)
            {
                string errmsg = ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_ERROR, filename);
                WriteOnce.Error(errmsg);
                throw new Exception(errmsg + e.Message + "\n" + e.StackTrace);
            }
        }
예제 #16
0
        void ProcessTarGzFile(string filename)
        {
            WriteOnce.SafeLog(string.Format("Analyzing .tar.gz file: [{0}]", filename), LogLevel.Trace);

            _appProfile.MetaData.TotalFiles = GetTarGzFileCount(filename);

            using (var fileStream = new FileStream(filename, FileMode.Open, FileAccess.Read))
                using (var gzipStream = new GZipInputStream(fileStream))
                    using (var memoryStream = new MemoryStream())
                    {
                        var      tarStream = new TarInputStream(gzipStream);
                        TarEntry tarEntry;
                        while ((tarEntry = tarStream.GetNextEntry()) != null)
                        {
                            if (tarEntry.IsDirectory)
                            {
                                continue;
                            }
                            tarStream.CopyEntryContents(memoryStream);
                            if (tarEntry.Size > MAX_FILESIZE)
                            {
                                _appProfile.MetaData.FilesSkipped++;
                                WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILESIZE_SKIPPED, tarEntry.Name), LogLevel.Warn);
                                tarStream.Close();
                                continue;
                            }

                            var mimeType = MimeTypeMap.GetMimeType(Path.GetExtension(tarEntry.Name));
                            if (IgnoreMimeRegex.IsMatch(mimeType) && new FileInfo(tarEntry.Name).Extension != ".ts")
                            {
                                _appProfile.MetaData.FilesSkipped++;
                                WriteOnce.SafeLog(string.Format("Ignoring tar entry [{0}]", tarEntry.Name), LogLevel.Error);
                            }
                            else
                            {
                                //file name may contain slashes; remove prior to call
                                byte[] streamByteArray = memoryStream.ToArray();
                                ProcessInMemory(Path.GetFileName(tarEntry.Name), Encoding.UTF8.GetString(streamByteArray, 0, streamByteArray.Length));
                            }

                            memoryStream.SetLength(0); // Clear out the stream
                        }

                        tarStream.Close();
                    }
        }
예제 #17
0
 public static void OpenBrowser(string url)
 {
     if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
     {
         try
         {
             Process.Start(new ProcessStartInfo("cmd", $"/c start {url}"));
             WriteOnce.General(ErrMsg.GetString(ErrMsg.ID.BROWSER_START_SUCCESS));
         }
         catch (Exception)
         {
             WriteOnce.General(ErrMsg.GetString(ErrMsg.ID.BROWSER_START_FAIL));
         }
     }
     else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
     {
         if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("BROWSER")))
         {
             try
             {
                 Process.Start("xdg-open", url);
                 WriteOnce.General(ErrMsg.GetString(ErrMsg.ID.BROWSER_START_SUCCESS));
             }
             catch (Exception)
             {
                 WriteOnce.SafeLog("Unable to open browser using BROWSER environment var", NLog.LogLevel.Error);
             }
         }
         else
         {
             WriteOnce.General(ErrMsg.GetString(ErrMsg.ID.BROWSER_ENVIRONMENT_VAR));
         }
     }
     else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
     {
         try
         {
             Process.Start("open", url);
             WriteOnce.General(ErrMsg.GetString(ErrMsg.ID.BROWSER_START_SUCCESS));
         }
         catch (Exception)
         {
             WriteOnce.General(ErrMsg.GetString(ErrMsg.ID.BROWSER_START_FAIL));
         }
     }
 }
예제 #18
0
        void ProcessZipFile(string filename)
        {
            WriteOnce.SafeLog(string.Format("Analyzing .zip file: [{0}])", filename), LogLevel.Trace);

            ZipFile zipFile;

            using (var fileStream = new FileStream(filename, FileMode.Open, FileAccess.Read))
                using (var memoryStream = new MemoryStream())
                {
                    zipFile = new ZipFile(fileStream);
                    _appProfile.MetaData.TotalFiles = (int)zipFile.Count;
                    foreach (ZipEntry zipEntry in zipFile)
                    {
                        if (zipEntry.IsDirectory)
                        {
                            continue;
                        }
                        byte[] buffer    = new byte[4096];
                        var    zipStream = zipFile.GetInputStream(zipEntry);
                        if (zipEntry.Size > MAX_FILESIZE)
                        {
                            _appProfile.MetaData.FilesSkipped++;
                            WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILESIZE_SKIPPED, zipEntry.Name), LogLevel.Warn);
                            zipFile.Close();
                            continue;
                        }

                        StreamUtils.Copy(zipStream, memoryStream, buffer);
                        var mimeType = MimeTypeMap.GetMimeType(Path.GetExtension(zipEntry.Name));
                        if (IgnoreMimeRegex.IsMatch(mimeType) && new FileInfo(zipEntry.Name).Extension != ".ts")
                        {
                            _appProfile.MetaData.FilesSkipped++;
                            WriteOnce.SafeLog(string.Format("Ignoring zip entry [{0}]", zipEntry.Name), LogLevel.Error);
                        }
                        else
                        {
                            byte[] streamByteArray = memoryStream.ToArray();
                            ProcessInMemory(Path.GetFileName(zipEntry.Name), Encoding.UTF8.GetString(streamByteArray, 0, streamByteArray.Length));
                        }
                        memoryStream.SetLength(0); // Clear out the stream
                    }

                    zipFile.Close();
                }
        }
        public TagDiffCommand(TagDiffCommandOptions opt)
        {
            _arg_src1       = opt.SourcePath1;
            _arg_src2       = opt.SourcePath2;
            _arg_rulesPath  = opt.CustomRulesPath;
            _arg_outputFile = opt.OutputFilePath;
            if (!Enum.TryParse(opt.ConsoleVerbosityLevel, true, out _arg_consoleVerbosityLevel))
            {
                throw new OpException(String.Format(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_ARG_VALUE, "-x")));
            }
            WriteOnce.Verbosity = _arg_consoleVerbosityLevel;

            if (!Enum.TryParse(opt.TestType, true, out _arg_tagTestType))
            {
                throw new OpException(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_ARG_VALUE, opt.TestType));
            }

            _arg_ignoreDefault = opt.IgnoreDefaultRules;
        }
예제 #20
0
        static void SetupLogging(string logFilePath, string logFileLevel)
        {
            var config = new NLog.Config.LoggingConfiguration();


            if (String.IsNullOrEmpty(logFilePath))
            {
                logFilePath = Utils.GetPath(Utils.AppPath.defaultLog);
                //if using default app log path clean up previous for convenience in reading
                if (File.Exists(logFilePath))
                {
                    File.Delete(logFilePath);
                }
            }

            LogLevel log_level = LogLevel.Error;//default

            try
            {
                log_level = LogLevel.FromString(logFileLevel);
            }
            catch (Exception)
            {
                throw new OpException(String.Format(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_ARG_VALUE, "-v")));
            }

            using (var fileTarget = new FileTarget()
            {
                Name = "LogFile",
                FileName = logFilePath,
                Layout = @"${date:universalTime=true:format=s} ${threadid} ${level:uppercase=true} - ${message}",
                ForceMutexConcurrentWrites = true
            })
            {
                config.AddTarget(fileTarget);
                config.LoggingRules.Add(new LoggingRule("*", log_level, fileTarget));
            }

            LogManager.Configuration = config;
            Logger = LogManager.GetCurrentClassLogger();
            Logger.Info("[" + DateTime.Now.ToLocalTime() + "] //////////////////////////////////////////////////////////");
            WriteOnce.Log = Logger;//allows log to be written to as well as console or output file
        }
예제 #21
0
        void ConfigRules()
        {
            _rulePaths = new List <string>();
            if (!_arg_ignoreDefaultRules)
            {
                _rulePaths.Add(Utils.GetPath(Utils.AppPath.defaultRules));
            }

            if (!string.IsNullOrEmpty(_arg_customRulesPath))
            {
                _rulePaths.Add(_arg_customRulesPath);
            }

            if (_rulePaths.Count == 0)
            {
                throw new OpException(ErrMsg.GetString(ErrMsg.ID.CMD_NORULES_SPECIFIED));
            }

            //validation of paths is delayed for Run in this cmd
        }
예제 #22
0
        public void FlushAll()
        {
            if (_outputWriter != null)
            {
                _outputWriter.WriteApp(_appProfile);

                if (_outputWriter.TextWriter != null && _arg_fileFormat != "html")
                {
                    _outputWriter.FlushAndClose();//not required for html formal i.e. multiple files already closed
                    _outputWriter = null;
                    if (!String.IsNullOrEmpty(_arg_outputFile))
                    {
                        WriteOnce.Any(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_OUTPUT_FILE, _arg_outputFile));
                    }
                    else
                    {
                        WriteOnce.NewLine();
                    }
                }
            }
        }
        /// <summary>
        /// Common validation called by ProcessAsFile and UnzipAndProcess to ensure same order and checks made
        /// </summary>
        /// <param name="filePath"></param>
        /// <param name="languageInfo"></param>
        /// <param name="fileLength">should be > zero if called from unzip method</param>
        /// <returns></returns>
        bool FileChecksPassed(string filePath, ref LanguageInfo languageInfo, long fileLength = 0)
        {
            _appProfile.MetaData.FileExtensions.Add(Path.GetExtension(filePath).Replace('.', ' ').TrimStart());

            // 1. Skip files written in unknown language
            if (!Language.FromFileName(filePath, ref languageInfo))
            {
                WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_LANGUAGE_NOTFOUND, filePath), LogLevel.Warn);
                _appProfile.MetaData.FilesSkipped++;
                return(false);
            }

            _appProfile.MetaData.AddLanguage(languageInfo.Name);

            // 2. Skip excluded files i.e. sample, test or similar unless ignore filter requested
            if (_fileExclusionList.Any(v => filePath.ToLower().Contains(v)))
            {
                WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_EXCLUDED_TYPE_SKIPPED, filePath), LogLevel.Warn);
                _appProfile.MetaData.FilesSkipped++;
                return(false);
            }

            // 3. Skip if exceeds file size limits
            try
            {
                fileLength = fileLength <= 0 ? new FileInfo(filePath).Length : fileLength;
                if (fileLength > MAX_FILESIZE)
                {
                    WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILESIZE_SKIPPED, filePath), LogLevel.Warn);
                    _appProfile.MetaData.FilesSkipped++;
                    return(false);
                }
            }
            catch (Exception)
            {
                throw new OpException(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_FILE_OR_DIR, filePath));
            }

            return(true);
        }
        bool CompareTags(string[] fileTags1, string[] fileTags2)
        {
            bool found = true;

            //are all tags in file1 found in file2
            foreach (string s1 in fileTags1)
            {
                if (!fileTags2.Contains(s1))
                {
                    found = false;
                    WriteOnce.Result(s1, true, WriteOnce.ConsoleVerbosity.High);
                }
            }

            //none missing
            if (found)
            {
                WriteOnce.Result(ErrMsg.GetString(ErrMsg.ID.TAGTEST_RESULTS_NONE), true, WriteOnce.ConsoleVerbosity.High);
            }

            return(found);
        }
        public int Run()
        {
            WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_RUNNING, "Exporttags"));

            SortedDictionary <string, string> uniqueTags = new SortedDictionary <string, string>();

            foreach (Rule r in _rules)
            {
                //builds a list of unique tags
                foreach (string t in r.Tags)
                {
                    if (uniqueTags.ContainsKey(t))
                    {
                        continue;
                    }
                    else
                    {
                        uniqueTags.Add(t, t);
                    }
                }
            }

            //separate loop so results are sorted (Sorted type)
            foreach (string s in uniqueTags.Values)
            {
                WriteOnce.Result(s, true);
            }

            WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_COMPLETED, "Exporttags"));
            WriteOnce.FlushAll();
            if (!String.IsNullOrEmpty(_arg_outputFile))
            {
                WriteOnce.Any(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_OUTPUT_FILE, _arg_outputFile), true, ConsoleColor.Gray, WriteOnce.ConsoleVerbosity.Low);
            }


            return((int)ExitCode.Success);
        }
        /// <summary>
        /// Wrapper for files that are on disk and ready to read to allow separation of core
        /// scan evaluation for use by decompression methods as well
        /// </summary>
        /// <param name="filename"></param>
        void ProcessAsFile(string filename)
        {
            try
            {
                if (new FileInfo(filename).Length > MAX_FILESIZE)
                {
                    WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILESIZE_SKIPPED, filename), LogLevel.Warn);
                    _appProfile.MetaData.FilesSkipped++;
                    return;
                }
            }
            catch (Exception)
            {
                throw new OpException(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_FILE_OR_DIR, filename));
            }


            _appProfile.MetaData.FileNames.Add(filename);
            _appProfile.MetaData.PackageTypes.Add(ErrMsg.GetString(ErrMsg.ID.ANALYZE_UNCOMPRESSED_FILETYPE));
            string fileText = File.ReadAllText(filename);

            ProcessInMemory(filename, fileText);
        }
        void ConfigRules()
        {
            List <string> rulePaths = new List <string>();

            if (!_arg_ignoreDefaultRules)
            {
                rulePaths.Add(Utils.GetPath(Utils.AppPath.defaultRules));
            }

            if (!string.IsNullOrEmpty(_arg_customRulesPath))
            {
                rulePaths.Add(_arg_customRulesPath);
            }

            foreach (string rulePath in rulePaths)
            {
                if (Directory.Exists(rulePath))
                {
                    _rules.AddDirectory(rulePath);
                }
                else if (File.Exists(rulePath))
                {
                    _rules.AddFile(rulePath);
                }
                else
                {
                    throw new OpException(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_RULE_PATH, rulePath));
                }
            }

            //error check based on ruleset not path enumeration
            if (_rules.Count() == 0)
            {
                throw new OpException(ErrMsg.GetString(ErrMsg.ID.CMD_NORULES_SPECIFIED));
            }
        }
예제 #28
0
        void UnZipAndProcess(string filename)
        {
            if (!File.Exists(filename))
            {
                throw new OpException(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_FILE_OR_DIR, filename));
            }

            // Ignore images and other junk like that
            var  fileExtension = new FileInfo(filename).Extension;
            var  mimeType      = MimeTypeMap.GetMimeType(fileExtension);
            bool mimeMatch     = false;

            if (!IgnoreMimeRegex.IsMatch(mimeType))
            {
                var isValidExtension = COMPRESSED_EXTENSIONS.Any(fileExtension.Contains);
                if (isValidExtension || fileExtension == "ts")
                {
                    mimeMatch = true;
                }
                else if (mimeType.Contains("zip", StringComparison.CurrentCultureIgnoreCase) || // Should have been caught in file extensions above, but still OK.
                         mimeType.Contains("tar", StringComparison.CurrentCultureIgnoreCase) ||
                         mimeType.Contains("compressed", StringComparison.CurrentCultureIgnoreCase))
                {
                    mimeMatch = true;
                }

                if (mimeMatch)
                {
                    // Now process the file
                    switch (fileExtension)
                    {
                    case ".tgz":
                        ProcessTarGzFile(filename);
                        break;

                    case ".gz":
                        if (filename.Contains(".tar.gz"))
                        {
                            ProcessTarGzFile(filename);
                        }
                        else
                        {
                            WriteOnce.SafeLog("no support for .gz unless .tar.gz: " + fileExtension, LogLevel.Warn);
                            _appProfile.MetaData.PackageTypes.Add("compressed-unsupported");
                        }
                        break;

                    case ".jar":
                    case ".zip":
                        ProcessZipFile(filename);
                        break;

                    case ".gem":
                    case ".tar":
                    case ".nupkg":
                        WriteOnce.SafeLog($"Processing of {fileExtension} not implemented yet.", LogLevel.Warn);
                        break;

                    default:
                        WriteOnce.SafeLog("no support for compressed type: " + fileExtension, LogLevel.Warn);
                        break;
                    }

                    _appProfile.MetaData.PackageTypes.Add("compressed");
                }
                else
                {
                    _appProfile.MetaData.PackageTypes.Add("compressed-unsupported");
                }
            }
        }
예제 #29
0
        /// <summary>
        /// Main WORKHORSE for analyzing file; called from file based or decompression functions
        /// </summary>
        /// <param name="filename"></param>
        /// <param name="fileText"></param>
        void ProcessInMemory(string filePath, string fileText)
        {
            #region quickvalidation
            if (fileText.Length > MAX_FILESIZE)
            {
                WriteOnce.SafeLog("File too large: " + filePath, LogLevel.Trace);
                WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILESIZE_SKIPPED, filePath), LogLevel.Error);
                _appProfile.MetaData.FilesSkipped++;
                return;
            }

            if (!_arg_allowSampleFiles && _fileExclusionList.Any(v => filePath.Contains(v)))
            {
                WriteOnce.SafeLog("Part of excluded list: " + filePath, LogLevel.Trace);
                WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILESIZE_SKIPPED, filePath), LogLevel.Error);
                _appProfile.MetaData.FilesSkipped++;
                return;
            }

            //determine if file is a compressed item to unpackage for processing
            string language = Language.FromFileName(filePath);

            // Skip files written in unknown language
            if (string.IsNullOrEmpty(language))
            {
                WriteOnce.SafeLog("Language not found for file: " + filePath, LogLevel.Trace);
                language = Path.GetFileName(filePath);
                _appProfile.MetaData.FilesSkipped++;
                return;
            }
            else
            {
                WriteOnce.SafeLog("Preparing to process file: " + filePath, LogLevel.Trace);
            }

            #endregion

            #region minorRollupTrackingAndProgress

            _appProfile.MetaData.FilesAnalyzed++;
            _appProfile.MetaData.AddLanguage(language);
            _appProfile.MetaData.FileExtensions.Add(Path.GetExtension(filePath).Replace('.', ' ').TrimStart());
            LastUpdated = File.GetLastWriteTime(filePath);

            int totalFilesReviewed = _appProfile.MetaData.FilesAnalyzed + _appProfile.MetaData.FilesSkipped;
            int percentCompleted   = (int)((float)totalFilesReviewed / (float)_appProfile.MetaData.TotalFiles * 100);
            WriteOnce.General("\r" + ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILES_PROCESSED_PCNT, percentCompleted), false);

            #endregion

            //process file against rules
            Issue[] matches = _rulesProcessor.Analyze(fileText, language);

            //if any matches found for this file...
            if (matches.Count() > 0)
            {
                _appProfile.MetaData.FilesAffected++;
                _appProfile.MetaData.TotalMatchesCount += matches.Count();

                HashSet <string> uniqueTagsControl = new HashSet <string>();

                // Iterate through each match issue
                foreach (Issue match in matches)
                {
                    WriteOnce.SafeLog(string.Format("Processing pattern matches for ruleId {0}, ruleName {1} file {2}", match.Rule.Id, match.Rule.Name, filePath), LogLevel.Trace);

                    //maintain a list of unique tags; multi-purpose but primarily for filtering -u option
                    bool dupTagFound = false;
                    foreach (string t in match.Rule.Tags)
                    {
                        dupTagFound = !uniqueTagsControl.Add(t);
                    }

                    //save all unique dependendencies even if Dependency tag pattern is not-unique
                    var    tagPatternRegex = new Regex("Dependency.SourceInclude", RegexOptions.IgnoreCase);
                    String textMatch;
                    if (match.Rule.Tags.Any(v => tagPatternRegex.IsMatch(v)))
                    {
                        textMatch = ExtractDependency(fileText, match.Boundary.Index, match.PatternMatch, language);
                    }
                    else
                    {
                        textMatch = ExtractTextSample(fileText, match.Boundary.Index, match.Boundary.Length);
                    }

                    //wrap rule issue result to add metadata
                    MatchRecord record = new MatchRecord()
                    {
                        Filename   = filePath,
                        Language   = language,
                        Filesize   = fileText.Length,
                        TextSample = textMatch,
                        Excerpt    = ExtractExcerpt(fileText, match.StartLocation.Line),
                        Issue      = match
                    };

                    //preserve issue level characteristics as rolled up meta data of interest
                    bool valid = _appProfile.MetaData.AddStandardProperties(record);

                    //bail after extracting any dependency unique items IF user requested
                    if (_arg_outputUniqueTagsOnly && dupTagFound)
                    {
                        continue;
                    }
                    else if (valid)
                    {
                        _appProfile.MatchList.Add(record);
                    }
                }
            }
            else
            {
                WriteOnce.SafeLog("No pattern matches detected for file: " + filePath, LogLevel.Trace);
            }
        }
예제 #30
0
        /// <summary>
        /// Add default and/or custom rules paths
        /// Iterate paths and add to ruleset
        /// </summary>
        void ConfigRules()
        {
            WriteOnce.SafeLog("AnalyzeCommand::ConfigRules", LogLevel.Trace);

            RuleSet       rulesSet  = new RuleSet(Program.Logger);
            List <string> rulePaths = new List <string>();

            if (!_arg_ignoreDefaultRules)
            {
                rulePaths.Add(Utils.GetPath(Utils.AppPath.defaultRules));
            }

            if (!string.IsNullOrEmpty(_arg_customRulesPath))
            {
                rulePaths.Add(_arg_customRulesPath);
            }

            foreach (string rulePath in rulePaths)
            {
                if (Directory.Exists(rulePath))
                {
                    rulesSet.AddDirectory(rulePath);
                }
                else if (File.Exists(rulePath))
                {
                    rulesSet.AddFile(rulePath);
                }
                else
                {
                    throw new OpException(ErrMsg.FormatString(ErrMsg.ID.CMD_INVALID_RULE_PATH, rulePath));
                }
            }

            //error check based on ruleset not path enumeration
            if (rulesSet.Count() == 0)
            {
                throw new OpException(ErrMsg.GetString(ErrMsg.ID.CMD_NORULES_SPECIFIED));
            }

            //instantiate a RuleProcessor with the added rules and exception for dependency
            _rulesProcessor = new RuleProcessor(rulesSet, _arg_confidence, _arg_outputUniqueTagsOnly, _arg_simpleTagsOnly, Program.Logger);

            if (_arg_outputUniqueTagsOnly)
            {
                List <TagException> tagExceptions;
                if (File.Exists(Utils.GetPath(Utils.AppPath.tagCounterPref)))
                {
                    tagExceptions = JsonConvert.DeserializeObject <List <TagException> >(File.ReadAllText(Utils.GetPath(Utils.AppPath.tagCounterPref)));
                    string[] exceptions = new string[tagExceptions.Count];
                    for (int i = 0; i < tagExceptions.Count; i++)
                    {
                        exceptions[i] = tagExceptions[i].Tag;
                    }
                    _rulesProcessor.UniqueTagExceptions = exceptions;
                }
            }

            _appProfile      = new AppProfile(_arg_sourcePath, rulePaths, false, _arg_simpleTagsOnly, _arg_outputUniqueTagsOnly);
            _appProfile.Args = "analyze -f " + _arg_fileFormat + " -u " + _arg_outputUniqueTagsOnly.ToString().ToLower() + " -v " +
                               WriteOnce.Verbosity.ToString() + " -x " + _arg_confidence + " -i " + _arg_ignoreDefaultRules.ToString().ToLower();
        }