Пример #1
0
        /// <summary>
        /// Main entry point to start analysis; handles setting up rules, directory enumeration
        /// file type detection and handoff
        /// Pre: All Configure Methods have been called already and we are ready to SCAN
        /// </summary>
        /// <returns></returns>
        public int Run()
        {
            WriteOnce.SafeLog("AnalyzeCommand::Run", LogLevel.Trace);

            DateTime start = DateTime.Now;

            WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_RUNNING, "Analyze"));

            _appProfile.MetaData.TotalFiles = _srcfileList.Count();//updated for zipped files later

            // Iterate through all files and process against rules
            foreach (string filename in _srcfileList)
            {
                var fileExtension = new FileInfo(filename).Extension;
                if (COMPRESSED_EXTENSIONS.Any(fileExtension.Contains))
                {
                    UnZipAndProcess(filename); //determine if file is a compressed item to unpackage for processing
                }
                else
                {
                    ProcessAsFile(filename);
                }
            }

            WriteOnce.General("\r" + ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILES_PROCESSED_PCNT, 100));
            WriteOnce.Operation(ErrMsg.GetString(ErrMsg.ID.CMD_PREPARING_REPORT));

            //Prepare report results
            _appProfile.MetaData.LastUpdated = LastUpdated.ToString();
            _appProfile.DateScanned          = DateScanned.ToString();
            _appProfile.PrepareReport();
            TimeSpan timeSpan = start - DateTime.Now;

            WriteOnce.SafeLog(String.Format("Processing time: seconds:{0}", timeSpan.TotalSeconds * -1), LogLevel.Trace);
            FlushAll();

            //wrapup result status
            if (_appProfile.MetaData.TotalFiles == _appProfile.MetaData.FilesSkipped)
            {
                WriteOnce.Error(ErrMsg.GetString(ErrMsg.ID.ANALYZE_NOSUPPORTED_FILETYPES));
            }
            else if (_appProfile.MatchList.Count == 0)
            {
                WriteOnce.Error(ErrMsg.GetString(ErrMsg.ID.ANALYZE_NOPATTERNS));
            }
            else
            {
                WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_COMPLETED, "Analyze"));
            }

            //html report size warning
            if (_arg_fileFormat == "html" && new FileInfo("output.html").Length > MAX_HTML_REPORT_FILE_SIZE)
            {
                WriteOnce.Info(ErrMsg.GetString(ErrMsg.ID.ANALYZE_REPORTSIZE_WARN));
            }

            return(_appProfile.MatchList.Count() == 0 ? (int)ExitCode.NoMatches :
                   (int)ExitCode.MatchesFound);
        }
        /// <summary>
        /// Main entry point to start analysis; handles setting up rules, directory enumeration
        /// file type detection and handoff
        /// Pre: All Configure Methods have been called already and we are ready to SCAN
        /// </summary>
        /// <returns></returns>
        public int Run()
        {
            WriteOnce.SafeLog("AnalyzeCommand::Run", LogLevel.Trace);

            DateTime start = DateTime.Now;

            WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_RUNNING, "Analyze"));

            _appProfile.MetaData.TotalFiles = _srcfileList.Count();//updated for zipped files later

            // Iterate through all files and process against rules
            foreach (string filename in _srcfileList)
            {
                ArchiveFileType archiveFileType = MiniMagic.DetectFileType(filename);
                if (archiveFileType == ArchiveFileType.UNKNOWN)//not a known zipped file type
                {
                    ProcessAsFile(filename);
                }
                else
                {
                    UnZipAndProcess(filename, archiveFileType);
                }
            }

            WriteOnce.General("\r" + ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILES_PROCESSED_PCNT, 100));
            WriteOnce.Operation(ErrMsg.GetString(ErrMsg.ID.CMD_PREPARING_REPORT));

            //Prepare report results
            _appProfile.MetaData.LastUpdated = LastUpdated.ToString();
            _appProfile.DateScanned          = DateScanned.ToString();
            _appProfile.PrepareReport();
            TimeSpan timeSpan = start - DateTime.Now;

            WriteOnce.SafeLog(String.Format("Processing time: seconds:{0}", timeSpan.TotalSeconds * -1), LogLevel.Trace);
            FlushAll();

            //wrapup result status
            if (_appProfile.MetaData.TotalFiles == _appProfile.MetaData.FilesSkipped)
            {
                WriteOnce.Error(ErrMsg.GetString(ErrMsg.ID.ANALYZE_NOSUPPORTED_FILETYPES));
            }
            else if (_appProfile.MatchList.Count == 0)
            {
                WriteOnce.Error(ErrMsg.GetString(ErrMsg.ID.ANALYZE_NOPATTERNS));
            }
            else
            {
                WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_COMPLETED, "Analyze"));
                if (!_arg_autoBrowserOpen)
                {
                    WriteOnce.Any(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_OUTPUT_FILE, "output.html"));
                }
            }

            return(_appProfile.MatchList.Count() == 0 ? (int)ExitCode.NoMatches :
                   (int)ExitCode.MatchesFound);
        }
        void UnZipAndProcess(string filename, ArchiveFileType archiveFileType)
        {
            //zip itself may be too huge for timely processing
            if (new FileInfo(filename).Length > WARN_ZIP_FILE_SIZE)
            {
                WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_FILESIZE_WARN));
            }
            else
            {
                WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_PROCESSING));
            }

            try
            {
                IEnumerable <FileEntry> files = Extractor.ExtractFile(filename);

                if (files.Count() > 0)
                {
                    _appProfile.MetaData.TotalFiles += files.Count();//additive in case additional child zip files processed
                    _appProfile.MetaData.PackageTypes.Add(ErrMsg.GetString(ErrMsg.ID.ANALYZE_COMPRESSED_FILETYPE));

                    foreach (FileEntry file in files)
                    {
                        if (file.Content.Length > MAX_FILESIZE)
                        {
                            WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILESIZE_SKIPPED, file.FullPath), LogLevel.Warn);
                            _appProfile.MetaData.FilesSkipped++;
                            continue;
                        }

                        //dup check vs Run() for zip contents; exclude sample, test or similar files by default or as specified in exclusion list
                        if (!_arg_allowSampleFiles && _fileExclusionList.Any(v => file.FullPath.ToLower().Contains(v)))
                        {
                            WriteOnce.SafeLog("Part of excluded list: " + file.FullPath, LogLevel.Trace);
                            WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILESIZE_SKIPPED, file.FullPath), LogLevel.Trace);
                            _appProfile.MetaData.FilesSkipped++;
                            continue;
                        }

                        WriteOnce.Log.Trace("processing zip file entry: " + file.FullPath);
                        byte[] streamByteArray = file.Content.ToArray();
                        ProcessInMemory(file.FullPath, Encoding.UTF8.GetString(streamByteArray, 0, streamByteArray.Length), true);
                    }
                }
                else
                {
                    throw new OpException(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_ERROR, filename));
                }
            }
            catch (Exception e)
            {
                string errmsg = ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_ERROR, filename);
                WriteOnce.Error(errmsg);
                throw new Exception(errmsg + e.Message + "\n" + e.StackTrace);
            }
        }
        void UnZipAndProcess(string filename, ArchiveFileType archiveFileType)
        {
            // zip itself may be in excluded list i.e. sample, test or similar unless ignore filter requested
            if (_fileExclusionList.Any(v => filename.ToLower().Contains(v)))
            {
                WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_EXCLUDED_TYPE_SKIPPED, filename), LogLevel.Warn);
                _appProfile.MetaData.FilesSkipped++;
                return;
            }

            //zip itself may be too huge for timely processing
            if (new FileInfo(filename).Length > WARN_ZIP_FILE_SIZE)
            {
                WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_FILESIZE_WARN));
            }
            else
            {
                WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_PROCESSING));
            }

            LastUpdated = File.GetLastWriteTime(filename);
            _appProfile.MetaData.PackageTypes.Add(ErrMsg.GetString(ErrMsg.ID.ANALYZE_COMPRESSED_FILETYPE));

            try
            {
                IEnumerable <FileEntry> files = Extractor.ExtractFile(filename);

                if (files.Count() > 0)
                {
                    _appProfile.MetaData.TotalFiles += files.Count();//additive in case additional child zip files processed

                    foreach (FileEntry file in files)
                    {
                        //check uncompressed file passes standard checks
                        LanguageInfo languageInfo = new LanguageInfo();
                        if (FileChecksPassed(file.FullPath, ref languageInfo, file.Content.Length))
                        {
                            byte[] streamByteArray = file.Content.ToArray();
                            ProcessInMemory(file.FullPath, Encoding.UTF8.GetString(streamByteArray, 0, streamByteArray.Length), languageInfo);
                        }
                    }
                }
                else
                {
                    WriteOnce.SafeLog(string.Format("Decompression found no files in {0}", filename), LogLevel.Warn);//zero results can be valid
                }
            }
            catch (Exception e)
            {
                string errmsg = ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_ERROR, filename);
                WriteOnce.Error(errmsg);
                throw new Exception(errmsg + e.Message + "\n" + e.StackTrace);
            }
        }
Пример #5
0
 public static void OpenBrowser(string url)
 {
     if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
     {
         try
         {
             Process.Start(new ProcessStartInfo("cmd", $"/c start {url}"));
             WriteOnce.General(ErrMsg.GetString(ErrMsg.ID.BROWSER_START_SUCCESS));
         }
         catch (Exception)
         {
             WriteOnce.General(ErrMsg.GetString(ErrMsg.ID.BROWSER_START_FAIL));
         }
     }
     else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
     {
         if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("BROWSER")))
         {
             try
             {
                 Process.Start("xdg-open", url);
                 WriteOnce.General(ErrMsg.GetString(ErrMsg.ID.BROWSER_START_SUCCESS));
             }
             catch (Exception)
             {
                 WriteOnce.SafeLog("Unable to open browser using BROWSER environment var", NLog.LogLevel.Error);
             }
         }
         else
         {
             WriteOnce.General(ErrMsg.GetString(ErrMsg.ID.BROWSER_ENVIRONMENT_VAR));
         }
     }
     else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX))
     {
         try
         {
             Process.Start("open", url);
             WriteOnce.General(ErrMsg.GetString(ErrMsg.ID.BROWSER_START_SUCCESS));
         }
         catch (Exception)
         {
             WriteOnce.General(ErrMsg.GetString(ErrMsg.ID.BROWSER_START_FAIL));
         }
     }
 }
Пример #6
0
        void UnZipAndProcess(string filename, ArchiveFileType archiveFileType)
        {
            //zip itself may be too huge for timely processing
            if (new FileInfo(filename).Length > WARN_ZIP_FILE_SIZE)
            {
                WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_FILESIZE_WARN));
            }
            else
            {
                WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_PROCESSING));
            }

            LastUpdated = File.GetLastWriteTime(filename);
            _appProfile.MetaData.PackageTypes.Add(ErrMsg.GetString(ErrMsg.ID.ANALYZE_COMPRESSED_FILETYPE));

            try
            {
                IEnumerable <FileEntry> files = Extractor.ExtractFile(filename);

                if (files.Count() > 0)
                {
                    _appProfile.MetaData.TotalFiles += files.Count();//additive in case additional child zip files processed

                    foreach (FileEntry file in files)
                    {
                        //check for supported language
                        LanguageInfo languageInfo = new LanguageInfo();
                        if (FileChecksPassed(file.FullPath, ref languageInfo, file.Content.Length))
                        {
                            byte[] streamByteArray = file.Content.ToArray();
                            ProcessInMemory(file.FullPath, Encoding.UTF8.GetString(streamByteArray, 0, streamByteArray.Length), languageInfo);
                        }
                    }
                }
                else
                {
                    throw new OpException(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_ERROR, filename));
                }
            }
            catch (Exception e)
            {
                string errmsg = ErrMsg.FormatString(ErrMsg.ID.ANALYZE_COMPRESSED_ERROR, filename);
                WriteOnce.Error(errmsg);
                throw new Exception(errmsg + e.Message + "\n" + e.StackTrace);
            }
        }
Пример #7
0
        /// <summary>
        /// Main WORKHORSE for analyzing file; called from file based or decompression functions
        /// </summary>
        /// <param name="filename"></param>
        /// <param name="fileText"></param>
        void ProcessInMemory(string filePath, string fileText)
        {
            #region quickvalidation
            if (fileText.Length > MAX_FILESIZE)
            {
                WriteOnce.SafeLog("File too large: " + filePath, LogLevel.Trace);
                WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILESIZE_SKIPPED, filePath), LogLevel.Error);
                _appProfile.MetaData.FilesSkipped++;
                return;
            }

            if (!_arg_allowSampleFiles && _fileExclusionList.Any(v => filePath.Contains(v)))
            {
                WriteOnce.SafeLog("Part of excluded list: " + filePath, LogLevel.Trace);
                WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILESIZE_SKIPPED, filePath), LogLevel.Error);
                _appProfile.MetaData.FilesSkipped++;
                return;
            }

            //determine if file is a compressed item to unpackage for processing
            string language = Language.FromFileName(filePath);

            // Skip files written in unknown language
            if (string.IsNullOrEmpty(language))
            {
                WriteOnce.SafeLog("Language not found for file: " + filePath, LogLevel.Trace);
                language = Path.GetFileName(filePath);
                _appProfile.MetaData.FilesSkipped++;
                return;
            }
            else
            {
                WriteOnce.SafeLog("Preparing to process file: " + filePath, LogLevel.Trace);
            }

            #endregion

            #region minorRollupTrackingAndProgress

            _appProfile.MetaData.FilesAnalyzed++;
            _appProfile.MetaData.AddLanguage(language);
            _appProfile.MetaData.FileExtensions.Add(Path.GetExtension(filePath).Replace('.', ' ').TrimStart());
            LastUpdated = File.GetLastWriteTime(filePath);

            int totalFilesReviewed = _appProfile.MetaData.FilesAnalyzed + _appProfile.MetaData.FilesSkipped;
            int percentCompleted   = (int)((float)totalFilesReviewed / (float)_appProfile.MetaData.TotalFiles * 100);
            WriteOnce.General("\r" + ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILES_PROCESSED_PCNT, percentCompleted), false);

            #endregion

            //process file against rules
            Issue[] matches = _rulesProcessor.Analyze(fileText, language);

            //if any matches found for this file...
            if (matches.Count() > 0)
            {
                _appProfile.MetaData.FilesAffected++;
                _appProfile.MetaData.TotalMatchesCount += matches.Count();

                HashSet <string> uniqueTagsControl = new HashSet <string>();

                // Iterate through each match issue
                foreach (Issue match in matches)
                {
                    WriteOnce.SafeLog(string.Format("Processing pattern matches for ruleId {0}, ruleName {1} file {2}", match.Rule.Id, match.Rule.Name, filePath), LogLevel.Trace);

                    //maintain a list of unique tags; multi-purpose but primarily for filtering -u option
                    bool dupTagFound = false;
                    foreach (string t in match.Rule.Tags)
                    {
                        dupTagFound = !uniqueTagsControl.Add(t);
                    }

                    //save all unique dependendencies even if Dependency tag pattern is not-unique
                    var    tagPatternRegex = new Regex("Dependency.SourceInclude", RegexOptions.IgnoreCase);
                    String textMatch;
                    if (match.Rule.Tags.Any(v => tagPatternRegex.IsMatch(v)))
                    {
                        textMatch = ExtractDependency(fileText, match.Boundary.Index, match.PatternMatch, language);
                    }
                    else
                    {
                        textMatch = ExtractTextSample(fileText, match.Boundary.Index, match.Boundary.Length);
                    }

                    //wrap rule issue result to add metadata
                    MatchRecord record = new MatchRecord()
                    {
                        Filename   = filePath,
                        Language   = language,
                        Filesize   = fileText.Length,
                        TextSample = textMatch,
                        Excerpt    = ExtractExcerpt(fileText, match.StartLocation.Line),
                        Issue      = match
                    };

                    //preserve issue level characteristics as rolled up meta data of interest
                    bool valid = _appProfile.MetaData.AddStandardProperties(record);

                    //bail after extracting any dependency unique items IF user requested
                    if (_arg_outputUniqueTagsOnly && dupTagFound)
                    {
                        continue;
                    }
                    else if (valid)
                    {
                        _appProfile.MatchList.Add(record);
                    }
                }
            }
            else
            {
                WriteOnce.SafeLog("No pattern matches detected for file: " + filePath, LogLevel.Trace);
            }
        }
        /// <summary>
        /// Main WORKHORSE for analyzing file; called from file based or decompression functions
        /// </summary>
        /// <param name="filename"></param>
        /// <param name="fileText"></param>
        void ProcessInMemory(string filePath, string fileText, LanguageInfo languageInfo)
        {
            #region minorRollupTrackingAndProgress

            WriteOnce.SafeLog("Preparing to process file: " + filePath, LogLevel.Trace);

            _appProfile.MetaData.FilesAnalyzed++;

            int totalFilesReviewed = _appProfile.MetaData.FilesAnalyzed + _appProfile.MetaData.FilesSkipped;
            int percentCompleted   = (int)((float)totalFilesReviewed / (float)_appProfile.MetaData.TotalFiles * 100);
            //earlier issue now resolved so app handles mixed zipped/zipped and unzipped/zipped directories but catch all for non-critical UI
            if (percentCompleted > 100)
            {
                percentCompleted = 100;
            }

            WriteOnce.General("\r" + ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILES_PROCESSED_PCNT, percentCompleted), false);

            #endregion

            //process file against rules
            Issue[] matches = _rulesProcessor.Analyze(fileText, languageInfo);

            //if any matches found for this file...
            if (matches.Count() > 0)
            {
                _appProfile.MetaData.FilesAffected++;
                _appProfile.MetaData.TotalMatchesCount += matches.Count();

                // Iterate through each match issue
                foreach (Issue match in matches)
                {
                    WriteOnce.SafeLog(string.Format("Processing pattern matches for ruleId {0}, ruleName {1} file {2}", match.Rule.Id, match.Rule.Name, filePath), LogLevel.Trace);

                    //maintain a list of unique tags; multi-purpose but primarily for filtering -d option
                    bool dupTagFound = false;
                    foreach (string t in match.Rule.Tags)
                    {
                        dupTagFound = !_uniqueTagsControl.Add(t);
                    }

                    //save all unique dependencies even if Dependency tag pattern is not-unique
                    var    tagPatternRegex = new Regex("Dependency.SourceInclude", RegexOptions.IgnoreCase);
                    String textMatch;
                    if (match.Rule.Tags.Any(v => tagPatternRegex.IsMatch(v)))
                    {
                        textMatch = ExtractDependency(fileText, match.Boundary.Index, match.PatternMatch, languageInfo.Name);
                    }
                    else
                    {
                        textMatch = ExtractTextSample(fileText, match.Boundary.Index, match.Boundary.Length);
                    }

                    //wrap rule issue result to add metadata
                    MatchRecord record = new MatchRecord()
                    {
                        Filename   = filePath,
                        Language   = languageInfo,
                        Filesize   = fileText.Length,
                        TextSample = textMatch,
                        Excerpt    = ExtractExcerpt(fileText, match.StartLocation.Line),
                        Issue      = match
                    };

                    //preserve issue level characteristics as rolled up meta data of interest
                    bool addAsFeatureMatch = _appProfile.MetaData.AddStandardProperties(ref record);

                    //bail after extracting any dependency unique items IF user requested
                    if (_arg_outputUniqueTagsOnly && dupTagFound)
                    {
                        continue;
                    }
                    else if (addAsFeatureMatch)
                    {
                        _appProfile.MatchList.Add(record);
                    }
                }
            }
            else
            {
                WriteOnce.SafeLog("No pattern matches detected for file: " + filePath, LogLevel.Trace);
            }
        }
Пример #9
0
        /// <summary>
        /// Main WORKHORSE for analyzing file; called from file based or decompression functions
        /// </summary>
        /// <param name="filename"></param>
        /// <param name="fileText"></param>
        void ProcessInMemory(string filePath, string fileText)
        {
            #region quickvalidation
            if (fileText.Length > MAX_FILESIZE)
            {
                WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILESIZE_SKIPPED, filePath), LogLevel.Warn);
                _appProfile.MetaData.FilesSkipped++;
                return;
            }

            //exclude sample, test or similar files by default or as specified in exclusion list
            if (!_arg_allowSampleFiles && _fileExclusionList.Any(v => filePath.ToLower().Contains(v)))
            {
                WriteOnce.SafeLog("Part of excluded list: " + filePath, LogLevel.Trace);
                WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILESIZE_SKIPPED, filePath), LogLevel.Trace);
                _appProfile.MetaData.FilesSkipped++;
                return;
            }

            //check for supported language
            LanguageInfo languageInfo = new LanguageInfo();
            // Skip files written in unknown language
            if (!Language.FromFileName(filePath, ref languageInfo))
            {
                WriteOnce.SafeLog("Language not found for file: " + filePath, LogLevel.Trace);
                _appProfile.MetaData.FilesSkipped++;
                return;
            }
            else
            {
                WriteOnce.SafeLog("Preparing to process file: " + filePath, LogLevel.Trace);
            }

            #endregion

            #region minorRollupTrackingAndProgress

            _appProfile.MetaData.FilesAnalyzed++;
            _appProfile.MetaData.AddLanguage(languageInfo.Name);
            _appProfile.MetaData.FileExtensions.Add(Path.GetExtension(filePath).Replace('.', ' ').TrimStart());
            LastUpdated = File.GetLastWriteTime(filePath);

            int totalFilesReviewed = _appProfile.MetaData.FilesAnalyzed + _appProfile.MetaData.FilesSkipped;
            int percentCompleted   = (int)((float)totalFilesReviewed / (float)_appProfile.MetaData.TotalFiles * 100);
            //reported: if a zip contains more zip files in it the total count may be off -complex.  ~workaround: freeze UI
            if (percentCompleted > 100)
            {
                percentCompleted = 100;
            }

            WriteOnce.General("\r" + ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILES_PROCESSED_PCNT, percentCompleted), false);

            #endregion

            //process file against rules
            Issue[] matches = _rulesProcessor.Analyze(fileText, languageInfo.Name);

            //if any matches found for this file...
            if (matches.Count() > 0)
            {
                _appProfile.MetaData.FilesAffected++;
                _appProfile.MetaData.TotalMatchesCount += matches.Count();

                // Iterate through each match issue
                foreach (Issue match in matches)
                {
                    WriteOnce.SafeLog(string.Format("Processing pattern matches for ruleId {0}, ruleName {1} file {2}", match.Rule.Id, match.Rule.Name, filePath), LogLevel.Trace);

                    //do not accept features from build type files (only metadata) to avoid false positives that are not part of the executable program
                    if (languageInfo.Type == LanguageInfo.LangFileType.Build && match.Rule.Tags.Any(v => !v.Contains("Metadata")))
                    {
                        continue;
                    }

                    //maintain a list of unique tags; multi-purpose but primarily for filtering -d option
                    bool dupTagFound = false;
                    foreach (string t in match.Rule.Tags)
                    {
                        dupTagFound = !_uniqueTagsControl.Add(t);
                    }

                    //save all unique dependendencies even if Dependency tag pattern is not-unique
                    var    tagPatternRegex = new Regex("Dependency.SourceInclude", RegexOptions.IgnoreCase);
                    String textMatch;
                    if (match.Rule.Tags.Any(v => tagPatternRegex.IsMatch(v)))
                    {
                        textMatch = ExtractDependency(fileText, match.Boundary.Index, match.PatternMatch, languageInfo.Name);
                    }
                    else
                    {
                        textMatch = ExtractTextSample(fileText, match.Boundary.Index, match.Boundary.Length);
                    }

                    //wrap rule issue result to add metadata
                    MatchRecord record = new MatchRecord()
                    {
                        Filename   = filePath,
                        Language   = languageInfo,
                        Filesize   = fileText.Length,
                        TextSample = textMatch,
                        Excerpt    = ExtractExcerpt(fileText, match.StartLocation.Line),
                        Issue      = match
                    };

                    //preserve issue level characteristics as rolled up meta data of interest
                    bool addAsFeatureMatch = _appProfile.MetaData.AddStandardProperties(record);

                    //bail after extracting any dependency unique items IF user requested
                    if (_arg_outputUniqueTagsOnly && dupTagFound)
                    {
                        continue;
                    }
                    else if (addAsFeatureMatch)
                    {
                        _appProfile.MatchList.Add(record);
                    }
                }
            }
            else
            {
                WriteOnce.SafeLog("No pattern matches detected for file: " + filePath, LogLevel.Trace);
            }
        }
        /// <summary>
        /// Main entry point to start analysis; handles setting up rules, directory enumeration
        /// file type detection and handoff
        /// Pre: All Configure Methods have been called already and we are ready to SCAN
        /// </summary>
        /// <returns></returns>
        public int Run()
        {
            WriteOnce.SafeLog("AnalyzeCommand::Run", LogLevel.Trace);

            DateTime start = DateTime.Now;

            WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_RUNNING, "Analyze"));

            _appProfile.MetaData.TotalFiles = _srcfileList.Count();//updated for zipped files later

            // Iterate through all files and process against rules
            foreach (string filename in _srcfileList)
            {
                //exclude sample, test or similar files by default or as specified in exclusion list
                if (!_arg_allowSampleFiles && _fileExclusionList.Any(v => filename.ToLower().Contains(v)))
                {
                    WriteOnce.SafeLog("Part of excluded list: " + filename, LogLevel.Trace);
                    WriteOnce.SafeLog(ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILESIZE_SKIPPED, filename), LogLevel.Trace);
                    _appProfile.MetaData.FilesSkipped++;
                    continue;
                }

                ArchiveFileType archiveFileType = MiniMagic.DetectFileType(filename);
                if (archiveFileType == ArchiveFileType.UNKNOWN)
                {
                    ProcessAsFile(filename);
                }
                else
                {
                    UnZipAndProcess(filename, archiveFileType);
                }
            }

            WriteOnce.General("\r" + ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILES_PROCESSED_PCNT, 100));
            WriteOnce.Operation(ErrMsg.GetString(ErrMsg.ID.CMD_PREPARING_REPORT));

            //Prepare report results
            _appProfile.MetaData.LastUpdated = LastUpdated.ToString();
            _appProfile.DateScanned          = DateScanned.ToString();
            _appProfile.PrepareReport();
            TimeSpan timeSpan = start - DateTime.Now;

            WriteOnce.SafeLog(String.Format("Processing time: seconds:{0}", timeSpan.TotalSeconds * -1), LogLevel.Trace);
            FlushAll();

            //wrapup result status
            if (_appProfile.MetaData.TotalFiles == _appProfile.MetaData.FilesSkipped)
            {
                WriteOnce.Error(ErrMsg.GetString(ErrMsg.ID.ANALYZE_NOSUPPORTED_FILETYPES));
            }
            else if (_appProfile.MatchList.Count == 0)
            {
                WriteOnce.Error(ErrMsg.GetString(ErrMsg.ID.ANALYZE_NOPATTERNS));
            }
            else
            {
                WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_COMPLETED, "Analyze"));
            }

            return(_appProfile.MatchList.Count() == 0 ? (int)ExitCode.NoMatches :
                   (int)ExitCode.MatchesFound);
        }
        public int Run()
        {
            WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_RUNNING, "Tagdiff"));

            //setup output
            TextWriter outputWriter;

            if (!string.IsNullOrEmpty(_arg_outputFile))
            {
                outputWriter = File.CreateText(_arg_outputFile);
                outputWriter.WriteLine(Program.GetVersionString());
                WriteOnce.Writer = outputWriter;
            }

            if (_arg_src1 == _arg_src2)
            {
                throw new OpException(ErrMsg.GetString(ErrMsg.ID.TAGDIFF_SAME_FILE_ARG));
            }
            else if (string.IsNullOrEmpty(_arg_src1) || string.IsNullOrEmpty(_arg_src2))
            {
                throw new OpException(ErrMsg.GetString(ErrMsg.ID.CMD_INVALID_ARG_VALUE));
            }

            #region setup analyze calls

            //save to quiet analyze cmd
            WriteOnce.ConsoleVerbosity saveVerbosity = WriteOnce.Verbosity;

            string tmp1 = Path.GetTempFileName();
            string tmp2 = Path.GetTempFileName();

            AnalyzeCommand.ExitCode result1 = AnalyzeCommand.ExitCode.CriticalError;
            AnalyzeCommand.ExitCode result2 = AnalyzeCommand.ExitCode.CriticalError;

            try
            {
                AnalyzeCommand cmd1 = new AnalyzeCommand(new AnalyzeCommandOptions
                {
                    SourcePath            = _arg_src1,
                    OutputFilePath        = tmp1,
                    OutputFileFormat      = "json",
                    CustomRulesPath       = _arg_rulesPath,
                    IgnoreDefaultRules    = _arg_ignoreDefault,
                    SimpleTagsOnly        = true,
                    AllowDupTags          = false,
                    FilePathExclusions    = "sample,example,test,docs,.vs,.git",
                    ConsoleVerbosityLevel = "None"
                });
                AnalyzeCommand cmd2 = new AnalyzeCommand(new AnalyzeCommandOptions
                {
                    SourcePath            = _arg_src2,
                    OutputFilePath        = tmp2,
                    OutputFileFormat      = "json",
                    CustomRulesPath       = _arg_rulesPath,
                    IgnoreDefaultRules    = _arg_ignoreDefault,
                    SimpleTagsOnly        = true,
                    AllowDupTags          = false,
                    FilePathExclusions    = "sample,example,test,docs,.vs,.git",
                    ConsoleVerbosityLevel = "None"
                });


                result1 = (AnalyzeCommand.ExitCode)cmd1.Run();
                result2 = (AnalyzeCommand.ExitCode)cmd2.Run();
            }
            catch (Exception e)
            {
                //restore
                WriteOnce.Verbosity = saveVerbosity;
                throw e;
            }

            //restore
            WriteOnce.Verbosity = saveVerbosity;

            #endregion

            bool successResult;
            bool equal1 = true;
            bool equal2 = true;

            //process results for each analyze call before comparing results
            if (result1 == AnalyzeCommand.ExitCode.CriticalError)
            {
                throw new OpException(ErrMsg.FormatString(ErrMsg.ID.CMD_CRITICAL_FILE_ERR, _arg_src1));
            }
            else if (result2 == AnalyzeCommand.ExitCode.CriticalError)
            {
                throw new OpException(ErrMsg.FormatString(ErrMsg.ID.CMD_CRITICAL_FILE_ERR, _arg_src2));
            }
            else if (result1 == AnalyzeCommand.ExitCode.NoMatches || result2 == AnalyzeCommand.ExitCode.NoMatches)
            {
                throw new OpException(ErrMsg.GetString(ErrMsg.ID.TAGDIFF_NO_TAGS_FOUND));
            }
            else //compare tag results; assumed (result1&2 == AnalyzeCommand.ExitCode.MatchesFound)
            {
                string file1TagsJson = File.ReadAllText(tmp1);
                string file2TagsJson = File.ReadAllText(tmp2);

                var file1Tags = JsonConvert.DeserializeObject <TagsFile[]>(file1TagsJson).First();
                var file2Tags = JsonConvert.DeserializeObject <TagsFile[]>(file2TagsJson).First();

                //can't simply compare counts as content may differ; must compare both in directions in two passes a->b; b->a
                //first pass
                WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.TAGDIFF_RESULTS_GAP, Path.GetFileName(_arg_src1), Path.GetFileName(_arg_src2)),
                                  true, WriteOnce.ConsoleVerbosity.High);
                equal1 = CompareTags(file1Tags.Tags, file2Tags.Tags);

                //reverse order for second pass
                WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.TAGDIFF_RESULTS_GAP, Path.GetFileName(_arg_src2), Path.GetFileName(_arg_src1)),
                                  true, WriteOnce.ConsoleVerbosity.High);
                equal2 = CompareTags(file2Tags.Tags, file1Tags.Tags);

                //final results
                bool resultsDiffer = !(equal1 && equal2);
                if (_arg_tagTestType == TagTestType.Inequality && resultsDiffer)
                {
                    successResult = true;
                }
                else if (_arg_tagTestType == TagTestType.Equality && !resultsDiffer)
                {
                    successResult = true;
                }
                else
                {
                    successResult = false;
                }

                WriteOnce.General(ErrMsg.GetString(ErrMsg.ID.TAGDIFF_RESULTS_DIFFER), false);
                WriteOnce.Result(resultsDiffer.ToString());
            }

            //cleanup
            try
            {
                File.Delete(tmp1);
                File.Delete(tmp2);
            }
            catch
            {
                //no action needed;
            }

            WriteOnce.FlushAll();
            WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_COMPLETED, "Tagdiff"));

            return(successResult ? (int)ExitCode.TestPassed : (int)ExitCode.TestFailed);
        }
Пример #12
0
        public int Run()
        {
            WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_RUNNING, "tagtest"));

            //init based on true or false present argument value
            bool testSuccess = true;

            //one file vs ruleset
            string tmp1 = Path.GetTempFileName();

            WriteOnce.ConsoleVerbosity saveVerbosity = WriteOnce.Verbosity;
            AnalyzeCommand.ExitCode    result        = AnalyzeCommand.ExitCode.CriticalError;

            //setup analyze call with silent option
            #region analyzesetup
            try
            {
                AnalyzeCommand cmd1 = new AnalyzeCommand(new AnalyzeCommandOptions
                {
                    SourcePath            = _arg_srcPath,
                    OutputFilePath        = tmp1,
                    OutputFileFormat      = "json",
                    CustomRulesPath       = _arg_customRulesPath,
                    IgnoreDefaultRules    = _arg_ignoreDefaultRules,
                    SimpleTagsOnly        = true,
                    AllowDupTags          = false,
                    ConsoleVerbosityLevel = "None"
                });


                //quiet analysis commands
                result = (AnalyzeCommand.ExitCode)cmd1.Run();
            }
            catch (Exception e)
            {
                WriteOnce.Verbosity = saveVerbosity;
                throw e;
            }

            //restore
            WriteOnce.Verbosity = saveVerbosity;

            if (result == AnalyzeCommand.ExitCode.CriticalError)
            {
                throw new OpException(ErrMsg.GetString(ErrMsg.ID.CMD_CRITICAL_FILE_ERR));
            }
            else if (result == AnalyzeCommand.ExitCode.NoMatches)
            {
                //results
                WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.TAGTEST_RESULTS_TEST_TYPE, _arg_tagTestType.ToString()), false, WriteOnce.ConsoleVerbosity.Low);
                if (_arg_tagTestType == TagTestType.RulesPresent)
                {
                    WriteOnce.Any(ErrMsg.GetString(ErrMsg.ID.TAGTEST_RESULTS_FAIL), true, ConsoleColor.Red, WriteOnce.ConsoleVerbosity.Low);
                }
                else
                {
                    WriteOnce.Any(ErrMsg.GetString(ErrMsg.ID.TAGTEST_RESULTS_SUCCESS), true, ConsoleColor.Green, WriteOnce.ConsoleVerbosity.Low);
                }

                WriteOnce.FlushAll();
                WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_COMPLETED, "Tagtest"));

                return((int)ExitCode.CriticalError);
            }

            #endregion

            //assumed (result == AnalyzeCommand.ExitCode.MatchesFound)
            string file1TagsJson = File.ReadAllText(tmp1);
            var    file1TagsObj  = JsonConvert.DeserializeObject <TagsFile[]>(file1TagsJson);
            var    file1Tags     = file1TagsObj.First(); // here we have a single FileList object
            File.Delete(tmp1);

            foreach (Rule r in _rulesSet)
            {
                //supports both directions by generalizing
                string[] testList1 = _arg_tagTestType == TagTestType.RulesNotPresent ?
                                     r.Tags : file1Tags.Tags;

                string[] testList2 = _arg_tagTestType == TagTestType.RulesNotPresent ?
                                     file1Tags.Tags : r.Tags;

                foreach (string t in testList2)
                {
                    if (TagTest(testList1, t))
                    {
                        WriteOnce.Result(ErrMsg.FormatString(ErrMsg.ID.TAGTEST_RESULTS_TAGS_FOUND, t), true, WriteOnce.ConsoleVerbosity.High);
                    }
                    else
                    {
                        testSuccess = false;
                        WriteOnce.Result(ErrMsg.FormatString(ErrMsg.ID.TAGTEST_RESULTS_TAGS_MISSING, t), true, WriteOnce.ConsoleVerbosity.High);
                    }
                }
            }

            //results
            WriteOnce.General(ErrMsg.FormatString(ErrMsg.ID.TAGTEST_RESULTS_TEST_TYPE, _arg_tagTestType.ToString()), false, WriteOnce.ConsoleVerbosity.Low);
            if (testSuccess)
            {
                WriteOnce.Any(ErrMsg.GetString(ErrMsg.ID.TAGTEST_RESULTS_SUCCESS), true, ConsoleColor.Green, WriteOnce.ConsoleVerbosity.Low);
            }
            else
            {
                WriteOnce.Any(ErrMsg.GetString(ErrMsg.ID.TAGTEST_RESULTS_FAIL), true, ConsoleColor.Red, WriteOnce.ConsoleVerbosity.Low);
            }

            WriteOnce.FlushAll();
            WriteOnce.Operation(ErrMsg.FormatString(ErrMsg.ID.CMD_COMPLETED, "Tagtest"));

            return(testSuccess ? (int)ExitCode.NoDiff : (int)ExitCode.DiffFound);
        }