/// <summary>
        /// Attempt to map application type tags or file type or language to identify
        /// WebApplications, Windows Services, Client Apps, WebServices, Azure Functions etc.
        /// </summary>
        /// <param name="match"></param>
        public string DetectSolutionType(MatchRecord match)
        {
            string result = "";

            if (match.Tags is not null && match.Tags.Any(s => s.Contains("Application.Type")))
            {
                foreach (string tag in match.Tags ?? new string[] { })
                {
                    int index = tag.IndexOf("Application.Type");
                    if (-1 != index)
                    {
                        result = tag.Substring(index + 17);
                        break;
                    }
                }
            }
        /// <summary>
        /// Assist in aggregating reporting properties of matches as they are added
        /// Keeps helpers isolated from MetaData class which is used as a result object to keep pure
        /// </summary>
        /// <param name="matchRecord"></param>
        public void AddTagsFromMatchRecord(MatchRecord matchRecord)
        {
            //special handling for standard characteristics in report
            foreach (var tag in matchRecord.Tags ?? Array.Empty <string>())
            {
                switch (tag)
                {
                case "Metadata.Application.Author":
                case "Metadata.Application.Publisher":
                    Metadata.Authors = ExtractValue(matchRecord.Sample);
                    break;

                case "Metadata.Application.Description":
                    Metadata.Description = ExtractValue(matchRecord.Sample);
                    break;

                case "Metadata.Application.Name":
                    Metadata.ApplicationName = ExtractValue(matchRecord.Sample);
                    break;

                case "Metadata.Application.Version":
                    Metadata.SourceVersion = ExtractValue(matchRecord.Sample);
                    break;

                case "Metadata.Application.Target.Processor":
                    CPUTargets.TryAdd(ExtractValue(matchRecord.Sample).ToLower(), 0);
                    break;

                case "Metadata.Application.Output.Type":
                    Outputs.TryAdd(ExtractValue(matchRecord.Sample).ToLower(), 0);
                    break;

                case "Dependency.SourceInclude":
                    return;     //design to keep noise out of detailed match list

                default:
                    if (tag.Split('.').Contains("Metric"))
                    {
                        _ = TagCounters.TryAdd(tag, new MetricTagCounter()
                        {
                            Tag = tag
                        });
                    }
                    else if (tag.Contains(".Platform.OS"))
                    {
                        OSTargets.TryAdd(tag[(tag.LastIndexOf('.', tag.Length - 1) + 1)..], 0);
        public void WriteMatch(MatchRecord match)
        {
            string output = _formatString.Replace("%F", match.Filename);

            output = output.Replace("%l", match.Language.Name);
            output = output.Replace("%t", match.Language.Type.ToString());
            output = output.Replace("%L", match.Issue.StartLocation.Line.ToString());
            output = output.Replace("%C", match.Issue.StartLocation.Column.ToString());
            output = output.Replace("%l", match.Issue.EndLocation.Line.ToString());
            output = output.Replace("%c", match.Issue.EndLocation.Column.ToString());
            output = output.Replace("%I", match.Issue.Boundary.Index.ToString());
            output = output.Replace("%i", match.Issue.Boundary.Length.ToString());
            output = output.Replace("%R", match.Issue.Rule.Id);
            output = output.Replace("%N", match.Issue.Rule.Name);
            output = output.Replace("%S", match.Issue.Rule.Severity.ToString());
            output = output.Replace("%X", match.Issue.Confidence.ToString());                  //override rule confidence because of unstructured text vs source
            output = output.Replace("%D", match.Issue.Rule.Description);
            output = output.Replace("%m", System.Net.WebUtility.HtmlEncode(match.TextSample)); //readability for non-browser format type
            output = output.Replace("%T", string.Join(',', match.Issue.Rule.Tags));

            TextWriter.WriteLine(output);
        }
示例#4
0
 public LimitedMatchRecord(MatchRecord matchRecord)
 {
     FileName            = matchRecord.Filename;
     SourceLabel         = matchRecord.Language.Name;
     SourceType          = matchRecord.Language.Type.ToString();
     StartLocationLine   = matchRecord.Issue.StartLocation.Line;
     StartLocationColumn = matchRecord.Issue.StartLocation.Column;
     EndLocationLine     = matchRecord.Issue.EndLocation.Line;
     EndLocationColumn   = matchRecord.Issue.EndLocation.Column;
     BoundaryIndex       = matchRecord.Issue.Boundary.Index;
     BoundaryLength      = matchRecord.Issue.Boundary.Length;
     RuleId            = matchRecord.Issue.Rule.Id;
     Severity          = matchRecord.Issue.Rule.Severity.ToString();
     RuleName          = matchRecord.Issue.Rule.Name;
     RuleDescription   = matchRecord.Issue.Rule.Description;
     PatternConfidence = matchRecord.Issue.Confidence.ToString();
     PatternType       = matchRecord.Issue.PatternMatch.PatternType.ToString();
     MatchingPattern   = matchRecord.Issue.PatternMatch.Pattern;
     Sample            = matchRecord.TextSample;
     Excerpt           = matchRecord.Excerpt;
     Tags = matchRecord.Issue.Rule.Tags;
 }
        /// <summary>
        /// Main WORKHORSE for analyzing file; called from file based or decompression functions
        /// </summary>
        /// <param name="filename"></param>
        /// <param name="fileText"></param>
        void ProcessInMemory(string filePath, string fileText, LanguageInfo languageInfo)
        {
            #region minorRollupTrackingAndProgress

            WriteOnce.SafeLog("Preparing to process file: " + filePath, LogLevel.Trace);

            _appProfile.MetaData.FilesAnalyzed++;

            int totalFilesReviewed = _appProfile.MetaData.FilesAnalyzed + _appProfile.MetaData.FilesSkipped;
            int percentCompleted   = (int)((float)totalFilesReviewed / (float)_appProfile.MetaData.TotalFiles * 100);
            //earlier issue now resolved so app handles mixed zipped/zipped and unzipped/zipped directories but catch all for non-critical UI
            if (percentCompleted > 100)
            {
                percentCompleted = 100;
            }
            else if (percentCompleted < 100) //caller already reports @100% so avoid 2x for file output
            {
                WriteOnce.General("\r" + ErrMsg.FormatString(ErrMsg.ID.ANALYZE_FILES_PROCESSED_PCNT, percentCompleted), false);
            }

            #endregion

            //process file against rules
            Issue[] matches = _rulesProcessor.Analyze(fileText, languageInfo);

            //if any matches found for this file...
            if (matches.Count() > 0)
            {
                _appProfile.MetaData.FilesAffected++;
                _appProfile.MetaData.TotalMatchesCount += matches.Count();

                // Iterate through each match issue
                foreach (Issue match in matches)
                {
                    WriteOnce.SafeLog(string.Format("Processing pattern matches for ruleId {0}, ruleName {1} file {2}", match.Rule.Id, match.Rule.Name, filePath), LogLevel.Trace);

                    //maintain a list of unique tags; multi-purpose but primarily for filtering -d option
                    bool dupTagFound = false;
                    foreach (string t in match.Rule.Tags)
                    {
                        dupTagFound = !_uniqueTagsControl.Add(t);
                    }

                    //save all unique dependencies even if Dependency tag pattern is not-unique
                    var    tagPatternRegex = new Regex("Dependency.SourceInclude", RegexOptions.IgnoreCase);
                    String textMatch;
                    if (match.Rule.Tags.Any(v => tagPatternRegex.IsMatch(v)))
                    {
                        textMatch = ExtractDependency(fileText, match.Boundary.Index, match.PatternMatch, languageInfo.Name);
                    }
                    else
                    {
                        textMatch = ExtractTextSample(fileText, match.Boundary.Index, match.Boundary.Length);
                    }

                    //wrap rule issue result to add metadata
                    MatchRecord record = new MatchRecord()
                    {
                        Filename   = filePath,
                        Language   = languageInfo,
                        Filesize   = fileText.Length,
                        TextSample = textMatch,
                        Excerpt    = ExtractExcerpt(fileText, match.StartLocation.Line),
                        Issue      = match
                    };

                    //preserve issue level characteristics as rolled up meta data of interest
                    bool addAsFeatureMatch = _appProfile.MetaData.AddStandardProperties(ref record);

                    //bail after extracting any dependency unique items IF user requested
                    if (_arg_outputUniqueTagsOnly && dupTagFound)
                    {
                        continue;
                    }
                    else if (addAsFeatureMatch)
                    {
                        _appProfile.MatchList.Add(record);
                    }
                    else
                    {
                        _appProfile.MetaData.TotalMatchesCount -= 1;//reduce e.g. tag counters only as per preferences file
                    }
                }
            }
            else
            {
                WriteOnce.SafeLog("No pattern matches detected for file: " + filePath, LogLevel.Trace);
            }
        }
        /// <summary>
        /// Attempt to map application type tags or file type or language to identify
        /// WebApplications, Windows Services, Client Apps, WebServices, Azure Functions etc.
        /// </summary>
        /// <param name="match"></param>
        static public String DetectSolutionType(MatchRecord match)
        {
            string result = "";

            if (match.Issue.Rule.Tags.Any(s => s.Contains("Application.Type")))
            {
                foreach (string tag in match.Issue.Rule.Tags)
                {
                    int index = tag.IndexOf("Application.Type");
                    if (-1 != index)
                    {
                        result = tag.Substring(index + 17);
                        break;
                    }
                }
            }
            else
            {
                switch (match.Filename)
                {
                case "web.config":
                    result = "Web.Application";
                    break;

                case "app.config":
                    result = ".NETclient";
                    break;

                default:
                    switch (Path.GetExtension(match.Filename))
                    {
                    case ".cshtml":
                        result = "Web.Application";
                        break;

                    case ".htm":
                    case ".html":
                    case ".js":
                    case ".ts":
                        result = "Web.Application";
                        break;

                    case "powershell":
                    case "shellscript":
                    case "wincmdscript":
                        result = "script";
                        break;

                    default:
                        switch (match.Language.Name)
                        {
                        case "ruby":
                        case "perl":
                        case "php":
                            result = "Web.Application";
                            break;
                        }
                        break;
                    }
                    break;
                }
            }

            return(result.ToLower());
        }
示例#7
0
        /// <summary>
        /// Part of post processing to test for matches against app defined properties
        /// defined in MetaData class
        /// Exludes a match if specified in preferences as a counted tag with exclude true
        /// </summary>
        /// <param name="matchRecord"></param>
        public bool AddStandardProperties(ref MatchRecord matchRecord)
        {
            bool includeAsMatch = true;

            //testing for presence of a tag against the specified set in preferences for report org
            foreach (string key in _propertyTagSearchPatterns.Keys)
            {
                var tagPatternRegex = new Regex(_propertyTagSearchPatterns[key], RegexOptions.IgnoreCase);
                if (matchRecord.Issue.Rule.Tags.Any(v => tagPatternRegex.IsMatch(v)))
                {
                    KeyedPropertyLists[key].Add(matchRecord.TextSample);
                }
            }

            // Author etc. or STANDARD METADATA properties we capture from any supported file type; others just captured as general tag matches...
            if (matchRecord.Issue.Rule.Tags.Any(v => v.Contains("Metadata.Application.Author")))
            {
                this.Authors = ExtractValue(matchRecord.TextSample);
            }
            if (matchRecord.Issue.Rule.Tags.Any(v => v.Contains("Metadata.Application.Publisher")))
            {
                this.Authors = ExtractValue(matchRecord.TextSample);
            }
            if (matchRecord.Issue.Rule.Tags.Any(v => v.Contains("Metadata.Application.Description")))
            {
                this.Description = ExtractValue(matchRecord.TextSample);
            }
            if (matchRecord.Issue.Rule.Tags.Any(v => v.Contains("Metadata.Application.Name")))
            {
                this.ApplicationName = ExtractValue(matchRecord.TextSample);
            }
            if (matchRecord.Issue.Rule.Tags.Any(v => v.Contains("Metadata.Application.Version")))
            {
                this.SourceVersion = ExtractValue(matchRecord.TextSample);
            }
            if (matchRecord.Issue.Rule.Tags.Any(v => v.Contains("Metadata.Application.Target.Processor")))
            {
                this.CPUTargets.Add(ExtractValue(matchRecord.TextSample).ToLower());
            }
            if (matchRecord.Issue.Rule.Tags.Any(v => v.Contains("Metadata.Application.Output.Type")))
            {
                this.Outputs.Add(ExtractValue(matchRecord.TextSample).ToLower());
            }
            if (matchRecord.Issue.Rule.Tags.Any(v => v.Contains("Platform.OS")))
            {
                this.OSTargets.Add(ExtractValue(matchRecord.TextSample).ToLower());
            }

            //Special handling; attempt to detect app types...review for multiple pattern rule limitation
            String solutionType = Utils.DetectSolutionType(matchRecord);

            if (!string.IsNullOrEmpty(solutionType))
            {
                AppTypes.Add(solutionType);
            }

            //Update metric counters for default or user specified tags
            foreach (TagCounter counter in TagCounters)
            {
                if (matchRecord.Issue.Rule.Tags.Any(v => v.Contains(counter.Tag)))
                {
                    counter.Count++;
                    includeAsMatch = counter.IncludeAsMatch;//Exclude as feature matches per preferences from reporting full match details
                }
            }

            //once patterns checked; prepare text for output blocking browser xss
            matchRecord.TextSample = System.Net.WebUtility.HtmlEncode(matchRecord.TextSample);

            return(includeAsMatch);
        }
示例#8
0
        /// <summary>
        /// Assist in aggregating reporting properties of matches as they are added
        /// Keeps helpers isolated from MetaData class which is used as a result object to keep pure
        /// </summary>
        /// <param name="matchRecord"></param>
        public void AddMatchRecord(MatchRecord matchRecord)
        {
            //aggregate lists of matches against standard set of properties to report on
            foreach (string key in _propertyTagSearchPatterns.Keys)
            {
                var tagPatternRegex = new Regex(_propertyTagSearchPatterns[key], RegexOptions.IgnoreCase);
                if (matchRecord.Tags.Any(v => tagPatternRegex.IsMatch(v)))
                {
                    Metadata.KeyedPropertyLists[key].Add(matchRecord.Sample);
                }
            }

            // single standard properties we capture from any supported file type; others just captured as general tag matches...
            if (matchRecord.Tags.Any(v => v.Contains("Metadata.Application.Author")))
            {
                Metadata.Authors = ExtractValue(matchRecord.Sample);
            }

            if (matchRecord.Tags.Any(v => v.Contains("Metadata.Application.Publisher")))
            {
                Metadata.Authors = ExtractValue(matchRecord.Sample);
            }

            if (matchRecord.Tags.Any(v => v.Contains("Metadata.Application.Description")))
            {
                Metadata.Description = ExtractValue(matchRecord.Sample);
            }

            if (matchRecord.Tags.Any(v => v.Contains("Metadata.Application.Name")))
            {
                Metadata.ApplicationName = ExtractValue(matchRecord.Sample);
            }

            if (matchRecord.Tags.Any(v => v.Contains("Metadata.Application.Version")))
            {
                Metadata.SourceVersion = ExtractValue(matchRecord.Sample);
            }

            if (matchRecord.Tags.Any(v => v.Contains("Metadata.Application.Target.Processor")))
            {
                Metadata.CPUTargets.Add(ExtractValue(matchRecord.Sample).ToLower());
            }

            if (matchRecord.Tags.Any(v => v.Contains("Metadata.Application.Output.Type")))
            {
                Metadata.Outputs.Add(ExtractValue(matchRecord.Sample).ToLower());
            }

            if (matchRecord.Tags.Any(v => v.Contains("Platform.OS")))
            {
                Metadata.OSTargets.Add(ExtractValue(matchRecord.Sample).ToLower());
            }

            if (matchRecord.Tags.Any(v => v.Contains("Metric.")))
            {
                Metadata.TagCounters.Add(new MetricTagCounter()
                {
                    Tag   = matchRecord.Tags[0],
                    Count = 0
                });
            }

            //safeguard sample output now that we've matched properties for blocking browser xss
            matchRecord.Sample = System.Net.WebUtility.HtmlEncode(matchRecord.Sample);

            //Special handling; attempt to detect app types...review for multiple pattern rule limitation
            String solutionType = DetectSolutionType(matchRecord);

            if (!string.IsNullOrEmpty(solutionType))
            {
                Metadata.AppTypes.Add(solutionType);
            }

            //Update metric counters for default or user specified tags; don't add as match detail
            bool counterOnlyTag = false;

            foreach (MetricTagCounter counter in Metadata.TagCounters)
            {
                if (matchRecord.Tags.Any(v => v.Contains(counter.Tag)))
                {
                    counterOnlyTag = true;
                    counter.Count++;
                    break;
                }
            }

            //omit adding if only a counter metric tag
            if (!counterOnlyTag)
            {
                //update list of unique tags as we go
                foreach (string tag in matchRecord.Tags)
                {
                    Metadata.UniqueTags.Add(tag);
                }

                Metadata.Matches.Add(matchRecord);
            }
            else
            {
                Metadata.TotalMatchesCount -= 1;//reduce e.g. tag counters not included as detailed match
            }
        }
        /// <summary>
        /// Assist in aggregating reporting properties of matches as they are added
        /// Keeps helpers isolated from MetaData class which is used as a result object to keep pure
        /// </summary>
        /// <param name="matchRecord"></param>
        public void AddMatchRecord(MatchRecord matchRecord)
        {
            //aggregate lists of matches against standard set of properties to report on
            foreach (string key in _propertyTagSearchPatterns.Keys)
            {
                if (matchRecord.Tags.Any(v => _propertyTagSearchPatterns[key].IsMatch(v)))
                {
                    _ = Metadata.KeyedPropertyLists[key].TryAdd(matchRecord.Sample, 0);
                }
            }

            //Update metric counters for default or user specified tags; don't add as match detail
            foreach (var tag in matchRecord.Tags)
            {
                switch (tag)
                {
                case "Metadata.Application.Author":
                case "Metadata.Application.Publisher":
                    Metadata.Authors = ExtractValue(matchRecord.Sample);
                    break;

                case "Metadata.Application.Description":
                    Metadata.Description = ExtractValue(matchRecord.Sample);
                    break;

                case "Metadata.Application.Name":
                    Metadata.ApplicationName = ExtractValue(matchRecord.Sample);
                    break;

                case "Metadata.Application.Version":
                    Metadata.SourceVersion = ExtractValue(matchRecord.Sample);
                    break;

                case "Metadata.Application.Target.Processor":
                    _ = Metadata.CPUTargets.TryAdd(ExtractValue(matchRecord.Sample).ToLower(), 0);
                    break;

                case "Metadata.Application.Output.Type":
                    _ = Metadata.Outputs.TryAdd(ExtractValue(matchRecord.Sample).ToLower(), 0);
                    break;

                case "Platform.OS":
                    _ = Metadata.OSTargets.TryAdd(ExtractValue(matchRecord.Sample).ToLower(), 0);
                    break;

                default:
                    if (tag.Contains("Metric."))
                    {
                        Metadata.TagCounters.Push(new MetricTagCounter()
                        {
                            Tag = tag
                        });
                    }
                    break;
                }
            }

            //safeguard sample output now that we've matched properties for blocking browser xss
            matchRecord.Sample = System.Net.WebUtility.HtmlEncode(matchRecord.Sample);

            //Special handling; attempt to detect app types...review for multiple pattern rule limitation
            string solutionType = DetectSolutionType(matchRecord);

            if (!string.IsNullOrEmpty(solutionType))
            {
                _ = Metadata.AppTypes.TryAdd(solutionType, 0);
            }

            bool CounterOnlyTagSet = false;
            var  selected          = Metadata.TagCounters.Where(x => matchRecord.Tags.Any(y => y.Contains(x.Tag)));

            foreach (var select in selected)
            {
                CounterOnlyTagSet = true;
                select.IncrementCount();
            }

            //omit adding if ther a counter metric tag
            if (!CounterOnlyTagSet)
            {
                //update list of unique tags as we go
                foreach (string tag in matchRecord.Tags)
                {
                    _ = Metadata.UniqueTags.TryAdd(tag, 0);
                }

                Metadata.Matches.Add(matchRecord);
            }
            else
            {
                Metadata.IncrementTotalMatchesCount(-1);//reduce e.g. tag counters not included as detailed match
            }
        }
示例#10
0
        /// <summary>
        /// Assist in aggregating reporting properties of matches as they are added
        /// Keeps helpers isolated from MetaData class which is used as a result object to keep pure
        /// </summary>
        /// <param name="matchRecord"></param>
        public void AddMatchRecord(MatchRecord matchRecord)
        {
            bool allowAdd = true;

            //special handling for standard characteristics in report
            foreach (var tag in matchRecord.Tags ?? new string[] { })
            {
                switch (tag)
                {
                case "Metadata.Application.Author":
                case "Metadata.Application.Publisher":
                    Metadata.Authors = ExtractValue(matchRecord.Sample);
                    break;

                case "Metadata.Application.Description":
                    Metadata.Description = ExtractValue(matchRecord.Sample);
                    break;

                case "Metadata.Application.Name":
                    Metadata.ApplicationName = ExtractValue(matchRecord.Sample);
                    break;

                case "Metadata.Application.Version":
                    Metadata.SourceVersion = ExtractValue(matchRecord.Sample);
                    break;

                case "Metadata.Application.Target.Processor":
                    _ = CPUTargets.TryAdd(ExtractValue(matchRecord.Sample).ToLower(), 0);
                    break;

                case "Metadata.Application.Output.Type":
                    _ = Outputs.TryAdd(ExtractValue(matchRecord.Sample).ToLower(), 0);
                    break;

                case "Dependency.SourceInclude":
                    allowAdd = false;     //design to keep noise out of detailed match list
                    break;

                default:
                    if (tag.Contains("Metric."))
                    {
                        _ = TagCounters.TryAdd(tag, new MetricTagCounter()
                        {
                            Tag = tag
                        });
                    }
                    else if (tag.Contains(".Platform.OS"))
                    {
                        _ = OSTargets.TryAdd(tag.Substring(tag.LastIndexOf('.', tag.Length - 1) + 1), 0);
                    }
                    else if (tag.Contains("CloudServices.Hosting"))
                    {
                        _ = CloudTargets.TryAdd(tag.Substring(tag.LastIndexOf('.', tag.Length - 1) + 1), 0);
                    }
                    break;
                }
            }

            //Special handling; attempt to detect app types...review for multiple pattern rule limitation
            string solutionType = DetectSolutionType(matchRecord);

            if (!string.IsNullOrEmpty(solutionType))
            {
                _ = AppTypes.TryAdd(solutionType, 0);
            }

            bool CounterOnlyTagSet = false;
            var  selected          = TagCounters.Where(x => matchRecord.Tags.Any(y => y.Contains(x.Value.Tag ?? "")));

            foreach (var select in selected)
            {
                CounterOnlyTagSet = true;
                select.Value.IncrementCount();
            }

            //omit adding if ther a counter metric tag
            if (!CounterOnlyTagSet)
            {
                //update list of unique tags as we go
                foreach (string tag in matchRecord.Tags ?? new string[] { })
                {
                    _ = UniqueTags.TryAdd(tag, 0);
                }

                if (allowAdd)
                {
                    Metadata?.Matches?.Add(matchRecord);
                }
            }
            else
            {
                Metadata.IncrementTotalMatchesCount(-1);//reduce e.g. tag counters not included as detailed match
            }
        }