Exemplo n.º 1
0
        /// <summary>
        /// Identify if new scan result is a better match than previous i.e. has a higher confidence.  Assumes unique list of MatchRecords.
        /// </summary>
        /// <param name="MatchRecords"></param>
        /// <param name="compareResult"></param>
        /// <param name="removeOld"></param>
        /// <returns></returns>
        private bool BetterMatch(List <MatchRecord> MatchRecords, MatchRecord newMatchRecord, bool removeOld = true)
        {
            bool betterMatch = false;
            bool noMatch     = true;

            //if list is empty the new match is the best match
            if (!MatchRecords.Any())
            {
                return(true);
            }

            MatchRecord?matchRecordToRemove = null;

            foreach (MatchRecord MatchRecord in MatchRecords)
            {
                foreach (string matchRecordTag in MatchRecord.Rule.Tags ?? new string[] { "" })
                {
                    foreach (string newMatchRecordTag in newMatchRecord.Rule.Tags ?? new string[] { "" })
                    {
                        if (matchRecordTag == newMatchRecordTag)
                        {
                            if (newMatchRecord.Tags.Any(x => x.Contains("AzureKeyVault")))
                            {
                            }

                            noMatch = false;
                            if (newMatchRecord.Confidence > MatchRecord.Confidence)
                            {
                                if (removeOld)
                                {
                                    matchRecordToRemove = MatchRecord;
                                }

                                betterMatch = true;
                                break;//as this method is used with uniquematche=true only one to worry about
                            }
                        }
                    }

                    if (betterMatch)
                    {
                        break;
                    }
                }
            }

            if (removeOld && matchRecordToRemove != null)
            {
                MatchRecords.Remove(matchRecordToRemove);//safer to remove outside for enumeration
            }

            return(betterMatch || noMatch);
        }
Exemplo n.º 2
0
        /// <summary>
        /// Analyzes given line of code returning matching scan results for the
        /// file passed in only; Use AllResults to get results across the entire set
        /// </summary>
        /// <param name="text">Source code</param>
        /// <param name="languages">List of languages</param>
        /// <returns>Array of matches</returns>
        public MatchRecord[] AnalyzeFile(string filePath, string text, LanguageInfo languageInfo)
        {
            // Get rules for the given content type
            IEnumerable <ConvertedOatRule> rules       = GetRulesForSingleLanguage(languageInfo.Name).Where(x => !x.AppInspectorRule.Disabled && SeverityLevel.HasFlag(x.AppInspectorRule.Severity));
            List <MatchRecord>             resultsList = new List <MatchRecord>();//matches for this file only
            TextContainer textContainer = new TextContainer(text, languageInfo.Name);

            foreach (var ruleCapture in analyzer.GetCaptures(rules, textContainer))
            {
                // If we have within captures it means we had conditions, and we only want the conditioned captures
                var withinCaptures = ruleCapture.Captures.Where(x => x.Clause is WithinClause);
                if (withinCaptures.Any())
                {
                    foreach (var cap in withinCaptures)
                    {
                        ProcessBoundary(cap);
                    }
                }
                // Otherwise we can use all the captures
                else
                {
                    foreach (var cap in ruleCapture.Captures)
                    {
                        ProcessBoundary(cap);
                    }
                }

                void ProcessBoundary(ClauseCapture cap)
                {
                    List <MatchRecord> newMatches = new List <MatchRecord>();//matches for this rule clause only

                    if (cap is TypedClauseCapture <List <(int, Boundary)> > tcc)
                    {
                        if (ruleCapture.Rule is ConvertedOatRule oatRule)
                        {
                            if (tcc?.Result is List <(int, Boundary)> captureResults)
                            {
                                foreach (var match in captureResults)
                                {
                                    var patternIndex = match.Item1;
                                    var boundary     = match.Item2;

                                    //restrict adds from build files to tags with "metadata" only to avoid false feature positives that are not part of executable code
                                    if (languageInfo.Type == LanguageInfo.LangFileType.Build && oatRule.AppInspectorRule.Tags.Any(v => !v.Contains("Metadata")))
                                    {
                                        continue;
                                    }

                                    if (patternIndex < 0 || patternIndex > oatRule.AppInspectorRule.Patterns.Length)
                                    {
                                        _logger?.Error("Index out of range for patterns for rule: " + oatRule.AppInspectorRule.Name);
                                        continue;
                                    }

                                    if (!ConfidenceLevelFilter.HasFlag(oatRule.AppInspectorRule.Patterns[patternIndex].Confidence))
                                    {
                                        continue;
                                    }

                                    Location    StartLocation = textContainer.GetLocation(boundary.Index);
                                    Location    EndLocation   = textContainer.GetLocation(boundary.Index + boundary.Length);
                                    MatchRecord newMatch      = new MatchRecord(oatRule.AppInspectorRule)
                                    {
                                        FileName          = filePath,
                                        FullText          = textContainer.FullContent,
                                        LanguageInfo      = languageInfo,
                                        Boundary          = boundary,
                                        StartLocationLine = StartLocation.Line,
                                        EndLocationLine   = EndLocation.Line != 0 ? EndLocation.Line : StartLocation.Line + 1, //match is on last line
                                        MatchingPattern   = oatRule.AppInspectorRule.Patterns[patternIndex],
                                        Excerpt           = ExtractExcerpt(textContainer.FullContent, StartLocation.Line),
                                        Sample            = ExtractTextSample(textContainer.FullContent, boundary.Index, boundary.Length)
                                    };

                                    bool addNewRecord = true;
                                    if (_uniqueTagMatchesOnly)
                                    {
                                        if (!RuleTagsAreUniqueOrAllowed(newMatch.Rule.Tags))
                                        {
                                            if (_stopAfterFirstMatch)
                                            {
                                                addNewRecord = false;                      //we've seen already; don't improve the match
                                            }
                                            else if (newMatch.Confidence > Confidence.Low) //user prefers highest confidence match over first match
                                            {
                                                addNewRecord = BetterMatch(newMatches, newMatch) && BetterMatch(resultsList, newMatch);
                                                if (addNewRecord)
                                                {
                                                    lock (_controllRunningListAdd)
                                                    {
                                                        addNewRecord = BetterMatch(_runningResultsList, newMatch);//check current rule matches and previous processed files
                                                    }
                                                }
                                            }
                                        }
                                    }

                                    if (addNewRecord)
                                    {
                                        newMatches.Add(newMatch);
                                        lock (_controllRunningListAdd)
                                        {
                                            _runningResultsList.Add(newMatch);
                                        }
                                    }

                                    AddRuleTagHashes(oatRule.AppInspectorRule.Tags ?? new string[] { "" });
                                }
                            }
                        }
                    }

                    resultsList.AddRange(newMatches);
                }
            }

            if (resultsList.Any(x => x.Rule.Overrides != null && x.Rule.Overrides.Length > 0))
            {
                // Deal with overrides
                List <MatchRecord> removes = new List <MatchRecord>();
                foreach (MatchRecord m in resultsList)
                {
                    if (m.Rule.Overrides != null && m.Rule.Overrides.Length > 0)
                    {
                        foreach (string ovrd in m.Rule.Overrides)
                        {
                            // Find all overriden rules and mark them for removal from issues list
                            foreach (MatchRecord om in resultsList.FindAll(x => x.Rule.Id == ovrd))
                            {
                                if (om.Boundary?.Index >= m.Boundary?.Index &&
                                    om.Boundary?.Index <= m.Boundary?.Index + m.Boundary?.Length)
                                {
                                    removes.Add(om);
                                }
                            }
                        }
                    }
                }

                // Remove overriden rules
                resultsList.RemoveAll(x => removes.Contains(x));
            }

            return(resultsList.ToArray());
        }
Exemplo n.º 3
0
        /// <summary>
        /// Analyzes given line of code returning matching scan results for the
        /// file passed in only; Use AllResults to get results across the entire set
        /// </summary>
        /// <param name="text">Source code</param>
        /// <param name="languages">List of languages</param>
        /// <returns>Array of matches</returns>
        public MatchRecord[] AnalyzeFile(string filePath, string text, LanguageInfo languageInfo)
        {
            // Get rules for the given content type
            var rulesByLanguage              = GetRulesByLanguage(languageInfo.Name).Where(x => !x.AppInspectorRule.Disabled && SeverityLevel.HasFlag(x.AppInspectorRule.Severity));
            var rules                        = rulesByLanguage.Union(GetRulesByFileName(filePath).Where(x => !x.AppInspectorRule.Disabled && SeverityLevel.HasFlag(x.AppInspectorRule.Severity)));
            List <MatchRecord> resultsList   = new List <MatchRecord>();//matches for this file only
            TextContainer      textContainer = new TextContainer(text, languageInfo.Name);

            foreach (var ruleCapture in analyzer.GetCaptures(rules, textContainer))
            {
                foreach (var cap in ruleCapture.Captures)
                {
                    ProcessBoundary(cap);
                }

                void ProcessBoundary(ClauseCapture cap)
                {
                    List <MatchRecord> newMatches = new List <MatchRecord>();//matches for this rule clause only

                    if (cap is TypedClauseCapture <List <(int, Boundary)> > tcc)
                    {
                        if (ruleCapture.Rule is ConvertedOatRule oatRule)
                        {
                            if (tcc?.Result is List <(int, Boundary)> captureResults)
                            {
                                foreach (var match in captureResults)
                                {
                                    var patternIndex = match.Item1;
                                    var boundary     = match.Item2;

                                    //restrict adds from build files to tags with "metadata" only to avoid false feature positives that are not part of executable code
                                    if (!_treatEverythingAsCode && languageInfo.Type == LanguageInfo.LangFileType.Build && oatRule.AppInspectorRule.Tags.Any(v => !v.Contains("Metadata")))
                                    {
                                        continue;
                                    }

                                    if (patternIndex < 0 || patternIndex > oatRule.AppInspectorRule.Patterns.Length)
                                    {
                                        _logger?.Error("Index out of range for patterns for rule: " + oatRule.AppInspectorRule.Name);
                                        continue;
                                    }

                                    if (!ConfidenceLevelFilter.HasFlag(oatRule.AppInspectorRule.Patterns[patternIndex].Confidence))
                                    {
                                        continue;
                                    }

                                    Location    StartLocation = textContainer.GetLocation(boundary.Index);
                                    Location    EndLocation   = textContainer.GetLocation(boundary.Index + boundary.Length);
                                    MatchRecord newMatch      = new MatchRecord(oatRule.AppInspectorRule)
                                    {
                                        FileName          = filePath,
                                        FullTextContainer = textContainer,
                                        LanguageInfo      = languageInfo,
                                        Boundary          = boundary,
                                        StartLocationLine = StartLocation.Line,
                                        EndLocationLine   = EndLocation.Line != 0 ? EndLocation.Line : StartLocation.Line + 1, //match is on last line
                                        MatchingPattern   = oatRule.AppInspectorRule.Patterns[patternIndex],
                                        Excerpt           = ExtractExcerpt(textContainer.FullContent, StartLocation.Line),
                                        Sample            = ExtractTextSample(textContainer.FullContent, boundary.Index, boundary.Length)
                                    };

                                    newMatches.Add(newMatch);

                                    if (oatRule.AppInspectorRule.Tags != null && oatRule.AppInspectorRule.Tags.Any())
                                    {
                                        AddRuleTagHashes(oatRule.AppInspectorRule.Tags);
                                    }
                                }
                            }
                        }
                    }

                    resultsList.AddRange(newMatches);
                }
            }

            if (_uniqueTagMatchesOnly)
            {
                var replacementList = new List <MatchRecord>();
                foreach (var entry in resultsList)
                {
                    if (!RuleTagsAreUniqueOrAllowed(entry.Rule.Tags))
                    {
                        if (!_stopAfterFirstMatch)
                        {
                            var replaceable = replacementList.Where(x => x.Tags.All(y => entry.Tags.Contains(y)));
                            if (replaceable.Any())
                            {
                                replaceable = replaceable.Where(x => x.Confidence < entry.Confidence).ToList();
                                if (replaceable.Any())
                                {
                                    replacementList.RemoveAll(x => replaceable.Contains(x));
                                    replacementList.Add(entry);
                                }
                            }
                            else
                            {
                                replacementList.Add(entry);
                            }
                        }
                    }
                    else
                    {
                        replacementList.Add(entry);
                    }
                }
                resultsList = replacementList;
            }
            List <MatchRecord> removes = new List <MatchRecord>();

            foreach (MatchRecord m in resultsList.Where(x => x.Rule.Overrides != null && x.Rule.Overrides.Length > 0))
            {
                if (m.Rule.Overrides != null && m.Rule.Overrides.Length > 0)
                {
                    foreach (string ovrd in m.Rule.Overrides)
                    {
                        // Find all overriden rules and mark them for removal from issues list
                        foreach (MatchRecord om in resultsList.FindAll(x => x.Rule.Id == ovrd))
                        {
                            if (om.Boundary?.Index >= m.Boundary?.Index &&
                                om.Boundary?.Index <= m.Boundary?.Index + m.Boundary?.Length)
                            {
                                removes.Add(om);
                            }
                        }
                    }
                }
                // Remove overriden rules
            }
            resultsList.RemoveAll(x => removes.Contains(x));

            foreach (var entry in resultsList)
            {
                _runningResultsList.Enqueue(entry);
            }

            return(resultsList.ToArray());
        }