Пример #1
0
        /// <summary>
        /// Returns language for given file name
        /// </summary>
        /// <param name="fileName">File name</param>
        /// <returns>Language</returns>
        public static bool FromFileName(string fileName, ref LanguageInfo info)
        {
            if (fileName == null)
            {
                return(false);
            }

            string file = Path.GetFileName(fileName).ToLower(CultureInfo.CurrentCulture);
            string ext  = Path.GetExtension(file);

            // Look for whole filename first
            foreach (LanguageInfo item in Instance.Languages)
            {
                if (item.Name == file)
                {
                    info = item;
                    return(true);
                }
            }

            // Look for extension only ext is defined
            if (!string.IsNullOrEmpty(ext))
            {
                foreach (LanguageInfo item in Instance.Languages)
                {
                    if (item.Name == "typescript-config")//special case where extension used for exact match to a single type
                    {
                        if (item.Extensions?.Any(x => x.ToLower().Equals(file)) ?? false)
                        {
                            info = item;
                            return(true);
                        }
                    }
                    else if (Array.Exists(item.Extensions ?? Array.Empty <string>(), x => x.EndsWith(ext, StringComparison.InvariantCultureIgnoreCase)))
                    {
                        info = item;
                        return(true);
                    }
                }
            }

            return(false);
        }
Пример #2
0
        /// <summary>
        /// Returns language for given file name
        /// </summary>
        /// <param name="fileName">File name</param>
        /// <returns>Language</returns>
        public static bool FromFileName(string fileName, ref LanguageInfo info)
        {
            if (fileName == null)
            {
                return(false);
            }

            bool result = false;

            string file = Path.GetFileName(fileName).ToLower(CultureInfo.CurrentCulture);
            string ext  = Path.GetExtension(file);

            // Look for whole filename first
            foreach (LanguageInfo item in Instance.Languages)
            {
                if (item.Name == file)
                {
                    info = item;
                    return(true);
                }
            }

            // Look for extension only ext is defined
            if (!string.IsNullOrEmpty(ext))
            {
                foreach (LanguageInfo item in Instance.Languages)
                {
                    if (Array.Exists(item.Extensions, x => x.EndsWith(ext)))
                    {
                        info = item;
                        return(true);
                    }
                }
            }

            return(result);
        }
        /// <summary>
        /// Analyzes given line of code
        /// </summary>
        /// <param name="text">Source code</param>
        /// <param name="languages">List of languages</param>
        /// <returns>Array of matches</returns>
        public ScanResult[] Analyze(string text, LanguageInfo languageInfo)
        {
            string[] languages = new string[] { languageInfo.Name };
            // Get rules for the given content type
            IEnumerable <Rule> rules         = GetRulesForLanguages(languages);
            List <ScanResult>  resultsList   = new List <ScanResult>();
            TextContainer      textContainer = new TextContainer(text, (languages.Length > 0) ? languages[0] : string.Empty, _stopAfterFirstPatternMatch);

            // Go through each rule
            foreach (Rule rule in rules)
            {
                if (_logger != null)
                {
                    _logger.Debug("Processing for rule: " + rule.Id);
                }

                // Skip rules that don't apply based on settings
                if (rule.Disabled || !SeverityLevel.HasFlag(rule.Severity))
                {
                    continue;
                }

                // Skip further processing of rule for efficiency if user requested first match only (speed/quality)
                if (_stopAfterFirstPatternMatch && _uniqueTagMatchesOnly && !UniqueTagsCheck(rule.Tags))
                {
                    continue;
                }

                List <ScanResult> matchList = new List <ScanResult>();

                // Go through each matching pattern of the rule
                foreach (SearchPattern pattern in rule.Patterns)
                {
                    //Skill patterns that don't apply based on settings
                    if (!ConfidenceLevelFilter.HasFlag(pattern.Confidence))
                    {
                        continue;
                    }

                    // Process all matches for the patttern (this may be only 1 is _stopAfterFirstMatch is set
                    foreach (Boundary match in textContainer.EnumerateMatchingBoundaries(pattern))
                    {
                        bool passedConditions = true;
                        foreach (SearchCondition condition in rule.Conditions)
                        {
                            bool res = textContainer.IsPatternMatch(condition.Pattern, match, condition);
                            if (res && condition.NegateFinding)
                            {
                                passedConditions = false;
                                break;
                            }
                            if (!res && condition.NegateFinding)
                            {
                                passedConditions = true;
                                break;
                            }
                            if (!res)
                            {
                                passedConditions = false;
                                break;
                            }
                        }

                        //restrict tags from build files to tags with "metadata" to avoid false feature positives that are not part of executable code
                        if (languageInfo.Type == LanguageInfo.LangFileType.Build && rule.Tags.Any(v => !v.Contains("Metadata")))
                        {
                            passedConditions = false;
                        }

                        if (passedConditions)
                        {
                            ScanResult newMatch = new ScanResult()
                            {
                                Boundary      = match,
                                StartLocation = textContainer.GetLocation(match.Index),
                                EndLocation   = textContainer.GetLocation(match.Index + match.Length),
                                PatternMatch  = pattern,
                                Confidence    = pattern.Confidence,
                                Rule          = rule
                            };

                            if (_uniqueTagMatchesOnly)
                            {
                                if (!UniqueTagsCheck(newMatch.Rule.Tags))                  //tag(s) previously seen
                                {
                                    if (_stopAfterFirstPatternMatch)                       //recheck stop at pattern level also within same rule
                                    {
                                        passedConditions = false;                          //user performance option i.e. only wants to identify if tag is detected nothing more
                                    }
                                    else if (newMatch.Confidence > Confidence.Low)         //user prefers highest confidence match over first match
                                    {
                                        passedConditions = BestMatch(matchList, newMatch); //; check all patterns in current rule

                                        if (passedConditions)
                                        {
                                            passedConditions = BestMatch(resultsList, newMatch);//check all rules in permanent list
                                        }
                                    }
                                }
                            }

                            if (passedConditions)
                            {
                                matchList.Add(newMatch);
                            }

                            AddRuleTagHashes(rule.Tags);
                        }
                    }
                }

                resultsList.AddRange(matchList);
            }

            // Deal with overrides
            List <ScanResult> removes = new List <ScanResult>();

            foreach (ScanResult scanResult in resultsList)
            {
                if (scanResult.Rule.Overrides is string[] overrides)
                {
                    foreach (string @override in overrides)
                    {
                        // Find all overriden rules and mark them for removal from issues list
                        foreach (ScanResult overRideMatch in resultsList.FindAll(x => x.Rule.Id == @override))
                        {
                            if (overRideMatch.Boundary.Index >= scanResult.Boundary.Index &&
                                overRideMatch.Boundary.Index <= scanResult.Boundary.Index + scanResult.Boundary.Length)
                            {
                                removes.Add(overRideMatch);
                            }
                        }
                    }
                }
            }

            if (removes.Count > 0)
            {
                // Remove overriden rules
                resultsList.RemoveAll(x => removes.Contains(x));
            }

            return(resultsList.ToArray());
        }
Пример #4
0
        /// <summary>
        /// Analyzes given line of code returning matching scan results for the
        /// file passed in only; Use AllResults to get results across the entire set
        /// </summary>
        /// <param name="text">Source code</param>
        /// <param name="languages">List of languages</param>
        /// <returns>Array of matches</returns>
        public MatchRecord[] AnalyzeFile(string filePath, string text, LanguageInfo languageInfo)
        {
            // Get rules for the given content type
            IEnumerable <ConvertedOatRule> rules       = GetRulesForSingleLanguage(languageInfo.Name).Where(x => !x.AppInspectorRule.Disabled && SeverityLevel.HasFlag(x.AppInspectorRule.Severity));
            List <MatchRecord>             resultsList = new List <MatchRecord>();//matches for this file only
            TextContainer textContainer = new TextContainer(text, languageInfo.Name);

            foreach (var ruleCapture in analyzer.GetCaptures(rules, textContainer))
            {
                // If we have within captures it means we had conditions, and we only want the conditioned captures
                var withinCaptures = ruleCapture.Captures.Where(x => x.Clause is WithinClause);
                if (withinCaptures.Any())
                {
                    foreach (var cap in withinCaptures)
                    {
                        ProcessBoundary(cap);
                    }
                }
                // Otherwise we can use all the captures
                else
                {
                    foreach (var cap in ruleCapture.Captures)
                    {
                        ProcessBoundary(cap);
                    }
                }

                void ProcessBoundary(ClauseCapture cap)
                {
                    List <MatchRecord> newMatches = new List <MatchRecord>();//matches for this rule clause only

                    if (cap is TypedClauseCapture <List <(int, Boundary)> > tcc)
                    {
                        if (ruleCapture.Rule is ConvertedOatRule oatRule)
                        {
                            if (tcc?.Result is List <(int, Boundary)> captureResults)
                            {
                                foreach (var match in captureResults)
                                {
                                    var patternIndex = match.Item1;
                                    var boundary     = match.Item2;

                                    //restrict adds from build files to tags with "metadata" only to avoid false feature positives that are not part of executable code
                                    if (languageInfo.Type == LanguageInfo.LangFileType.Build && oatRule.AppInspectorRule.Tags.Any(v => !v.Contains("Metadata")))
                                    {
                                        continue;
                                    }

                                    if (patternIndex < 0 || patternIndex > oatRule.AppInspectorRule.Patterns.Length)
                                    {
                                        _logger?.Error("Index out of range for patterns for rule: " + oatRule.AppInspectorRule.Name);
                                        continue;
                                    }

                                    if (!ConfidenceLevelFilter.HasFlag(oatRule.AppInspectorRule.Patterns[patternIndex].Confidence))
                                    {
                                        continue;
                                    }

                                    Location    StartLocation = textContainer.GetLocation(boundary.Index);
                                    Location    EndLocation   = textContainer.GetLocation(boundary.Index + boundary.Length);
                                    MatchRecord newMatch      = new MatchRecord(oatRule.AppInspectorRule)
                                    {
                                        FileName          = filePath,
                                        FullText          = textContainer.FullContent,
                                        LanguageInfo      = languageInfo,
                                        Boundary          = boundary,
                                        StartLocationLine = StartLocation.Line,
                                        EndLocationLine   = EndLocation.Line != 0 ? EndLocation.Line : StartLocation.Line + 1, //match is on last line
                                        MatchingPattern   = oatRule.AppInspectorRule.Patterns[patternIndex],
                                        Excerpt           = ExtractExcerpt(textContainer.FullContent, StartLocation.Line),
                                        Sample            = ExtractTextSample(textContainer.FullContent, boundary.Index, boundary.Length)
                                    };

                                    bool addNewRecord = true;
                                    if (_uniqueTagMatchesOnly)
                                    {
                                        if (!RuleTagsAreUniqueOrAllowed(newMatch.Rule.Tags))
                                        {
                                            if (_stopAfterFirstMatch)
                                            {
                                                addNewRecord = false;                      //we've seen already; don't improve the match
                                            }
                                            else if (newMatch.Confidence > Confidence.Low) //user prefers highest confidence match over first match
                                            {
                                                addNewRecord = BetterMatch(newMatches, newMatch) && BetterMatch(resultsList, newMatch);
                                                if (addNewRecord)
                                                {
                                                    lock (_controllRunningListAdd)
                                                    {
                                                        addNewRecord = BetterMatch(_runningResultsList, newMatch);//check current rule matches and previous processed files
                                                    }
                                                }
                                            }
                                        }
                                    }

                                    if (addNewRecord)
                                    {
                                        newMatches.Add(newMatch);
                                        lock (_controllRunningListAdd)
                                        {
                                            _runningResultsList.Add(newMatch);
                                        }
                                    }

                                    AddRuleTagHashes(oatRule.AppInspectorRule.Tags ?? new string[] { "" });
                                }
                            }
                        }
                    }

                    resultsList.AddRange(newMatches);
                }
            }

            if (resultsList.Any(x => x.Rule.Overrides != null && x.Rule.Overrides.Length > 0))
            {
                // Deal with overrides
                List <MatchRecord> removes = new List <MatchRecord>();
                foreach (MatchRecord m in resultsList)
                {
                    if (m.Rule.Overrides != null && m.Rule.Overrides.Length > 0)
                    {
                        foreach (string ovrd in m.Rule.Overrides)
                        {
                            // Find all overriden rules and mark them for removal from issues list
                            foreach (MatchRecord om in resultsList.FindAll(x => x.Rule.Id == ovrd))
                            {
                                if (om.Boundary?.Index >= m.Boundary?.Index &&
                                    om.Boundary?.Index <= m.Boundary?.Index + m.Boundary?.Length)
                                {
                                    removes.Add(om);
                                }
                            }
                        }
                    }
                }

                // Remove overriden rules
                resultsList.RemoveAll(x => removes.Contains(x));
            }

            return(resultsList.ToArray());
        }
Пример #5
0
        /// <summary>
        /// Returns the language for a given file name if detected.
        /// </summary>
        /// <param name="fileName">The FileName to check.</param>
        /// <param name="info">If this returns true, a valid LanguageInfo object. If false, undefined.</param>
        /// <returns>True if the language could be detected based on the filename.</returns>
        public static bool FromFileNameOut(string fileName, out LanguageInfo info)
        {
            info = new LanguageInfo();

            return(FromFileName(fileName, ref info));
        }
Пример #6
0
        /// <summary>
        /// Analyzes given line of code returning matching scan results for the
        /// file passed in only; Use AllResults to get results across the entire set
        /// </summary>
        /// <param name="text">Source code</param>
        /// <param name="languages">List of languages</param>
        /// <returns>Array of matches</returns>
        public MatchRecord[] AnalyzeFile(string filePath, string text, LanguageInfo languageInfo)
        {
            // Get rules for the given content type
            var rulesByLanguage              = GetRulesByLanguage(languageInfo.Name).Where(x => !x.AppInspectorRule.Disabled && SeverityLevel.HasFlag(x.AppInspectorRule.Severity));
            var rules                        = rulesByLanguage.Union(GetRulesByFileName(filePath).Where(x => !x.AppInspectorRule.Disabled && SeverityLevel.HasFlag(x.AppInspectorRule.Severity)));
            List <MatchRecord> resultsList   = new List <MatchRecord>();//matches for this file only
            TextContainer      textContainer = new TextContainer(text, languageInfo.Name);

            foreach (var ruleCapture in analyzer.GetCaptures(rules, textContainer))
            {
                foreach (var cap in ruleCapture.Captures)
                {
                    ProcessBoundary(cap);
                }

                void ProcessBoundary(ClauseCapture cap)
                {
                    List <MatchRecord> newMatches = new List <MatchRecord>();//matches for this rule clause only

                    if (cap is TypedClauseCapture <List <(int, Boundary)> > tcc)
                    {
                        if (ruleCapture.Rule is ConvertedOatRule oatRule)
                        {
                            if (tcc?.Result is List <(int, Boundary)> captureResults)
                            {
                                foreach (var match in captureResults)
                                {
                                    var patternIndex = match.Item1;
                                    var boundary     = match.Item2;

                                    //restrict adds from build files to tags with "metadata" only to avoid false feature positives that are not part of executable code
                                    if (!_treatEverythingAsCode && languageInfo.Type == LanguageInfo.LangFileType.Build && oatRule.AppInspectorRule.Tags.Any(v => !v.Contains("Metadata")))
                                    {
                                        continue;
                                    }

                                    if (patternIndex < 0 || patternIndex > oatRule.AppInspectorRule.Patterns.Length)
                                    {
                                        _logger?.Error("Index out of range for patterns for rule: " + oatRule.AppInspectorRule.Name);
                                        continue;
                                    }

                                    if (!ConfidenceLevelFilter.HasFlag(oatRule.AppInspectorRule.Patterns[patternIndex].Confidence))
                                    {
                                        continue;
                                    }

                                    Location    StartLocation = textContainer.GetLocation(boundary.Index);
                                    Location    EndLocation   = textContainer.GetLocation(boundary.Index + boundary.Length);
                                    MatchRecord newMatch      = new MatchRecord(oatRule.AppInspectorRule)
                                    {
                                        FileName          = filePath,
                                        FullTextContainer = textContainer,
                                        LanguageInfo      = languageInfo,
                                        Boundary          = boundary,
                                        StartLocationLine = StartLocation.Line,
                                        EndLocationLine   = EndLocation.Line != 0 ? EndLocation.Line : StartLocation.Line + 1, //match is on last line
                                        MatchingPattern   = oatRule.AppInspectorRule.Patterns[patternIndex],
                                        Excerpt           = ExtractExcerpt(textContainer.FullContent, StartLocation.Line),
                                        Sample            = ExtractTextSample(textContainer.FullContent, boundary.Index, boundary.Length)
                                    };

                                    newMatches.Add(newMatch);

                                    if (oatRule.AppInspectorRule.Tags != null && oatRule.AppInspectorRule.Tags.Any())
                                    {
                                        AddRuleTagHashes(oatRule.AppInspectorRule.Tags);
                                    }
                                }
                            }
                        }
                    }

                    resultsList.AddRange(newMatches);
                }
            }

            if (_uniqueTagMatchesOnly)
            {
                var replacementList = new List <MatchRecord>();
                foreach (var entry in resultsList)
                {
                    if (!RuleTagsAreUniqueOrAllowed(entry.Rule.Tags))
                    {
                        if (!_stopAfterFirstMatch)
                        {
                            var replaceable = replacementList.Where(x => x.Tags.All(y => entry.Tags.Contains(y)));
                            if (replaceable.Any())
                            {
                                replaceable = replaceable.Where(x => x.Confidence < entry.Confidence).ToList();
                                if (replaceable.Any())
                                {
                                    replacementList.RemoveAll(x => replaceable.Contains(x));
                                    replacementList.Add(entry);
                                }
                            }
                            else
                            {
                                replacementList.Add(entry);
                            }
                        }
                    }
                    else
                    {
                        replacementList.Add(entry);
                    }
                }
                resultsList = replacementList;
            }
            List <MatchRecord> removes = new List <MatchRecord>();

            foreach (MatchRecord m in resultsList.Where(x => x.Rule.Overrides != null && x.Rule.Overrides.Length > 0))
            {
                if (m.Rule.Overrides != null && m.Rule.Overrides.Length > 0)
                {
                    foreach (string ovrd in m.Rule.Overrides)
                    {
                        // Find all overriden rules and mark them for removal from issues list
                        foreach (MatchRecord om in resultsList.FindAll(x => x.Rule.Id == ovrd))
                        {
                            if (om.Boundary?.Index >= m.Boundary?.Index &&
                                om.Boundary?.Index <= m.Boundary?.Index + m.Boundary?.Length)
                            {
                                removes.Add(om);
                            }
                        }
                    }
                }
                // Remove overriden rules
            }
            resultsList.RemoveAll(x => removes.Contains(x));

            foreach (var entry in resultsList)
            {
                _runningResultsList.Enqueue(entry);
            }

            return(resultsList.ToArray());
        }
        /// <summary>
        /// Analyzes given line of code
        /// </summary>
        /// <param name="text">Source code</param>
        /// <param name="languages">List of languages</param>
        /// <returns>Array of matches</returns>
        public Issue[] Analyze(string text, LanguageInfo languageInfo)
        {
            string[] languages = new string[] { languageInfo.Name };
            // Get rules for the given content type
            IEnumerable <Rule> rules         = GetRulesForLanguages(languages);
            List <Issue>       resultsList   = new List <Issue>();
            TextContainer      textContainer = new TextContainer(text, (languages.Length > 0) ? languages[0] : string.Empty, _stopAfterFirstPatternMatch);

            // Go through each rule
            foreach (Rule rule in rules)
            {
                if (_logger != null)
                {
                    _logger.Debug("Processing for rule: " + rule.Id);
                }

                // Skip pattern matching this rule if uniquetag option and not in exceptions list
                bool multipleMatchesOk = !_uniqueTagMatchesOnly || UniqueTagsCheck(rule.Tags);
                if (!multipleMatchesOk)
                {
                    continue;
                }

                List <Issue> matchList = new List <Issue>();

                // Skip rules that don't apply based on settings
                if (rule.Disabled || !SeverityLevel.HasFlag(rule.Severity))
                {
                    continue;
                }

                // Go through each matching pattern of the rule
                foreach (SearchPattern pattern in rule.Patterns)
                {
                    //Skill patterns that don't apply based on settings
                    if (!ConfidenceLevelFilter.HasFlag(pattern.Confidence))
                    {
                        continue;
                    }

                    // Get all matches for the patttern
                    List <Boundary> matches = textContainer.MatchPattern(pattern);

                    if (matches.Count > 0)
                    {
                        foreach (Boundary match in matches)
                        {
                            bool passedConditions = true;
                            foreach (SearchCondition condition in rule.Conditions)
                            {
                                bool res = textContainer.MatchPattern(condition.Pattern, match, condition);
                                if (res && condition.NegateFinding)
                                {
                                    passedConditions = false;
                                    break;
                                }
                                if (!res && condition.NegateFinding)
                                {
                                    passedConditions = true;
                                    break;
                                }
                                if (!res)
                                {
                                    passedConditions = false;
                                    break;
                                }
                            }

                            //do not accept features from build type files (only metadata) to avoid false positives that are not part of the executable program
                            if (languageInfo.Type == LanguageInfo.LangFileType.Build && rule.Tags.Any(v => !v.Contains("Metadata")))
                            {
                                passedConditions = false;
                            }

                            if (passedConditions)
                            {
                                Issue issue = new Issue()
                                {
                                    Boundary      = match,
                                    StartLocation = textContainer.GetLocation(match.Index),
                                    EndLocation   = textContainer.GetLocation(match.Index + match.Length),
                                    PatternMatch  = pattern,
                                    Confidence    = pattern.Confidence,
                                    Rule          = rule
                                };

                                //check at pattern level to avoid adding duplicates
                                if (_uniqueTagMatchesOnly && !UniqueTagsCheck(rule.Tags))
                                {
                                    break;
                                }

                                AddRuleTagHashes(rule.Tags);
                                matchList.Add(issue);
                            }
                        }
                    }
                }

                // We got matching rule and suppression are enabled,
                // let's see if we have a supression on the line
                if (EnableSuppressions && matchList.Count > 0)
                {
                    Suppression supp;
                    foreach (Issue result in matchList)
                    {
                        supp = new Suppression(textContainer.GetLineContent(result.StartLocation.Line));
                        // If rule is NOT being suppressed then report it
                        SuppressedIssue supissue = supp.GetSuppressedIssue(result.Rule.Id);
                        if (supissue == null)
                        {
                            resultsList.Add(result);
                        }
                        // Otherwise add the suppression info instead
                        else
                        {
                            Boundary bound = textContainer.GetLineBoundary(result.Boundary.Index);
                            bound.Index += supissue.Boundary.Index;
                            bound.Length = supissue.Boundary.Length;

                            //resultsList.Add();
                            Issue info = new Issue()
                            {
                                IsSuppressionInfo = true,
                                Boundary          = bound,
                                StartLocation     = textContainer.GetLocation(bound.Index),
                                EndLocation       = textContainer.GetLocation(bound.Index + bound.Length),
                                Rule = result.Rule
                            };

                            // Add info only if it's not on the same location
                            if (resultsList.FirstOrDefault(x => x.Rule.Id == info.Rule.Id && x.Boundary.Index == info.Boundary.Index) == null)
                            {
                                resultsList.Add(info);
                            }
                            else if (_logger != null)
                            {
                                _logger.Debug("Not added due to proximity to another rule");
                            }
                        }
                    }
                }
                // Otherwise put matchlist to resultlist
                else
                {
                    resultsList.AddRange(matchList);
                }
            }

            // Deal with overrides
            List <Issue> removes = new List <Issue>();

            foreach (Issue m in resultsList)
            {
                if (m.Rule.Overrides != null && m.Rule.Overrides.Length > 0)
                {
                    foreach (string ovrd in m.Rule.Overrides)
                    {
                        // Find all overriden rules and mark them for removal from issues list
                        foreach (Issue om in resultsList.FindAll(x => x.Rule.Id == ovrd))
                        {
                            if (om.Boundary.Index >= m.Boundary.Index &&
                                om.Boundary.Index <= m.Boundary.Index + m.Boundary.Length)
                            {
                                removes.Add(om);
                            }
                        }
                    }
                }
            }

            // Remove overriden rules
            resultsList.RemoveAll(x => removes.Contains(x));

            return(resultsList.ToArray());
        }