private void RunRules(StringBuilder sb, string inputFileName)
        {
            string fileName          = Path.GetFileName(inputFileName);
            string actualDirectory   = Path.Combine(Path.GetDirectoryName(inputFileName), "Actual");
            string expectedDirectory = Path.Combine(Path.GetDirectoryName(inputFileName), "Expected");

            if (Directory.Exists(actualDirectory))
            {
                Directory.Delete(actualDirectory, true);
            }
            Directory.CreateDirectory(actualDirectory);

            string expectedFileName = Path.Combine(expectedDirectory, fileName + ".sarif");
            string actualFileName   = Path.Combine(actualDirectory, fileName + ".sarif");

            AnalyzeCommand command = new AnalyzeCommand();
            AnalyzeOptions options = new AnalyzeOptions();

            options.TargetFileSpecifiers = new string[] { inputFileName };
            options.OutputFilePath       = actualFileName;
            options.Verbose = true;
            options.Recurse = false;
            options.ConfigurationFilePath = "default";

            int result = command.Run(options);

            Assert.Equal(0, result);

            JsonSerializerSettings settings = new JsonSerializerSettings()
            {
                ContractResolver = SarifContractResolver.Instance,
                Formatting       = Formatting.Indented
            };

            string expectedText = File.ReadAllText(expectedFileName);
            string actualText   = File.ReadAllText(actualFileName);

            // Make sure we can successfully deserialize what was just generated
            ResultLog expectedLog = JsonConvert.DeserializeObject <ResultLog>(expectedText, settings);
            ResultLog actualLog   = JsonConvert.DeserializeObject <ResultLog>(actualText, settings);

            var visitor = new ResultDiffingVisitor(expectedLog);

            if (!visitor.Diff(actualLog.RunLogs[0].Results))
            {
                string errorMessage = "The output of the tool did not match for input {0}.";
                sb.AppendLine(String.Format(CultureInfo.CurrentCulture, errorMessage, inputFileName));
                sb.AppendLine("Check differences with:");
                sb.AppendLine(GenerateDiffCommand(expectedFileName, actualFileName));
            }
        }
Beispiel #2
0
        private void RunRules(StringBuilder sb, string inputFileName)
        {
            string fileName        = Path.GetFileName(inputFileName);
            string actualDirectory = Path.Combine(Path.GetDirectoryName(inputFileName), "Actual");
            string expectedDirectory;

            if (PlatformSpecificHelpers.RunningOnWindows())
            {
                expectedDirectory = Path.Combine(Path.GetDirectoryName(inputFileName), "Expected");
            }
            else
            {
                expectedDirectory = Path.Combine(Path.GetDirectoryName(inputFileName), "NonWindowsExpected");
            }
            if (!Directory.Exists(actualDirectory))
            {
                Directory.CreateDirectory(actualDirectory);
            }

            string expectedFileName = Path.Combine(expectedDirectory, fileName + ".sarif");
            string actualFileName   = Path.Combine(actualDirectory, fileName + ".sarif");

            var command = new AnalyzeCommand();
            var options = new AnalyzeOptions
            {
                Force                 = true,
                Verbose               = true,
                Recurse               = false,
                PrettyPrint           = true,
                ComputeFileHashes     = true,
                OutputFilePath        = actualFileName,
                ConfigurationFilePath = "default",
                SarifOutputVersion    = SarifVersion.Current,
                TargetFileSpecifiers  = new string[] { inputFileName }
            };

            int result = command.Run(options);

            // Note that we don't ensure a success code. That is because we
            // are running end-to-end tests for valid and invalid files

            var settings = new JsonSerializerSettings()
            {
                Formatting = Newtonsoft.Json.Formatting.Indented
            };

            string expectedText = File.ReadAllText(expectedFileName);
            string actualText   = File.ReadAllText(actualFileName);

            // Replace repository root absolute path with Z:\ for machine and enlistment independence
            string repoRoot = Path.GetFullPath(Path.Combine(actualDirectory, "..", "..", "..", ".."));

            actualText = actualText.Replace(repoRoot.Replace(@"\", @"\\"), @"Z:");
            actualText = actualText.Replace(repoRoot.Replace(@"\", @"/"), @"Z:");

            // Remove stack traces as they can change due to inlining differences by configuration and runtime.
            actualText = Regex.Replace(actualText, @"\\r\\n   at [^""]+", "");

            actualText = actualText.Replace(@"""Sarif""", @"""BinSkim""");
            actualText = actualText.Replace(@"        ""fileVersion"": ""15.0.0""," + Environment.NewLine, string.Empty);

            actualText = Regex.Replace(actualText, @"\s*""fullName""[^\n]+?\n", Environment.NewLine);
            actualText = Regex.Replace(actualText, @"\s*""semanticVersion""[^\n]+?\n", Environment.NewLine);
            actualText = Regex.Replace(actualText, @"\s*""sarifLoggerVersion""[^\n]+?\n", Environment.NewLine);
            actualText = Regex.Replace(actualText, @"\s*""dottedQuadFileVersion""[^\n]+?\n", Environment.NewLine);
            actualText = Regex.Replace(actualText, @"\s*""Comments""[^\n]+?\n", Environment.NewLine);
            actualText = Regex.Replace(actualText, @"\s*""CompanyName""[^\n]+?\n", Environment.NewLine);
            actualText = Regex.Replace(actualText, @"\s*""ProductName""[^\n]+?\n", Environment.NewLine);

            actualText = Regex.Replace(actualText, @"\s*""time""[^\n]+?\n", Environment.NewLine);
            actualText = Regex.Replace(actualText, @"\s*""endTimeUtc""[^\n]+?\n", Environment.NewLine);
            actualText = Regex.Replace(actualText, @"\s*""startTimeUtc""[^\n]+?\n", Environment.NewLine);
            actualText = Regex.Replace(actualText, @"\s*""processId""[^\n]+?\n", Environment.NewLine);
            actualText = Regex.Replace(actualText, @"      ""id""[^,]+,\s+""tool""", @"      ""tool""", RegexOptions.Multiline);

            // Write back the normalized actual text so that the diff command given on failure shows what was actually compared.

            Encoding utf8encoding = new UTF8Encoding(true);

            using (var textWriter = new StreamWriter(actualFileName, false, utf8encoding))
            {
                textWriter.Write(actualText);
            }

            // Make sure we can successfully deserialize what was just generated
            SarifLog expectedLog = PrereleaseCompatibilityTransformer.UpdateToCurrentVersion(
                expectedText,
                settings.Formatting,
                out expectedText);

            SarifLog actualLog = JsonConvert.DeserializeObject <SarifLog>(actualText, settings);

            var visitor = new ResultDiffingVisitor(expectedLog);

            if (!visitor.Diff(actualLog.Runs[0].Results))
            {
                string errorMessage = "The output of the tool did not match for input {0}.";
                sb.AppendLine(string.Format(CultureInfo.CurrentCulture, errorMessage, inputFileName));
                sb.AppendLine("Check differences with:");
                sb.AppendLine(this.GenerateDiffCommand(expectedFileName, actualFileName));
            }
        }
Beispiel #3
0
        private void RunRules(StringBuilder sb, string inputFileName)
        {
            string fileName          = Path.GetFileName(inputFileName);
            string actualDirectory   = Path.Combine(Path.GetDirectoryName(inputFileName), "Actual");
            string expectedDirectory = Path.Combine(Path.GetDirectoryName(inputFileName), "Expected");

            if (!Directory.Exists(actualDirectory))
            {
                Directory.CreateDirectory(actualDirectory);
            }

            string expectedFileName = Path.Combine(expectedDirectory, fileName + ".sarif");
            string actualFileName   = Path.Combine(actualDirectory, fileName + ".sarif");

            AnalyzeCommand command = new AnalyzeCommand();
            AnalyzeOptions options = new AnalyzeOptions();

            options.TargetFileSpecifiers = new string[] { inputFileName };
            options.OutputFilePath       = actualFileName;
            options.Verbose               = true;
            options.Recurse               = false;
            options.ComputeFileHashes     = true;
            options.ConfigurationFilePath = "default";

            int result = command.Run(options);

            // Note that we don't ensure a success code. That is because we
            // are running end-to-end tests for valid and invalid files

            JsonSerializerSettings settings = new JsonSerializerSettings()
            {
                ContractResolver = SarifContractResolver.Instance,
                Formatting       = Formatting.Indented
            };

            string expectedText = File.ReadAllText(expectedFileName);
            string actualText   = File.ReadAllText(actualFileName);

            // Replace repository root absolute path with Z:\ for machine and enlistment independence
            string repoRoot = Path.GetFullPath(Path.Combine(actualDirectory, "..", "..", "..", ".."));

            actualText = actualText.Replace(repoRoot.Replace(@"\", @"\\"), @"Z:");
            actualText = actualText.Replace(repoRoot.Replace(@"\", @"/"), @"Z:");

            // Remove stack traces as they can change due to inlining differences by configuration and runtime.
            actualText = Regex.Replace(actualText, @"\\r\\n   at [^""]+", "");

            // Write back the normalized actual text so that the diff command given on failure shows what was actually compared.
            File.WriteAllText(actualFileName, actualText);

            // Make sure we can successfully deserialize what was just generated
            SarifLog expectedLog = JsonConvert.DeserializeObject <SarifLog>(expectedText, settings);
            SarifLog actualLog   = JsonConvert.DeserializeObject <SarifLog>(actualText, settings);

            var visitor = new ResultDiffingVisitor(expectedLog);

            if (!visitor.Diff(actualLog.Runs[0].Results))
            {
                string errorMessage = "The output of the tool did not match for input {0}.";
                sb.AppendLine(String.Format(CultureInfo.CurrentCulture, errorMessage, inputFileName));
                sb.AppendLine("Check differences with:");
                sb.AppendLine(GenerateDiffCommand(expectedFileName, actualFileName));
            }
        }