private async Task RunAsync(Options options) { if (options.ShowAllDifferences) { options.ShowDifferences = true; } // Validate strategies (before we do any other processing IEnumerable <Type>?runSpecificStrategies = null; if (options.SpecificStrategies != null) { IEnumerable <string>?requestedStrategies = options.SpecificStrategies.Split(',').Select(s => s.Trim().ToLowerInvariant()).Distinct(); runSpecificStrategies = typeof(BaseStrategy).Assembly.GetTypes() .Where(t => t.IsSubclassOf(typeof(BaseStrategy))) .Where(t => requestedStrategies.Contains(t.Name.ToLowerInvariant())); Logger.Debug("Specific strategies requested: {0}", string.Join(", ", runSpecificStrategies.Select(t => t.Name))); if (requestedStrategies.Count() != runSpecificStrategies.Count()) { Logger.Debug("Invalid strategies."); Console.WriteLine("Invalid strategy, available options are:"); IEnumerable <Type>?allStrategies = typeof(BaseStrategy).Assembly.GetTypes().Where(t => t.IsSubclassOf(typeof(BaseStrategy))); foreach (Type?s in allStrategies) { Console.WriteLine($" * {s.Name}"); } Console.WriteLine("Example: oss-reproducible --specific-strategies AutoBuildProducesSamePackage,PackageMatchesSourceStrategy pkg:npm/[email protected]"); return; } } // Expand targets List <string>?targets = new List <string>(); foreach (string?target in options.Targets ?? Array.Empty <string>()) { PackageURL? purl = new PackageURL(target); PackageDownloader?downloader = new PackageDownloader(purl, ProjectManagerFactory, "temp"); foreach (PackageURL?version in downloader.PackageVersions) { targets.Add(version.ToString()); } } List <ReproducibleToolResult>?finalResults = new List <ReproducibleToolResult>(); foreach (string?target in targets) { try { Console.WriteLine($"Analyzing {target}..."); Logger.Debug("Processing: {0}", target); PackageURL?purl = new PackageURL(target); if (purl.Version == null) { Logger.Error("Package is missing a version, which is required for this tool."); continue; } string?tempDirectoryName = Path.Join(Path.GetTempPath(), Guid.NewGuid().ToString().Substring(0, 8)); FileSystemHelper.RetryDeleteDirectory(tempDirectoryName); // Download the package Console.WriteLine("Downloading package..."); PackageDownloader?packageDownloader = new PackageDownloader(purl, ProjectManagerFactory, Path.Join(tempDirectoryName, "package")); List <string>? downloadResults = await packageDownloader.DownloadPackageLocalCopy(purl, false, true); if (!downloadResults.Any()) { Logger.Debug("Unable to download package."); } // Locate the source Console.WriteLine("Locating source..."); FindSourceTool?findSourceTool = new FindSourceTool(ProjectManagerFactory); Dictionary <PackageURL, double>?sourceMap = await findSourceTool.FindSourceAsync(purl); if (sourceMap.Any()) { List <KeyValuePair <PackageURL, double> >?sourceMapList = sourceMap.ToList(); sourceMapList.Sort((a, b) => a.Value.CompareTo(b.Value)); PackageURL?bestSourcePurl = sourceMapList.Last().Key; if (string.IsNullOrEmpty(bestSourcePurl.Version)) { // Tie back the original version to the new PackageURL bestSourcePurl = new PackageURL(bestSourcePurl.Type, bestSourcePurl.Namespace, bestSourcePurl.Name, purl.Version, bestSourcePurl.Qualifiers, bestSourcePurl.Subpath); } Logger.Debug("Identified best source code repository: {0}", bestSourcePurl); // Download the source Console.WriteLine("Downloading source..."); foreach (string?reference in new[] { bestSourcePurl.Version, options.OverrideSourceReference, "master", "main" }) { if (string.IsNullOrWhiteSpace(reference)) { continue; } Logger.Debug("Trying to download package, version/reference [{0}].", reference); PackageURL?purlRef = new PackageURL(bestSourcePurl.Type, bestSourcePurl.Namespace, bestSourcePurl.Name, reference, bestSourcePurl.Qualifiers, bestSourcePurl.Subpath); packageDownloader = new PackageDownloader(purlRef, ProjectManagerFactory, Path.Join(tempDirectoryName, "src")); downloadResults = await packageDownloader.DownloadPackageLocalCopy(purlRef, false, true); if (downloadResults.Any()) { break; } } if (!downloadResults.Any()) { Logger.Debug("Unable to download source."); } } else { Logger.Debug("Unable to locate source repository."); } // Execute all available strategies StrategyOptions?strategyOptions = new StrategyOptions() { PackageDirectory = Path.Join(tempDirectoryName, "package"), SourceDirectory = Path.Join(tempDirectoryName, "src"), PackageUrl = purl, TemporaryDirectory = Path.GetFullPath(tempDirectoryName), DiffTechnique = options.DiffTechnique }; // First, check to see how many strategies apply IEnumerable <Type>?strategies = runSpecificStrategies; if (strategies == null || !strategies.Any()) { strategies = BaseStrategy.GetStrategies(strategyOptions) ?? Array.Empty <Type>(); } int numStrategiesApplies = 0; foreach (Type?strategy in strategies) { ConstructorInfo?ctor = strategy.GetConstructor(new Type[] { typeof(StrategyOptions) }); if (ctor != null) { BaseStrategy?strategyObject = (BaseStrategy)(ctor.Invoke(new object?[] { strategyOptions })); if (strategyObject.StrategyApplies()) { numStrategiesApplies++; } } } Console.Write($"Out of {Yellow(strategies.Count().ToString())} potential strategies, {Yellow(numStrategiesApplies.ToString())} apply. "); if (options.AllStrategies) { Console.WriteLine("Analysis will continue even after a successful strategy is found."); } else { Console.WriteLine("Analysis will stop after the first successful strategy is found."); } List <StrategyResult> strategyResults = new List <StrategyResult>(); Console.WriteLine($"\n{Blue("Results: ")}"); bool hasSuccessfulStrategy = false; foreach (Type?strategy in strategies) { ConstructorInfo?ctor = strategy.GetConstructor(new Type[] { typeof(StrategyOptions) }); if (ctor != null) { // Create a temporary directory, copy the contents from source/package // so that this strategy can modify the contents without affecting other strategies. StrategyOptions?tempStrategyOptions = new StrategyOptions { PackageDirectory = Path.Join(strategyOptions.TemporaryDirectory, strategy.Name, "package"), SourceDirectory = Path.Join(strategyOptions.TemporaryDirectory, strategy.Name, "src"), TemporaryDirectory = Path.Join(strategyOptions.TemporaryDirectory, strategy.Name), PackageUrl = strategyOptions.PackageUrl, IncludeDiffoscope = options.ShowDifferences }; try { OssReproducibleHelpers.DirectoryCopy(strategyOptions.PackageDirectory, tempStrategyOptions.PackageDirectory); OssReproducibleHelpers.DirectoryCopy(strategyOptions.SourceDirectory, tempStrategyOptions.SourceDirectory); } catch (Exception ex) { Logger.Debug(ex, "Error copying directory for strategy. Aborting execution."); } System.IO.Directory.CreateDirectory(tempStrategyOptions.PackageDirectory); System.IO.Directory.CreateDirectory(tempStrategyOptions.SourceDirectory); try { BaseStrategy? strategyObject = (BaseStrategy)(ctor.Invoke(new object?[] { tempStrategyOptions })); StrategyResult?strategyResult = strategyObject.Execute(); if (strategyResult != null) { strategyResults.Add(strategyResult); } if (strategyResult != null) { if (strategyResult.IsSuccess) { Console.WriteLine($" [{Bold().Yellow("PASS")}] {Yellow(strategy.Name)}"); hasSuccessfulStrategy = true; } else { Console.WriteLine($" [{Red("FAIL")}] {Red(strategy.Name)}"); } if (options.ShowDifferences) { foreach (StrategyResultMessage?resultMessage in strategyResult.Messages) { if (resultMessage.Filename != null && resultMessage.CompareFilename != null) { Console.WriteLine($" {Bright.Black("(")}{Blue("P ")}{Bright.Black(")")} {resultMessage.Filename}"); Console.WriteLine($" {Bright.Black("(")}{Blue(" S")}{Bright.Black(")")} {resultMessage.CompareFilename}"); } else if (resultMessage.Filename != null) { Console.WriteLine($" {Bright.Black("(")}{Blue("P+")}{Bright.Black(")")} {resultMessage.Filename}"); } else if (resultMessage.CompareFilename != null) { Console.WriteLine($" {Bright.Black("(")}{Blue("S+")}{Bright.Black(")")} {resultMessage.CompareFilename}"); } IEnumerable <DiffPiece>?differences = resultMessage.Differences ?? Array.Empty <DiffPiece>(); int maxShowDifferences = 20; int numShowDifferences = 0; foreach (DiffPiece?diff in differences) { if (!options.ShowAllDifferences && numShowDifferences > maxShowDifferences) { Console.WriteLine(Background.Blue(Bold().White("NOTE: Additional differences exist but are not shown. Pass --show-all-differences to view them all."))); break; } switch (diff.Type) { case ChangeType.Inserted: Console.WriteLine($"{Bright.Black(diff.Position + ")")}\t{Red("+")} {Blue(diff.Text)}"); ++numShowDifferences; break; case ChangeType.Deleted: Console.WriteLine($"\t{Green("-")} {Green(diff.Text)}"); ++numShowDifferences; break; default: break; } } if (numShowDifferences > 0) { Console.WriteLine(); } } string?diffoscopeFile = Guid.NewGuid().ToString() + ".html"; File.WriteAllText(diffoscopeFile, strategyResult.Diffoscope); Console.WriteLine($" Diffoscope results written to {diffoscopeFile}."); } } else { Console.WriteLine(Green($" [-] {strategy.Name}")); } } catch (Exception ex) { Logger.Warn(ex, "Error processing {0}: {1}", strategy, ex.Message); Logger.Debug(ex.StackTrace); } } if (hasSuccessfulStrategy && !options.AllStrategies) { break; // We don't need to continue } } ReproducibleToolResult?reproducibilityToolResult = new ReproducibleToolResult { PackageUrl = purl.ToString(), Results = strategyResults }; finalResults.Add(reproducibilityToolResult); (double score, string scoreText) = GetReproducibilityScore(reproducibilityToolResult); Console.WriteLine($"\n{Blue("Summary:")}"); string?scoreDisplay = $"{(score * 100.0):0.#}"; if (reproducibilityToolResult.IsReproducible) { Console.WriteLine($" [{Yellow(scoreDisplay + "%")}] {Yellow(scoreText)}"); } else { Console.WriteLine($" [{Red(scoreDisplay + "%")}] {Red(scoreText)}"); } if (options.LeaveIntermediateFiles) { Console.WriteLine(); Console.WriteLine($"Intermediate files are located in [{tempDirectoryName}]."); } else { FileSystemHelper.RetryDeleteDirectory(tempDirectoryName); } } catch (Exception ex) { Logger.Warn(ex, "Error processing {0}: {1}", target, ex.Message); Logger.Debug(ex.StackTrace); } } if (finalResults.Any()) { // Write the output somewhere string?jsonResults = Newtonsoft.Json.JsonConvert.SerializeObject(finalResults, Newtonsoft.Json.Formatting.Indented); if (!string.IsNullOrWhiteSpace(options.OutputFile) && !string.Equals(options.OutputFile, "-", StringComparison.InvariantCultureIgnoreCase)) { try { File.WriteAllText(options.OutputFile, jsonResults); Console.WriteLine($"Detailed results are available in {options.OutputFile}."); } catch (Exception ex) { Logger.Warn(ex, "Unable to write to {0}. Writing to console instead.", options.OutputFile); Console.WriteLine(jsonResults); } } else if (string.Equals(options.OutputFile, "-", StringComparison.InvariantCultureIgnoreCase)) { Console.WriteLine(jsonResults); } } else { Logger.Debug("No results were produced."); } }
/// <summary> /// Main entrypoint for the download program. /// </summary> /// <param name="args">parameters passed in from the user</param> private static async Task Main(string[] args) { ShowToolBanner(); DetectCryptographyTool detectCryptographyTool = new DetectCryptographyTool(); detectCryptographyTool.ParseOptions(args); // select output destination and format detectCryptographyTool.SelectOutput((string?)detectCryptographyTool.Options["output-file"] ?? ""); IOutputBuilder outputBuilder = detectCryptographyTool.SelectFormat((string?)detectCryptographyTool.Options["format"] ?? "text"); if (detectCryptographyTool.Options["target"] is IList <string> targetList && targetList.Count > 0) { StringBuilder?sb = new StringBuilder(); foreach (string?target in targetList) { sb.Clear(); try { List <IssueRecord>?results = null; if (target.StartsWith("pkg:", StringComparison.InvariantCulture)) { PackageURL?purl = new PackageURL(target); results = await(detectCryptographyTool.AnalyzePackage(purl, (string?)detectCryptographyTool.Options["download-directory"] ?? string.Empty, (bool?)detectCryptographyTool.Options["use-cache"] == true) ?? Task.FromResult(new List <IssueRecord>())); } else if (System.IO.Directory.Exists(target)) { results = await(detectCryptographyTool.AnalyzeDirectory(target) ?? Task.FromResult(new List <IssueRecord>())); } else if (File.Exists(target)) { string?targetDirectoryName = null; while (targetDirectoryName == null || System.IO.Directory.Exists(targetDirectoryName)) { targetDirectoryName = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName()); } string?path = await ArchiveHelper.ExtractArchiveAsync(targetDirectoryName, Path.GetFileName(target), File.OpenRead(target)); results = await detectCryptographyTool.AnalyzeDirectory(path); // Clean up after ourselves } else { Logger.Warn($"{target} was neither a Package URL, directory, nor a file."); continue; } if (results == null) { Logger.Warn("No results were generated."); continue; } else { sb.AppendLine("Summary Results:"); sb.AppendLine(Blue("Cryptographic Implementations:")); IOrderedEnumerable <string>?implementations = results.SelectMany(r => r.Issue.Rule.Tags ?? Array.Empty <string>()) .Distinct() .Where(t => t.StartsWith("Cryptography.Implementation.")) .Select(t => t.Replace("Cryptography.Implementation.", "")) .OrderBy(s => s); if (implementations.Any()) { foreach (string?tag in implementations) { sb.AppendLine(Bright.Blue($" * {tag}")); } } else { sb.AppendLine(Bright.Black(" No implementations found.")); } sb.AppendLine(); sb.AppendLine(Red("Cryptographic Library References:")); IOrderedEnumerable <string>?references = results.SelectMany(r => r.Issue.Rule.Tags ?? Array.Empty <string>()) .Distinct() .Where(t => t.StartsWith("Cryptography.Reference.")) .Select(t => t.Replace("Cryptography.Reference.", "")) .OrderBy(s => s); if (references.Any()) { foreach (string?tag in references) { sb.AppendLine(Bright.Red($" * {tag}")); } } else { sb.AppendLine(Bright.Black(" No library references found.")); } sb.AppendLine(); sb.AppendLine(Green("Other Cryptographic Characteristics:")); IOrderedEnumerable <string>?characteristics = results.SelectMany(r => r.Issue.Rule.Tags ?? Array.Empty <string>()) .Distinct() .Where(t => t.Contains("Crypto", StringComparison.InvariantCultureIgnoreCase) && !t.StartsWith("Cryptography.Implementation.") && !t.StartsWith("Cryptography.Reference.")) .Select(t => t.Replace("Cryptography.", "")) .OrderBy(s => s); if (characteristics.Any()) { foreach (string?tag in characteristics) { sb.AppendLine(Bright.Green($" * {tag}")); } } else { sb.AppendLine(Bright.Black(" No additional characteristics found.")); } if ((bool?)detectCryptographyTool.Options["verbose"] == true) { IOrderedEnumerable <IGrouping <string, IssueRecord> >?items = results.GroupBy(k => k.Issue.Rule.Name).OrderByDescending(k => k.Count()); foreach (IGrouping <string, IssueRecord>?item in items) { sb.AppendLine(); sb.AppendLine($"There were {item.Count()} finding(s) of type [{item.Key}]."); foreach (IssueRecord?result in results) { if (result.Issue.Rule.Name == item.Key) { sb.AppendLine($" {result.Filename}:"); if (result.Issue.Rule.Id == "_CRYPTO_DENSITY") { // No excerpt for cryptographic density // TODO: We stuffed the density in the unused 'Description' field. This is code smell. sb.AppendLine($" | The maximum cryptographic density is {result.Issue.Rule.Description}."); } else { // Show the excerpt foreach (string?line in result.TextSample.Split(new char[] { '\n', '\r' })) { if (!string.IsNullOrWhiteSpace(line)) { sb.AppendLine($" | {line.Trim()}"); } } } sb.AppendLine(); } } } } if (Logger.IsDebugEnabled) { foreach (IssueRecord?result in results) { Logger.Debug($"Result: {result.Filename} {result.Issue.Rule.Name} {result.TextSample}"); } } } Console.WriteLine(sb.ToString()); } catch (Exception ex) { Logger.Warn(ex, "Error processing {0}: {1}", target, ex.Message); Logger.Warn(ex.StackTrace); } } } else { Logger.Warn("No target provided; nothing to analyze."); DetectCryptographyTool.ShowUsage(); Environment.Exit(1); } }
/// <summary> /// Main entrypoint for the download program. /// </summary> /// <param name="args"> parameters passed in from the user </param> private static async Task Main(string[] args) { ShowToolBanner(); DetectBackdoorTool?detectBackdoorTool = new DetectBackdoorTool(); Options? parsedOptions = detectBackdoorTool.ParseOptions <Options>(args).Value; List <Dictionary <string, AnalyzeResult?> >?detectionResults = await detectBackdoorTool.RunAsync(parsedOptions); foreach (Dictionary <string, AnalyzeResult?>?result in detectionResults) { foreach (KeyValuePair <string, AnalyzeResult?> entry in result) { if (entry.Value == null || entry.Value.Metadata == null || entry.Value.Metadata.Matches == null) { continue; } if (parsedOptions.Format == "text") { IOrderedEnumerable <MatchRecord>?matchEntries = entry.Value.Metadata.Matches.OrderByDescending(x => x.Confidence); int matchEntriesCount = matchEntries.Count(); int matchIndex = 1; foreach (MatchRecord?match in matchEntries) { WriteMatch(match, matchIndex, matchEntriesCount); matchIndex++; } Console.WriteLine($"{entry.Value.Metadata.TotalMatchesCount} matches found."); } void WriteMatch(MatchRecord match, int index, int matchCount) { string?filename = match.FileName; if (filename == null) { return; } int?sourcePathLength = entry.Value.Metadata.SourcePath?.Length; if (sourcePathLength.HasValue) { if (entry.Value.Metadata.SourcePath != null && filename.StartsWith(entry.Value.Metadata.SourcePath)) { filename = filename[sourcePathLength.Value..]; } } Console.WriteLine(Red($"--[ ") + Blue("Match #") + Yellow(index.ToString()) + Blue(" of ") + Yellow(matchCount.ToString()) + Red(" ]--")); Console.WriteLine(" Rule Id: " + Blue(match.Rule.Id)); Console.WriteLine(" Tag: " + Blue(match.Tags?.First())); Console.WriteLine(" Severity: " + Cyan(match.Severity.ToString()) + ", Confidence: " + Cyan(match.Confidence.ToString())); Console.WriteLine(" Filename: " + Yellow(filename)); Console.WriteLine(" Pattern: " + Green(match.MatchingPattern.Pattern)); foreach (string?line in match.Excerpt.Split(new[] { "\r", "\n", "\r\n" }, StringSplitOptions.None)) { string?s = line; if (s.Length > 100) { s = s.Substring(0, 100); } Console.WriteLine(Bright.Black(" | ") + Magenta(s)); } Console.WriteLine(); }