public async Task GetDeps(XElement project, List <Package> packages)
{
IEnumerable <NuGetFramework> frameworks =
project.Descendants()
.Where(x => x.Name.LocalName == "TargetFramework" || x.Name.LocalName == "TargetFrameworks")
.SingleOrDefault()
.Value.Split(';')
.Select(f => NuGetFramework.ParseFolder(f));
AuditEnvironment.Info("{0}", frameworks.First().Framework);
var nugetPackages = packages.Select(p => new PackageIdentity(p.Name, NuGetVersion.Parse(p.Version)));
var settings = Settings.LoadDefaultSettings(root: null);
var sourceRepositoryProvider = new SourceRepositoryProvider(settings, Repository.Provider.GetCoreV3());
var logger = NullLogger.Instance;
using (var cacheContext = new SourceCacheContext())
{
foreach (var np in nugetPackages)
{
foreach (var sourceRepository in sourceRepositoryProvider.GetRepositories())
{
var dependencyInfoResource = await sourceRepository.GetResourceAsync <DependencyInfoResource>();
var dependencyInfo = await dependencyInfoResource.ResolvePackage(
np, frameworks.First(), cacheContext, logger, CancellationToken.None);
if (dependencyInfo != null)
{
AuditEnvironment.Info("Dependency info: {0}.", dependencyInfo);
}
}
}
}
}
protected override string GetVersion()
{
AuditEnvironment.ProcessExecuteStatus process_status;
string process_output;
string process_error;
AuditEnvironment.Execute(ApplicationBinary.FullName, "-v", out process_status, out process_output, out process_error);
if (process_status == AuditEnvironment.ProcessExecuteStatus.Completed && (process_output.Contains("nginx version: ") || process_error.Contains("nginx version: ")))
{
if (!string.IsNullOrEmpty(process_error) && string.IsNullOrEmpty(process_output))
{
process_output = process_error;
}
this.Version = process_output.Substring("nginx version: ".Length);
this.VersionInitialised = true;
this.AuditEnvironment.Success("Got Nginx version {0}.", this.Version);
return(this.Version);
}
else if (process_output.Contains("nginx version: ") || process_error.Contains("nginx version: "))
{
if (!string.IsNullOrEmpty(process_error) && string.IsNullOrEmpty(process_output))
{
process_output = process_error;
}
this.Version = process_output.Substring("nginx version: ".Length);
this.VersionInitialised = true;
this.AuditEnvironment.Success("Got Nginx version {0}.", this.Version);
return(this.Version);
}
else
{
throw new Exception(string.Format("Did not execute process {0} successfully or could not parse output. Process output: {1}.\nProcess error: {2}.", ApplicationBinary.Name, process_output, process_error));
}
}
public AuditDirectoryInfo(AuditEnvironment env, string dir_path)
{
this.FullName = dir_path;
this.AuditEnvironment = env;
this.PathSeparator = this.AuditEnvironment.OS.Platform == PlatformID.Win32NT ? "\\" : "/";
this.Name = this.GetPathComponents().Last();
}
protected IEnumerable <Package> FilterPackagesUsingProfile()
{
if (this.Packages == null || this.Packages.Count() == 0 || this.AuditProfile == null || this.AuditProfile.Rules == null)
{
return(this.Packages);
}
else if (this.AuditProfile.Rules.Any(r => r.Category == "exclude" && r.Target == this.PackageManagerId))
{
List <Package> packages = this.Packages.ToList();
List <AuditProfileRule> exclude_rules = this.AuditProfile.Rules.Where(r => r.Category == "exclude" && r.Target == this.PackageManagerId).ToList();
foreach (AuditProfileRule r in exclude_rules.Where(r => !string.IsNullOrEmpty(r.MatchName)))
{
try
{
if (packages.Any(p => Regex.IsMatch(p.Name, r.MatchName)))
{
int c = packages.RemoveAll(p => Regex.IsMatch(p.Name, r.MatchName) &&
(string.IsNullOrEmpty(r.MatchVersion) || (!string.IsNullOrEmpty(r.MatchVersion) && this.IsVulnerabilityVersionInPackageVersionRange(r.MatchVersion, p.Version))));
AuditEnvironment.Info("Excluded {0} package(s) using audit profile rules.", c);
}
}
catch (Exception e)
{
AuditEnvironment.Warning("Error attempting to match name {0} with a package name: {1}. Skipping rule.", r.MatchName, e.Message);
}
}
return(packages);
}
else
{
return(this.Packages);
}
}
protected override string GetVersion()
{
Stopwatch sw = new Stopwatch();
sw.Start();
AuditEnvironment.ProcessExecuteStatus process_status;
string process_output;
string process_error;
AuditEnvironment.Execute(this.ApplicationBinary.FullName, "-V", out process_status, out process_output, out process_error);
sw.Stop();
if (process_status == AuditEnvironment.ProcessExecuteStatus.Completed)
{
string v0 = process_output.Substring(process_output.IndexOf("Ver") + 4);
string v1 = v0.Substring(0, v0.IndexOf(" "));
this.Version = new string(v1.TakeWhile(v => char.IsDigit(v) || v == '.').ToArray());
this.VersionInitialised = true;
this.AuditEnvironment.Success("Got {0} version {1} in {2} ms.", this.ApplicationLabel, this.Version, sw.ElapsedMilliseconds);
return(this.Version);
}
else
{
throw new Exception(string.Format("Did not execute process {0} successfully. Error: {1}.", this.ApplicationBinary.Name, process_error));
}
}
public HttpDataSource(AuditEnvironment host_env, Dictionary <string, object> datasource_options)
{
if (!datasource_options.ContainsKey("ApiUrl"))
{
throw new ArgumentException("The ApiUrl option is not specified.");
}
this.DataSourceOptions = datasource_options;
this.HostEnvironment = host_env;
string api_url_s = (string)this.DataSourceOptions["ApiUrl"];
Uri api_url = null;
if (!Uri.TryCreate(api_url_s, UriKind.Absolute, out api_url))
{
this.HostEnvironment.Error("Could not create Uri from {0}.", api_url_s);
this.DataSourceInitialised = false;
return;
}
else
{
this.ApiUrl = api_url;
}
if (this.DataSourceOptions.ContainsKey("HttpsProxy"))
{
HttpsProxy = (Uri)this.DataSourceOptions["HttpsProxy"];
}
}
public LocalDataSource(AuditEnvironment host_env, Dictionary <string, object> datasource_options)
{
if (!datasource_options.ContainsKey("DirectoryPath"))
{
throw new ArgumentException("The datasource options does not contain the DirectoryPath");
}
this.DataSourceOptions = datasource_options;
this.HostEnvironment = host_env;
string dir_path = (string)this.DataSourceOptions["DirectoryPath"];
try
{
DirectoryInfo dir = new DirectoryInfo(dir_path);
if (!dir.Exists)
{
HostEnvironment.Error("The directory {0} does not exist.", dir.FullName);
this.DataSourceInitialised = false;
return;
}
else
{
Directory = dir;
}
}
catch (Exception e)
{
this.HostEnvironment.Error(e, "An error occurred attempting to access the directory {0}.", dir_path);
this.DataSourceInitialised = false;
return;
}
}
public void Error(Span span, ErrorInfo info, params string[] argsOpt)
{
Errors.Add(new ErrorInstance()
{
Span = span,
Error = info,
Args = argsOpt,
});
AuditEnvironment.Debug("Error written to PHP parser sink.");
}
private void AddApiAuthenticationIfPresent(HttpClient client)
{
if (!string.IsNullOrEmpty(this.ApiToken) && !string.IsNullOrEmpty(this.ApiUser))
{
AuditEnvironment.Info("Using API authentication for OSS Index user {0}.", ApiUser);
var byteArray = Encoding.ASCII.GetBytes($"{ApiUser}:{ApiToken}");
client.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic",
Convert.ToBase64String(byteArray));
}
}
public override async Task <bool> ReportPackageSourceAudit()
{
if (!AuditOptions.ContainsKey("GitHubReportOwner") || !AuditOptions.ContainsKey("GitHubReportName") || !AuditOptions.ContainsKey("GitHubToken"))
{
throw new ArgumentException("The GitHubReportOwner, GitHubReportName, and GitHubReportOwner audit options must be present.");
}
if (AuditOptions.ContainsKey("GitHubReportTitle"))
{
IssueTitle = (string)AuditOptions["GitHubReportTitle"];
}
else
{
IssueTitle = string.Format("[DevAudit] {2} audit on {0} {1}", DateTime.UtcNow.ToShortDateString(), DateTime.UtcNow.ToShortTimeString(), Source.PackageManagerLabel);
}
GitHubClient client;
client = new GitHubClient(new ProductHeaderValue("DevAudit"));
client.Credentials = new Credentials((string)AuditOptions["GitHubToken"]);
Repository repository;
try
{
repository = await client.Repository.Get((string)AuditOptions["GitHubReportOwner"], (string)AuditOptions["GitHubReportName"]);
}
catch (Exception)
{
AuditEnvironment.Warning("Could not get repository {0}/{1}.", (string)AuditOptions["GitHubReportOwner"], (string)AuditOptions["GitHubReportName"]);
}
NewIssue issue = new NewIssue(IssueTitle);
BuildPackageSourceAuditReport();
issue.Body = IssueText.ToString();
try
{
Issue i = await client.Issue.Create((string)AuditOptions["GitHubReportOwner"], (string)AuditOptions["GitHubReportName"], issue);
AuditEnvironment.Info("Created issue #{0} {1} in GitHub repository {2}/{3}.", i.Number, IssueTitle, (string)AuditOptions["GitHubReportOwner"], (string)AuditOptions["GitHubReportName"]);
}
catch (AggregateException ae)
{
AuditEnvironment.Error(ae, "Error creating new issue for repository {0}/{1}.", (string)AuditOptions["GitHubReportOwner"], (string)AuditOptions["GitHubReportName"]);
return(false);
}
catch (Exception e)
{
AuditEnvironment.Error(e, "Error creating new issue for repository {0}/{1}.", (string)AuditOptions["GitHubReportOwner"], (string)AuditOptions["GitHubReportName"]);
return(false);
}
return(true);
}
public IQServerReporter(PackageSource source) : base(source)
{
if (!AuditOptions.ContainsKey("IQServerUrl") || !AuditOptions.ContainsKey("IQServerUser") || !AuditOptions.ContainsKey("IQServerPass") || !AuditOptions.ContainsKey("IQServerAppId"))
{
throw new ArgumentException("The IQServerUrl, IQServerUser, IQServerPass, IQServerAppId audit options must all be present.");
}
IQServerUrl = (Uri)AuditOptions["IQServerUrl"];
IQServerUser = (string)AuditOptions["IQServerUser"];
IQServerPass = (string)AuditOptions["IQServerPass"];
IQServerAppId = (string)AuditOptions["IQServerAppId"];
HttpClient = CreateHttpClient();
var byteArray = Encoding.ASCII.GetBytes($"{IQServerUser}:{IQServerPass}");
HttpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic",
Convert.ToBase64String(byteArray));
AuditEnvironment.Info("Authenticating with IQ Server user {0}.", IQServerUser);
}
//Get list of installed programs from 3 registry locations.
public override IEnumerable <Package> GetPackages(params string[] o)
{
var wmi = AuditEnvironment.Execute("wmic", "product get name,version", out var process_status, out var process_output, out var process_error);
if (process_status != AuditEnvironment.ProcessExecuteStatus.Completed)
{
throw new Exception("The wmic command did not execute successfully.");
}
var lines = process_output.Split(AuditEnvironment.LineTerminator.ToCharArray());
var regex = new Regex(@"(\S.+)\s+\d(\S+)", RegexOptions.Compiled);
foreach (var s in lines.Skip(1))
{
var m = regex.Match(s);
}
throw new NotImplementedException();
}
public AuditProfile(AuditEnvironment env, AuditFileInfo pf)
{
this.AuditEnvironment = env;
this.ProfileFile = pf;
this.AuditEnvironment.Info("Using profile file {0}.", pf.FullName);
Deserializer yaml_deserializer = new Deserializer(namingConvention: new CamelCaseNamingConvention(), ignoreUnmatched: true);
try
{
this.Rules = yaml_deserializer.Deserialize <List <AuditProfileRule> >(new StringReader(this.ProfileFile.ReadAsText()));
AuditEnvironment.Info("Loaded {0} rule(s) from audit profile.", this.Rules.Count);
}
catch (Exception e)
{
this.AuditEnvironment.Error(e, "Error occurred reading audit profile from {0}.", this.ProfileFile.FullName);
}
}
public override IEnumerable <Package> GetPackages(params string[] o)
{
Stopwatch sw = new Stopwatch();
List <Package> packages = new List <Package>();
string command = @"dpkg-query";
string arguments = @"-W -f '${package} ${version} ${architecture}|'";
Regex process_output_pattern = new Regex(@"^(\S+)\s(\S+)\s(\S+)$", RegexOptions.Compiled);
AuditEnvironment.ProcessExecuteStatus process_status;
string process_output, process_error;
sw.Start();
if (AuditEnvironment.Execute(command, arguments, out process_status, out process_output, out process_error))
{
string[] p = process_output.Split("|".ToCharArray(), StringSplitOptions.RemoveEmptyEntries);
for (int i = 0; i < p.Count(); i++)
{
if (string.IsNullOrEmpty(p[i]) || string.IsNullOrWhiteSpace(p[i]))
{
continue;
}
Match m = process_output_pattern.Match(p[i].Trim());
if (!m.Success)
{
sw.Stop();
this.AuditEnvironment.Error(this.AuditEnvironment.Here(), "Could not parse dpkg command output at row {0}: {1}.", i, p[i]);
throw new Exception(string.Format("Could not parse dpkg command output at row {0}: {1}.", i, p[i]));
}
else
{
packages.Add(new Package(this.PackageManagerId, m.Groups[1].Value, m.Groups[2].Value, null, null, m.Groups[3].Value));
}
}
sw.Stop();
this.AuditEnvironment.Info("Retrieved {0} packages from {1} package manager in {2} ms.", packages.Count, this.PackageManagerLabel, sw.ElapsedMilliseconds);
}
else
{
sw.Stop();
throw new Exception(string.Format("Error running {0} {1} command in audit environment: {2} {3}.", command,
arguments, process_error, process_output));
}
return(packages);
}
public PHPAuditSourceUnit(AuditEnvironment audit_env, string code, FileInfo file) : base(code, file.FullName, Encoding.UTF8)
{
this.ErrorSink = new PHPAuditErrorSink(this.AuditEnvironment);
BasicNodesFactory factory = new BasicNodesFactory(this);
try
{
this.Parse(factory, this.ErrorSink);
if (this.Ast != null)
{
this.DTV = new DeclarationsTreeVisitor();
this.Ast.VisitMe(this.DTV);
}
}
catch (Exception e)
{
this.AuditEnvironment.Error("Parsing file {0} through an exception.", file.FullName);
this.AuditEnvironment.Error(e);
}
}
protected override string GetVersion()
{
AuditEnvironment.ProcessExecuteStatus process_status;
string process_output;
string process_error;
AuditEnvironment.Execute(ApplicationBinary.FullName, "-v", out process_status, out process_output, out process_error);
if (process_status == AuditEnvironment.ProcessExecuteStatus.Completed)
{
if (!string.IsNullOrEmpty(process_error) && string.IsNullOrEmpty(process_output))
{
process_output = process_error;
}
this.Version = process_output.Split(Environment.NewLine.ToCharArray())[0];
this.VersionInitialised = true;
return(this.Version);
}
else
{
throw new Exception(string.Format("Did not execute process {0} successfully. Error: {1}.", ApplicationBinary.Name, process_error));
}
}
protected override string GetVersion()
{
Stopwatch sw = new Stopwatch();
sw.Start();
this.AuditEnvironment.Status("Scanning {0} version.", this.ApplicationLabel);
AuditEnvironment.ProcessExecuteStatus process_status;
string process_output;
string process_error;
AuditEnvironment.Execute(this.ApplicationBinary.FullName, "-?", out process_status, out process_output, out process_error);
if (process_status == AuditEnvironment.ProcessExecuteStatus.Completed)
{
if (!string.IsNullOrEmpty(process_error) && string.IsNullOrEmpty(process_output))
{
process_output = process_error;
}
this.Version = process_output.Split("\n".ToCharArray(), StringSplitOptions.RemoveEmptyEntries)[1];
sw.Stop();
this.AuditEnvironment.Success("Got {0} version {1} in {2} ms.", this.ApplicationLabel, this.Version, sw.ElapsedMilliseconds);
this.VersionInitialised = true;
return(this.Version);
}
else if (!string.IsNullOrEmpty(process_error) && string.IsNullOrEmpty(process_output) && process_error.Contains("unknown option"))
{
this.Version = process_error.Split("\n".ToCharArray(), StringSplitOptions.RemoveEmptyEntries)[1];
sw.Stop();
this.AuditEnvironment.Success("Got {0} version {1} in {2} ms.", this.ApplicationLabel, this.Version, sw.ElapsedMilliseconds);
this.VersionInitialised = true;
return(this.Version);
}
else
{
sw.Stop();
throw new Exception(string.Format("Did not execute process {0} successfully. Error: {1}.", this.ApplicationBinary.Name, process_error));
}
}
public LibrariesdotIODataSource(AuditEnvironment host_env, Dictionary <string, object> datasource_options) : base(host_env, datasource_options)
{
}
public override IEnumerable <Package> GetPackages(params string[] o)
{
AuditFileInfo config_file = this.AuditEnvironment.ConstructFile(this.PackageManagerConfigurationFile);
List <Package> packages = new List <Package>();
if (config_file.Name.EndsWith(".csproj"))
{
this.AuditEnvironment.Info("Reading packages from .NET Core C# .csproj file.");
string _byteOrderMarkUtf8 = Encoding.UTF8.GetString(Encoding.UTF8.GetPreamble());
string xml = config_file.ReadAsText();
if (xml.StartsWith(_byteOrderMarkUtf8, StringComparison.Ordinal))
{
var lastIndexOfUtf8 = _byteOrderMarkUtf8.Length;
xml = xml.Remove(0, lastIndexOfUtf8);
}
XElement root = XElement.Parse(xml);
if (root.Name.LocalName == "Project")
{
packages =
root
.Descendants()
.Where(x => x.Name.LocalName == "PackageReference" && x.Attribute("Include") != null && x.Attribute("Version") != null)
.SelectMany(r => GetDeveloperPackages(r.Attribute("Include").Value, r.Attribute("Version").Value))
.ToList();
IEnumerable <string> skipped_packages =
root
.Descendants()
.Where(x => x.Name.LocalName == "PackageReference" && x.Attribute("Include") != null && x.Attribute("Version") == null)
.Select(r => r.Attribute("Include").Value);
if (skipped_packages.Count() > 0)
{
this.AuditEnvironment.Warning("{0} package(s) do not have a version specified and will not be audited: {1}.", skipped_packages.Count(),
skipped_packages.Aggregate((s1, s2) => s1 + "," + s2));
}
var helper = new NuGetApiHelper(this.AuditEnvironment, config_file.DirectoryName);
foreach (var framework in helper.GetFrameworks(root))
{
AuditEnvironment.Info("Scanning NuGet transitive dependencies for {0}...", framework.GetFrameworkString());
var deps = helper.GetPackageDependencies(packages, framework);
Task.WaitAll(deps);
packages = helper.AddPackageDependencies(deps.Result, packages);
}
return(packages);
}
else
{
this.AuditEnvironment.Error("{0} is not a .NET Core format .csproj file.", config_file.FullName);
return(packages);
}
}
else if (config_file.Name.EndsWith(".deps.json"))
{
try
{
this.AuditEnvironment.Info("Reading packages from .NET Core depedencies manifest..");
JObject json = (JObject)JToken.Parse(config_file.ReadAsText());
JObject libraries = (JObject)json["libraries"];
if (libraries != null)
{
foreach (JProperty p in libraries.Properties())
{
string[] name = p.Name.Split('/');
packages.Add(new Package("nuget", name[0], name[1]));
}
}
return(packages);
}
catch (Exception e)
{
this.AuditEnvironment.Error(e, "Error reading .NET Core dependencies manifest {0}.", config_file.FullName);
return(packages);
}
}
else
{
this.AuditEnvironment.Error("Unknown .NET Core project file type: {0}.", config_file.FullName);
return(packages);
}
}
public virtual AuditResult Audit(CancellationToken ct)
{
CallerInformation here = this.AuditEnvironment.Here();
this.GetPackagesTask(ct);
this.GetVulnerableCredentialStorageTask(ct);
try
{
Task.WaitAll(this.PackagesTask, this.VulnerableCredentialStorageTask);
if (!this.SkipPackagesAudit)
{
AuditEnvironment.Success("Scanned {0} {1} packages.", this.Packages.Count(), this.PackageManagerLabel);
}
}
catch (AggregateException ae)
{
this.AuditEnvironment.Error(here, ae, "Error in {0} method in GetPackages task.", ae.InnerException.TargetSite.Name);
return(AuditResult.ERROR_SCANNING_PACKAGES);
}
this.Packages = this.FilterPackagesUsingProfile();
this.GetArtifactsTask(ct);
try
{
this.ArtifactsTask.Wait();
}
catch (AggregateException ae)
{
this.AuditEnvironment.Error("Error in GetArtifacts task.", ae);
return(AuditResult.ERROR_SEARCHING_ARTIFACTS);
}
this.GetVulnerabilitiesTask(ct);
try
{
this.VulnerabilitiesTask.Wait();
}
catch (AggregateException ae)
{
this.AuditEnvironment.Error(here, ae, "Error in GetVulnerabilities task");
return(AuditResult.ERROR_SEARCHING_VULNERABILITIES);
}
if (this.Vulnerabilities.Count == 0)
{
this.EvaluateVulnerabilitiesTask = Task.CompletedTask;
}
else
{
this.EvaluateVulnerabilitiesTask = Task.Run(() => this.EvaluateVulnerabilities(), ct);
}
try
{
this.EvaluateVulnerabilitiesTask.Wait();
}
catch (AggregateException ae)
{
this.AuditEnvironment.Error(here, ae.InnerException, "Error in {0} task.", ae.InnerException.TargetSite.Name);
return(AuditResult.ERROR_EVALUATING_VULNERABILITIES);
}
if (this.Vulnerabilities.Count == 0)
{
this.ReportAuditTask = Task.CompletedTask;
this.AuditEnvironment.Info("Not reporting package source audit with zero vulnerabilities.");
}
else
{
this.ReportAuditTask = Task.Run(() => this.ReportAudit(), ct);
}
try
{
this.ReportAuditTask.Wait();
}
catch (AggregateException ae)
{
this.AuditEnvironment.Error(here, ae, "Error in {0} task.", ae.InnerException.TargetSite.Name);
}
return(AuditResult.SUCCESS);
}
public virtual AuditResult Audit(CancellationToken ct)
{
CallerInformation caller = this.AuditEnvironment.Here();
this.GetPackagesTask(ct);
try
{
this.PackagesTask.Wait();
if (!this.SkipPackagesAudit)
{
AuditEnvironment.Success("Scanned {0} {1} packages.", this.Packages.Count(), this.PackageManagerLabel);
}
}
catch (AggregateException ae)
{
this.AuditEnvironment.Error(caller, ae, "Exception thrown in {0} method in GetPackages task.", ae.InnerException.TargetSite.Name);
return(AuditResult.ERROR_SCANNING_PACKAGES);
}
if (this.ListPackages || this.Packages.Count() == 0)
{
this.ArtifactsTask = this.VulnerabilitiesTask = this.EvaluateVulnerabilitiesTask = Task.CompletedTask;
}
else if (this.ListArtifacts)
{
this.ArtifactsTask = Task.Run(() => this.GetArtifacts(), ct);
}
else
{
this.ArtifactsTask = Task.CompletedTask;
}
try
{
this.ArtifactsTask.Wait();
}
catch (AggregateException ae)
{
this.AuditEnvironment.Error("Exception thrown in GetArtifacts task.", ae.InnerException);
return(AuditResult.ERROR_SEARCHING_ARTIFACTS);
}
if (this.ListPackages || this.Packages.Count() == 0 || this.ListArtifacts)
{
this.VulnerabilitiesTask = this.EvaluateVulnerabilitiesTask = Task.CompletedTask;
}
else
{
this.VulnerabilitiesTask = Task.Run(() => this.GetVulnerabiltiesApiv2(), ct);
}
try
{
this.VulnerabilitiesTask.Wait();
}
catch (AggregateException ae)
{
this.AuditEnvironment.Error(caller, ae.InnerException, "Exception thrown in GetVulnerabilities task");
return(AuditResult.ERROR_SEARCHING_VULNERABILITIES);
}
if (this.Vulnerabilities.Count == 0)
{
this.EvaluateVulnerabilitiesTask = Task.CompletedTask;
}
else
{
this.EvaluateVulnerabilitiesTask = Task.Run(() => this.EvaluateVulnerabilities(), ct);
}
try
{
this.EvaluateVulnerabilitiesTask.Wait();
}
catch (AggregateException ae)
{
this.AuditEnvironment.Error(caller, ae.InnerException, "Exception thrown in {0} task.", ae.InnerException.TargetSite.Name);
return(AuditResult.ERROR_EVALUATING_VULNERABILITIES);
}
return(AuditResult.SUCCESS);
}
public override async Task <bool> ReportPackageSourceAudit()
{
if (!this.AuditOptions.ContainsKey("GitLabReportUrl") || !this.AuditOptions.ContainsKey("GitLabReportName") || !this.AuditOptions.ContainsKey("GitLabToken"))
{
throw new ArgumentException("A required audit option for the GitLab environment is missing.");
}
HostUrl = (string)this.AuditOptions["GitLabReportUrl"];
Token = (string)this.AuditOptions["GitLabToken"];
ProjectName = (string)this.AuditOptions["GitLabReportName"];
GitLabClient client = null;
try
{
this.AuditEnvironment.Info("Connecting to project {0} at {1}", ProjectName, HostUrl);
client = new GitLabClient(HostUrl, Token);
IEnumerable <Project> projects = await client.Projects.Owned();
Project = projects.Where(p => p.Name == ProjectName).FirstOrDefault();
this.AuditEnvironment.Info("Connected to project {0}.", Project.PathWithNamespace);
}
catch (AggregateException ae)
{
AuditEnvironment.Error(ae, "Could not get project {0} at url {1}.", ProjectName, HostUrl);
return(false);
}
catch (Exception e)
{
AuditEnvironment.Error(e, "Could not get project {0} at url {1}.", ProjectName, HostUrl);
return(false);
}
if (Project == null)
{
AuditEnvironment.Error("Could not find the project {0}.", Project.Name);
return(false);
}
if (!Project.IssuesEnabled)
{
AuditEnvironment.Error("Issues are not enabled for the project {0}/{1}.", Project.Owner, Project.Name);
return(false);
}
if (AuditOptions.ContainsKey("GitLabReportTitle"))
{
IssueTitle = (string)AuditOptions["GitLabReportTitle"];
}
else
{
IssueTitle = string.Format("[DevAudit] {2} audit on {0} {1}", DateTime.UtcNow.ToShortDateString(), DateTime.UtcNow.ToShortTimeString(), Source.PackageManagerLabel);
}
BuildPackageSourceAuditReport();
try
{
IssueCreate ic = new IssueCreate
{
ProjectId = Project.Id,
Title = IssueTitle,
Description = IssueText.ToString()
};
Issue issue = await client.Issues.CreateAsync(ic);
if (issue != null)
{
AuditEnvironment.Success("Created issue #{0} '{1}' in GitLab project {2}/{3} at host url {4}.", issue.IssueId, issue.Title, Project.Owner, ProjectName, HostUrl);
return(true);
}
else
{
AuditEnvironment.Error("Error creating new issue for project {0} at host url {1}. The issue object is null.", ProjectName, HostUrl);
return(false);
}
}
catch (AggregateException ae)
{
AuditEnvironment.Error(ae, "Error creating new issue for project {0} at host url {1}.", ProjectName, HostUrl);
return(false);
}
catch (Exception e)
{
AuditEnvironment.Error(e, "Error creating new issue for project {0} at host url {1}.", ProjectName, HostUrl);
return(false);
}
}
public override async Task <bool> ReportPackageSourceAudit()
{
HttpClient.DefaultRequestHeaders.Accept.Clear();
HttpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
var r = await HttpClient.GetStringAsync("/api/v2/applications?publicId=" + IQServerAppId);
var apps = JsonConvert.DeserializeObject <IQServerApplications>(r);
if (apps.Applications.Length == 0)
{
AuditEnvironment.Error("The app id {0} does not exist on the IQ Server at {1}.", IQServerAppId, IQServerUrl.ToString());
return(false);
}
IQServerInternalAppId = apps.Applications.First().Id;
SBom.AppendFormat("<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>\n<bom xmlns=\"http://cyclonedx.org/schema/bom/1.1\" version=\"1\" serialNumber=\"urn:uuid:{0}\"\nxmlns:v=\"http://cyclonedx.org/schema/ext/vulnerability/1.0\">\n\t<components>\n",
Guid.NewGuid().ToString("D"));
int packages_count = Source.Vulnerabilities.Count;
int packages_processed = 0;
int c = 0;
foreach (var pv in Source.Vulnerabilities.OrderByDescending(sv => sv.Value.Count(v => v.PackageVersionIsInRange)))
{
IPackage package = pv.Key;
List <IVulnerability> package_vulnerabilities = pv.Value;
var purl = string.IsNullOrEmpty(package.Group) ? string.Format("pkg:nuget/{0}@{1}", package.Name, package.Version) : string.Format("pkg:nuget/{0}/{1}@{2}", package.Group, package.Name, package.Version);
SBom.AppendLine("\t\t<component type =\"library\">");
if (!string.IsNullOrEmpty(package.Group))
{
SBom.AppendFormat("\t\t\t<group>{0}</group>\n", package.Group);
}
SBom.AppendFormat("\t\t\t<name>{0}</name>\n", package.Name);
SBom.AppendFormat("\t\t\t<version>{0}</version>\n", package.Version);
SBom.AppendFormat("\t\t\t<purl>{0}</purl>\n", purl);
if ((package_vulnerabilities.Count() != 0) && (package_vulnerabilities.Count(v => v.PackageVersionIsInRange) > 0))
{
SBom.AppendLine("\t\t\t<v:vulnerabilities>");
var matched_vulnerabilities = package_vulnerabilities.Where(v => v.PackageVersionIsInRange).ToList();
int matched_vulnerabilities_count = matched_vulnerabilities.Count;
matched_vulnerabilities.ForEach(v =>
{
SBom.AppendFormat("\t\t\t\t<v:vulnerability ref=\"{0}\">\n", purl);
SBom.AppendFormat("\t\t\t\t\t<v:id>{0}</v:id>\n", v.CVE != null && v.CVE.Length > 0 ? v.CVE.First() : v.Id);
SBom.AppendLine("\t\t\t\t</v:vulnerability>");
c++;
});
SBom.AppendLine("\t\t\t</v:vulnerabilities>");
}
SBom.AppendLine("\t\t</component>");
packages_processed++;
}
SBom.AppendLine("\t</components>\n</bom>");
AuditEnvironment.Debug("SBOM:\n{0}", SBom.ToString());
HttpClient.DefaultRequestHeaders.Accept.Clear();
//HttpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/xml"));
var req = string.Format("/api/v2/scan/applications/{0}/sources/devaudit", IQServerInternalAppId);
var content = new StringContent(SBom.ToString(), Encoding.UTF8, "application/xml");
var response = await HttpClient.PostAsync(req, content);
if (!response.IsSuccessStatusCode)
{
return(false);
}
string json = await response.Content.ReadAsStringAsync();
AuditEnvironment.Debug("JSON response from IQ Server at {0}: {1}", IQServerUrl.ToString(), json);
var status = JsonConvert.DeserializeObject <IQServerStatus>(json);
AuditEnvironment.Info("IQ Server report complete. {0} components with {1} vulnerabilities submitted. Scan status is at: {2}{3}", packages_count, c, IQServerUrl.ToString(), status.statusUrl);
return(true);
}
public PHPAuditErrorSink(AuditEnvironment audit_env)
{
this.AuditEnvironment = audit_env;
}
private List <Package> GetPackagesFromProjectFile(string filename, bool isCsProj)
{
List <Package> packages = new List <Package>();
AuditFileInfo config_file = this.AuditEnvironment.ConstructFile(filename);
var fileType = isCsProj ? ".csproj" : "build targets";
this.AuditEnvironment.Info($"Reading packages from .NET Core C# {fileType} file.");
string _byteOrderMarkUtf8 = Encoding.UTF8.GetString(Encoding.UTF8.GetPreamble());
string xml = config_file.ReadAsText();
if (xml.StartsWith(_byteOrderMarkUtf8, StringComparison.Ordinal))
{
var lastIndexOfUtf8 = _byteOrderMarkUtf8.Length;
xml = xml.Remove(0, lastIndexOfUtf8);
}
XElement root = XElement.Parse(xml);
var package = isCsProj ? "Include" : "Update";
if (root.Name.LocalName == "Project")
{
packages =
root
.Descendants()
.Where(x => x.Name.LocalName == "PackageReference" && x.Attribute(package) != null && x.Attribute("Version") != null)
.SelectMany(r => GetDeveloperPackages(r.Attribute(package).Value, r.Attribute("Version").Value))
.ToList();
IEnumerable <string> skipped_packages =
root
.Descendants()
.Where(x => x.Name.LocalName == "PackageReference" && x.Attribute(package) != null && x.Attribute("Version") == null)
.Select(r => r.Attribute(package).Value);
if (skipped_packages.Count() > 0)
{
this.AuditEnvironment.Warning("{0} package(s) do not have a version specified and will not be audited: {1}.", skipped_packages.Count(),
skipped_packages.Aggregate((s1, s2) => s1 + "," + s2));
}
var helper = new NuGetApiHelper(this.AuditEnvironment, config_file.DirectoryName);
var nuGetFrameworks = helper.GetFrameworks(root);
if (!nuGetFrameworks.Any())
{
AuditEnvironment.Warning("Scanning from project file found 0 packages, checking for packages.config file. ");
nuGetFrameworks = helper.GetFrameworks();
}
if (!nuGetFrameworks.Any())
{
AuditEnvironment.Warning("Scanning NuGet transitive dependencies failed because no target framework is found in {0}...", config_file.Name);
}
foreach (var framework in nuGetFrameworks)
{
AuditEnvironment.Info("Scanning NuGet transitive dependencies for {0}...", framework.GetFrameworkString());
var deps = helper.GetPackageDependencies(packages, framework);
Task.WaitAll(deps);
packages = helper.AddPackageDependencies(deps.Result, packages);
}
return(packages);
}
else
{
this.AuditEnvironment.Error("{0} is not a .NET Core format .csproj file.", config_file.FullName);
return(packages);
}
}
public override Task <bool> ReportPackageSourceAudit()
{
if (!AuditOptions.ContainsKey("BitBucketReportAccount") || !AuditOptions.ContainsKey("BitBucketReportName") || !AuditOptions.ContainsKey("BitBucketKey"))
{
throw new ArgumentException("The BitBucketReportAccount, BitBucketReportName, and BitBucketReportKey audit options must be present.");
}
string key = (string)AuditOptions["BitBucketKey"];
string[] k = key.Split('|');
if (k.Count() != 2)
{
throw new ArgumentException("The BitBucketReportKey audit option must have the format consumer_key|secret.");
}
string consumer = k[0], secret = k[1];
string account = (string)AuditOptions["BitBucketReportAccount"];
string repository = (string)AuditOptions["BitBucketReportName"];
if (AuditOptions.ContainsKey("BitBucketReportTitle"))
{
IssueTitle = (string)AuditOptions["BitBucketReportTitle"];
}
else
{
IssueTitle = string.Format("[DevAudit] {0} audit on {1} {2}", Source.PackageManagerLabel, DateTime.UtcNow.ToShortDateString(), DateTime.UtcNow.ToShortTimeString());
}
SharpBucketV2 sharp_bucket = new SharpBucketV2();
sharp_bucket.OAuth2LeggedAuthentication(consumer, secret);
RepositoriesEndPoint repository_endpoint = sharp_bucket.RepositoriesEndPoint(account, repository);
IssuesResource r;
try
{
r = repository_endpoint.IssuesResource();
}
catch (Exception e)
{
AuditEnvironment.Error(e, "Could not get issues resource for repository {0}/{1}.", account, repository);
return(Task.FromResult(false));
}
BuildPackageSourceAuditReport();
Issue issue = new Issue()
{
title = IssueTitle,
content = IssueText.ToString(),
status = "new",
priority = "major",
kind = "bug"
};
try
{
Issue i = r.PostIssue(issue);
if (i == null)
{
AuditEnvironment.Error("Could not post issue to repository {0}/{1}.", account, repository);
return(Task.FromResult(false));
}
else
{
AuditEnvironment.Success("Created issue {0} at {1}.", i.title, i.resource_uri);
}
}
catch (Exception e)
{
AuditEnvironment.Error(e, "Could not post issue to repository {0}/{1}.", account, repository);
return(Task.FromResult(false));
}
return(Task.FromResult(true));
}
public override AuditResult Audit(CancellationToken ct)
{
CallerInformation here = this.AuditEnvironment.Here();
try
{
this.GetModules();
this.GetVersion();
}
catch (Exception e)
{
if (e is NotImplementedException && e.TargetSite.Name == "GetVersion")
{
this.AuditEnvironment.Debug("{0} application does not implement standalone GetVersion method.", this.ApplicationLabel);
}
else if (e.TargetSite.Name == "GetVersion")
{
this.AuditEnvironment.Error(e, "There was an error scanning the {0} application version.", this.ApplicationLabel);
return(AuditResult.ERROR_SCANNING_VERSION);
}
else
{
this.AuditEnvironment.Error(e, "There was an error scanning the {0} application modules.", this.ApplicationLabel);
return(AuditResult.ERROR_SCANNING_MODULES);
}
}
this.GetPackagesTask(ct);
this.GetConfigurationTask(ct);
try
{
Task.WaitAll(this.PackagesTask, this.ConfigurationTask);
if (!this.SkipPackagesAudit && !this.PrintConfiguration && this.PackageSourceInitialized && this.PackagesTask.Status == TaskStatus.RanToCompletion)
{
AuditEnvironment.Success("Scanned {0} {1} packages.", this.Packages.Count(), this.PackageManagerLabel);
}
}
catch (AggregateException ae)
{
if (ae.InnerException is NotImplementedException && ae.InnerException.TargetSite.Name == "GetConfiguration")
{
this.AuditEnvironment.Debug("{0} application doe not implement standalone GetConfiguration method.", this.ApplicationId);
}
else
{
this.AuditEnvironment.Error(here, ae, "Error occurred in {0} task.", ae.InnerException.TargetSite.Name);
if (ae.TargetSite.Name == "GetPackages")
{
return(AuditResult.ERROR_SCANNING_PACKAGES);
}
else
{
return(AuditResult.ERROR_SCANNING_CONFIGURATION);
}
}
}
this.GetArtifactsTask(ct);
try
{
this.ArtifactsTask.Wait();
}
catch (AggregateException ae)
{
this.AuditEnvironment.Error("Exception thrown in GetArtifacts task.", ae.InnerException);
return(AuditResult.ERROR_SEARCHING_ARTIFACTS);
}
this.GetVulnerableCredentialStorageTask(ct);
this.GetVulnerabilitiesTask(ct);
this.GetConfigurationRulesTask(ct);
if (this.ListPackages || this.ListArtifacts || !this.ConfigurationInitialised || this.PrintConfiguration)
{
this.DefaultConfigurationRulesTask = Task.CompletedTask;
}
else
{
this.DefaultConfigurationRulesTask = Task.Run(() => this.GetDefaultConfigurationRules());
}
if (this.ListPackages || this.ListArtifacts || this.PrintConfiguration)
{
this.GetAnalyzersTask = Task.CompletedTask;
}
else if (this.ModulesInitialised && !string.IsNullOrEmpty(this.AnalyzerType))
{
this.GetAnalyzersTask = Task.Run(() => this.GetAnalyzers());
}
else
{
this.GetAnalyzersTask = Task.CompletedTask;
}
try
{
Task.WaitAll(this.VulnerableCredentialStorageTask, this.VulnerabilitiesTask, this.ConfigurationRulesTask, this.DefaultConfigurationRulesTask, this.GetAnalyzersTask);
}
catch (AggregateException ae)
{
if (ae.InnerException.TargetSite.Name == "GetVulnerabilities")
{
this.AuditEnvironment.Error(here, ae.InnerException, "Exception thrown in GetVulnerabilities task.");
return(AuditResult.ERROR_SEARCHING_VULNERABILITIES);
}
else if (ae.InnerException.TargetSite.Name == "GetVulnerableCredentialStorage")
{
this.AuditEnvironment.Error(here, ae.InnerException, "Exception thrown in GetVulnerableCredentialStorage task.");
return(AuditResult.ERROR_SCANNING_VULNERABLE_CREDENTIAL_STORAGE);
}
else if (ae.InnerException.TargetSite.Name == "GetConfigurationRules")
{
this.AuditEnvironment.Error(here, ae.InnerException, "Exception thrown in GetDefaultConfigurationRules task.");
return(AuditResult.ERROR_SCANNING_DEFAULT_CONFIGURATION_RULES);
}
else if (ae.InnerException.TargetSite.Name == "GetAnalyzers")
{
this.AuditEnvironment.Error(here, ae.InnerException, "Exception thrown in GetAnalyzers task.");
return(AuditResult.ERROR_SCANNING_ANALYZERS);
}
else
{
this.AuditEnvironment.Error(here, ae.InnerException);
return(AuditResult.ERROR_SEARCHING_VULNERABILITIES);
}
}
if (this.ListPackages || this.ListArtifacts || this.ListConfigurationRules || this.Vulnerabilities.Count == 0 || this.PrintConfiguration)
{
this.EvaluateVulnerabilitiesTask = Task.CompletedTask;
}
else
{
this.EvaluateVulnerabilitiesTask = Task.Run(() => this.EvaluateVulnerabilities(), ct);
}
if (this.ListPackages || this.ListArtifacts || this.ListConfigurationRules || this.PrintConfiguration)
{
this.EvaluateConfigurationRulesTask = Task.CompletedTask;
}
else
{
this.EvaluateConfigurationRulesTask = Task.Run(() => this.EvaluateProjectConfigurationRules(), ct);
}
if (this.ListPackages || this.ListArtifacts || this.ListConfigurationRules || this.PrintConfiguration)
{
this.GetAnalyzersResultsTask = Task.CompletedTask;
}
else if (this.AnalyzersInitialized)
{
this.GetAnalyzersResultsTask = Task.Run(() => this.GetAnalyzerResults());
}
else
{
this.GetAnalyzersResultsTask = Task.CompletedTask;
}
try
{
Task.WaitAll(this.EvaluateVulnerabilitiesTask, this.EvaluateConfigurationRulesTask, this.GetAnalyzersResultsTask);
}
catch (AggregateException ae)
{
this.AuditEnvironment.Error(here, ae.InnerException, "Exception thrown in {0} task.", ae.InnerException.TargetSite.Name);
return(AuditResult.ERROR_EVALUATING_CONFIGURATION_RULES);
}
return(AuditResult.SUCCESS);
}
public NuGetApiHelper(AuditEnvironment env, string projRoot)
{
Environment = env;
_projRoot = projRoot;
}
