protected IEnumerable <Package> FilterPackagesUsingProfile()
{
if (this.Packages == null || this.Packages.Count() == 0 || this.AuditProfile == null || this.AuditProfile.Rules == null)
{
return(this.Packages);
}
else if (this.AuditProfile.Rules.Any(r => r.Category == "exclude" && r.Target == this.PackageManagerId))
{
List <Package> packages = this.Packages.ToList();
List <AuditProfileRule> exclude_rules = this.AuditProfile.Rules.Where(r => r.Category == "exclude" && r.Target == this.PackageManagerId).ToList();
foreach (AuditProfileRule r in exclude_rules.Where(r => !string.IsNullOrEmpty(r.MatchName)))
{
try
{
if (packages.Any(p => Regex.IsMatch(p.Name, r.MatchName)))
{
int c = packages.RemoveAll(p => Regex.IsMatch(p.Name, r.MatchName) &&
(string.IsNullOrEmpty(r.MatchVersion) || (!string.IsNullOrEmpty(r.MatchVersion) && this.IsVulnerabilityVersionInPackageVersionRange(r.MatchVersion, p.Version))));
AuditEnvironment.Info("Excluded {0} package(s) using audit profile rules.", c);
}
}
catch (Exception e)
{
AuditEnvironment.Warning("Error attempting to match name {0} with a package name: {1}. Skipping rule.", r.MatchName, e.Message);
}
}
return(packages);
}
else
{
return(this.Packages);
}
}
public async Task GetDeps(XElement project, List <Package> packages)
{
IEnumerable <NuGetFramework> frameworks =
project.Descendants()
.Where(x => x.Name.LocalName == "TargetFramework" || x.Name.LocalName == "TargetFrameworks")
.SingleOrDefault()
.Value.Split(';')
.Select(f => NuGetFramework.ParseFolder(f));
AuditEnvironment.Info("{0}", frameworks.First().Framework);
var nugetPackages = packages.Select(p => new PackageIdentity(p.Name, NuGetVersion.Parse(p.Version)));
var settings = Settings.LoadDefaultSettings(root: null);
var sourceRepositoryProvider = new SourceRepositoryProvider(settings, Repository.Provider.GetCoreV3());
var logger = NullLogger.Instance;
using (var cacheContext = new SourceCacheContext())
{
foreach (var np in nugetPackages)
{
foreach (var sourceRepository in sourceRepositoryProvider.GetRepositories())
{
var dependencyInfoResource = await sourceRepository.GetResourceAsync <DependencyInfoResource>();
var dependencyInfo = await dependencyInfoResource.ResolvePackage(
np, frameworks.First(), cacheContext, logger, CancellationToken.None);
if (dependencyInfo != null)
{
AuditEnvironment.Info("Dependency info: {0}.", dependencyInfo);
}
}
}
}
}
private void AddApiAuthenticationIfPresent(HttpClient client)
{
if (!string.IsNullOrEmpty(this.ApiToken) && !string.IsNullOrEmpty(this.ApiUser))
{
AuditEnvironment.Info("Using API authentication for OSS Index user {0}.", ApiUser);
var byteArray = Encoding.ASCII.GetBytes($"{ApiUser}:{ApiToken}");
client.DefaultRequestHeaders.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic",
Convert.ToBase64String(byteArray));
}
}
public override async Task <bool> ReportPackageSourceAudit()
{
if (!AuditOptions.ContainsKey("GitHubReportOwner") || !AuditOptions.ContainsKey("GitHubReportName") || !AuditOptions.ContainsKey("GitHubToken"))
{
throw new ArgumentException("The GitHubReportOwner, GitHubReportName, and GitHubReportOwner audit options must be present.");
}
if (AuditOptions.ContainsKey("GitHubReportTitle"))
{
IssueTitle = (string)AuditOptions["GitHubReportTitle"];
}
else
{
IssueTitle = string.Format("[DevAudit] {2} audit on {0} {1}", DateTime.UtcNow.ToShortDateString(), DateTime.UtcNow.ToShortTimeString(), Source.PackageManagerLabel);
}
GitHubClient client;
client = new GitHubClient(new ProductHeaderValue("DevAudit"));
client.Credentials = new Credentials((string)AuditOptions["GitHubToken"]);
Repository repository;
try
{
repository = await client.Repository.Get((string)AuditOptions["GitHubReportOwner"], (string)AuditOptions["GitHubReportName"]);
}
catch (Exception)
{
AuditEnvironment.Warning("Could not get repository {0}/{1}.", (string)AuditOptions["GitHubReportOwner"], (string)AuditOptions["GitHubReportName"]);
}
NewIssue issue = new NewIssue(IssueTitle);
BuildPackageSourceAuditReport();
issue.Body = IssueText.ToString();
try
{
Issue i = await client.Issue.Create((string)AuditOptions["GitHubReportOwner"], (string)AuditOptions["GitHubReportName"], issue);
AuditEnvironment.Info("Created issue #{0} {1} in GitHub repository {2}/{3}.", i.Number, IssueTitle, (string)AuditOptions["GitHubReportOwner"], (string)AuditOptions["GitHubReportName"]);
}
catch (AggregateException ae)
{
AuditEnvironment.Error(ae, "Error creating new issue for repository {0}/{1}.", (string)AuditOptions["GitHubReportOwner"], (string)AuditOptions["GitHubReportName"]);
return(false);
}
catch (Exception e)
{
AuditEnvironment.Error(e, "Error creating new issue for repository {0}/{1}.", (string)AuditOptions["GitHubReportOwner"], (string)AuditOptions["GitHubReportName"]);
return(false);
}
return(true);
}
public IQServerReporter(PackageSource source) : base(source)
{
if (!AuditOptions.ContainsKey("IQServerUrl") || !AuditOptions.ContainsKey("IQServerUser") || !AuditOptions.ContainsKey("IQServerPass") || !AuditOptions.ContainsKey("IQServerAppId"))
{
throw new ArgumentException("The IQServerUrl, IQServerUser, IQServerPass, IQServerAppId audit options must all be present.");
}
IQServerUrl = (Uri)AuditOptions["IQServerUrl"];
IQServerUser = (string)AuditOptions["IQServerUser"];
IQServerPass = (string)AuditOptions["IQServerPass"];
IQServerAppId = (string)AuditOptions["IQServerAppId"];
HttpClient = CreateHttpClient();
var byteArray = Encoding.ASCII.GetBytes($"{IQServerUser}:{IQServerPass}");
HttpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic",
Convert.ToBase64String(byteArray));
AuditEnvironment.Info("Authenticating with IQ Server user {0}.", IQServerUser);
}
public AuditProfile(AuditEnvironment env, AuditFileInfo pf)
{
this.AuditEnvironment = env;
this.ProfileFile = pf;
this.AuditEnvironment.Info("Using profile file {0}.", pf.FullName);
Deserializer yaml_deserializer = new Deserializer(namingConvention: new CamelCaseNamingConvention(), ignoreUnmatched: true);
try
{
this.Rules = yaml_deserializer.Deserialize <List <AuditProfileRule> >(new StringReader(this.ProfileFile.ReadAsText()));
AuditEnvironment.Info("Loaded {0} rule(s) from audit profile.", this.Rules.Count);
}
catch (Exception e)
{
this.AuditEnvironment.Error(e, "Error occurred reading audit profile from {0}.", this.ProfileFile.FullName);
}
}
private List <Package> GetPackagesFromProjectFile(string filename, bool isCsProj)
{
List <Package> packages = new List <Package>();
AuditFileInfo config_file = this.AuditEnvironment.ConstructFile(filename);
var fileType = isCsProj ? ".csproj" : "build targets";
this.AuditEnvironment.Info($"Reading packages from .NET Core C# {fileType} file.");
string _byteOrderMarkUtf8 = Encoding.UTF8.GetString(Encoding.UTF8.GetPreamble());
string xml = config_file.ReadAsText();
if (xml.StartsWith(_byteOrderMarkUtf8, StringComparison.Ordinal))
{
var lastIndexOfUtf8 = _byteOrderMarkUtf8.Length;
xml = xml.Remove(0, lastIndexOfUtf8);
}
XElement root = XElement.Parse(xml);
var package = isCsProj ? "Include" : "Update";
if (root.Name.LocalName == "Project")
{
packages =
root
.Descendants()
.Where(x => x.Name.LocalName == "PackageReference" && x.Attribute(package) != null && x.Attribute("Version") != null)
.SelectMany(r => GetDeveloperPackages(r.Attribute(package).Value, r.Attribute("Version").Value))
.ToList();
IEnumerable <string> skipped_packages =
root
.Descendants()
.Where(x => x.Name.LocalName == "PackageReference" && x.Attribute(package) != null && x.Attribute("Version") == null)
.Select(r => r.Attribute(package).Value);
if (skipped_packages.Count() > 0)
{
this.AuditEnvironment.Warning("{0} package(s) do not have a version specified and will not be audited: {1}.", skipped_packages.Count(),
skipped_packages.Aggregate((s1, s2) => s1 + "," + s2));
}
var helper = new NuGetApiHelper(this.AuditEnvironment, config_file.DirectoryName);
var nuGetFrameworks = helper.GetFrameworks(root);
if (!nuGetFrameworks.Any())
{
AuditEnvironment.Warning("Scanning from project file found 0 packages, checking for packages.config file. ");
nuGetFrameworks = helper.GetFrameworks();
}
if (!nuGetFrameworks.Any())
{
AuditEnvironment.Warning("Scanning NuGet transitive dependencies failed because no target framework is found in {0}...", config_file.Name);
}
foreach (var framework in nuGetFrameworks)
{
AuditEnvironment.Info("Scanning NuGet transitive dependencies for {0}...", framework.GetFrameworkString());
var deps = helper.GetPackageDependencies(packages, framework);
Task.WaitAll(deps);
packages = helper.AddPackageDependencies(deps.Result, packages);
}
return(packages);
}
else
{
this.AuditEnvironment.Error("{0} is not a .NET Core format .csproj file.", config_file.FullName);
return(packages);
}
}
public override async Task <bool> ReportPackageSourceAudit()
{
HttpClient.DefaultRequestHeaders.Accept.Clear();
HttpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
var r = await HttpClient.GetStringAsync("/api/v2/applications?publicId=" + IQServerAppId);
var apps = JsonConvert.DeserializeObject <IQServerApplications>(r);
if (apps.Applications.Length == 0)
{
AuditEnvironment.Error("The app id {0} does not exist on the IQ Server at {1}.", IQServerAppId, IQServerUrl.ToString());
return(false);
}
IQServerInternalAppId = apps.Applications.First().Id;
SBom.AppendFormat("<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>\n<bom xmlns=\"http://cyclonedx.org/schema/bom/1.1\" version=\"1\" serialNumber=\"urn:uuid:{0}\"\nxmlns:v=\"http://cyclonedx.org/schema/ext/vulnerability/1.0\">\n\t<components>\n",
Guid.NewGuid().ToString("D"));
int packages_count = Source.Vulnerabilities.Count;
int packages_processed = 0;
int c = 0;
foreach (var pv in Source.Vulnerabilities.OrderByDescending(sv => sv.Value.Count(v => v.PackageVersionIsInRange)))
{
IPackage package = pv.Key;
List <IVulnerability> package_vulnerabilities = pv.Value;
var purl = string.IsNullOrEmpty(package.Group) ? string.Format("pkg:nuget/{0}@{1}", package.Name, package.Version) : string.Format("pkg:nuget/{0}/{1}@{2}", package.Group, package.Name, package.Version);
SBom.AppendLine("\t\t<component type =\"library\">");
if (!string.IsNullOrEmpty(package.Group))
{
SBom.AppendFormat("\t\t\t<group>{0}</group>\n", package.Group);
}
SBom.AppendFormat("\t\t\t<name>{0}</name>\n", package.Name);
SBom.AppendFormat("\t\t\t<version>{0}</version>\n", package.Version);
SBom.AppendFormat("\t\t\t<purl>{0}</purl>\n", purl);
if ((package_vulnerabilities.Count() != 0) && (package_vulnerabilities.Count(v => v.PackageVersionIsInRange) > 0))
{
SBom.AppendLine("\t\t\t<v:vulnerabilities>");
var matched_vulnerabilities = package_vulnerabilities.Where(v => v.PackageVersionIsInRange).ToList();
int matched_vulnerabilities_count = matched_vulnerabilities.Count;
matched_vulnerabilities.ForEach(v =>
{
SBom.AppendFormat("\t\t\t\t<v:vulnerability ref=\"{0}\">\n", purl);
SBom.AppendFormat("\t\t\t\t\t<v:id>{0}</v:id>\n", v.CVE != null && v.CVE.Length > 0 ? v.CVE.First() : v.Id);
SBom.AppendLine("\t\t\t\t</v:vulnerability>");
c++;
});
SBom.AppendLine("\t\t\t</v:vulnerabilities>");
}
SBom.AppendLine("\t\t</component>");
packages_processed++;
}
SBom.AppendLine("\t</components>\n</bom>");
AuditEnvironment.Debug("SBOM:\n{0}", SBom.ToString());
HttpClient.DefaultRequestHeaders.Accept.Clear();
//HttpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/xml"));
var req = string.Format("/api/v2/scan/applications/{0}/sources/devaudit", IQServerInternalAppId);
var content = new StringContent(SBom.ToString(), Encoding.UTF8, "application/xml");
var response = await HttpClient.PostAsync(req, content);
if (!response.IsSuccessStatusCode)
{
return(false);
}
string json = await response.Content.ReadAsStringAsync();
AuditEnvironment.Debug("JSON response from IQ Server at {0}: {1}", IQServerUrl.ToString(), json);
var status = JsonConvert.DeserializeObject <IQServerStatus>(json);
AuditEnvironment.Info("IQ Server report complete. {0} components with {1} vulnerabilities submitted. Scan status is at: {2}{3}", packages_count, c, IQServerUrl.ToString(), status.statusUrl);
return(true);
}
public override IEnumerable <Package> GetPackages(params string[] o)
{
AuditFileInfo config_file = this.AuditEnvironment.ConstructFile(this.PackageManagerConfigurationFile);
List <Package> packages = new List <Package>();
if (config_file.Name.EndsWith(".csproj"))
{
this.AuditEnvironment.Info("Reading packages from .NET Core C# .csproj file.");
string _byteOrderMarkUtf8 = Encoding.UTF8.GetString(Encoding.UTF8.GetPreamble());
string xml = config_file.ReadAsText();
if (xml.StartsWith(_byteOrderMarkUtf8, StringComparison.Ordinal))
{
var lastIndexOfUtf8 = _byteOrderMarkUtf8.Length;
xml = xml.Remove(0, lastIndexOfUtf8);
}
XElement root = XElement.Parse(xml);
if (root.Name.LocalName == "Project")
{
packages =
root
.Descendants()
.Where(x => x.Name.LocalName == "PackageReference" && x.Attribute("Include") != null && x.Attribute("Version") != null)
.SelectMany(r => GetDeveloperPackages(r.Attribute("Include").Value, r.Attribute("Version").Value))
.ToList();
IEnumerable <string> skipped_packages =
root
.Descendants()
.Where(x => x.Name.LocalName == "PackageReference" && x.Attribute("Include") != null && x.Attribute("Version") == null)
.Select(r => r.Attribute("Include").Value);
if (skipped_packages.Count() > 0)
{
this.AuditEnvironment.Warning("{0} package(s) do not have a version specified and will not be audited: {1}.", skipped_packages.Count(),
skipped_packages.Aggregate((s1, s2) => s1 + "," + s2));
}
var helper = new NuGetApiHelper(this.AuditEnvironment, config_file.DirectoryName);
foreach (var framework in helper.GetFrameworks(root))
{
AuditEnvironment.Info("Scanning NuGet transitive dependencies for {0}...", framework.GetFrameworkString());
var deps = helper.GetPackageDependencies(packages, framework);
Task.WaitAll(deps);
packages = helper.AddPackageDependencies(deps.Result, packages);
}
return(packages);
}
else
{
this.AuditEnvironment.Error("{0} is not a .NET Core format .csproj file.", config_file.FullName);
return(packages);
}
}
else if (config_file.Name.EndsWith(".deps.json"))
{
try
{
this.AuditEnvironment.Info("Reading packages from .NET Core depedencies manifest..");
JObject json = (JObject)JToken.Parse(config_file.ReadAsText());
JObject libraries = (JObject)json["libraries"];
if (libraries != null)
{
foreach (JProperty p in libraries.Properties())
{
string[] name = p.Name.Split('/');
packages.Add(new Package("nuget", name[0], name[1]));
}
}
return(packages);
}
catch (Exception e)
{
this.AuditEnvironment.Error(e, "Error reading .NET Core dependencies manifest {0}.", config_file.FullName);
return(packages);
}
}
else
{
this.AuditEnvironment.Error("Unknown .NET Core project file type: {0}.", config_file.FullName);
return(packages);
}
}
