public async static void AddVulnerabilitiesAsync(List <OSSIndexVulnerability> vulnerabilities)
{
int i = 0;
IEnumerable <IGrouping <int, OSSIndexVulnerability> > packages_vulnerabilities = vulnerabilities.GroupBy(x => i++ / 10).ToList();
OSSIndexHttpClient c = new OSSIndexHttpClient("1.1e", ProgramOptions.User, ProgramOptions.Password);
for (int g = 0; g < packages_vulnerabilities.Count(); g++)
{
IEnumerable <OSSIndexVulnerability> task_vulnerabilities = packages_vulnerabilities.Where(pv => pv.Key == g).SelectMany(s => s);
{
List <Tuple <OSSIndexVulnerability, Task <bool> > > tasks = task_vulnerabilities.Select(tv =>
new Tuple <OSSIndexVulnerability, Task <bool> >(tv,
Task <bool> .Run(async() => await c.AddVulnerabilityAsync(tv)))).ToList();
while (tasks.Count > 0)
{
Task <bool> completed_task = await Task.WhenAny(tasks.Select(t => t.Item2).ToArray());
bool r = await completed_task;
List <Tuple <OSSIndexVulnerability, Task <bool> > > completed = tasks.Where(t => t.Item2.IsCompleted).ToList();
List <Tuple <OSSIndexVulnerability, Task <bool> > > faulted = tasks.Where(t => t.Item2.IsFaulted).ToList();
List <Tuple <OSSIndexVulnerability, Task <bool> > > cancelled = tasks.Where(t => t.Item2.IsCanceled).ToList();
Database.PutVulnerabilities(completed.Select(cv => cv.Item1).ToList());
completed.ForEach(cv => logger.Info("Added vulnerability with id {0} for package {1} to OSS Index and local database cache.", cv.Item2.Id, cv.Item1.Name));
cancelled.ForEach(cv => logger.Info("The task to add vulnerability with id {0} to OSS Index did not complete successfully.", cv.Item2.Id));
faulted.ForEach(cv => logger.Info("The task to add vulnerability with id {0} to OSS Index did not complete successfully.", cv.Item2.Id));
tasks.RemoveAll(t => t.Item2.IsCompleted || t.Item2.IsFaulted || t.Item2.IsCanceled);
}
}
}
}
public override async Task CanGetVulnerabilities()
{
OSSIndexHttpClient http_client = new OSSIndexHttpClient("1.1");
List <OSSIndexProjectVulnerability> v1 = (await http_client.GetVulnerabilitiesForIdAsync("8396615975")).ToList();
Assert.NotNull(v1);
Assert.Equal(v1[0].Title, "[CVE-2006-4067] Cross-site scripting (XSS) vulnerability in cake/libs/error.php in CakePHP befor...");
}
public override async Task CanGetVulnerabilities()
{
OSSIndexHttpClient http_client = new OSSIndexHttpClient("1.0");
List <OSSIndexProjectVulnerability> v1 = (await http_client.GetVulnerabilitiesForIdAsync("284089289")).ToList();
Assert.NotNull(v1);
Assert.Equal(v1[1].Title, "CVE-2015-4670] Improper Limitation of a Pathname to a Restricted Directory");
}
public override async Task CanGetVulnerabilities()
{
OSSIndexHttpClient http_client = new OSSIndexHttpClient("1.1");
List <OSSIndexProjectVulnerability> v1 = (await http_client.GetVulnerabilitiesForIdAsync("8396559329")).ToList();
Assert.NotNull(v1);
Assert.True(v1.Any(v => v.Uri == "cve:/CVE-2011-4969"));
}
public override async Task CanGetVulnerabilities()
{
OSSIndexHttpClient http_client = new OSSIndexHttpClient("1.1");
List <OSSIndexProjectVulnerability> v1 = (await http_client.GetVulnerabilitiesForIdAsync("8396615975")).ToList();
Assert.NotNull(v1);
Assert.Equal(v1[0].Title, "[CVE-2006-4067] Improper Neutralization of Input During Web Page Generation (\"Cross-site Scripting\")");
}
public void CanAddVulnerabilityToOSSIndex()
{
List <DebianPackage> packages = DebianPackage.ParseDebianJsonFile(json_2);
List <OSSIndexVulnerability> vulns = packages.First().MapToOSSIndexVulnerabilities();
vulns.ForEach(v => v.Url += "vbot_unit_test" + v.Url + "_" + DateTime.UtcNow.Ticks.ToString());
OSSIndexHttpClient c = new OSSIndexHttpClient("1.1e", "*****@*****.**", "d8gh#beharry");
Assert.True(c.AddVulnerability(packages.Where(p => p.Name == "rsyslog").First().MapToOSSIndexVulnerabilities().First()));
}
public override async Task CanGetProjects()
{
OSSIndexHttpClient http_client = new OSSIndexHttpClient("1.1");
OSSIndexProject p1 = await http_client.GetProjectForIdAsync("8396831047");
Assert.NotNull(p1);
Assert.Equal(p1.Id, 8396831047);
Assert.Equal(p1.Name, "Modernizr");
Assert.Equal(p1.HasVulnerability, false);
}
public override async Task CanGetProjects()
{
OSSIndexHttpClient http_client = new OSSIndexHttpClient("1.1");
OSSIndexProject p1 = await http_client.GetProjectForIdAsync("8396615975");
Assert.NotNull(p1);
Assert.Equal(p1.Id, 8396615975);
Assert.Equal(p1.Name, "Cakephp");
Assert.Equal(p1.HasVulnerability, true);
Assert.Equal(p1.Vulnerabilities, "https://ossindex.net/v1.1/project/8396615975/vulnerabilities");
}
public override async Task CanGetProjects()
{
OSSIndexHttpClient http_client = new OSSIndexHttpClient("1.1");
OSSIndexProject p1 = await http_client.GetProjectForIdAsync("8396559329");
Assert.NotNull(p1);
Assert.Equal(p1.Id, 8396559329);
Assert.Equal(p1.Name, "JQuery");
Assert.Equal(p1.HasVulnerability, true);
Assert.Equal(p1.Vulnerabilities, "http://ossindex.net:8080/v1.1/project/8396559329/vulnerabilities");
}
public void CanAddVulnerabilities()
{
List <DebianPackage> packages = DebianPackage.ParseDebianJsonFile(json_1);
OSSIndexHttpClient c = new OSSIndexHttpClient("1.1e", "*****@*****.**", "d8gh#beharry");
List <Tuple <OSSIndexVulnerability, Task <bool> > > tasks = c.AddVulnerabilities(packages.First().MapToOSSIndexVulnerabilities());
while (tasks.Count > 0)
{
Task.WaitAny(tasks.Select(t => t.Item2).ToArray());
List <Tuple <OSSIndexVulnerability, Task <bool> > > completed = tasks.Where(t => t.Item2.IsCompleted).ToList();
tasks.RemoveAll(t => t.Item2.IsCompleted || t.Item2.IsFaulted || t.Item2.IsCanceled);
}
}
public override int Extract(int vulnerabilitiesLimit, Dictionary <string, string> options)
{
OSSIndexHttpClient client = null;
if (!string.IsNullOrEmpty(Authentication))
{
client = new OSSIndexHttpClient("2.0", User, Token);
}
else
{
client = new OSSIndexHttpClient("2.0");
}
List <ExtractedRecord> records = new List <ExtractedRecord>();
VulnerablityComparator vc = new VulnerablityComparator();
ExtractedRecordComparator erc = new ExtractedRecordComparator();
bool hasNext = false;
long from = 0, till = -1;
do
{
QueryResponse response = client.GetPackages(PackageManager, from, till).Result;
L.Information("Got {ps} package entries with {vuln} distinct vulnerability entries for package manager {pm}.",
response.packages.Select(p => p.Id).Distinct().Count(),
response.packages.SelectMany(p => p.Vulnerabilities).Distinct(vc).Count(), PackageManager);
hasNext = !string.IsNullOrEmpty(response.NextUrl);
var duplicates = response.packages.SelectMany(p => p.Vulnerabilities).GroupBy(v => v.Id)
.Where(g => g.Count() > 1)
.Select(y => y.Key)
.ToList();
L.Information("Got {0} duplicate vulnerabilities.", duplicates.Count);
foreach (Package package in response.packages)
{
records.AddRange(
package.Vulnerabilities.Select(v => new
ExtractedRecord
{
PackageManager = this.PackageManager,
PackageId = package.Id,
PackageName = package.Name,
VulnerabilityId = v.Id,
Title = FilterNonText(v.Title),
Description = FilterNonText(v.Description), //strip out any URLs
References = v.References,
Updated = v.Updated.HasValue ? v.Updated.Value : DateTime.MinValue,
Published = v.Published.HasValue ? v.Published.Value : DateTime.MinValue
})
);
}
hasNext = !string.IsNullOrEmpty(response.NextUrl);
if (hasNext)
{
Uri n = new Uri(response.NextUrl);
till = Int64.Parse(n.Segments[7]);
}
if (PackagesLimit > 0 && records.Distinct(erc).Select(r => r.PackageId).Distinct().Count() > PackagesLimit)
{
hasNext = false;
}
if (vulnerabilitiesLimit > 0 && records.Distinct(erc).Count() > vulnerabilitiesLimit)
{
hasNext = false;
}
}while (hasNext);
ExtractedRecords = records.Distinct(erc).Select(er => (Record)er).ToList();
L.Information("Extracted {packages} packages with {vulnd} distinct vulnerabilities.", records.Distinct(erc).Select(r => r.PackageId).Distinct().Count(),
ExtractedRecords.Count());
return(ExtractedRecords.Count);
}
static int Main(string[] args)
{
if (!CommandLine.Parser.Default.ParseArguments(args, ProgramOptions))
{
return((int)ProgramExitStatus.InvalidArguments);
}
else
{
if (!string.IsNullOrEmpty(ProgramOptions.LocalFile))
{
if (!File.Exists(ProgramOptions.LocalFile))
{
logger.Info("The local file {0} does not exist, exiting.", ProgramOptions.LocalFile);
return((int)ProgramExitStatus.InvalidArguments);
}
}
if ((!ProgramOptions.DumpDatabase && !ProgramOptions.ExtractPrint) &&
((string.IsNullOrEmpty(ProgramOptions.User) || (string.IsNullOrEmpty(ProgramOptions.Password)))))
{
logger.Info("The user and password options must be specified.");
return((int)ProgramExitStatus.InvalidArguments);
}
}
Config = core.Configuration.ReadConfiguration();
if (Config == null)
{
logger.Info("No configuration present, creating with initial values.");
Config = new Dictionary <string, string>()
{
{ "LastRun", "" },
{ "LastHash", "" }
};
core.Configuration.WriteConfiguration(Config);
}
Configure();
if (ProgramOptions.DumpDatabase)
{
Database.PrintAllVulnerabilities();
return((int)ProgramExitStatus.Success);
}
FileInfo f = null;
if (string.IsNullOrEmpty(ProgramOptions.LocalFile))
{
if (!Directory.Exists("work"))
{
Directory.CreateDirectory("work");
}
f = new FileInfo(Path.Combine("work", DateTime.UtcNow.Ticks.ToString()));
int percentage_completed = 0;
long bytes_received = 0;
DownloadProgressChangedEventHandler d = delegate(object sender, DownloadProgressChangedEventArgs e)
{
if (e.ProgressPercentage > 0 && (e.ProgressPercentage - percentage_completed > 10))
{
logger.Debug("Received {0} KB, {1} percentage completed.", e.BytesReceived / 1024, e.ProgressPercentage);
percentage_completed = e.ProgressPercentage;
}
else
{
if ((e.BytesReceived - bytes_received) > 1024 * 1024)
{
bytes_received = e.BytesReceived;
logger.Debug("No progress percentage available, received {0} bytes.", bytes_received);
}
}
};
VBotHttpFileDownload vhfd = new VBotHttpFileDownload(security_tracker_json_url, f, d);
logger.Debug("Downloading {0} to {1}...", vhfd.url.ToString(), f.Name);
vhfd.StartTask().Wait();
if (!vhfd.CompletedSuccessfully)
{
logger.Info("The download of {0} did not complete successfully.");
if (vhfd.Error != null)
{
logger.Error(vhfd.Error);
}
vhfd = null;
logger.Info("Nothing to do exiting.");
return((int)ProgramExitStatus.DownloadFailed);
}
vhfd = null;
string hash = Cryptography.ComputeFileSHA1Hash(f);
logger.Debug("Downloaded file SHA1 hash: {0}.", hash);
if (hash == LastHash)
{
logger.Info("File hash is the same as previous run: {0}. Nothing to do, exiting.", Program.LastHash);
return((int)ProgramExitStatus.Success);
}
else
{
Config["LastHash"] = hash;
Program.LastHash = hash;
Configuration.WriteConfiguration(Config);
}
}
else
{
f = new FileInfo(ProgramOptions.LocalFile);
}
List <DebianPackage> packages = DebianPackage.ParseDebianJsonFile(f);
List <OSSIndexVulnerability> vulnerabilities = packages.SelectMany(p => p.MapToOSSIndexVulnerabilities()).ToList();
logger.Info("{0} total vulnerabilities extracted.", vulnerabilities.Count);
if (!string.IsNullOrEmpty(ProgramOptions.PackageName))
{
vulnerabilities = vulnerabilities.Where(v => v.Name == ProgramOptions.PackageName).ToList();
logger.Info("Found {0} new or updated vulnerabilities for package {1}.", vulnerabilities.Count, ProgramOptions.PackageName);
}
if (ProgramOptions.ExtractPrint)
{
foreach (OSSIndexVulnerability v in vulnerabilities)
{
logger.Info("\nId: {0}\nPackage:{1}\nVersion: {2}\nUrl:{3}\n", v.Vid, v.Name, v.Version, v.Url);
}
return((int)ProgramExitStatus.Success);
}
List <OSSIndexVulnerability> cached_vulnerabilities = new List <OSSIndexVulnerability>();
foreach (OSSIndexVulnerability v in vulnerabilities)
{
OSSIndexVulnerability cached_v = null;
if (Database.GetVulnerability(string.IsNullOrEmpty(v.Vid) ? v.Url : v.Vid + "#" + v.Group, out cached_v))
{
if (v.EqualValues(cached_v))
{
cached_vulnerabilities.Add(v);
}
}
}
vulnerabilities = vulnerabilities.Except(cached_vulnerabilities).ToList();
logger.Info("{0} vulnerabilities are cached and have already been submitted to the OSS Index server.", cached_vulnerabilities.Count);
OSSIndexHttpClient client = new OSSIndexHttpClient("1.1e", ProgramOptions.User, ProgramOptions.Password);
OSSIndexHttpClient c = new OSSIndexHttpClient("1.1e", ProgramOptions.User, ProgramOptions.Password);
int i = 0;
IEnumerable <IGrouping <int, OSSIndexVulnerability> > packages_vulnerabilities = vulnerabilities.GroupBy(x => i++ / 10).ToList();
for (int g = 0; g < packages_vulnerabilities.Count(); g++)
{
List <Tuple <OSSIndexVulnerability, Task <bool> > > tasks = c.AddVulnerabilities(packages_vulnerabilities.Where(pv => pv.Key == g).SelectMany(s => s).ToList());
while (tasks.Count > 0)
{
Task.WaitAny(tasks.Select(t => t.Item2).ToArray());
List <Tuple <OSSIndexVulnerability, Task <bool> > > completed = tasks.Where(t => t.Item2.IsCompleted).ToList();
List <Tuple <OSSIndexVulnerability, Task <bool> > > faulted = tasks.Where(t => t.Item2.IsFaulted).ToList();
List <Tuple <OSSIndexVulnerability, Task <bool> > > cancelled = tasks.Where(t => t.Item2.IsCanceled).ToList();
Database.PutVulnerabilities(completed.Select(cv => cv.Item1).ToList());
completed.ForEach(cv => logger.Info("Added vulnerability with id {0} for package {1} to OSS Index and local database cache.", string.IsNullOrEmpty(cv.Item1.Vid) ? cv.Item1.Url : cv.Item1.Vid, cv.Item1.Name));
cancelled.ForEach(cv => logger.Info("The task to add vulnerability with id {0} for package {1} to OSS Index and local database cache.", string.IsNullOrEmpty(cv.Item1.Vid) ? cv.Item1.Url : cv.Item1.Vid, cv.Item1.Name));
faulted.ForEach(cv => logger.Info("The task to add vulnerability with id {0} for package {1} to OSS Index and local database cache.", string.IsNullOrEmpty(cv.Item1.Vid) ? cv.Item1.Url : cv.Item1.Vid, cv.Item1.Name));
tasks.RemoveAll(t => completed.Contains(t) || cancelled.Contains(t) || faulted.Contains(t));
}
}
return((int)ProgramExitStatus.Success);
}
