public void CanParseDebianJson()
{
List <DebianPackage> packages_1 = DebianPackage.ParseDebianJsonFile(json_1);
Assert.NotEmpty(packages_1);
Assert.Equal(packages_1.Count, 1);
Assert.Equal(packages_1.First().Name, "prototypejs");
}
public void CanAddVulnerabilityToOSSIndex()
{
List <DebianPackage> packages = DebianPackage.ParseDebianJsonFile(json_2);
List <OSSIndexVulnerability> vulns = packages.First().MapToOSSIndexVulnerabilities();
vulns.ForEach(v => v.Url += "vbot_unit_test" + v.Url + "_" + DateTime.UtcNow.Ticks.ToString());
OSSIndexHttpClient c = new OSSIndexHttpClient("1.1e", "*****@*****.**", "d8gh#beharry");
Assert.True(c.AddVulnerability(packages.Where(p => p.Name == "rsyslog").First().MapToOSSIndexVulnerabilities().First()));
}
public void CanPutPackageVulnerabilities()
{
List <DebianPackage> packages_2 = DebianPackage.ParseDebianJsonFile(json_2);
List <OSSIndexVulnerability> vulns = packages_2.First().MapToOSSIndexVulnerabilities();
vulns.ForEach(v => v.Url += "_" + DateTime.UtcNow.Ticks.ToString());
Assert.True(Database.PutVulnerabilities(vulns));
OSSIndexVulnerability o;
vulns.ForEach(v => Assert.True(Database.GetVulnerability(v.Url, out o)));
}
public void CanAddVulnerabilities()
{
List <DebianPackage> packages = DebianPackage.ParseDebianJsonFile(json_1);
OSSIndexHttpClient c = new OSSIndexHttpClient("1.1e", "*****@*****.**", "d8gh#beharry");
List <Tuple <OSSIndexVulnerability, Task <bool> > > tasks = c.AddVulnerabilities(packages.First().MapToOSSIndexVulnerabilities());
while (tasks.Count > 0)
{
Task.WaitAny(tasks.Select(t => t.Item2).ToArray());
List <Tuple <OSSIndexVulnerability, Task <bool> > > completed = tasks.Where(t => t.Item2.IsCompleted).ToList();
tasks.RemoveAll(t => t.Item2.IsCompleted || t.Item2.IsFaulted || t.Item2.IsCanceled);
}
}
static int Main(string[] args)
{
if (!CommandLine.Parser.Default.ParseArguments(args, ProgramOptions))
{
return((int)ProgramExitStatus.InvalidArguments);
}
else
{
if (!string.IsNullOrEmpty(ProgramOptions.LocalFile))
{
if (!File.Exists(ProgramOptions.LocalFile))
{
logger.Info("The local file {0} does not exist, exiting.", ProgramOptions.LocalFile);
return((int)ProgramExitStatus.InvalidArguments);
}
}
if ((!ProgramOptions.DumpDatabase && !ProgramOptions.ExtractPrint) &&
((string.IsNullOrEmpty(ProgramOptions.User) || (string.IsNullOrEmpty(ProgramOptions.Password)))))
{
logger.Info("The user and password options must be specified.");
return((int)ProgramExitStatus.InvalidArguments);
}
}
Config = core.Configuration.ReadConfiguration();
if (Config == null)
{
logger.Info("No configuration present, creating with initial values.");
Config = new Dictionary <string, string>()
{
{ "LastRun", "" },
{ "LastHash", "" }
};
core.Configuration.WriteConfiguration(Config);
}
Configure();
if (ProgramOptions.DumpDatabase)
{
Database.PrintAllVulnerabilities();
return((int)ProgramExitStatus.Success);
}
FileInfo f = null;
if (string.IsNullOrEmpty(ProgramOptions.LocalFile))
{
if (!Directory.Exists("work"))
{
Directory.CreateDirectory("work");
}
f = new FileInfo(Path.Combine("work", DateTime.UtcNow.Ticks.ToString()));
int percentage_completed = 0;
long bytes_received = 0;
DownloadProgressChangedEventHandler d = delegate(object sender, DownloadProgressChangedEventArgs e)
{
if (e.ProgressPercentage > 0 && (e.ProgressPercentage - percentage_completed > 10))
{
logger.Debug("Received {0} KB, {1} percentage completed.", e.BytesReceived / 1024, e.ProgressPercentage);
percentage_completed = e.ProgressPercentage;
}
else
{
if ((e.BytesReceived - bytes_received) > 1024 * 1024)
{
bytes_received = e.BytesReceived;
logger.Debug("No progress percentage available, received {0} bytes.", bytes_received);
}
}
};
VBotHttpFileDownload vhfd = new VBotHttpFileDownload(security_tracker_json_url, f, d);
logger.Debug("Downloading {0} to {1}...", vhfd.url.ToString(), f.Name);
vhfd.StartTask().Wait();
if (!vhfd.CompletedSuccessfully)
{
logger.Info("The download of {0} did not complete successfully.");
if (vhfd.Error != null)
{
logger.Error(vhfd.Error);
}
vhfd = null;
logger.Info("Nothing to do exiting.");
return((int)ProgramExitStatus.DownloadFailed);
}
vhfd = null;
string hash = Cryptography.ComputeFileSHA1Hash(f);
logger.Debug("Downloaded file SHA1 hash: {0}.", hash);
if (hash == LastHash)
{
logger.Info("File hash is the same as previous run: {0}. Nothing to do, exiting.", Program.LastHash);
return((int)ProgramExitStatus.Success);
}
else
{
Config["LastHash"] = hash;
Program.LastHash = hash;
Configuration.WriteConfiguration(Config);
}
}
else
{
f = new FileInfo(ProgramOptions.LocalFile);
}
List <DebianPackage> packages = DebianPackage.ParseDebianJsonFile(f);
List <OSSIndexVulnerability> vulnerabilities = packages.SelectMany(p => p.MapToOSSIndexVulnerabilities()).ToList();
logger.Info("{0} total vulnerabilities extracted.", vulnerabilities.Count);
if (!string.IsNullOrEmpty(ProgramOptions.PackageName))
{
vulnerabilities = vulnerabilities.Where(v => v.Name == ProgramOptions.PackageName).ToList();
logger.Info("Found {0} new or updated vulnerabilities for package {1}.", vulnerabilities.Count, ProgramOptions.PackageName);
}
if (ProgramOptions.ExtractPrint)
{
foreach (OSSIndexVulnerability v in vulnerabilities)
{
logger.Info("\nId: {0}\nPackage:{1}\nVersion: {2}\nUrl:{3}\n", v.Vid, v.Name, v.Version, v.Url);
}
return((int)ProgramExitStatus.Success);
}
List <OSSIndexVulnerability> cached_vulnerabilities = new List <OSSIndexVulnerability>();
foreach (OSSIndexVulnerability v in vulnerabilities)
{
OSSIndexVulnerability cached_v = null;
if (Database.GetVulnerability(string.IsNullOrEmpty(v.Vid) ? v.Url : v.Vid + "#" + v.Group, out cached_v))
{
if (v.EqualValues(cached_v))
{
cached_vulnerabilities.Add(v);
}
}
}
vulnerabilities = vulnerabilities.Except(cached_vulnerabilities).ToList();
logger.Info("{0} vulnerabilities are cached and have already been submitted to the OSS Index server.", cached_vulnerabilities.Count);
OSSIndexHttpClient client = new OSSIndexHttpClient("1.1e", ProgramOptions.User, ProgramOptions.Password);
OSSIndexHttpClient c = new OSSIndexHttpClient("1.1e", ProgramOptions.User, ProgramOptions.Password);
int i = 0;
IEnumerable <IGrouping <int, OSSIndexVulnerability> > packages_vulnerabilities = vulnerabilities.GroupBy(x => i++ / 10).ToList();
for (int g = 0; g < packages_vulnerabilities.Count(); g++)
{
List <Tuple <OSSIndexVulnerability, Task <bool> > > tasks = c.AddVulnerabilities(packages_vulnerabilities.Where(pv => pv.Key == g).SelectMany(s => s).ToList());
while (tasks.Count > 0)
{
Task.WaitAny(tasks.Select(t => t.Item2).ToArray());
List <Tuple <OSSIndexVulnerability, Task <bool> > > completed = tasks.Where(t => t.Item2.IsCompleted).ToList();
List <Tuple <OSSIndexVulnerability, Task <bool> > > faulted = tasks.Where(t => t.Item2.IsFaulted).ToList();
List <Tuple <OSSIndexVulnerability, Task <bool> > > cancelled = tasks.Where(t => t.Item2.IsCanceled).ToList();
Database.PutVulnerabilities(completed.Select(cv => cv.Item1).ToList());
completed.ForEach(cv => logger.Info("Added vulnerability with id {0} for package {1} to OSS Index and local database cache.", string.IsNullOrEmpty(cv.Item1.Vid) ? cv.Item1.Url : cv.Item1.Vid, cv.Item1.Name));
cancelled.ForEach(cv => logger.Info("The task to add vulnerability with id {0} for package {1} to OSS Index and local database cache.", string.IsNullOrEmpty(cv.Item1.Vid) ? cv.Item1.Url : cv.Item1.Vid, cv.Item1.Name));
faulted.ForEach(cv => logger.Info("The task to add vulnerability with id {0} for package {1} to OSS Index and local database cache.", string.IsNullOrEmpty(cv.Item1.Vid) ? cv.Item1.Url : cv.Item1.Vid, cv.Item1.Name));
tasks.RemoveAll(t => completed.Contains(t) || cancelled.Contains(t) || faulted.Contains(t));
}
}
return((int)ProgramExitStatus.Success);
}
