public void CanParseDebianJson() { List <DebianPackage> packages_1 = DebianPackage.ParseDebianJsonFile(json_1); Assert.NotEmpty(packages_1); Assert.Equal(packages_1.Count, 1); Assert.Equal(packages_1.First().Name, "prototypejs"); }
public void CanAddVulnerabilityToOSSIndex() { List <DebianPackage> packages = DebianPackage.ParseDebianJsonFile(json_2); List <OSSIndexVulnerability> vulns = packages.First().MapToOSSIndexVulnerabilities(); vulns.ForEach(v => v.Url += "vbot_unit_test" + v.Url + "_" + DateTime.UtcNow.Ticks.ToString()); OSSIndexHttpClient c = new OSSIndexHttpClient("1.1e", "*****@*****.**", "d8gh#beharry"); Assert.True(c.AddVulnerability(packages.Where(p => p.Name == "rsyslog").First().MapToOSSIndexVulnerabilities().First())); }
public void CanPutPackageVulnerabilities() { List <DebianPackage> packages_2 = DebianPackage.ParseDebianJsonFile(json_2); List <OSSIndexVulnerability> vulns = packages_2.First().MapToOSSIndexVulnerabilities(); vulns.ForEach(v => v.Url += "_" + DateTime.UtcNow.Ticks.ToString()); Assert.True(Database.PutVulnerabilities(vulns)); OSSIndexVulnerability o; vulns.ForEach(v => Assert.True(Database.GetVulnerability(v.Url, out o))); }
public void CanAddVulnerabilities() { List <DebianPackage> packages = DebianPackage.ParseDebianJsonFile(json_1); OSSIndexHttpClient c = new OSSIndexHttpClient("1.1e", "*****@*****.**", "d8gh#beharry"); List <Tuple <OSSIndexVulnerability, Task <bool> > > tasks = c.AddVulnerabilities(packages.First().MapToOSSIndexVulnerabilities()); while (tasks.Count > 0) { Task.WaitAny(tasks.Select(t => t.Item2).ToArray()); List <Tuple <OSSIndexVulnerability, Task <bool> > > completed = tasks.Where(t => t.Item2.IsCompleted).ToList(); tasks.RemoveAll(t => t.Item2.IsCompleted || t.Item2.IsFaulted || t.Item2.IsCanceled); } }
static int Main(string[] args) { if (!CommandLine.Parser.Default.ParseArguments(args, ProgramOptions)) { return((int)ProgramExitStatus.InvalidArguments); } else { if (!string.IsNullOrEmpty(ProgramOptions.LocalFile)) { if (!File.Exists(ProgramOptions.LocalFile)) { logger.Info("The local file {0} does not exist, exiting.", ProgramOptions.LocalFile); return((int)ProgramExitStatus.InvalidArguments); } } if ((!ProgramOptions.DumpDatabase && !ProgramOptions.ExtractPrint) && ((string.IsNullOrEmpty(ProgramOptions.User) || (string.IsNullOrEmpty(ProgramOptions.Password))))) { logger.Info("The user and password options must be specified."); return((int)ProgramExitStatus.InvalidArguments); } } Config = core.Configuration.ReadConfiguration(); if (Config == null) { logger.Info("No configuration present, creating with initial values."); Config = new Dictionary <string, string>() { { "LastRun", "" }, { "LastHash", "" } }; core.Configuration.WriteConfiguration(Config); } Configure(); if (ProgramOptions.DumpDatabase) { Database.PrintAllVulnerabilities(); return((int)ProgramExitStatus.Success); } FileInfo f = null; if (string.IsNullOrEmpty(ProgramOptions.LocalFile)) { if (!Directory.Exists("work")) { Directory.CreateDirectory("work"); } f = new FileInfo(Path.Combine("work", DateTime.UtcNow.Ticks.ToString())); int percentage_completed = 0; long bytes_received = 0; DownloadProgressChangedEventHandler d = delegate(object sender, DownloadProgressChangedEventArgs e) { if (e.ProgressPercentage > 0 && (e.ProgressPercentage - percentage_completed > 10)) { logger.Debug("Received {0} KB, {1} percentage completed.", e.BytesReceived / 1024, e.ProgressPercentage); percentage_completed = e.ProgressPercentage; } else { if ((e.BytesReceived - bytes_received) > 1024 * 1024) { bytes_received = e.BytesReceived; logger.Debug("No progress percentage available, received {0} bytes.", bytes_received); } } }; VBotHttpFileDownload vhfd = new VBotHttpFileDownload(security_tracker_json_url, f, d); logger.Debug("Downloading {0} to {1}...", vhfd.url.ToString(), f.Name); vhfd.StartTask().Wait(); if (!vhfd.CompletedSuccessfully) { logger.Info("The download of {0} did not complete successfully."); if (vhfd.Error != null) { logger.Error(vhfd.Error); } vhfd = null; logger.Info("Nothing to do exiting."); return((int)ProgramExitStatus.DownloadFailed); } vhfd = null; string hash = Cryptography.ComputeFileSHA1Hash(f); logger.Debug("Downloaded file SHA1 hash: {0}.", hash); if (hash == LastHash) { logger.Info("File hash is the same as previous run: {0}. Nothing to do, exiting.", Program.LastHash); return((int)ProgramExitStatus.Success); } else { Config["LastHash"] = hash; Program.LastHash = hash; Configuration.WriteConfiguration(Config); } } else { f = new FileInfo(ProgramOptions.LocalFile); } List <DebianPackage> packages = DebianPackage.ParseDebianJsonFile(f); List <OSSIndexVulnerability> vulnerabilities = packages.SelectMany(p => p.MapToOSSIndexVulnerabilities()).ToList(); logger.Info("{0} total vulnerabilities extracted.", vulnerabilities.Count); if (!string.IsNullOrEmpty(ProgramOptions.PackageName)) { vulnerabilities = vulnerabilities.Where(v => v.Name == ProgramOptions.PackageName).ToList(); logger.Info("Found {0} new or updated vulnerabilities for package {1}.", vulnerabilities.Count, ProgramOptions.PackageName); } if (ProgramOptions.ExtractPrint) { foreach (OSSIndexVulnerability v in vulnerabilities) { logger.Info("\nId: {0}\nPackage:{1}\nVersion: {2}\nUrl:{3}\n", v.Vid, v.Name, v.Version, v.Url); } return((int)ProgramExitStatus.Success); } List <OSSIndexVulnerability> cached_vulnerabilities = new List <OSSIndexVulnerability>(); foreach (OSSIndexVulnerability v in vulnerabilities) { OSSIndexVulnerability cached_v = null; if (Database.GetVulnerability(string.IsNullOrEmpty(v.Vid) ? v.Url : v.Vid + "#" + v.Group, out cached_v)) { if (v.EqualValues(cached_v)) { cached_vulnerabilities.Add(v); } } } vulnerabilities = vulnerabilities.Except(cached_vulnerabilities).ToList(); logger.Info("{0} vulnerabilities are cached and have already been submitted to the OSS Index server.", cached_vulnerabilities.Count); OSSIndexHttpClient client = new OSSIndexHttpClient("1.1e", ProgramOptions.User, ProgramOptions.Password); OSSIndexHttpClient c = new OSSIndexHttpClient("1.1e", ProgramOptions.User, ProgramOptions.Password); int i = 0; IEnumerable <IGrouping <int, OSSIndexVulnerability> > packages_vulnerabilities = vulnerabilities.GroupBy(x => i++ / 10).ToList(); for (int g = 0; g < packages_vulnerabilities.Count(); g++) { List <Tuple <OSSIndexVulnerability, Task <bool> > > tasks = c.AddVulnerabilities(packages_vulnerabilities.Where(pv => pv.Key == g).SelectMany(s => s).ToList()); while (tasks.Count > 0) { Task.WaitAny(tasks.Select(t => t.Item2).ToArray()); List <Tuple <OSSIndexVulnerability, Task <bool> > > completed = tasks.Where(t => t.Item2.IsCompleted).ToList(); List <Tuple <OSSIndexVulnerability, Task <bool> > > faulted = tasks.Where(t => t.Item2.IsFaulted).ToList(); List <Tuple <OSSIndexVulnerability, Task <bool> > > cancelled = tasks.Where(t => t.Item2.IsCanceled).ToList(); Database.PutVulnerabilities(completed.Select(cv => cv.Item1).ToList()); completed.ForEach(cv => logger.Info("Added vulnerability with id {0} for package {1} to OSS Index and local database cache.", string.IsNullOrEmpty(cv.Item1.Vid) ? cv.Item1.Url : cv.Item1.Vid, cv.Item1.Name)); cancelled.ForEach(cv => logger.Info("The task to add vulnerability with id {0} for package {1} to OSS Index and local database cache.", string.IsNullOrEmpty(cv.Item1.Vid) ? cv.Item1.Url : cv.Item1.Vid, cv.Item1.Name)); faulted.ForEach(cv => logger.Info("The task to add vulnerability with id {0} for package {1} to OSS Index and local database cache.", string.IsNullOrEmpty(cv.Item1.Vid) ? cv.Item1.Url : cv.Item1.Vid, cv.Item1.Name)); tasks.RemoveAll(t => completed.Contains(t) || cancelled.Contains(t) || faulted.Contains(t)); } } return((int)ProgramExitStatus.Success); }