public async Task ProcessAuditHappyPath() { // Arrange await using var context = JosekiTestsDb.CreateUniqueContext(); var parser = new ConfigurationParser("config.sample.yaml"); var checksCache = new ChecksCache(parser, context, new MemoryCache(new MemoryCacheOptions())); var blobsMock = new Mock <IBlobStorageProcessor>(MockBehavior.Strict); var dbMock = new Mock <IJosekiDatabase>(); var queueMock = new Mock <IQueue>(); var ownershipCache = new OwnershipCache(context, new MemoryCache(new MemoryCacheOptions())); var postProcessor = new Mock <ExtractOwnershipProcessor>(context, ownershipCache); var processor = new PolarisAuditProcessor(blobsMock.Object, dbMock.Object, checksCache, queueMock.Object, postProcessor.Object); var container = new ScannerContainer(Path.Combine("audits", "samples", "polaris")) { Metadata = new ScannerMetadata { Type = ScannerType.Polaris, Id = Guid.NewGuid().ToString(), }, }; var audit = new AuditBlob { Name = "meta.json", ParentContainer = container }; blobsMock .Setup(i => i.GetUnprocessedAudits(container)) .ReturnsAsync(new[] { audit }) .Verifiable(); blobsMock .Setup(i => i.DownloadFile($"{container.Name}/{audit.Name}")) .ReturnsAsync(File.OpenRead($"{container.Name}/{audit.Name}")) .Verifiable(); blobsMock .Setup(i => i.DownloadFile($"{container.Name}/audit.json")) .ReturnsAsync(File.OpenRead($"{container.Name}/audit.json")) .Verifiable(); blobsMock .Setup(i => i.DownloadFile($"{container.Name}/k8s-meta.json")) .ReturnsAsync(File.OpenRead($"{container.Name}/k8s-meta.json")) .Verifiable(); blobsMock .Setup(i => i.MarkAsProcessed(audit)) .Returns(Task.CompletedTask) .Verifiable(); // Act & Assert await processor.Process(container, CancellationToken.None); blobsMock.Verify(); dbMock.Verify(i => i.GetNotExpiredImageScans(It.Is <string[]>(tags => tags.Length == UniqueImageTagCount))); dbMock.Verify(i => i.SaveInProgressImageScan(It.IsAny <ImageScanResultWithCVEs>()), Times.Exactly(UniqueImageTagCount)); queueMock.Verify(i => i.EnqueueImageScanRequest(It.IsAny <ImageScanResultWithCVEs>()), Times.Exactly(UniqueImageTagCount)); dbMock.Verify(i => i.SaveAuditResult(It.Is <Audit>(a => VerifyHappyPathAudit(a, container)))); }
private async Task ProcessSingleAudit(AuditBlob auditBlob) { var path = $"{auditBlob.ParentContainer.Name}/{auditBlob.Name}"; var stream = await this.blobStorage.DownloadFile(path); using var sr = new StreamReader(stream); var metadataString = sr.ReadToEnd(); var auditMetadata = JsonConvert.DeserializeObject <AuditMetadata>(metadataString); if (auditMetadata.AuditResult != "succeeded") { Logger.Warning( "Audit {AuditPath} result is {AuditResult} due: {FailureReason}", path, auditMetadata.AuditResult, auditMetadata.FailureDescription); } else { try { var tasks = auditMetadata .AzSkAuditPaths .Where(i => i.EndsWith(".json")) .Select(i => this.blobStorage.DownloadFile($"{auditBlob.ParentContainer.Name}/{i}")) .Select(this.ToJArray) .ToArray(); await Task.WhenAll(tasks); var audit = await this.NormalizeRawData(tasks.Select(i => i.Result).ToArray(), auditBlob, auditMetadata); // TODO: Counter functions could be converted into single iterator Logger.Information( "Successfully processed audit {AuditId} of {AuditDate} with {CheckResultCount} check results, where succeeded {Succeeded}; failed {Failed}; no-data {NoData}", audit.Id, audit.Date, audit.CheckResults.Count, audit.CheckResults.Count(i => i.Value == CheckValue.Succeeded), audit.CheckResults.Count(i => i.Value == CheckValue.Failed), audit.CheckResults.Count(i => i.Value == CheckValue.NoData)); await this.db.SaveAuditResult(audit); // TODO: use a cancellation token inside this process? await this.extractOwnershipProcessor.Process(audit, CancellationToken.None); } catch (Exception ex) { Logger.Error(ex, "Failed to process audit {AuditPath}", path); } } await this.blobStorage.MarkAsProcessed(auditBlob); }
public async Task ProcessAuditHappyPath() { // Arrange await using var context = JosekiTestsDb.CreateUniqueContext(); var parser = new ConfigurationParser("config.sample.yaml"); var checksCache = new ChecksCache(parser, context, new MemoryCache(new MemoryCacheOptions())); var blobsMock = new Mock <IBlobStorageProcessor>(MockBehavior.Strict); var dbMock = new Mock <IJosekiDatabase>(); var ownershipCache = new OwnershipCache(context, new MemoryCache(new MemoryCacheOptions())); var postProcessor = new Mock <ExtractOwnershipProcessor>(context, ownershipCache); var processor = new AzskAuditProcessor(blobsMock.Object, dbMock.Object, checksCache, postProcessor.Object); var container = new ScannerContainer(Path.Combine("audits", "samples", "azsk")) { Metadata = new ScannerMetadata { Type = ScannerType.Azsk, Id = Guid.NewGuid().ToString(), }, }; var audit = new AuditBlob { Name = "meta.json", ParentContainer = container }; blobsMock .Setup(i => i.GetUnprocessedAudits(container)) .ReturnsAsync(new[] { audit }) .Verifiable(); blobsMock .Setup(i => i.DownloadFile($"{container.Name}/{audit.Name}")) .ReturnsAsync(File.OpenRead($"{container.Name}/{audit.Name}")) .Verifiable(); blobsMock .Setup(i => i.DownloadFile($"{container.Name}/resources.json")) .ReturnsAsync(File.OpenRead($"{container.Name}/resources.json")) .Verifiable(); blobsMock .Setup(i => i.DownloadFile($"{container.Name}/subscription.json")) .ReturnsAsync(File.OpenRead($"{container.Name}/subscription.json")) .Verifiable(); blobsMock .Setup(i => i.MarkAsProcessed(audit)) .Returns(Task.CompletedTask) .Verifiable(); // Act & Assert await processor.Process(container, CancellationToken.None); blobsMock.Verify(); dbMock.Verify(i => i.SaveAuditResult(It.Is <Audit>(a => VerifyHappyPathAudit(a, container)))); }
private async Task <Audit> NormalizeRawData(AuditBlob auditBlob, AuditMetadata auditMetadata) { var clusterId = string.IsNullOrEmpty(auditMetadata.ClusterId) ? auditBlob.ParentContainer.Metadata.Id : auditMetadata.ClusterId; var auditJson = await this.GetJsonObject($"{auditBlob.ParentContainer.Name}/{auditMetadata.PolarisAuditPaths}"); var checks = await this.ParseChecksResults(auditJson, auditMetadata, clusterId); var k8sJson = await this.GetJsonObject($"{auditBlob.ParentContainer.Name}/{auditMetadata.KubeMetadataPaths}"); try { var imageScanCheckResults = await this.EnrichAuditWithImageScans(auditMetadata.AuditId, k8sJson, clusterId); checks.AddRange(imageScanCheckResults); } catch (Exception ex) { // Failure here should not impact the rest of audit Logger.Warning(ex, "Failed to enrich audit {AuditId} Check Results with image scans", auditMetadata.AuditId); } var k8sMetadata = new JObject { { "scanner", JToken.FromObject(auditBlob.ParentContainer.Metadata) }, { "audit", JToken.FromObject(auditMetadata) }, { "cluster", k8sJson }, }; var auditDate = DateTimeOffset.FromUnixTimeSeconds(auditMetadata.Timestamp).DateTime; var infraComponentId = $"/k8s/{auditMetadata.ClusterId}"; var k8sName = this.ParseClusterName(k8sJson) ?? infraComponentId; var audit = new Audit { Id = auditMetadata.AuditId, Date = auditDate, ScannerId = $"{auditBlob.ParentContainer.Metadata.Type}/{auditBlob.ParentContainer.Metadata.Id}", ComponentId = infraComponentId, ComponentName = k8sName, CheckResults = checks, MetadataKube = new MetadataKube { AuditId = auditMetadata.AuditId, Date = auditDate, JSON = k8sMetadata.ToString(Formatting.None), }, }; return(audit); }
public async Task ProcessScanResultWithoutCVEs() { // Arrange await using var context = JosekiTestsDb.CreateUniqueContext(); var parser = new ConfigurationParser("config.sample.yaml"); var cveCache = new CveCache(parser, context, new MemoryCache(new MemoryCacheOptions())); var blobsMock = new Mock <IBlobStorageProcessor>(MockBehavior.Strict); var dbMock = new Mock <IJosekiDatabase>(); var processor = new TrivyAuditProcessor(blobsMock.Object, dbMock.Object, cveCache); var container = new ScannerContainer(Path.Combine("audits", "samples", "trivy")) { Metadata = new ScannerMetadata { Type = ScannerType.Trivy, Id = Guid.NewGuid().ToString(), }, }; var audit = new AuditBlob { Name = "meta_no_cve.json", ParentContainer = container }; blobsMock .Setup(i => i.GetUnprocessedAudits(container)) .ReturnsAsync(new[] { audit }) .Verifiable(); blobsMock .Setup(i => i.DownloadFile($"{container.Name}/{audit.Name}")) .ReturnsAsync(File.OpenRead($"{container.Name}/{audit.Name}")) .Verifiable(); blobsMock .Setup(i => i.DownloadFile($"{container.Name}/result_no_cves.json")) .ReturnsAsync(File.OpenRead($"{container.Name}/result_no_cves.json")) .Verifiable(); blobsMock .Setup(i => i.MarkAsProcessed(audit)) .Returns(Task.CompletedTask) .Verifiable(); // Act & Assert await processor.Process(container, CancellationToken.None); blobsMock.Verify(); dbMock.Verify(i => i.SaveImageScanResult(It.Is <ImageScanResultWithCVEs>(a => VerifyNoCveScan(a)))); }
private async Task <Audit> NormalizeRawData(JArray[] rawData, AuditBlob auditBlob, AuditMetadata auditMetadata) { var(scope, id, name) = this.GetSubscriptionMeta(rawData); var(resources, checks) = await this.ParseResourcesAndChecks(rawData, auditMetadata); var azMetadata = new JObject { { "scanner", JToken.FromObject(auditBlob.ParentContainer.Metadata) }, { "audit", JToken.FromObject(auditMetadata) }, { "subscription", new JObject { { "scope", scope }, { "id", id }, { "name", name }, { "resources", resources }, } }, }; var auditDate = DateTimeOffset.FromUnixTimeSeconds(auditMetadata.Timestamp).DateTime; var audit = new Audit { Id = auditMetadata.AuditId, Date = auditDate, ScannerId = $"{auditBlob.ParentContainer.Metadata.Type}/{auditBlob.ParentContainer.Metadata.Id}", ComponentId = $"/subscriptions/{id}", ComponentName = name, CheckResults = checks, MetadataAzure = new MetadataAzure { AuditId = auditMetadata.AuditId, Date = auditDate, JSON = azMetadata.ToString(Formatting.None), }, }; return(audit); }
private async Task <ImageScanResultWithCVEs> NormalizeRawData(AuditBlob auditBlob, AuditMetadata auditMetadata) { var auditDate = DateTimeOffset.FromUnixTimeSeconds(auditMetadata.Timestamp).DateTime; var scanResult = new ImageScanResultWithCVEs { Id = auditMetadata.AuditId, Date = auditDate, ImageTag = auditMetadata.ImageTag, }; if (auditMetadata.AuditResult != "succeeded") { var path = $"{auditBlob.ParentContainer.Name}/{auditBlob.Name}"; Logger.Warning( "Audit {AuditPath} result is {AuditResult} due: {FailureReason}", path, auditMetadata.AuditResult, auditMetadata.FailureDescription); scanResult.Status = ImageScanStatus.Failed; scanResult.Description = TrivyScanDescriptionNormalizer.ToHumanReadable(auditMetadata.FailureDescription); } else { var auditResultFilePath = $"{auditBlob.ParentContainer.Name}/{auditMetadata.TrivyAuditPath}"; var(entities, counters) = await this.ParseScanTargets(auditResultFilePath); scanResult.FoundCVEs = entities; scanResult.Counters = counters; scanResult.Status = ImageScanStatus.Succeeded; Logger.Information( "Successfully processed {ImageTag} image scan of {AuditDate} with {ScanSummary}", scanResult.ImageTag, scanResult.Date, scanResult.GetCheckResultMessage()); } return(scanResult); }
private async Task ProcessSingleAudit(AuditBlob auditBlob) { var path = $"{auditBlob.ParentContainer.Name}/{auditBlob.Name}"; var stream = await this.blobStorage.DownloadFile(path); using var sr = new StreamReader(stream); var metadataString = sr.ReadToEnd(); var auditMetadata = JsonConvert.DeserializeObject <AuditMetadata>(metadataString); try { var imageScanResult = await this.NormalizeRawData(auditBlob, auditMetadata); await this.db.SaveImageScanResult(imageScanResult); } catch (Exception ex) { Logger.Error(ex, "Failed to process audit {AuditPath}", path); } await this.blobStorage.MarkAsProcessed(auditBlob); }
private async Task ProcessSingleAudit(AuditBlob auditBlob) { var path = $"{auditBlob.ParentContainer.Name}/{auditBlob.Name}"; var stream = await this.blobStorage.DownloadFile(path); using var sr = new StreamReader(stream); var metadataString = sr.ReadToEnd(); var auditMetadata = JsonConvert.DeserializeObject <AuditMetadata>(metadataString); if (auditMetadata.AuditResult != "succeeded") { Logger.Warning( "Audit {AuditPath} result is {AuditResult} due: {FailureReason}", path, auditMetadata.AuditResult, auditMetadata.FailureDescription); } else { try { var audit = await this.NormalizeRawData(auditBlob, auditMetadata); int succeeded = 0, failed = 0, nodata = 0, inprogress = 0; foreach (var checkResult in audit.CheckResults) { switch (checkResult.Value) { case CheckValue.Succeeded: succeeded++; break; case CheckValue.Failed: failed++; break; case CheckValue.NoData: nodata++; break; case CheckValue.InProgress: inprogress++; break; } } Logger.Information( "Successfully processed audit {AuditId} of {AuditDate} with {CheckResultCount} check results, where succeeded {Succeeded}; failed {Failed}; in-progress {InProgress}; no-data {NoData}", audit.Id, audit.Date, audit.CheckResults.Count, succeeded, failed, nodata, inprogress); await this.db.SaveAuditResult(audit); // TODO: use a cancellation token inside this process? await this.extractOwnershipProcessor.Process(audit, CancellationToken.None); } catch (Exception ex) { Logger.Error(ex, "Failed to process audit {AuditPath}", path); } } await this.blobStorage.MarkAsProcessed(auditBlob); }