public async Task ProcessAuditHappyPath()
{
// Arrange
await using var context = JosekiTestsDb.CreateUniqueContext();
var parser = new ConfigurationParser("config.sample.yaml");
var checksCache = new ChecksCache(parser, context, new MemoryCache(new MemoryCacheOptions()));
var blobsMock = new Mock <IBlobStorageProcessor>(MockBehavior.Strict);
var dbMock = new Mock <IJosekiDatabase>();
var queueMock = new Mock <IQueue>();
var ownershipCache = new OwnershipCache(context, new MemoryCache(new MemoryCacheOptions()));
var postProcessor = new Mock <ExtractOwnershipProcessor>(context, ownershipCache);
var processor = new PolarisAuditProcessor(blobsMock.Object, dbMock.Object, checksCache, queueMock.Object, postProcessor.Object);
var container = new ScannerContainer(Path.Combine("audits", "samples", "polaris"))
{
Metadata = new ScannerMetadata
{
Type = ScannerType.Polaris,
Id = Guid.NewGuid().ToString(),
},
};
var audit = new AuditBlob {
Name = "meta.json", ParentContainer = container
};
blobsMock
.Setup(i => i.GetUnprocessedAudits(container))
.ReturnsAsync(new[] { audit })
.Verifiable();
blobsMock
.Setup(i => i.DownloadFile($"{container.Name}/{audit.Name}"))
.ReturnsAsync(File.OpenRead($"{container.Name}/{audit.Name}"))
.Verifiable();
blobsMock
.Setup(i => i.DownloadFile($"{container.Name}/audit.json"))
.ReturnsAsync(File.OpenRead($"{container.Name}/audit.json"))
.Verifiable();
blobsMock
.Setup(i => i.DownloadFile($"{container.Name}/k8s-meta.json"))
.ReturnsAsync(File.OpenRead($"{container.Name}/k8s-meta.json"))
.Verifiable();
blobsMock
.Setup(i => i.MarkAsProcessed(audit))
.Returns(Task.CompletedTask)
.Verifiable();
// Act & Assert
await processor.Process(container, CancellationToken.None);
blobsMock.Verify();
dbMock.Verify(i => i.GetNotExpiredImageScans(It.Is <string[]>(tags => tags.Length == UniqueImageTagCount)));
dbMock.Verify(i => i.SaveInProgressImageScan(It.IsAny <ImageScanResultWithCVEs>()), Times.Exactly(UniqueImageTagCount));
queueMock.Verify(i => i.EnqueueImageScanRequest(It.IsAny <ImageScanResultWithCVEs>()), Times.Exactly(UniqueImageTagCount));
dbMock.Verify(i => i.SaveAuditResult(It.Is <Audit>(a => VerifyHappyPathAudit(a, container))));
}
private async Task ProcessSingleAudit(AuditBlob auditBlob)
{
var path = $"{auditBlob.ParentContainer.Name}/{auditBlob.Name}";
var stream = await this.blobStorage.DownloadFile(path);
using var sr = new StreamReader(stream);
var metadataString = sr.ReadToEnd();
var auditMetadata = JsonConvert.DeserializeObject <AuditMetadata>(metadataString);
if (auditMetadata.AuditResult != "succeeded")
{
Logger.Warning(
"Audit {AuditPath} result is {AuditResult} due: {FailureReason}",
path,
auditMetadata.AuditResult,
auditMetadata.FailureDescription);
}
else
{
try
{
var tasks = auditMetadata
.AzSkAuditPaths
.Where(i => i.EndsWith(".json"))
.Select(i => this.blobStorage.DownloadFile($"{auditBlob.ParentContainer.Name}/{i}"))
.Select(this.ToJArray)
.ToArray();
await Task.WhenAll(tasks);
var audit = await this.NormalizeRawData(tasks.Select(i => i.Result).ToArray(), auditBlob, auditMetadata);
// TODO: Counter functions could be converted into single iterator
Logger.Information(
"Successfully processed audit {AuditId} of {AuditDate} with {CheckResultCount} check results, where succeeded {Succeeded}; failed {Failed}; no-data {NoData}",
audit.Id,
audit.Date,
audit.CheckResults.Count,
audit.CheckResults.Count(i => i.Value == CheckValue.Succeeded),
audit.CheckResults.Count(i => i.Value == CheckValue.Failed),
audit.CheckResults.Count(i => i.Value == CheckValue.NoData));
await this.db.SaveAuditResult(audit);
// TODO: use a cancellation token inside this process?
await this.extractOwnershipProcessor.Process(audit, CancellationToken.None);
}
catch (Exception ex)
{
Logger.Error(ex, "Failed to process audit {AuditPath}", path);
}
}
await this.blobStorage.MarkAsProcessed(auditBlob);
}
public async Task ProcessAuditHappyPath()
{
// Arrange
await using var context = JosekiTestsDb.CreateUniqueContext();
var parser = new ConfigurationParser("config.sample.yaml");
var checksCache = new ChecksCache(parser, context, new MemoryCache(new MemoryCacheOptions()));
var blobsMock = new Mock <IBlobStorageProcessor>(MockBehavior.Strict);
var dbMock = new Mock <IJosekiDatabase>();
var ownershipCache = new OwnershipCache(context, new MemoryCache(new MemoryCacheOptions()));
var postProcessor = new Mock <ExtractOwnershipProcessor>(context, ownershipCache);
var processor = new AzskAuditProcessor(blobsMock.Object, dbMock.Object, checksCache, postProcessor.Object);
var container = new ScannerContainer(Path.Combine("audits", "samples", "azsk"))
{
Metadata = new ScannerMetadata
{
Type = ScannerType.Azsk,
Id = Guid.NewGuid().ToString(),
},
};
var audit = new AuditBlob {
Name = "meta.json", ParentContainer = container
};
blobsMock
.Setup(i => i.GetUnprocessedAudits(container))
.ReturnsAsync(new[] { audit })
.Verifiable();
blobsMock
.Setup(i => i.DownloadFile($"{container.Name}/{audit.Name}"))
.ReturnsAsync(File.OpenRead($"{container.Name}/{audit.Name}"))
.Verifiable();
blobsMock
.Setup(i => i.DownloadFile($"{container.Name}/resources.json"))
.ReturnsAsync(File.OpenRead($"{container.Name}/resources.json"))
.Verifiable();
blobsMock
.Setup(i => i.DownloadFile($"{container.Name}/subscription.json"))
.ReturnsAsync(File.OpenRead($"{container.Name}/subscription.json"))
.Verifiable();
blobsMock
.Setup(i => i.MarkAsProcessed(audit))
.Returns(Task.CompletedTask)
.Verifiable();
// Act & Assert
await processor.Process(container, CancellationToken.None);
blobsMock.Verify();
dbMock.Verify(i => i.SaveAuditResult(It.Is <Audit>(a => VerifyHappyPathAudit(a, container))));
}
private async Task <Audit> NormalizeRawData(AuditBlob auditBlob, AuditMetadata auditMetadata)
{
var clusterId = string.IsNullOrEmpty(auditMetadata.ClusterId) ? auditBlob.ParentContainer.Metadata.Id : auditMetadata.ClusterId;
var auditJson = await this.GetJsonObject($"{auditBlob.ParentContainer.Name}/{auditMetadata.PolarisAuditPaths}");
var checks = await this.ParseChecksResults(auditJson, auditMetadata, clusterId);
var k8sJson = await this.GetJsonObject($"{auditBlob.ParentContainer.Name}/{auditMetadata.KubeMetadataPaths}");
try
{
var imageScanCheckResults = await this.EnrichAuditWithImageScans(auditMetadata.AuditId, k8sJson, clusterId);
checks.AddRange(imageScanCheckResults);
}
catch (Exception ex)
{
// Failure here should not impact the rest of audit
Logger.Warning(ex, "Failed to enrich audit {AuditId} Check Results with image scans", auditMetadata.AuditId);
}
var k8sMetadata = new JObject
{
{ "scanner", JToken.FromObject(auditBlob.ParentContainer.Metadata) },
{ "audit", JToken.FromObject(auditMetadata) },
{ "cluster", k8sJson },
};
var auditDate = DateTimeOffset.FromUnixTimeSeconds(auditMetadata.Timestamp).DateTime;
var infraComponentId = $"/k8s/{auditMetadata.ClusterId}";
var k8sName = this.ParseClusterName(k8sJson) ?? infraComponentId;
var audit = new Audit
{
Id = auditMetadata.AuditId,
Date = auditDate,
ScannerId = $"{auditBlob.ParentContainer.Metadata.Type}/{auditBlob.ParentContainer.Metadata.Id}",
ComponentId = infraComponentId,
ComponentName = k8sName,
CheckResults = checks,
MetadataKube = new MetadataKube
{
AuditId = auditMetadata.AuditId,
Date = auditDate,
JSON = k8sMetadata.ToString(Formatting.None),
},
};
return(audit);
}
public async Task ProcessScanResultWithoutCVEs()
{
// Arrange
await using var context = JosekiTestsDb.CreateUniqueContext();
var parser = new ConfigurationParser("config.sample.yaml");
var cveCache = new CveCache(parser, context, new MemoryCache(new MemoryCacheOptions()));
var blobsMock = new Mock <IBlobStorageProcessor>(MockBehavior.Strict);
var dbMock = new Mock <IJosekiDatabase>();
var processor = new TrivyAuditProcessor(blobsMock.Object, dbMock.Object, cveCache);
var container = new ScannerContainer(Path.Combine("audits", "samples", "trivy"))
{
Metadata = new ScannerMetadata
{
Type = ScannerType.Trivy,
Id = Guid.NewGuid().ToString(),
},
};
var audit = new AuditBlob {
Name = "meta_no_cve.json", ParentContainer = container
};
blobsMock
.Setup(i => i.GetUnprocessedAudits(container))
.ReturnsAsync(new[] { audit })
.Verifiable();
blobsMock
.Setup(i => i.DownloadFile($"{container.Name}/{audit.Name}"))
.ReturnsAsync(File.OpenRead($"{container.Name}/{audit.Name}"))
.Verifiable();
blobsMock
.Setup(i => i.DownloadFile($"{container.Name}/result_no_cves.json"))
.ReturnsAsync(File.OpenRead($"{container.Name}/result_no_cves.json"))
.Verifiable();
blobsMock
.Setup(i => i.MarkAsProcessed(audit))
.Returns(Task.CompletedTask)
.Verifiable();
// Act & Assert
await processor.Process(container, CancellationToken.None);
blobsMock.Verify();
dbMock.Verify(i => i.SaveImageScanResult(It.Is <ImageScanResultWithCVEs>(a => VerifyNoCveScan(a))));
}
private async Task <Audit> NormalizeRawData(JArray[] rawData, AuditBlob auditBlob, AuditMetadata auditMetadata)
{
var(scope, id, name) = this.GetSubscriptionMeta(rawData);
var(resources, checks) = await this.ParseResourcesAndChecks(rawData, auditMetadata);
var azMetadata = new JObject
{
{ "scanner", JToken.FromObject(auditBlob.ParentContainer.Metadata) },
{ "audit", JToken.FromObject(auditMetadata) },
{
"subscription", new JObject
{
{ "scope", scope },
{ "id", id },
{ "name", name },
{ "resources", resources },
}
},
};
var auditDate = DateTimeOffset.FromUnixTimeSeconds(auditMetadata.Timestamp).DateTime;
var audit = new Audit
{
Id = auditMetadata.AuditId,
Date = auditDate,
ScannerId = $"{auditBlob.ParentContainer.Metadata.Type}/{auditBlob.ParentContainer.Metadata.Id}",
ComponentId = $"/subscriptions/{id}",
ComponentName = name,
CheckResults = checks,
MetadataAzure = new MetadataAzure
{
AuditId = auditMetadata.AuditId,
Date = auditDate,
JSON = azMetadata.ToString(Formatting.None),
},
};
return(audit);
}
private async Task <ImageScanResultWithCVEs> NormalizeRawData(AuditBlob auditBlob, AuditMetadata auditMetadata)
{
var auditDate = DateTimeOffset.FromUnixTimeSeconds(auditMetadata.Timestamp).DateTime;
var scanResult = new ImageScanResultWithCVEs
{
Id = auditMetadata.AuditId,
Date = auditDate,
ImageTag = auditMetadata.ImageTag,
};
if (auditMetadata.AuditResult != "succeeded")
{
var path = $"{auditBlob.ParentContainer.Name}/{auditBlob.Name}";
Logger.Warning(
"Audit {AuditPath} result is {AuditResult} due: {FailureReason}",
path,
auditMetadata.AuditResult,
auditMetadata.FailureDescription);
scanResult.Status = ImageScanStatus.Failed;
scanResult.Description = TrivyScanDescriptionNormalizer.ToHumanReadable(auditMetadata.FailureDescription);
}
else
{
var auditResultFilePath = $"{auditBlob.ParentContainer.Name}/{auditMetadata.TrivyAuditPath}";
var(entities, counters) = await this.ParseScanTargets(auditResultFilePath);
scanResult.FoundCVEs = entities;
scanResult.Counters = counters;
scanResult.Status = ImageScanStatus.Succeeded;
Logger.Information(
"Successfully processed {ImageTag} image scan of {AuditDate} with {ScanSummary}",
scanResult.ImageTag,
scanResult.Date,
scanResult.GetCheckResultMessage());
}
return(scanResult);
}
private async Task ProcessSingleAudit(AuditBlob auditBlob)
{
var path = $"{auditBlob.ParentContainer.Name}/{auditBlob.Name}";
var stream = await this.blobStorage.DownloadFile(path);
using var sr = new StreamReader(stream);
var metadataString = sr.ReadToEnd();
var auditMetadata = JsonConvert.DeserializeObject <AuditMetadata>(metadataString);
try
{
var imageScanResult = await this.NormalizeRawData(auditBlob, auditMetadata);
await this.db.SaveImageScanResult(imageScanResult);
}
catch (Exception ex)
{
Logger.Error(ex, "Failed to process audit {AuditPath}", path);
}
await this.blobStorage.MarkAsProcessed(auditBlob);
}
private async Task ProcessSingleAudit(AuditBlob auditBlob)
{
var path = $"{auditBlob.ParentContainer.Name}/{auditBlob.Name}";
var stream = await this.blobStorage.DownloadFile(path);
using var sr = new StreamReader(stream);
var metadataString = sr.ReadToEnd();
var auditMetadata = JsonConvert.DeserializeObject <AuditMetadata>(metadataString);
if (auditMetadata.AuditResult != "succeeded")
{
Logger.Warning(
"Audit {AuditPath} result is {AuditResult} due: {FailureReason}",
path,
auditMetadata.AuditResult,
auditMetadata.FailureDescription);
}
else
{
try
{
var audit = await this.NormalizeRawData(auditBlob, auditMetadata);
int succeeded = 0, failed = 0, nodata = 0, inprogress = 0;
foreach (var checkResult in audit.CheckResults)
{
switch (checkResult.Value)
{
case CheckValue.Succeeded:
succeeded++;
break;
case CheckValue.Failed:
failed++;
break;
case CheckValue.NoData:
nodata++;
break;
case CheckValue.InProgress:
inprogress++;
break;
}
}
Logger.Information(
"Successfully processed audit {AuditId} of {AuditDate} with {CheckResultCount} check results, where succeeded {Succeeded}; failed {Failed}; in-progress {InProgress}; no-data {NoData}",
audit.Id,
audit.Date,
audit.CheckResults.Count,
succeeded,
failed,
nodata,
inprogress);
await this.db.SaveAuditResult(audit);
// TODO: use a cancellation token inside this process?
await this.extractOwnershipProcessor.Process(audit, CancellationToken.None);
}
catch (Exception ex)
{
Logger.Error(ex, "Failed to process audit {AuditPath}", path);
}
}
await this.blobStorage.MarkAsProcessed(auditBlob);
}
