public async Task ProcessAuditHappyPath()
{
// Arrange
await using var context = JosekiTestsDb.CreateUniqueContext();
var parser = new ConfigurationParser("config.sample.yaml");
var checksCache = new ChecksCache(parser, context, new MemoryCache(new MemoryCacheOptions()));
var blobsMock = new Mock <IBlobStorageProcessor>(MockBehavior.Strict);
var dbMock = new Mock <IJosekiDatabase>();
var queueMock = new Mock <IQueue>();
var ownershipCache = new OwnershipCache(context, new MemoryCache(new MemoryCacheOptions()));
var postProcessor = new Mock <ExtractOwnershipProcessor>(context, ownershipCache);
var processor = new PolarisAuditProcessor(blobsMock.Object, dbMock.Object, checksCache, queueMock.Object, postProcessor.Object);
var container = new ScannerContainer(Path.Combine("audits", "samples", "polaris"))
{
Metadata = new ScannerMetadata
{
Type = ScannerType.Polaris,
Id = Guid.NewGuid().ToString(),
},
};
var audit = new AuditBlob {
Name = "meta.json", ParentContainer = container
};
blobsMock
.Setup(i => i.GetUnprocessedAudits(container))
.ReturnsAsync(new[] { audit })
.Verifiable();
blobsMock
.Setup(i => i.DownloadFile($"{container.Name}/{audit.Name}"))
.ReturnsAsync(File.OpenRead($"{container.Name}/{audit.Name}"))
.Verifiable();
blobsMock
.Setup(i => i.DownloadFile($"{container.Name}/audit.json"))
.ReturnsAsync(File.OpenRead($"{container.Name}/audit.json"))
.Verifiable();
blobsMock
.Setup(i => i.DownloadFile($"{container.Name}/k8s-meta.json"))
.ReturnsAsync(File.OpenRead($"{container.Name}/k8s-meta.json"))
.Verifiable();
blobsMock
.Setup(i => i.MarkAsProcessed(audit))
.Returns(Task.CompletedTask)
.Verifiable();
// Act & Assert
await processor.Process(container, CancellationToken.None);
blobsMock.Verify();
dbMock.Verify(i => i.GetNotExpiredImageScans(It.Is <string[]>(tags => tags.Length == UniqueImageTagCount)));
dbMock.Verify(i => i.SaveInProgressImageScan(It.IsAny <ImageScanResultWithCVEs>()), Times.Exactly(UniqueImageTagCount));
queueMock.Verify(i => i.EnqueueImageScanRequest(It.IsAny <ImageScanResultWithCVEs>()), Times.Exactly(UniqueImageTagCount));
dbMock.Verify(i => i.SaveAuditResult(It.Is <Audit>(a => VerifyHappyPathAudit(a, container))));
}
public async Task ProcessAuditHappyPath()
{
// Arrange
await using var context = JosekiTestsDb.CreateUniqueContext();
var parser = new ConfigurationParser("config.sample.yaml");
var checksCache = new ChecksCache(parser, context, new MemoryCache(new MemoryCacheOptions()));
var blobsMock = new Mock <IBlobStorageProcessor>(MockBehavior.Strict);
var dbMock = new Mock <IJosekiDatabase>();
var ownershipCache = new OwnershipCache(context, new MemoryCache(new MemoryCacheOptions()));
var postProcessor = new Mock <ExtractOwnershipProcessor>(context, ownershipCache);
var processor = new AzskAuditProcessor(blobsMock.Object, dbMock.Object, checksCache, postProcessor.Object);
var container = new ScannerContainer(Path.Combine("audits", "samples", "azsk"))
{
Metadata = new ScannerMetadata
{
Type = ScannerType.Azsk,
Id = Guid.NewGuid().ToString(),
},
};
var audit = new AuditBlob {
Name = "meta.json", ParentContainer = container
};
blobsMock
.Setup(i => i.GetUnprocessedAudits(container))
.ReturnsAsync(new[] { audit })
.Verifiable();
blobsMock
.Setup(i => i.DownloadFile($"{container.Name}/{audit.Name}"))
.ReturnsAsync(File.OpenRead($"{container.Name}/{audit.Name}"))
.Verifiable();
blobsMock
.Setup(i => i.DownloadFile($"{container.Name}/resources.json"))
.ReturnsAsync(File.OpenRead($"{container.Name}/resources.json"))
.Verifiable();
blobsMock
.Setup(i => i.DownloadFile($"{container.Name}/subscription.json"))
.ReturnsAsync(File.OpenRead($"{container.Name}/subscription.json"))
.Verifiable();
blobsMock
.Setup(i => i.MarkAsProcessed(audit))
.Returns(Task.CompletedTask)
.Verifiable();
// Act & Assert
await processor.Process(container, CancellationToken.None);
blobsMock.Verify();
dbMock.Verify(i => i.SaveAuditResult(It.Is <Audit>(a => VerifyHappyPathAudit(a, container))));
}
public async Task ProcessScanResultWithoutCVEs()
{
// Arrange
await using var context = JosekiTestsDb.CreateUniqueContext();
var parser = new ConfigurationParser("config.sample.yaml");
var cveCache = new CveCache(parser, context, new MemoryCache(new MemoryCacheOptions()));
var blobsMock = new Mock <IBlobStorageProcessor>(MockBehavior.Strict);
var dbMock = new Mock <IJosekiDatabase>();
var processor = new TrivyAuditProcessor(blobsMock.Object, dbMock.Object, cveCache);
var container = new ScannerContainer(Path.Combine("audits", "samples", "trivy"))
{
Metadata = new ScannerMetadata
{
Type = ScannerType.Trivy,
Id = Guid.NewGuid().ToString(),
},
};
var audit = new AuditBlob {
Name = "meta_no_cve.json", ParentContainer = container
};
blobsMock
.Setup(i => i.GetUnprocessedAudits(container))
.ReturnsAsync(new[] { audit })
.Verifiable();
blobsMock
.Setup(i => i.DownloadFile($"{container.Name}/{audit.Name}"))
.ReturnsAsync(File.OpenRead($"{container.Name}/{audit.Name}"))
.Verifiable();
blobsMock
.Setup(i => i.DownloadFile($"{container.Name}/result_no_cves.json"))
.ReturnsAsync(File.OpenRead($"{container.Name}/result_no_cves.json"))
.Verifiable();
blobsMock
.Setup(i => i.MarkAsProcessed(audit))
.Returns(Task.CompletedTask)
.Verifiable();
// Act & Assert
await processor.Process(container, CancellationToken.None);
blobsMock.Verify();
dbMock.Verify(i => i.SaveImageScanResult(It.Is <ImageScanResultWithCVEs>(a => VerifyNoCveScan(a))));
}
private static bool VerifyHappyPathAudit(Audit audit, ScannerContainer container)
{
const string auditId = "ec783b2d-1da0-4a30-8e52-b72dd3174eae";
var auditDate = DateTimeOffset.FromUnixTimeSeconds(1583892106).DateTime;
audit.Id.Should().Be(auditId);
audit.Date.Should().BeCloseTo(auditDate, TimeSpan.FromMinutes(1));
audit.ScannerId.Should().Be($"{container.Metadata.Type}/{container.Metadata.Id}");
audit.ComponentId.Should().Be("/subscriptions/bc76e4b4-bc3e-46f0-b0c1-955c4c5fbe2a");
audit.ComponentName.Should().Be("Test Subscription");
audit.MetadataAzure.AuditId.Should().Be(auditId);
audit.MetadataAzure.Date.Should().BeCloseTo(auditDate, TimeSpan.FromMinutes(1));
audit.MetadataAzure.JSON.Should().NotBeNullOrWhiteSpace();
// TODO: maybe check each check in more details?
audit.CheckResults.Should().HaveCount(ExpectedCheckResults);
return(true);
}
private static bool VerifyHappyPathAudit(Audit audit, ScannerContainer container)
{
const string auditId = "3aab7985-5756-4cd6-8e3f-3353aa4b98a4";
var auditDate = DateTimeOffset.FromUnixTimeSeconds(1595556010).DateTime;
audit.Id.Should().Be(auditId);
audit.Date.Should().BeCloseTo(auditDate, TimeSpan.FromMinutes(1));
audit.ScannerId.Should().Be($"{container.Metadata.Type}/{container.Metadata.Id}");
audit.ComponentId.Should().Be("/k8s/3aab7985-d1a2-161d-8236-3353aa4b98a4");
audit.ComponentName.Should().Be("10.0.0.1");
audit.MetadataKube.AuditId.Should().Be(auditId);
audit.MetadataKube.Date.Should().BeCloseTo(auditDate, TimeSpan.FromMinutes(1));
audit.MetadataKube.JSON.Should().NotBeNullOrWhiteSpace();
// TODO: maybe check each check in more details?
audit.CheckResults.Should().HaveCount(ExpectedCheckResults);
return(true);
}
private async Task <ScannerContainer> DownloadAndParseMetadata(ScannerContainer container)
{
const int oneHourSeconds = 3600;
try
{
var stream = await this.blobStorage.DownloadFile(container.MetadataFilePath);
using var sr = new StreamReader(stream);
var metadataString = sr.ReadToEnd();
container.Metadata = JsonConvert.DeserializeObject <ScannerMetadata>(metadataString);
var unixNow = DateTimeOffset.UtcNow.ToUnixTimeSeconds();
var delta = unixNow - container.Metadata.Heartbeat;
if (delta < container.Metadata.HeartbeatPeriodicity)
{
// do nothing
}
else if (delta < oneHourSeconds || delta < container.Metadata.HeartbeatPeriodicity * 2)
{
var lastExecution = DateTimeOffset.FromUnixTimeSeconds(container.Metadata.Heartbeat);
Logger.Warning("Scanner {ScannerContainer} keeps silence since {LastExecution}", container.Name, lastExecution);
}
else
{
var lastExecution = DateTimeOffset.FromUnixTimeSeconds(container.Metadata.Heartbeat);
Logger.Error("Scanner {ScannerContainer} keeps silence since {LastExecution}", container.Name, lastExecution);
}
return(container);
}
catch (JosekiException ex)
{
Logger.Warning(ex, "Failed to download amd parse metadata of {ScannerContainer}", container.Name);
return(ScannerContainer.Empty);
}
}
/// <inheritdoc />
public async Task Process(ScannerContainer container, CancellationToken token)
{
var audits = await this.blobStorage.GetUnprocessedAudits(container);
foreach (var audit in audits)
{
if (token.IsCancellationRequested)
{
return;
}
try
{
Logger.Information("Processing audit result from {AuditName} container was {ProcessingState}", audit.Name, "started");
await this.ProcessSingleAudit(audit);
Logger.Information("Processing audit result from {AuditName} container was {ProcessingState}", audit.Name, "finished");
}
catch (Exception ex)
{
Logger.Error(ex, "Processing audit result from {AuditName} container was {ProcessingState}", audit.Name, "failed");
}
}
}
/// <inheritdoc />
public Task Process(ScannerContainer container, CancellationToken token)
{
return(Task.CompletedTask);
}
