public void BlobPolling_IgnoresClockSkew()
{
int testScanBlobLimitPerPoll = 3;
var now = DateTimeOffset.UtcNow;
var timeMap = new Dictionary <string, DateTimeOffset>();
var container = blobContainerMock.Object;
IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager(), _exceptionHandler, NullLogger <BlobListener> .Instance);
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
typeof(ScanBlobScanLogHybridPollingStrategy)
.GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic)
.SetValue(product, testScanBlobLimitPerPoll);
product.Register(blobClientMock.Object, container, executor);
product.Start();
List <string> expectedNames = new List <string>();
expectedNames.Add(CreateBlobAndUploadToContainer(blobContainerMock, blobItems));
timeMap[expectedNames.Single()] = now.AddSeconds(-60);
RunExecuterWithExpectedBlobs(expectedNames, product, executor);
expectedNames.Add(CreateBlobAndUploadToContainer(blobContainerMock, blobItems));
timeMap[expectedNames.Last()] = now.AddSeconds(-59);
// We should see the new item. We'll see 2 blobs, but only process 1 (due to receipt).
RunExecuterWithExpectedBlobs(expectedNames, product, executor, 1);
blobContainerMock.Verify(x => x.GetBlobsAsync(It.IsAny <BlobTraits>(), It.IsAny <BlobStates>(), It.IsAny <string>(), It.IsAny <CancellationToken>()),
Times.Exactly(2));
Assert.Equal(expectedNames, executor.BlobReceipts);
}
public async Task TestBlobListenerWithContainerBiggerThanThreshold()
{
int testScanBlobLimitPerPoll = 1;
string containerName = Path.GetRandomFileName();
var account = CreateFakeStorageAccount();
var container = account.CreateCloudBlobClient().GetContainerReference(containerName);
IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager());
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
typeof(ScanBlobScanLogHybridPollingStrategy)
.GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic)
.SetValue(product, testScanBlobLimitPerPoll);
product.Register(container, executor);
product.Start();
// populate with 5 blobs
List <string> expectedNames = new List <string>();
for (int i = 0; i < 5; i++)
{
expectedNames.Add(await CreateBlobAndUploadToContainer(container));
}
RunExecuteWithMultiPollingInterval(expectedNames, product, executor, testScanBlobLimitPerPoll);
// Now run again; shouldn't show up.
RunExecuterWithExpectedBlobs(new List <string>(), product, executor);
}
public void TestBlobListenerWithContainerBiggerThanThreshold()
{
int testScanBlobLimitPerPoll = 1;
var container = blobContainerMock.Object;
IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager(), _exceptionHandler, NullLogger <BlobListener> .Instance);
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
typeof(ScanBlobScanLogHybridPollingStrategy)
.GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic)
.SetValue(product, testScanBlobLimitPerPoll);
product.Register(blobClientMock.Object, container, executor);
product.Start();
// populate with 5 blobs
List <string> expectedNames = new List <string>();
for (int i = 0; i < 5; i++)
{
expectedNames.Add(CreateBlobAndUploadToContainer(blobContainerMock, blobItems));
}
RunExecuteWithMultiPollingInterval(expectedNames, product, executor, testScanBlobLimitPerPoll);
// Now run again; shouldn't show up.
RunExecuterWithExpectedBlobs(new List <string>(), product, executor);
}
public void ScanBlobScanLogHybridPollingStrategyTestBlobListener()
{
var container = blobContainerMock.Object;
IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager(), _exceptionHandler, _logger);
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
product.Register(blobClientMock.Object, container, executor);
product.Start();
RunExecuterWithExpectedBlobs(new List <string>(), product, executor);
string expectedBlobName = CreateBlobAndUploadToContainer(blobContainerMock, blobItems);
RunExecuterWithExpectedBlobs(new List <string>()
{
expectedBlobName
}, product, executor);
// Now run again; shouldn't show up.
RunExecuterWithExpectedBlobs(new List <string>(), product, executor);
// Verify happy-path logging.
var logMessages = _loggerProvider.GetAllLogMessages().ToArray();
Assert.Equal(4, logMessages.Length);
// 1 initialization log
var initLog = logMessages.Single(m => m.EventId.Name == "InitializedScanInfo");
Assert.Equal(Microsoft.Extensions.Logging.LogLevel.Debug, initLog.Level);
Assert.Equal(3, initLog.State.Count());
Assert.Equal(ContainerName, initLog.GetStateValue <string>("containerName"));
Assert.True(!string.IsNullOrWhiteSpace(initLog.GetStateValue <string>("latestScanInfo")));
Assert.True(!string.IsNullOrWhiteSpace(initLog.GetStateValue <string>("{OriginalFormat}")));
// 3 polling logs
var pollLogs = logMessages.Where(m => m.EventId.Name == "PollBlobContainer").ToArray();
Assert.Equal(3, pollLogs.Length);
void ValidatePollingLog(LogMessage pollingLog, int expectedBlobCount)
{
Assert.Equal(Microsoft.Extensions.Logging.LogLevel.Debug, pollingLog.Level);
Assert.Equal(7, pollingLog.State.Count());
Assert.Equal(ContainerName, pollingLog.GetStateValue <string>("containerName"));
Assert.Equal(expectedBlobCount, pollingLog.GetStateValue <int>("blobCount"));
Assert.True(!string.IsNullOrWhiteSpace(pollingLog.GetStateValue <string>("pollMinimumTime")));
Assert.True(!string.IsNullOrWhiteSpace(pollingLog.GetStateValue <string>("clientRequestId")));
Assert.True(pollingLog.GetStateValue <long>("pollLatency") >= 0);
Assert.False(pollingLog.GetStateValue <bool>("hasContinuationToken"));
Assert.True(!string.IsNullOrWhiteSpace(pollingLog.GetStateValue <string>("{OriginalFormat}")));
}
ValidatePollingLog(pollLogs[0], 0);
ValidatePollingLog(pollLogs[1], 1);
ValidatePollingLog(pollLogs[2], 1);
}
public async Task TestBlobListenerWithMultipleContainers()
{
int testScanBlobLimitPerPoll = 6, containerCount = 2;
string firstContainerName = Path.GetRandomFileName();
string secondContainerName = Path.GetRandomFileName();
var account = CreateFakeStorageAccount();
var firstContainer = account.CreateCloudBlobClient().GetContainerReference(firstContainerName);
var secondContainer = account.CreateCloudBlobClient().GetContainerReference(secondContainerName);
IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager());
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
typeof(ScanBlobScanLogHybridPollingStrategy)
.GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic)
.SetValue(product, testScanBlobLimitPerPoll);
product.Register(firstContainer, executor);
product.Register(secondContainer, executor);
product.Start();
// populate first container with 5 blobs > page size and second with 2 blobs < page size
// page size is going to be testScanBlobLimitPerPoll / number of container 6/2 = 3
List <string> firstContainerExpectedNames = new List <string>();
for (int i = 0; i < 5; i++)
{
firstContainerExpectedNames.Add(await CreateBlobAndUploadToContainer(firstContainer));
}
RunExecuteWithMultiPollingInterval(firstContainerExpectedNames, product, executor, testScanBlobLimitPerPoll / containerCount);
Thread.Sleep(10);
List <string> secondContainerExpectedNames = new List <string>();
for (int i = 0; i < 2; i++)
{
secondContainerExpectedNames.Add(await CreateBlobAndUploadToContainer(secondContainer));
}
RunExecuteWithMultiPollingInterval(secondContainerExpectedNames, product, executor, testScanBlobLimitPerPoll / containerCount);
// Now run again; shouldn't show up.
RunExecuterWithExpectedBlobs(new List <string>(), product, executor);
}
public void TestBlobListenerWithMultipleContainers()
{
int testScanBlobLimitPerPoll = 6, containerCount = 2;
var firstContainer = blobContainerMock.Object;
var secondContainer = secondBlobContainerMock.Object;
IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager(), _exceptionHandler, NullLogger <BlobListener> .Instance);
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
typeof(ScanBlobScanLogHybridPollingStrategy)
.GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic)
.SetValue(product, testScanBlobLimitPerPoll);
product.Register(blobClientMock.Object, firstContainer, executor);
product.Register(blobClientMock.Object, secondContainer, executor);
product.Start();
// populate first container with 5 blobs > page size and second with 2 blobs < page size
// page size is going to be testScanBlobLimitPerPoll / number of container 6/2 = 3
List <string> firstContainerExpectedNames = new List <string>();
for (int i = 0; i < 5; i++)
{
firstContainerExpectedNames.Add(CreateBlobAndUploadToContainer(blobContainerMock, blobItems));
}
RunExecuteWithMultiPollingInterval(firstContainerExpectedNames, product, executor, testScanBlobLimitPerPoll / containerCount);
Thread.Sleep(10);
List <string> secondContainerExpectedNames = new List <string>();
for (int i = 0; i < 2; i++)
{
secondContainerExpectedNames.Add(CreateBlobAndUploadToContainer(secondBlobContainerMock, secondBlobItems));
}
RunExecuteWithMultiPollingInterval(secondContainerExpectedNames, product, executor, testScanBlobLimitPerPoll / containerCount);
// Now run again; shouldn't show up.
RunExecuterWithExpectedBlobs(new List <string>(), product, executor);
}
public async Task BlobPolling_IncludesPreviousBatch()
{
// Blob timestamps are rounded to the nearest second, so make sure we continue to poll
// the previous second to catch any blobs that came in slightly after our previous attempt.
int testScanBlobLimitPerPoll = 3;
string containerName = Path.GetRandomFileName();
var account = CreateFakeStorageAccount();
var client = account.CreateCloudBlobClient();
// Strip off milliseconds.
var now = DateTimeOffset.UtcNow;
now = new DateTimeOffset(now.Year, now.Month, now.Day, now.Hour, now.Minute, now.Second, now.Offset);
var container = new SkewableFakeStorageBlobContainer(containerName, client as FakeStorageBlobClient,
(blobs) =>
{
foreach (ICloudBlob blob in blobs.Results)
{
blob.Properties.SetLastModified(now);
}
});
IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager());
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
typeof(ScanBlobScanLogHybridPollingStrategy)
.GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic)
.SetValue(product, testScanBlobLimitPerPoll);
product.Register(container, executor);
product.Start();
List <string> expectedNames = new List <string>();
expectedNames.Add(await CreateBlobAndUploadToContainer(container));
RunExecuterWithExpectedBlobs(expectedNames, product, executor);
expectedNames.Add(await CreateBlobAndUploadToContainer(container));
// We should see the new item. We'll see 2 blobs, but only process 1 (due to receipt).
RunExecuterWithExpectedBlobs(expectedNames, product, executor, 1);
Assert.Equal(2, container.CallCount);
Assert.Equal(expectedNames, executor.BlobReceipts);
}
public async Task BlobPolling_IgnoresClockSkew()
{
int testScanBlobLimitPerPoll = 3;
string containerName = Path.GetRandomFileName();
var account = CreateFakeStorageAccount();
var client = account.CreateCloudBlobClient();
var now = DateTimeOffset.UtcNow;
var timeMap = new Dictionary <string, DateTimeOffset>();
var container = new SkewableFakeStorageBlobContainer(containerName, client as FakeStorageBlobClient,
(blobs) =>
{
// Simulate some extreme clock skew -- the first one's LastUpdated
// wll be 60 seconds ago and the second will be 59 seconds ago.
foreach (ICloudBlob blob in blobs.Results)
{
blob.Properties.SetLastModified(timeMap[blob.Name]);
}
});
IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager());
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
typeof(ScanBlobScanLogHybridPollingStrategy)
.GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic)
.SetValue(product, testScanBlobLimitPerPoll);
product.Register(container, executor);
product.Start();
List <string> expectedNames = new List <string>();
expectedNames.Add(await CreateBlobAndUploadToContainer(container));
timeMap[expectedNames.Single()] = now.AddSeconds(-60);
RunExecuterWithExpectedBlobs(expectedNames, product, executor);
expectedNames.Clear();
expectedNames.Add(await CreateBlobAndUploadToContainer(container));
timeMap[expectedNames.Single()] = now.AddSeconds(-59);
// We should see the new item.
RunExecuterWithExpectedBlobs(expectedNames, product, executor);
Assert.Equal(2, container.CallCount);
}
public void BlobPolling_IncludesPreviousBatch()
{
// Blob timestamps are rounded to the nearest second, so make sure we continue to poll
// the previous second to catch any blobs that came in slightly after our previous attempt.
int testScanBlobLimitPerPoll = 3;
string containerName = Path.GetRandomFileName();
// Strip off milliseconds.
var now = DateTimeOffset.UtcNow;
now = new DateTimeOffset(now.Year, now.Month, now.Day, now.Hour, now.Minute, now.Second, now.Offset);
var container = blobContainerMock.Object;
IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager(), _exceptionHandler, NullLogger <BlobListener> .Instance);
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
typeof(ScanBlobScanLogHybridPollingStrategy)
.GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic)
.SetValue(product, testScanBlobLimitPerPoll);
product.Register(blobClientMock.Object, container, executor);
product.Start();
List <string> expectedNames = new List <string>();
expectedNames.Add(CreateBlobAndUploadToContainer(blobContainerMock, blobItems, lastModified: now));
RunExecuterWithExpectedBlobs(expectedNames, product, executor);
expectedNames.Add(CreateBlobAndUploadToContainer(blobContainerMock, blobItems, lastModified: now));
// We should see the new item. We'll see 2 blobs, but only process 1 (due to receipt).
RunExecuterWithExpectedBlobs(expectedNames, product, executor, 1);
blobContainerMock.Verify(x => x.GetBlobsAsync(It.IsAny <BlobTraits>(), It.IsAny <BlobStates>(), It.IsAny <string>(), It.IsAny <CancellationToken>()),
Times.Exactly(2));
Assert.Equal(expectedNames, executor.BlobReceipts);
}
public async Task ScanBlobScanLogHybridPollingStrategyTestBlobListener()
{
string containerName = Path.GetRandomFileName();
var account = CreateFakeStorageAccount();
var container = account.CreateCloudBlobClient().GetContainerReference(containerName);
IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager());
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
product.Register(container, executor);
product.Start();
RunExecuterWithExpectedBlobs(new List <string>(), product, executor);
string expectedBlobName = await CreateBlobAndUploadToContainer(container);
RunExecuterWithExpectedBlobs(new List <string>()
{
expectedBlobName
}, product, executor);
// Now run again; shouldn't show up.
RunExecuterWithExpectedBlobs(new List <string>(), product, executor);
}