public async Task RegisterAsync_InitializesWithScanInfoManager()
{
var container = _blobContainerMock.Object;
TestBlobScanInfoManager scanInfoManager = new TestBlobScanInfoManager();
IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(scanInfoManager, _exceptionHandler, NullLogger <BlobListener> .Instance);
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
// Create a few blobs.
for (int i = 0; i < 5; i++)
{
CreateBlobAndUploadToContainer(_blobContainerMock, _blobItems);
}
// delay slightly so we guarantee a later timestamp
await Task.Delay(10);
await scanInfoManager.UpdateLatestScanAsync(AccountName, ContainerName, DateTime.UtcNow);
await product.RegisterAsync(_blobClientMock.Object, container, executor, CancellationToken.None);
// delay slightly so we guarantee a later timestamp
await Task.Delay(10);
var expectedNames = new List <string>();
expectedNames.Add(CreateBlobAndUploadToContainer(_blobContainerMock, _blobItems));
RunExecuterWithExpectedBlobs(expectedNames, product, executor);
}
public void BlobPolling_IgnoresClockSkew()
{
int testScanBlobLimitPerPoll = 3;
var now = DateTimeOffset.UtcNow;
var timeMap = new Dictionary <string, DateTimeOffset>();
var container = _blobContainerMock.Object;
IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager(), _exceptionHandler, NullLogger <BlobListener> .Instance);
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
typeof(ScanBlobScanLogHybridPollingStrategy)
.GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic)
.SetValue(product, testScanBlobLimitPerPoll);
product.Register(_blobClientMock.Object, container, executor);
product.Start();
List <string> expectedNames = new List <string>();
expectedNames.Add(CreateBlobAndUploadToContainer(_blobContainerMock, _blobItems));
timeMap[expectedNames.Single()] = now.AddSeconds(-60);
RunExecuterWithExpectedBlobs(expectedNames, product, executor);
expectedNames.Add(CreateBlobAndUploadToContainer(_blobContainerMock, _blobItems));
timeMap[expectedNames.Last()] = now.AddSeconds(-59);
// We should see the new item. We'll see 2 blobs, but only process 1 (due to receipt).
RunExecuterWithExpectedBlobs(expectedNames, product, executor, 1);
_blobContainerMock.Verify(x => x.GetBlobsAsync(It.IsAny <BlobTraits>(), It.IsAny <BlobStates>(), It.IsAny <string>(), It.IsAny <CancellationToken>()),
Times.Exactly(2));
Assert.AreEqual(expectedNames, executor.BlobReceipts);
}
public void TestBlobListenerWithContainerBiggerThanThreshold()
{
int testScanBlobLimitPerPoll = 1;
var container = _blobContainerMock.Object;
IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager(), _exceptionHandler, NullLogger <BlobListener> .Instance);
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
typeof(ScanBlobScanLogHybridPollingStrategy)
.GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic)
.SetValue(product, testScanBlobLimitPerPoll);
product.Register(_blobClientMock.Object, container, executor);
product.Start();
// populate with 5 blobs
List <string> expectedNames = new List <string>();
for (int i = 0; i < 5; i++)
{
expectedNames.Add(CreateBlobAndUploadToContainer(_blobContainerMock, _blobItems));
}
RunExecuteWithMultiPollingInterval(expectedNames, product, executor, testScanBlobLimitPerPoll);
// Now run again; shouldn't show up.
RunExecuterWithExpectedBlobs(new List <string>(), product, executor);
}
public void ScanBlobScanLogHybridPollingStrategyTestBlobListener()
{
var container = _blobContainerMock.Object;
IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager(), _exceptionHandler, _logger);
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
product.Register(_blobClientMock.Object, container, executor);
product.Start();
RunExecuterWithExpectedBlobs(new List <string>(), product, executor);
string expectedBlobName = CreateBlobAndUploadToContainer(_blobContainerMock, _blobItems);
RunExecuterWithExpectedBlobs(new List <string>()
{
expectedBlobName
}, product, executor);
// Now run again; shouldn't show up.
RunExecuterWithExpectedBlobs(new List <string>(), product, executor);
// Verify happy-path logging.
var logMessages = _loggerProvider.GetAllLogMessages().ToArray();
Assert.AreEqual(4, logMessages.Length);
// 1 initialization log
var initLog = logMessages.Single(m => m.EventId.Name == "InitializedScanInfo");
Assert.AreEqual(Microsoft.Extensions.Logging.LogLevel.Debug, initLog.Level);
Assert.AreEqual(3, initLog.State.Count());
Assert.AreEqual(ContainerName, initLog.GetStateValue <string>("containerName"));
Assert.True(!string.IsNullOrWhiteSpace(initLog.GetStateValue <string>("latestScanInfo")));
Assert.True(!string.IsNullOrWhiteSpace(initLog.GetStateValue <string>("{OriginalFormat}")));
// 3 polling logs
var pollLogs = logMessages.Where(m => m.EventId.Name == "PollBlobContainer").ToArray();
Assert.AreEqual(3, pollLogs.Length);
void ValidatePollingLog(LogMessage pollingLog, int expectedBlobCount)
{
Assert.AreEqual(Microsoft.Extensions.Logging.LogLevel.Debug, pollingLog.Level);
Assert.AreEqual(7, pollingLog.State.Count());
Assert.AreEqual(ContainerName, pollingLog.GetStateValue <string>("containerName"));
Assert.AreEqual(expectedBlobCount, pollingLog.GetStateValue <int>("blobCount"));
Assert.True(!string.IsNullOrWhiteSpace(pollingLog.GetStateValue <string>("pollMinimumTime")));
Assert.True(!string.IsNullOrWhiteSpace(pollingLog.GetStateValue <string>("clientRequestId")));
Assert.True(pollingLog.GetStateValue <long>("pollLatency") >= 0);
Assert.False(pollingLog.GetStateValue <bool>("hasContinuationToken"));
Assert.True(!string.IsNullOrWhiteSpace(pollingLog.GetStateValue <string>("{OriginalFormat}")));
}
ValidatePollingLog(pollLogs[0], 0);
ValidatePollingLog(pollLogs[1], 1);
ValidatePollingLog(pollLogs[2], 1);
}
public async Task ExecuteAsync_UpdatesScanInfoManager()
{
int testScanBlobLimitPerPoll = 6;
BlobContainerClient firstContainer = _blobContainerMock.Object;
BlobContainerClient secondContainer = _secondBlobContainerMock.Object;
TestBlobScanInfoManager testScanInfoManager = new TestBlobScanInfoManager();
string accountName = AccountName;
testScanInfoManager.SetScanInfo(accountName, ContainerName, DateTime.MinValue);
testScanInfoManager.SetScanInfo(accountName, SecondContainerName, DateTime.MinValue);
IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(testScanInfoManager, _exceptionHandler, NullLogger <BlobListener> .Instance);
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
typeof(ScanBlobScanLogHybridPollingStrategy)
.GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic)
.SetValue(product, testScanBlobLimitPerPoll);
await product.RegisterAsync(_blobClientMock.Object, firstContainer, executor, CancellationToken.None);
await product.RegisterAsync(_blobClientMock.Object, secondContainer, executor, CancellationToken.None);
var firstExpectedNames = new List <string>();
for (int i = 0; i < 3; i++)
{
firstExpectedNames.Add(CreateBlobAndUploadToContainer(_blobContainerMock, _blobItems));
}
RunExecuteWithMultiPollingInterval(firstExpectedNames, product, executor, testScanBlobLimitPerPoll / 2);
// only expect the first container to have updated its scanInfo
Assert.AreEqual(1, testScanInfoManager.UpdateCounts[accountName][ContainerName]);
int count;
testScanInfoManager.UpdateCounts[accountName].TryGetValue(SecondContainerName, out count);
Assert.AreEqual(0, count);
await Task.Delay(10);
var secondExpectedNames = new List <string>();
for (int i = 0; i < 7; i++)
{
secondExpectedNames.Add(CreateBlobAndUploadToContainer(_secondBlobContainerMock, _secondBlobItems));
}
RunExecuteWithMultiPollingInterval(secondExpectedNames, product, executor, testScanBlobLimitPerPoll / 2);
// this time, only expect the second container to have updated its scanInfo
Assert.AreEqual(1, testScanInfoManager.UpdateCounts[accountName][ContainerName]);
Assert.AreEqual(1, testScanInfoManager.UpdateCounts[accountName][SecondContainerName]);
}
public async Task ExecuteAsync_UpdatesScanInfo_WithEarliestFailure()
{
int testScanBlobLimitPerPoll = 6;
// we'll introduce multiple errors to make sure we take the earliest timestamp
DateTime earliestErrorTime = DateTime.UtcNow;
var container = _blobContainerMock.Object;
TestBlobScanInfoManager testScanInfoManager = new TestBlobScanInfoManager();
string accountName = AccountName;
testScanInfoManager.SetScanInfo(accountName, ContainerName, DateTime.MinValue);
IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(testScanInfoManager, _exceptionHandler, NullLogger <BlobListener> .Instance);
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
typeof(ScanBlobScanLogHybridPollingStrategy)
.GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic)
.SetValue(product, testScanBlobLimitPerPoll);
await product.RegisterAsync(_blobClientMock.Object, container, executor, CancellationToken.None);
// Induce a failure to make sure the timestamp is earlier than the failure.
var expectedNames = new List <string>();
for (int i = 0; i < 7; i++)
{
string name;
if (i % 3 == 0)
{
name = CreateBlobAndUploadToContainer(_blobContainerMock, _blobItems, "throw", earliestErrorTime.AddMinutes(i));
}
else
{
name = CreateBlobAndUploadToContainer(_blobContainerMock, _blobItems, "test", earliestErrorTime.AddMinutes(10));
}
expectedNames.Add(name);
}
RunExecuteWithMultiPollingInterval(expectedNames, product, executor, testScanBlobLimitPerPoll);
DateTime?storedTime = await testScanInfoManager.LoadLatestScanAsync(accountName, ContainerName);
Assert.True(storedTime < earliestErrorTime);
Assert.AreEqual(1, testScanInfoManager.UpdateCounts[accountName][ContainerName]);
_blobContainerMock.Verify(x => x.GetBlobsAsync(It.IsAny <BlobTraits>(), It.IsAny <BlobStates>(), It.IsAny <string>(), It.IsAny <CancellationToken>()),
Times.Exactly(2));
}
public void TestBlobListenerWithMultipleContainers()
{
int testScanBlobLimitPerPoll = 6, containerCount = 2;
var firstContainer = _blobContainerMock.Object;
var secondContainer = _secondBlobContainerMock.Object;
IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager(), _exceptionHandler, NullLogger <BlobListener> .Instance);
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
typeof(ScanBlobScanLogHybridPollingStrategy)
.GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic)
.SetValue(product, testScanBlobLimitPerPoll);
product.Register(_blobClientMock.Object, firstContainer, executor);
product.Register(_blobClientMock.Object, secondContainer, executor);
product.Start();
// populate first container with 5 blobs > page size and second with 2 blobs < page size
// page size is going to be testScanBlobLimitPerPoll / number of container 6/2 = 3
List <string> firstContainerExpectedNames = new List <string>();
for (int i = 0; i < 5; i++)
{
firstContainerExpectedNames.Add(CreateBlobAndUploadToContainer(_blobContainerMock, _blobItems));
}
RunExecuteWithMultiPollingInterval(firstContainerExpectedNames, product, executor, testScanBlobLimitPerPoll / containerCount);
Thread.Sleep(10);
List <string> secondContainerExpectedNames = new List <string>();
for (int i = 0; i < 2; i++)
{
secondContainerExpectedNames.Add(CreateBlobAndUploadToContainer(_secondBlobContainerMock, _secondBlobItems));
}
RunExecuteWithMultiPollingInterval(secondContainerExpectedNames, product, executor, testScanBlobLimitPerPoll / containerCount);
// Now run again; shouldn't show up.
RunExecuterWithExpectedBlobs(new List <string>(), product, executor);
}
public void BlobPolling_IncludesPreviousBatch()
{
// Blob timestamps are rounded to the nearest second, so make sure we continue to poll
// the previous second to catch any blobs that came in slightly after our previous attempt.
int testScanBlobLimitPerPoll = 3;
string containerName = Path.GetRandomFileName();
// Strip off milliseconds.
var now = DateTimeOffset.UtcNow;
now = new DateTimeOffset(now.Year, now.Month, now.Day, now.Hour, now.Minute, now.Second, now.Offset);
var container = _blobContainerMock.Object;
IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager(), _exceptionHandler, NullLogger <BlobListener> .Instance);
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
typeof(ScanBlobScanLogHybridPollingStrategy)
.GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic)
.SetValue(product, testScanBlobLimitPerPoll);
product.Register(_blobClientMock.Object, container, executor);
product.Start();
List <string> expectedNames = new List <string>();
expectedNames.Add(CreateBlobAndUploadToContainer(_blobContainerMock, _blobItems, lastModified: now));
RunExecuterWithExpectedBlobs(expectedNames, product, executor);
expectedNames.Add(CreateBlobAndUploadToContainer(_blobContainerMock, _blobItems, lastModified: now));
// We should see the new item. We'll see 2 blobs, but only process 1 (due to receipt).
RunExecuterWithExpectedBlobs(expectedNames, product, executor, 1);
_blobContainerMock.Verify(x => x.GetBlobsAsync(It.IsAny <BlobTraits>(), It.IsAny <BlobStates>(), It.IsAny <string>(), It.IsAny <CancellationToken>()),
Times.Exactly(2));
Assert.AreEqual(expectedNames, executor.BlobReceipts);
}
