public async Task RegisterAsync_InitializesWithScanInfoManager()
{
string containerName = Guid.NewGuid().ToString();
var account = CreateFakeStorageAccount();
var container = account.CreateCloudBlobClient().GetContainerReference(containerName);
TestBlobScanInfoManager scanInfoManager = new TestBlobScanInfoManager();
IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(scanInfoManager);
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
// Create a few blobs.
for (int i = 0; i < 5; i++)
{
await CreateBlobAndUploadToContainer(container);
}
await scanInfoManager.UpdateLatestScanAsync(account.Name, containerName, DateTime.UtcNow);
await product.RegisterAsync(container, executor, CancellationToken.None);
// delay slightly so we guarantee a later timestamp
await Task.Delay(10);
var expectedNames = new List <string>();
expectedNames.Add(await CreateBlobAndUploadToContainer(container));
RunExecuterWithExpectedBlobs(expectedNames, product, executor);
}
public async Task TestBlobListenerWithContainerBiggerThanThreshold()
{
int testScanBlobLimitPerPoll = 1;
string containerName = Path.GetRandomFileName();
var account = CreateFakeStorageAccount();
var container = account.CreateCloudBlobClient().GetContainerReference(containerName);
IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager());
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
typeof(ScanBlobScanLogHybridPollingStrategy)
.GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic)
.SetValue(product, testScanBlobLimitPerPoll);
product.Register(container, executor);
product.Start();
// populate with 5 blobs
List <string> expectedNames = new List <string>();
for (int i = 0; i < 5; i++)
{
expectedNames.Add(await CreateBlobAndUploadToContainer(container));
}
RunExecuteWithMultiPollingInterval(expectedNames, product, executor, testScanBlobLimitPerPoll);
// Now run again; shouldn't show up.
RunExecuterWithExpectedBlobs(new List <string>(), product, executor);
}
public async Task RegisterAsync_InitializesWithScanInfoManager()
{
var container = blobContainerMock.Object;
TestBlobScanInfoManager scanInfoManager = new TestBlobScanInfoManager();
IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(scanInfoManager, _exceptionHandler, NullLogger <BlobListener> .Instance);
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
// Create a few blobs.
for (int i = 0; i < 5; i++)
{
CreateBlobAndUploadToContainer(blobContainerMock, blobItems);
}
// delay slightly so we guarantee a later timestamp
await Task.Delay(10);
await scanInfoManager.UpdateLatestScanAsync(AccountName, ContainerName, DateTime.UtcNow);
await product.RegisterAsync(blobClientMock.Object, container, executor, CancellationToken.None);
// delay slightly so we guarantee a later timestamp
await Task.Delay(10);
var expectedNames = new List <string>();
expectedNames.Add(CreateBlobAndUploadToContainer(blobContainerMock, blobItems));
RunExecuterWithExpectedBlobs(expectedNames, product, executor);
}
public void BlobPolling_IgnoresClockSkew()
{
int testScanBlobLimitPerPoll = 3;
var now = DateTimeOffset.UtcNow;
var timeMap = new Dictionary <string, DateTimeOffset>();
var container = blobContainerMock.Object;
IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager(), _exceptionHandler, NullLogger <BlobListener> .Instance);
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
typeof(ScanBlobScanLogHybridPollingStrategy)
.GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic)
.SetValue(product, testScanBlobLimitPerPoll);
product.Register(blobClientMock.Object, container, executor);
product.Start();
List <string> expectedNames = new List <string>();
expectedNames.Add(CreateBlobAndUploadToContainer(blobContainerMock, blobItems));
timeMap[expectedNames.Single()] = now.AddSeconds(-60);
RunExecuterWithExpectedBlobs(expectedNames, product, executor);
expectedNames.Add(CreateBlobAndUploadToContainer(blobContainerMock, blobItems));
timeMap[expectedNames.Last()] = now.AddSeconds(-59);
// We should see the new item. We'll see 2 blobs, but only process 1 (due to receipt).
RunExecuterWithExpectedBlobs(expectedNames, product, executor, 1);
blobContainerMock.Verify(x => x.GetBlobsAsync(It.IsAny <BlobTraits>(), It.IsAny <BlobStates>(), It.IsAny <string>(), It.IsAny <CancellationToken>()),
Times.Exactly(2));
Assert.Equal(expectedNames, executor.BlobReceipts);
}
public void TestBlobListenerWithContainerBiggerThanThreshold()
{
int testScanBlobLimitPerPoll = 1;
var container = blobContainerMock.Object;
IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager(), _exceptionHandler, NullLogger <BlobListener> .Instance);
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
typeof(ScanBlobScanLogHybridPollingStrategy)
.GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic)
.SetValue(product, testScanBlobLimitPerPoll);
product.Register(blobClientMock.Object, container, executor);
product.Start();
// populate with 5 blobs
List <string> expectedNames = new List <string>();
for (int i = 0; i < 5; i++)
{
expectedNames.Add(CreateBlobAndUploadToContainer(blobContainerMock, blobItems));
}
RunExecuteWithMultiPollingInterval(expectedNames, product, executor, testScanBlobLimitPerPoll);
// Now run again; shouldn't show up.
RunExecuterWithExpectedBlobs(new List <string>(), product, executor);
}
public async Task ExecuteAsync_UpdatesScanInfo_WithEarliestFailure()
{
int testScanBlobLimitPerPoll = 6;
string containerName = Guid.NewGuid().ToString();
// we'll introduce multiple errors to make sure we take the earliest timestamp
DateTime earliestErrorTime = DateTime.UtcNow;
var timeMap = new Dictionary <string, DateTimeOffset>();
IStorageAccount account = CreateFakeStorageAccount();
IStorageBlobContainer container = new SkewableFakeStorageBlobContainer(new MemoryBlobStore(), containerName,
account.CreateBlobClient(), blobs =>
{
// Set a blob with "throw" to a specific date and time. Make sure the error blob
// is earlier than the others.
foreach (IStorageBlob blob in blobs.Results)
{
((FakeStorageBlobProperties)blob.Properties).LastModified = timeMap[blob.Name];
}
});
TestBlobScanInfoManager testScanInfoManager = new TestBlobScanInfoManager();
string accountName = account.Credentials.AccountName;
testScanInfoManager.SetScanInfo(accountName, containerName, DateTime.MinValue);
IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(testScanInfoManager);
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
typeof(ScanBlobScanLogHybridPollingStrategy)
.GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic)
.SetValue(product, testScanBlobLimitPerPoll);
await product.RegisterAsync(container, executor, CancellationToken.None);
// Induce a failure to make sure the timestamp is earlier than the failure.
var expectedNames = new List <string>();
for (int i = 0; i < 7; i++)
{
string name;
if (i % 3 == 0)
{
name = CreateAblobAndUploadToContainer(container, "throw");
timeMap[name] = earliestErrorTime.AddMinutes(i);
}
else
{
name = CreateAblobAndUploadToContainer(container, "test");
timeMap[name] = earliestErrorTime.AddMinutes(10);
}
expectedNames.Add(name);
}
RunExecuteWithMultiPollingInterval(expectedNames, product, executor, testScanBlobLimitPerPoll);
DateTime?storedTime = await testScanInfoManager.LoadLatestScanAsync(accountName, containerName);
Assert.True(storedTime < earliestErrorTime);
Assert.Equal(1, testScanInfoManager.UpdateCounts[accountName][containerName]);
}
public void ScanBlobScanLogHybridPollingStrategyTestBlobListener()
{
var container = blobContainerMock.Object;
IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager(), _exceptionHandler, _logger);
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
product.Register(blobClientMock.Object, container, executor);
product.Start();
RunExecuterWithExpectedBlobs(new List <string>(), product, executor);
string expectedBlobName = CreateBlobAndUploadToContainer(blobContainerMock, blobItems);
RunExecuterWithExpectedBlobs(new List <string>()
{
expectedBlobName
}, product, executor);
// Now run again; shouldn't show up.
RunExecuterWithExpectedBlobs(new List <string>(), product, executor);
// Verify happy-path logging.
var logMessages = _loggerProvider.GetAllLogMessages().ToArray();
Assert.Equal(4, logMessages.Length);
// 1 initialization log
var initLog = logMessages.Single(m => m.EventId.Name == "InitializedScanInfo");
Assert.Equal(Microsoft.Extensions.Logging.LogLevel.Debug, initLog.Level);
Assert.Equal(3, initLog.State.Count());
Assert.Equal(ContainerName, initLog.GetStateValue <string>("containerName"));
Assert.True(!string.IsNullOrWhiteSpace(initLog.GetStateValue <string>("latestScanInfo")));
Assert.True(!string.IsNullOrWhiteSpace(initLog.GetStateValue <string>("{OriginalFormat}")));
// 3 polling logs
var pollLogs = logMessages.Where(m => m.EventId.Name == "PollBlobContainer").ToArray();
Assert.Equal(3, pollLogs.Length);
void ValidatePollingLog(LogMessage pollingLog, int expectedBlobCount)
{
Assert.Equal(Microsoft.Extensions.Logging.LogLevel.Debug, pollingLog.Level);
Assert.Equal(7, pollingLog.State.Count());
Assert.Equal(ContainerName, pollingLog.GetStateValue <string>("containerName"));
Assert.Equal(expectedBlobCount, pollingLog.GetStateValue <int>("blobCount"));
Assert.True(!string.IsNullOrWhiteSpace(pollingLog.GetStateValue <string>("pollMinimumTime")));
Assert.True(!string.IsNullOrWhiteSpace(pollingLog.GetStateValue <string>("clientRequestId")));
Assert.True(pollingLog.GetStateValue <long>("pollLatency") >= 0);
Assert.False(pollingLog.GetStateValue <bool>("hasContinuationToken"));
Assert.True(!string.IsNullOrWhiteSpace(pollingLog.GetStateValue <string>("{OriginalFormat}")));
}
ValidatePollingLog(pollLogs[0], 0);
ValidatePollingLog(pollLogs[1], 1);
ValidatePollingLog(pollLogs[2], 1);
}
public async Task ExecuteAsync_UpdatesScanInfoManager()
{
int testScanBlobLimitPerPoll = 6;
string firstContainerName = Guid.NewGuid().ToString();
string secondContainerName = Guid.NewGuid().ToString();
var account = CreateFakeStorageAccount();
CloudBlobContainer firstContainer = account.CreateCloudBlobClient().GetContainerReference(firstContainerName);
CloudBlobContainer secondContainer = account.CreateCloudBlobClient().GetContainerReference(secondContainerName);
TestBlobScanInfoManager testScanInfoManager = new TestBlobScanInfoManager();
string accountName = account.Name;
testScanInfoManager.SetScanInfo(accountName, firstContainerName, DateTime.MinValue);
testScanInfoManager.SetScanInfo(accountName, secondContainerName, DateTime.MinValue);
IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(testScanInfoManager);
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
typeof(ScanBlobScanLogHybridPollingStrategy)
.GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic)
.SetValue(product, testScanBlobLimitPerPoll);
await product.RegisterAsync(firstContainer, executor, CancellationToken.None);
await product.RegisterAsync(secondContainer, executor, CancellationToken.None);
var firstExpectedNames = new List <string>();
for (int i = 0; i < 3; i++)
{
firstExpectedNames.Add(await CreateBlobAndUploadToContainer(firstContainer));
}
RunExecuteWithMultiPollingInterval(firstExpectedNames, product, executor, testScanBlobLimitPerPoll / 2);
// only expect the first container to have updated its scanInfo
Assert.Equal(1, testScanInfoManager.UpdateCounts[accountName][firstContainerName]);
int count;
testScanInfoManager.UpdateCounts[accountName].TryGetValue(secondContainerName, out count);
Assert.Equal(0, count);
await Task.Delay(10);
var secondExpectedNames = new List <string>();
for (int i = 0; i < 7; i++)
{
secondExpectedNames.Add(await CreateBlobAndUploadToContainer(secondContainer));
}
RunExecuteWithMultiPollingInterval(secondExpectedNames, product, executor, testScanBlobLimitPerPoll / 2);
// this time, only expect the second container to have updated its scanInfo
Assert.Equal(1, testScanInfoManager.UpdateCounts[accountName][firstContainerName]);
Assert.Equal(1, testScanInfoManager.UpdateCounts[accountName][secondContainerName]);
}
public async Task ExecuteAsync_UpdatesScanInfoManager()
{
int testScanBlobLimitPerPoll = 6;
BlobContainerClient firstContainer = blobContainerMock.Object;
BlobContainerClient secondContainer = secondBlobContainerMock.Object;
TestBlobScanInfoManager testScanInfoManager = new TestBlobScanInfoManager();
string accountName = AccountName;
testScanInfoManager.SetScanInfo(accountName, ContainerName, DateTime.MinValue);
testScanInfoManager.SetScanInfo(accountName, SecondContainerName, DateTime.MinValue);
IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(testScanInfoManager, _exceptionHandler, NullLogger <BlobListener> .Instance);
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
typeof(ScanBlobScanLogHybridPollingStrategy)
.GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic)
.SetValue(product, testScanBlobLimitPerPoll);
await product.RegisterAsync(blobClientMock.Object, firstContainer, executor, CancellationToken.None);
await product.RegisterAsync(blobClientMock.Object, secondContainer, executor, CancellationToken.None);
var firstExpectedNames = new List <string>();
for (int i = 0; i < 3; i++)
{
firstExpectedNames.Add(CreateBlobAndUploadToContainer(blobContainerMock, blobItems));
}
RunExecuteWithMultiPollingInterval(firstExpectedNames, product, executor, testScanBlobLimitPerPoll / 2);
// only expect the first container to have updated its scanInfo
Assert.Equal(1, testScanInfoManager.UpdateCounts[accountName][ContainerName]);
int count;
testScanInfoManager.UpdateCounts[accountName].TryGetValue(SecondContainerName, out count);
Assert.Equal(0, count);
await Task.Delay(10);
var secondExpectedNames = new List <string>();
for (int i = 0; i < 7; i++)
{
secondExpectedNames.Add(CreateBlobAndUploadToContainer(secondBlobContainerMock, secondBlobItems));
}
RunExecuteWithMultiPollingInterval(secondExpectedNames, product, executor, testScanBlobLimitPerPoll / 2);
// this time, only expect the second container to have updated its scanInfo
Assert.Equal(1, testScanInfoManager.UpdateCounts[accountName][ContainerName]);
Assert.Equal(1, testScanInfoManager.UpdateCounts[accountName][SecondContainerName]);
}
public async Task ExecuteAsync_UpdatesScanInfo_WithEarliestFailure()
{
int testScanBlobLimitPerPoll = 6;
// we'll introduce multiple errors to make sure we take the earliest timestamp
DateTime earliestErrorTime = DateTime.UtcNow;
var container = blobContainerMock.Object;
TestBlobScanInfoManager testScanInfoManager = new TestBlobScanInfoManager();
string accountName = AccountName;
testScanInfoManager.SetScanInfo(accountName, ContainerName, DateTime.MinValue);
IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(testScanInfoManager, _exceptionHandler, NullLogger <BlobListener> .Instance);
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
typeof(ScanBlobScanLogHybridPollingStrategy)
.GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic)
.SetValue(product, testScanBlobLimitPerPoll);
await product.RegisterAsync(blobClientMock.Object, container, executor, CancellationToken.None);
// Induce a failure to make sure the timestamp is earlier than the failure.
var expectedNames = new List <string>();
for (int i = 0; i < 7; i++)
{
string name;
if (i % 3 == 0)
{
name = CreateBlobAndUploadToContainer(blobContainerMock, blobItems, "throw", earliestErrorTime.AddMinutes(i));
}
else
{
name = CreateBlobAndUploadToContainer(blobContainerMock, blobItems, "test", earliestErrorTime.AddMinutes(10));
}
expectedNames.Add(name);
}
RunExecuteWithMultiPollingInterval(expectedNames, product, executor, testScanBlobLimitPerPoll);
DateTime?storedTime = await testScanInfoManager.LoadLatestScanAsync(accountName, ContainerName);
Assert.True(storedTime < earliestErrorTime);
Assert.Equal(1, testScanInfoManager.UpdateCounts[accountName][ContainerName]);
blobContainerMock.Verify(x => x.GetBlobsAsync(It.IsAny <BlobTraits>(), It.IsAny <BlobStates>(), It.IsAny <string>(), It.IsAny <CancellationToken>()),
Times.Exactly(2));
}
public async Task BlobPolling_IncludesPreviousBatch()
{
// Blob timestamps are rounded to the nearest second, so make sure we continue to poll
// the previous second to catch any blobs that came in slightly after our previous attempt.
int testScanBlobLimitPerPoll = 3;
string containerName = Path.GetRandomFileName();
var account = CreateFakeStorageAccount();
var client = account.CreateCloudBlobClient();
// Strip off milliseconds.
var now = DateTimeOffset.UtcNow;
now = new DateTimeOffset(now.Year, now.Month, now.Day, now.Hour, now.Minute, now.Second, now.Offset);
var container = new SkewableFakeStorageBlobContainer(containerName, client as FakeStorageBlobClient,
(blobs) =>
{
foreach (ICloudBlob blob in blobs.Results)
{
blob.Properties.SetLastModified(now);
}
});
IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager());
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
typeof(ScanBlobScanLogHybridPollingStrategy)
.GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic)
.SetValue(product, testScanBlobLimitPerPoll);
product.Register(container, executor);
product.Start();
List <string> expectedNames = new List <string>();
expectedNames.Add(await CreateBlobAndUploadToContainer(container));
RunExecuterWithExpectedBlobs(expectedNames, product, executor);
expectedNames.Add(await CreateBlobAndUploadToContainer(container));
// We should see the new item. We'll see 2 blobs, but only process 1 (due to receipt).
RunExecuterWithExpectedBlobs(expectedNames, product, executor, 1);
Assert.Equal(2, container.CallCount);
Assert.Equal(expectedNames, executor.BlobReceipts);
}
public async Task TestBlobListenerWithMultipleContainers()
{
int testScanBlobLimitPerPoll = 6, containerCount = 2;
string firstContainerName = Path.GetRandomFileName();
string secondContainerName = Path.GetRandomFileName();
var account = CreateFakeStorageAccount();
var firstContainer = account.CreateCloudBlobClient().GetContainerReference(firstContainerName);
var secondContainer = account.CreateCloudBlobClient().GetContainerReference(secondContainerName);
IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager());
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
typeof(ScanBlobScanLogHybridPollingStrategy)
.GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic)
.SetValue(product, testScanBlobLimitPerPoll);
product.Register(firstContainer, executor);
product.Register(secondContainer, executor);
product.Start();
// populate first container with 5 blobs > page size and second with 2 blobs < page size
// page size is going to be testScanBlobLimitPerPoll / number of container 6/2 = 3
List <string> firstContainerExpectedNames = new List <string>();
for (int i = 0; i < 5; i++)
{
firstContainerExpectedNames.Add(await CreateBlobAndUploadToContainer(firstContainer));
}
RunExecuteWithMultiPollingInterval(firstContainerExpectedNames, product, executor, testScanBlobLimitPerPoll / containerCount);
Thread.Sleep(10);
List <string> secondContainerExpectedNames = new List <string>();
for (int i = 0; i < 2; i++)
{
secondContainerExpectedNames.Add(await CreateBlobAndUploadToContainer(secondContainer));
}
RunExecuteWithMultiPollingInterval(secondContainerExpectedNames, product, executor, testScanBlobLimitPerPoll / containerCount);
// Now run again; shouldn't show up.
RunExecuterWithExpectedBlobs(new List <string>(), product, executor);
}
public async Task BlobPolling_IgnoresClockSkew()
{
int testScanBlobLimitPerPoll = 3;
string containerName = Path.GetRandomFileName();
var account = CreateFakeStorageAccount();
var client = account.CreateCloudBlobClient();
var now = DateTimeOffset.UtcNow;
var timeMap = new Dictionary <string, DateTimeOffset>();
var container = new SkewableFakeStorageBlobContainer(containerName, client as FakeStorageBlobClient,
(blobs) =>
{
// Simulate some extreme clock skew -- the first one's LastUpdated
// wll be 60 seconds ago and the second will be 59 seconds ago.
foreach (ICloudBlob blob in blobs.Results)
{
blob.Properties.SetLastModified(timeMap[blob.Name]);
}
});
IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager());
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
typeof(ScanBlobScanLogHybridPollingStrategy)
.GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic)
.SetValue(product, testScanBlobLimitPerPoll);
product.Register(container, executor);
product.Start();
List <string> expectedNames = new List <string>();
expectedNames.Add(await CreateBlobAndUploadToContainer(container));
timeMap[expectedNames.Single()] = now.AddSeconds(-60);
RunExecuterWithExpectedBlobs(expectedNames, product, executor);
expectedNames.Clear();
expectedNames.Add(await CreateBlobAndUploadToContainer(container));
timeMap[expectedNames.Single()] = now.AddSeconds(-59);
// We should see the new item.
RunExecuterWithExpectedBlobs(expectedNames, product, executor);
Assert.Equal(2, container.CallCount);
}
public void TestBlobListenerWithMultipleContainers()
{
int testScanBlobLimitPerPoll = 6, containerCount = 2;
var firstContainer = blobContainerMock.Object;
var secondContainer = secondBlobContainerMock.Object;
IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager(), _exceptionHandler, NullLogger <BlobListener> .Instance);
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
typeof(ScanBlobScanLogHybridPollingStrategy)
.GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic)
.SetValue(product, testScanBlobLimitPerPoll);
product.Register(blobClientMock.Object, firstContainer, executor);
product.Register(blobClientMock.Object, secondContainer, executor);
product.Start();
// populate first container with 5 blobs > page size and second with 2 blobs < page size
// page size is going to be testScanBlobLimitPerPoll / number of container 6/2 = 3
List <string> firstContainerExpectedNames = new List <string>();
for (int i = 0; i < 5; i++)
{
firstContainerExpectedNames.Add(CreateBlobAndUploadToContainer(blobContainerMock, blobItems));
}
RunExecuteWithMultiPollingInterval(firstContainerExpectedNames, product, executor, testScanBlobLimitPerPoll / containerCount);
Thread.Sleep(10);
List <string> secondContainerExpectedNames = new List <string>();
for (int i = 0; i < 2; i++)
{
secondContainerExpectedNames.Add(CreateBlobAndUploadToContainer(secondBlobContainerMock, secondBlobItems));
}
RunExecuteWithMultiPollingInterval(secondContainerExpectedNames, product, executor, testScanBlobLimitPerPoll / containerCount);
// Now run again; shouldn't show up.
RunExecuterWithExpectedBlobs(new List <string>(), product, executor);
}
public void BlobPolling_IncludesPreviousBatch()
{
// Blob timestamps are rounded to the nearest second, so make sure we continue to poll
// the previous second to catch any blobs that came in slightly after our previous attempt.
int testScanBlobLimitPerPoll = 3;
string containerName = Path.GetRandomFileName();
// Strip off milliseconds.
var now = DateTimeOffset.UtcNow;
now = new DateTimeOffset(now.Year, now.Month, now.Day, now.Hour, now.Minute, now.Second, now.Offset);
var container = blobContainerMock.Object;
IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager(), _exceptionHandler, NullLogger <BlobListener> .Instance);
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
typeof(ScanBlobScanLogHybridPollingStrategy)
.GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic)
.SetValue(product, testScanBlobLimitPerPoll);
product.Register(blobClientMock.Object, container, executor);
product.Start();
List <string> expectedNames = new List <string>();
expectedNames.Add(CreateBlobAndUploadToContainer(blobContainerMock, blobItems, lastModified: now));
RunExecuterWithExpectedBlobs(expectedNames, product, executor);
expectedNames.Add(CreateBlobAndUploadToContainer(blobContainerMock, blobItems, lastModified: now));
// We should see the new item. We'll see 2 blobs, but only process 1 (due to receipt).
RunExecuterWithExpectedBlobs(expectedNames, product, executor, 1);
blobContainerMock.Verify(x => x.GetBlobsAsync(It.IsAny <BlobTraits>(), It.IsAny <BlobStates>(), It.IsAny <string>(), It.IsAny <CancellationToken>()),
Times.Exactly(2));
Assert.Equal(expectedNames, executor.BlobReceipts);
}
public async Task ScanBlobScanLogHybridPollingStrategyTestBlobListener()
{
string containerName = Path.GetRandomFileName();
var account = CreateFakeStorageAccount();
var container = account.CreateCloudBlobClient().GetContainerReference(containerName);
IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager());
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
product.Register(container, executor);
product.Start();
RunExecuterWithExpectedBlobs(new List <string>(), product, executor);
string expectedBlobName = await CreateBlobAndUploadToContainer(container);
RunExecuterWithExpectedBlobs(new List <string>()
{
expectedBlobName
}, product, executor);
// Now run again; shouldn't show up.
RunExecuterWithExpectedBlobs(new List <string>(), product, executor);
}