public async Task ExecuteAsync_UpdatesScanInfo_WithEarliestFailure()
{
int testScanBlobLimitPerPoll = 6;
string containerName = Guid.NewGuid().ToString();
// we'll introduce multiple errors to make sure we take the earliest timestamp
DateTime earliestErrorTime = DateTime.UtcNow;
var timeMap = new Dictionary <string, DateTimeOffset>();
IStorageAccount account = CreateFakeStorageAccount();
IStorageBlobContainer container = new SkewableFakeStorageBlobContainer(new MemoryBlobStore(), containerName,
account.CreateBlobClient(), blobs =>
{
// Set a blob with "throw" to a specific date and time. Make sure the error blob
// is earlier than the others.
foreach (IStorageBlob blob in blobs.Results)
{
((FakeStorageBlobProperties)blob.Properties).LastModified = timeMap[blob.Name];
}
});
TestBlobScanInfoManager testScanInfoManager = new TestBlobScanInfoManager();
string accountName = account.Credentials.AccountName;
testScanInfoManager.SetScanInfo(accountName, containerName, DateTime.MinValue);
IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(testScanInfoManager);
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
typeof(ScanBlobScanLogHybridPollingStrategy)
.GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic)
.SetValue(product, testScanBlobLimitPerPoll);
await product.RegisterAsync(container, executor, CancellationToken.None);
// Induce a failure to make sure the timestamp is earlier than the failure.
var expectedNames = new List <string>();
for (int i = 0; i < 7; i++)
{
string name;
if (i % 3 == 0)
{
name = CreateAblobAndUploadToContainer(container, "throw");
timeMap[name] = earliestErrorTime.AddMinutes(i);
}
else
{
name = CreateAblobAndUploadToContainer(container, "test");
timeMap[name] = earliestErrorTime.AddMinutes(10);
}
expectedNames.Add(name);
}
RunExecuteWithMultiPollingInterval(expectedNames, product, executor, testScanBlobLimitPerPoll);
DateTime?storedTime = await testScanInfoManager.LoadLatestScanAsync(accountName, containerName);
Assert.True(storedTime < earliestErrorTime);
Assert.Equal(1, testScanInfoManager.UpdateCounts[accountName][containerName]);
}
public async Task BlobPolling_IncludesPreviousBatch()
{
// Blob timestamps are rounded to the nearest second, so make sure we continue to poll
// the previous second to catch any blobs that came in slightly after our previous attempt.
int testScanBlobLimitPerPoll = 3;
string containerName = Path.GetRandomFileName();
var account = CreateFakeStorageAccount();
var client = account.CreateCloudBlobClient();
// Strip off milliseconds.
var now = DateTimeOffset.UtcNow;
now = new DateTimeOffset(now.Year, now.Month, now.Day, now.Hour, now.Minute, now.Second, now.Offset);
var container = new SkewableFakeStorageBlobContainer(containerName, client as FakeStorageBlobClient,
(blobs) =>
{
foreach (ICloudBlob blob in blobs.Results)
{
blob.Properties.SetLastModified(now);
}
});
IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager());
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
typeof(ScanBlobScanLogHybridPollingStrategy)
.GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic)
.SetValue(product, testScanBlobLimitPerPoll);
product.Register(container, executor);
product.Start();
List <string> expectedNames = new List <string>();
expectedNames.Add(await CreateBlobAndUploadToContainer(container));
RunExecuterWithExpectedBlobs(expectedNames, product, executor);
expectedNames.Add(await CreateBlobAndUploadToContainer(container));
// We should see the new item. We'll see 2 blobs, but only process 1 (due to receipt).
RunExecuterWithExpectedBlobs(expectedNames, product, executor, 1);
Assert.Equal(2, container.CallCount);
Assert.Equal(expectedNames, executor.BlobReceipts);
}
public async Task BlobPolling_IgnoresClockSkew()
{
int testScanBlobLimitPerPoll = 3;
string containerName = Path.GetRandomFileName();
var account = CreateFakeStorageAccount();
var client = account.CreateCloudBlobClient();
var now = DateTimeOffset.UtcNow;
var timeMap = new Dictionary <string, DateTimeOffset>();
var container = new SkewableFakeStorageBlobContainer(containerName, client as FakeStorageBlobClient,
(blobs) =>
{
// Simulate some extreme clock skew -- the first one's LastUpdated
// wll be 60 seconds ago and the second will be 59 seconds ago.
foreach (ICloudBlob blob in blobs.Results)
{
blob.Properties.SetLastModified(timeMap[blob.Name]);
}
});
IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager());
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
typeof(ScanBlobScanLogHybridPollingStrategy)
.GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic)
.SetValue(product, testScanBlobLimitPerPoll);
product.Register(container, executor);
product.Start();
List <string> expectedNames = new List <string>();
expectedNames.Add(await CreateBlobAndUploadToContainer(container));
timeMap[expectedNames.Single()] = now.AddSeconds(-60);
RunExecuterWithExpectedBlobs(expectedNames, product, executor);
expectedNames.Clear();
expectedNames.Add(await CreateBlobAndUploadToContainer(container));
timeMap[expectedNames.Single()] = now.AddSeconds(-59);
// We should see the new item.
RunExecuterWithExpectedBlobs(expectedNames, product, executor);
Assert.Equal(2, container.CallCount);
}
