public async Task ExecuteAsync_UpdatesScanInfo_WithEarliestFailure() { int testScanBlobLimitPerPoll = 6; string containerName = Guid.NewGuid().ToString(); // we'll introduce multiple errors to make sure we take the earliest timestamp DateTime earliestErrorTime = DateTime.UtcNow; var timeMap = new Dictionary <string, DateTimeOffset>(); IStorageAccount account = CreateFakeStorageAccount(); IStorageBlobContainer container = new SkewableFakeStorageBlobContainer(new MemoryBlobStore(), containerName, account.CreateBlobClient(), blobs => { // Set a blob with "throw" to a specific date and time. Make sure the error blob // is earlier than the others. foreach (IStorageBlob blob in blobs.Results) { ((FakeStorageBlobProperties)blob.Properties).LastModified = timeMap[blob.Name]; } }); TestBlobScanInfoManager testScanInfoManager = new TestBlobScanInfoManager(); string accountName = account.Credentials.AccountName; testScanInfoManager.SetScanInfo(accountName, containerName, DateTime.MinValue); IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(testScanInfoManager); LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor(); typeof(ScanBlobScanLogHybridPollingStrategy) .GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic) .SetValue(product, testScanBlobLimitPerPoll); await product.RegisterAsync(container, executor, CancellationToken.None); // Induce a failure to make sure the timestamp is earlier than the failure. var expectedNames = new List <string>(); for (int i = 0; i < 7; i++) { string name; if (i % 3 == 0) { name = CreateAblobAndUploadToContainer(container, "throw"); timeMap[name] = earliestErrorTime.AddMinutes(i); } else { name = CreateAblobAndUploadToContainer(container, "test"); timeMap[name] = earliestErrorTime.AddMinutes(10); } expectedNames.Add(name); } RunExecuteWithMultiPollingInterval(expectedNames, product, executor, testScanBlobLimitPerPoll); DateTime?storedTime = await testScanInfoManager.LoadLatestScanAsync(accountName, containerName); Assert.True(storedTime < earliestErrorTime); Assert.Equal(1, testScanInfoManager.UpdateCounts[accountName][containerName]); }
public async Task BlobPolling_IncludesPreviousBatch() { // Blob timestamps are rounded to the nearest second, so make sure we continue to poll // the previous second to catch any blobs that came in slightly after our previous attempt. int testScanBlobLimitPerPoll = 3; string containerName = Path.GetRandomFileName(); var account = CreateFakeStorageAccount(); var client = account.CreateCloudBlobClient(); // Strip off milliseconds. var now = DateTimeOffset.UtcNow; now = new DateTimeOffset(now.Year, now.Month, now.Day, now.Hour, now.Minute, now.Second, now.Offset); var container = new SkewableFakeStorageBlobContainer(containerName, client as FakeStorageBlobClient, (blobs) => { foreach (ICloudBlob blob in blobs.Results) { blob.Properties.SetLastModified(now); } }); IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager()); LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor(); typeof(ScanBlobScanLogHybridPollingStrategy) .GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic) .SetValue(product, testScanBlobLimitPerPoll); product.Register(container, executor); product.Start(); List <string> expectedNames = new List <string>(); expectedNames.Add(await CreateBlobAndUploadToContainer(container)); RunExecuterWithExpectedBlobs(expectedNames, product, executor); expectedNames.Add(await CreateBlobAndUploadToContainer(container)); // We should see the new item. We'll see 2 blobs, but only process 1 (due to receipt). RunExecuterWithExpectedBlobs(expectedNames, product, executor, 1); Assert.Equal(2, container.CallCount); Assert.Equal(expectedNames, executor.BlobReceipts); }
public async Task BlobPolling_IgnoresClockSkew() { int testScanBlobLimitPerPoll = 3; string containerName = Path.GetRandomFileName(); var account = CreateFakeStorageAccount(); var client = account.CreateCloudBlobClient(); var now = DateTimeOffset.UtcNow; var timeMap = new Dictionary <string, DateTimeOffset>(); var container = new SkewableFakeStorageBlobContainer(containerName, client as FakeStorageBlobClient, (blobs) => { // Simulate some extreme clock skew -- the first one's LastUpdated // wll be 60 seconds ago and the second will be 59 seconds ago. foreach (ICloudBlob blob in blobs.Results) { blob.Properties.SetLastModified(timeMap[blob.Name]); } }); IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager()); LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor(); typeof(ScanBlobScanLogHybridPollingStrategy) .GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic) .SetValue(product, testScanBlobLimitPerPoll); product.Register(container, executor); product.Start(); List <string> expectedNames = new List <string>(); expectedNames.Add(await CreateBlobAndUploadToContainer(container)); timeMap[expectedNames.Single()] = now.AddSeconds(-60); RunExecuterWithExpectedBlobs(expectedNames, product, executor); expectedNames.Clear(); expectedNames.Add(await CreateBlobAndUploadToContainer(container)); timeMap[expectedNames.Single()] = now.AddSeconds(-59); // We should see the new item. RunExecuterWithExpectedBlobs(expectedNames, product, executor); Assert.Equal(2, container.CallCount); }