public void BlobPolling_IgnoresClockSkew()
        {
            int testScanBlobLimitPerPoll = 3;
            var now       = DateTimeOffset.UtcNow;
            var timeMap   = new Dictionary <string, DateTimeOffset>();
            var container = blobContainerMock.Object;
            IBlobListenerStrategy     product  = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager(), _exceptionHandler, NullLogger <BlobListener> .Instance);
            LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();

            typeof(ScanBlobScanLogHybridPollingStrategy)
            .GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic)
            .SetValue(product, testScanBlobLimitPerPoll);

            product.Register(blobClientMock.Object, container, executor);
            product.Start();

            List <string> expectedNames = new List <string>();

            expectedNames.Add(CreateBlobAndUploadToContainer(blobContainerMock, blobItems));
            timeMap[expectedNames.Single()] = now.AddSeconds(-60);
            RunExecuterWithExpectedBlobs(expectedNames, product, executor);

            expectedNames.Add(CreateBlobAndUploadToContainer(blobContainerMock, blobItems));
            timeMap[expectedNames.Last()] = now.AddSeconds(-59);

            // We should see the new item. We'll see 2 blobs, but only process 1 (due to receipt).
            RunExecuterWithExpectedBlobs(expectedNames, product, executor, 1);

            blobContainerMock.Verify(x => x.GetBlobsAsync(It.IsAny <BlobTraits>(), It.IsAny <BlobStates>(), It.IsAny <string>(), It.IsAny <CancellationToken>()),
                                     Times.Exactly(2));
            Assert.Equal(expectedNames, executor.BlobReceipts);
        }
        public async Task TestBlobListenerWithContainerBiggerThanThreshold()
        {
            int    testScanBlobLimitPerPoll = 1;
            string containerName            = Path.GetRandomFileName();
            var    account   = CreateFakeStorageAccount();
            var    container = account.CreateCloudBlobClient().GetContainerReference(containerName);
            IBlobListenerStrategy     product  = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager());
            LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();

            typeof(ScanBlobScanLogHybridPollingStrategy)
            .GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic)
            .SetValue(product, testScanBlobLimitPerPoll);

            product.Register(container, executor);
            product.Start();

            // populate with 5 blobs
            List <string> expectedNames = new List <string>();

            for (int i = 0; i < 5; i++)
            {
                expectedNames.Add(await CreateBlobAndUploadToContainer(container));
            }

            RunExecuteWithMultiPollingInterval(expectedNames, product, executor, testScanBlobLimitPerPoll);

            // Now run again; shouldn't show up.
            RunExecuterWithExpectedBlobs(new List <string>(), product, executor);
        }
        public void TestBlobListenerWithContainerBiggerThanThreshold()
        {
            int testScanBlobLimitPerPoll = 1;
            var container = blobContainerMock.Object;
            IBlobListenerStrategy     product  = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager(), _exceptionHandler, NullLogger <BlobListener> .Instance);
            LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();

            typeof(ScanBlobScanLogHybridPollingStrategy)
            .GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic)
            .SetValue(product, testScanBlobLimitPerPoll);

            product.Register(blobClientMock.Object, container, executor);
            product.Start();

            // populate with 5 blobs
            List <string> expectedNames = new List <string>();

            for (int i = 0; i < 5; i++)
            {
                expectedNames.Add(CreateBlobAndUploadToContainer(blobContainerMock, blobItems));
            }

            RunExecuteWithMultiPollingInterval(expectedNames, product, executor, testScanBlobLimitPerPoll);

            // Now run again; shouldn't show up.
            RunExecuterWithExpectedBlobs(new List <string>(), product, executor);
        }
        public void ScanBlobScanLogHybridPollingStrategyTestBlobListener()
        {
            var container = blobContainerMock.Object;
            IBlobListenerStrategy     product  = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager(), _exceptionHandler, _logger);
            LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();

            product.Register(blobClientMock.Object, container, executor);
            product.Start();

            RunExecuterWithExpectedBlobs(new List <string>(), product, executor);

            string expectedBlobName = CreateBlobAndUploadToContainer(blobContainerMock, blobItems);

            RunExecuterWithExpectedBlobs(new List <string>()
            {
                expectedBlobName
            }, product, executor);

            // Now run again; shouldn't show up.
            RunExecuterWithExpectedBlobs(new List <string>(), product, executor);

            // Verify happy-path logging.
            var logMessages = _loggerProvider.GetAllLogMessages().ToArray();

            Assert.Equal(4, logMessages.Length);

            // 1 initialization log
            var initLog = logMessages.Single(m => m.EventId.Name == "InitializedScanInfo");

            Assert.Equal(Microsoft.Extensions.Logging.LogLevel.Debug, initLog.Level);
            Assert.Equal(3, initLog.State.Count());
            Assert.Equal(ContainerName, initLog.GetStateValue <string>("containerName"));
            Assert.True(!string.IsNullOrWhiteSpace(initLog.GetStateValue <string>("latestScanInfo")));
            Assert.True(!string.IsNullOrWhiteSpace(initLog.GetStateValue <string>("{OriginalFormat}")));

            // 3 polling logs
            var pollLogs = logMessages.Where(m => m.EventId.Name == "PollBlobContainer").ToArray();

            Assert.Equal(3, pollLogs.Length);

            void ValidatePollingLog(LogMessage pollingLog, int expectedBlobCount)
            {
                Assert.Equal(Microsoft.Extensions.Logging.LogLevel.Debug, pollingLog.Level);
                Assert.Equal(7, pollingLog.State.Count());
                Assert.Equal(ContainerName, pollingLog.GetStateValue <string>("containerName"));
                Assert.Equal(expectedBlobCount, pollingLog.GetStateValue <int>("blobCount"));
                Assert.True(!string.IsNullOrWhiteSpace(pollingLog.GetStateValue <string>("pollMinimumTime")));
                Assert.True(!string.IsNullOrWhiteSpace(pollingLog.GetStateValue <string>("clientRequestId")));
                Assert.True(pollingLog.GetStateValue <long>("pollLatency") >= 0);
                Assert.False(pollingLog.GetStateValue <bool>("hasContinuationToken"));
                Assert.True(!string.IsNullOrWhiteSpace(pollingLog.GetStateValue <string>("{OriginalFormat}")));
            }

            ValidatePollingLog(pollLogs[0], 0);
            ValidatePollingLog(pollLogs[1], 1);
            ValidatePollingLog(pollLogs[2], 1);
        }
コード例 #5
0
        public async Task BlobPolling_IncludesPreviousBatch()
        {
            // Blob timestamps are rounded to the nearest second, so make sure we continue to poll
            // the previous second to catch any blobs that came in slightly after our previous attempt.
            int    testScanBlobLimitPerPoll = 3;
            string containerName            = Path.GetRandomFileName();
            var    account = CreateFakeStorageAccount();
            var    client  = account.CreateCloudBlobClient();

            // Strip off milliseconds.
            var now = DateTimeOffset.UtcNow;

            now = new DateTimeOffset(now.Year, now.Month, now.Day, now.Hour, now.Minute, now.Second, now.Offset);

            var container = new SkewableFakeStorageBlobContainer(containerName, client as FakeStorageBlobClient,
                                                                 (blobs) =>
            {
                foreach (ICloudBlob blob in blobs.Results)
                {
                    blob.Properties.SetLastModified(now);
                }
            });

            IBlobListenerStrategy     product  = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager());
            LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();

            typeof(ScanBlobScanLogHybridPollingStrategy)
            .GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic)
            .SetValue(product, testScanBlobLimitPerPoll);

            product.Register(container, executor);
            product.Start();

            List <string> expectedNames = new List <string>();

            expectedNames.Add(await CreateBlobAndUploadToContainer(container));

            RunExecuterWithExpectedBlobs(expectedNames, product, executor);

            expectedNames.Add(await CreateBlobAndUploadToContainer(container));

            // We should see the new item. We'll see 2 blobs, but only process 1 (due to receipt).
            RunExecuterWithExpectedBlobs(expectedNames, product, executor, 1);

            Assert.Equal(2, container.CallCount);
            Assert.Equal(expectedNames, executor.BlobReceipts);
        }
        public async Task TestBlobListenerWithMultipleContainers()
        {
            int    testScanBlobLimitPerPoll = 6, containerCount = 2;
            string firstContainerName  = Path.GetRandomFileName();
            string secondContainerName = Path.GetRandomFileName();
            var    account                     = CreateFakeStorageAccount();
            var    firstContainer              = account.CreateCloudBlobClient().GetContainerReference(firstContainerName);
            var    secondContainer             = account.CreateCloudBlobClient().GetContainerReference(secondContainerName);
            IBlobListenerStrategy     product  = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager());
            LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();

            typeof(ScanBlobScanLogHybridPollingStrategy)
            .GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic)
            .SetValue(product, testScanBlobLimitPerPoll);

            product.Register(firstContainer, executor);
            product.Register(secondContainer, executor);
            product.Start();

            // populate first container with 5 blobs > page size and second with 2 blobs < page size
            // page size is going to be testScanBlobLimitPerPoll / number of container 6/2 = 3
            List <string> firstContainerExpectedNames = new List <string>();

            for (int i = 0; i < 5; i++)
            {
                firstContainerExpectedNames.Add(await CreateBlobAndUploadToContainer(firstContainer));
            }

            RunExecuteWithMultiPollingInterval(firstContainerExpectedNames, product, executor, testScanBlobLimitPerPoll / containerCount);

            Thread.Sleep(10);

            List <string> secondContainerExpectedNames = new List <string>();

            for (int i = 0; i < 2; i++)
            {
                secondContainerExpectedNames.Add(await CreateBlobAndUploadToContainer(secondContainer));
            }

            RunExecuteWithMultiPollingInterval(secondContainerExpectedNames, product, executor, testScanBlobLimitPerPoll / containerCount);

            // Now run again; shouldn't show up.
            RunExecuterWithExpectedBlobs(new List <string>(), product, executor);
        }
        public async Task BlobPolling_IgnoresClockSkew()
        {
            int    testScanBlobLimitPerPoll = 3;
            string containerName            = Path.GetRandomFileName();
            var    account   = CreateFakeStorageAccount();
            var    client    = account.CreateCloudBlobClient();
            var    now       = DateTimeOffset.UtcNow;
            var    timeMap   = new Dictionary <string, DateTimeOffset>();
            var    container = new SkewableFakeStorageBlobContainer(containerName, client as FakeStorageBlobClient,
                                                                    (blobs) =>
            {
                // Simulate some extreme clock skew -- the first one's LastUpdated
                // wll be 60 seconds ago and the second will be 59 seconds ago.
                foreach (ICloudBlob blob in blobs.Results)
                {
                    blob.Properties.SetLastModified(timeMap[blob.Name]);
                }
            });
            IBlobListenerStrategy     product  = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager());
            LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();

            typeof(ScanBlobScanLogHybridPollingStrategy)
            .GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic)
            .SetValue(product, testScanBlobLimitPerPoll);

            product.Register(container, executor);
            product.Start();

            List <string> expectedNames = new List <string>();

            expectedNames.Add(await CreateBlobAndUploadToContainer(container));
            timeMap[expectedNames.Single()] = now.AddSeconds(-60);
            RunExecuterWithExpectedBlobs(expectedNames, product, executor);

            expectedNames.Clear();

            expectedNames.Add(await CreateBlobAndUploadToContainer(container));
            timeMap[expectedNames.Single()] = now.AddSeconds(-59);

            // We should see the new item.
            RunExecuterWithExpectedBlobs(expectedNames, product, executor);

            Assert.Equal(2, container.CallCount);
        }
        public void TestBlobListenerWithMultipleContainers()
        {
            int testScanBlobLimitPerPoll = 6, containerCount = 2;
            var firstContainer                 = blobContainerMock.Object;
            var secondContainer                = secondBlobContainerMock.Object;
            IBlobListenerStrategy     product  = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager(), _exceptionHandler, NullLogger <BlobListener> .Instance);
            LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();

            typeof(ScanBlobScanLogHybridPollingStrategy)
            .GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic)
            .SetValue(product, testScanBlobLimitPerPoll);

            product.Register(blobClientMock.Object, firstContainer, executor);
            product.Register(blobClientMock.Object, secondContainer, executor);
            product.Start();

            // populate first container with 5 blobs > page size and second with 2 blobs < page size
            // page size is going to be testScanBlobLimitPerPoll / number of container 6/2 = 3
            List <string> firstContainerExpectedNames = new List <string>();

            for (int i = 0; i < 5; i++)
            {
                firstContainerExpectedNames.Add(CreateBlobAndUploadToContainer(blobContainerMock, blobItems));
            }

            RunExecuteWithMultiPollingInterval(firstContainerExpectedNames, product, executor, testScanBlobLimitPerPoll / containerCount);

            Thread.Sleep(10);

            List <string> secondContainerExpectedNames = new List <string>();

            for (int i = 0; i < 2; i++)
            {
                secondContainerExpectedNames.Add(CreateBlobAndUploadToContainer(secondBlobContainerMock, secondBlobItems));
            }

            RunExecuteWithMultiPollingInterval(secondContainerExpectedNames, product, executor, testScanBlobLimitPerPoll / containerCount);

            // Now run again; shouldn't show up.
            RunExecuterWithExpectedBlobs(new List <string>(), product, executor);
        }
        public void BlobPolling_IncludesPreviousBatch()
        {
            // Blob timestamps are rounded to the nearest second, so make sure we continue to poll
            // the previous second to catch any blobs that came in slightly after our previous attempt.
            int    testScanBlobLimitPerPoll = 3;
            string containerName            = Path.GetRandomFileName();

            // Strip off milliseconds.
            var now = DateTimeOffset.UtcNow;

            now = new DateTimeOffset(now.Year, now.Month, now.Day, now.Hour, now.Minute, now.Second, now.Offset);

            var container = blobContainerMock.Object;

            IBlobListenerStrategy     product  = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager(), _exceptionHandler, NullLogger <BlobListener> .Instance);
            LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();

            typeof(ScanBlobScanLogHybridPollingStrategy)
            .GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic)
            .SetValue(product, testScanBlobLimitPerPoll);

            product.Register(blobClientMock.Object, container, executor);
            product.Start();

            List <string> expectedNames = new List <string>();

            expectedNames.Add(CreateBlobAndUploadToContainer(blobContainerMock, blobItems, lastModified: now));

            RunExecuterWithExpectedBlobs(expectedNames, product, executor);

            expectedNames.Add(CreateBlobAndUploadToContainer(blobContainerMock, blobItems, lastModified: now));

            // We should see the new item. We'll see 2 blobs, but only process 1 (due to receipt).
            RunExecuterWithExpectedBlobs(expectedNames, product, executor, 1);

            blobContainerMock.Verify(x => x.GetBlobsAsync(It.IsAny <BlobTraits>(), It.IsAny <BlobStates>(), It.IsAny <string>(), It.IsAny <CancellationToken>()),
                                     Times.Exactly(2));
            Assert.Equal(expectedNames, executor.BlobReceipts);
        }
        public async Task ScanBlobScanLogHybridPollingStrategyTestBlobListener()
        {
            string containerName = Path.GetRandomFileName();
            var    account       = CreateFakeStorageAccount();
            var    container     = account.CreateCloudBlobClient().GetContainerReference(containerName);
            IBlobListenerStrategy     product  = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager());
            LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();

            product.Register(container, executor);
            product.Start();

            RunExecuterWithExpectedBlobs(new List <string>(), product, executor);

            string expectedBlobName = await CreateBlobAndUploadToContainer(container);

            RunExecuterWithExpectedBlobs(new List <string>()
            {
                expectedBlobName
            }, product, executor);

            // Now run again; shouldn't show up.
            RunExecuterWithExpectedBlobs(new List <string>(), product, executor);
        }