public IEnumerable <IStorageBlob> ListBlobs(MemoryBlobStore store, IStorageBlobContainer parent,
BlobListingDetails blobListingDetails)
{
if (blobListingDetails != BlobListingDetails.None && blobListingDetails != BlobListingDetails.Metadata)
{
throw new NotImplementedException();
}
List <IStorageBlob> results = new List <IStorageBlob>();
foreach (KeyValuePair <string, Blob> item in _items)
{
string blobName = item.Key;
IStorageBlob blob = new FakeStorageBlockBlob(store, blobName, parent);
if ((blobListingDetails | BlobListingDetails.Metadata) == BlobListingDetails.Metadata)
{
Blob storeBlob = item.Value;
IReadOnlyDictionary <string, string> storeMetadata = storeBlob.Metadata;
foreach (KeyValuePair <string, string> pair in storeMetadata)
{
blob.Metadata.Add(pair.Key, pair.Value);
}
}
results.Add(blob);
}
return(results);
}
public Task <IStorageBlob> ConvertAsync(string input, CancellationToken cancellationToken)
{
BlobPath path = BlobPath.ParseAndValidate(input);
IStorageBlobContainer container = _client.GetContainerReference(path.ContainerName);
return(container.GetBlobReferenceFromServerAsync(path.BlobName, cancellationToken));
}
public void BlobTrigger_IfBindingAlwaysFails_MovesToPoisonQueue()
{
// Arrange
IStorageAccount account = CreateFakeStorageAccount();
IStorageBlobContainer container = CreateContainer(account, ContainerName);
IStorageBlockBlob blob = container.GetBlockBlobReference(BlobName);
CloudBlockBlob expectedBlob = blob.SdkObject;
blob.UploadText("ignore");
// Act
string result = RunTrigger <string>(account, typeof(PoisonBlobProgram),
(s) => PoisonBlobProgram.TaskSource = s,
new string[] { typeof(PoisonBlobProgram).FullName + ".PutInPoisonQueue" });
// Assert
BlobTriggerMessage message = JsonConvert.DeserializeObject <BlobTriggerMessage>(result);
Assert.NotNull(message);
Assert.Equal(typeof(PoisonBlobProgram).FullName + ".PutInPoisonQueue", message.FunctionId);
Assert.Equal(StorageBlobType.BlockBlob, message.BlobType);
Assert.Equal(ContainerName, message.ContainerName);
Assert.Equal(BlobName, message.BlobName);
Assert.NotEmpty(message.ETag);
}
public Task RegisterAsync(IStorageBlobContainer container, ITriggerExecutor<IStorageBlob> triggerExecutor,
CancellationToken cancellationToken)
{
// Register and Execute are not concurrency-safe.
// Avoiding calling Register while Execute is running is the caller's responsibility.
ICollection<ITriggerExecutor<IStorageBlob>> containerRegistrations;
if (_registrations.ContainsKey(container))
{
containerRegistrations = _registrations[container];
}
else
{
containerRegistrations = new List<ITriggerExecutor<IStorageBlob>>();
_registrations.Add(container, containerRegistrations);
}
containerRegistrations.Add(triggerExecutor);
if (!_lastModifiedTimestamps.ContainsKey(container))
{
_lastModifiedTimestamps.Add(container, DateTime.MinValue);
}
return Task.FromResult(0);
}
private async Task NotifyRegistrationsAsync(IStorageBlob blob, ICollection <IStorageBlob> failedNotifications,
CancellationToken cancellationToken)
{
IStorageBlobContainer container = blob.Container;
ContainerScanInfo containerScanInfo;
// Blob written notifications are host-wide, so filter out things that aren't in the container list.
if (!_scanInfo.TryGetValue(container, out containerScanInfo))
{
return;
}
foreach (ITriggerExecutor <IStorageBlob> registration in containerScanInfo.Registrations)
{
cancellationToken.ThrowIfCancellationRequested();
FunctionResult result = await registration.ExecuteAsync(blob, cancellationToken);
if (!result.Succeeded)
{
// If notification failed, try again on the next iteration.
failedNotifications.Add(blob);
}
}
}
/// <summary>
/// This method is called each polling interval for all containers. The method divides the
/// budget of allocated number of blobs to query, for each container we query a page of
/// that size and we keep the continuation token for the next time. AS a curser, we use
/// the time stamp when the current cycle on the container started. blobs newer than that
/// time will be considered new and registrations will be notified
/// </summary>
/// <param name="container"></param>
/// <param name="containerScanInfo"> Information that includes the last cycle start
/// the continuation token and the current cycle start for a container</param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public async Task <IEnumerable <IStorageBlob> > PollNewBlobsAsync(
IStorageBlobContainer container, ContainerScanInfo containerScanInfo, CancellationToken cancellationToken)
{
IEnumerable <IStorageListBlobItem> currentBlobs;
IStorageBlobResultSegment blobSegment;
int blobPollLimitPerContainer = _scanBlobLimitPerPoll / _scanInfo.Count;
BlobContinuationToken continuationToken = containerScanInfo.ContinuationToken;
// if starting the cycle, keep the current time stamp to be used as curser
if (continuationToken == null)
{
containerScanInfo.CurrentSweepCycleStartTime = DateTime.UtcNow;
}
try
{
blobSegment = await container.ListBlobsSegmentedAsync(prefix : null, useFlatBlobListing : true,
blobListingDetails : BlobListingDetails.None, maxResults : blobPollLimitPerContainer, currentToken : continuationToken,
options : null, operationContext : null, cancellationToken : cancellationToken);
currentBlobs = blobSegment.Results;
}
catch (StorageException exception)
{
if (exception.IsNotFound())
{
return(Enumerable.Empty <IStorageBlob>());
}
else
{
throw;
}
}
List <IStorageBlob> newBlobs = new List <IStorageBlob>();
// Type cast to IStorageBlob is safe due to useFlatBlobListing: true above.
foreach (IStorageBlob currentBlob in currentBlobs)
{
cancellationToken.ThrowIfCancellationRequested();
IStorageBlobProperties properties = currentBlob.Properties;
DateTime lastModifiedTimestamp = properties.LastModified.Value.UtcDateTime;
if (lastModifiedTimestamp > containerScanInfo.LastSweepCycleStartTime)
{
newBlobs.Add(currentBlob);
}
}
// record continuation token for next chunk retrieval
containerScanInfo.ContinuationToken = blobSegment.ContinuationToken;
// if ending a cycle then copy currentSweepCycleStartTime to lastSweepCycleStartTime
if (blobSegment.ContinuationToken == null)
{
containerScanInfo.LastSweepCycleStartTime = containerScanInfo.CurrentSweepCycleStartTime;
}
return(newBlobs);
}
public void TestBlobListenerWithContainerBiggerThanThreshold()
{
int testScanBlobLimitPerPoll = 1;
string containerName = Path.GetRandomFileName();
IStorageAccount account = CreateFakeStorageAccount();
IStorageBlobContainer container = account.CreateBlobClient().GetContainerReference(containerName);
IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager());
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
typeof(ScanBlobScanLogHybridPollingStrategy)
.GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic)
.SetValue(product, testScanBlobLimitPerPoll);
product.Register(container, executor);
product.Start();
// populate with 5 blobs
List <string> expectedNames = new List <string>();
for (int i = 0; i < 5; i++)
{
expectedNames.Add(CreateAblobAndUploadToContainer(container));
}
RunExecuteWithMultiPollingInterval(expectedNames, product, executor, testScanBlobLimitPerPoll);
// Now run again; shouldn't show up.
RunExecuterWithExpectedBlobs(new List <string>(), product, executor);
}
public async Task <IEnumerable <IStorageBlob> > GetRecentBlobWritesAsync(CancellationToken cancellationToken,
int hoursWindow = DefaultScanHoursWindow)
{
List <IStorageBlob> blobs = new List <IStorageBlob>();
var time = DateTime.UtcNow; // will scan back 2 hours, which is enough to deal with clock sqew
foreach (var blob in await ListRecentLogFilesAsync(_blobClient, time, cancellationToken, hoursWindow))
{
bool isAdded = _scannedBlobNames.Add(blob.Name);
if (!isAdded)
{
continue;
}
// Need to clear out cache.
if (_scannedBlobNames.Count > 100 * 1000)
{
_scannedBlobNames.Clear();
}
IEnumerable <StorageAnalyticsLogEntry> entries = await _parser.ParseLogAsync(blob, cancellationToken);
IEnumerable <BlobPath> filteredBlobs = GetPathsForValidBlobWrites(entries);
foreach (BlobPath path in filteredBlobs)
{
IStorageBlobContainer container = _blobClient.GetContainerReference(path.ContainerName);
blobs.Add(container.GetBlockBlobReference(path.BlobName));
}
}
return(blobs);
}
public async Task RegisterAsync(IStorageBlobContainer container, ITriggerExecutor<IStorageBlob> triggerExecutor,
CancellationToken cancellationToken)
{
ThrowIfDisposed();
// Initial background scans for all containers happen on first Execute call.
// Prevent accidental late registrations.
// (Also prevents incorrect concurrent execution of Register with Execute.)
if (_initialScanThread.ThreadState != ThreadState.Unstarted)
{
throw new InvalidOperationException("All registrations must be created before execution begins.");
}
ICollection<ITriggerExecutor<IStorageBlob>> containerRegistrations;
if (_registrations.ContainsKey(container))
{
containerRegistrations = _registrations[container];
}
else
{
containerRegistrations = new List<ITriggerExecutor<IStorageBlob>>();
_registrations.Add(container, containerRegistrations);
}
containerRegistrations.Add(triggerExecutor);
IStorageBlobClient client = container.ServiceClient;
if (!_logListeners.ContainsKey(client))
{
BlobLogListener logListener = await BlobLogListener.CreateAsync(client, cancellationToken);
_logListeners.Add(client, logListener);
}
}
public BlobListenerFactory(IHostIdProvider hostIdProvider,
IQueueConfiguration queueConfiguration,
JobHostBlobsConfiguration blobsConfiguration,
IWebJobsExceptionHandler exceptionHandler,
IContextSetter <IBlobWrittenWatcher> blobWrittenWatcherSetter,
IContextSetter <IMessageEnqueuedWatcher> messageEnqueuedWatcherSetter,
ISharedContextProvider sharedContextProvider,
ILoggerFactory loggerFactory,
string functionId,
IStorageAccount hostAccount,
IStorageAccount dataAccount,
IStorageBlobContainer container,
IBlobPathSource input,
ITriggeredFunctionExecutor executor,
SingletonManager singletonManager)
{
_hostIdProvider = hostIdProvider ?? throw new ArgumentNullException(nameof(hostIdProvider));
_queueConfiguration = queueConfiguration ?? throw new ArgumentNullException(nameof(queueConfiguration));
_blobsConfiguration = blobsConfiguration ?? throw new ArgumentNullException(nameof(blobsConfiguration));
_exceptionHandler = exceptionHandler ?? throw new ArgumentNullException(nameof(exceptionHandler));
_blobWrittenWatcherSetter = blobWrittenWatcherSetter ?? throw new ArgumentNullException(nameof(blobWrittenWatcherSetter));
_messageEnqueuedWatcherSetter = messageEnqueuedWatcherSetter ?? throw new ArgumentNullException(nameof(messageEnqueuedWatcherSetter));
_sharedContextProvider = sharedContextProvider ?? throw new ArgumentNullException(nameof(sharedContextProvider));
_loggerFactory = loggerFactory;
_functionId = functionId;
_hostAccount = hostAccount ?? throw new ArgumentNullException(nameof(hostAccount));
_dataAccount = dataAccount ?? throw new ArgumentNullException(nameof(dataAccount));
_container = container ?? throw new ArgumentNullException(nameof(container));
_input = input ?? throw new ArgumentNullException(nameof(input));
_executor = executor ?? throw new ArgumentNullException(nameof(executor));
_singletonManager = singletonManager ?? throw new ArgumentNullException(nameof(singletonManager));
}
public async Task RegisterAsync(IStorageBlobContainer container, ITriggerExecutor <IStorageBlob> triggerExecutor, CancellationToken cancellationToken)
{
// Register and Execute are not concurrency-safe.
// Avoiding calling Register while Execute is running is the caller's responsibility.
ThrowIfDisposed();
// Register all in logPolling, there is no problem if we get 2 notifications of the new blob
await _pollLogStrategy.RegisterAsync(container, triggerExecutor, cancellationToken);
ContainerScanInfo containerScanInfo;
if (!_scanInfo.TryGetValue(container, out containerScanInfo))
{
// First, try to load serialized scanInfo for this container.
DateTime?latestStoredScan = await _blobScanInfoManager.LoadLatestScanAsync(container.ServiceClient.Credentials.AccountName, container.Name);
containerScanInfo = new ContainerScanInfo()
{
Registrations = new List <ITriggerExecutor <IStorageBlob> >(),
LastSweepCycleLatestModified = latestStoredScan ?? DateTime.MinValue,
CurrentSweepCycleLatestModified = DateTime.MinValue,
ContinuationToken = null
};
_scanInfo.Add(container, containerScanInfo);
}
containerScanInfo.Registrations.Add(triggerExecutor);
}
public async Task RegisterAsync(IStorageBlobContainer container, ITriggerExecutor <IStorageBlob> triggerExecutor, CancellationToken cancellationToken)
{
// Register and Execute are not concurrency-safe.
// Avoiding calling Register while Execute is running is the caller's responsibility.
ThrowIfDisposed();
// Register all in logPolling, there is no problem if we get 2 notifications of the new blob
await _pollLogStrategy.RegisterAsync(container, triggerExecutor, cancellationToken);
ContainerScanInfo containerScanInfo;
if (!_scanInfo.TryGetValue(container, out containerScanInfo))
{
containerScanInfo = new ContainerScanInfo()
{
Registrations = new List <ITriggerExecutor <IStorageBlob> >(),
LastSweepCycleLatestModified = DateTime.MinValue,
CurrentSweepCycleLatestModified = DateTime.MinValue,
ContinuationToken = null
};
_scanInfo.Add(container, containerScanInfo);
}
containerScanInfo.Registrations.Add(triggerExecutor);
}
private async Task PollAndNotify(IStorageBlobContainer container, ContainerScanInfo containerScanInfo, CancellationToken cancellationToken, List <IStorageBlob> failedNotifications)
{
cancellationToken.ThrowIfCancellationRequested();
DateTime lastScan = containerScanInfo.LastSweepCycleLatestModified;
IEnumerable <IStorageBlob> newBlobs = await PollNewBlobsAsync(container, containerScanInfo, cancellationToken);
foreach (IStorageBlob newBlob in newBlobs)
{
cancellationToken.ThrowIfCancellationRequested();
await NotifyRegistrationsAsync(newBlob, failedNotifications, cancellationToken);
}
// if the 'LatestModified' has changed, update it in the manager
if (containerScanInfo.LastSweepCycleLatestModified > lastScan)
{
DateTime latestScan = containerScanInfo.LastSweepCycleLatestModified;
// It's possible that we had some blobs that we failed to move to the queue. We want to make sure
// we continue to find these if the host needs to restart.
if (failedNotifications.Any())
{
latestScan = failedNotifications.Min(n => n.Properties.LastModified.Value.UtcDateTime);
}
// Store our timestamp slightly earlier than the last timestamp. This is a failsafe for any blobs that created
// milliseconds after our last scan (blob timestamps round to the second). This way we make sure to pick those
// up on a host restart.
await _blobScanInfoManager.UpdateLatestScanAsync(container.ServiceClient.Credentials.AccountName,
container.Name, latestScan.AddMilliseconds(-1));
}
}
public Task RegisterAsync(IStorageBlobContainer container, ITriggerExecutor <IStorageBlob> triggerExecutor,
CancellationToken cancellationToken)
{
// Register and Execute are not concurrency-safe.
// Avoiding calling Register while Execute is running is the caller's responsibility.
ICollection <ITriggerExecutor <IStorageBlob> > containerRegistrations;
if (_registrations.ContainsKey(container))
{
containerRegistrations = _registrations[container];
}
else
{
containerRegistrations = new List <ITriggerExecutor <IStorageBlob> >();
_registrations.Add(container, containerRegistrations);
}
containerRegistrations.Add(triggerExecutor);
if (!_lastModifiedTimestamps.ContainsKey(container))
{
_lastModifiedTimestamps.Add(container, DateTime.MinValue);
}
return(Task.FromResult(0));
}
public async Task RegisterAsync_InitializesWithScanInfoManager()
{
string containerName = Guid.NewGuid().ToString();
IStorageAccount account = CreateFakeStorageAccount();
IStorageBlobContainer container = account.CreateBlobClient().GetContainerReference(containerName);
TestBlobScanInfoManager scanInfoManager = new TestBlobScanInfoManager();
IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(scanInfoManager);
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
// Create a few blobs.
for (int i = 0; i < 5; i++)
{
CreateAblobAndUploadToContainer(container);
}
await scanInfoManager.UpdateLatestScanAsync(account.Credentials.AccountName, containerName, DateTime.UtcNow);
await product.RegisterAsync(container, executor, CancellationToken.None);
// delay slightly so we guarantee a later timestamp
await Task.Delay(10);
var expectedNames = new List <string>();
expectedNames.Add(CreateAblobAndUploadToContainer(container));
RunExecuterWithExpectedBlobs(expectedNames, product, executor);
}
public async Task <IEnumerable <IStorageBlob> > GetRecentBlobWritesAsync(CancellationToken cancellationToken,
int hoursWindow = DefaultScanHoursWindow)
{
List <IStorageBlob> blobs = new List <IStorageBlob>();
var time = DateTime.UtcNow; // will scan back 2 hours, which is enough to deal with clock sqew
foreach (var blob in await ListRecentLogFilesAsync(_blobClient, time, cancellationToken, hoursWindow))
{
bool isAdded = _scannedBlobNames.Add(blob.Name);
if (!isAdded)
{
continue;
}
// Need to clear out cache.
if (_scannedBlobNames.Count > 100 * 1000)
{
_scannedBlobNames.Clear();
}
var parsedBlobPaths = from entry in await _parser.ParseLogAsync(blob, cancellationToken)
where entry.IsBlobWrite
select entry.ToBlobPath();
foreach (BlobPath path in parsedBlobPaths.Where(p => p != null))
{
IStorageBlobContainer container = _blobClient.GetContainerReference(path.ContainerName);
blobs.Add(container.GetBlockBlobReference(path.BlobName));
}
}
return(blobs);
}
private static IStorageBlobContainer CreateContainer(IStorageAccount account, string containerName)
{
IStorageBlobClient client = account.CreateBlobClient();
IStorageBlobContainer container = client.GetContainerReference(containerName);
container.CreateIfNotExists();
return(container);
}
private static IStorageBlob CreateBlobReference(string containerName, string blobName)
{
IStorageAccount account = CreateAccount();
IStorageBlobClient client = account.CreateBlobClient();
IStorageBlobContainer container = client.GetContainerReference(containerName);
return(container.GetBlockBlobReference(blobName));
}
public async Task <IValueProvider> BindAsync(BindingContext context)
{
BlobPath boundPath = _path.Bind(context.BindingData);
IStorageBlobContainer container = _client.GetContainerReference(boundPath.ContainerName);
ValueBindingContext containerContext = new BlobValueBindingContext(boundPath, context.ValueContext);
return(await BindBlobContainerAsync(container, containerContext));
}
public IStorageBlob GetBlobReferenceFromServer(IStorageBlobContainer parent, string containerName, string blobName)
{
if (!_items.ContainsKey(containerName))
{
throw StorageExceptionFactory.Create(404);
}
return(_items[containerName].GetBlobReferenceFromServer(this, parent, blobName));
}
public IStorageBlob GetBlobReferenceFromServer(IStorageBlobContainer parent, string containerName, string blobName)
{
if (!_items.ContainsKey(containerName))
{
throw StorageExceptionFactory.Create(404);
}
return _items[containerName].GetBlobReferenceFromServer(this, parent, blobName);
}
public FakeStoragePageBlob(MemoryBlobStore store, string blobName, IStorageBlobContainer parent)
{
_store = store;
_blobName = blobName;
_parent = parent;
_containerName = parent.Name;
_metadata = new Dictionary <string, string>();
_sdkObject = new CloudPageBlob(new Uri("http://localhost/" + _containerName + "/" + blobName));
}
private string CreateAblobAndUploadToContainer(IStorageBlobContainer container, string blobContent = "test")
{
string blobName = Path.GetRandomFileName().Replace(".", "");
IStorageBlockBlob blob = container.GetBlockBlobReference(blobName);
container.CreateIfNotExists();
blob.UploadText(blobContent);
return(blobName);
}
public FakeStoragePageBlob(MemoryBlobStore store, string blobName, IStorageBlobContainer parent)
{
_store = store;
_blobName = blobName;
_parent = parent;
_containerName = parent.Name;
_metadata = new Dictionary<string, string>();
_sdkObject = new CloudPageBlob(new Uri("http://localhost/" + _containerName + "/" + blobName));
}
public static bool Exists(this IStorageBlobContainer container)
{
if (container == null)
{
throw new ArgumentNullException("container");
}
return(container.ExistsAsync(CancellationToken.None).GetAwaiter().GetResult());
}
public static void CreateIfNotExists(this IStorageBlobContainer container)
{
if (container == null)
{
throw new ArgumentNullException("container");
}
container.CreateIfNotExistsAsync(CancellationToken.None).GetAwaiter().GetResult();
}
public static void Register(this IBlobListenerStrategy strategy, IStorageBlobContainer container,
ITriggerExecutor <IStorageBlob> triggerExecutor)
{
if (strategy == null)
{
throw new ArgumentNullException("strategy");
}
strategy.RegisterAsync(container, triggerExecutor, CancellationToken.None).GetAwaiter().GetResult();
}
public static void Start(this IBlobListenerStrategy strategy, IStorageBlobContainer container,
ITriggerExecutor<IStorageBlob> triggerExecutor)
{
if (strategy == null)
{
throw new ArgumentNullException("strategy");
}
strategy.Start();
}
public static void Register(this IBlobListenerStrategy strategy, IStorageBlobContainer container,
ITriggerExecutor<IStorageBlob> triggerExecutor)
{
if (strategy == null)
{
throw new ArgumentNullException("strategy");
}
strategy.RegisterAsync(container, triggerExecutor, CancellationToken.None).GetAwaiter().GetResult();
}
public static void Start(this IBlobListenerStrategy strategy, IStorageBlobContainer container,
ITriggerExecutor <IStorageBlob> triggerExecutor)
{
if (strategy == null)
{
throw new ArgumentNullException("strategy");
}
strategy.Start();
}
public Task RegisterAsync(IStorageBlobContainer container, ITriggerExecutor <IStorageBlob> triggerExecutor,
CancellationToken cancellationToken)
{
if (_started)
{
throw new InvalidOperationException(
"Registrations may not be added while the shared listener is running.");
}
return(_strategy.RegisterAsync(container, triggerExecutor, cancellationToken));
}
public Task RegisterAsync(IStorageBlobContainer container, ITriggerExecutor<IStorageBlob> triggerExecutor,
CancellationToken cancellationToken)
{
if (_started)
{
throw new InvalidOperationException(
"Registrations may not be added while the shared listener is running.");
}
return _strategy.RegisterAsync(container, triggerExecutor, cancellationToken);
}
private async Task PollAndNotify(IStorageBlobContainer container, ContainerScanInfo containerInfo, CancellationToken cancellationToken, List <IStorageBlob> failedNotifications)
{
cancellationToken.ThrowIfCancellationRequested();
IEnumerable <IStorageBlob> newBlobs = await PollNewBlobsAsync(container, containerInfo, cancellationToken);
foreach (IStorageBlob newBlob in newBlobs)
{
cancellationToken.ThrowIfCancellationRequested();
await NotifyRegistrationsAsync(newBlob, failedNotifications, cancellationToken);
}
}
private async Task <IStorageBlobContainer> GetContainerAsync(
BlobAttribute blobAttribute,
CancellationToken cancellationToken)
{
IStorageBlobClient client = await GetClientAsync(blobAttribute, cancellationToken);
BlobPath boundPath = BlobPath.ParseAndValidate(blobAttribute.BlobPath, isContainerBinding: true);
IStorageBlobContainer container = client.GetContainerReference(boundPath.ContainerName);
return(container);
}
public async Task <IValueProvider> BindAsync(object value, ValueBindingContext context)
{
BlobPath path = null;
IStorageBlobContainer container = null;
if (TryConvert(value, _client, out container, out path))
{
return(await BindBlobContainerAsync(container, new BlobValueBindingContext(path, context)));
}
throw new InvalidOperationException("Unable to convert value to CloudBlobContainer.");
}
public void TryConvert_ConvertString_Failure(string value)
{
Mock <IStorageBlobClient> mockStorageClient = new Mock <IStorageBlobClient>(MockBehavior.Strict);
IStorageBlobContainer container = null;
BlobPath path = null;
Assert.Throws <FormatException>(() =>
{
BlobContainerBinding.TryConvert(value, mockStorageClient.Object, out container, out path);
});
}
public async Task ExecuteAsync_UpdatesScanInfoManager()
{
int testScanBlobLimitPerPoll = 6;
string firstContainerName = Guid.NewGuid().ToString();
string secondContainerName = Guid.NewGuid().ToString();
IStorageAccount account = CreateFakeStorageAccount();
IStorageBlobContainer firstContainer = account.CreateBlobClient().GetContainerReference(firstContainerName);
IStorageBlobContainer secondContainer = account.CreateBlobClient().GetContainerReference(secondContainerName);
TestBlobScanInfoManager testScanInfoManager = new TestBlobScanInfoManager();
string accountName = account.Credentials.AccountName;
testScanInfoManager.SetScanInfo(accountName, firstContainerName, DateTime.MinValue);
testScanInfoManager.SetScanInfo(accountName, secondContainerName, DateTime.MinValue);
IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(testScanInfoManager);
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
typeof(ScanBlobScanLogHybridPollingStrategy)
.GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic)
.SetValue(product, testScanBlobLimitPerPoll);
await product.RegisterAsync(firstContainer, executor, CancellationToken.None);
await product.RegisterAsync(secondContainer, executor, CancellationToken.None);
var firstExpectedNames = new List <string>();
for (int i = 0; i < 3; i++)
{
firstExpectedNames.Add(CreateAblobAndUploadToContainer(firstContainer));
}
RunExecuteWithMultiPollingInterval(firstExpectedNames, product, executor, testScanBlobLimitPerPoll / 2);
// only expect the first container to have updated its scanInfo
Assert.Equal(1, testScanInfoManager.UpdateCounts[accountName][firstContainerName]);
int count;
testScanInfoManager.UpdateCounts[accountName].TryGetValue(secondContainerName, out count);
Assert.Equal(0, count);
await Task.Delay(10);
var secondExpectedNames = new List <string>();
for (int i = 0; i < 7; i++)
{
secondExpectedNames.Add(CreateAblobAndUploadToContainer(secondContainer));
}
RunExecuteWithMultiPollingInterval(secondExpectedNames, product, executor, testScanBlobLimitPerPoll / 2);
// this time, only expect the second container to have updated its scanInfo
Assert.Equal(1, testScanInfoManager.UpdateCounts[accountName][firstContainerName]);
Assert.Equal(1, testScanInfoManager.UpdateCounts[accountName][secondContainerName]);
}
public FunctionStatusLogger(IHostIdProvider hostIdProvider, IStorageBlobClient blobClient)
{
if (hostIdProvider == null)
{
throw new ArgumentNullException("hostIdProvider");
}
if (blobClient == null)
{
throw new ArgumentNullException("blobClient");
}
_hostIdProvider = hostIdProvider;
_blobClient = blobClient;
_hostContainer = _blobClient.GetContainerReference(HostContainerNames.Hosts);
}
internal static bool TryConvert(object value, IStorageBlobClient client, out IStorageBlobContainer container, out BlobPath path)
{
container = null;
path = null;
string fullPath = value as string;
if (fullPath != null)
{
path = BlobPath.ParseAndValidate(fullPath, isContainerBinding: true);
container = client.GetContainerReference(path.ContainerName);
return true;
}
return false;
}
public FakeStorageBlockBlob(MemoryBlobStore store, string blobName, IStorageBlobContainer parent, FakeStorageBlobProperties properties = null)
{
_store = store;
_blobName = blobName;
_parent = parent;
_containerName = parent.Name;
_metadata = new Dictionary<string, string>();
if (properties != null )
{
_properties = properties;
}
else
{
_properties = new FakeStorageBlobProperties();
}
_sdkObject = new CloudBlockBlob(new Uri("http://localhost/" + _containerName + "/" + blobName));
}
private static async Task<IStorageBlob> GetExistingOrNewBlockBlobReferenceAsync(IStorageBlobContainer container,
string blobName, CancellationToken cancellationToken)
{
try
{
return await container.GetBlobReferenceFromServerAsync(blobName, cancellationToken);
}
catch (StorageException exception)
{
RequestResult result = exception.RequestInformation;
if (result == null || result.HttpStatusCode != 404)
{
throw;
}
else
{
return container.GetBlockBlobReference(blobName);
}
}
}
public async Task RegisterAsync(IStorageBlobContainer container, ITriggerExecutor<IStorageBlob> triggerExecutor, CancellationToken cancellationToken)
{
// Register and Execute are not concurrency-safe.
// Avoiding calling Register while Execute is running is the caller's responsibility.
ThrowIfDisposed();
// Register all in logPolling, there is no problem if we get 2 notifications of the new blob
await _pollLogStrategy.RegisterAsync(container, triggerExecutor, cancellationToken);
ContainerScanInfo containerScanInfo;
if (!_scanInfo.TryGetValue(container, out containerScanInfo))
{
containerScanInfo = new ContainerScanInfo()
{
Registrations = new List<ITriggerExecutor<IStorageBlob>>(),
LastSweepCycleStartTime = DateTime.MinValue,
CurrentSweepCycleStartTime = DateTime.MinValue,
ContinuationToken = null
};
_scanInfo.Add(container, containerScanInfo);
}
containerScanInfo.Registrations.Add(triggerExecutor);
}
/// <summary>
/// This method is called each polling interval for all containers. The method divides the
/// budget of allocated number of blobs to query, for each container we query a page of
/// that size and we keep the continuation token for the next time. AS a curser, we use
/// the time stamp when the current cycle on the container started. blobs newer than that
/// time will be considered new and registrations will be notified
/// </summary>
/// <param name="container"></param>
/// <param name="containerScanInfo"> Information that includes the last cycle start
/// the continuation token and the current cycle start for a container</param>
/// <param name="cancellationToken"></param>
/// <returns></returns>
public async Task<IEnumerable<IStorageBlob>> PollNewBlobsAsync(
IStorageBlobContainer container, ContainerScanInfo containerScanInfo, CancellationToken cancellationToken)
{
IEnumerable<IStorageListBlobItem> currentBlobs;
IStorageBlobResultSegment blobSegment;
int blobPollLimitPerContainer = _scanBlobLimitPerPoll / _scanInfo.Count;
BlobContinuationToken continuationToken = containerScanInfo.ContinuationToken;
// if starting the cycle, keep the current time stamp to be used as curser
if (continuationToken == null)
{
containerScanInfo.CurrentSweepCycleStartTime = DateTime.UtcNow;
}
try
{
blobSegment = await container.ListBlobsSegmentedAsync(prefix: null, useFlatBlobListing: true,
blobListingDetails: BlobListingDetails.None, maxResults: blobPollLimitPerContainer, currentToken: continuationToken,
options: null, operationContext: null, cancellationToken: cancellationToken);
currentBlobs = blobSegment.Results;
}
catch (StorageException exception)
{
if (exception.IsNotFound())
{
return Enumerable.Empty<IStorageBlob>();
}
else
{
throw;
}
}
List<IStorageBlob> newBlobs = new List<IStorageBlob>();
// Type cast to IStorageBlob is safe due to useFlatBlobListing: true above.
foreach (IStorageBlob currentBlob in currentBlobs)
{
cancellationToken.ThrowIfCancellationRequested();
IStorageBlobProperties properties = currentBlob.Properties;
DateTime lastModifiedTimestamp = properties.LastModified.Value.UtcDateTime;
if (lastModifiedTimestamp > containerScanInfo.LastSweepCycleStartTime)
{
newBlobs.Add(currentBlob);
}
}
// record continuation token for next chunk retrieval
containerScanInfo.ContinuationToken = blobSegment.ContinuationToken;
// if ending a cycle then copy currentSweepCycleStartTime to lastSweepCycleStartTime
if (blobSegment.ContinuationToken == null)
{
containerScanInfo.LastSweepCycleStartTime = containerScanInfo.CurrentSweepCycleStartTime;
}
return newBlobs;
}
public BlobListenerFactory(IHostIdProvider hostIdProvider,
IQueueConfiguration queueConfiguration,
IBackgroundExceptionDispatcher backgroundExceptionDispatcher,
IContextSetter<IBlobWrittenWatcher> blobWrittenWatcherSetter,
IContextSetter<IMessageEnqueuedWatcher> messageEnqueuedWatcherSetter,
ISharedContextProvider sharedContextProvider,
TraceWriter trace,
string functionId,
IStorageAccount hostAccount,
IStorageAccount dataAccount,
IStorageBlobContainer container,
IBlobPathSource input,
ITriggeredFunctionExecutor executor,
SingletonManager singletonManager)
{
if (hostIdProvider == null)
{
throw new ArgumentNullException("hostIdProvider");
}
if (queueConfiguration == null)
{
throw new ArgumentNullException("queueConfiguration");
}
if (backgroundExceptionDispatcher == null)
{
throw new ArgumentNullException("backgroundExceptionDispatcher");
}
if (blobWrittenWatcherSetter == null)
{
throw new ArgumentNullException("blobWrittenWatcherSetter");
}
if (messageEnqueuedWatcherSetter == null)
{
throw new ArgumentNullException("messageEnqueuedWatcherSetter");
}
if (sharedContextProvider == null)
{
throw new ArgumentNullException("sharedContextProvider");
}
if (trace == null)
{
throw new ArgumentNullException("trace");
}
if (hostAccount == null)
{
throw new ArgumentNullException("hostAccount");
}
if (dataAccount == null)
{
throw new ArgumentNullException("dataAccount");
}
if (container == null)
{
throw new ArgumentNullException("container");
}
if (input == null)
{
throw new ArgumentNullException("input");
}
if (executor == null)
{
throw new ArgumentNullException("executor");
}
if (singletonManager == null)
{
throw new ArgumentNullException("singletonManager");
}
_hostIdProvider = hostIdProvider;
_queueConfiguration = queueConfiguration;
_backgroundExceptionDispatcher = backgroundExceptionDispatcher;
_blobWrittenWatcherSetter = blobWrittenWatcherSetter;
_messageEnqueuedWatcherSetter = messageEnqueuedWatcherSetter;
_sharedContextProvider = sharedContextProvider;
_trace = trace;
_functionId = functionId;
_hostAccount = hostAccount;
_dataAccount = dataAccount;
_container = container;
_input = input;
_executor = executor;
_singletonManager = singletonManager;
}
internal static IStorageListBlobItem ToStorageListBlobItem(IStorageBlobContainer parent, IListBlobItem sdkItem)
{
if (sdkItem == null)
{
return null;
}
ICloudBlob sdkBlob = sdkItem as ICloudBlob;
if (sdkBlob != null)
{
return ToStorageBlob(parent, sdkBlob);
}
else
{
return new StorageBlobDirectory(parent, (CloudBlobDirectory)sdkItem);
}
}
/// <summary>Initializes a new instance of the <see cref="StoragePageBlob"/> class.</summary>
/// <param name="parent">The parent blob container.</param>
/// <param name="sdk">The SDK blob to wrap.</param>
public StoragePageBlob(IStorageBlobContainer parent, CloudPageBlob sdk)
{
_parent = parent;
_sdk = sdk;
_properties = new StorageBlobProperties(sdk);
}
private static IStorageBlob ToStorageBlob(IStorageBlobContainer parent, ICloudBlob sdkBlob)
{
Debug.Assert(sdkBlob != null);
CloudBlockBlob blockBlob = sdkBlob as CloudBlockBlob;
if (blockBlob != null)
{
return new StorageBlockBlob(parent, blockBlob);
}
else
{
CloudPageBlob cloudPageBlob = sdkBlob as CloudPageBlob;
Debug.Assert(cloudPageBlob != null);
return new StoragePageBlob(parent, cloudPageBlob);
}
}
private async Task PollAndNotify(IStorageBlobContainer container, ContainerScanInfo containerInfo, CancellationToken cancellationToken, List<IStorageBlob> failedNotifications)
{
cancellationToken.ThrowIfCancellationRequested();
IEnumerable<IStorageBlob> newBlobs = await PollNewBlobsAsync(container, containerInfo, cancellationToken);
foreach (IStorageBlob newBlob in newBlobs)
{
cancellationToken.ThrowIfCancellationRequested();
await NotifyRegistrationsAsync(newBlob, failedNotifications, cancellationToken);
}
}
public BlobListenerFactory(IHostIdProvider hostIdProvider,
IQueueConfiguration queueConfiguration,
IBackgroundExceptionDispatcher backgroundExceptionDispatcher,
IContextSetter<IBlobWrittenWatcher> blobWrittenWatcherSetter,
IContextSetter<IMessageEnqueuedWatcher> messageEnqueuedWatcherSetter,
ISharedContextProvider sharedContextProvider,
TextWriter log,
string functionId,
IStorageAccount account,
IStorageBlobContainer container,
IBlobPathSource input,
ITriggeredFunctionExecutor<IStorageBlob> executor)
{
if (hostIdProvider == null)
{
throw new ArgumentNullException("hostIdProvider");
}
if (queueConfiguration == null)
{
throw new ArgumentNullException("queueConfiguration");
}
if (backgroundExceptionDispatcher == null)
{
throw new ArgumentNullException("backgroundExceptionDispatcher");
}
if (blobWrittenWatcherSetter == null)
{
throw new ArgumentNullException("blobWrittenWatcherSetter");
}
if (messageEnqueuedWatcherSetter == null)
{
throw new ArgumentNullException("messageEnqueuedWatcherSetter");
}
if (sharedContextProvider == null)
{
throw new ArgumentNullException("sharedContextProvider");
}
if (log == null)
{
throw new ArgumentNullException("log");
}
if (account == null)
{
throw new ArgumentNullException("account");
}
if (container == null)
{
throw new ArgumentNullException("container");
}
if (input == null)
{
throw new ArgumentNullException("input");
}
if (executor == null)
{
throw new ArgumentNullException("executor");
}
_hostIdProvider = hostIdProvider;
_queueConfiguration = queueConfiguration;
_backgroundExceptionDispatcher = backgroundExceptionDispatcher;
_blobWrittenWatcherSetter = blobWrittenWatcherSetter;
_messageEnqueuedWatcherSetter = messageEnqueuedWatcherSetter;
_sharedContextProvider = sharedContextProvider;
_log = log;
_functionId = functionId;
_account = account;
_container = container;
_input = input;
_executor = executor;
}
private Task<IValueProvider> BindBlobContainerAsync(IStorageBlobContainer value, ValueBindingContext context)
{
return _argumentBinding.BindAsync(value, context);
}
public FakeStorageBlobDirectory(MemoryBlobStore store, string relativeAddress, IStorageBlobContainer parent)
{
_store = store;
_relativeAddress = relativeAddress;
_parent = parent;
}
public IEnumerable<IStorageBlob> ListBlobs(MemoryBlobStore store, IStorageBlobContainer parent,
BlobListingDetails blobListingDetails)
{
if (blobListingDetails != BlobListingDetails.None && blobListingDetails != BlobListingDetails.Metadata)
{
throw new NotImplementedException();
}
List<IStorageBlob> results = new List<IStorageBlob>();
foreach (KeyValuePair<string, Blob> item in _items)
{
string blobName = item.Key;
IStorageBlob blob = new FakeStorageBlockBlob(store, blobName, parent);
if ((blobListingDetails | BlobListingDetails.Metadata) == BlobListingDetails.Metadata)
{
Blob storeBlob = item.Value;
IReadOnlyDictionary<string, string> storeMetadata = storeBlob.Metadata;
foreach (KeyValuePair<string, string> pair in storeMetadata)
{
blob.Metadata.Add(pair.Key, pair.Value);
}
}
results.Add(blob);
}
return results;
}
public IEnumerable<IStorageBlob> ListBlobs(MemoryBlobStore store, IStorageBlobContainer parent,
BlobListingDetails blobListingDetails)
{
if (blobListingDetails != BlobListingDetails.None && blobListingDetails != BlobListingDetails.Metadata)
{
throw new NotImplementedException();
}
List<IStorageBlob> results = new List<IStorageBlob>();
foreach (KeyValuePair<string, Blob> item in _items)
{
string blobName = item.Key;
// Etag and LastModifiedTime is always passed in listBlobs
FakeStorageBlobProperties properties = new FakeStorageBlobProperties()
{
ETag = item.Value.ETag,
LastModified = item.Value.LastModified,
};
IStorageBlob blob = new FakeStorageBlockBlob(store, blobName, parent, properties);
if ((blobListingDetails | BlobListingDetails.Metadata) == BlobListingDetails.Metadata)
{
Blob storeBlob = item.Value;
IReadOnlyDictionary<string, string> storeMetadata = storeBlob.Metadata;
foreach (KeyValuePair<string, string> pair in storeMetadata)
{
blob.Metadata.Add(pair.Key, pair.Value);
}
}
results.Add(blob);
}
return results;
}
public IStorageBlob GetBlobReferenceFromServer(MemoryBlobStore store, IStorageBlobContainer parent, string blobName)
{
if (!_items.ContainsKey(blobName))
{
throw StorageExceptionFactory.Create(404);
}
Blob blob = _items[blobName];
if (blob.BlobType == StorageBlobType.BlockBlob)
{
return new FakeStorageBlockBlob(store, blobName, parent);
}
else
{
return new FakeStoragePageBlob(store, blobName, parent);
}
}
/// <summary>Initializes a new instance of the <see cref="StorageBlobDirectory"/> class.</summary>
/// <param name="parent">The parent blob container.</param>
/// <param name="sdk">The SDK directory to wrap.</param>
public StorageBlobDirectory(IStorageBlobContainer parent, CloudBlobDirectory sdk)
{
_parent = parent;
_sdk = sdk;
}
public static async Task<Tuple<IEnumerable<IStorageBlob>, DateTime>> PollNewBlobsAsync(
IStorageBlobContainer container, DateTime previousTimestamp, CancellationToken cancellationToken)
{
DateTime updatedTimestamp = previousTimestamp;
IList<IStorageListBlobItem> currentBlobs;
try
{
currentBlobs = (await container.ListBlobsAsync(prefix: null, useFlatBlobListing: true,
cancellationToken: cancellationToken)).ToList();
}
catch (StorageException exception)
{
if (exception.IsNotFound())
{
return new Tuple<IEnumerable<IStorageBlob>, DateTime>(
Enumerable.Empty<IStorageBlob>(), updatedTimestamp);
}
else
{
throw;
}
}
List<IStorageBlob> newBlobs = new List<IStorageBlob>();
// Type cast to IStorageBlob is safe due to useFlatBlobListing: true above.
foreach (IStorageBlob currentBlob in currentBlobs)
{
cancellationToken.ThrowIfCancellationRequested();
try
{
await currentBlob.FetchAttributesAsync(cancellationToken);
}
catch (StorageException exception)
{
if (exception.IsNotFound())
{
continue;
}
else
{
throw;
}
}
IStorageBlobProperties properties = currentBlob.Properties;
DateTime lastModifiedTimestamp = properties.LastModified.Value.UtcDateTime;
if (lastModifiedTimestamp > updatedTimestamp)
{
updatedTimestamp = lastModifiedTimestamp;
}
if (lastModifiedTimestamp > previousTimestamp)
{
newBlobs.Add(currentBlob);
}
}
return new Tuple<IEnumerable<IStorageBlob>, DateTime>(newBlobs, updatedTimestamp);
}
