public FakeStorageBlobContainer(MemoryBlobStore store, string containerName, IStorageBlobClient parent)
{
_store = store;
_containerName = containerName;
_parent = parent;
_uri = new Uri("http://localhost/" + containerName);
}
public BlobFunctionOutputDefinition(IStorageBlobClient client, LocalBlobDescriptor outputBlob,
LocalBlobDescriptor parameterLogBlob)
{
_client = client;
_outputBlob = outputBlob;
_parameterLogBlob = parameterLogBlob;
}
public BlobContainerBinding(string parameterName, IArgumentBinding<IStorageBlobContainer> argumentBinding, IStorageBlobClient client,
IBindableBlobPath path)
{
_parameterName = parameterName;
_argumentBinding = argumentBinding;
_client = client;
_path = path;
}
public SingletonManager(IStorageBlobClient blobClient, IBackgroundExceptionDispatcher backgroundExceptionDispatcher, SingletonConfiguration config, TraceWriter trace)
{
_backgroundExceptionDispatcher = backgroundExceptionDispatcher;
_directory = blobClient.GetContainerReference(HostContainerNames.Hosts)
.GetDirectoryReference(HostDirectoryNames.SingletonLocks);
_config = config;
_trace = trace;
}
public BlobBinding(string parameterName, IBlobArgumentBinding argumentBinding, IStorageBlobClient client, IBindableBlobPath path)
{
_parameterName = parameterName;
_argumentBinding = argumentBinding;
_client = client;
_accountName = BlobClient.GetAccountName(client);
_path = path;
_converter = CreateConverter(_client, path, argumentBinding.ValueType);
}
public static string GetAccountName(IStorageBlobClient client)
{
if (client == null)
{
return null;
}
return StorageClient.GetAccountName(client.Credentials);
}
private static IAsyncObjectToTypeConverter<IStorageBlob> CreateConverter(IStorageBlobClient client,
IBindableBlobPath path, Type argumentType)
{
return new CompositeAsyncObjectToTypeConverter<IStorageBlob>(
new BlobOutputConverter<IStorageBlob>(new AsyncConverter<IStorageBlob, IStorageBlob>(
new IdentityConverter<IStorageBlob>())),
new BlobOutputConverter<ICloudBlob>(new AsyncConverter<ICloudBlob, IStorageBlob>(
new CloudBlobToStorageBlobConverter())),
new BlobOutputConverter<string>(new StringToStorageBlobConverter(client, path, argumentType)));
}
internal static bool TryConvert(object value, IStorageBlobClient client, out IStorageBlobContainer container, out BlobPath path)
{
container = null;
path = null;
string fullPath = value as string;
if (fullPath != null)
{
path = BlobPath.ParseAndValidate(fullPath, isContainerBinding: true);
container = client.GetContainerReference(path.ContainerName);
return true;
}
return false;
}
public FunctionStatusLogger(IHostIdProvider hostIdProvider, IStorageBlobClient blobClient)
{
if (hostIdProvider == null)
{
throw new ArgumentNullException("hostIdProvider");
}
if (blobClient == null)
{
throw new ArgumentNullException("blobClient");
}
_hostIdProvider = hostIdProvider;
_blobClient = blobClient;
_hostContainer = _blobClient.GetContainerReference(HostContainerNames.Hosts);
}
public void BlobPolling_IgnoresClockSkew()
{
int testScanBlobLimitPerPoll = 3;
string containerName = Path.GetRandomFileName();
IStorageAccount account = CreateFakeStorageAccount();
IStorageBlobClient client = account.CreateBlobClient();
var now = DateTimeOffset.UtcNow;
var timeMap = new Dictionary <string, DateTimeOffset>();
IStorageBlobContainer container = new SkewableFakeStorageBlobContainer(new MemoryBlobStore(), containerName, client,
(IStorageBlobResultSegment blobs) =>
{
// Simulate some extreme clock skew -- the first one's LastUpdated
// wll be 60 seconds ago and the second will be 59 seconds ago.
foreach (IStorageBlob blob in blobs.Results)
{
((FakeStorageBlobProperties)blob.Properties).LastModified = timeMap[blob.Name];
}
});
IBlobListenerStrategy product = new ScanBlobScanLogHybridPollingStrategy(new TestBlobScanInfoManager());
LambdaBlobTriggerExecutor executor = new LambdaBlobTriggerExecutor();
typeof(ScanBlobScanLogHybridPollingStrategy)
.GetField("_scanBlobLimitPerPoll", BindingFlags.Instance | BindingFlags.NonPublic)
.SetValue(product, testScanBlobLimitPerPoll);
product.Register(container, executor);
product.Start();
List <string> expectedNames = new List <string>();
expectedNames.Add(CreateAblobAndUploadToContainer(container));
timeMap[expectedNames.Single()] = now.AddSeconds(-60);
RunExecuterWithExpectedBlobs(expectedNames, product, executor);
expectedNames.Clear();
expectedNames.Add(CreateAblobAndUploadToContainer(container));
timeMap[expectedNames.Single()] = now.AddSeconds(-59);
// We should see the new item.
RunExecuterWithExpectedBlobs(expectedNames, product, executor);
}
private async Task <IStorageBlob> GetBlobAsync(
BlobAttribute blobAttribute,
CancellationToken cancellationToken,
Type requestedType = null)
{
IStorageBlobClient client = await GetClientAsync(blobAttribute, cancellationToken);
BlobPath boundPath = BlobPath.ParseAndValidate(blobAttribute.BlobPath);
IStorageBlobContainer container = client.GetContainerReference(boundPath.ContainerName);
if (blobAttribute.Access != FileAccess.Read)
{
await container.CreateIfNotExistsAsync(cancellationToken);
}
IStorageBlob blob = await container.GetBlobReferenceForArgumentTypeAsync(
boundPath.BlobName, requestedType, cancellationToken);
return(blob);
}
public void ExecuteAsync_IfBlobDoesNotMatchPattern_ReturnsTrue()
{
// Arrange
IStorageAccount account = CreateAccount();
IStorageBlobClient client = account.CreateBlobClient();
string containerName = "container";
IStorageBlobContainer container = client.GetContainerReference(containerName);
IStorageBlobContainer otherContainer = client.GetContainerReference("other");
IBlobPathSource input = BlobPathSource.Create(containerName + "/{name}");
ITriggerExecutor <IStorageBlob> product = CreateProductUnderTest(input);
IStorageBlob blob = otherContainer.GetBlockBlobReference("nonmatch");
// Act
Task <bool> task = product.ExecuteAsync(blob, CancellationToken.None);
// Assert
Assert.True(task.Result);
}
internal IStorageBlobDirectory GetLockDirectory(string accountName)
{
if (string.IsNullOrEmpty(accountName))
{
accountName = ConnectionStringNames.Storage;
}
IStorageBlobDirectory storageDirectory = null;
if (!_lockDirectoryMap.TryGetValue(accountName, out storageDirectory))
{
Task <IStorageAccount> task = _accountProvider.GetAccountAsync(accountName, CancellationToken.None);
IStorageAccount storageAccount = task.Result;
IStorageBlobClient blobClient = storageAccount.CreateBlobClient();
storageDirectory = blobClient.GetContainerReference(HostContainerNames.Hosts)
.GetDirectoryReference(HostDirectoryNames.SingletonLocks);
_lockDirectoryMap[accountName] = storageDirectory;
}
return(storageDirectory);
}
private IListener CreateQueueMessageToTriggerExecutionListener(
ISharedContextProvider sharedContextProvider,
SharedQueueWatcher sharedQueueWatcher,
IStorageQueueClient queueClient,
IStorageQueue hostBlobTriggerQueue,
IStorageBlobClient blobClient,
IBlobWrittenWatcher blobWrittenWatcher)
{
SharedBlobQueueListener sharedListener = sharedContextProvider.GetOrCreate <SharedBlobQueueListener>(
new SharedBlobQueueListenerFactory(sharedQueueWatcher, queueClient, hostBlobTriggerQueue,
_queueConfiguration, _backgroundExceptionDispatcher, _trace, blobWrittenWatcher));
BlobQueueRegistration registration = new BlobQueueRegistration
{
Executor = _executor,
BlobClient = blobClient
};
sharedListener.Register(_functionId, registration);
return(new BlobListener(sharedListener));
}
public BlobTriggerBinding(ParameterInfo parameter,
IStorageAccount hostAccount,
IStorageAccount dataAccount,
IBlobPathSource path,
IHostIdProvider hostIdProvider,
IQueueConfiguration queueConfiguration,
JobHostBlobsConfiguration blobsConfiguration,
IWebJobsExceptionHandler exceptionHandler,
IContextSetter <IBlobWrittenWatcher> blobWrittenWatcherSetter,
IContextSetter <IMessageEnqueuedWatcher> messageEnqueuedWatcherSetter,
ISharedContextProvider sharedContextProvider,
SingletonManager singletonManager,
ILoggerFactory loggerFactory)
{
_parameter = parameter ?? throw new ArgumentNullException(nameof(parameter));
_hostAccount = hostAccount ?? throw new ArgumentNullException(nameof(hostAccount));
_dataAccount = dataAccount ?? throw new ArgumentNullException(nameof(dataAccount));
StorageClientFactoryContext context = new StorageClientFactoryContext
{
Parameter = parameter
};
_blobClient = dataAccount.CreateBlobClient(context);
_accountName = BlobClient.GetAccountName(_blobClient);
_path = path ?? throw new ArgumentNullException(nameof(path));
_hostIdProvider = hostIdProvider ?? throw new ArgumentNullException(nameof(hostIdProvider));
_queueConfiguration = queueConfiguration ?? throw new ArgumentNullException(nameof(queueConfiguration));
_blobsConfiguration = blobsConfiguration ?? throw new ArgumentNullException(nameof(blobsConfiguration));
_exceptionHandler = exceptionHandler ?? throw new ArgumentNullException(nameof(exceptionHandler));
_blobWrittenWatcherSetter = blobWrittenWatcherSetter ?? throw new ArgumentNullException(nameof(blobWrittenWatcherSetter));
_messageEnqueuedWatcherSetter = messageEnqueuedWatcherSetter ?? throw new ArgumentNullException(nameof(messageEnqueuedWatcherSetter));
_sharedContextProvider = sharedContextProvider ?? throw new ArgumentNullException(nameof(sharedContextProvider));
_singletonManager = singletonManager ?? throw new ArgumentNullException(nameof(singletonManager));
_loggerFactory = loggerFactory;
_converter = CreateConverter(_blobClient);
_bindingDataContract = CreateBindingDataContract(path);
}
public StorageBlobScanInfoManager(string hostId, IStorageBlobClient blobClient)
{
if (string.IsNullOrEmpty(hostId))
{
throw new ArgumentNullException("hostId");
}
if (blobClient == null)
{
throw new ArgumentNullException("blobClient");
}
_hostId = hostId;
JsonSerializerSettings settings = new JsonSerializerSettings
{
DateFormatHandling = DateFormatHandling.IsoDateFormat
};
_serializer = JsonSerializer.Create(settings);
string blobScanInfoDirectoryPath = string.Format(CultureInfo.InvariantCulture, "blobscaninfo/{0}", _hostId);
_blobScanInfoDirectory = blobClient.GetContainerReference(HostContainerNames.Hosts).GetDirectoryReference(blobScanInfoDirectoryPath);
}
internal IStorageBlobDirectory GetLockDirectory(string accountName)
{
if (string.IsNullOrEmpty(accountName))
{
accountName = ConnectionStringNames.Storage;
}
IStorageBlobDirectory storageDirectory = null;
if (!_lockDirectoryMap.TryGetValue(accountName, out storageDirectory))
{
Task <IStorageAccount> task = _accountProvider.GetStorageAccountAsync(accountName, CancellationToken.None);
IStorageAccount storageAccount = task.Result;
// singleton requires block blobs, cannot be premium
storageAccount.AssertTypeOneOf(StorageAccountType.GeneralPurpose, StorageAccountType.BlobOnly);
IStorageBlobClient blobClient = storageAccount.CreateBlobClient();
storageDirectory = blobClient.GetContainerReference(HostContainerNames.Hosts)
.GetDirectoryReference(HostDirectoryNames.SingletonLocks);
_lockDirectoryMap[accountName] = storageDirectory;
}
return(storageDirectory);
}
public async Task RegisterAsync(IStorageBlobContainer container, ITriggerExecutor <IStorageBlob> triggerExecutor,
CancellationToken cancellationToken)
{
ThrowIfDisposed();
// Initial background scans for all containers happen on first Execute call.
// Prevent accidental late registrations.
// (Also prevents incorrect concurrent execution of Register with Execute.)
if (_initialScanThread.ThreadState != ThreadState.Unstarted)
{
throw new InvalidOperationException("All registrations must be created before execution begins.");
}
ICollection <ITriggerExecutor <IStorageBlob> > containerRegistrations;
if (_registrations.ContainsKey(container))
{
containerRegistrations = _registrations[container];
}
else
{
containerRegistrations = new List <ITriggerExecutor <IStorageBlob> >();
_registrations.Add(container, containerRegistrations);
}
containerRegistrations.Add(triggerExecutor);
IStorageBlobClient client = container.ServiceClient;
if (!_logListeners.ContainsKey(client))
{
BlobLogListener logListener = await BlobLogListener.CreateAsync(client, cancellationToken);
_logListeners.Add(client, logListener);
}
}
private static async Task ValidateCredentialsAsyncCore(IStorageAccount account,
CancellationToken cancellationToken)
{
// Verify the credentials are correct.
// Have to actually ping a storage operation.
IStorageBlobClient client = account.CreateBlobClient();
try
{
// This can hang for a long time if the account name is wrong.
// If will fail fast if the password is incorrect.
await client.GetServicePropertiesAsync(cancellationToken);
}
catch (OperationCanceledException)
{
throw;
}
catch
{
string message = String.Format(CultureInfo.CurrentCulture,
"The account credentials for '{0}' are incorrect.", account.Credentials.AccountName);
throw new InvalidOperationException(message);
}
}
private async Task <IListener> CreateBlobDiscoveryToQueueMessageListenerAsync(string hostId,
SharedBlobListener sharedBlobListener,
IStorageBlobClient blobClient,
IStorageQueue hostBlobTriggerQueue,
IMessageEnqueuedWatcher messageEnqueuedWatcher,
CancellationToken cancellationToken)
{
BlobTriggerExecutor triggerExecutor = new BlobTriggerExecutor(hostId, _functionId, _input,
BlobETagReader.Instance, new BlobReceiptManager(blobClient),
new BlobTriggerQueueWriter(hostBlobTriggerQueue, messageEnqueuedWatcher));
await sharedBlobListener.RegisterAsync(_container, triggerExecutor, cancellationToken);
return(new BlobListener(sharedBlobListener));
}
// Test that the credentials are valid and classify the account.Type as one of StorageAccountTypes
private static async Task ValidateCredentialsAsyncCore(IStorageAccount account, CancellationToken cancellationToken)
{
// Verify the credentials are correct.
// Have to actually ping a storage operation.
IStorageBlobClient client = account.CreateBlobClient();
try
{
// This can hang for a long time if the account name is wrong.
// If will fail fast if the password is incorrect.
await client.GetServicePropertiesAsync(cancellationToken);
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception e)
{
var storageException = e as StorageException;
if (storageException?.RequestInformation?.HttpStatusCode == 400 &&
storageException?.RequestInformation?.ExtendedErrorInformation?.ErrorCode == "InvalidQueryParameterValue")
{
// Premium storage accounts do not support the GetServicePropertiesAsync call, and respond with a 400 'InvalidQueryParameterValue'.
// If we see this error response classify the account as a premium account
account.Type = StorageAccountType.Premium;
return;
}
else
{
// If not a recognized error, the credentials are invalid
string message = String.Format(CultureInfo.CurrentCulture,
"Invalid storage account '{0}'. Please make sure your credentials are correct.",
account.Credentials.AccountName);
throw new InvalidOperationException(message);
}
}
IStorageQueueClient queueClient = account.CreateQueueClient();
IStorageQueue queue = queueClient.GetQueueReference("name");
try
{
await queue.ExistsAsync(cancellationToken);
}
catch (OperationCanceledException)
{
throw;
}
catch (StorageException exception)
{
WebException webException = exception.GetBaseException() as WebException;
if (webException?.Status == WebExceptionStatus.NameResolutionFailure)
{
// Blob-only storage accounts do not support services other than Blob.
// If we see a name resolution failure on the queue endpoint classify as a blob-only account
account.Type = StorageAccountType.BlobOnly;
return;
}
throw;
}
}
public BlobFunctionOutputLogger(IStorageBlobClient client)
: this(client.GetContainerReference(
HostContainerNames.Hosts).GetDirectoryReference(HostDirectoryNames.OutputLogs))
{
}
public BlobTriggerBinding(string parameterName,
IArgumentBinding <IStorageBlob> argumentBinding,
IStorageAccount account,
IBlobPathSource path,
IHostIdProvider hostIdProvider,
IQueueConfiguration queueConfiguration,
IBackgroundExceptionDispatcher backgroundExceptionDispatcher,
IContextSetter <IBlobWrittenWatcher> blobWrittenWatcherSetter,
IContextSetter <IMessageEnqueuedWatcher> messageEnqueuedWatcherSetter,
ISharedContextProvider sharedContextProvider,
TextWriter log)
{
if (argumentBinding == null)
{
throw new ArgumentNullException("argumentBinding");
}
if (account == null)
{
throw new ArgumentNullException("account");
}
if (path == null)
{
throw new ArgumentNullException("path");
}
if (hostIdProvider == null)
{
throw new ArgumentNullException("hostIdProvider");
}
if (queueConfiguration == null)
{
throw new ArgumentNullException("queueConfiguration");
}
if (backgroundExceptionDispatcher == null)
{
throw new ArgumentNullException("backgroundExceptionDispatcher");
}
if (blobWrittenWatcherSetter == null)
{
throw new ArgumentNullException("blobWrittenWatcherSetter");
}
if (messageEnqueuedWatcherSetter == null)
{
throw new ArgumentNullException("messageEnqueuedWatcherSetter");
}
if (sharedContextProvider == null)
{
throw new ArgumentNullException("sharedContextProvider");
}
if (log == null)
{
throw new ArgumentNullException("log");
}
_parameterName = parameterName;
_argumentBinding = argumentBinding;
_account = account;
_client = account.CreateBlobClient();
_accountName = BlobClient.GetAccountName(_client);
_path = path;
_hostIdProvider = hostIdProvider;
_queueConfiguration = queueConfiguration;
_backgroundExceptionDispatcher = backgroundExceptionDispatcher;
_blobWrittenWatcherSetter = blobWrittenWatcherSetter;
_messageEnqueuedWatcherSetter = messageEnqueuedWatcherSetter;
_sharedContextProvider = sharedContextProvider;
_log = log;
_converter = CreateConverter(_client);
_bindingDataContract = CreateBindingDataContract(path);
}
public BlobTriggerBinding(ParameterInfo parameter,
IArgumentBinding<IStorageBlob> argumentBinding,
IStorageAccount hostAccount,
IStorageAccount dataAccount,
IBlobPathSource path,
IHostIdProvider hostIdProvider,
IQueueConfiguration queueConfiguration,
IBackgroundExceptionDispatcher backgroundExceptionDispatcher,
IContextSetter<IBlobWrittenWatcher> blobWrittenWatcherSetter,
IContextSetter<IMessageEnqueuedWatcher> messageEnqueuedWatcherSetter,
ISharedContextProvider sharedContextProvider,
TraceWriter trace)
{
if (parameter == null)
{
throw new ArgumentNullException("parameter");
}
if (argumentBinding == null)
{
throw new ArgumentNullException("argumentBinding");
}
if (hostAccount == null)
{
throw new ArgumentNullException("hostAccount");
}
if (dataAccount == null)
{
throw new ArgumentNullException("dataAccount");
}
if (path == null)
{
throw new ArgumentNullException("path");
}
if (hostIdProvider == null)
{
throw new ArgumentNullException("hostIdProvider");
}
if (queueConfiguration == null)
{
throw new ArgumentNullException("queueConfiguration");
}
if (backgroundExceptionDispatcher == null)
{
throw new ArgumentNullException("backgroundExceptionDispatcher");
}
if (blobWrittenWatcherSetter == null)
{
throw new ArgumentNullException("blobWrittenWatcherSetter");
}
if (messageEnqueuedWatcherSetter == null)
{
throw new ArgumentNullException("messageEnqueuedWatcherSetter");
}
if (sharedContextProvider == null)
{
throw new ArgumentNullException("sharedContextProvider");
}
if (trace == null)
{
throw new ArgumentNullException("trace");
}
_parameter = parameter;
_argumentBinding = argumentBinding;
_hostAccount = hostAccount;
_dataAccount = dataAccount;
StorageClientFactoryContext context = new StorageClientFactoryContext
{
Parameter = parameter
};
_blobClient = dataAccount.CreateBlobClient(context);
_accountName = BlobClient.GetAccountName(_blobClient);
_path = path;
_hostIdProvider = hostIdProvider;
_queueConfiguration = queueConfiguration;
_backgroundExceptionDispatcher = backgroundExceptionDispatcher;
_blobWrittenWatcherSetter = blobWrittenWatcherSetter;
_messageEnqueuedWatcherSetter = messageEnqueuedWatcherSetter;
_sharedContextProvider = sharedContextProvider;
_trace = trace;
_converter = CreateConverter(_blobClient);
_bindingDataContract = CreateBindingDataContract(path);
}
/// <summary>Initializes a new instance of the <see cref="StorageBlobContainer"/> class.</summary>
/// <param name="parent">The parent blob client.</param>
/// <param name="sdk">The SDK container to wrap.</param>
public StorageBlobContainer(IStorageBlobClient parent, CloudBlobContainer sdk)
{
_parent = parent;
_sdk = sdk;
}
public BlobFunctionOutputLogger(IStorageBlobClient client)
: this(client.GetContainerReference(HostContainerNames.Hosts).GetDirectoryReference(HostDirectoryNames.OutputLogs))
{
}
public async Task <IListener> CreateAsync(CancellationToken cancellationToken)
{
// Note that these clients are intentionally for the storage account rather than for the dashboard account.
// We use the storage, not dashboard, account for the blob receipt container and blob trigger queues.
IStorageQueueClient primaryQueueClient = _hostAccount.CreateQueueClient();
IStorageBlobClient primaryBlobClient = _hostAccount.CreateBlobClient();
// Important: We're using the storage account of the function target here, which is the account that the
// function the listener is for is targeting. This is the account that will be used
// to read the trigger blob.
IStorageBlobClient targetBlobClient = _dataAccount.CreateBlobClient();
IStorageQueueClient targetQueueClient = _dataAccount.CreateQueueClient();
string hostId = await _hostIdProvider.GetHostIdAsync(cancellationToken);
string hostBlobTriggerQueueName = HostQueueNames.GetHostBlobTriggerQueueName(hostId);
IStorageQueue hostBlobTriggerQueue = primaryQueueClient.GetQueueReference(hostBlobTriggerQueueName);
SharedQueueWatcher sharedQueueWatcher = _sharedContextProvider.GetOrCreateInstance <SharedQueueWatcher>(
new SharedQueueWatcherFactory(_messageEnqueuedWatcherSetter));
SharedBlobListener sharedBlobListener = _sharedContextProvider.GetOrCreateInstance <SharedBlobListener>(
new SharedBlobListenerFactory(hostId, _hostAccount, _exceptionHandler, _blobWrittenWatcherSetter));
// Register the blob container we wish to monitor with the shared blob listener.
await RegisterWithSharedBlobListenerAsync(hostId, sharedBlobListener, primaryBlobClient,
hostBlobTriggerQueue, sharedQueueWatcher, cancellationToken);
// Create a "bridge" listener that will monitor the blob
// notification queue and dispatch to the target job function.
SharedBlobQueueListener sharedBlobQueueListener = _sharedContextProvider.GetOrCreateInstance <SharedBlobQueueListener>(
new SharedBlobQueueListenerFactory(_hostAccount, sharedQueueWatcher, hostBlobTriggerQueue,
_queueConfiguration, _exceptionHandler, _trace, _loggerFactory, sharedBlobListener.BlobWritterWatcher));
var queueListener = new BlobListener(sharedBlobQueueListener);
// determine which client to use for the poison queue
// by default this should target the same storage account
// as the blob container we're monitoring
var poisonQueueClient = targetQueueClient;
if (_dataAccount.Type != StorageAccountType.GeneralPurpose ||
_blobsConfiguration.CentralizedPoisonQueue)
{
// use the primary storage account if the centralize flag is true,
// or if the target storage account doesn't support queues
poisonQueueClient = primaryQueueClient;
}
// Register our function with the shared blob queue listener
RegisterWithSharedBlobQueueListenerAsync(sharedBlobQueueListener, targetBlobClient, poisonQueueClient);
// check a flag in the shared context to see if we've created the singleton
// shared blob listener in this host instance
object singletonListenerCreated = false;
if (!_sharedContextProvider.TryGetValue(SingletonBlobListenerScopeId, out singletonListenerCreated))
{
// Create a singleton shared blob listener, since we only
// want a single instance of the blob poll/scan logic to be running
// across host instances
var singletonBlobListener = _singletonManager.CreateHostSingletonListener(
new BlobListener(sharedBlobListener), SingletonBlobListenerScopeId);
_sharedContextProvider.SetValue(SingletonBlobListenerScopeId, true);
return(new CompositeListener(singletonBlobListener, queueListener));
}
else
{
// We've already created the singleton blob listener
// so just return our queue listener. Note that while we want the
// blob scan to be singleton, the shared queue listener needs to run
// on ALL instances so load can be scaled out
return(queueListener);
}
}
// Populate the List<> with blob logs for the given prefix.
// http://blogs.msdn.com/b/windowsazurestorage/archive/2011/08/03/windows-azure-storage-logging-using-logs-to-track-storage-requests.aspx
private static async Task GetLogsWithPrefixAsync(List <IStorageBlob> selectedLogs, IStorageBlobClient blobClient,
string prefix, CancellationToken cancellationToken)
{
// List the blobs using the prefix
IEnumerable <IStorageListBlobItem> blobs = await blobClient.ListBlobsAsync(prefix,
useFlatBlobListing : true, blobListingDetails : BlobListingDetails.Metadata,
cancellationToken : cancellationToken);
// iterate through each blob and figure the start and end times in the metadata
// Type cast to IStorageBlob is safe due to useFlatBlobListing: true above.
foreach (IStorageBlob item in blobs)
{
IStorageBlob log = item as IStorageBlob;
if (log != null)
{
// we will exclude the file if the file does not have log entries in the interested time range.
string logType = log.Metadata[LogType];
bool hasWrites = logType.Contains("write");
if (hasWrites)
{
selectedLogs.Add(log);
}
}
}
}
/// <summary>Initializes a new instance of the <see cref="StorageBlobContainer"/> class.</summary>
/// <param name="parent">The parent blob client.</param>
/// <param name="sdk">The SDK container to wrap.</param>
public StorageBlobContainer(IStorageBlobClient parent, CloudBlobContainer sdk)
{
_parent = parent;
_sdk = sdk;
}
public BlobContainerBinding(string parameterName, IArgumentBinding <IStorageBlobContainer> argumentBinding, IStorageBlobClient client,
IBindableBlobPath path)
{
_parameterName = parameterName;
_argumentBinding = argumentBinding;
_client = client;
_path = path;
}
public StringToStorageBlobConverter(IStorageBlobClient client, IBindableBlobPath defaultPath, Type argumentType)
{
_client = client;
_defaultPath = defaultPath;
_argumentType = argumentType;
}
public SkewableFakeStorageBlobContainer(MemoryBlobStore store, string containerName,
IStorageBlobClient parent, Action <IStorageBlobResultSegment> onListBlobsSegmented)
: base(store, containerName, parent)
{
_onListBlobsSegmented = onListBlobsSegmented;
}
public static async Task <IEnumerable <IStorageListBlobItem> > ListBlobsAsync(this IStorageBlobClient client,
string prefix, bool useFlatBlobListing, BlobListingDetails blobListingDetails,
CancellationToken cancellationToken)
{
if (client == null)
{
throw new ArgumentNullException("client");
}
List <IStorageListBlobItem> allResults = new List <IStorageListBlobItem>();
BlobContinuationToken currentToken = null;
IStorageBlobResultSegment result;
do
{
result = await client.ListBlobsSegmentedAsync(prefix, useFlatBlobListing, blobListingDetails,
maxResults : null, currentToken : currentToken, options : null, operationContext : null,
cancellationToken : cancellationToken);
if (result != null)
{
IEnumerable <IStorageListBlobItem> currentResults = result.Results;
if (currentResults != null)
{
allResults.AddRange(currentResults);
}
}
}while (result != null && result.ContinuationToken != null);
return(allResults);
}
private IListener CreateQueueMessageToTriggerExecutionListener(
ISharedContextProvider sharedContextProvider,
SharedQueueWatcher sharedQueueWatcher,
IStorageQueueClient queueClient,
IStorageQueue hostBlobTriggerQueue,
IStorageBlobClient blobClient,
IBlobWrittenWatcher blobWrittenWatcher)
{
SharedBlobQueueListener sharedListener = sharedContextProvider.GetOrCreate<SharedBlobQueueListener>(
new SharedBlobQueueListenerFactory(sharedQueueWatcher, queueClient, hostBlobTriggerQueue,
blobClient, _queueConfiguration, _backgroundExceptionDispatcher, _log, blobWrittenWatcher));
sharedListener.Register(_functionId, _executor);
return new BlobListener(sharedListener);
}
public BlobReceiptManager(IStorageBlobClient client)
: this(client.GetContainerReference(HostContainerNames.Hosts)
.GetDirectoryReference(HostDirectoryNames.BlobReceipts))
{
}
public BlobBinding(string parameterName, IBlobArgumentBinding argumentBinding, IStorageBlobClient client,
IBindableBlobPath path)
{
_parameterName = parameterName;
_argumentBinding = argumentBinding;
_client = client;
_accountName = BlobClient.GetAccountName(client);
_path = path;
_converter = CreateConverter(_client, path, argumentBinding.ValueType);
}
private void RegisterWithSharedBlobQueueListenerAsync(
SharedBlobQueueListener sharedBlobQueueListener,
IStorageBlobClient blobClient)
{
BlobQueueRegistration registration = new BlobQueueRegistration
{
Executor = _executor,
BlobClient = blobClient
};
sharedBlobQueueListener.Register(_functionId, registration);
}
private static IStorageBlobContainer GetContainerReference(IStorageAccount account, string containerName)
{
IStorageBlobClient client = account.CreateBlobClient();
return(client.GetContainerReference(ContainerName));
}
private BlobLogListener(IStorageBlobClient blobClient)
{
_blobClient = blobClient;
}
private async Task<IListener> CreateBlobDiscoveryToQueueMessageListenerAsync(string hostId,
SharedBlobListener sharedBlobListener,
IStorageBlobClient blobClient,
IStorageQueue hostBlobTriggerQueue,
IMessageEnqueuedWatcher messageEnqueuedWatcher,
CancellationToken cancellationToken)
{
BlobTriggerExecutor triggerExecutor = new BlobTriggerExecutor(hostId, _functionId, _input,
BlobETagReader.Instance, new BlobReceiptManager(blobClient),
new BlobTriggerQueueWriter(hostBlobTriggerQueue, messageEnqueuedWatcher));
await sharedBlobListener.RegisterAsync(_container, triggerExecutor, cancellationToken);
return new BlobListener(sharedBlobListener);
}
public SharedBlobQueueListenerFactory(IFunctionExecutor executor,
SharedQueueWatcher sharedQueueWatcher,
IStorageQueueClient queueClient,
IStorageQueue hostBlobTriggerQueue,
IStorageBlobClient blobClient,
IQueueConfiguration queueConfiguration,
IBackgroundExceptionDispatcher backgroundExceptionDispatcher,
TextWriter log,
IBlobWrittenWatcher blobWrittenWatcher)
{
if (executor == null)
{
throw new ArgumentNullException("executor");
}
if (sharedQueueWatcher == null)
{
throw new ArgumentNullException("sharedQueueWatcher");
}
if (queueClient == null)
{
throw new ArgumentNullException("queueClient");
}
if (hostBlobTriggerQueue == null)
{
throw new ArgumentNullException("hostBlobTriggerQueue");
}
if (blobClient == null)
{
throw new ArgumentNullException("blobClient");
}
if (queueConfiguration == null)
{
throw new ArgumentNullException("queueConfiguration");
}
if (backgroundExceptionDispatcher == null)
{
throw new ArgumentNullException("backgroundExceptionDispatcher");
}
if (log == null)
{
throw new ArgumentNullException("log");
}
if (blobWrittenWatcher == null)
{
throw new ArgumentNullException("blobWrittenWatcher");
}
_executor = executor;
_sharedQueueWatcher = sharedQueueWatcher;
_queueClient = queueClient;
_hostBlobTriggerQueue = hostBlobTriggerQueue;
_blobClient = blobClient;
_queueConfiguration = queueConfiguration;
_backgroundExceptionDispatcher = backgroundExceptionDispatcher;
_log = log;
_blobWrittenWatcher = blobWrittenWatcher;
}
public BlobQueueTriggerExecutor(IStorageBlobClient client, IBlobWrittenWatcher blobWrittenWatcher)
: this(client, BlobETagReader.Instance, BlobCausalityReader.Instance, blobWrittenWatcher)
{
}
/// <summary>Initializes a new instance of the <see cref="PersistentQueueWriter{T}"/> class.</summary>
/// <param name="client">
/// A blob client for the storage account into which host output messages are written.
/// </param>
public PersistentQueueWriter(IStorageBlobClient client)
: this(client.GetContainerReference(ContainerNames.HostOutput))
{
}
public BlobFunctionOutputDefinition(IStorageBlobClient client, LocalBlobDescriptor outputBlob, LocalBlobDescriptor parameterLogBlob)
{
_client = client;
_outputBlob = outputBlob;
_parameterLogBlob = parameterLogBlob;
}
public BlobTriggerBinding(ParameterInfo parameter,
IArgumentBinding <IStorageBlob> argumentBinding,
IStorageAccount hostAccount,
IStorageAccount dataAccount,
IBlobPathSource path,
IHostIdProvider hostIdProvider,
IQueueConfiguration queueConfiguration,
IBackgroundExceptionDispatcher backgroundExceptionDispatcher,
IContextSetter <IBlobWrittenWatcher> blobWrittenWatcherSetter,
IContextSetter <IMessageEnqueuedWatcher> messageEnqueuedWatcherSetter,
ISharedContextProvider sharedContextProvider,
SingletonManager singletonManager,
TraceWriter trace)
{
if (parameter == null)
{
throw new ArgumentNullException("parameter");
}
if (argumentBinding == null)
{
throw new ArgumentNullException("argumentBinding");
}
if (hostAccount == null)
{
throw new ArgumentNullException("hostAccount");
}
if (dataAccount == null)
{
throw new ArgumentNullException("dataAccount");
}
if (path == null)
{
throw new ArgumentNullException("path");
}
if (hostIdProvider == null)
{
throw new ArgumentNullException("hostIdProvider");
}
if (queueConfiguration == null)
{
throw new ArgumentNullException("queueConfiguration");
}
if (backgroundExceptionDispatcher == null)
{
throw new ArgumentNullException("backgroundExceptionDispatcher");
}
if (blobWrittenWatcherSetter == null)
{
throw new ArgumentNullException("blobWrittenWatcherSetter");
}
if (messageEnqueuedWatcherSetter == null)
{
throw new ArgumentNullException("messageEnqueuedWatcherSetter");
}
if (sharedContextProvider == null)
{
throw new ArgumentNullException("sharedContextProvider");
}
if (singletonManager == null)
{
throw new ArgumentNullException("singletonManager");
}
if (trace == null)
{
throw new ArgumentNullException("trace");
}
_parameter = parameter;
_argumentBinding = argumentBinding;
_hostAccount = hostAccount;
_dataAccount = dataAccount;
StorageClientFactoryContext context = new StorageClientFactoryContext
{
Parameter = parameter
};
_blobClient = dataAccount.CreateBlobClient(context);
_accountName = BlobClient.GetAccountName(_blobClient);
_path = path;
_hostIdProvider = hostIdProvider;
_queueConfiguration = queueConfiguration;
_backgroundExceptionDispatcher = backgroundExceptionDispatcher;
_blobWrittenWatcherSetter = blobWrittenWatcherSetter;
_messageEnqueuedWatcherSetter = messageEnqueuedWatcherSetter;
_sharedContextProvider = sharedContextProvider;
_singletonManager = singletonManager;
_trace = trace;
_converter = CreateConverter(_blobClient);
_bindingDataContract = CreateBindingDataContract(path);
}
private static IAsyncObjectToTypeConverter<IStorageBlob> CreateConverter(IStorageBlobClient client)
{
return new CompositeAsyncObjectToTypeConverter<IStorageBlob>(
new OutputConverter<IStorageBlob>(new AsyncConverter<IStorageBlob, IStorageBlob>(
new IdentityConverter<IStorageBlob>())),
new OutputConverter<ICloudBlob>(new AsyncConverter<ICloudBlob, IStorageBlob>(
new CloudBlobToStorageBlobConverter())),
new OutputConverter<string>(new StringToStorageBlobConverter(client)));
}
public BlobTriggerBinding(string parameterName,
IArgumentBinding<IStorageBlob> argumentBinding,
IStorageAccount account,
IBlobPathSource path,
IHostIdProvider hostIdProvider,
IQueueConfiguration queueConfiguration,
IBackgroundExceptionDispatcher backgroundExceptionDispatcher,
IContextSetter<IBlobWrittenWatcher> blobWrittenWatcherSetter,
IContextSetter<IMessageEnqueuedWatcher> messageEnqueuedWatcherSetter,
ISharedContextProvider sharedContextProvider,
TextWriter log)
{
if (argumentBinding == null)
{
throw new ArgumentNullException("argumentBinding");
}
if (account == null)
{
throw new ArgumentNullException("account");
}
if (path == null)
{
throw new ArgumentNullException("path");
}
if (hostIdProvider == null)
{
throw new ArgumentNullException("hostIdProvider");
}
if (queueConfiguration == null)
{
throw new ArgumentNullException("queueConfiguration");
}
if (backgroundExceptionDispatcher == null)
{
throw new ArgumentNullException("backgroundExceptionDispatcher");
}
if (blobWrittenWatcherSetter == null)
{
throw new ArgumentNullException("blobWrittenWatcherSetter");
}
if (messageEnqueuedWatcherSetter == null)
{
throw new ArgumentNullException("messageEnqueuedWatcherSetter");
}
if (sharedContextProvider == null)
{
throw new ArgumentNullException("sharedContextProvider");
}
if (log == null)
{
throw new ArgumentNullException("log");
}
_parameterName = parameterName;
_argumentBinding = argumentBinding;
_account = account;
_client = account.CreateBlobClient();
_accountName = BlobClient.GetAccountName(_client);
_path = path;
_hostIdProvider = hostIdProvider;
_queueConfiguration = queueConfiguration;
_backgroundExceptionDispatcher = backgroundExceptionDispatcher;
_blobWrittenWatcherSetter = blobWrittenWatcherSetter;
_messageEnqueuedWatcherSetter = messageEnqueuedWatcherSetter;
_sharedContextProvider = sharedContextProvider;
_log = log;
_converter = CreateConverter(_client);
_bindingDataContract = CreateBindingDataContract(path);
}
public BlobReceiptManager(IStorageBlobClient client)
: this(client.GetContainerReference(HostContainerNames.Hosts)
.GetDirectoryReference(HostDirectoryNames.BlobReceipts))
{
}
